From 1a263127f5fcca0196eb874b5cc77765e1f43ca0 Mon Sep 17 00:00:00 2001 From: PaulKoudelka Date: Fri, 14 Aug 2026 12:37:50 +0200 Subject: [PATCH] replaced embedding_state with index_store as a sqlite name --- .../Assistants/I18N/allTexts.lua | 20 +++--- .../Tools/Databases/DatabaseClientProvider.cs | 14 ++-- .../Tools/Databases/DatabaseRole.cs | 2 +- .../IndexStoreClient.cs} | 8 +-- .../IndexStoreDbContext.cs} | 22 +++--- .../IndexStoreDesignTimeDbContextFactory.cs} | 8 +-- .../IndexStoreSchemaMigrator.cs} | 6 +- .../20260804000000_InitialRagIndex.cs | 4 +- .../IndexStoreDbContextModelSnapshot.cs} | 28 ++++---- .../NoIndexStoreClient.cs} | 10 +-- .../SqliteIndexStoreClientImplementation.cs} | 30 ++++---- .../DataSourceEmbeddingService.Files.cs | 2 +- .../DataSourceEmbeddingService.State.cs | 14 ++-- .../Services/DataSourceEmbeddingService.cs | 72 +++++++++---------- .../DataSourceLocalRetrievalService.cs | 18 ++--- 15 files changed, 129 insertions(+), 129 deletions(-) rename app/MindWork AI Studio/Tools/Databases/{EmbeddingState/EmbeddingStateClient.cs => IndexStore/IndexStoreClient.cs} (85%) rename app/MindWork AI Studio/Tools/Databases/{EmbeddingState/EmbeddingStateDbContext.cs => IndexStore/IndexStoreDbContext.cs} (91%) rename app/MindWork AI Studio/Tools/Databases/{EmbeddingState/EmbeddingStateDesignTimeDbContextFactory.cs => IndexStore/IndexStoreDesignTimeDbContextFactory.cs} (51%) rename app/MindWork AI Studio/Tools/Databases/{EmbeddingState/EmbeddingStateSchemaMigrator.cs => IndexStore/IndexStoreSchemaMigrator.cs} (61%) rename app/MindWork AI Studio/Tools/Databases/{EmbeddingState => IndexStore}/Migrations/20260804000000_InitialRagIndex.cs (98%) rename app/MindWork AI Studio/Tools/Databases/{EmbeddingState/Migrations/EmbeddingStateDbContextModelSnapshot.cs => IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs} (88%) rename app/MindWork AI Studio/Tools/Databases/{EmbeddingState/NoEmbeddingStateClient.cs => IndexStore/NoIndexStoreClient.cs} (77%) rename app/MindWork AI Studio/Tools/Databases/{EmbeddingState/SqliteEmbeddingStateClientImplementation.cs => IndexStore/SqliteIndexStoreClientImplementation.cs} (90%) diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index f87866e4..64943c4b 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -8882,34 +8882,34 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T3893997203"] = " UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T4107860491"] = "Trust all LLM providers" -- Reason -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T1093747001"] = "Reason" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1093747001"] = "Reason" -- Starting -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T1233211769"] = "Starting" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1233211769"] = "Starting" -- Unavailable -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T3662391977"] = "Unavailable" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T3662391977"] = "Unavailable" -- Status -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T6222351"] = "Status" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T6222351"] = "Status" -- Database path -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T1100578143"] = "Database path" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T1100578143"] = "Database path" -- Storage size -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T1230141403"] = "Storage size" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T1230141403"] = "Storage size" -- Indexed files -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T2235289713"] = "Indexed files" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T2235289713"] = "Indexed files" -- Search chunks -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T2333737457"] = "Search chunks" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T2333737457"] = "Search chunks" -- Indexed data sources -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T3524534748"] = "Indexed data sources" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T3524534748"] = "Indexed data sources" -- Reported version -UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T3556099842"] = "Reported version" +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T3556099842"] = "Reported version" -- Reason UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T1093747001"] = "Reason" diff --git a/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs b/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs index f28c0779..7ca28ff5 100644 --- a/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs +++ b/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs @@ -1,4 +1,4 @@ -using AIStudio.Tools.Databases.EmbeddingState; +using AIStudio.Tools.Databases.IndexStore; using AIStudio.Tools.Databases.VectorStore; using AIStudio.Tools.Services; @@ -57,13 +57,13 @@ public sealed partial class DatabaseClientProvider(RustService rustService, ILog client.Status); } - public async Task GetEmbeddingStateAsync(CancellationToken cancellationToken = default) + public async Task GetIndexStoreAsync(CancellationToken cancellationToken = default) { - var client = await this.GetClientAsync(DatabaseRole.EMBEDDING_STATE, cancellationToken); - if (client is EmbeddingStateClient embeddingState) - return embeddingState; + var client = await this.GetClientAsync(DatabaseRole.INDEX_STORE, cancellationToken); + if (client is IndexStoreClient indexStore) + return indexStore; - return new NoEmbeddingStateClient( + return new NoIndexStoreClient( client.Name, "The configured database client does not support local RAG index operations.", client.Status); @@ -105,7 +105,7 @@ public sealed partial class DatabaseClientProvider(RustService rustService, ILog private async Task CreateClientAsync(DatabaseRole databaseRole, CancellationToken cancellationToken) => databaseRole switch { DatabaseRole.VECTOR_STORE => await QdrantEdgeClientImplementation.CreateAsync(rustService, this.logger, this.databaseClientLogger, cancellationToken), - DatabaseRole.EMBEDDING_STATE => await SqliteEmbeddingStateClientImplementation.CreateAsync(this.logger, this.databaseClientLogger, cancellationToken), + DatabaseRole.INDEX_STORE => await SqliteIndexStoreClientImplementation.CreateAsync(this.logger, this.databaseClientLogger, cancellationToken), _ => new NoDatabaseClient(databaseRole.ToString(), "The requested database role is not supported.") }; diff --git a/app/MindWork AI Studio/Tools/Databases/DatabaseRole.cs b/app/MindWork AI Studio/Tools/Databases/DatabaseRole.cs index 3e41b9cb..fcf9765b 100644 --- a/app/MindWork AI Studio/Tools/Databases/DatabaseRole.cs +++ b/app/MindWork AI Studio/Tools/Databases/DatabaseRole.cs @@ -3,5 +3,5 @@ namespace AIStudio.Tools.Databases; public enum DatabaseRole { VECTOR_STORE, - EMBEDDING_STATE, + INDEX_STORE, } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs similarity index 85% rename from app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs rename to app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs index 7ebb4851..9ea68584 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs @@ -1,8 +1,8 @@ using AIStudio.Tools.Services; -namespace AIStudio.Tools.Databases.EmbeddingState; +namespace AIStudio.Tools.Databases.IndexStore; -public abstract class EmbeddingStateClient(string name, string path) : DatabaseClient(name, path) +public abstract class IndexStoreClient(string name, string path) : DatabaseClient(name, path) { public abstract Task GetManifestAsync(string dataSourceId, CancellationToken token); @@ -26,7 +26,7 @@ public abstract class EmbeddingStateClient(string name, string path) : DatabaseC public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList chunks, CancellationToken token); - public abstract Task> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token); + public abstract Task> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token); public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token); } @@ -54,7 +54,7 @@ public sealed record EmbeddingStateChunk( string ChunkText, DateTimeOffset EmbeddedAtUtc); -public sealed record EmbeddingStateSearchResult( +public sealed record IndexStoreSearchResult( string ChunkId, string ParentFileId, string DataSourceId, diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDbContext.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs similarity index 91% rename from app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDbContext.cs rename to app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs index 1000ef53..726026f4 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDbContext.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs @@ -4,11 +4,11 @@ using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Storage.ValueConversion; -namespace AIStudio.Tools.Databases.EmbeddingState; +namespace AIStudio.Tools.Databases.IndexStore; -internal sealed class EmbeddingStateDbContext(DbContextOptions options) : DbContext(options) +internal sealed class IndexStoreDbContext(DbContextOptions options) : DbContext(options) { - public static DbContextOptions CreateOptions(string databasePath) => new DbContextOptionsBuilder() + public static DbContextOptions CreateOptions(string databasePath) => new DbContextOptionsBuilder() .UseSqlite(BuildConnectionString(databasePath)) .Options; @@ -18,11 +18,11 @@ internal sealed class EmbeddingStateDbContext(DbContextOptions EmbeddingChunks => this.Set(); - public DbSet SearchResults => this.Set(); + public DbSet SearchResults => this.Set(); protected override void OnModelCreating(ModelBuilder modelBuilder) { - var utcDateTimeOffsetConverter = new EmbeddingStateDateTimeOffsetConverter(); + var utcDateTimeOffsetConverter = new IndexStoreDateTimeOffsetConverter(); modelBuilder.Entity(entity => { @@ -97,7 +97,7 @@ internal sealed class EmbeddingStateDbContext(DbContextOptions new { chunk.ParentFileId, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index").IsUnique(); }); - modelBuilder.Entity(entity => + modelBuilder.Entity(entity => { entity.HasNoKey(); entity.ToView("embedding_chunk_search_results"); @@ -193,7 +193,7 @@ internal sealed class EmbeddingStateChunkEntity public EmbeddingStateFileEntity? File { get; set; } } -internal sealed class EmbeddingStateSearchResultEntity +internal sealed class IndexStoreSearchResultEntity { public string ChunkId { get; set; } = string.Empty; @@ -238,11 +238,11 @@ internal sealed class EmbeddingStateSearchResultEntity public int ConfidenceLevelRank { get; set; } } -internal sealed class EmbeddingStateDateTimeOffsetConverter() : ValueConverter( - value => EmbeddingStateDateTimeOffset.ToUtcText(value), - value => EmbeddingStateDateTimeOffset.ParseUtc(value)); +internal sealed class IndexStoreDateTimeOffsetConverter() : ValueConverter( + value => IndexStoreDateTimeOffset.ToUtcText(value), + value => IndexStoreDateTimeOffset.ParseUtc(value)); -internal static class EmbeddingStateDateTimeOffset +internal static class IndexStoreDateTimeOffset { public static string ToUtcText(DateTimeOffset dateTime) { diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDesignTimeDbContextFactory.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDesignTimeDbContextFactory.cs similarity index 51% rename from app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDesignTimeDbContextFactory.cs rename to app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDesignTimeDbContextFactory.cs index b055f75a..ec92c7de 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDesignTimeDbContextFactory.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDesignTimeDbContextFactory.cs @@ -1,15 +1,15 @@ using Microsoft.EntityFrameworkCore.Design; -namespace AIStudio.Tools.Databases.EmbeddingState; +namespace AIStudio.Tools.Databases.IndexStore; -internal sealed class EmbeddingStateDesignTimeDbContextFactory : IDesignTimeDbContextFactory +internal sealed class IndexStoreDesignTimeDbContextFactory : IDesignTimeDbContextFactory { - public EmbeddingStateDbContext CreateDbContext(string[] args) + public IndexStoreDbContext CreateDbContext(string[] args) { var databasePath = args.FirstOrDefault(argument => argument.EndsWith(".sqlite3", StringComparison.OrdinalIgnoreCase)); if (string.IsNullOrWhiteSpace(databasePath)) databasePath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-rag-index-design.sqlite3"); - return new EmbeddingStateDbContext(EmbeddingStateDbContext.CreateOptions(databasePath)); + return new IndexStoreDbContext(IndexStoreDbContext.CreateOptions(databasePath)); } } diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateSchemaMigrator.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs similarity index 61% rename from app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateSchemaMigrator.cs rename to app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs index c590c280..bf44f43b 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateSchemaMigrator.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs @@ -2,12 +2,12 @@ using System.Diagnostics.CodeAnalysis; using Microsoft.EntityFrameworkCore; -namespace AIStudio.Tools.Databases.EmbeddingState; +namespace AIStudio.Tools.Databases.IndexStore; -internal static class EmbeddingStateSchemaMigrator +internal static class IndexStoreSchemaMigrator { [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))] - public static async Task MigrateAsync(EmbeddingStateDbContext context, CancellationToken token) + public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token) { await context.Database.MigrateAsync(token); await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL;", token); diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/20260804000000_InitialRagIndex.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260804000000_InitialRagIndex.cs similarity index 98% rename from app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/20260804000000_InitialRagIndex.cs rename to app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260804000000_InitialRagIndex.cs index 2eaa31d1..0845cdac 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/20260804000000_InitialRagIndex.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260804000000_InitialRagIndex.cs @@ -3,9 +3,9 @@ using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Migrations; -namespace AIStudio.Tools.Databases.EmbeddingState.Migrations; +namespace AIStudio.Tools.Databases.IndexStore.Migrations; -[DbContext(typeof(EmbeddingStateDbContext))] +[DbContext(typeof(IndexStoreDbContext))] [Migration("20260804000000_InitialRagIndex")] public partial class InitialRagIndex : Migration { diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/EmbeddingStateDbContextModelSnapshot.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs similarity index 88% rename from app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/EmbeddingStateDbContextModelSnapshot.cs rename to app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs index 3721b8a2..f0faf9ad 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/EmbeddingStateDbContextModelSnapshot.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs @@ -3,18 +3,18 @@ using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Infrastructure; -namespace AIStudio.Tools.Databases.EmbeddingState.Migrations; +namespace AIStudio.Tools.Databases.IndexStore.Migrations; -[DbContext(typeof(EmbeddingStateDbContext))] -partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot +[DbContext(typeof(IndexStoreDbContext))] +partial class IndexStoreDbContextModelSnapshot : ModelSnapshot { protected override void BuildModel(ModelBuilder modelBuilder) { #pragma warning disable 612, 618 modelBuilder.HasAnnotation("ProductVersion", "9.0.18"); - var utcDateTimeOffsetConverter = new EmbeddingStateDateTimeOffsetConverter(); + var utcDateTimeOffsetConverter = new IndexStoreDateTimeOffsetConverter(); - modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity => + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity => { entity.Property("DataSourceId") .HasColumnType("TEXT") @@ -61,7 +61,7 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot entity.ToTable("data_sources"); }); - modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity => + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity => { entity.Property("ParentFileId") .HasColumnType("TEXT") @@ -151,7 +151,7 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot entity.ToTable("embedded_files"); }); - modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateChunkEntity", entity => + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateChunkEntity", entity => { entity.Property("Id") .ValueGeneratedOnAdd() @@ -206,7 +206,7 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot entity.ToTable("embedding_chunks"); }); - modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateSearchResultEntity", entity => + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.IndexStoreSearchResultEntity", entity => { entity.Property("AbsolutePath") .IsRequired() @@ -291,9 +291,9 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot entity.ToView("embedding_chunk_search_results"); }); - modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity => + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity => { - entity.HasOne("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", "DataSource") + entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", "DataSource") .WithMany("Files") .HasForeignKey("DataSourceId") .OnDelete(DeleteBehavior.Cascade) @@ -302,9 +302,9 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot entity.Navigation("DataSource"); }); - modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateChunkEntity", entity => + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateChunkEntity", entity => { - entity.HasOne("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", "File") + entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", "File") .WithMany("Chunks") .HasForeignKey("ParentFileId") .OnDelete(DeleteBehavior.Cascade) @@ -313,12 +313,12 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot entity.Navigation("File"); }); - modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity => + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity => { entity.Navigation("Files"); }); - modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity => + modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity => { entity.Navigation("Chunks"); }); diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs similarity index 77% rename from app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs rename to app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs index e7696cfa..76f93f25 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs @@ -1,11 +1,11 @@ using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Services; -namespace AIStudio.Tools.Databases.EmbeddingState; +namespace AIStudio.Tools.Databases.IndexStore; -public sealed class NoEmbeddingStateClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : EmbeddingStateClient(name, string.Empty) +public sealed class NoIndexStoreClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : IndexStoreClient(name, string.Empty) { - private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoEmbeddingStateClient).Namespace, nameof(NoEmbeddingStateClient)); + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoIndexStoreClient).Namespace, nameof(NoIndexStoreClient)); public override DatabaseClientStatus Status => status; @@ -46,8 +46,8 @@ public sealed class NoEmbeddingStateClient(string name, string? unavailableReaso public override Task UpsertChunksAsync(string dataSourceId, IReadOnlyList chunks, CancellationToken token) => Task.CompletedTask; - public override Task> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) => - Task.FromResult>([]); + public override Task> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) => + Task.FromResult>([]); public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask; diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs similarity index 90% rename from app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs rename to app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs index fe132378..7ccb06d2 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs @@ -7,13 +7,13 @@ using AIStudio.Tools.Services; using Microsoft.EntityFrameworkCore; -namespace AIStudio.Tools.Databases.EmbeddingState; +namespace AIStudio.Tools.Databases.IndexStore; -public sealed class SqliteEmbeddingStateClientImplementation( +public sealed class SqliteIndexStoreClientImplementation( string name, string databasePath, string basePath, - string version) : EmbeddingStateClient(name, basePath) + string version) : IndexStoreClient(name, basePath) { private const string DATABASE_NAME = "Local RAG Index"; private const string DATABASE_FILENAME = "rag-index.sqlite3"; @@ -23,9 +23,9 @@ public sealed class SqliteEmbeddingStateClientImplementation( private static readonly Regex FTS_TOKEN_REGEX = new(@"[\p{L}\p{Nd}_]+", RegexOptions.Compiled | RegexOptions.CultureInvariant); private readonly string databasePath = databasePath; - private readonly DbContextOptions dbContextOptions = EmbeddingStateDbContext.CreateOptions(databasePath); + private readonly DbContextOptions dbContextOptions = IndexStoreDbContext.CreateOptions(databasePath); - private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SqliteEmbeddingStateClientImplementation).Namespace, nameof(SqliteEmbeddingStateClientImplementation)); + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SqliteIndexStoreClientImplementation).Namespace, nameof(SqliteIndexStoreClientImplementation)); public override string CacheKey => $"{this.Name}:{this.databasePath}:{version}"; @@ -35,7 +35,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( CancellationToken cancellationToken) { if (string.IsNullOrWhiteSpace(SettingsManager.DataDirectory)) - return CreateNoEmbeddingStateClient(DATABASE_NAME, "The application data directory is not available yet.", DatabaseClientStatus.STARTING, databaseClientLogger); + return CreateNoIndexStoreClient(DATABASE_NAME, "The application data directory is not available yet.", DatabaseClientStatus.STARTING, databaseClientLogger); try { @@ -45,18 +45,18 @@ public sealed class SqliteEmbeddingStateClientImplementation( Directory.CreateDirectory(basePath); var databasePath = Path.Combine(basePath, DATABASE_FILENAME); - var client = new SqliteEmbeddingStateClientImplementation(DATABASE_NAME, databasePath, basePath, string.Empty); + var client = new SqliteIndexStoreClientImplementation(DATABASE_NAME, databasePath, basePath, string.Empty); await client.InitializeAsync(cancellationToken); var version = await client.GetSqliteVersionAsync(cancellationToken); - client = new SqliteEmbeddingStateClientImplementation(DATABASE_NAME, databasePath, basePath, version); + client = new SqliteIndexStoreClientImplementation(DATABASE_NAME, databasePath, basePath, version); client.SetLogger(databaseClientLogger); return client; } catch (Exception exception) { logger.LogWarning(exception, "{DatabaseName} is not available. Indexed file fingerprints and search chunks are disabled.", DATABASE_NAME); - return CreateNoEmbeddingStateClient(DATABASE_NAME, exception.Message, DatabaseClientStatus.UNAVAILABLE, databaseClientLogger); + return CreateNoIndexStoreClient(DATABASE_NAME, exception.Message, DatabaseClientStatus.UNAVAILABLE, databaseClientLogger); } } @@ -236,7 +236,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( await transaction.CommitAsync(token); } - public override async Task> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) + public override async Task> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) { if (maxMatches <= 0) return []; @@ -314,7 +314,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( private async Task InitializeAsync(CancellationToken token) { await using var context = this.CreateContext(); - await EmbeddingStateSchemaMigrator.MigrateAsync(context, token); + await IndexStoreSchemaMigrator.MigrateAsync(context, token); } private async Task GetSqliteVersionAsync(CancellationToken token) @@ -326,7 +326,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( return versions.FirstOrDefault() ?? string.Empty; } - private EmbeddingStateDbContext CreateContext() => new(this.dbContextOptions); + private IndexStoreDbContext CreateContext() => new(this.dbContextOptions); private static void ApplyDataSource( EmbeddingStateDataSourceEntity dataSource, @@ -373,7 +373,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( chunkEntity.EmbeddedAtUtc = chunk.EmbeddedAtUtc; } - private static EmbeddingStateSearchResult ToSearchResult(EmbeddingStateSearchResultEntity result) => new( + private static IndexStoreSearchResult ToSearchResult(IndexStoreSearchResultEntity result) => new( result.ChunkId, result.ParentFileId, result.DataSourceId, @@ -410,9 +410,9 @@ public sealed class SqliteEmbeddingStateClientImplementation( return terms.Count == 0 ? string.Empty : string.Join(" OR ", terms); } - private static NoEmbeddingStateClient CreateNoEmbeddingStateClient(string name, string? unavailableReason, DatabaseClientStatus status, ILogger databaseClientLogger) + private static NoIndexStoreClient CreateNoIndexStoreClient(string name, string? unavailableReason, DatabaseClientStatus status, ILogger databaseClientLogger) { - var client = new NoEmbeddingStateClient(name, unavailableReason, status); + var client = new NoIndexStoreClient(name, unavailableReason, status); client.SetLogger(databaseClientLogger); return client; } diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 551852fd..5c46c289 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -5,7 +5,7 @@ using System.Text.RegularExpressions; using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; -using AIStudio.Tools.Databases.EmbeddingState; +using AIStudio.Tools.Databases.IndexStore; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs index fc318f02..06ce878f 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs @@ -1,4 +1,4 @@ -using AIStudio.Tools.Databases.EmbeddingState; +using AIStudio.Tools.Databases.IndexStore; using AIStudio.Tools.Databases.VectorStore; namespace AIStudio.Tools.Services; @@ -8,19 +8,19 @@ public sealed partial class DataSourceEmbeddingService private async Task ResetPersistedStateAsync( string dataSourceId, VectorStoreClient? vectorStore, - EmbeddingStateClient? embeddingState, + IndexStoreClient? indexStore, CancellationToken token) { await this.DeleteCollectionAsync(DataSourceEmbeddingNames.GetCollectionName(dataSourceId), vectorStore, token); - embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token); - if (!embeddingState.IsAvailable) + indexStore ??= await databaseClientProvider.GetIndexStoreAsync(token); + if (!indexStore.IsAvailable) { - logger.LogWarning("Could not delete local RAG index state for data source '{DataSourceId}' because the database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name); + logger.LogWarning("Could not delete local RAG embedding state for data source '{DataSourceId}' because the database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name); return; } - await embeddingState.DeleteDataSourceAsync(dataSourceId, token); - logger.LogInformation("Reset persisted local RAG index state for data source '{DataSourceId}'.", dataSourceId); + await indexStore.DeleteDataSourceAsync(dataSourceId, token); + logger.LogInformation("Reset persisted local RAG embedding state for data source '{DataSourceId}'.", dataSourceId); } } diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index f7a56717..42ce0de1 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -7,7 +7,7 @@ using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.Databases; -using AIStudio.Tools.Databases.EmbeddingState; +using AIStudio.Tools.Databases.IndexStore; using AIStudio.Tools.Databases.VectorStore; using AIStudio.Tools.PluginSystem; @@ -188,14 +188,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM public async Task ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default) { - var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token); - if (!embeddingState.IsAvailable) + var indexStore = await databaseClientProvider.GetIndexStoreAsync(token); + if (!indexStore.IsAvailable) { - logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name); + logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name); return true; } - var manifest = await embeddingState.GetManifestAsync(dataSourceId, token); + var manifest = await indexStore.GetManifestAsync(dataSourceId, token); return !string.IsNullOrWhiteSpace(manifest.EmbeddingProviderId) || !string.IsNullOrWhiteSpace(manifest.EmbeddingSignature) || !string.IsNullOrWhiteSpace(manifest.SourceHash) @@ -410,7 +410,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM token.ThrowIfCancellationRequested(); var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token); - var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token); + var indexStore = await databaseClientProvider.GetIndexStoreAsync(token); token.ThrowIfCancellationRequested(); if (!vectorStore.IsAvailable) @@ -425,20 +425,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM return; } - if (!embeddingState.IsAvailable) + if (!indexStore.IsAvailable) { logger.LogWarning( "Skipping background embeddings for data source '{DataSourceName}' ({DataSourceId}) because the database client '{DatabaseName}' is unavailable.", dataSource.Name, dataSource.Id, - embeddingState.Name); + indexStore.Name); token.ThrowIfCancellationRequested(); this.UpsertStatus(this.GetFallbackStatus(dataSource, "The local RAG index database is not available.")); return; } var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id); - var persistedManifest = await embeddingState.GetManifestAsync(dataSource.Id, token); + var persistedManifest = await indexStore.GetManifestAsync(dataSource.Id, token); if (persistedManifest.VectorSize > 0) { var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedManifest.VectorSize, token); @@ -449,7 +449,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM collectionName, dataSource.Name, dataSource.Id); - await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, embeddingState, token); + await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, indexStore, token); } } @@ -484,7 +484,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Name, dataSource.Id); - var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, embeddingState, token); + var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, indexStore, token); token.ThrowIfCancellationRequested(); var inputFiles = this.GetInputFiles(dataSource); @@ -510,7 +510,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM collectionName); var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles); - var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(vectorStore, embeddingState, dataSource, collectionName, manifest, indexedFiles, token); + var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(vectorStore, indexStore, dataSource, collectionName, manifest, indexedFiles, token); var optimizationTracker = new VectorStoreOptimizationTracker(); if (removedMissingFiles > 0) optimizationTracker.MarkChanged(); @@ -543,7 +543,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM token); token.ThrowIfCancellationRequested(); - await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token); + await indexStore.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token); this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, indexedFiles.Count, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures)); return; } @@ -602,7 +602,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM skippedFiles + completedFiles + 1, totalFiles); var startedAtUtc = DateTimeOffset.UtcNow; - var chunkCount = await this.IndexOneFileAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token); + var chunkCount = await this.IndexOneFileAsync(indexStore, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token); token.ThrowIfCancellationRequested(); var fingerprintAfterEmbedding = BuildFileMetadataHash(file); if (!string.Equals(fingerprint, fingerprintAfterEmbedding, StringComparison.Ordinal)) @@ -615,7 +615,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM new DateTimeOffset(file.LastWriteTimeUtc), embeddedAtUtc, chunkCount); - await embeddingState.UpsertFileAsync( + await indexStore.UpsertFileAsync( dataSource.Id, this.CreateEmbeddingStateFile(dataSource, file, fingerprint, chunkCount, embeddedAtUtc), token); @@ -644,7 +644,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM lastError = exception.Message; failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message)); manifest.Files.Remove(file.FullName); - await this.CleanupFailedFileAsync(embeddingState, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token); + await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token); logger.LogWarning(exception, "Failed to embed file '{FilePath}' for data source '{DataSourceName}'.", file.FullName, dataSource.Name); this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, exception.Message, failureDetails)); @@ -662,7 +662,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM token); token.ThrowIfCancellationRequested(); - await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token); + await indexStore.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token); token.ThrowIfCancellationRequested(); this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, skippedFiles + completedFiles, failedFiles, lastError, failureDetails)); @@ -682,7 +682,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } private async Task IndexOneFileAsync( - EmbeddingStateClient embeddingState, + IndexStoreClient indexStore, VectorStoreClient vectorStore, IDataSource dataSource, FileInfo file, @@ -700,10 +700,10 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM collectionName); await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token); optimizationTracker.MarkChanged(); - await embeddingState.DeleteFileAsync(dataSource.Id, file.FullName, token); + await indexStore.DeleteFileAsync(dataSource.Id, file.FullName, token); var parentFile = this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow); - await embeddingState.UpsertFileAsync(dataSource.Id, parentFile, token); + await indexStore.UpsertFileAsync(dataSource.Id, parentFile, token); var embeddingBatchSize = Math.Max(1, embeddingProvider.EffectiveEmbeddingBatchSize); var batch = new List(embeddingBatchSize); @@ -715,11 +715,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM totalChunkCount++; if (batch.Count >= embeddingBatchSize) - await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token); + await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token); } if (batch.Count > 0) - await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token); + await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token); if (totalChunkCount == 0) throw new InvalidOperationException($"The file '{file.Name}' did not yield any text chunks."); @@ -735,7 +735,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } private async Task FlushBatchAsync( - EmbeddingStateClient embeddingState, + IndexStoreClient indexStore, VectorStoreClient vectorStore, IDataSource dataSource, FileInfo file, @@ -805,7 +805,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly."); } - await embeddingState.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token); + await indexStore.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token); manifest.VectorSize = vectorSize; logger.LogInformation( "Created embedding collection '{CollectionName}' with vector size {VectorSize} for data source '{DataSourceName}' ({DataSourceId}).", @@ -829,7 +829,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM embeddedAtUtc, token); token.ThrowIfCancellationRequested(); - await embeddingState.UpsertChunksAsync( + await indexStore.UpsertChunksAsync( dataSource.Id, this.CreateEmbeddingStateChunks(parentFile, batch, embeddedAtUtc), token); @@ -897,7 +897,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } private async Task CleanupFailedFileAsync( - EmbeddingStateClient embeddingState, + IndexStoreClient indexStore, VectorStoreClient vectorStore, IDataSource dataSource, string collectionName, @@ -926,7 +926,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM try { - await embeddingState.DeleteFileAsync(dataSource.Id, filePath, token); + await indexStore.DeleteFileAsync(dataSource.Id, filePath, token); } catch (OperationCanceledException) when (token.IsCancellationRequested) { @@ -1018,7 +1018,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM .ToList(); logger.LogInformation( - "Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed local RAG index state will be retried during this pass. File watchers will be activated after this check completes.", + "Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed local RAG embedding state will be retried during this pass. File watchers will be activated after this check completes.", supportedDataSources.Count); foreach (var dataSource in supportedDataSources) @@ -1074,12 +1074,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM EmbeddingProvider embeddingProvider, string collectionName, VectorStoreClient vectorStore, - EmbeddingStateClient embeddingState, + IndexStoreClient indexStore, CancellationToken token) { var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider); var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); - var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token); + var manifest = await indexStore.GetManifestAsync(dataSource.Id, token); logger.LogInformation( "Loaded persisted local RAG index manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.", @@ -1093,7 +1093,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM if (!string.Equals(manifest.EmbeddingSignature, embeddingSignature, StringComparison.Ordinal)) { logger.LogInformation( - "Embedding configuration changed for data source '{DataSourceName}' ({DataSourceId}). Resetting persisted state and collection '{CollectionName}'.", + "Embedding configuration changed for data source '{DataSourceName}' ({DataSourceId}). Resetting persisted embedding state and collection '{CollectionName}'.", dataSource.Name, dataSource.Id, collectionName); @@ -1103,8 +1103,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Id, manifest.EmbeddingSignature, embeddingSignature); - await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, embeddingState, token); - manifest = await embeddingState.GetManifestAsync(dataSource.Id, token); + await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, indexStore, token); + manifest = await indexStore.GetManifestAsync(dataSource.Id, token); } if (!string.Equals(manifest.EmbeddingProviderId, embeddingProvider.Id, StringComparison.OrdinalIgnoreCase) || @@ -1114,7 +1114,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM manifest.EmbeddingSignature = embeddingSignature; } - await embeddingState.UpsertDataSourceAsync( + await indexStore.UpsertDataSourceAsync( dataSource.Id, dataSource.Name, dataSource.Type.ToString(), @@ -1129,7 +1129,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM private async Task RemoveMissingFileEmbeddingsAsync( VectorStoreClient vectorStore, - EmbeddingStateClient embeddingState, + IndexStoreClient indexStore, IDataSource dataSource, string collectionName, DataSourceEmbeddingManifest manifest, @@ -1144,7 +1144,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM foreach (var removedFilePath in manifest.Files.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList()) { await this.DeleteFilePointsAsync(vectorStore, collectionName, removedFilePath, token); - await embeddingState.DeleteFileAsync(dataSource.Id, removedFilePath, token); + await indexStore.DeleteFileAsync(dataSource.Id, removedFilePath, token); manifest.Files.Remove(removedFilePath); removedFiles++; logger.LogInformation( diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index 723ccbd8..973c6ecc 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -3,7 +3,7 @@ using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.Databases; -using AIStudio.Tools.Databases.EmbeddingState; +using AIStudio.Tools.Databases.IndexStore; using AIStudio.Tools.Databases.VectorStore; using AIStudio.Tools.RAG; using AIStudio.Tools.Rust; @@ -188,25 +188,25 @@ public sealed class DataSourceLocalRetrievalService( return true; } - private async Task> SearchBm25Async(IInternalDataSource dataSource, string query, int maxMatches, CancellationToken token) + private async Task> SearchBm25Async(IInternalDataSource dataSource, string query, int maxMatches, CancellationToken token) { try { - var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token); - if (!embeddingState.IsAvailable) + var indexStore = await databaseClientProvider.GetIndexStoreAsync(token); + if (!indexStore.IsAvailable) { logger.LogWarning( "Skipping BM25 retrieval for data source '{DataSourceName}' ({DataSourceId}) because local RAG index '{DatabaseName}' is unavailable.", dataSource.Name, dataSource.Id, - embeddingState.Name); + indexStore.Name); return []; } var results = this.LimitSearchResults( dataSource, "BM25", - await embeddingState.SearchChunksAsync(dataSource.Id, query, maxMatches, token), + await indexStore.SearchChunksAsync(dataSource.Id, query, maxMatches, token), maxMatches); this.LogBm25Results(dataSource, results); return results; @@ -240,7 +240,7 @@ public sealed class DataSourceLocalRetrievalService( private static IReadOnlyList MergeResults( IReadOnlyList vectorResults, - IReadOnlyList bm25Results, + IReadOnlyList bm25Results, int maxMatches) { // Future reranking should replace this deterministic channel merge. @@ -299,7 +299,7 @@ public sealed class DataSourceLocalRetrievalService( result.ConfidenceLevel, result.ConfidenceLevelRank); - private static LocalRetrievalHit FromBm25Result(EmbeddingStateSearchResult result, int rank) => + private static LocalRetrievalHit FromBm25Result(IndexStoreSearchResult result, int rank) => new( RetrievalChannel.BM25, result.ChunkId, @@ -418,7 +418,7 @@ public sealed class DataSourceLocalRetrievalService( } } - private void LogBm25Results(IInternalDataSource dataSource, IReadOnlyList results) + private void LogBm25Results(IInternalDataSource dataSource, IReadOnlyList results) { if (results.Count == 0) {