replaced embedding_state with index_store as a sqlite name

This commit is contained in:
PaulKoudelka 2026-08-14 12:37:50 +02:00
parent 50aa5c485d
commit 1a263127f5
15 changed files with 129 additions and 129 deletions

View File

@ -8882,34 +8882,34 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T3893997203"] = "
UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T4107860491"] = "Trust all LLM providers" UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T4107860491"] = "Trust all LLM providers"
-- Reason -- Reason
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T1093747001"] = "Reason" UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1093747001"] = "Reason"
-- Starting -- Starting
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T1233211769"] = "Starting" UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1233211769"] = "Starting"
-- Unavailable -- Unavailable
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T3662391977"] = "Unavailable" UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T3662391977"] = "Unavailable"
-- Status -- Status
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T6222351"] = "Status" UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T6222351"] = "Status"
-- Database path -- Database path
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T1100578143"] = "Database path" UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T1100578143"] = "Database path"
-- Storage size -- Storage size
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T1230141403"] = "Storage size" UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T1230141403"] = "Storage size"
-- Indexed files -- Indexed files
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T2235289713"] = "Indexed files" UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T2235289713"] = "Indexed files"
-- Search chunks -- Search chunks
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T2333737457"] = "Search chunks" UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T2333737457"] = "Search chunks"
-- Indexed data sources -- Indexed data sources
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T3524534748"] = "Indexed data sources" UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T3524534748"] = "Indexed data sources"
-- Reported version -- Reported version
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T3556099842"] = "Reported version" UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T3556099842"] = "Reported version"
-- Reason -- Reason
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T1093747001"] = "Reason" UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T1093747001"] = "Reason"

View File

@ -1,4 +1,4 @@
using AIStudio.Tools.Databases.EmbeddingState; using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.Databases.VectorStore; using AIStudio.Tools.Databases.VectorStore;
using AIStudio.Tools.Services; using AIStudio.Tools.Services;
@ -57,13 +57,13 @@ public sealed partial class DatabaseClientProvider(RustService rustService, ILog
client.Status); client.Status);
} }
public async Task<EmbeddingStateClient> GetEmbeddingStateAsync(CancellationToken cancellationToken = default) public async Task<IndexStoreClient> GetIndexStoreAsync(CancellationToken cancellationToken = default)
{ {
var client = await this.GetClientAsync(DatabaseRole.EMBEDDING_STATE, cancellationToken); var client = await this.GetClientAsync(DatabaseRole.INDEX_STORE, cancellationToken);
if (client is EmbeddingStateClient embeddingState) if (client is IndexStoreClient indexStore)
return embeddingState; return indexStore;
return new NoEmbeddingStateClient( return new NoIndexStoreClient(
client.Name, client.Name,
"The configured database client does not support local RAG index operations.", "The configured database client does not support local RAG index operations.",
client.Status); client.Status);
@ -105,7 +105,7 @@ public sealed partial class DatabaseClientProvider(RustService rustService, ILog
private async Task<DatabaseClient> CreateClientAsync(DatabaseRole databaseRole, CancellationToken cancellationToken) => databaseRole switch private async Task<DatabaseClient> CreateClientAsync(DatabaseRole databaseRole, CancellationToken cancellationToken) => databaseRole switch
{ {
DatabaseRole.VECTOR_STORE => await QdrantEdgeClientImplementation.CreateAsync(rustService, this.logger, this.databaseClientLogger, cancellationToken), DatabaseRole.VECTOR_STORE => await QdrantEdgeClientImplementation.CreateAsync(rustService, this.logger, this.databaseClientLogger, cancellationToken),
DatabaseRole.EMBEDDING_STATE => await SqliteEmbeddingStateClientImplementation.CreateAsync(this.logger, this.databaseClientLogger, cancellationToken), DatabaseRole.INDEX_STORE => await SqliteIndexStoreClientImplementation.CreateAsync(this.logger, this.databaseClientLogger, cancellationToken),
_ => new NoDatabaseClient(databaseRole.ToString(), "The requested database role is not supported.") _ => new NoDatabaseClient(databaseRole.ToString(), "The requested database role is not supported.")
}; };

View File

@ -3,5 +3,5 @@ namespace AIStudio.Tools.Databases;
public enum DatabaseRole public enum DatabaseRole
{ {
VECTOR_STORE, VECTOR_STORE,
EMBEDDING_STATE, INDEX_STORE,
} }

View File

@ -1,8 +1,8 @@
using AIStudio.Tools.Services; using AIStudio.Tools.Services;
namespace AIStudio.Tools.Databases.EmbeddingState; namespace AIStudio.Tools.Databases.IndexStore;
public abstract class EmbeddingStateClient(string name, string path) : DatabaseClient(name, path) public abstract class IndexStoreClient(string name, string path) : DatabaseClient(name, path)
{ {
public abstract Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token); public abstract Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token);
@ -26,7 +26,7 @@ public abstract class EmbeddingStateClient(string name, string path) : DatabaseC
public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token); public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token);
public abstract Task<IReadOnlyList<EmbeddingStateSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token); public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token); public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
} }
@ -54,7 +54,7 @@ public sealed record EmbeddingStateChunk(
string ChunkText, string ChunkText,
DateTimeOffset EmbeddedAtUtc); DateTimeOffset EmbeddedAtUtc);
public sealed record EmbeddingStateSearchResult( public sealed record IndexStoreSearchResult(
string ChunkId, string ChunkId,
string ParentFileId, string ParentFileId,
string DataSourceId, string DataSourceId,

View File

@ -4,11 +4,11 @@ using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion; using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
namespace AIStudio.Tools.Databases.EmbeddingState; namespace AIStudio.Tools.Databases.IndexStore;
internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbContext> options) : DbContext(options) internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext> options) : DbContext(options)
{ {
public static DbContextOptions<EmbeddingStateDbContext> CreateOptions(string databasePath) => new DbContextOptionsBuilder<EmbeddingStateDbContext>() public static DbContextOptions<IndexStoreDbContext> CreateOptions(string databasePath) => new DbContextOptionsBuilder<IndexStoreDbContext>()
.UseSqlite(BuildConnectionString(databasePath)) .UseSqlite(BuildConnectionString(databasePath))
.Options; .Options;
@ -18,11 +18,11 @@ internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbC
public DbSet<EmbeddingStateChunkEntity> EmbeddingChunks => this.Set<EmbeddingStateChunkEntity>(); public DbSet<EmbeddingStateChunkEntity> EmbeddingChunks => this.Set<EmbeddingStateChunkEntity>();
public DbSet<EmbeddingStateSearchResultEntity> SearchResults => this.Set<EmbeddingStateSearchResultEntity>(); public DbSet<IndexStoreSearchResultEntity> SearchResults => this.Set<IndexStoreSearchResultEntity>();
protected override void OnModelCreating(ModelBuilder modelBuilder) protected override void OnModelCreating(ModelBuilder modelBuilder)
{ {
var utcDateTimeOffsetConverter = new EmbeddingStateDateTimeOffsetConverter(); var utcDateTimeOffsetConverter = new IndexStoreDateTimeOffsetConverter();
modelBuilder.Entity<EmbeddingStateDataSourceEntity>(entity => modelBuilder.Entity<EmbeddingStateDataSourceEntity>(entity =>
{ {
@ -97,7 +97,7 @@ internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbC
entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index").IsUnique(); entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index").IsUnique();
}); });
modelBuilder.Entity<EmbeddingStateSearchResultEntity>(entity => modelBuilder.Entity<IndexStoreSearchResultEntity>(entity =>
{ {
entity.HasNoKey(); entity.HasNoKey();
entity.ToView("embedding_chunk_search_results"); entity.ToView("embedding_chunk_search_results");
@ -193,7 +193,7 @@ internal sealed class EmbeddingStateChunkEntity
public EmbeddingStateFileEntity? File { get; set; } public EmbeddingStateFileEntity? File { get; set; }
} }
internal sealed class EmbeddingStateSearchResultEntity internal sealed class IndexStoreSearchResultEntity
{ {
public string ChunkId { get; set; } = string.Empty; public string ChunkId { get; set; } = string.Empty;
@ -238,11 +238,11 @@ internal sealed class EmbeddingStateSearchResultEntity
public int ConfidenceLevelRank { get; set; } public int ConfidenceLevelRank { get; set; }
} }
internal sealed class EmbeddingStateDateTimeOffsetConverter() : ValueConverter<DateTimeOffset, string>( internal sealed class IndexStoreDateTimeOffsetConverter() : ValueConverter<DateTimeOffset, string>(
value => EmbeddingStateDateTimeOffset.ToUtcText(value), value => IndexStoreDateTimeOffset.ToUtcText(value),
value => EmbeddingStateDateTimeOffset.ParseUtc(value)); value => IndexStoreDateTimeOffset.ParseUtc(value));
internal static class EmbeddingStateDateTimeOffset internal static class IndexStoreDateTimeOffset
{ {
public static string ToUtcText(DateTimeOffset dateTime) public static string ToUtcText(DateTimeOffset dateTime)
{ {

View File

@ -1,15 +1,15 @@
using Microsoft.EntityFrameworkCore.Design; using Microsoft.EntityFrameworkCore.Design;
namespace AIStudio.Tools.Databases.EmbeddingState; namespace AIStudio.Tools.Databases.IndexStore;
internal sealed class EmbeddingStateDesignTimeDbContextFactory : IDesignTimeDbContextFactory<EmbeddingStateDbContext> internal sealed class IndexStoreDesignTimeDbContextFactory : IDesignTimeDbContextFactory<IndexStoreDbContext>
{ {
public EmbeddingStateDbContext CreateDbContext(string[] args) public IndexStoreDbContext CreateDbContext(string[] args)
{ {
var databasePath = args.FirstOrDefault(argument => argument.EndsWith(".sqlite3", StringComparison.OrdinalIgnoreCase)); var databasePath = args.FirstOrDefault(argument => argument.EndsWith(".sqlite3", StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrWhiteSpace(databasePath)) if (string.IsNullOrWhiteSpace(databasePath))
databasePath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-rag-index-design.sqlite3"); databasePath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-rag-index-design.sqlite3");
return new EmbeddingStateDbContext(EmbeddingStateDbContext.CreateOptions(databasePath)); return new IndexStoreDbContext(IndexStoreDbContext.CreateOptions(databasePath));
} }
} }

View File

@ -2,12 +2,12 @@ using System.Diagnostics.CodeAnalysis;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
namespace AIStudio.Tools.Databases.EmbeddingState; namespace AIStudio.Tools.Databases.IndexStore;
internal static class EmbeddingStateSchemaMigrator internal static class IndexStoreSchemaMigrator
{ {
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
public static async Task MigrateAsync(EmbeddingStateDbContext context, CancellationToken token) public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
{ {
await context.Database.MigrateAsync(token); await context.Database.MigrateAsync(token);
await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL;", token); await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL;", token);

View File

@ -3,9 +3,9 @@
using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations; using Microsoft.EntityFrameworkCore.Migrations;
namespace AIStudio.Tools.Databases.EmbeddingState.Migrations; namespace AIStudio.Tools.Databases.IndexStore.Migrations;
[DbContext(typeof(EmbeddingStateDbContext))] [DbContext(typeof(IndexStoreDbContext))]
[Migration("20260804000000_InitialRagIndex")] [Migration("20260804000000_InitialRagIndex")]
public partial class InitialRagIndex : Migration public partial class InitialRagIndex : Migration
{ {

View File

@ -3,18 +3,18 @@
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure; using Microsoft.EntityFrameworkCore.Infrastructure;
namespace AIStudio.Tools.Databases.EmbeddingState.Migrations; namespace AIStudio.Tools.Databases.IndexStore.Migrations;
[DbContext(typeof(EmbeddingStateDbContext))] [DbContext(typeof(IndexStoreDbContext))]
partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
{ {
protected override void BuildModel(ModelBuilder modelBuilder) protected override void BuildModel(ModelBuilder modelBuilder)
{ {
#pragma warning disable 612, 618 #pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "9.0.18"); modelBuilder.HasAnnotation("ProductVersion", "9.0.18");
var utcDateTimeOffsetConverter = new EmbeddingStateDateTimeOffsetConverter(); var utcDateTimeOffsetConverter = new IndexStoreDateTimeOffsetConverter();
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity => modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity =>
{ {
entity.Property<string>("DataSourceId") entity.Property<string>("DataSourceId")
.HasColumnType("TEXT") .HasColumnType("TEXT")
@ -61,7 +61,7 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
entity.ToTable("data_sources"); entity.ToTable("data_sources");
}); });
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity => modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
{ {
entity.Property<string>("ParentFileId") entity.Property<string>("ParentFileId")
.HasColumnType("TEXT") .HasColumnType("TEXT")
@ -151,7 +151,7 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
entity.ToTable("embedded_files"); entity.ToTable("embedded_files");
}); });
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateChunkEntity", entity => modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateChunkEntity", entity =>
{ {
entity.Property<int>("Id") entity.Property<int>("Id")
.ValueGeneratedOnAdd() .ValueGeneratedOnAdd()
@ -206,7 +206,7 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
entity.ToTable("embedding_chunks"); entity.ToTable("embedding_chunks");
}); });
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateSearchResultEntity", entity => modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.IndexStoreSearchResultEntity", entity =>
{ {
entity.Property<string>("AbsolutePath") entity.Property<string>("AbsolutePath")
.IsRequired() .IsRequired()
@ -291,9 +291,9 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
entity.ToView("embedding_chunk_search_results"); entity.ToView("embedding_chunk_search_results");
}); });
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity => modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
{ {
entity.HasOne("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", "DataSource") entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", "DataSource")
.WithMany("Files") .WithMany("Files")
.HasForeignKey("DataSourceId") .HasForeignKey("DataSourceId")
.OnDelete(DeleteBehavior.Cascade) .OnDelete(DeleteBehavior.Cascade)
@ -302,9 +302,9 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
entity.Navigation("DataSource"); entity.Navigation("DataSource");
}); });
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateChunkEntity", entity => modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateChunkEntity", entity =>
{ {
entity.HasOne("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", "File") entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", "File")
.WithMany("Chunks") .WithMany("Chunks")
.HasForeignKey("ParentFileId") .HasForeignKey("ParentFileId")
.OnDelete(DeleteBehavior.Cascade) .OnDelete(DeleteBehavior.Cascade)
@ -313,12 +313,12 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
entity.Navigation("File"); entity.Navigation("File");
}); });
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity => modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity =>
{ {
entity.Navigation("Files"); entity.Navigation("Files");
}); });
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity => modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
{ {
entity.Navigation("Chunks"); entity.Navigation("Chunks");
}); });

View File

@ -1,11 +1,11 @@
using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Services; using AIStudio.Tools.Services;
namespace AIStudio.Tools.Databases.EmbeddingState; namespace AIStudio.Tools.Databases.IndexStore;
public sealed class NoEmbeddingStateClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : EmbeddingStateClient(name, string.Empty) public sealed class NoIndexStoreClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : IndexStoreClient(name, string.Empty)
{ {
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoEmbeddingStateClient).Namespace, nameof(NoEmbeddingStateClient)); private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoIndexStoreClient).Namespace, nameof(NoIndexStoreClient));
public override DatabaseClientStatus Status => status; public override DatabaseClientStatus Status => status;
@ -46,8 +46,8 @@ public sealed class NoEmbeddingStateClient(string name, string? unavailableReaso
public override Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token) => Task.CompletedTask; public override Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token) => Task.CompletedTask;
public override Task<IReadOnlyList<EmbeddingStateSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) => public override Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) =>
Task.FromResult<IReadOnlyList<EmbeddingStateSearchResult>>([]); Task.FromResult<IReadOnlyList<IndexStoreSearchResult>>([]);
public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask; public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask;

View File

@ -7,13 +7,13 @@ using AIStudio.Tools.Services;
using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore;
namespace AIStudio.Tools.Databases.EmbeddingState; namespace AIStudio.Tools.Databases.IndexStore;
public sealed class SqliteEmbeddingStateClientImplementation( public sealed class SqliteIndexStoreClientImplementation(
string name, string name,
string databasePath, string databasePath,
string basePath, string basePath,
string version) : EmbeddingStateClient(name, basePath) string version) : IndexStoreClient(name, basePath)
{ {
private const string DATABASE_NAME = "Local RAG Index"; private const string DATABASE_NAME = "Local RAG Index";
private const string DATABASE_FILENAME = "rag-index.sqlite3"; private const string DATABASE_FILENAME = "rag-index.sqlite3";
@ -23,9 +23,9 @@ public sealed class SqliteEmbeddingStateClientImplementation(
private static readonly Regex FTS_TOKEN_REGEX = new(@"[\p{L}\p{Nd}_]+", RegexOptions.Compiled | RegexOptions.CultureInvariant); private static readonly Regex FTS_TOKEN_REGEX = new(@"[\p{L}\p{Nd}_]+", RegexOptions.Compiled | RegexOptions.CultureInvariant);
private readonly string databasePath = databasePath; private readonly string databasePath = databasePath;
private readonly DbContextOptions<EmbeddingStateDbContext> dbContextOptions = EmbeddingStateDbContext.CreateOptions(databasePath); private readonly DbContextOptions<IndexStoreDbContext> dbContextOptions = IndexStoreDbContext.CreateOptions(databasePath);
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SqliteEmbeddingStateClientImplementation).Namespace, nameof(SqliteEmbeddingStateClientImplementation)); private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SqliteIndexStoreClientImplementation).Namespace, nameof(SqliteIndexStoreClientImplementation));
public override string CacheKey => $"{this.Name}:{this.databasePath}:{version}"; public override string CacheKey => $"{this.Name}:{this.databasePath}:{version}";
@ -35,7 +35,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
CancellationToken cancellationToken) CancellationToken cancellationToken)
{ {
if (string.IsNullOrWhiteSpace(SettingsManager.DataDirectory)) if (string.IsNullOrWhiteSpace(SettingsManager.DataDirectory))
return CreateNoEmbeddingStateClient(DATABASE_NAME, "The application data directory is not available yet.", DatabaseClientStatus.STARTING, databaseClientLogger); return CreateNoIndexStoreClient(DATABASE_NAME, "The application data directory is not available yet.", DatabaseClientStatus.STARTING, databaseClientLogger);
try try
{ {
@ -45,18 +45,18 @@ public sealed class SqliteEmbeddingStateClientImplementation(
Directory.CreateDirectory(basePath); Directory.CreateDirectory(basePath);
var databasePath = Path.Combine(basePath, DATABASE_FILENAME); var databasePath = Path.Combine(basePath, DATABASE_FILENAME);
var client = new SqliteEmbeddingStateClientImplementation(DATABASE_NAME, databasePath, basePath, string.Empty); var client = new SqliteIndexStoreClientImplementation(DATABASE_NAME, databasePath, basePath, string.Empty);
await client.InitializeAsync(cancellationToken); await client.InitializeAsync(cancellationToken);
var version = await client.GetSqliteVersionAsync(cancellationToken); var version = await client.GetSqliteVersionAsync(cancellationToken);
client = new SqliteEmbeddingStateClientImplementation(DATABASE_NAME, databasePath, basePath, version); client = new SqliteIndexStoreClientImplementation(DATABASE_NAME, databasePath, basePath, version);
client.SetLogger(databaseClientLogger); client.SetLogger(databaseClientLogger);
return client; return client;
} }
catch (Exception exception) catch (Exception exception)
{ {
logger.LogWarning(exception, "{DatabaseName} is not available. Indexed file fingerprints and search chunks are disabled.", DATABASE_NAME); logger.LogWarning(exception, "{DatabaseName} is not available. Indexed file fingerprints and search chunks are disabled.", DATABASE_NAME);
return CreateNoEmbeddingStateClient(DATABASE_NAME, exception.Message, DatabaseClientStatus.UNAVAILABLE, databaseClientLogger); return CreateNoIndexStoreClient(DATABASE_NAME, exception.Message, DatabaseClientStatus.UNAVAILABLE, databaseClientLogger);
} }
} }
@ -236,7 +236,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
await transaction.CommitAsync(token); await transaction.CommitAsync(token);
} }
public override async Task<IReadOnlyList<EmbeddingStateSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) public override async Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token)
{ {
if (maxMatches <= 0) if (maxMatches <= 0)
return []; return [];
@ -314,7 +314,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
private async Task InitializeAsync(CancellationToken token) private async Task InitializeAsync(CancellationToken token)
{ {
await using var context = this.CreateContext(); await using var context = this.CreateContext();
await EmbeddingStateSchemaMigrator.MigrateAsync(context, token); await IndexStoreSchemaMigrator.MigrateAsync(context, token);
} }
private async Task<string> GetSqliteVersionAsync(CancellationToken token) private async Task<string> GetSqliteVersionAsync(CancellationToken token)
@ -326,7 +326,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
return versions.FirstOrDefault() ?? string.Empty; return versions.FirstOrDefault() ?? string.Empty;
} }
private EmbeddingStateDbContext CreateContext() => new(this.dbContextOptions); private IndexStoreDbContext CreateContext() => new(this.dbContextOptions);
private static void ApplyDataSource( private static void ApplyDataSource(
EmbeddingStateDataSourceEntity dataSource, EmbeddingStateDataSourceEntity dataSource,
@ -373,7 +373,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
chunkEntity.EmbeddedAtUtc = chunk.EmbeddedAtUtc; chunkEntity.EmbeddedAtUtc = chunk.EmbeddedAtUtc;
} }
private static EmbeddingStateSearchResult ToSearchResult(EmbeddingStateSearchResultEntity result) => new( private static IndexStoreSearchResult ToSearchResult(IndexStoreSearchResultEntity result) => new(
result.ChunkId, result.ChunkId,
result.ParentFileId, result.ParentFileId,
result.DataSourceId, result.DataSourceId,
@ -410,9 +410,9 @@ public sealed class SqliteEmbeddingStateClientImplementation(
return terms.Count == 0 ? string.Empty : string.Join(" OR ", terms); return terms.Count == 0 ? string.Empty : string.Join(" OR ", terms);
} }
private static NoEmbeddingStateClient CreateNoEmbeddingStateClient(string name, string? unavailableReason, DatabaseClientStatus status, ILogger<DatabaseClient> databaseClientLogger) private static NoIndexStoreClient CreateNoIndexStoreClient(string name, string? unavailableReason, DatabaseClientStatus status, ILogger<DatabaseClient> databaseClientLogger)
{ {
var client = new NoEmbeddingStateClient(name, unavailableReason, status); var client = new NoIndexStoreClient(name, unavailableReason, status);
client.SetLogger(databaseClientLogger); client.SetLogger(databaseClientLogger);
return client; return client;
} }

View File

@ -5,7 +5,7 @@ using System.Text.RegularExpressions;
using AIStudio.Provider; using AIStudio.Provider;
using AIStudio.Settings; using AIStudio.Settings;
using AIStudio.Settings.DataModel; using AIStudio.Settings.DataModel;
using AIStudio.Tools.Databases.EmbeddingState; using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Rust; using AIStudio.Tools.Rust;

View File

@ -1,4 +1,4 @@
using AIStudio.Tools.Databases.EmbeddingState; using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.Databases.VectorStore; using AIStudio.Tools.Databases.VectorStore;
namespace AIStudio.Tools.Services; namespace AIStudio.Tools.Services;
@ -8,19 +8,19 @@ public sealed partial class DataSourceEmbeddingService
private async Task ResetPersistedStateAsync( private async Task ResetPersistedStateAsync(
string dataSourceId, string dataSourceId,
VectorStoreClient? vectorStore, VectorStoreClient? vectorStore,
EmbeddingStateClient? embeddingState, IndexStoreClient? indexStore,
CancellationToken token) CancellationToken token)
{ {
await this.DeleteCollectionAsync(DataSourceEmbeddingNames.GetCollectionName(dataSourceId), vectorStore, token); await this.DeleteCollectionAsync(DataSourceEmbeddingNames.GetCollectionName(dataSourceId), vectorStore, token);
embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token); indexStore ??= await databaseClientProvider.GetIndexStoreAsync(token);
if (!embeddingState.IsAvailable) if (!indexStore.IsAvailable)
{ {
logger.LogWarning("Could not delete local RAG index state for data source '{DataSourceId}' because the database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name); logger.LogWarning("Could not delete local RAG embedding state for data source '{DataSourceId}' because the database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
return; return;
} }
await embeddingState.DeleteDataSourceAsync(dataSourceId, token); await indexStore.DeleteDataSourceAsync(dataSourceId, token);
logger.LogInformation("Reset persisted local RAG index state for data source '{DataSourceId}'.", dataSourceId); logger.LogInformation("Reset persisted local RAG embedding state for data source '{DataSourceId}'.", dataSourceId);
} }
} }

View File

@ -7,7 +7,7 @@ using AIStudio.Provider;
using AIStudio.Settings; using AIStudio.Settings;
using AIStudio.Settings.DataModel; using AIStudio.Settings.DataModel;
using AIStudio.Tools.Databases; using AIStudio.Tools.Databases;
using AIStudio.Tools.Databases.EmbeddingState; using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.Databases.VectorStore; using AIStudio.Tools.Databases.VectorStore;
using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem;
@ -188,14 +188,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
public async Task<bool> ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default) public async Task<bool> ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default)
{ {
var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token); var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
if (!embeddingState.IsAvailable) if (!indexStore.IsAvailable)
{ {
logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name); logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
return true; return true;
} }
var manifest = await embeddingState.GetManifestAsync(dataSourceId, token); var manifest = await indexStore.GetManifestAsync(dataSourceId, token);
return !string.IsNullOrWhiteSpace(manifest.EmbeddingProviderId) return !string.IsNullOrWhiteSpace(manifest.EmbeddingProviderId)
|| !string.IsNullOrWhiteSpace(manifest.EmbeddingSignature) || !string.IsNullOrWhiteSpace(manifest.EmbeddingSignature)
|| !string.IsNullOrWhiteSpace(manifest.SourceHash) || !string.IsNullOrWhiteSpace(manifest.SourceHash)
@ -410,7 +410,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
token.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested();
var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token); var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token);
var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token); var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
token.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested();
if (!vectorStore.IsAvailable) if (!vectorStore.IsAvailable)
@ -425,20 +425,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
return; return;
} }
if (!embeddingState.IsAvailable) if (!indexStore.IsAvailable)
{ {
logger.LogWarning( logger.LogWarning(
"Skipping background embeddings for data source '{DataSourceName}' ({DataSourceId}) because the database client '{DatabaseName}' is unavailable.", "Skipping background embeddings for data source '{DataSourceName}' ({DataSourceId}) because the database client '{DatabaseName}' is unavailable.",
dataSource.Name, dataSource.Name,
dataSource.Id, dataSource.Id,
embeddingState.Name); indexStore.Name);
token.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested();
this.UpsertStatus(this.GetFallbackStatus(dataSource, "The local RAG index database is not available.")); this.UpsertStatus(this.GetFallbackStatus(dataSource, "The local RAG index database is not available."));
return; return;
} }
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id); var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
var persistedManifest = await embeddingState.GetManifestAsync(dataSource.Id, token); var persistedManifest = await indexStore.GetManifestAsync(dataSource.Id, token);
if (persistedManifest.VectorSize > 0) if (persistedManifest.VectorSize > 0)
{ {
var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedManifest.VectorSize, token); var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedManifest.VectorSize, token);
@ -449,7 +449,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
collectionName, collectionName,
dataSource.Name, dataSource.Name,
dataSource.Id); dataSource.Id);
await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, embeddingState, token); await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, indexStore, token);
} }
} }
@ -484,7 +484,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Name, dataSource.Name,
dataSource.Id); dataSource.Id);
var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, embeddingState, token); var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, indexStore, token);
token.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested();
var inputFiles = this.GetInputFiles(dataSource); var inputFiles = this.GetInputFiles(dataSource);
@ -510,7 +510,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
collectionName); collectionName);
var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles); var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles);
var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(vectorStore, embeddingState, dataSource, collectionName, manifest, indexedFiles, token); var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(vectorStore, indexStore, dataSource, collectionName, manifest, indexedFiles, token);
var optimizationTracker = new VectorStoreOptimizationTracker(); var optimizationTracker = new VectorStoreOptimizationTracker();
if (removedMissingFiles > 0) if (removedMissingFiles > 0)
optimizationTracker.MarkChanged(); optimizationTracker.MarkChanged();
@ -543,7 +543,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
token); token);
token.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested();
await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token); await indexStore.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token);
this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, indexedFiles.Count, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures)); this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, indexedFiles.Count, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures));
return; return;
} }
@ -602,7 +602,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
skippedFiles + completedFiles + 1, skippedFiles + completedFiles + 1,
totalFiles); totalFiles);
var startedAtUtc = DateTimeOffset.UtcNow; var startedAtUtc = DateTimeOffset.UtcNow;
var chunkCount = await this.IndexOneFileAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token); var chunkCount = await this.IndexOneFileAsync(indexStore, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token);
token.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested();
var fingerprintAfterEmbedding = BuildFileMetadataHash(file); var fingerprintAfterEmbedding = BuildFileMetadataHash(file);
if (!string.Equals(fingerprint, fingerprintAfterEmbedding, StringComparison.Ordinal)) if (!string.Equals(fingerprint, fingerprintAfterEmbedding, StringComparison.Ordinal))
@ -615,7 +615,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
new DateTimeOffset(file.LastWriteTimeUtc), new DateTimeOffset(file.LastWriteTimeUtc),
embeddedAtUtc, embeddedAtUtc,
chunkCount); chunkCount);
await embeddingState.UpsertFileAsync( await indexStore.UpsertFileAsync(
dataSource.Id, dataSource.Id,
this.CreateEmbeddingStateFile(dataSource, file, fingerprint, chunkCount, embeddedAtUtc), this.CreateEmbeddingStateFile(dataSource, file, fingerprint, chunkCount, embeddedAtUtc),
token); token);
@ -644,7 +644,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
lastError = exception.Message; lastError = exception.Message;
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message)); failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message));
manifest.Files.Remove(file.FullName); manifest.Files.Remove(file.FullName);
await this.CleanupFailedFileAsync(embeddingState, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token); await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token);
logger.LogWarning(exception, "Failed to embed file '{FilePath}' for data source '{DataSourceName}'.", file.FullName, dataSource.Name); logger.LogWarning(exception, "Failed to embed file '{FilePath}' for data source '{DataSourceName}'.", file.FullName, dataSource.Name);
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, exception.Message, failureDetails)); this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, exception.Message, failureDetails));
@ -662,7 +662,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
token); token);
token.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested();
await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token); await indexStore.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token);
token.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested();
this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, skippedFiles + completedFiles, failedFiles, lastError, failureDetails)); this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, skippedFiles + completedFiles, failedFiles, lastError, failureDetails));
@ -682,7 +682,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
} }
private async Task<int> IndexOneFileAsync( private async Task<int> IndexOneFileAsync(
EmbeddingStateClient embeddingState, IndexStoreClient indexStore,
VectorStoreClient vectorStore, VectorStoreClient vectorStore,
IDataSource dataSource, IDataSource dataSource,
FileInfo file, FileInfo file,
@ -700,10 +700,10 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
collectionName); collectionName);
await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token); await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token);
optimizationTracker.MarkChanged(); optimizationTracker.MarkChanged();
await embeddingState.DeleteFileAsync(dataSource.Id, file.FullName, token); await indexStore.DeleteFileAsync(dataSource.Id, file.FullName, token);
var parentFile = this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow); var parentFile = this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow);
await embeddingState.UpsertFileAsync(dataSource.Id, parentFile, token); await indexStore.UpsertFileAsync(dataSource.Id, parentFile, token);
var embeddingBatchSize = Math.Max(1, embeddingProvider.EffectiveEmbeddingBatchSize); var embeddingBatchSize = Math.Max(1, embeddingProvider.EffectiveEmbeddingBatchSize);
var batch = new List<EmbeddingChunkDraft>(embeddingBatchSize); var batch = new List<EmbeddingChunkDraft>(embeddingBatchSize);
@ -715,11 +715,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
totalChunkCount++; totalChunkCount++;
if (batch.Count >= embeddingBatchSize) if (batch.Count >= embeddingBatchSize)
await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token); await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token);
} }
if (batch.Count > 0) if (batch.Count > 0)
await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token); await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token);
if (totalChunkCount == 0) if (totalChunkCount == 0)
throw new InvalidOperationException($"The file '{file.Name}' did not yield any text chunks."); throw new InvalidOperationException($"The file '{file.Name}' did not yield any text chunks.");
@ -735,7 +735,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
} }
private async Task FlushBatchAsync( private async Task FlushBatchAsync(
EmbeddingStateClient embeddingState, IndexStoreClient indexStore,
VectorStoreClient vectorStore, VectorStoreClient vectorStore,
IDataSource dataSource, IDataSource dataSource,
FileInfo file, FileInfo file,
@ -805,7 +805,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly."); throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly.");
} }
await embeddingState.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token); await indexStore.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token);
manifest.VectorSize = vectorSize; manifest.VectorSize = vectorSize;
logger.LogInformation( logger.LogInformation(
"Created embedding collection '{CollectionName}' with vector size {VectorSize} for data source '{DataSourceName}' ({DataSourceId}).", "Created embedding collection '{CollectionName}' with vector size {VectorSize} for data source '{DataSourceName}' ({DataSourceId}).",
@ -829,7 +829,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
embeddedAtUtc, embeddedAtUtc,
token); token);
token.ThrowIfCancellationRequested(); token.ThrowIfCancellationRequested();
await embeddingState.UpsertChunksAsync( await indexStore.UpsertChunksAsync(
dataSource.Id, dataSource.Id,
this.CreateEmbeddingStateChunks(parentFile, batch, embeddedAtUtc), this.CreateEmbeddingStateChunks(parentFile, batch, embeddedAtUtc),
token); token);
@ -897,7 +897,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
} }
private async Task CleanupFailedFileAsync( private async Task CleanupFailedFileAsync(
EmbeddingStateClient embeddingState, IndexStoreClient indexStore,
VectorStoreClient vectorStore, VectorStoreClient vectorStore,
IDataSource dataSource, IDataSource dataSource,
string collectionName, string collectionName,
@ -926,7 +926,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
try try
{ {
await embeddingState.DeleteFileAsync(dataSource.Id, filePath, token); await indexStore.DeleteFileAsync(dataSource.Id, filePath, token);
} }
catch (OperationCanceledException) when (token.IsCancellationRequested) catch (OperationCanceledException) when (token.IsCancellationRequested)
{ {
@ -1018,7 +1018,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
.ToList(); .ToList();
logger.LogInformation( logger.LogInformation(
"Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed local RAG index state will be retried during this pass. File watchers will be activated after this check completes.", "Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed local RAG embedding state will be retried during this pass. File watchers will be activated after this check completes.",
supportedDataSources.Count); supportedDataSources.Count);
foreach (var dataSource in supportedDataSources) foreach (var dataSource in supportedDataSources)
@ -1074,12 +1074,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
EmbeddingProvider embeddingProvider, EmbeddingProvider embeddingProvider,
string collectionName, string collectionName,
VectorStoreClient vectorStore, VectorStoreClient vectorStore,
EmbeddingStateClient embeddingState, IndexStoreClient indexStore,
CancellationToken token) CancellationToken token)
{ {
var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider); var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token); var manifest = await indexStore.GetManifestAsync(dataSource.Id, token);
logger.LogInformation( logger.LogInformation(
"Loaded persisted local RAG index manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.", "Loaded persisted local RAG index manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.",
@ -1093,7 +1093,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
if (!string.Equals(manifest.EmbeddingSignature, embeddingSignature, StringComparison.Ordinal)) if (!string.Equals(manifest.EmbeddingSignature, embeddingSignature, StringComparison.Ordinal))
{ {
logger.LogInformation( logger.LogInformation(
"Embedding configuration changed for data source '{DataSourceName}' ({DataSourceId}). Resetting persisted state and collection '{CollectionName}'.", "Embedding configuration changed for data source '{DataSourceName}' ({DataSourceId}). Resetting persisted embedding state and collection '{CollectionName}'.",
dataSource.Name, dataSource.Name,
dataSource.Id, dataSource.Id,
collectionName); collectionName);
@ -1103,8 +1103,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Id, dataSource.Id,
manifest.EmbeddingSignature, manifest.EmbeddingSignature,
embeddingSignature); embeddingSignature);
await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, embeddingState, token); await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, indexStore, token);
manifest = await embeddingState.GetManifestAsync(dataSource.Id, token); manifest = await indexStore.GetManifestAsync(dataSource.Id, token);
} }
if (!string.Equals(manifest.EmbeddingProviderId, embeddingProvider.Id, StringComparison.OrdinalIgnoreCase) || if (!string.Equals(manifest.EmbeddingProviderId, embeddingProvider.Id, StringComparison.OrdinalIgnoreCase) ||
@ -1114,7 +1114,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
manifest.EmbeddingSignature = embeddingSignature; manifest.EmbeddingSignature = embeddingSignature;
} }
await embeddingState.UpsertDataSourceAsync( await indexStore.UpsertDataSourceAsync(
dataSource.Id, dataSource.Id,
dataSource.Name, dataSource.Name,
dataSource.Type.ToString(), dataSource.Type.ToString(),
@ -1129,7 +1129,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
private async Task<int> RemoveMissingFileEmbeddingsAsync( private async Task<int> RemoveMissingFileEmbeddingsAsync(
VectorStoreClient vectorStore, VectorStoreClient vectorStore,
EmbeddingStateClient embeddingState, IndexStoreClient indexStore,
IDataSource dataSource, IDataSource dataSource,
string collectionName, string collectionName,
DataSourceEmbeddingManifest manifest, DataSourceEmbeddingManifest manifest,
@ -1144,7 +1144,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
foreach (var removedFilePath in manifest.Files.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList()) foreach (var removedFilePath in manifest.Files.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList())
{ {
await this.DeleteFilePointsAsync(vectorStore, collectionName, removedFilePath, token); await this.DeleteFilePointsAsync(vectorStore, collectionName, removedFilePath, token);
await embeddingState.DeleteFileAsync(dataSource.Id, removedFilePath, token); await indexStore.DeleteFileAsync(dataSource.Id, removedFilePath, token);
manifest.Files.Remove(removedFilePath); manifest.Files.Remove(removedFilePath);
removedFiles++; removedFiles++;
logger.LogInformation( logger.LogInformation(

View File

@ -3,7 +3,7 @@ using AIStudio.Provider;
using AIStudio.Settings; using AIStudio.Settings;
using AIStudio.Settings.DataModel; using AIStudio.Settings.DataModel;
using AIStudio.Tools.Databases; using AIStudio.Tools.Databases;
using AIStudio.Tools.Databases.EmbeddingState; using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.Databases.VectorStore; using AIStudio.Tools.Databases.VectorStore;
using AIStudio.Tools.RAG; using AIStudio.Tools.RAG;
using AIStudio.Tools.Rust; using AIStudio.Tools.Rust;
@ -188,25 +188,25 @@ public sealed class DataSourceLocalRetrievalService(
return true; return true;
} }
private async Task<IReadOnlyList<EmbeddingStateSearchResult>> SearchBm25Async(IInternalDataSource dataSource, string query, int maxMatches, CancellationToken token) private async Task<IReadOnlyList<IndexStoreSearchResult>> SearchBm25Async(IInternalDataSource dataSource, string query, int maxMatches, CancellationToken token)
{ {
try try
{ {
var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token); var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
if (!embeddingState.IsAvailable) if (!indexStore.IsAvailable)
{ {
logger.LogWarning( logger.LogWarning(
"Skipping BM25 retrieval for data source '{DataSourceName}' ({DataSourceId}) because local RAG index '{DatabaseName}' is unavailable.", "Skipping BM25 retrieval for data source '{DataSourceName}' ({DataSourceId}) because local RAG index '{DatabaseName}' is unavailable.",
dataSource.Name, dataSource.Name,
dataSource.Id, dataSource.Id,
embeddingState.Name); indexStore.Name);
return []; return [];
} }
var results = this.LimitSearchResults( var results = this.LimitSearchResults(
dataSource, dataSource,
"BM25", "BM25",
await embeddingState.SearchChunksAsync(dataSource.Id, query, maxMatches, token), await indexStore.SearchChunksAsync(dataSource.Id, query, maxMatches, token),
maxMatches); maxMatches);
this.LogBm25Results(dataSource, results); this.LogBm25Results(dataSource, results);
return results; return results;
@ -240,7 +240,7 @@ public sealed class DataSourceLocalRetrievalService(
private static IReadOnlyList<LocalRetrievalHit> MergeResults( private static IReadOnlyList<LocalRetrievalHit> MergeResults(
IReadOnlyList<VectorSearchResult> vectorResults, IReadOnlyList<VectorSearchResult> vectorResults,
IReadOnlyList<EmbeddingStateSearchResult> bm25Results, IReadOnlyList<IndexStoreSearchResult> bm25Results,
int maxMatches) int maxMatches)
{ {
// Future reranking should replace this deterministic channel merge. // Future reranking should replace this deterministic channel merge.
@ -299,7 +299,7 @@ public sealed class DataSourceLocalRetrievalService(
result.ConfidenceLevel, result.ConfidenceLevel,
result.ConfidenceLevelRank); result.ConfidenceLevelRank);
private static LocalRetrievalHit FromBm25Result(EmbeddingStateSearchResult result, int rank) => private static LocalRetrievalHit FromBm25Result(IndexStoreSearchResult result, int rank) =>
new( new(
RetrievalChannel.BM25, RetrievalChannel.BM25,
result.ChunkId, result.ChunkId,
@ -418,7 +418,7 @@ public sealed class DataSourceLocalRetrievalService(
} }
} }
private void LogBm25Results(IInternalDataSource dataSource, IReadOnlyList<EmbeddingStateSearchResult> results) private void LogBm25Results(IInternalDataSource dataSource, IReadOnlyList<IndexStoreSearchResult> results)
{ {
if (results.Count == 0) if (results.Count == 0)
{ {