mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-08-16 00:02:12 +00:00
replaced embedding_state with index_store as a sqlite name
This commit is contained in:
PaulKoudelka
parent
50aa5c485d
commit
1a263127f5
@ -8882,34 +8882,34 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T3893997203"] = "
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T4107860491"] = "Trust all LLM providers"
|
||||
|
||||
-- Reason
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T1093747001"] = "Reason"
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1093747001"] = "Reason"
|
||||
|
||||
-- Starting
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T1233211769"] = "Starting"
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1233211769"] = "Starting"
|
||||
|
||||
-- Unavailable
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T3662391977"] = "Unavailable"
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T3662391977"] = "Unavailable"
|
||||
|
||||
-- Status
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::NOEMBEDDINGSTATECLIENT::T6222351"] = "Status"
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T6222351"] = "Status"
|
||||
|
||||
-- Database path
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T1100578143"] = "Database path"
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T1100578143"] = "Database path"
|
||||
|
||||
-- Storage size
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T1230141403"] = "Storage size"
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T1230141403"] = "Storage size"
|
||||
|
||||
-- Indexed files
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T2235289713"] = "Indexed files"
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T2235289713"] = "Indexed files"
|
||||
|
||||
-- Search chunks
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T2333737457"] = "Search chunks"
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T2333737457"] = "Search chunks"
|
||||
|
||||
-- Indexed data sources
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T3524534748"] = "Indexed data sources"
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T3524534748"] = "Indexed data sources"
|
||||
|
||||
-- Reported version
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T3556099842"] = "Reported version"
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::SQLITEINDEXSTORECLIENTIMPLEMENTATION::T3556099842"] = "Reported version"
|
||||
|
||||
-- Reason
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T1093747001"] = "Reason"
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
using AIStudio.Tools.Databases.EmbeddingState;
|
||||
using AIStudio.Tools.Databases.IndexStore;
|
||||
using AIStudio.Tools.Databases.VectorStore;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
@ -57,13 +57,13 @@ public sealed partial class DatabaseClientProvider(RustService rustService, ILog
|
||||
client.Status);
|
||||
}
|
||||
|
||||
public async Task<EmbeddingStateClient> GetEmbeddingStateAsync(CancellationToken cancellationToken = default)
|
||||
public async Task<IndexStoreClient> GetIndexStoreAsync(CancellationToken cancellationToken = default)
|
||||
{
|
||||
var client = await this.GetClientAsync(DatabaseRole.EMBEDDING_STATE, cancellationToken);
|
||||
if (client is EmbeddingStateClient embeddingState)
|
||||
return embeddingState;
|
||||
var client = await this.GetClientAsync(DatabaseRole.INDEX_STORE, cancellationToken);
|
||||
if (client is IndexStoreClient indexStore)
|
||||
return indexStore;
|
||||
|
||||
return new NoEmbeddingStateClient(
|
||||
return new NoIndexStoreClient(
|
||||
client.Name,
|
||||
"The configured database client does not support local RAG index operations.",
|
||||
client.Status);
|
||||
@ -105,7 +105,7 @@ public sealed partial class DatabaseClientProvider(RustService rustService, ILog
|
||||
private async Task<DatabaseClient> CreateClientAsync(DatabaseRole databaseRole, CancellationToken cancellationToken) => databaseRole switch
|
||||
{
|
||||
DatabaseRole.VECTOR_STORE => await QdrantEdgeClientImplementation.CreateAsync(rustService, this.logger, this.databaseClientLogger, cancellationToken),
|
||||
DatabaseRole.EMBEDDING_STATE => await SqliteEmbeddingStateClientImplementation.CreateAsync(this.logger, this.databaseClientLogger, cancellationToken),
|
||||
DatabaseRole.INDEX_STORE => await SqliteIndexStoreClientImplementation.CreateAsync(this.logger, this.databaseClientLogger, cancellationToken),
|
||||
_ => new NoDatabaseClient(databaseRole.ToString(), "The requested database role is not supported.")
|
||||
};
|
||||
|
||||
|
||||
@ -3,5 +3,5 @@ namespace AIStudio.Tools.Databases;
|
||||
public enum DatabaseRole
|
||||
{
|
||||
VECTOR_STORE,
|
||||
EMBEDDING_STATE,
|
||||
INDEX_STORE,
|
||||
}
|
||||
@ -1,8 +1,8 @@
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState;
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
public abstract class EmbeddingStateClient(string name, string path) : DatabaseClient(name, path)
|
||||
public abstract class IndexStoreClient(string name, string path) : DatabaseClient(name, path)
|
||||
{
|
||||
public abstract Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token);
|
||||
|
||||
@ -26,7 +26,7 @@ public abstract class EmbeddingStateClient(string name, string path) : DatabaseC
|
||||
|
||||
public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token);
|
||||
|
||||
public abstract Task<IReadOnlyList<EmbeddingStateSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
|
||||
public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
|
||||
|
||||
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
|
||||
}
|
||||
@ -54,7 +54,7 @@ public sealed record EmbeddingStateChunk(
|
||||
string ChunkText,
|
||||
DateTimeOffset EmbeddedAtUtc);
|
||||
|
||||
public sealed record EmbeddingStateSearchResult(
|
||||
public sealed record IndexStoreSearchResult(
|
||||
string ChunkId,
|
||||
string ParentFileId,
|
||||
string DataSourceId,
|
||||
@ -4,11 +4,11 @@ using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState;
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbContext> options) : DbContext(options)
|
||||
internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext> options) : DbContext(options)
|
||||
{
|
||||
public static DbContextOptions<EmbeddingStateDbContext> CreateOptions(string databasePath) => new DbContextOptionsBuilder<EmbeddingStateDbContext>()
|
||||
public static DbContextOptions<IndexStoreDbContext> CreateOptions(string databasePath) => new DbContextOptionsBuilder<IndexStoreDbContext>()
|
||||
.UseSqlite(BuildConnectionString(databasePath))
|
||||
.Options;
|
||||
|
||||
@ -18,11 +18,11 @@ internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbC
|
||||
|
||||
public DbSet<EmbeddingStateChunkEntity> EmbeddingChunks => this.Set<EmbeddingStateChunkEntity>();
|
||||
|
||||
public DbSet<EmbeddingStateSearchResultEntity> SearchResults => this.Set<EmbeddingStateSearchResultEntity>();
|
||||
public DbSet<IndexStoreSearchResultEntity> SearchResults => this.Set<IndexStoreSearchResultEntity>();
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
{
|
||||
var utcDateTimeOffsetConverter = new EmbeddingStateDateTimeOffsetConverter();
|
||||
var utcDateTimeOffsetConverter = new IndexStoreDateTimeOffsetConverter();
|
||||
|
||||
modelBuilder.Entity<EmbeddingStateDataSourceEntity>(entity =>
|
||||
{
|
||||
@ -97,7 +97,7 @@ internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbC
|
||||
entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index").IsUnique();
|
||||
});
|
||||
|
||||
modelBuilder.Entity<EmbeddingStateSearchResultEntity>(entity =>
|
||||
modelBuilder.Entity<IndexStoreSearchResultEntity>(entity =>
|
||||
{
|
||||
entity.HasNoKey();
|
||||
entity.ToView("embedding_chunk_search_results");
|
||||
@ -193,7 +193,7 @@ internal sealed class EmbeddingStateChunkEntity
|
||||
public EmbeddingStateFileEntity? File { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateSearchResultEntity
|
||||
internal sealed class IndexStoreSearchResultEntity
|
||||
{
|
||||
public string ChunkId { get; set; } = string.Empty;
|
||||
|
||||
@ -238,11 +238,11 @@ internal sealed class EmbeddingStateSearchResultEntity
|
||||
public int ConfidenceLevelRank { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateDateTimeOffsetConverter() : ValueConverter<DateTimeOffset, string>(
|
||||
value => EmbeddingStateDateTimeOffset.ToUtcText(value),
|
||||
value => EmbeddingStateDateTimeOffset.ParseUtc(value));
|
||||
internal sealed class IndexStoreDateTimeOffsetConverter() : ValueConverter<DateTimeOffset, string>(
|
||||
value => IndexStoreDateTimeOffset.ToUtcText(value),
|
||||
value => IndexStoreDateTimeOffset.ParseUtc(value));
|
||||
|
||||
internal static class EmbeddingStateDateTimeOffset
|
||||
internal static class IndexStoreDateTimeOffset
|
||||
{
|
||||
public static string ToUtcText(DateTimeOffset dateTime)
|
||||
{
|
||||
@ -1,15 +1,15 @@
|
||||
using Microsoft.EntityFrameworkCore.Design;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState;
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal sealed class EmbeddingStateDesignTimeDbContextFactory : IDesignTimeDbContextFactory<EmbeddingStateDbContext>
|
||||
internal sealed class IndexStoreDesignTimeDbContextFactory : IDesignTimeDbContextFactory<IndexStoreDbContext>
|
||||
{
|
||||
public EmbeddingStateDbContext CreateDbContext(string[] args)
|
||||
public IndexStoreDbContext CreateDbContext(string[] args)
|
||||
{
|
||||
var databasePath = args.FirstOrDefault(argument => argument.EndsWith(".sqlite3", StringComparison.OrdinalIgnoreCase));
|
||||
if (string.IsNullOrWhiteSpace(databasePath))
|
||||
databasePath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-rag-index-design.sqlite3");
|
||||
|
||||
return new EmbeddingStateDbContext(EmbeddingStateDbContext.CreateOptions(databasePath));
|
||||
return new IndexStoreDbContext(IndexStoreDbContext.CreateOptions(databasePath));
|
||||
}
|
||||
}
|
||||
@ -2,12 +2,12 @@ using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState;
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal static class EmbeddingStateSchemaMigrator
|
||||
internal static class IndexStoreSchemaMigrator
|
||||
{
|
||||
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
|
||||
public static async Task MigrateAsync(EmbeddingStateDbContext context, CancellationToken token)
|
||||
public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
|
||||
{
|
||||
await context.Database.MigrateAsync(token);
|
||||
await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL;", token);
|
||||
@ -3,9 +3,9 @@
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState.Migrations;
|
||||
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
|
||||
|
||||
[DbContext(typeof(EmbeddingStateDbContext))]
|
||||
[DbContext(typeof(IndexStoreDbContext))]
|
||||
[Migration("20260804000000_InitialRagIndex")]
|
||||
public partial class InitialRagIndex : Migration
|
||||
{
|
||||
@ -3,18 +3,18 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState.Migrations;
|
||||
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
|
||||
|
||||
[DbContext(typeof(EmbeddingStateDbContext))]
|
||||
partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
||||
[DbContext(typeof(IndexStoreDbContext))]
|
||||
partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
|
||||
{
|
||||
protected override void BuildModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder.HasAnnotation("ProductVersion", "9.0.18");
|
||||
var utcDateTimeOffsetConverter = new EmbeddingStateDateTimeOffsetConverter();
|
||||
var utcDateTimeOffsetConverter = new IndexStoreDateTimeOffsetConverter();
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity =>
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity =>
|
||||
{
|
||||
entity.Property<string>("DataSourceId")
|
||||
.HasColumnType("TEXT")
|
||||
@ -61,7 +61,7 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
||||
entity.ToTable("data_sources");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity =>
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
|
||||
{
|
||||
entity.Property<string>("ParentFileId")
|
||||
.HasColumnType("TEXT")
|
||||
@ -151,7 +151,7 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
||||
entity.ToTable("embedded_files");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateChunkEntity", entity =>
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateChunkEntity", entity =>
|
||||
{
|
||||
entity.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
@ -206,7 +206,7 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
||||
entity.ToTable("embedding_chunks");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateSearchResultEntity", entity =>
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.IndexStoreSearchResultEntity", entity =>
|
||||
{
|
||||
entity.Property<string>("AbsolutePath")
|
||||
.IsRequired()
|
||||
@ -291,9 +291,9 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
||||
entity.ToView("embedding_chunk_search_results");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity =>
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
|
||||
{
|
||||
entity.HasOne("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", "DataSource")
|
||||
entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", "DataSource")
|
||||
.WithMany("Files")
|
||||
.HasForeignKey("DataSourceId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
@ -302,9 +302,9 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
||||
entity.Navigation("DataSource");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateChunkEntity", entity =>
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateChunkEntity", entity =>
|
||||
{
|
||||
entity.HasOne("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", "File")
|
||||
entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", "File")
|
||||
.WithMany("Chunks")
|
||||
.HasForeignKey("ParentFileId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
@ -313,12 +313,12 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
||||
entity.Navigation("File");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity =>
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity =>
|
||||
{
|
||||
entity.Navigation("Files");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity =>
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
|
||||
{
|
||||
entity.Navigation("Chunks");
|
||||
});
|
||||
@ -1,11 +1,11 @@
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState;
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
public sealed class NoEmbeddingStateClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : EmbeddingStateClient(name, string.Empty)
|
||||
public sealed class NoIndexStoreClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : IndexStoreClient(name, string.Empty)
|
||||
{
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoEmbeddingStateClient).Namespace, nameof(NoEmbeddingStateClient));
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoIndexStoreClient).Namespace, nameof(NoIndexStoreClient));
|
||||
|
||||
public override DatabaseClientStatus Status => status;
|
||||
|
||||
@ -46,8 +46,8 @@ public sealed class NoEmbeddingStateClient(string name, string? unavailableReaso
|
||||
|
||||
public override Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task<IReadOnlyList<EmbeddingStateSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) =>
|
||||
Task.FromResult<IReadOnlyList<EmbeddingStateSearchResult>>([]);
|
||||
public override Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) =>
|
||||
Task.FromResult<IReadOnlyList<IndexStoreSearchResult>>([]);
|
||||
|
||||
public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
@ -7,13 +7,13 @@ using AIStudio.Tools.Services;
|
||||
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState;
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
public sealed class SqliteIndexStoreClientImplementation(
|
||||
string name,
|
||||
string databasePath,
|
||||
string basePath,
|
||||
string version) : EmbeddingStateClient(name, basePath)
|
||||
string version) : IndexStoreClient(name, basePath)
|
||||
{
|
||||
private const string DATABASE_NAME = "Local RAG Index";
|
||||
private const string DATABASE_FILENAME = "rag-index.sqlite3";
|
||||
@ -23,9 +23,9 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
private static readonly Regex FTS_TOKEN_REGEX = new(@"[\p{L}\p{Nd}_]+", RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
|
||||
private readonly string databasePath = databasePath;
|
||||
private readonly DbContextOptions<EmbeddingStateDbContext> dbContextOptions = EmbeddingStateDbContext.CreateOptions(databasePath);
|
||||
private readonly DbContextOptions<IndexStoreDbContext> dbContextOptions = IndexStoreDbContext.CreateOptions(databasePath);
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SqliteEmbeddingStateClientImplementation).Namespace, nameof(SqliteEmbeddingStateClientImplementation));
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SqliteIndexStoreClientImplementation).Namespace, nameof(SqliteIndexStoreClientImplementation));
|
||||
|
||||
public override string CacheKey => $"{this.Name}:{this.databasePath}:{version}";
|
||||
|
||||
@ -35,7 +35,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(SettingsManager.DataDirectory))
|
||||
return CreateNoEmbeddingStateClient(DATABASE_NAME, "The application data directory is not available yet.", DatabaseClientStatus.STARTING, databaseClientLogger);
|
||||
return CreateNoIndexStoreClient(DATABASE_NAME, "The application data directory is not available yet.", DatabaseClientStatus.STARTING, databaseClientLogger);
|
||||
|
||||
try
|
||||
{
|
||||
@ -45,18 +45,18 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
Directory.CreateDirectory(basePath);
|
||||
|
||||
var databasePath = Path.Combine(basePath, DATABASE_FILENAME);
|
||||
var client = new SqliteEmbeddingStateClientImplementation(DATABASE_NAME, databasePath, basePath, string.Empty);
|
||||
var client = new SqliteIndexStoreClientImplementation(DATABASE_NAME, databasePath, basePath, string.Empty);
|
||||
await client.InitializeAsync(cancellationToken);
|
||||
var version = await client.GetSqliteVersionAsync(cancellationToken);
|
||||
|
||||
client = new SqliteEmbeddingStateClientImplementation(DATABASE_NAME, databasePath, basePath, version);
|
||||
client = new SqliteIndexStoreClientImplementation(DATABASE_NAME, databasePath, basePath, version);
|
||||
client.SetLogger(databaseClientLogger);
|
||||
return client;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "{DatabaseName} is not available. Indexed file fingerprints and search chunks are disabled.", DATABASE_NAME);
|
||||
return CreateNoEmbeddingStateClient(DATABASE_NAME, exception.Message, DatabaseClientStatus.UNAVAILABLE, databaseClientLogger);
|
||||
return CreateNoIndexStoreClient(DATABASE_NAME, exception.Message, DatabaseClientStatus.UNAVAILABLE, databaseClientLogger);
|
||||
}
|
||||
}
|
||||
|
||||
@ -236,7 +236,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
await transaction.CommitAsync(token);
|
||||
}
|
||||
|
||||
public override async Task<IReadOnlyList<EmbeddingStateSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token)
|
||||
public override async Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token)
|
||||
{
|
||||
if (maxMatches <= 0)
|
||||
return [];
|
||||
@ -314,7 +314,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
private async Task InitializeAsync(CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
await EmbeddingStateSchemaMigrator.MigrateAsync(context, token);
|
||||
await IndexStoreSchemaMigrator.MigrateAsync(context, token);
|
||||
}
|
||||
|
||||
private async Task<string> GetSqliteVersionAsync(CancellationToken token)
|
||||
@ -326,7 +326,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
return versions.FirstOrDefault() ?? string.Empty;
|
||||
}
|
||||
|
||||
private EmbeddingStateDbContext CreateContext() => new(this.dbContextOptions);
|
||||
private IndexStoreDbContext CreateContext() => new(this.dbContextOptions);
|
||||
|
||||
private static void ApplyDataSource(
|
||||
EmbeddingStateDataSourceEntity dataSource,
|
||||
@ -373,7 +373,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
chunkEntity.EmbeddedAtUtc = chunk.EmbeddedAtUtc;
|
||||
}
|
||||
|
||||
private static EmbeddingStateSearchResult ToSearchResult(EmbeddingStateSearchResultEntity result) => new(
|
||||
private static IndexStoreSearchResult ToSearchResult(IndexStoreSearchResultEntity result) => new(
|
||||
result.ChunkId,
|
||||
result.ParentFileId,
|
||||
result.DataSourceId,
|
||||
@ -410,9 +410,9 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
return terms.Count == 0 ? string.Empty : string.Join(" OR ", terms);
|
||||
}
|
||||
|
||||
private static NoEmbeddingStateClient CreateNoEmbeddingStateClient(string name, string? unavailableReason, DatabaseClientStatus status, ILogger<DatabaseClient> databaseClientLogger)
|
||||
private static NoIndexStoreClient CreateNoIndexStoreClient(string name, string? unavailableReason, DatabaseClientStatus status, ILogger<DatabaseClient> databaseClientLogger)
|
||||
{
|
||||
var client = new NoEmbeddingStateClient(name, unavailableReason, status);
|
||||
var client = new NoIndexStoreClient(name, unavailableReason, status);
|
||||
client.SetLogger(databaseClientLogger);
|
||||
return client;
|
||||
}
|
||||
@ -5,7 +5,7 @@ using System.Text.RegularExpressions;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.Databases.EmbeddingState;
|
||||
using AIStudio.Tools.Databases.IndexStore;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.Rust;
|
||||
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
using AIStudio.Tools.Databases.EmbeddingState;
|
||||
using AIStudio.Tools.Databases.IndexStore;
|
||||
using AIStudio.Tools.Databases.VectorStore;
|
||||
|
||||
namespace AIStudio.Tools.Services;
|
||||
@ -8,19 +8,19 @@ public sealed partial class DataSourceEmbeddingService
|
||||
private async Task ResetPersistedStateAsync(
|
||||
string dataSourceId,
|
||||
VectorStoreClient? vectorStore,
|
||||
EmbeddingStateClient? embeddingState,
|
||||
IndexStoreClient? indexStore,
|
||||
CancellationToken token)
|
||||
{
|
||||
await this.DeleteCollectionAsync(DataSourceEmbeddingNames.GetCollectionName(dataSourceId), vectorStore, token);
|
||||
|
||||
embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token);
|
||||
if (!embeddingState.IsAvailable)
|
||||
indexStore ??= await databaseClientProvider.GetIndexStoreAsync(token);
|
||||
if (!indexStore.IsAvailable)
|
||||
{
|
||||
logger.LogWarning("Could not delete local RAG index state for data source '{DataSourceId}' because the database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
|
||||
logger.LogWarning("Could not delete local RAG embedding state for data source '{DataSourceId}' because the database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
|
||||
return;
|
||||
}
|
||||
|
||||
await embeddingState.DeleteDataSourceAsync(dataSourceId, token);
|
||||
logger.LogInformation("Reset persisted local RAG index state for data source '{DataSourceId}'.", dataSourceId);
|
||||
await indexStore.DeleteDataSourceAsync(dataSourceId, token);
|
||||
logger.LogInformation("Reset persisted local RAG embedding state for data source '{DataSourceId}'.", dataSourceId);
|
||||
}
|
||||
}
|
||||
|
||||
@ -7,7 +7,7 @@ using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.Databases;
|
||||
using AIStudio.Tools.Databases.EmbeddingState;
|
||||
using AIStudio.Tools.Databases.IndexStore;
|
||||
using AIStudio.Tools.Databases.VectorStore;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
@ -188,14 +188,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
|
||||
public async Task<bool> ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default)
|
||||
{
|
||||
var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token);
|
||||
if (!embeddingState.IsAvailable)
|
||||
var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
|
||||
if (!indexStore.IsAvailable)
|
||||
{
|
||||
logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
|
||||
logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
|
||||
return true;
|
||||
}
|
||||
|
||||
var manifest = await embeddingState.GetManifestAsync(dataSourceId, token);
|
||||
var manifest = await indexStore.GetManifestAsync(dataSourceId, token);
|
||||
return !string.IsNullOrWhiteSpace(manifest.EmbeddingProviderId)
|
||||
|| !string.IsNullOrWhiteSpace(manifest.EmbeddingSignature)
|
||||
|| !string.IsNullOrWhiteSpace(manifest.SourceHash)
|
||||
@ -410,7 +410,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token);
|
||||
var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token);
|
||||
var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
if (!vectorStore.IsAvailable)
|
||||
@ -425,20 +425,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
return;
|
||||
}
|
||||
|
||||
if (!embeddingState.IsAvailable)
|
||||
if (!indexStore.IsAvailable)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Skipping background embeddings for data source '{DataSourceName}' ({DataSourceId}) because the database client '{DatabaseName}' is unavailable.",
|
||||
dataSource.Name,
|
||||
dataSource.Id,
|
||||
embeddingState.Name);
|
||||
indexStore.Name);
|
||||
token.ThrowIfCancellationRequested();
|
||||
this.UpsertStatus(this.GetFallbackStatus(dataSource, "The local RAG index database is not available."));
|
||||
return;
|
||||
}
|
||||
|
||||
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
|
||||
var persistedManifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
|
||||
var persistedManifest = await indexStore.GetManifestAsync(dataSource.Id, token);
|
||||
if (persistedManifest.VectorSize > 0)
|
||||
{
|
||||
var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedManifest.VectorSize, token);
|
||||
@ -449,7 +449,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
collectionName,
|
||||
dataSource.Name,
|
||||
dataSource.Id);
|
||||
await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, embeddingState, token);
|
||||
await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, indexStore, token);
|
||||
}
|
||||
}
|
||||
|
||||
@ -484,7 +484,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
dataSource.Name,
|
||||
dataSource.Id);
|
||||
|
||||
var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, embeddingState, token);
|
||||
var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, indexStore, token);
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var inputFiles = this.GetInputFiles(dataSource);
|
||||
@ -510,7 +510,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
collectionName);
|
||||
|
||||
var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles);
|
||||
var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(vectorStore, embeddingState, dataSource, collectionName, manifest, indexedFiles, token);
|
||||
var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(vectorStore, indexStore, dataSource, collectionName, manifest, indexedFiles, token);
|
||||
var optimizationTracker = new VectorStoreOptimizationTracker();
|
||||
if (removedMissingFiles > 0)
|
||||
optimizationTracker.MarkChanged();
|
||||
@ -543,7 +543,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
token);
|
||||
|
||||
token.ThrowIfCancellationRequested();
|
||||
await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token);
|
||||
await indexStore.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token);
|
||||
this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, indexedFiles.Count, inputFiles.FailedFiles, inputFiles.LastError, inputFiles.Failures));
|
||||
return;
|
||||
}
|
||||
@ -602,7 +602,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
skippedFiles + completedFiles + 1,
|
||||
totalFiles);
|
||||
var startedAtUtc = DateTimeOffset.UtcNow;
|
||||
var chunkCount = await this.IndexOneFileAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token);
|
||||
var chunkCount = await this.IndexOneFileAsync(indexStore, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token);
|
||||
token.ThrowIfCancellationRequested();
|
||||
var fingerprintAfterEmbedding = BuildFileMetadataHash(file);
|
||||
if (!string.Equals(fingerprint, fingerprintAfterEmbedding, StringComparison.Ordinal))
|
||||
@ -615,7 +615,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
new DateTimeOffset(file.LastWriteTimeUtc),
|
||||
embeddedAtUtc,
|
||||
chunkCount);
|
||||
await embeddingState.UpsertFileAsync(
|
||||
await indexStore.UpsertFileAsync(
|
||||
dataSource.Id,
|
||||
this.CreateEmbeddingStateFile(dataSource, file, fingerprint, chunkCount, embeddedAtUtc),
|
||||
token);
|
||||
@ -644,7 +644,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
lastError = exception.Message;
|
||||
failureDetails.Add(new DataSourceEmbeddingFailure(file.FullName, exception.Message));
|
||||
manifest.Files.Remove(file.FullName);
|
||||
await this.CleanupFailedFileAsync(embeddingState, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token);
|
||||
await this.CleanupFailedFileAsync(indexStore, vectorStore, dataSource, collectionName, file.FullName, optimizationTracker, token);
|
||||
|
||||
logger.LogWarning(exception, "Failed to embed file '{FilePath}' for data source '{DataSourceName}'.", file.FullName, dataSource.Name);
|
||||
this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, exception.Message, failureDetails));
|
||||
@ -662,7 +662,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
token);
|
||||
|
||||
token.ThrowIfCancellationRequested();
|
||||
await embeddingState.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token);
|
||||
await indexStore.UpdateDataSourceHashAsync(dataSource.Id, metadataSnapshot.SourceHash, token);
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
this.UpsertStatus(this.CreateCompletedStatus(dataSource, totalFiles, skippedFiles + completedFiles, failedFiles, lastError, failureDetails));
|
||||
@ -682,7 +682,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
}
|
||||
|
||||
private async Task<int> IndexOneFileAsync(
|
||||
EmbeddingStateClient embeddingState,
|
||||
IndexStoreClient indexStore,
|
||||
VectorStoreClient vectorStore,
|
||||
IDataSource dataSource,
|
||||
FileInfo file,
|
||||
@ -700,10 +700,10 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
collectionName);
|
||||
await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token);
|
||||
optimizationTracker.MarkChanged();
|
||||
await embeddingState.DeleteFileAsync(dataSource.Id, file.FullName, token);
|
||||
await indexStore.DeleteFileAsync(dataSource.Id, file.FullName, token);
|
||||
|
||||
var parentFile = this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow);
|
||||
await embeddingState.UpsertFileAsync(dataSource.Id, parentFile, token);
|
||||
await indexStore.UpsertFileAsync(dataSource.Id, parentFile, token);
|
||||
|
||||
var embeddingBatchSize = Math.Max(1, embeddingProvider.EffectiveEmbeddingBatchSize);
|
||||
var batch = new List<EmbeddingChunkDraft>(embeddingBatchSize);
|
||||
@ -715,11 +715,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
totalChunkCount++;
|
||||
|
||||
if (batch.Count >= embeddingBatchSize)
|
||||
await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token);
|
||||
await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token);
|
||||
}
|
||||
|
||||
if (batch.Count > 0)
|
||||
await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token);
|
||||
await this.FlushBatchAsync(indexStore, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, optimizationTracker, collectionName, batch, token);
|
||||
|
||||
if (totalChunkCount == 0)
|
||||
throw new InvalidOperationException($"The file '{file.Name}' did not yield any text chunks.");
|
||||
@ -735,7 +735,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
}
|
||||
|
||||
private async Task FlushBatchAsync(
|
||||
EmbeddingStateClient embeddingState,
|
||||
IndexStoreClient indexStore,
|
||||
VectorStoreClient vectorStore,
|
||||
IDataSource dataSource,
|
||||
FileInfo file,
|
||||
@ -805,7 +805,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly.");
|
||||
}
|
||||
|
||||
await embeddingState.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token);
|
||||
await indexStore.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token);
|
||||
manifest.VectorSize = vectorSize;
|
||||
logger.LogInformation(
|
||||
"Created embedding collection '{CollectionName}' with vector size {VectorSize} for data source '{DataSourceName}' ({DataSourceId}).",
|
||||
@ -829,7 +829,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
embeddedAtUtc,
|
||||
token);
|
||||
token.ThrowIfCancellationRequested();
|
||||
await embeddingState.UpsertChunksAsync(
|
||||
await indexStore.UpsertChunksAsync(
|
||||
dataSource.Id,
|
||||
this.CreateEmbeddingStateChunks(parentFile, batch, embeddedAtUtc),
|
||||
token);
|
||||
@ -897,7 +897,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
}
|
||||
|
||||
private async Task CleanupFailedFileAsync(
|
||||
EmbeddingStateClient embeddingState,
|
||||
IndexStoreClient indexStore,
|
||||
VectorStoreClient vectorStore,
|
||||
IDataSource dataSource,
|
||||
string collectionName,
|
||||
@ -926,7 +926,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
|
||||
try
|
||||
{
|
||||
await embeddingState.DeleteFileAsync(dataSource.Id, filePath, token);
|
||||
await indexStore.DeleteFileAsync(dataSource.Id, filePath, token);
|
||||
}
|
||||
catch (OperationCanceledException) when (token.IsCancellationRequested)
|
||||
{
|
||||
@ -1018,7 +1018,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
.ToList();
|
||||
|
||||
logger.LogInformation(
|
||||
"Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed local RAG index state will be retried during this pass. File watchers will be activated after this check completes.",
|
||||
"Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed local RAG embedding state will be retried during this pass. File watchers will be activated after this check completes.",
|
||||
supportedDataSources.Count);
|
||||
|
||||
foreach (var dataSource in supportedDataSources)
|
||||
@ -1074,12 +1074,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
EmbeddingProvider embeddingProvider,
|
||||
string collectionName,
|
||||
VectorStoreClient vectorStore,
|
||||
EmbeddingStateClient embeddingState,
|
||||
IndexStoreClient indexStore,
|
||||
CancellationToken token)
|
||||
{
|
||||
var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
|
||||
var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
|
||||
var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
|
||||
var manifest = await indexStore.GetManifestAsync(dataSource.Id, token);
|
||||
|
||||
logger.LogInformation(
|
||||
"Loaded persisted local RAG index manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.",
|
||||
@ -1093,7 +1093,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
if (!string.Equals(manifest.EmbeddingSignature, embeddingSignature, StringComparison.Ordinal))
|
||||
{
|
||||
logger.LogInformation(
|
||||
"Embedding configuration changed for data source '{DataSourceName}' ({DataSourceId}). Resetting persisted state and collection '{CollectionName}'.",
|
||||
"Embedding configuration changed for data source '{DataSourceName}' ({DataSourceId}). Resetting persisted embedding state and collection '{CollectionName}'.",
|
||||
dataSource.Name,
|
||||
dataSource.Id,
|
||||
collectionName);
|
||||
@ -1103,8 +1103,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
dataSource.Id,
|
||||
manifest.EmbeddingSignature,
|
||||
embeddingSignature);
|
||||
await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, embeddingState, token);
|
||||
manifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
|
||||
await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, indexStore, token);
|
||||
manifest = await indexStore.GetManifestAsync(dataSource.Id, token);
|
||||
}
|
||||
|
||||
if (!string.Equals(manifest.EmbeddingProviderId, embeddingProvider.Id, StringComparison.OrdinalIgnoreCase) ||
|
||||
@ -1114,7 +1114,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
manifest.EmbeddingSignature = embeddingSignature;
|
||||
}
|
||||
|
||||
await embeddingState.UpsertDataSourceAsync(
|
||||
await indexStore.UpsertDataSourceAsync(
|
||||
dataSource.Id,
|
||||
dataSource.Name,
|
||||
dataSource.Type.ToString(),
|
||||
@ -1129,7 +1129,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
|
||||
private async Task<int> RemoveMissingFileEmbeddingsAsync(
|
||||
VectorStoreClient vectorStore,
|
||||
EmbeddingStateClient embeddingState,
|
||||
IndexStoreClient indexStore,
|
||||
IDataSource dataSource,
|
||||
string collectionName,
|
||||
DataSourceEmbeddingManifest manifest,
|
||||
@ -1144,7 +1144,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
foreach (var removedFilePath in manifest.Files.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList())
|
||||
{
|
||||
await this.DeleteFilePointsAsync(vectorStore, collectionName, removedFilePath, token);
|
||||
await embeddingState.DeleteFileAsync(dataSource.Id, removedFilePath, token);
|
||||
await indexStore.DeleteFileAsync(dataSource.Id, removedFilePath, token);
|
||||
manifest.Files.Remove(removedFilePath);
|
||||
removedFiles++;
|
||||
logger.LogInformation(
|
||||
|
||||
@ -3,7 +3,7 @@ using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.Databases;
|
||||
using AIStudio.Tools.Databases.EmbeddingState;
|
||||
using AIStudio.Tools.Databases.IndexStore;
|
||||
using AIStudio.Tools.Databases.VectorStore;
|
||||
using AIStudio.Tools.RAG;
|
||||
using AIStudio.Tools.Rust;
|
||||
@ -188,25 +188,25 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task<IReadOnlyList<EmbeddingStateSearchResult>> SearchBm25Async(IInternalDataSource dataSource, string query, int maxMatches, CancellationToken token)
|
||||
private async Task<IReadOnlyList<IndexStoreSearchResult>> SearchBm25Async(IInternalDataSource dataSource, string query, int maxMatches, CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token);
|
||||
if (!embeddingState.IsAvailable)
|
||||
var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
|
||||
if (!indexStore.IsAvailable)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Skipping BM25 retrieval for data source '{DataSourceName}' ({DataSourceId}) because local RAG index '{DatabaseName}' is unavailable.",
|
||||
dataSource.Name,
|
||||
dataSource.Id,
|
||||
embeddingState.Name);
|
||||
indexStore.Name);
|
||||
return [];
|
||||
}
|
||||
|
||||
var results = this.LimitSearchResults(
|
||||
dataSource,
|
||||
"BM25",
|
||||
await embeddingState.SearchChunksAsync(dataSource.Id, query, maxMatches, token),
|
||||
await indexStore.SearchChunksAsync(dataSource.Id, query, maxMatches, token),
|
||||
maxMatches);
|
||||
this.LogBm25Results(dataSource, results);
|
||||
return results;
|
||||
@ -240,7 +240,7 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
|
||||
private static IReadOnlyList<LocalRetrievalHit> MergeResults(
|
||||
IReadOnlyList<VectorSearchResult> vectorResults,
|
||||
IReadOnlyList<EmbeddingStateSearchResult> bm25Results,
|
||||
IReadOnlyList<IndexStoreSearchResult> bm25Results,
|
||||
int maxMatches)
|
||||
{
|
||||
// Future reranking should replace this deterministic channel merge.
|
||||
@ -299,7 +299,7 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
result.ConfidenceLevel,
|
||||
result.ConfidenceLevelRank);
|
||||
|
||||
private static LocalRetrievalHit FromBm25Result(EmbeddingStateSearchResult result, int rank) =>
|
||||
private static LocalRetrievalHit FromBm25Result(IndexStoreSearchResult result, int rank) =>
|
||||
new(
|
||||
RetrievalChannel.BM25,
|
||||
result.ChunkId,
|
||||
@ -418,7 +418,7 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
}
|
||||
}
|
||||
|
||||
private void LogBm25Results(IInternalDataSource dataSource, IReadOnlyList<EmbeddingStateSearchResult> results)
|
||||
private void LogBm25Results(IInternalDataSource dataSource, IReadOnlyList<IndexStoreSearchResult> results)
|
||||
{
|
||||
if (results.Count == 0)
|
||||
{
|
||||
|
||||
Loading…
Reference in New Issue
Block a user