mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-17 04:38:00 +00:00
replaced embedding_state with index_store as a sqlite name
This commit is contained in:
PaulKoudelka committed
1 parent
50aa5c485d
commit
1a263127f5
15 files changed
+129
-129
No files matched your search
@@ -0,0 +1,78 @@
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
public abstract class IndexStoreClient(string name, string path) : DatabaseClient(name, path)
|
||||
{
|
||||
public abstract Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token);
|
||||
|
||||
public abstract Task UpsertDataSourceAsync(
|
||||
string dataSourceId,
|
||||
string dataSourceName,
|
||||
string dataSourceType,
|
||||
string embeddingProviderId,
|
||||
string embeddingSignature,
|
||||
string sourceHash,
|
||||
int vectorSize,
|
||||
CancellationToken token);
|
||||
|
||||
public abstract Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token);
|
||||
|
||||
public abstract Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token);
|
||||
|
||||
public abstract Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token);
|
||||
|
||||
public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token);
|
||||
|
||||
public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token);
|
||||
|
||||
public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
|
||||
|
||||
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
|
||||
}
|
||||
|
||||
public sealed record EmbeddingStateFile(
|
||||
string ParentFileId,
|
||||
string AbsolutePath,
|
||||
string FileName,
|
||||
string RelativePath,
|
||||
string FileType,
|
||||
string Fingerprint,
|
||||
long FileSize,
|
||||
DateTimeOffset CreationUtc,
|
||||
DateTimeOffset LastWriteUtc,
|
||||
DateTimeOffset EmbeddedAtUtc,
|
||||
int ChunkCount,
|
||||
string ConfidenceLevel,
|
||||
int ConfidenceLevelRank);
|
||||
|
||||
public sealed record EmbeddingStateChunk(
|
||||
string ChunkId,
|
||||
string ParentFileId,
|
||||
int? PageNumber,
|
||||
int ChunkIndex,
|
||||
string ChunkText,
|
||||
DateTimeOffset EmbeddedAtUtc);
|
||||
|
||||
public sealed record IndexStoreSearchResult(
|
||||
string ChunkId,
|
||||
string ParentFileId,
|
||||
string DataSourceId,
|
||||
string DataSourceName,
|
||||
string DataSourceType,
|
||||
string AbsolutePath,
|
||||
string FileName,
|
||||
string RelativePath,
|
||||
string FileType,
|
||||
int? PageNumber,
|
||||
int ChunkIndex,
|
||||
string ChunkText,
|
||||
double Score,
|
||||
string Fingerprint,
|
||||
long FileSize,
|
||||
DateTimeOffset CreationUtc,
|
||||
DateTimeOffset LastWriteUtc,
|
||||
DateTimeOffset EmbeddedAtUtc,
|
||||
int ChunkCount,
|
||||
string ConfidenceLevel,
|
||||
int ConfidenceLevelRank);
|
||||
@@ -0,0 +1,258 @@
|
||||
using System.Globalization;
|
||||
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext> options) : DbContext(options)
|
||||
{
|
||||
public static DbContextOptions<IndexStoreDbContext> CreateOptions(string databasePath) => new DbContextOptionsBuilder<IndexStoreDbContext>()
|
||||
.UseSqlite(BuildConnectionString(databasePath))
|
||||
.Options;
|
||||
|
||||
public DbSet<EmbeddingStateDataSourceEntity> DataSources => this.Set<EmbeddingStateDataSourceEntity>();
|
||||
|
||||
public DbSet<EmbeddingStateFileEntity> EmbeddedFiles => this.Set<EmbeddingStateFileEntity>();
|
||||
|
||||
public DbSet<EmbeddingStateChunkEntity> EmbeddingChunks => this.Set<EmbeddingStateChunkEntity>();
|
||||
|
||||
public DbSet<IndexStoreSearchResultEntity> SearchResults => this.Set<IndexStoreSearchResultEntity>();
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
{
|
||||
var utcDateTimeOffsetConverter = new IndexStoreDateTimeOffsetConverter();
|
||||
|
||||
modelBuilder.Entity<EmbeddingStateDataSourceEntity>(entity =>
|
||||
{
|
||||
entity.ToTable("data_sources");
|
||||
entity.HasKey(dataSource => dataSource.DataSourceId);
|
||||
|
||||
entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id");
|
||||
entity.Property(dataSource => dataSource.DataSourceName).HasColumnName("data_source_name").IsRequired();
|
||||
entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired();
|
||||
entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired();
|
||||
entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired();
|
||||
entity.Property(dataSource => dataSource.SourceHash).HasColumnName("source_hash").IsRequired().HasDefaultValue(string.Empty);
|
||||
entity.Property(dataSource => dataSource.VectorSize).HasColumnName("vector_size").HasDefaultValue(0);
|
||||
entity.Property(dataSource => dataSource.UpdatedAtUtc).HasColumnName("updated_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
|
||||
|
||||
entity
|
||||
.HasMany(dataSource => dataSource.Files)
|
||||
.WithOne(file => file.DataSource)
|
||||
.HasForeignKey(file => file.DataSourceId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity<EmbeddingStateFileEntity>(entity =>
|
||||
{
|
||||
entity.ToTable("embedded_files");
|
||||
entity.HasKey(file => file.ParentFileId);
|
||||
|
||||
entity.Property(file => file.ParentFileId).HasColumnName("parent_file_id");
|
||||
entity.Property(file => file.DataSourceId).HasColumnName("data_source_id").IsRequired();
|
||||
entity.Property(file => file.AbsolutePath).HasColumnName("absolute_path").UseCollation("NOCASE").IsRequired();
|
||||
entity.Property(file => file.FileName).HasColumnName("file_name").IsRequired();
|
||||
entity.Property(file => file.RelativePath).HasColumnName("relative_path").IsRequired();
|
||||
entity.Property(file => file.FileType).HasColumnName("file_type").IsRequired();
|
||||
entity.Property(file => file.Fingerprint).HasColumnName("fingerprint").IsRequired();
|
||||
entity.Property(file => file.FileSize).HasColumnName("file_size");
|
||||
entity.Property(file => file.CreationUtc).HasColumnName("creation_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
|
||||
entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
|
||||
entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
|
||||
entity.Property(file => file.ChunkCount).HasColumnName("chunk_count");
|
||||
entity.Property(file => file.ConfidenceLevel).HasColumnName("confidence_level").IsRequired();
|
||||
entity.Property(file => file.ConfidenceLevelRank).HasColumnName("confidence_level_rank");
|
||||
|
||||
entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source");
|
||||
entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path");
|
||||
entity.HasIndex(file => file.FileType).HasDatabaseName("idx_embedded_files_file_type");
|
||||
entity.HasIndex(file => file.ConfidenceLevelRank).HasDatabaseName("idx_embedded_files_confidence");
|
||||
entity.HasIndex(file => new { file.DataSourceId, file.AbsolutePath }).HasDatabaseName("idx_embedded_files_data_source_absolute_path").IsUnique();
|
||||
|
||||
entity
|
||||
.HasMany(file => file.Chunks)
|
||||
.WithOne(chunk => chunk.File)
|
||||
.HasForeignKey(chunk => chunk.ParentFileId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity<EmbeddingStateChunkEntity>(entity =>
|
||||
{
|
||||
entity.ToTable("embedding_chunks");
|
||||
entity.HasKey(chunk => chunk.Id);
|
||||
|
||||
entity.Property(chunk => chunk.Id).HasColumnName("id").ValueGeneratedOnAdd();
|
||||
entity.Property(chunk => chunk.ChunkId).HasColumnName("chunk_id").IsRequired();
|
||||
entity.Property(chunk => chunk.ParentFileId).HasColumnName("parent_file_id").IsRequired();
|
||||
entity.Property(chunk => chunk.PageNumber).HasColumnName("page_number");
|
||||
entity.Property(chunk => chunk.ChunkIndex).HasColumnName("chunk_index");
|
||||
entity.Property(chunk => chunk.ChunkText).HasColumnName("chunk_text").IsRequired();
|
||||
entity.Property(chunk => chunk.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
|
||||
|
||||
entity.HasIndex(chunk => chunk.ChunkId).HasDatabaseName("idx_embedding_chunks_chunk_id").IsUnique();
|
||||
entity.HasIndex(chunk => chunk.ParentFileId).HasDatabaseName("idx_embedding_chunks_parent_file");
|
||||
entity.HasIndex(chunk => chunk.PageNumber).HasDatabaseName("idx_embedding_chunks_page");
|
||||
entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index").IsUnique();
|
||||
});
|
||||
|
||||
modelBuilder.Entity<IndexStoreSearchResultEntity>(entity =>
|
||||
{
|
||||
entity.HasNoKey();
|
||||
entity.ToView("embedding_chunk_search_results");
|
||||
|
||||
entity.Property(result => result.CreationUtc).HasConversion(utcDateTimeOffsetConverter);
|
||||
entity.Property(result => result.LastWriteUtc).HasConversion(utcDateTimeOffsetConverter);
|
||||
entity.Property(result => result.EmbeddedAtUtc).HasConversion(utcDateTimeOffsetConverter);
|
||||
});
|
||||
}
|
||||
|
||||
private static string BuildConnectionString(string databasePath) => new SqliteConnectionStringBuilder
|
||||
{
|
||||
DataSource = databasePath,
|
||||
Mode = SqliteOpenMode.ReadWriteCreate,
|
||||
Cache = SqliteCacheMode.Shared,
|
||||
ForeignKeys = true,
|
||||
DefaultTimeout = 30,
|
||||
}.ToString();
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateDataSourceEntity
|
||||
{
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceName { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceType { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingProviderId { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingSignature { get; set; } = string.Empty;
|
||||
|
||||
public string SourceHash { get; set; } = string.Empty;
|
||||
|
||||
public int VectorSize { get; set; }
|
||||
|
||||
public DateTimeOffset UpdatedAtUtc { get; set; }
|
||||
|
||||
public List<EmbeddingStateFileEntity> Files { get; set; } = [];
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateFileEntity
|
||||
{
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string AbsolutePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileName { get; set; } = string.Empty;
|
||||
|
||||
public string RelativePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileType { get; set; } = string.Empty;
|
||||
|
||||
public string Fingerprint { get; set; } = string.Empty;
|
||||
|
||||
public long FileSize { get; set; }
|
||||
|
||||
public DateTimeOffset CreationUtc { get; set; }
|
||||
|
||||
public DateTimeOffset LastWriteUtc { get; set; }
|
||||
|
||||
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||
|
||||
public int ChunkCount { get; set; }
|
||||
|
||||
public string ConfidenceLevel { get; set; } = string.Empty;
|
||||
|
||||
public int ConfidenceLevelRank { get; set; }
|
||||
|
||||
public EmbeddingStateDataSourceEntity? DataSource { get; set; }
|
||||
|
||||
public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateChunkEntity
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public string ChunkId { get; set; } = string.Empty;
|
||||
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public int? PageNumber { get; set; }
|
||||
|
||||
public int ChunkIndex { get; set; }
|
||||
|
||||
public string ChunkText { get; set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||
|
||||
public EmbeddingStateFileEntity? File { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class IndexStoreSearchResultEntity
|
||||
{
|
||||
public string ChunkId { get; set; } = string.Empty;
|
||||
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceName { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceType { get; set; } = string.Empty;
|
||||
|
||||
public string AbsolutePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileName { get; set; } = string.Empty;
|
||||
|
||||
public string RelativePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileType { get; set; } = string.Empty;
|
||||
|
||||
public int? PageNumber { get; set; }
|
||||
|
||||
public int ChunkIndex { get; set; }
|
||||
|
||||
public string ChunkText { get; set; } = string.Empty;
|
||||
|
||||
public double Score { get; set; }
|
||||
|
||||
public string Fingerprint { get; set; } = string.Empty;
|
||||
|
||||
public long FileSize { get; set; }
|
||||
|
||||
public DateTimeOffset CreationUtc { get; set; }
|
||||
|
||||
public DateTimeOffset LastWriteUtc { get; set; }
|
||||
|
||||
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||
|
||||
public int ChunkCount { get; set; }
|
||||
|
||||
public string ConfidenceLevel { get; set; } = string.Empty;
|
||||
|
||||
public int ConfidenceLevelRank { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class IndexStoreDateTimeOffsetConverter() : ValueConverter<DateTimeOffset, string>(
|
||||
value => IndexStoreDateTimeOffset.ToUtcText(value),
|
||||
value => IndexStoreDateTimeOffset.ParseUtc(value));
|
||||
|
||||
internal static class IndexStoreDateTimeOffset
|
||||
{
|
||||
public static string ToUtcText(DateTimeOffset dateTime)
|
||||
{
|
||||
return dateTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
public static DateTimeOffset ParseUtc(string value)
|
||||
{
|
||||
return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateTime)
|
||||
? dateTime.ToUniversalTime()
|
||||
: DateTimeOffset.UnixEpoch;
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
using Microsoft.EntityFrameworkCore.Design;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal sealed class IndexStoreDesignTimeDbContextFactory : IDesignTimeDbContextFactory<IndexStoreDbContext>
|
||||
{
|
||||
public IndexStoreDbContext CreateDbContext(string[] args)
|
||||
{
|
||||
var databasePath = args.FirstOrDefault(argument => argument.EndsWith(".sqlite3", StringComparison.OrdinalIgnoreCase));
|
||||
if (string.IsNullOrWhiteSpace(databasePath))
|
||||
databasePath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-rag-index-design.sqlite3");
|
||||
|
||||
return new IndexStoreDbContext(IndexStoreDbContext.CreateOptions(databasePath));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal static class IndexStoreSchemaMigrator
|
||||
{
|
||||
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
|
||||
public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
|
||||
{
|
||||
await context.Database.MigrateAsync(token);
|
||||
await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL;", token);
|
||||
}
|
||||
}
|
||||
+202
@@ -0,0 +1,202 @@
|
||||
#nullable disable
|
||||
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
|
||||
|
||||
[DbContext(typeof(IndexStoreDbContext))]
|
||||
[Migration("20260804000000_InitialRagIndex")]
|
||||
public partial class InitialRagIndex : Migration
|
||||
{
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.CreateTable(
|
||||
name: "data_sources",
|
||||
columns: table => new
|
||||
{
|
||||
data_source_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
data_source_name = table.Column<string>(type: "TEXT", nullable: false),
|
||||
data_source_type = table.Column<string>(type: "TEXT", nullable: false),
|
||||
embedding_provider_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
embedding_signature = table.Column<string>(type: "TEXT", nullable: false),
|
||||
source_hash = table.Column<string>(type: "TEXT", nullable: false, defaultValue: string.Empty),
|
||||
vector_size = table.Column<int>(type: "INTEGER", nullable: false, defaultValue: 0),
|
||||
updated_at_utc = table.Column<string>(type: "TEXT", nullable: false),
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_data_sources", source => source.data_source_id);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateTable(
|
||||
name: "embedded_files",
|
||||
columns: table => new
|
||||
{
|
||||
parent_file_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
data_source_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
absolute_path = table.Column<string>(type: "TEXT", nullable: false, collation: "NOCASE"),
|
||||
file_name = table.Column<string>(type: "TEXT", nullable: false),
|
||||
relative_path = table.Column<string>(type: "TEXT", nullable: false),
|
||||
file_type = table.Column<string>(type: "TEXT", nullable: false),
|
||||
fingerprint = table.Column<string>(type: "TEXT", nullable: false),
|
||||
file_size = table.Column<long>(type: "INTEGER", nullable: false),
|
||||
creation_utc = table.Column<string>(type: "TEXT", nullable: false),
|
||||
last_write_utc = table.Column<string>(type: "TEXT", nullable: false),
|
||||
embedded_at_utc = table.Column<string>(type: "TEXT", nullable: false),
|
||||
chunk_count = table.Column<int>(type: "INTEGER", nullable: false),
|
||||
confidence_level = table.Column<string>(type: "TEXT", nullable: false),
|
||||
confidence_level_rank = table.Column<int>(type: "INTEGER", nullable: false),
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_embedded_files", file => file.parent_file_id);
|
||||
table.ForeignKey(
|
||||
name: "FK_embedded_files_data_sources_data_source_id",
|
||||
column: file => file.data_source_id,
|
||||
principalTable: "data_sources",
|
||||
principalColumn: "data_source_id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateTable(
|
||||
name: "embedding_chunks",
|
||||
columns: table => new
|
||||
{
|
||||
id = table.Column<int>(type: "INTEGER", nullable: false)
|
||||
.Annotation("Sqlite:Autoincrement", true),
|
||||
chunk_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
parent_file_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
page_number = table.Column<int>(type: "INTEGER", nullable: true),
|
||||
chunk_index = table.Column<int>(type: "INTEGER", nullable: false),
|
||||
chunk_text = table.Column<string>(type: "TEXT", nullable: false),
|
||||
embedded_at_utc = table.Column<string>(type: "TEXT", nullable: false),
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_embedding_chunks", chunk => chunk.id);
|
||||
table.ForeignKey(
|
||||
name: "FK_embedding_chunks_embedded_files_parent_file_id",
|
||||
column: chunk => chunk.parent_file_id,
|
||||
principalTable: "embedded_files",
|
||||
principalColumn: "parent_file_id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedded_files_absolute_path",
|
||||
table: "embedded_files",
|
||||
column: "absolute_path");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedded_files_confidence",
|
||||
table: "embedded_files",
|
||||
column: "confidence_level_rank");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedded_files_data_source",
|
||||
table: "embedded_files",
|
||||
column: "data_source_id");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedded_files_data_source_absolute_path",
|
||||
table: "embedded_files",
|
||||
columns: ["data_source_id", "absolute_path"],
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedded_files_file_type",
|
||||
table: "embedded_files",
|
||||
column: "file_type");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedding_chunks_chunk_id",
|
||||
table: "embedding_chunks",
|
||||
column: "chunk_id",
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedding_chunks_page",
|
||||
table: "embedding_chunks",
|
||||
column: "page_number");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedding_chunks_parent_file",
|
||||
table: "embedding_chunks",
|
||||
column: "parent_file_id");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedding_chunks_parent_file_chunk_index",
|
||||
table: "embedding_chunks",
|
||||
columns: ["parent_file_id", "chunk_index"],
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.Sql("""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS embedding_chunks_fts
|
||||
USING fts5(chunk_id UNINDEXED, file_name, chunk_text);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedding_chunks_ai
|
||||
AFTER INSERT ON embedding_chunks
|
||||
BEGIN
|
||||
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
|
||||
VALUES (
|
||||
new.id,
|
||||
new.chunk_id,
|
||||
(SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id),
|
||||
new.chunk_text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedding_chunks_ad
|
||||
AFTER DELETE ON embedding_chunks
|
||||
BEGIN
|
||||
DELETE FROM embedding_chunks_fts
|
||||
WHERE rowid = old.id;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedding_chunks_au
|
||||
AFTER UPDATE ON embedding_chunks
|
||||
BEGIN
|
||||
DELETE FROM embedding_chunks_fts
|
||||
WHERE rowid = old.id;
|
||||
|
||||
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
|
||||
VALUES (
|
||||
new.id,
|
||||
new.chunk_id,
|
||||
(SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id),
|
||||
new.chunk_text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedded_files_file_name_au
|
||||
AFTER UPDATE OF file_name ON embedded_files
|
||||
BEGIN
|
||||
DELETE FROM embedding_chunks_fts
|
||||
WHERE rowid IN (
|
||||
SELECT id
|
||||
FROM embedding_chunks
|
||||
WHERE parent_file_id = new.parent_file_id
|
||||
);
|
||||
|
||||
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
|
||||
SELECT id, chunk_id, new.file_name, chunk_text
|
||||
FROM embedding_chunks
|
||||
WHERE parent_file_id = new.parent_file_id;
|
||||
END;
|
||||
""");
|
||||
}
|
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.Sql("""
|
||||
DROP TRIGGER IF EXISTS embedded_files_file_name_au;
|
||||
DROP TRIGGER IF EXISTS embedding_chunks_au;
|
||||
DROP TRIGGER IF EXISTS embedding_chunks_ad;
|
||||
DROP TRIGGER IF EXISTS embedding_chunks_ai;
|
||||
DROP TABLE IF EXISTS embedding_chunks_fts;
|
||||
""");
|
||||
|
||||
migrationBuilder.DropTable(name: "embedding_chunks");
|
||||
migrationBuilder.DropTable(name: "embedded_files");
|
||||
migrationBuilder.DropTable(name: "data_sources");
|
||||
}
|
||||
}
|
||||
+327
@@ -0,0 +1,327 @@
|
||||
#nullable disable
|
||||
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
|
||||
|
||||
[DbContext(typeof(IndexStoreDbContext))]
|
||||
partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
|
||||
{
|
||||
protected override void BuildModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder.HasAnnotation("ProductVersion", "9.0.18");
|
||||
var utcDateTimeOffsetConverter = new IndexStoreDateTimeOffsetConverter();
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity =>
|
||||
{
|
||||
entity.Property<string>("DataSourceId")
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("data_source_id");
|
||||
|
||||
entity.Property<string>("DataSourceName")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("data_source_name");
|
||||
|
||||
entity.Property<string>("DataSourceType")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("data_source_type");
|
||||
|
||||
entity.Property<string>("EmbeddingProviderId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("embedding_provider_id");
|
||||
|
||||
entity.Property<string>("EmbeddingSignature")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("embedding_signature");
|
||||
|
||||
entity.Property<string>("SourceHash")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("source_hash")
|
||||
.HasDefaultValue(string.Empty);
|
||||
|
||||
entity.Property<DateTimeOffset>("UpdatedAtUtc")
|
||||
.HasConversion(utcDateTimeOffsetConverter)
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("updated_at_utc");
|
||||
|
||||
entity.Property<int>("VectorSize")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("vector_size")
|
||||
.HasDefaultValue(0);
|
||||
|
||||
entity.HasKey("DataSourceId");
|
||||
|
||||
entity.ToTable("data_sources");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
|
||||
{
|
||||
entity.Property<string>("ParentFileId")
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("parent_file_id");
|
||||
|
||||
entity.Property<string>("AbsolutePath")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("absolute_path")
|
||||
.UseCollation("NOCASE");
|
||||
|
||||
entity.Property<int>("ChunkCount")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("chunk_count");
|
||||
|
||||
entity.Property<string>("ConfidenceLevel")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("confidence_level");
|
||||
|
||||
entity.Property<int>("ConfidenceLevelRank")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("confidence_level_rank");
|
||||
|
||||
entity.Property<DateTimeOffset>("CreationUtc")
|
||||
.HasConversion(utcDateTimeOffsetConverter)
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("creation_utc");
|
||||
|
||||
entity.Property<string>("DataSourceId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("data_source_id");
|
||||
|
||||
entity.Property<DateTimeOffset>("EmbeddedAtUtc")
|
||||
.HasConversion(utcDateTimeOffsetConverter)
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("embedded_at_utc");
|
||||
|
||||
entity.Property<string>("FileName")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("file_name");
|
||||
|
||||
entity.Property<long>("FileSize")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("file_size");
|
||||
|
||||
entity.Property<string>("FileType")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("file_type");
|
||||
|
||||
entity.Property<string>("Fingerprint")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("fingerprint");
|
||||
|
||||
entity.Property<DateTimeOffset>("LastWriteUtc")
|
||||
.HasConversion(utcDateTimeOffsetConverter)
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("last_write_utc");
|
||||
|
||||
entity.Property<string>("RelativePath")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("relative_path");
|
||||
|
||||
entity.HasKey("ParentFileId");
|
||||
|
||||
entity.HasIndex("AbsolutePath")
|
||||
.HasDatabaseName("idx_embedded_files_absolute_path");
|
||||
|
||||
entity.HasIndex("ConfidenceLevelRank")
|
||||
.HasDatabaseName("idx_embedded_files_confidence");
|
||||
|
||||
entity.HasIndex("DataSourceId")
|
||||
.HasDatabaseName("idx_embedded_files_data_source");
|
||||
|
||||
entity.HasIndex("DataSourceId", "AbsolutePath")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("idx_embedded_files_data_source_absolute_path");
|
||||
|
||||
entity.HasIndex("FileType")
|
||||
.HasDatabaseName("idx_embedded_files_file_type");
|
||||
|
||||
entity.ToTable("embedded_files");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateChunkEntity", entity =>
|
||||
{
|
||||
entity.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("id")
|
||||
.HasAnnotation("Sqlite:Autoincrement", true);
|
||||
|
||||
entity.Property<string>("ChunkId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("chunk_id");
|
||||
|
||||
entity.Property<int>("ChunkIndex")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("chunk_index");
|
||||
|
||||
entity.Property<string>("ChunkText")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("chunk_text");
|
||||
|
||||
entity.Property<DateTimeOffset>("EmbeddedAtUtc")
|
||||
.HasConversion(utcDateTimeOffsetConverter)
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("embedded_at_utc");
|
||||
|
||||
entity.Property<int?>("PageNumber")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("page_number");
|
||||
|
||||
entity.Property<string>("ParentFileId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("parent_file_id");
|
||||
|
||||
entity.HasKey("Id");
|
||||
|
||||
entity.HasIndex("ChunkId")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("idx_embedding_chunks_chunk_id");
|
||||
|
||||
entity.HasIndex("PageNumber")
|
||||
.HasDatabaseName("idx_embedding_chunks_page");
|
||||
|
||||
entity.HasIndex("ParentFileId")
|
||||
.HasDatabaseName("idx_embedding_chunks_parent_file");
|
||||
|
||||
entity.HasIndex("ParentFileId", "ChunkIndex")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index");
|
||||
|
||||
entity.ToTable("embedding_chunks");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.IndexStoreSearchResultEntity", entity =>
|
||||
{
|
||||
entity.Property<string>("AbsolutePath")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<int>("ChunkCount")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
entity.Property<string>("ChunkId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<int>("ChunkIndex")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
entity.Property<string>("ChunkText")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("ConfidenceLevel")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<int>("ConfidenceLevelRank")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
entity.Property<DateTimeOffset>("CreationUtc")
|
||||
.HasConversion(utcDateTimeOffsetConverter)
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("DataSourceId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("DataSourceName")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("DataSourceType")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<DateTimeOffset>("EmbeddedAtUtc")
|
||||
.HasConversion(utcDateTimeOffsetConverter)
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("FileName")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<long>("FileSize")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
entity.Property<string>("FileType")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("Fingerprint")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<DateTimeOffset>("LastWriteUtc")
|
||||
.HasConversion(utcDateTimeOffsetConverter)
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<int?>("PageNumber")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
entity.Property<string>("ParentFileId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("RelativePath")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<double>("Score")
|
||||
.HasColumnType("REAL");
|
||||
|
||||
entity.HasNoKey();
|
||||
|
||||
entity.ToView("embedding_chunk_search_results");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
|
||||
{
|
||||
entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", "DataSource")
|
||||
.WithMany("Files")
|
||||
.HasForeignKey("DataSourceId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
entity.Navigation("DataSource");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateChunkEntity", entity =>
|
||||
{
|
||||
entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", "File")
|
||||
.WithMany("Chunks")
|
||||
.HasForeignKey("ParentFileId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
entity.Navigation("File");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity =>
|
||||
{
|
||||
entity.Navigation("Files");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
|
||||
{
|
||||
entity.Navigation("Chunks");
|
||||
});
|
||||
#pragma warning restore 612, 618
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
public sealed class NoIndexStoreClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : IndexStoreClient(name, string.Empty)
|
||||
{
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoIndexStoreClient).Namespace, nameof(NoIndexStoreClient));
|
||||
|
||||
public override DatabaseClientStatus Status => status;
|
||||
|
||||
public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo()
|
||||
{
|
||||
yield return (TB("Status"), status switch
|
||||
{
|
||||
DatabaseClientStatus.STARTING => TB("Starting"),
|
||||
_ => TB("Unavailable")
|
||||
});
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(unavailableReason))
|
||||
yield return (TB("Reason"), unavailableReason);
|
||||
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
public override Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token) =>
|
||||
Task.FromResult(new DataSourceEmbeddingManifest());
|
||||
|
||||
public override Task UpsertDataSourceAsync(
|
||||
string dataSourceId,
|
||||
string dataSourceName,
|
||||
string dataSourceType,
|
||||
string embeddingProviderId,
|
||||
string embeddingSignature,
|
||||
string sourceHash,
|
||||
int vectorSize,
|
||||
CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) =>
|
||||
Task.FromResult<IReadOnlyList<IndexStoreSearchResult>>([]);
|
||||
|
||||
public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
+419
@@ -0,0 +1,419 @@
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
public sealed class SqliteIndexStoreClientImplementation(
|
||||
string name,
|
||||
string databasePath,
|
||||
string basePath,
|
||||
string version) : IndexStoreClient(name, basePath)
|
||||
{
|
||||
private const string DATABASE_NAME = "Local RAG Index";
|
||||
private const string DATABASE_FILENAME = "rag-index.sqlite3";
|
||||
private const int MAX_FTS_QUERY_TERMS = 32;
|
||||
private const int CHUNK_UPSERT_BATCH_SIZE = 500;
|
||||
|
||||
private static readonly Regex FTS_TOKEN_REGEX = new(@"[\p{L}\p{Nd}_]+", RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
|
||||
private readonly string databasePath = databasePath;
|
||||
private readonly DbContextOptions<IndexStoreDbContext> dbContextOptions = IndexStoreDbContext.CreateOptions(databasePath);
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SqliteIndexStoreClientImplementation).Namespace, nameof(SqliteIndexStoreClientImplementation));
|
||||
|
||||
public override string CacheKey => $"{this.Name}:{this.databasePath}:{version}";
|
||||
|
||||
public static async Task<DatabaseClient> CreateAsync(
|
||||
ILogger logger,
|
||||
ILogger<DatabaseClient> databaseClientLogger,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(SettingsManager.DataDirectory))
|
||||
return CreateNoIndexStoreClient(DATABASE_NAME, "The application data directory is not available yet.", DatabaseClientStatus.STARTING, databaseClientLogger);
|
||||
|
||||
try
|
||||
{
|
||||
SQLitePCL.Batteries_V2.Init();
|
||||
|
||||
var basePath = Path.Combine(SettingsManager.DataDirectory, "databases", "sqlite");
|
||||
Directory.CreateDirectory(basePath);
|
||||
|
||||
var databasePath = Path.Combine(basePath, DATABASE_FILENAME);
|
||||
var client = new SqliteIndexStoreClientImplementation(DATABASE_NAME, databasePath, basePath, string.Empty);
|
||||
await client.InitializeAsync(cancellationToken);
|
||||
var version = await client.GetSqliteVersionAsync(cancellationToken);
|
||||
|
||||
client = new SqliteIndexStoreClientImplementation(DATABASE_NAME, databasePath, basePath, version);
|
||||
client.SetLogger(databaseClientLogger);
|
||||
return client;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "{DatabaseName} is not available. Indexed file fingerprints and search chunks are disabled.", DATABASE_NAME);
|
||||
return CreateNoIndexStoreClient(DATABASE_NAME, exception.Message, DatabaseClientStatus.UNAVAILABLE, databaseClientLogger);
|
||||
}
|
||||
}
|
||||
|
||||
public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo()
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
|
||||
yield return (TB("Reported version"), version);
|
||||
yield return (TB("Database path"), this.databasePath);
|
||||
yield return (TB("Storage size"), this.GetStorageSize());
|
||||
yield return (TB("Indexed data sources"), (await context.DataSources.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Indexed files"), (await context.EmbeddedFiles.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Search chunks"), (await context.EmbeddingChunks.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
public override async Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
var manifest = new DataSourceEmbeddingManifest();
|
||||
|
||||
var dataSource = await context.DataSources
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token);
|
||||
|
||||
if (dataSource is null)
|
||||
return manifest;
|
||||
|
||||
manifest.EmbeddingProviderId = dataSource.EmbeddingProviderId;
|
||||
manifest.EmbeddingSignature = dataSource.EmbeddingSignature;
|
||||
manifest.SourceHash = dataSource.SourceHash;
|
||||
manifest.VectorSize = dataSource.VectorSize;
|
||||
|
||||
var files = await context.EmbeddedFiles
|
||||
.AsNoTracking()
|
||||
.Where(file => file.DataSourceId == dataSourceId && file.ChunkCount > 0)
|
||||
.ToListAsync(token);
|
||||
foreach (var file in files)
|
||||
{
|
||||
manifest.Files[file.AbsolutePath] = new EmbeddedFileRecord(
|
||||
file.Fingerprint,
|
||||
file.FileSize,
|
||||
file.LastWriteUtc,
|
||||
file.EmbeddedAtUtc,
|
||||
file.ChunkCount);
|
||||
}
|
||||
|
||||
return manifest;
|
||||
}
|
||||
|
||||
public override async Task UpsertDataSourceAsync(
|
||||
string dataSourceId,
|
||||
string dataSourceName,
|
||||
string dataSourceType,
|
||||
string embeddingProviderId,
|
||||
string embeddingSignature,
|
||||
string sourceHash,
|
||||
int vectorSize,
|
||||
CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
var dataSource = await context.DataSources.FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token);
|
||||
if (dataSource is null)
|
||||
{
|
||||
dataSource = new EmbeddingStateDataSourceEntity
|
||||
{
|
||||
DataSourceId = dataSourceId,
|
||||
};
|
||||
context.DataSources.Add(dataSource);
|
||||
}
|
||||
|
||||
ApplyDataSource(dataSource, dataSourceName, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize);
|
||||
await context.SaveChangesAsync(token);
|
||||
}
|
||||
|
||||
public override async Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
var dataSource = await context.DataSources.FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token);
|
||||
if (dataSource is null)
|
||||
return;
|
||||
|
||||
dataSource.VectorSize = vectorSize;
|
||||
dataSource.UpdatedAtUtc = DateTimeOffset.UtcNow;
|
||||
await context.SaveChangesAsync(token);
|
||||
}
|
||||
|
||||
public override async Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
var dataSource = await context.DataSources.FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token);
|
||||
if (dataSource is null)
|
||||
return;
|
||||
|
||||
dataSource.SourceHash = sourceHash;
|
||||
dataSource.UpdatedAtUtc = DateTimeOffset.UtcNow;
|
||||
await context.SaveChangesAsync(token);
|
||||
}
|
||||
|
||||
public override async Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
var fileEntity = await context.EmbeddedFiles.FirstOrDefaultAsync(entity => entity.ParentFileId == file.ParentFileId, token);
|
||||
if (fileEntity is null)
|
||||
{
|
||||
fileEntity = new EmbeddingStateFileEntity
|
||||
{
|
||||
ParentFileId = file.ParentFileId,
|
||||
};
|
||||
context.EmbeddedFiles.Add(fileEntity);
|
||||
}
|
||||
|
||||
ApplyFile(fileEntity, dataSourceId, file);
|
||||
await context.SaveChangesAsync(token);
|
||||
}
|
||||
|
||||
public override async Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
await using var transaction = await context.Database.BeginTransactionAsync(token);
|
||||
|
||||
var parentFileIds = await context.EmbeddedFiles
|
||||
.Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath)
|
||||
.Select(file => file.ParentFileId)
|
||||
.ToListAsync(token);
|
||||
|
||||
foreach (var parentFileIdBatch in parentFileIds.Chunk(CHUNK_UPSERT_BATCH_SIZE))
|
||||
await context.EmbeddingChunks
|
||||
.Where(chunk => parentFileIdBatch.Contains(chunk.ParentFileId))
|
||||
.ExecuteDeleteAsync(token);
|
||||
|
||||
await context.EmbeddedFiles
|
||||
.Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath)
|
||||
.ExecuteDeleteAsync(token);
|
||||
|
||||
await transaction.CommitAsync(token);
|
||||
}
|
||||
|
||||
public override async Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token)
|
||||
{
|
||||
if (chunks.Count == 0)
|
||||
return;
|
||||
|
||||
await using var context = this.CreateContext();
|
||||
await using var transaction = await context.Database.BeginTransactionAsync(token);
|
||||
|
||||
foreach (var chunkBatch in chunks.Chunk(CHUNK_UPSERT_BATCH_SIZE))
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var chunkIds = chunkBatch
|
||||
.Select(chunk => chunk.ChunkId)
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
var existingChunks = await context.EmbeddingChunks
|
||||
.Where(chunk => chunkIds.Contains(chunk.ChunkId))
|
||||
.ToDictionaryAsync(chunk => chunk.ChunkId, StringComparer.Ordinal, token);
|
||||
|
||||
foreach (var chunk in chunkBatch)
|
||||
{
|
||||
if (!existingChunks.TryGetValue(chunk.ChunkId, out var chunkEntity))
|
||||
{
|
||||
chunkEntity = new EmbeddingStateChunkEntity
|
||||
{
|
||||
ChunkId = chunk.ChunkId,
|
||||
};
|
||||
context.EmbeddingChunks.Add(chunkEntity);
|
||||
existingChunks[chunk.ChunkId] = chunkEntity;
|
||||
}
|
||||
|
||||
ApplyChunk(chunkEntity, chunk);
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(token);
|
||||
context.ChangeTracker.Clear();
|
||||
}
|
||||
|
||||
await transaction.CommitAsync(token);
|
||||
}
|
||||
|
||||
public override async Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token)
|
||||
{
|
||||
if (maxMatches <= 0)
|
||||
return [];
|
||||
|
||||
var ftsQuery = BuildFtsQuery(query);
|
||||
if (string.IsNullOrWhiteSpace(ftsQuery))
|
||||
return [];
|
||||
|
||||
await using var context = this.CreateContext();
|
||||
var results = await context.SearchResults
|
||||
.FromSqlInterpolated($"""
|
||||
SELECT
|
||||
c.chunk_id AS ChunkId,
|
||||
c.parent_file_id AS ParentFileId,
|
||||
ds.data_source_id AS DataSourceId,
|
||||
ds.data_source_name AS DataSourceName,
|
||||
ds.data_source_type AS DataSourceType,
|
||||
f.absolute_path AS AbsolutePath,
|
||||
f.file_name AS FileName,
|
||||
f.relative_path AS RelativePath,
|
||||
f.file_type AS FileType,
|
||||
c.page_number AS PageNumber,
|
||||
c.chunk_index AS ChunkIndex,
|
||||
c.chunk_text AS ChunkText,
|
||||
bm25(embedding_chunks_fts) AS Score,
|
||||
f.fingerprint AS Fingerprint,
|
||||
f.file_size AS FileSize,
|
||||
f.creation_utc AS CreationUtc,
|
||||
f.last_write_utc AS LastWriteUtc,
|
||||
c.embedded_at_utc AS EmbeddedAtUtc,
|
||||
f.chunk_count AS ChunkCount,
|
||||
f.confidence_level AS ConfidenceLevel,
|
||||
f.confidence_level_rank AS ConfidenceLevelRank
|
||||
FROM embedding_chunks_fts
|
||||
JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid
|
||||
JOIN embedded_files f ON f.parent_file_id = c.parent_file_id
|
||||
JOIN data_sources ds ON ds.data_source_id = f.data_source_id
|
||||
WHERE ds.data_source_id = {dataSourceId}
|
||||
AND embedding_chunks_fts MATCH {ftsQuery}
|
||||
ORDER BY Score
|
||||
LIMIT {maxMatches}
|
||||
""")
|
||||
.AsNoTracking()
|
||||
.ToListAsync(token);
|
||||
|
||||
return results.Select(ToSearchResult).ToList();
|
||||
}
|
||||
|
||||
public override async Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
await using var transaction = await context.Database.BeginTransactionAsync(token);
|
||||
|
||||
var parentFileIds = await context.EmbeddedFiles
|
||||
.Where(file => file.DataSourceId == dataSourceId)
|
||||
.Select(file => file.ParentFileId)
|
||||
.ToListAsync(token);
|
||||
|
||||
foreach (var parentFileIdBatch in parentFileIds.Chunk(CHUNK_UPSERT_BATCH_SIZE))
|
||||
await context.EmbeddingChunks
|
||||
.Where(chunk => parentFileIdBatch.Contains(chunk.ParentFileId))
|
||||
.ExecuteDeleteAsync(token);
|
||||
|
||||
await context.DataSources
|
||||
.Where(source => source.DataSourceId == dataSourceId)
|
||||
.ExecuteDeleteAsync(token);
|
||||
|
||||
await transaction.CommitAsync(token);
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
private async Task InitializeAsync(CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
await IndexStoreSchemaMigrator.MigrateAsync(context, token);
|
||||
}
|
||||
|
||||
private async Task<string> GetSqliteVersionAsync(CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
var versions = await context.Database
|
||||
.SqlQueryRaw<string>("SELECT sqlite_version() AS Value")
|
||||
.ToListAsync(token);
|
||||
return versions.FirstOrDefault() ?? string.Empty;
|
||||
}
|
||||
|
||||
private IndexStoreDbContext CreateContext() => new(this.dbContextOptions);
|
||||
|
||||
private static void ApplyDataSource(
|
||||
EmbeddingStateDataSourceEntity dataSource,
|
||||
string dataSourceName,
|
||||
string dataSourceType,
|
||||
string embeddingProviderId,
|
||||
string embeddingSignature,
|
||||
string sourceHash,
|
||||
int vectorSize)
|
||||
{
|
||||
dataSource.DataSourceName = dataSourceName;
|
||||
dataSource.DataSourceType = dataSourceType;
|
||||
dataSource.EmbeddingProviderId = embeddingProviderId;
|
||||
dataSource.EmbeddingSignature = embeddingSignature;
|
||||
dataSource.SourceHash = sourceHash;
|
||||
dataSource.VectorSize = vectorSize;
|
||||
dataSource.UpdatedAtUtc = DateTimeOffset.UtcNow;
|
||||
}
|
||||
|
||||
private static void ApplyFile(EmbeddingStateFileEntity fileEntity, string dataSourceId, EmbeddingStateFile file)
|
||||
{
|
||||
fileEntity.DataSourceId = dataSourceId;
|
||||
fileEntity.AbsolutePath = file.AbsolutePath;
|
||||
fileEntity.FileName = file.FileName;
|
||||
fileEntity.RelativePath = file.RelativePath;
|
||||
fileEntity.FileType = file.FileType;
|
||||
fileEntity.Fingerprint = file.Fingerprint;
|
||||
fileEntity.FileSize = file.FileSize;
|
||||
fileEntity.CreationUtc = file.CreationUtc;
|
||||
fileEntity.LastWriteUtc = file.LastWriteUtc;
|
||||
fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc;
|
||||
fileEntity.ChunkCount = file.ChunkCount;
|
||||
fileEntity.ConfidenceLevel = file.ConfidenceLevel;
|
||||
fileEntity.ConfidenceLevelRank = file.ConfidenceLevelRank;
|
||||
}
|
||||
|
||||
private static void ApplyChunk(EmbeddingStateChunkEntity chunkEntity, EmbeddingStateChunk chunk)
|
||||
{
|
||||
chunkEntity.ChunkId = chunk.ChunkId;
|
||||
chunkEntity.ParentFileId = chunk.ParentFileId;
|
||||
chunkEntity.PageNumber = chunk.PageNumber;
|
||||
chunkEntity.ChunkIndex = chunk.ChunkIndex;
|
||||
chunkEntity.ChunkText = chunk.ChunkText;
|
||||
chunkEntity.EmbeddedAtUtc = chunk.EmbeddedAtUtc;
|
||||
}
|
||||
|
||||
private static IndexStoreSearchResult ToSearchResult(IndexStoreSearchResultEntity result) => new(
|
||||
result.ChunkId,
|
||||
result.ParentFileId,
|
||||
result.DataSourceId,
|
||||
result.DataSourceName,
|
||||
result.DataSourceType,
|
||||
result.AbsolutePath,
|
||||
result.FileName,
|
||||
result.RelativePath,
|
||||
result.FileType,
|
||||
result.PageNumber,
|
||||
result.ChunkIndex,
|
||||
result.ChunkText,
|
||||
result.Score,
|
||||
result.Fingerprint,
|
||||
result.FileSize,
|
||||
result.CreationUtc,
|
||||
result.LastWriteUtc,
|
||||
result.EmbeddedAtUtc,
|
||||
result.ChunkCount,
|
||||
result.ConfidenceLevel,
|
||||
result.ConfidenceLevelRank);
|
||||
|
||||
private static string BuildFtsQuery(string query)
|
||||
{
|
||||
var terms = FTS_TOKEN_REGEX
|
||||
.Matches(query)
|
||||
.Select(match => match.Value)
|
||||
.Where(term => !string.IsNullOrWhiteSpace(term))
|
||||
.Distinct(StringComparer.OrdinalIgnoreCase)
|
||||
.Take(MAX_FTS_QUERY_TERMS)
|
||||
.Select(term => $"\"{term.Replace("\"", "\"\"", StringComparison.Ordinal)}\"")
|
||||
.ToList();
|
||||
|
||||
return terms.Count == 0 ? string.Empty : string.Join(" OR ", terms);
|
||||
}
|
||||
|
||||
private static NoIndexStoreClient CreateNoIndexStoreClient(string name, string? unavailableReason, DatabaseClientStatus status, ILogger<DatabaseClient> databaseClientLogger)
|
||||
{
|
||||
var client = new NoIndexStoreClient(name, unavailableReason, status);
|
||||
client.SetLogger(databaseClientLogger);
|
||||
return client;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user