mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-14 02:13:47 +00:00
changed to sqlite connection to ef core
This commit is contained in:
PaulKoudelka committed
1 parent
23416b21fc
commit
3d68d73767
8 files changed
+1222
-503
No files matched your search
@@ -0,0 +1,259 @@
|
||||
using System.Globalization;
|
||||
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState;
|
||||
|
||||
internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbContext> options) : DbContext(options)
|
||||
{
|
||||
public static DbContextOptions<EmbeddingStateDbContext> CreateOptions(string databasePath) => new DbContextOptionsBuilder<EmbeddingStateDbContext>()
|
||||
.UseSqlite(BuildConnectionString(databasePath))
|
||||
.Options;
|
||||
|
||||
public DbSet<EmbeddingStateDataSourceEntity> DataSources => this.Set<EmbeddingStateDataSourceEntity>();
|
||||
|
||||
public DbSet<EmbeddingStateFileEntity> EmbeddedFiles => this.Set<EmbeddingStateFileEntity>();
|
||||
|
||||
public DbSet<EmbeddingStateChunkEntity> EmbeddingChunks => this.Set<EmbeddingStateChunkEntity>();
|
||||
|
||||
public DbSet<EmbeddingStateSearchResultEntity> SearchResults => this.Set<EmbeddingStateSearchResultEntity>();
|
||||
|
||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||
{
|
||||
var utcDateTimeConverter = new EmbeddingStateDateTimeConverter();
|
||||
|
||||
modelBuilder.Entity<EmbeddingStateDataSourceEntity>(entity =>
|
||||
{
|
||||
entity.ToTable("data_sources");
|
||||
entity.HasKey(dataSource => dataSource.DataSourceId);
|
||||
|
||||
entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id");
|
||||
entity.Property(dataSource => dataSource.DataSourceName).HasColumnName("data_source_name").IsRequired();
|
||||
entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired();
|
||||
entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired();
|
||||
entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired();
|
||||
entity.Property(dataSource => dataSource.SourceHash).HasColumnName("source_hash").IsRequired().HasDefaultValue(string.Empty);
|
||||
entity.Property(dataSource => dataSource.VectorSize).HasColumnName("vector_size").HasDefaultValue(0);
|
||||
entity.Property(dataSource => dataSource.UpdatedAtUtc).HasColumnName("updated_at_utc").HasConversion(utcDateTimeConverter).IsRequired();
|
||||
|
||||
entity
|
||||
.HasMany(dataSource => dataSource.Files)
|
||||
.WithOne(file => file.DataSource)
|
||||
.HasForeignKey(file => file.DataSourceId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity<EmbeddingStateFileEntity>(entity =>
|
||||
{
|
||||
entity.ToTable("embedded_files");
|
||||
entity.HasKey(file => file.ParentFileId);
|
||||
|
||||
entity.Property(file => file.ParentFileId).HasColumnName("parent_file_id");
|
||||
entity.Property(file => file.DataSourceId).HasColumnName("data_source_id").IsRequired();
|
||||
entity.Property(file => file.AbsolutePath).HasColumnName("absolute_path").UseCollation("NOCASE").IsRequired();
|
||||
entity.Property(file => file.FileName).HasColumnName("file_name").IsRequired();
|
||||
entity.Property(file => file.RelativePath).HasColumnName("relative_path").IsRequired();
|
||||
entity.Property(file => file.FileType).HasColumnName("file_type").IsRequired();
|
||||
entity.Property(file => file.Fingerprint).HasColumnName("fingerprint").IsRequired();
|
||||
entity.Property(file => file.FileSize).HasColumnName("file_size");
|
||||
entity.Property(file => file.CreationUtc).HasColumnName("creation_utc").HasConversion(utcDateTimeConverter).IsRequired();
|
||||
entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeConverter).IsRequired();
|
||||
entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeConverter).IsRequired();
|
||||
entity.Property(file => file.ChunkCount).HasColumnName("chunk_count");
|
||||
entity.Property(file => file.ComplianceLevel).HasColumnName("compliance_level").IsRequired();
|
||||
entity.Property(file => file.ComplianceLevelRank).HasColumnName("compliance_level_rank");
|
||||
|
||||
entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source");
|
||||
entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path");
|
||||
entity.HasIndex(file => file.FileType).HasDatabaseName("idx_embedded_files_file_type");
|
||||
entity.HasIndex(file => file.ComplianceLevelRank).HasDatabaseName("idx_embedded_files_compliance");
|
||||
entity.HasIndex(file => new { file.DataSourceId, file.AbsolutePath }).HasDatabaseName("idx_embedded_files_data_source_absolute_path").IsUnique();
|
||||
|
||||
entity
|
||||
.HasMany(file => file.Chunks)
|
||||
.WithOne(chunk => chunk.File)
|
||||
.HasForeignKey(chunk => chunk.ParentFileId)
|
||||
.OnDelete(DeleteBehavior.Cascade);
|
||||
});
|
||||
|
||||
modelBuilder.Entity<EmbeddingStateChunkEntity>(entity =>
|
||||
{
|
||||
entity.ToTable("embedding_chunks");
|
||||
entity.HasKey(chunk => chunk.Id);
|
||||
|
||||
entity.Property(chunk => chunk.Id).HasColumnName("id").ValueGeneratedOnAdd();
|
||||
entity.Property(chunk => chunk.ChunkId).HasColumnName("chunk_id").IsRequired();
|
||||
entity.Property(chunk => chunk.ParentFileId).HasColumnName("parent_file_id").IsRequired();
|
||||
entity.Property(chunk => chunk.PageNumber).HasColumnName("page_number");
|
||||
entity.Property(chunk => chunk.ChunkIndex).HasColumnName("chunk_index");
|
||||
entity.Property(chunk => chunk.ChunkText).HasColumnName("chunk_text").IsRequired();
|
||||
entity.Property(chunk => chunk.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeConverter).IsRequired();
|
||||
|
||||
entity.HasIndex(chunk => chunk.ChunkId).HasDatabaseName("idx_embedding_chunks_chunk_id").IsUnique();
|
||||
entity.HasIndex(chunk => chunk.ParentFileId).HasDatabaseName("idx_embedding_chunks_parent_file");
|
||||
entity.HasIndex(chunk => chunk.PageNumber).HasDatabaseName("idx_embedding_chunks_page");
|
||||
entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index").IsUnique();
|
||||
});
|
||||
|
||||
modelBuilder.Entity<EmbeddingStateSearchResultEntity>(entity =>
|
||||
{
|
||||
entity.HasNoKey();
|
||||
entity.ToView("embedding_chunk_search_results");
|
||||
|
||||
entity.Property(result => result.CreationUtc).HasConversion(utcDateTimeConverter);
|
||||
entity.Property(result => result.LastWriteUtc).HasConversion(utcDateTimeConverter);
|
||||
entity.Property(result => result.EmbeddedAtUtc).HasConversion(utcDateTimeConverter);
|
||||
});
|
||||
}
|
||||
|
||||
private static string BuildConnectionString(string databasePath) => new SqliteConnectionStringBuilder
|
||||
{
|
||||
DataSource = databasePath,
|
||||
Mode = SqliteOpenMode.ReadWriteCreate,
|
||||
Cache = SqliteCacheMode.Shared,
|
||||
ForeignKeys = true,
|
||||
DefaultTimeout = 30,
|
||||
}.ToString();
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateDataSourceEntity
|
||||
{
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceName { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceType { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingProviderId { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingSignature { get; set; } = string.Empty;
|
||||
|
||||
public string SourceHash { get; set; } = string.Empty;
|
||||
|
||||
public int VectorSize { get; set; }
|
||||
|
||||
public DateTime UpdatedAtUtc { get; set; }
|
||||
|
||||
public List<EmbeddingStateFileEntity> Files { get; set; } = [];
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateFileEntity
|
||||
{
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string AbsolutePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileName { get; set; } = string.Empty;
|
||||
|
||||
public string RelativePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileType { get; set; } = string.Empty;
|
||||
|
||||
public string Fingerprint { get; set; } = string.Empty;
|
||||
|
||||
public long FileSize { get; set; }
|
||||
|
||||
public DateTime CreationUtc { get; set; }
|
||||
|
||||
public DateTime LastWriteUtc { get; set; }
|
||||
|
||||
public DateTime EmbeddedAtUtc { get; set; }
|
||||
|
||||
public int ChunkCount { get; set; }
|
||||
|
||||
public string ComplianceLevel { get; set; } = string.Empty;
|
||||
|
||||
public int ComplianceLevelRank { get; set; }
|
||||
|
||||
public EmbeddingStateDataSourceEntity? DataSource { get; set; }
|
||||
|
||||
public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateChunkEntity
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public string ChunkId { get; set; } = string.Empty;
|
||||
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public int? PageNumber { get; set; }
|
||||
|
||||
public int ChunkIndex { get; set; }
|
||||
|
||||
public string ChunkText { get; set; } = string.Empty;
|
||||
|
||||
public DateTime EmbeddedAtUtc { get; set; }
|
||||
|
||||
public EmbeddingStateFileEntity? File { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateSearchResultEntity
|
||||
{
|
||||
public string ChunkId { get; set; } = string.Empty;
|
||||
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceName { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceType { get; set; } = string.Empty;
|
||||
|
||||
public string AbsolutePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileName { get; set; } = string.Empty;
|
||||
|
||||
public string RelativePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileType { get; set; } = string.Empty;
|
||||
|
||||
public int? PageNumber { get; set; }
|
||||
|
||||
public int ChunkIndex { get; set; }
|
||||
|
||||
public string ChunkText { get; set; } = string.Empty;
|
||||
|
||||
public double Score { get; set; }
|
||||
|
||||
public string Fingerprint { get; set; } = string.Empty;
|
||||
|
||||
public long FileSize { get; set; }
|
||||
|
||||
public DateTime CreationUtc { get; set; }
|
||||
|
||||
public DateTime LastWriteUtc { get; set; }
|
||||
|
||||
public DateTime EmbeddedAtUtc { get; set; }
|
||||
|
||||
public int ChunkCount { get; set; }
|
||||
|
||||
public string ComplianceLevel { get; set; } = string.Empty;
|
||||
|
||||
public int ComplianceLevelRank { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateDateTimeConverter() : ValueConverter<DateTime, string>(
|
||||
value => EmbeddingStateDateTime.ToUtcText(value),
|
||||
value => EmbeddingStateDateTime.ParseUtc(value));
|
||||
|
||||
internal static class EmbeddingStateDateTime
|
||||
{
|
||||
public static string ToUtcText(DateTime dateTime)
|
||||
{
|
||||
var utc = dateTime.Kind is DateTimeKind.Utc ? dateTime : dateTime.ToUniversalTime();
|
||||
return utc.ToString("O", CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
public static DateTime ParseUtc(string value)
|
||||
{
|
||||
return DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var dateTime)
|
||||
? dateTime.ToUniversalTime()
|
||||
: DateTime.UnixEpoch;
|
||||
}
|
||||
}
|
||||
+15
@@ -0,0 +1,15 @@
|
||||
using Microsoft.EntityFrameworkCore.Design;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState;
|
||||
|
||||
internal sealed class EmbeddingStateDesignTimeDbContextFactory : IDesignTimeDbContextFactory<EmbeddingStateDbContext>
|
||||
{
|
||||
public EmbeddingStateDbContext CreateDbContext(string[] args)
|
||||
{
|
||||
var databasePath = args.FirstOrDefault(argument => argument.EndsWith(".sqlite3", StringComparison.OrdinalIgnoreCase));
|
||||
if (string.IsNullOrWhiteSpace(databasePath))
|
||||
databasePath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-rag-index-design.sqlite3");
|
||||
|
||||
return new EmbeddingStateDbContext(EmbeddingStateDbContext.CreateOptions(databasePath));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
using System.Diagnostics.CodeAnalysis;
|
||||
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState;
|
||||
|
||||
internal static class EmbeddingStateSchemaMigrator
|
||||
{
|
||||
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))]
|
||||
public static async Task MigrateAsync(EmbeddingStateDbContext context, CancellationToken token)
|
||||
{
|
||||
await context.Database.MigrateAsync(token);
|
||||
await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL;", token);
|
||||
}
|
||||
}
|
||||
+202
@@ -0,0 +1,202 @@
|
||||
#nullable disable
|
||||
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
using Microsoft.EntityFrameworkCore.Migrations;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState.Migrations;
|
||||
|
||||
[DbContext(typeof(EmbeddingStateDbContext))]
|
||||
[Migration("20260804000000_InitialRagIndex")]
|
||||
public partial class InitialRagIndex : Migration
|
||||
{
|
||||
protected override void Up(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.CreateTable(
|
||||
name: "data_sources",
|
||||
columns: table => new
|
||||
{
|
||||
data_source_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
data_source_name = table.Column<string>(type: "TEXT", nullable: false),
|
||||
data_source_type = table.Column<string>(type: "TEXT", nullable: false),
|
||||
embedding_provider_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
embedding_signature = table.Column<string>(type: "TEXT", nullable: false),
|
||||
source_hash = table.Column<string>(type: "TEXT", nullable: false, defaultValue: string.Empty),
|
||||
vector_size = table.Column<int>(type: "INTEGER", nullable: false, defaultValue: 0),
|
||||
updated_at_utc = table.Column<string>(type: "TEXT", nullable: false),
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_data_sources", source => source.data_source_id);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateTable(
|
||||
name: "embedded_files",
|
||||
columns: table => new
|
||||
{
|
||||
parent_file_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
data_source_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
absolute_path = table.Column<string>(type: "TEXT", nullable: false, collation: "NOCASE"),
|
||||
file_name = table.Column<string>(type: "TEXT", nullable: false),
|
||||
relative_path = table.Column<string>(type: "TEXT", nullable: false),
|
||||
file_type = table.Column<string>(type: "TEXT", nullable: false),
|
||||
fingerprint = table.Column<string>(type: "TEXT", nullable: false),
|
||||
file_size = table.Column<long>(type: "INTEGER", nullable: false),
|
||||
creation_utc = table.Column<string>(type: "TEXT", nullable: false),
|
||||
last_write_utc = table.Column<string>(type: "TEXT", nullable: false),
|
||||
embedded_at_utc = table.Column<string>(type: "TEXT", nullable: false),
|
||||
chunk_count = table.Column<int>(type: "INTEGER", nullable: false),
|
||||
compliance_level = table.Column<string>(type: "TEXT", nullable: false),
|
||||
compliance_level_rank = table.Column<int>(type: "INTEGER", nullable: false),
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_embedded_files", file => file.parent_file_id);
|
||||
table.ForeignKey(
|
||||
name: "FK_embedded_files_data_sources_data_source_id",
|
||||
column: file => file.data_source_id,
|
||||
principalTable: "data_sources",
|
||||
principalColumn: "data_source_id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateTable(
|
||||
name: "embedding_chunks",
|
||||
columns: table => new
|
||||
{
|
||||
id = table.Column<int>(type: "INTEGER", nullable: false)
|
||||
.Annotation("Sqlite:Autoincrement", true),
|
||||
chunk_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
parent_file_id = table.Column<string>(type: "TEXT", nullable: false),
|
||||
page_number = table.Column<int>(type: "INTEGER", nullable: true),
|
||||
chunk_index = table.Column<int>(type: "INTEGER", nullable: false),
|
||||
chunk_text = table.Column<string>(type: "TEXT", nullable: false),
|
||||
embedded_at_utc = table.Column<string>(type: "TEXT", nullable: false),
|
||||
},
|
||||
constraints: table =>
|
||||
{
|
||||
table.PrimaryKey("PK_embedding_chunks", chunk => chunk.id);
|
||||
table.ForeignKey(
|
||||
name: "FK_embedding_chunks_embedded_files_parent_file_id",
|
||||
column: chunk => chunk.parent_file_id,
|
||||
principalTable: "embedded_files",
|
||||
principalColumn: "parent_file_id",
|
||||
onDelete: ReferentialAction.Cascade);
|
||||
});
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedded_files_absolute_path",
|
||||
table: "embedded_files",
|
||||
column: "absolute_path");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedded_files_compliance",
|
||||
table: "embedded_files",
|
||||
column: "compliance_level_rank");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedded_files_data_source",
|
||||
table: "embedded_files",
|
||||
column: "data_source_id");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedded_files_data_source_absolute_path",
|
||||
table: "embedded_files",
|
||||
columns: ["data_source_id", "absolute_path"],
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedded_files_file_type",
|
||||
table: "embedded_files",
|
||||
column: "file_type");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedding_chunks_chunk_id",
|
||||
table: "embedding_chunks",
|
||||
column: "chunk_id",
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedding_chunks_page",
|
||||
table: "embedding_chunks",
|
||||
column: "page_number");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedding_chunks_parent_file",
|
||||
table: "embedding_chunks",
|
||||
column: "parent_file_id");
|
||||
|
||||
migrationBuilder.CreateIndex(
|
||||
name: "idx_embedding_chunks_parent_file_chunk_index",
|
||||
table: "embedding_chunks",
|
||||
columns: ["parent_file_id", "chunk_index"],
|
||||
unique: true);
|
||||
|
||||
migrationBuilder.Sql("""
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS embedding_chunks_fts
|
||||
USING fts5(chunk_id UNINDEXED, file_name, chunk_text);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedding_chunks_ai
|
||||
AFTER INSERT ON embedding_chunks
|
||||
BEGIN
|
||||
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
|
||||
VALUES (
|
||||
new.id,
|
||||
new.chunk_id,
|
||||
(SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id),
|
||||
new.chunk_text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedding_chunks_ad
|
||||
AFTER DELETE ON embedding_chunks
|
||||
BEGIN
|
||||
DELETE FROM embedding_chunks_fts
|
||||
WHERE rowid = old.id;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedding_chunks_au
|
||||
AFTER UPDATE ON embedding_chunks
|
||||
BEGIN
|
||||
DELETE FROM embedding_chunks_fts
|
||||
WHERE rowid = old.id;
|
||||
|
||||
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
|
||||
VALUES (
|
||||
new.id,
|
||||
new.chunk_id,
|
||||
(SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id),
|
||||
new.chunk_text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedded_files_file_name_au
|
||||
AFTER UPDATE OF file_name ON embedded_files
|
||||
BEGIN
|
||||
DELETE FROM embedding_chunks_fts
|
||||
WHERE rowid IN (
|
||||
SELECT id
|
||||
FROM embedding_chunks
|
||||
WHERE parent_file_id = new.parent_file_id
|
||||
);
|
||||
|
||||
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
|
||||
SELECT id, chunk_id, new.file_name, chunk_text
|
||||
FROM embedding_chunks
|
||||
WHERE parent_file_id = new.parent_file_id;
|
||||
END;
|
||||
""");
|
||||
}
|
||||
|
||||
protected override void Down(MigrationBuilder migrationBuilder)
|
||||
{
|
||||
migrationBuilder.Sql("""
|
||||
DROP TRIGGER IF EXISTS embedded_files_file_name_au;
|
||||
DROP TRIGGER IF EXISTS embedding_chunks_au;
|
||||
DROP TRIGGER IF EXISTS embedding_chunks_ad;
|
||||
DROP TRIGGER IF EXISTS embedding_chunks_ai;
|
||||
DROP TABLE IF EXISTS embedding_chunks_fts;
|
||||
""");
|
||||
|
||||
migrationBuilder.DropTable(name: "embedding_chunks");
|
||||
migrationBuilder.DropTable(name: "embedded_files");
|
||||
migrationBuilder.DropTable(name: "data_sources");
|
||||
}
|
||||
}
|
||||
+327
@@ -0,0 +1,327 @@
|
||||
#nullable disable
|
||||
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Infrastructure;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState.Migrations;
|
||||
|
||||
[DbContext(typeof(EmbeddingStateDbContext))]
|
||||
partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
||||
{
|
||||
protected override void BuildModel(ModelBuilder modelBuilder)
|
||||
{
|
||||
#pragma warning disable 612, 618
|
||||
modelBuilder.HasAnnotation("ProductVersion", "9.0.18");
|
||||
var utcDateTimeConverter = new EmbeddingStateDateTimeConverter();
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity =>
|
||||
{
|
||||
entity.Property<string>("DataSourceId")
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("data_source_id");
|
||||
|
||||
entity.Property<string>("DataSourceName")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("data_source_name");
|
||||
|
||||
entity.Property<string>("DataSourceType")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("data_source_type");
|
||||
|
||||
entity.Property<string>("EmbeddingProviderId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("embedding_provider_id");
|
||||
|
||||
entity.Property<string>("EmbeddingSignature")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("embedding_signature");
|
||||
|
||||
entity.Property<string>("SourceHash")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("source_hash")
|
||||
.HasDefaultValue(string.Empty);
|
||||
|
||||
entity.Property<DateTime>("UpdatedAtUtc")
|
||||
.HasConversion(utcDateTimeConverter)
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("updated_at_utc");
|
||||
|
||||
entity.Property<int>("VectorSize")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("vector_size")
|
||||
.HasDefaultValue(0);
|
||||
|
||||
entity.HasKey("DataSourceId");
|
||||
|
||||
entity.ToTable("data_sources");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity =>
|
||||
{
|
||||
entity.Property<string>("ParentFileId")
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("parent_file_id");
|
||||
|
||||
entity.Property<string>("AbsolutePath")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("absolute_path")
|
||||
.UseCollation("NOCASE");
|
||||
|
||||
entity.Property<int>("ChunkCount")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("chunk_count");
|
||||
|
||||
entity.Property<string>("ComplianceLevel")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("compliance_level");
|
||||
|
||||
entity.Property<int>("ComplianceLevelRank")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("compliance_level_rank");
|
||||
|
||||
entity.Property<DateTime>("CreationUtc")
|
||||
.HasConversion(utcDateTimeConverter)
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("creation_utc");
|
||||
|
||||
entity.Property<string>("DataSourceId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("data_source_id");
|
||||
|
||||
entity.Property<DateTime>("EmbeddedAtUtc")
|
||||
.HasConversion(utcDateTimeConverter)
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("embedded_at_utc");
|
||||
|
||||
entity.Property<string>("FileName")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("file_name");
|
||||
|
||||
entity.Property<long>("FileSize")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("file_size");
|
||||
|
||||
entity.Property<string>("FileType")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("file_type");
|
||||
|
||||
entity.Property<string>("Fingerprint")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("fingerprint");
|
||||
|
||||
entity.Property<DateTime>("LastWriteUtc")
|
||||
.HasConversion(utcDateTimeConverter)
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("last_write_utc");
|
||||
|
||||
entity.Property<string>("RelativePath")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("relative_path");
|
||||
|
||||
entity.HasKey("ParentFileId");
|
||||
|
||||
entity.HasIndex("AbsolutePath")
|
||||
.HasDatabaseName("idx_embedded_files_absolute_path");
|
||||
|
||||
entity.HasIndex("ComplianceLevelRank")
|
||||
.HasDatabaseName("idx_embedded_files_compliance");
|
||||
|
||||
entity.HasIndex("DataSourceId")
|
||||
.HasDatabaseName("idx_embedded_files_data_source");
|
||||
|
||||
entity.HasIndex("DataSourceId", "AbsolutePath")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("idx_embedded_files_data_source_absolute_path");
|
||||
|
||||
entity.HasIndex("FileType")
|
||||
.HasDatabaseName("idx_embedded_files_file_type");
|
||||
|
||||
entity.ToTable("embedded_files");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateChunkEntity", entity =>
|
||||
{
|
||||
entity.Property<int>("Id")
|
||||
.ValueGeneratedOnAdd()
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("id")
|
||||
.HasAnnotation("Sqlite:Autoincrement", true);
|
||||
|
||||
entity.Property<string>("ChunkId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("chunk_id");
|
||||
|
||||
entity.Property<int>("ChunkIndex")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("chunk_index");
|
||||
|
||||
entity.Property<string>("ChunkText")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("chunk_text");
|
||||
|
||||
entity.Property<DateTime>("EmbeddedAtUtc")
|
||||
.HasConversion(utcDateTimeConverter)
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("embedded_at_utc");
|
||||
|
||||
entity.Property<int?>("PageNumber")
|
||||
.HasColumnType("INTEGER")
|
||||
.HasColumnName("page_number");
|
||||
|
||||
entity.Property<string>("ParentFileId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT")
|
||||
.HasColumnName("parent_file_id");
|
||||
|
||||
entity.HasKey("Id");
|
||||
|
||||
entity.HasIndex("ChunkId")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("idx_embedding_chunks_chunk_id");
|
||||
|
||||
entity.HasIndex("PageNumber")
|
||||
.HasDatabaseName("idx_embedding_chunks_page");
|
||||
|
||||
entity.HasIndex("ParentFileId")
|
||||
.HasDatabaseName("idx_embedding_chunks_parent_file");
|
||||
|
||||
entity.HasIndex("ParentFileId", "ChunkIndex")
|
||||
.IsUnique()
|
||||
.HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index");
|
||||
|
||||
entity.ToTable("embedding_chunks");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateSearchResultEntity", entity =>
|
||||
{
|
||||
entity.Property<string>("AbsolutePath")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<int>("ChunkCount")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
entity.Property<string>("ChunkId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<int>("ChunkIndex")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
entity.Property<string>("ChunkText")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("ComplianceLevel")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<int>("ComplianceLevelRank")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
entity.Property<DateTime>("CreationUtc")
|
||||
.HasConversion(utcDateTimeConverter)
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("DataSourceId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("DataSourceName")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("DataSourceType")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<DateTime>("EmbeddedAtUtc")
|
||||
.HasConversion(utcDateTimeConverter)
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("FileName")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<long>("FileSize")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
entity.Property<string>("FileType")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("Fingerprint")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<DateTime>("LastWriteUtc")
|
||||
.HasConversion(utcDateTimeConverter)
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<int?>("PageNumber")
|
||||
.HasColumnType("INTEGER");
|
||||
|
||||
entity.Property<string>("ParentFileId")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<string>("RelativePath")
|
||||
.IsRequired()
|
||||
.HasColumnType("TEXT");
|
||||
|
||||
entity.Property<double>("Score")
|
||||
.HasColumnType("REAL");
|
||||
|
||||
entity.HasNoKey();
|
||||
|
||||
entity.ToView("embedding_chunk_search_results");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity =>
|
||||
{
|
||||
entity.HasOne("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", "DataSource")
|
||||
.WithMany("Files")
|
||||
.HasForeignKey("DataSourceId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
entity.Navigation("DataSource");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateChunkEntity", entity =>
|
||||
{
|
||||
entity.HasOne("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", "File")
|
||||
.WithMany("Chunks")
|
||||
.HasForeignKey("ParentFileId")
|
||||
.OnDelete(DeleteBehavior.Cascade)
|
||||
.IsRequired();
|
||||
|
||||
entity.Navigation("File");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity =>
|
||||
{
|
||||
entity.Navigation("Files");
|
||||
});
|
||||
|
||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity =>
|
||||
{
|
||||
entity.Navigation("Chunks");
|
||||
});
|
||||
#pragma warning restore 612, 618
|
||||
}
|
||||
}
|
||||
+241
-452
@@ -5,7 +5,7 @@ using AIStudio.Settings;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace AIStudio.Tools.Databases.EmbeddingState;
|
||||
|
||||
@@ -18,18 +18,12 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
private const string DATABASE_NAME = "Local RAG Index";
|
||||
private const string DATABASE_FILENAME = "rag-index.sqlite3";
|
||||
private const int MAX_FTS_QUERY_TERMS = 32;
|
||||
private const int CHUNK_UPSERT_BATCH_SIZE = 500;
|
||||
|
||||
private static readonly Regex FTS_TOKEN_REGEX = new(@"[\p{L}\p{Nd}_]+", RegexOptions.Compiled | RegexOptions.CultureInvariant);
|
||||
|
||||
private readonly string databasePath = databasePath;
|
||||
private readonly string connectionString = new SqliteConnectionStringBuilder
|
||||
{
|
||||
DataSource = databasePath,
|
||||
Mode = SqliteOpenMode.ReadWriteCreate,
|
||||
Cache = SqliteCacheMode.Shared,
|
||||
ForeignKeys = true,
|
||||
DefaultTimeout = 30,
|
||||
}.ToString();
|
||||
private readonly DbContextOptions<EmbeddingStateDbContext> dbContextOptions = EmbeddingStateDbContext.CreateOptions(databasePath);
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SqliteEmbeddingStateClientImplementation).Namespace, nameof(SqliteEmbeddingStateClientImplementation));
|
||||
|
||||
@@ -68,54 +62,45 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
|
||||
public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo()
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
|
||||
yield return (TB("Reported version"), version);
|
||||
yield return (TB("Database path"), this.databasePath);
|
||||
yield return (TB("Storage size"), this.GetStorageSize());
|
||||
yield return (TB("Indexed data sources"), (await this.CountAsync("data_sources", CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Indexed files"), (await this.CountAsync("embedded_files", CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Search chunks"), (await this.CountAsync("embedding_chunks", CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Indexed data sources"), (await context.DataSources.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Indexed files"), (await context.EmbeddedFiles.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Search chunks"), (await context.EmbeddingChunks.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
public override async Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token)
|
||||
{
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await using var context = this.CreateContext();
|
||||
var manifest = new DataSourceEmbeddingManifest();
|
||||
|
||||
await using (var command = CreateCommand(connection, """
|
||||
SELECT embedding_provider_id, embedding_signature, source_hash, vector_size
|
||||
FROM data_sources
|
||||
WHERE data_source_id = $dataSourceId
|
||||
"""))
|
||||
{
|
||||
command.Parameters.AddWithValue("$dataSourceId", dataSourceId);
|
||||
await using var reader = await command.ExecuteReaderAsync(token);
|
||||
if (await reader.ReadAsync(token))
|
||||
{
|
||||
manifest.EmbeddingProviderId = reader.GetString(0);
|
||||
manifest.EmbeddingSignature = reader.GetString(1);
|
||||
manifest.SourceHash = reader.GetString(2);
|
||||
manifest.VectorSize = reader.GetInt32(3);
|
||||
}
|
||||
}
|
||||
var dataSource = await context.DataSources
|
||||
.AsNoTracking()
|
||||
.FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token);
|
||||
|
||||
await using (var command = CreateCommand(connection, """
|
||||
SELECT absolute_path, fingerprint, file_size, last_write_utc, embedded_at_utc, chunk_count
|
||||
FROM embedded_files
|
||||
WHERE data_source_id = $dataSourceId
|
||||
AND chunk_count > 0
|
||||
"""))
|
||||
if (dataSource is null)
|
||||
return manifest;
|
||||
|
||||
manifest.EmbeddingProviderId = dataSource.EmbeddingProviderId;
|
||||
manifest.EmbeddingSignature = dataSource.EmbeddingSignature;
|
||||
manifest.SourceHash = dataSource.SourceHash;
|
||||
manifest.VectorSize = dataSource.VectorSize;
|
||||
|
||||
var files = await context.EmbeddedFiles
|
||||
.AsNoTracking()
|
||||
.Where(file => file.DataSourceId == dataSourceId && file.ChunkCount > 0)
|
||||
.ToListAsync(token);
|
||||
foreach (var file in files)
|
||||
{
|
||||
command.Parameters.AddWithValue("$dataSourceId", dataSourceId);
|
||||
await using var reader = await command.ExecuteReaderAsync(token);
|
||||
while (await reader.ReadAsync(token))
|
||||
{
|
||||
manifest.Files[reader.GetString(0)] = new EmbeddedFileRecord(
|
||||
reader.GetString(1),
|
||||
reader.GetInt64(2),
|
||||
ParseUtc(reader.GetString(3)),
|
||||
ParseUtc(reader.GetString(4)),
|
||||
reader.GetInt32(5));
|
||||
}
|
||||
manifest.Files[file.AbsolutePath] = new EmbeddedFileRecord(
|
||||
file.Fingerprint,
|
||||
file.FileSize,
|
||||
file.LastWriteUtc,
|
||||
file.EmbeddedAtUtc,
|
||||
file.ChunkCount);
|
||||
}
|
||||
|
||||
return manifest;
|
||||
@@ -131,148 +116,82 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
int vectorSize,
|
||||
CancellationToken token)
|
||||
{
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await ExecuteNonQueryAsync(connection, """
|
||||
INSERT INTO data_sources (
|
||||
data_source_id,
|
||||
data_source_name,
|
||||
data_source_type,
|
||||
embedding_provider_id,
|
||||
embedding_signature,
|
||||
source_hash,
|
||||
vector_size,
|
||||
updated_at_utc)
|
||||
VALUES (
|
||||
$dataSourceId,
|
||||
$dataSourceName,
|
||||
$dataSourceType,
|
||||
$embeddingProviderId,
|
||||
$embeddingSignature,
|
||||
$sourceHash,
|
||||
$vectorSize,
|
||||
$updatedAtUtc)
|
||||
ON CONFLICT(data_source_id) DO UPDATE SET
|
||||
data_source_name = excluded.data_source_name,
|
||||
data_source_type = excluded.data_source_type,
|
||||
embedding_provider_id = excluded.embedding_provider_id,
|
||||
embedding_signature = excluded.embedding_signature,
|
||||
source_hash = excluded.source_hash,
|
||||
vector_size = excluded.vector_size,
|
||||
updated_at_utc = excluded.updated_at_utc
|
||||
""", token,
|
||||
("$dataSourceId", dataSourceId),
|
||||
("$dataSourceName", dataSourceName),
|
||||
("$dataSourceType", dataSourceType),
|
||||
("$embeddingProviderId", embeddingProviderId),
|
||||
("$embeddingSignature", embeddingSignature),
|
||||
("$sourceHash", sourceHash),
|
||||
("$vectorSize", vectorSize),
|
||||
("$updatedAtUtc", ToUtcText(DateTime.UtcNow)));
|
||||
await using var context = this.CreateContext();
|
||||
var dataSource = await context.DataSources.FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token);
|
||||
if (dataSource is null)
|
||||
{
|
||||
dataSource = new EmbeddingStateDataSourceEntity
|
||||
{
|
||||
DataSourceId = dataSourceId,
|
||||
};
|
||||
context.DataSources.Add(dataSource);
|
||||
}
|
||||
|
||||
ApplyDataSource(dataSource, dataSourceName, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize);
|
||||
await context.SaveChangesAsync(token);
|
||||
}
|
||||
|
||||
public override async Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token)
|
||||
{
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await ExecuteNonQueryAsync(connection, """
|
||||
UPDATE data_sources
|
||||
SET vector_size = $vectorSize,
|
||||
updated_at_utc = $updatedAtUtc
|
||||
WHERE data_source_id = $dataSourceId
|
||||
""", token,
|
||||
("$dataSourceId", dataSourceId),
|
||||
("$vectorSize", vectorSize),
|
||||
("$updatedAtUtc", ToUtcText(DateTime.UtcNow)));
|
||||
await using var context = this.CreateContext();
|
||||
var dataSource = await context.DataSources.FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token);
|
||||
if (dataSource is null)
|
||||
return;
|
||||
|
||||
dataSource.VectorSize = vectorSize;
|
||||
dataSource.UpdatedAtUtc = DateTime.UtcNow;
|
||||
await context.SaveChangesAsync(token);
|
||||
}
|
||||
|
||||
public override async Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token)
|
||||
{
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await ExecuteNonQueryAsync(connection, """
|
||||
UPDATE data_sources
|
||||
SET source_hash = $sourceHash,
|
||||
updated_at_utc = $updatedAtUtc
|
||||
WHERE data_source_id = $dataSourceId
|
||||
""", token,
|
||||
("$dataSourceId", dataSourceId),
|
||||
("$sourceHash", sourceHash),
|
||||
("$updatedAtUtc", ToUtcText(DateTime.UtcNow)));
|
||||
await using var context = this.CreateContext();
|
||||
var dataSource = await context.DataSources.FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token);
|
||||
if (dataSource is null)
|
||||
return;
|
||||
|
||||
dataSource.SourceHash = sourceHash;
|
||||
dataSource.UpdatedAtUtc = DateTime.UtcNow;
|
||||
await context.SaveChangesAsync(token);
|
||||
}
|
||||
|
||||
public override async Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token)
|
||||
{
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await ExecuteNonQueryAsync(connection, """
|
||||
INSERT INTO embedded_files (
|
||||
parent_file_id,
|
||||
data_source_id,
|
||||
absolute_path,
|
||||
file_name,
|
||||
relative_path,
|
||||
file_type,
|
||||
fingerprint,
|
||||
file_size,
|
||||
creation_utc,
|
||||
last_write_utc,
|
||||
embedded_at_utc,
|
||||
chunk_count,
|
||||
compliance_level,
|
||||
compliance_level_rank)
|
||||
VALUES (
|
||||
$parentFileId,
|
||||
$dataSourceId,
|
||||
$absolutePath,
|
||||
$fileName,
|
||||
$relativePath,
|
||||
$fileType,
|
||||
$fingerprint,
|
||||
$fileSize,
|
||||
$creationUtc,
|
||||
$lastWriteUtc,
|
||||
$embeddedAtUtc,
|
||||
$chunkCount,
|
||||
$complianceLevel,
|
||||
$complianceLevelRank)
|
||||
ON CONFLICT(parent_file_id) DO UPDATE SET
|
||||
data_source_id = excluded.data_source_id,
|
||||
absolute_path = excluded.absolute_path,
|
||||
file_name = excluded.file_name,
|
||||
relative_path = excluded.relative_path,
|
||||
file_type = excluded.file_type,
|
||||
fingerprint = excluded.fingerprint,
|
||||
file_size = excluded.file_size,
|
||||
creation_utc = excluded.creation_utc,
|
||||
last_write_utc = excluded.last_write_utc,
|
||||
embedded_at_utc = excluded.embedded_at_utc,
|
||||
chunk_count = excluded.chunk_count,
|
||||
compliance_level = excluded.compliance_level,
|
||||
compliance_level_rank = excluded.compliance_level_rank
|
||||
""", token,
|
||||
("$parentFileId", file.ParentFileId),
|
||||
("$dataSourceId", dataSourceId),
|
||||
("$absolutePath", file.AbsolutePath),
|
||||
("$fileName", file.FileName),
|
||||
("$relativePath", file.RelativePath),
|
||||
("$fileType", file.FileType),
|
||||
("$fingerprint", file.Fingerprint),
|
||||
("$fileSize", file.FileSize),
|
||||
("$creationUtc", ToUtcText(file.CreationUtc)),
|
||||
("$lastWriteUtc", ToUtcText(file.LastWriteUtc)),
|
||||
("$embeddedAtUtc", ToUtcText(file.EmbeddedAtUtc)),
|
||||
("$chunkCount", file.ChunkCount),
|
||||
("$complianceLevel", file.ComplianceLevel),
|
||||
("$complianceLevelRank", file.ComplianceLevelRank));
|
||||
await using var context = this.CreateContext();
|
||||
var fileEntity = await context.EmbeddedFiles.FirstOrDefaultAsync(entity => entity.ParentFileId == file.ParentFileId, token);
|
||||
if (fileEntity is null)
|
||||
{
|
||||
fileEntity = new EmbeddingStateFileEntity
|
||||
{
|
||||
ParentFileId = file.ParentFileId,
|
||||
};
|
||||
context.EmbeddedFiles.Add(fileEntity);
|
||||
}
|
||||
|
||||
ApplyFile(fileEntity, dataSourceId, file);
|
||||
await context.SaveChangesAsync(token);
|
||||
}
|
||||
|
||||
public override async Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token)
|
||||
{
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await ExecuteNonQueryAsync(connection, """
|
||||
DELETE FROM embedded_files
|
||||
WHERE data_source_id = $dataSourceId
|
||||
AND absolute_path = $filePath
|
||||
""", token,
|
||||
("$dataSourceId", dataSourceId),
|
||||
("$filePath", filePath));
|
||||
await using var context = this.CreateContext();
|
||||
await using var transaction = await context.Database.BeginTransactionAsync(token);
|
||||
|
||||
var parentFileIds = await context.EmbeddedFiles
|
||||
.Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath)
|
||||
.Select(file => file.ParentFileId)
|
||||
.ToListAsync(token);
|
||||
|
||||
foreach (var parentFileIdBatch in parentFileIds.Chunk(CHUNK_UPSERT_BATCH_SIZE))
|
||||
await context.EmbeddingChunks
|
||||
.Where(chunk => parentFileIdBatch.Contains(chunk.ParentFileId))
|
||||
.ExecuteDeleteAsync(token);
|
||||
|
||||
await context.EmbeddedFiles
|
||||
.Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath)
|
||||
.ExecuteDeleteAsync(token);
|
||||
|
||||
await transaction.CommitAsync(token);
|
||||
}
|
||||
|
||||
public override async Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token)
|
||||
@@ -280,41 +199,38 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
if (chunks.Count == 0)
|
||||
return;
|
||||
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(token);
|
||||
await using var context = this.CreateContext();
|
||||
await using var transaction = await context.Database.BeginTransactionAsync(token);
|
||||
|
||||
foreach (var chunk in chunks)
|
||||
foreach (var chunkBatch in chunks.Chunk(CHUNK_UPSERT_BATCH_SIZE))
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
await ExecuteNonQueryAsync(connection, transaction, """
|
||||
INSERT INTO embedding_chunks (
|
||||
chunk_id,
|
||||
parent_file_id,
|
||||
page_number,
|
||||
chunk_index,
|
||||
chunk_text,
|
||||
embedded_at_utc)
|
||||
VALUES (
|
||||
$chunkId,
|
||||
$parentFileId,
|
||||
$pageNumber,
|
||||
$chunkIndex,
|
||||
$chunkText,
|
||||
$embeddedAtUtc)
|
||||
ON CONFLICT(chunk_id) DO UPDATE SET
|
||||
parent_file_id = excluded.parent_file_id,
|
||||
page_number = excluded.page_number,
|
||||
chunk_index = excluded.chunk_index,
|
||||
chunk_text = excluded.chunk_text,
|
||||
embedded_at_utc = excluded.embedded_at_utc
|
||||
""", token,
|
||||
("$chunkId", chunk.ChunkId),
|
||||
("$parentFileId", chunk.ParentFileId),
|
||||
("$pageNumber", chunk.PageNumber),
|
||||
("$chunkIndex", chunk.ChunkIndex),
|
||||
("$chunkText", chunk.ChunkText),
|
||||
("$embeddedAtUtc", ToUtcText(chunk.EmbeddedAtUtc)));
|
||||
var chunkIds = chunkBatch
|
||||
.Select(chunk => chunk.ChunkId)
|
||||
.Distinct(StringComparer.Ordinal)
|
||||
.ToArray();
|
||||
var existingChunks = await context.EmbeddingChunks
|
||||
.Where(chunk => chunkIds.Contains(chunk.ChunkId))
|
||||
.ToDictionaryAsync(chunk => chunk.ChunkId, StringComparer.Ordinal, token);
|
||||
|
||||
foreach (var chunk in chunkBatch)
|
||||
{
|
||||
if (!existingChunks.TryGetValue(chunk.ChunkId, out var chunkEntity))
|
||||
{
|
||||
chunkEntity = new EmbeddingStateChunkEntity
|
||||
{
|
||||
ChunkId = chunk.ChunkId,
|
||||
};
|
||||
context.EmbeddingChunks.Add(chunkEntity);
|
||||
existingChunks[chunk.ChunkId] = chunkEntity;
|
||||
}
|
||||
|
||||
ApplyChunk(chunkEntity, chunk);
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync(token);
|
||||
context.ChangeTracker.Clear();
|
||||
}
|
||||
|
||||
await transaction.CommitAsync(token);
|
||||
@@ -329,83 +245,66 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
if (string.IsNullOrWhiteSpace(ftsQuery))
|
||||
return [];
|
||||
|
||||
var results = new List<EmbeddingStateSearchResult>(maxMatches);
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await using var command = CreateCommand(connection, """
|
||||
SELECT
|
||||
c.chunk_id,
|
||||
c.parent_file_id,
|
||||
ds.data_source_id,
|
||||
ds.data_source_name,
|
||||
ds.data_source_type,
|
||||
f.absolute_path,
|
||||
f.file_name,
|
||||
f.relative_path,
|
||||
f.file_type,
|
||||
c.page_number,
|
||||
c.chunk_index,
|
||||
c.chunk_text,
|
||||
bm25(embedding_chunks_fts) AS score,
|
||||
f.fingerprint,
|
||||
f.file_size,
|
||||
f.creation_utc,
|
||||
f.last_write_utc,
|
||||
c.embedded_at_utc,
|
||||
f.chunk_count,
|
||||
f.compliance_level,
|
||||
f.compliance_level_rank
|
||||
FROM embedding_chunks_fts
|
||||
JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid
|
||||
JOIN embedded_files f ON f.parent_file_id = c.parent_file_id
|
||||
JOIN data_sources ds ON ds.data_source_id = f.data_source_id
|
||||
WHERE ds.data_source_id = $dataSourceId
|
||||
AND embedding_chunks_fts MATCH $query
|
||||
ORDER BY score
|
||||
LIMIT $maxMatches
|
||||
""");
|
||||
await using var context = this.CreateContext();
|
||||
var results = await context.SearchResults
|
||||
.FromSqlInterpolated($"""
|
||||
SELECT
|
||||
c.chunk_id AS ChunkId,
|
||||
c.parent_file_id AS ParentFileId,
|
||||
ds.data_source_id AS DataSourceId,
|
||||
ds.data_source_name AS DataSourceName,
|
||||
ds.data_source_type AS DataSourceType,
|
||||
f.absolute_path AS AbsolutePath,
|
||||
f.file_name AS FileName,
|
||||
f.relative_path AS RelativePath,
|
||||
f.file_type AS FileType,
|
||||
c.page_number AS PageNumber,
|
||||
c.chunk_index AS ChunkIndex,
|
||||
c.chunk_text AS ChunkText,
|
||||
bm25(embedding_chunks_fts) AS Score,
|
||||
f.fingerprint AS Fingerprint,
|
||||
f.file_size AS FileSize,
|
||||
f.creation_utc AS CreationUtc,
|
||||
f.last_write_utc AS LastWriteUtc,
|
||||
c.embedded_at_utc AS EmbeddedAtUtc,
|
||||
f.chunk_count AS ChunkCount,
|
||||
f.compliance_level AS ComplianceLevel,
|
||||
f.compliance_level_rank AS ComplianceLevelRank
|
||||
FROM embedding_chunks_fts
|
||||
JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid
|
||||
JOIN embedded_files f ON f.parent_file_id = c.parent_file_id
|
||||
JOIN data_sources ds ON ds.data_source_id = f.data_source_id
|
||||
WHERE ds.data_source_id = {dataSourceId}
|
||||
AND embedding_chunks_fts MATCH {ftsQuery}
|
||||
ORDER BY Score
|
||||
LIMIT {maxMatches}
|
||||
""")
|
||||
.AsNoTracking()
|
||||
.ToListAsync(token);
|
||||
|
||||
command.Parameters.AddWithValue("$dataSourceId", dataSourceId);
|
||||
command.Parameters.AddWithValue("$query", ftsQuery);
|
||||
command.Parameters.AddWithValue("$maxMatches", maxMatches);
|
||||
|
||||
await using var reader = await command.ExecuteReaderAsync(token);
|
||||
while (await reader.ReadAsync(token))
|
||||
{
|
||||
results.Add(new EmbeddingStateSearchResult(
|
||||
reader.GetString(0),
|
||||
reader.GetString(1),
|
||||
reader.GetString(2),
|
||||
reader.GetString(3),
|
||||
reader.GetString(4),
|
||||
reader.GetString(5),
|
||||
reader.GetString(6),
|
||||
reader.GetString(7),
|
||||
reader.GetString(8),
|
||||
reader.IsDBNull(9) ? null : reader.GetInt32(9),
|
||||
reader.GetInt32(10),
|
||||
reader.GetString(11),
|
||||
reader.GetDouble(12),
|
||||
reader.GetString(13),
|
||||
reader.GetInt64(14),
|
||||
ParseUtc(reader.GetString(15)),
|
||||
ParseUtc(reader.GetString(16)),
|
||||
ParseUtc(reader.GetString(17)),
|
||||
reader.GetInt32(18),
|
||||
reader.GetString(19),
|
||||
reader.GetInt32(20)));
|
||||
}
|
||||
|
||||
return results;
|
||||
return results.Select(ToSearchResult).ToList();
|
||||
}
|
||||
|
||||
public override async Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token)
|
||||
{
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await ExecuteNonQueryAsync(connection, """
|
||||
DELETE FROM data_sources
|
||||
WHERE data_source_id = $dataSourceId
|
||||
""", token,
|
||||
("$dataSourceId", dataSourceId));
|
||||
await using var context = this.CreateContext();
|
||||
await using var transaction = await context.Database.BeginTransactionAsync(token);
|
||||
|
||||
var parentFileIds = await context.EmbeddedFiles
|
||||
.Where(file => file.DataSourceId == dataSourceId)
|
||||
.Select(file => file.ParentFileId)
|
||||
.ToListAsync(token);
|
||||
|
||||
foreach (var parentFileIdBatch in parentFileIds.Chunk(CHUNK_UPSERT_BATCH_SIZE))
|
||||
await context.EmbeddingChunks
|
||||
.Where(chunk => parentFileIdBatch.Contains(chunk.ParentFileId))
|
||||
.ExecuteDeleteAsync(token);
|
||||
|
||||
await context.DataSources
|
||||
.Where(source => source.DataSourceId == dataSourceId)
|
||||
.ExecuteDeleteAsync(token);
|
||||
|
||||
await transaction.CommitAsync(token);
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
@@ -414,198 +313,88 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
||||
|
||||
private async Task InitializeAsync(CancellationToken token)
|
||||
{
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await ExecuteNonQueryAsync(connection, """
|
||||
PRAGMA journal_mode=WAL;
|
||||
PRAGMA foreign_keys=ON;
|
||||
|
||||
CREATE TABLE IF NOT EXISTS data_sources (
|
||||
data_source_id TEXT PRIMARY KEY,
|
||||
data_source_name TEXT NOT NULL,
|
||||
data_source_type TEXT NOT NULL,
|
||||
embedding_provider_id TEXT NOT NULL,
|
||||
embedding_signature TEXT NOT NULL,
|
||||
source_hash TEXT NOT NULL DEFAULT '',
|
||||
vector_size INTEGER NOT NULL DEFAULT 0,
|
||||
updated_at_utc TEXT NOT NULL
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS embedded_files (
|
||||
parent_file_id TEXT PRIMARY KEY,
|
||||
data_source_id TEXT NOT NULL,
|
||||
absolute_path TEXT COLLATE NOCASE NOT NULL,
|
||||
file_name TEXT NOT NULL,
|
||||
relative_path TEXT NOT NULL,
|
||||
file_type TEXT NOT NULL,
|
||||
fingerprint TEXT NOT NULL,
|
||||
file_size INTEGER NOT NULL,
|
||||
creation_utc TEXT NOT NULL,
|
||||
last_write_utc TEXT NOT NULL,
|
||||
embedded_at_utc TEXT NOT NULL,
|
||||
chunk_count INTEGER NOT NULL,
|
||||
compliance_level TEXT NOT NULL,
|
||||
compliance_level_rank INTEGER NOT NULL,
|
||||
FOREIGN KEY (data_source_id)
|
||||
REFERENCES data_sources(data_source_id)
|
||||
ON DELETE CASCADE,
|
||||
UNIQUE(data_source_id, absolute_path)
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS embedding_chunks (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
chunk_id TEXT NOT NULL UNIQUE,
|
||||
parent_file_id TEXT NOT NULL,
|
||||
page_number INTEGER NULL,
|
||||
chunk_index INTEGER NOT NULL,
|
||||
chunk_text TEXT NOT NULL,
|
||||
embedded_at_utc TEXT NOT NULL,
|
||||
FOREIGN KEY (parent_file_id)
|
||||
REFERENCES embedded_files(parent_file_id)
|
||||
ON DELETE CASCADE,
|
||||
UNIQUE(parent_file_id, chunk_index)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_embedded_files_data_source
|
||||
ON embedded_files(data_source_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_embedded_files_absolute_path
|
||||
ON embedded_files(absolute_path);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_embedded_files_file_type
|
||||
ON embedded_files(file_type);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_embedded_files_compliance
|
||||
ON embedded_files(compliance_level_rank);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_embedding_chunks_parent_file
|
||||
ON embedding_chunks(parent_file_id);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_embedding_chunks_page
|
||||
ON embedding_chunks(page_number);
|
||||
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS embedding_chunks_fts
|
||||
USING fts5(chunk_id UNINDEXED, file_name, chunk_text);
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedding_chunks_ai
|
||||
AFTER INSERT ON embedding_chunks
|
||||
BEGIN
|
||||
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
|
||||
VALUES (
|
||||
new.id,
|
||||
new.chunk_id,
|
||||
(SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id),
|
||||
new.chunk_text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedding_chunks_ad
|
||||
AFTER DELETE ON embedding_chunks
|
||||
BEGIN
|
||||
DELETE FROM embedding_chunks_fts
|
||||
WHERE rowid = old.id;
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedding_chunks_au
|
||||
AFTER UPDATE ON embedding_chunks
|
||||
BEGIN
|
||||
DELETE FROM embedding_chunks_fts
|
||||
WHERE rowid = old.id;
|
||||
|
||||
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
|
||||
VALUES (
|
||||
new.id,
|
||||
new.chunk_id,
|
||||
(SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id),
|
||||
new.chunk_text);
|
||||
END;
|
||||
|
||||
CREATE TRIGGER IF NOT EXISTS embedded_files_file_name_au
|
||||
AFTER UPDATE OF file_name ON embedded_files
|
||||
BEGIN
|
||||
DELETE FROM embedding_chunks_fts
|
||||
WHERE rowid IN (
|
||||
SELECT id
|
||||
FROM embedding_chunks
|
||||
WHERE parent_file_id = new.parent_file_id
|
||||
);
|
||||
|
||||
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
|
||||
SELECT id, chunk_id, new.file_name, chunk_text
|
||||
FROM embedding_chunks
|
||||
WHERE parent_file_id = new.parent_file_id;
|
||||
END;
|
||||
""", token);
|
||||
await using var context = this.CreateContext();
|
||||
await EmbeddingStateSchemaMigrator.MigrateAsync(context, token);
|
||||
}
|
||||
|
||||
private async Task<string> GetSqliteVersionAsync(CancellationToken token)
|
||||
{
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await using var command = CreateCommand(connection, "SELECT sqlite_version()");
|
||||
var versionObject = await command.ExecuteScalarAsync(token);
|
||||
return Convert.ToString(versionObject, CultureInfo.InvariantCulture) ?? string.Empty;
|
||||
await using var context = this.CreateContext();
|
||||
var versions = await context.Database
|
||||
.SqlQueryRaw<string>("SELECT sqlite_version() AS Value")
|
||||
.ToListAsync(token);
|
||||
return versions.FirstOrDefault() ?? string.Empty;
|
||||
}
|
||||
|
||||
private async Task<long> CountAsync(string tableName, CancellationToken token)
|
||||
private EmbeddingStateDbContext CreateContext() => new(this.dbContextOptions);
|
||||
|
||||
private static void ApplyDataSource(
|
||||
EmbeddingStateDataSourceEntity dataSource,
|
||||
string dataSourceName,
|
||||
string dataSourceType,
|
||||
string embeddingProviderId,
|
||||
string embeddingSignature,
|
||||
string sourceHash,
|
||||
int vectorSize)
|
||||
{
|
||||
await using var connection = await this.OpenConnectionAsync(token);
|
||||
await using var command = CreateCommand(connection, $"SELECT COUNT(*) FROM {tableName}");
|
||||
var countObject = await command.ExecuteScalarAsync(token);
|
||||
return Convert.ToInt64(countObject, CultureInfo.InvariantCulture);
|
||||
dataSource.DataSourceName = dataSourceName;
|
||||
dataSource.DataSourceType = dataSourceType;
|
||||
dataSource.EmbeddingProviderId = embeddingProviderId;
|
||||
dataSource.EmbeddingSignature = embeddingSignature;
|
||||
dataSource.SourceHash = sourceHash;
|
||||
dataSource.VectorSize = vectorSize;
|
||||
dataSource.UpdatedAtUtc = DateTime.UtcNow;
|
||||
}
|
||||
|
||||
private async Task<SqliteConnection> OpenConnectionAsync(CancellationToken token)
|
||||
private static void ApplyFile(EmbeddingStateFileEntity fileEntity, string dataSourceId, EmbeddingStateFile file)
|
||||
{
|
||||
var connection = new SqliteConnection(this.connectionString);
|
||||
await connection.OpenAsync(token);
|
||||
return connection;
|
||||
fileEntity.DataSourceId = dataSourceId;
|
||||
fileEntity.AbsolutePath = file.AbsolutePath;
|
||||
fileEntity.FileName = file.FileName;
|
||||
fileEntity.RelativePath = file.RelativePath;
|
||||
fileEntity.FileType = file.FileType;
|
||||
fileEntity.Fingerprint = file.Fingerprint;
|
||||
fileEntity.FileSize = file.FileSize;
|
||||
fileEntity.CreationUtc = file.CreationUtc;
|
||||
fileEntity.LastWriteUtc = file.LastWriteUtc;
|
||||
fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc;
|
||||
fileEntity.ChunkCount = file.ChunkCount;
|
||||
fileEntity.ComplianceLevel = file.ComplianceLevel;
|
||||
fileEntity.ComplianceLevelRank = file.ComplianceLevelRank;
|
||||
}
|
||||
|
||||
private static SqliteCommand CreateCommand(SqliteConnection connection, string commandText)
|
||||
private static void ApplyChunk(EmbeddingStateChunkEntity chunkEntity, EmbeddingStateChunk chunk)
|
||||
{
|
||||
var command = connection.CreateCommand();
|
||||
command.CommandText = commandText;
|
||||
return command;
|
||||
chunkEntity.ChunkId = chunk.ChunkId;
|
||||
chunkEntity.ParentFileId = chunk.ParentFileId;
|
||||
chunkEntity.PageNumber = chunk.PageNumber;
|
||||
chunkEntity.ChunkIndex = chunk.ChunkIndex;
|
||||
chunkEntity.ChunkText = chunk.ChunkText;
|
||||
chunkEntity.EmbeddedAtUtc = chunk.EmbeddedAtUtc;
|
||||
}
|
||||
|
||||
private static async Task ExecuteNonQueryAsync(
|
||||
SqliteConnection connection,
|
||||
string commandText,
|
||||
CancellationToken token,
|
||||
params (string Name, object? Value)[] parameters)
|
||||
{
|
||||
await using var command = CreateCommand(connection, commandText);
|
||||
foreach (var (name, value) in parameters)
|
||||
command.Parameters.AddWithValue(name, value ?? DBNull.Value);
|
||||
|
||||
await command.ExecuteNonQueryAsync(token);
|
||||
}
|
||||
|
||||
private static async Task ExecuteNonQueryAsync(
|
||||
SqliteConnection connection,
|
||||
SqliteTransaction transaction,
|
||||
string commandText,
|
||||
CancellationToken token,
|
||||
params (string Name, object? Value)[] parameters)
|
||||
{
|
||||
await using var command = CreateCommand(connection, commandText);
|
||||
command.Transaction = transaction;
|
||||
foreach (var (name, value) in parameters)
|
||||
command.Parameters.AddWithValue(name, value ?? DBNull.Value);
|
||||
|
||||
await command.ExecuteNonQueryAsync(token);
|
||||
}
|
||||
|
||||
private static string ToUtcText(DateTime dateTime)
|
||||
{
|
||||
var utc = dateTime.Kind is DateTimeKind.Utc ? dateTime : dateTime.ToUniversalTime();
|
||||
return utc.ToString("O", CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
private static DateTime ParseUtc(string value)
|
||||
{
|
||||
return DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var dateTime)
|
||||
? dateTime.ToUniversalTime()
|
||||
: DateTime.UnixEpoch;
|
||||
}
|
||||
private static EmbeddingStateSearchResult ToSearchResult(EmbeddingStateSearchResultEntity result) => new(
|
||||
result.ChunkId,
|
||||
result.ParentFileId,
|
||||
result.DataSourceId,
|
||||
result.DataSourceName,
|
||||
result.DataSourceType,
|
||||
result.AbsolutePath,
|
||||
result.FileName,
|
||||
result.RelativePath,
|
||||
result.FileType,
|
||||
result.PageNumber,
|
||||
result.ChunkIndex,
|
||||
result.ChunkText,
|
||||
result.Score,
|
||||
result.Fingerprint,
|
||||
result.FileSize,
|
||||
result.CreationUtc,
|
||||
result.LastWriteUtc,
|
||||
result.EmbeddedAtUtc,
|
||||
result.ChunkCount,
|
||||
result.ComplianceLevel,
|
||||
result.ComplianceLevelRank);
|
||||
|
||||
private static string BuildFtsQuery(string query)
|
||||
{
|
||||
|
||||
Reference in new issue
Block a user