diff --git a/app/MindWork AI Studio/MindWork AI Studio.csproj b/app/MindWork AI Studio/MindWork AI Studio.csproj index f302f321..182ba397 100644 --- a/app/MindWork AI Studio/MindWork AI Studio.csproj +++ b/app/MindWork AI Studio/MindWork AI Studio.csproj @@ -50,12 +50,13 @@ - - + + + - + diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDbContext.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDbContext.cs new file mode 100644 index 00000000..9088f9cf --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDbContext.cs @@ -0,0 +1,259 @@ +using System.Globalization; + +using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +namespace AIStudio.Tools.Databases.EmbeddingState; + +internal sealed class EmbeddingStateDbContext(DbContextOptions options) : DbContext(options) +{ + public static DbContextOptions CreateOptions(string databasePath) => new DbContextOptionsBuilder() + .UseSqlite(BuildConnectionString(databasePath)) + .Options; + + public DbSet DataSources => this.Set(); + + public DbSet EmbeddedFiles => this.Set(); + + public DbSet EmbeddingChunks => this.Set(); + + public DbSet SearchResults => this.Set(); + + protected override void OnModelCreating(ModelBuilder modelBuilder) + { + var utcDateTimeConverter = new EmbeddingStateDateTimeConverter(); + + modelBuilder.Entity(entity => + { + entity.ToTable("data_sources"); + entity.HasKey(dataSource => dataSource.DataSourceId); + + entity.Property(dataSource => dataSource.DataSourceId).HasColumnName("data_source_id"); + entity.Property(dataSource => dataSource.DataSourceName).HasColumnName("data_source_name").IsRequired(); + entity.Property(dataSource => dataSource.DataSourceType).HasColumnName("data_source_type").IsRequired(); + entity.Property(dataSource => dataSource.EmbeddingProviderId).HasColumnName("embedding_provider_id").IsRequired(); + entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired(); + entity.Property(dataSource => dataSource.SourceHash).HasColumnName("source_hash").IsRequired().HasDefaultValue(string.Empty); + entity.Property(dataSource => dataSource.VectorSize).HasColumnName("vector_size").HasDefaultValue(0); + entity.Property(dataSource => dataSource.UpdatedAtUtc).HasColumnName("updated_at_utc").HasConversion(utcDateTimeConverter).IsRequired(); + + entity + .HasMany(dataSource => dataSource.Files) + .WithOne(file => file.DataSource) + .HasForeignKey(file => file.DataSourceId) + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("embedded_files"); + entity.HasKey(file => file.ParentFileId); + + entity.Property(file => file.ParentFileId).HasColumnName("parent_file_id"); + entity.Property(file => file.DataSourceId).HasColumnName("data_source_id").IsRequired(); + entity.Property(file => file.AbsolutePath).HasColumnName("absolute_path").UseCollation("NOCASE").IsRequired(); + entity.Property(file => file.FileName).HasColumnName("file_name").IsRequired(); + entity.Property(file => file.RelativePath).HasColumnName("relative_path").IsRequired(); + entity.Property(file => file.FileType).HasColumnName("file_type").IsRequired(); + entity.Property(file => file.Fingerprint).HasColumnName("fingerprint").IsRequired(); + entity.Property(file => file.FileSize).HasColumnName("file_size"); + entity.Property(file => file.CreationUtc).HasColumnName("creation_utc").HasConversion(utcDateTimeConverter).IsRequired(); + entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeConverter).IsRequired(); + entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeConverter).IsRequired(); + entity.Property(file => file.ChunkCount).HasColumnName("chunk_count"); + entity.Property(file => file.ComplianceLevel).HasColumnName("compliance_level").IsRequired(); + entity.Property(file => file.ComplianceLevelRank).HasColumnName("compliance_level_rank"); + + entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source"); + entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path"); + entity.HasIndex(file => file.FileType).HasDatabaseName("idx_embedded_files_file_type"); + entity.HasIndex(file => file.ComplianceLevelRank).HasDatabaseName("idx_embedded_files_compliance"); + entity.HasIndex(file => new { file.DataSourceId, file.AbsolutePath }).HasDatabaseName("idx_embedded_files_data_source_absolute_path").IsUnique(); + + entity + .HasMany(file => file.Chunks) + .WithOne(chunk => chunk.File) + .HasForeignKey(chunk => chunk.ParentFileId) + .OnDelete(DeleteBehavior.Cascade); + }); + + modelBuilder.Entity(entity => + { + entity.ToTable("embedding_chunks"); + entity.HasKey(chunk => chunk.Id); + + entity.Property(chunk => chunk.Id).HasColumnName("id").ValueGeneratedOnAdd(); + entity.Property(chunk => chunk.ChunkId).HasColumnName("chunk_id").IsRequired(); + entity.Property(chunk => chunk.ParentFileId).HasColumnName("parent_file_id").IsRequired(); + entity.Property(chunk => chunk.PageNumber).HasColumnName("page_number"); + entity.Property(chunk => chunk.ChunkIndex).HasColumnName("chunk_index"); + entity.Property(chunk => chunk.ChunkText).HasColumnName("chunk_text").IsRequired(); + entity.Property(chunk => chunk.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeConverter).IsRequired(); + + entity.HasIndex(chunk => chunk.ChunkId).HasDatabaseName("idx_embedding_chunks_chunk_id").IsUnique(); + entity.HasIndex(chunk => chunk.ParentFileId).HasDatabaseName("idx_embedding_chunks_parent_file"); + entity.HasIndex(chunk => chunk.PageNumber).HasDatabaseName("idx_embedding_chunks_page"); + entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index").IsUnique(); + }); + + modelBuilder.Entity(entity => + { + entity.HasNoKey(); + entity.ToView("embedding_chunk_search_results"); + + entity.Property(result => result.CreationUtc).HasConversion(utcDateTimeConverter); + entity.Property(result => result.LastWriteUtc).HasConversion(utcDateTimeConverter); + entity.Property(result => result.EmbeddedAtUtc).HasConversion(utcDateTimeConverter); + }); + } + + private static string BuildConnectionString(string databasePath) => new SqliteConnectionStringBuilder + { + DataSource = databasePath, + Mode = SqliteOpenMode.ReadWriteCreate, + Cache = SqliteCacheMode.Shared, + ForeignKeys = true, + DefaultTimeout = 30, + }.ToString(); +} + +internal sealed class EmbeddingStateDataSourceEntity +{ + public string DataSourceId { get; set; } = string.Empty; + + public string DataSourceName { get; set; } = string.Empty; + + public string DataSourceType { get; set; } = string.Empty; + + public string EmbeddingProviderId { get; set; } = string.Empty; + + public string EmbeddingSignature { get; set; } = string.Empty; + + public string SourceHash { get; set; } = string.Empty; + + public int VectorSize { get; set; } + + public DateTime UpdatedAtUtc { get; set; } + + public List Files { get; set; } = []; +} + +internal sealed class EmbeddingStateFileEntity +{ + public string ParentFileId { get; set; } = string.Empty; + + public string DataSourceId { get; set; } = string.Empty; + + public string AbsolutePath { get; set; } = string.Empty; + + public string FileName { get; set; } = string.Empty; + + public string RelativePath { get; set; } = string.Empty; + + public string FileType { get; set; } = string.Empty; + + public string Fingerprint { get; set; } = string.Empty; + + public long FileSize { get; set; } + + public DateTime CreationUtc { get; set; } + + public DateTime LastWriteUtc { get; set; } + + public DateTime EmbeddedAtUtc { get; set; } + + public int ChunkCount { get; set; } + + public string ComplianceLevel { get; set; } = string.Empty; + + public int ComplianceLevelRank { get; set; } + + public EmbeddingStateDataSourceEntity? DataSource { get; set; } + + public List Chunks { get; set; } = []; +} + +internal sealed class EmbeddingStateChunkEntity +{ + public int Id { get; set; } + + public string ChunkId { get; set; } = string.Empty; + + public string ParentFileId { get; set; } = string.Empty; + + public int? PageNumber { get; set; } + + public int ChunkIndex { get; set; } + + public string ChunkText { get; set; } = string.Empty; + + public DateTime EmbeddedAtUtc { get; set; } + + public EmbeddingStateFileEntity? File { get; set; } +} + +internal sealed class EmbeddingStateSearchResultEntity +{ + public string ChunkId { get; set; } = string.Empty; + + public string ParentFileId { get; set; } = string.Empty; + + public string DataSourceId { get; set; } = string.Empty; + + public string DataSourceName { get; set; } = string.Empty; + + public string DataSourceType { get; set; } = string.Empty; + + public string AbsolutePath { get; set; } = string.Empty; + + public string FileName { get; set; } = string.Empty; + + public string RelativePath { get; set; } = string.Empty; + + public string FileType { get; set; } = string.Empty; + + public int? PageNumber { get; set; } + + public int ChunkIndex { get; set; } + + public string ChunkText { get; set; } = string.Empty; + + public double Score { get; set; } + + public string Fingerprint { get; set; } = string.Empty; + + public long FileSize { get; set; } + + public DateTime CreationUtc { get; set; } + + public DateTime LastWriteUtc { get; set; } + + public DateTime EmbeddedAtUtc { get; set; } + + public int ChunkCount { get; set; } + + public string ComplianceLevel { get; set; } = string.Empty; + + public int ComplianceLevelRank { get; set; } +} + +internal sealed class EmbeddingStateDateTimeConverter() : ValueConverter( + value => EmbeddingStateDateTime.ToUtcText(value), + value => EmbeddingStateDateTime.ParseUtc(value)); + +internal static class EmbeddingStateDateTime +{ + public static string ToUtcText(DateTime dateTime) + { + var utc = dateTime.Kind is DateTimeKind.Utc ? dateTime : dateTime.ToUniversalTime(); + return utc.ToString("O", CultureInfo.InvariantCulture); + } + + public static DateTime ParseUtc(string value) + { + return DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var dateTime) + ? dateTime.ToUniversalTime() + : DateTime.UnixEpoch; + } +} diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDesignTimeDbContextFactory.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDesignTimeDbContextFactory.cs new file mode 100644 index 00000000..b055f75a --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateDesignTimeDbContextFactory.cs @@ -0,0 +1,15 @@ +using Microsoft.EntityFrameworkCore.Design; + +namespace AIStudio.Tools.Databases.EmbeddingState; + +internal sealed class EmbeddingStateDesignTimeDbContextFactory : IDesignTimeDbContextFactory +{ + public EmbeddingStateDbContext CreateDbContext(string[] args) + { + var databasePath = args.FirstOrDefault(argument => argument.EndsWith(".sqlite3", StringComparison.OrdinalIgnoreCase)); + if (string.IsNullOrWhiteSpace(databasePath)) + databasePath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-rag-index-design.sqlite3"); + + return new EmbeddingStateDbContext(EmbeddingStateDbContext.CreateOptions(databasePath)); + } +} diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateSchemaMigrator.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateSchemaMigrator.cs new file mode 100644 index 00000000..c590c280 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateSchemaMigrator.cs @@ -0,0 +1,15 @@ +using System.Diagnostics.CodeAnalysis; + +using Microsoft.EntityFrameworkCore; + +namespace AIStudio.Tools.Databases.EmbeddingState; + +internal static class EmbeddingStateSchemaMigrator +{ + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))] + public static async Task MigrateAsync(EmbeddingStateDbContext context, CancellationToken token) + { + await context.Database.MigrateAsync(token); + await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL;", token); + } +} diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/20260804000000_InitialRagIndex.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/20260804000000_InitialRagIndex.cs new file mode 100644 index 00000000..990081e6 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/20260804000000_InitialRagIndex.cs @@ -0,0 +1,202 @@ +#nullable disable + +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace AIStudio.Tools.Databases.EmbeddingState.Migrations; + +[DbContext(typeof(EmbeddingStateDbContext))] +[Migration("20260804000000_InitialRagIndex")] +public partial class InitialRagIndex : Migration +{ + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.CreateTable( + name: "data_sources", + columns: table => new + { + data_source_id = table.Column(type: "TEXT", nullable: false), + data_source_name = table.Column(type: "TEXT", nullable: false), + data_source_type = table.Column(type: "TEXT", nullable: false), + embedding_provider_id = table.Column(type: "TEXT", nullable: false), + embedding_signature = table.Column(type: "TEXT", nullable: false), + source_hash = table.Column(type: "TEXT", nullable: false, defaultValue: string.Empty), + vector_size = table.Column(type: "INTEGER", nullable: false, defaultValue: 0), + updated_at_utc = table.Column(type: "TEXT", nullable: false), + }, + constraints: table => + { + table.PrimaryKey("PK_data_sources", source => source.data_source_id); + }); + + migrationBuilder.CreateTable( + name: "embedded_files", + columns: table => new + { + parent_file_id = table.Column(type: "TEXT", nullable: false), + data_source_id = table.Column(type: "TEXT", nullable: false), + absolute_path = table.Column(type: "TEXT", nullable: false, collation: "NOCASE"), + file_name = table.Column(type: "TEXT", nullable: false), + relative_path = table.Column(type: "TEXT", nullable: false), + file_type = table.Column(type: "TEXT", nullable: false), + fingerprint = table.Column(type: "TEXT", nullable: false), + file_size = table.Column(type: "INTEGER", nullable: false), + creation_utc = table.Column(type: "TEXT", nullable: false), + last_write_utc = table.Column(type: "TEXT", nullable: false), + embedded_at_utc = table.Column(type: "TEXT", nullable: false), + chunk_count = table.Column(type: "INTEGER", nullable: false), + compliance_level = table.Column(type: "TEXT", nullable: false), + compliance_level_rank = table.Column(type: "INTEGER", nullable: false), + }, + constraints: table => + { + table.PrimaryKey("PK_embedded_files", file => file.parent_file_id); + table.ForeignKey( + name: "FK_embedded_files_data_sources_data_source_id", + column: file => file.data_source_id, + principalTable: "data_sources", + principalColumn: "data_source_id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateTable( + name: "embedding_chunks", + columns: table => new + { + id = table.Column(type: "INTEGER", nullable: false) + .Annotation("Sqlite:Autoincrement", true), + chunk_id = table.Column(type: "TEXT", nullable: false), + parent_file_id = table.Column(type: "TEXT", nullable: false), + page_number = table.Column(type: "INTEGER", nullable: true), + chunk_index = table.Column(type: "INTEGER", nullable: false), + chunk_text = table.Column(type: "TEXT", nullable: false), + embedded_at_utc = table.Column(type: "TEXT", nullable: false), + }, + constraints: table => + { + table.PrimaryKey("PK_embedding_chunks", chunk => chunk.id); + table.ForeignKey( + name: "FK_embedding_chunks_embedded_files_parent_file_id", + column: chunk => chunk.parent_file_id, + principalTable: "embedded_files", + principalColumn: "parent_file_id", + onDelete: ReferentialAction.Cascade); + }); + + migrationBuilder.CreateIndex( + name: "idx_embedded_files_absolute_path", + table: "embedded_files", + column: "absolute_path"); + + migrationBuilder.CreateIndex( + name: "idx_embedded_files_compliance", + table: "embedded_files", + column: "compliance_level_rank"); + + migrationBuilder.CreateIndex( + name: "idx_embedded_files_data_source", + table: "embedded_files", + column: "data_source_id"); + + migrationBuilder.CreateIndex( + name: "idx_embedded_files_data_source_absolute_path", + table: "embedded_files", + columns: ["data_source_id", "absolute_path"], + unique: true); + + migrationBuilder.CreateIndex( + name: "idx_embedded_files_file_type", + table: "embedded_files", + column: "file_type"); + + migrationBuilder.CreateIndex( + name: "idx_embedding_chunks_chunk_id", + table: "embedding_chunks", + column: "chunk_id", + unique: true); + + migrationBuilder.CreateIndex( + name: "idx_embedding_chunks_page", + table: "embedding_chunks", + column: "page_number"); + + migrationBuilder.CreateIndex( + name: "idx_embedding_chunks_parent_file", + table: "embedding_chunks", + column: "parent_file_id"); + + migrationBuilder.CreateIndex( + name: "idx_embedding_chunks_parent_file_chunk_index", + table: "embedding_chunks", + columns: ["parent_file_id", "chunk_index"], + unique: true); + + migrationBuilder.Sql(""" + CREATE VIRTUAL TABLE IF NOT EXISTS embedding_chunks_fts + USING fts5(chunk_id UNINDEXED, file_name, chunk_text); + + CREATE TRIGGER IF NOT EXISTS embedding_chunks_ai + AFTER INSERT ON embedding_chunks + BEGIN + INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text) + VALUES ( + new.id, + new.chunk_id, + (SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id), + new.chunk_text); + END; + + CREATE TRIGGER IF NOT EXISTS embedding_chunks_ad + AFTER DELETE ON embedding_chunks + BEGIN + DELETE FROM embedding_chunks_fts + WHERE rowid = old.id; + END; + + CREATE TRIGGER IF NOT EXISTS embedding_chunks_au + AFTER UPDATE ON embedding_chunks + BEGIN + DELETE FROM embedding_chunks_fts + WHERE rowid = old.id; + + INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text) + VALUES ( + new.id, + new.chunk_id, + (SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id), + new.chunk_text); + END; + + CREATE TRIGGER IF NOT EXISTS embedded_files_file_name_au + AFTER UPDATE OF file_name ON embedded_files + BEGIN + DELETE FROM embedding_chunks_fts + WHERE rowid IN ( + SELECT id + FROM embedding_chunks + WHERE parent_file_id = new.parent_file_id + ); + + INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text) + SELECT id, chunk_id, new.file_name, chunk_text + FROM embedding_chunks + WHERE parent_file_id = new.parent_file_id; + END; + """); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(""" + DROP TRIGGER IF EXISTS embedded_files_file_name_au; + DROP TRIGGER IF EXISTS embedding_chunks_au; + DROP TRIGGER IF EXISTS embedding_chunks_ad; + DROP TRIGGER IF EXISTS embedding_chunks_ai; + DROP TABLE IF EXISTS embedding_chunks_fts; + """); + + migrationBuilder.DropTable(name: "embedding_chunks"); + migrationBuilder.DropTable(name: "embedded_files"); + migrationBuilder.DropTable(name: "data_sources"); + } +} diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/EmbeddingStateDbContextModelSnapshot.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/EmbeddingStateDbContextModelSnapshot.cs new file mode 100644 index 00000000..98910680 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/Migrations/EmbeddingStateDbContextModelSnapshot.cs @@ -0,0 +1,327 @@ +#nullable disable + +using Microsoft.EntityFrameworkCore; +using Microsoft.EntityFrameworkCore.Infrastructure; + +namespace AIStudio.Tools.Databases.EmbeddingState.Migrations; + +[DbContext(typeof(EmbeddingStateDbContext))] +partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot +{ + protected override void BuildModel(ModelBuilder modelBuilder) + { +#pragma warning disable 612, 618 + modelBuilder.HasAnnotation("ProductVersion", "9.0.18"); + var utcDateTimeConverter = new EmbeddingStateDateTimeConverter(); + + modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity => + { + entity.Property("DataSourceId") + .HasColumnType("TEXT") + .HasColumnName("data_source_id"); + + entity.Property("DataSourceName") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("data_source_name"); + + entity.Property("DataSourceType") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("data_source_type"); + + entity.Property("EmbeddingProviderId") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("embedding_provider_id"); + + entity.Property("EmbeddingSignature") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("embedding_signature"); + + entity.Property("SourceHash") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("source_hash") + .HasDefaultValue(string.Empty); + + entity.Property("UpdatedAtUtc") + .HasConversion(utcDateTimeConverter) + .HasColumnType("TEXT") + .HasColumnName("updated_at_utc"); + + entity.Property("VectorSize") + .HasColumnType("INTEGER") + .HasColumnName("vector_size") + .HasDefaultValue(0); + + entity.HasKey("DataSourceId"); + + entity.ToTable("data_sources"); + }); + + modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity => + { + entity.Property("ParentFileId") + .HasColumnType("TEXT") + .HasColumnName("parent_file_id"); + + entity.Property("AbsolutePath") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("absolute_path") + .UseCollation("NOCASE"); + + entity.Property("ChunkCount") + .HasColumnType("INTEGER") + .HasColumnName("chunk_count"); + + entity.Property("ComplianceLevel") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("compliance_level"); + + entity.Property("ComplianceLevelRank") + .HasColumnType("INTEGER") + .HasColumnName("compliance_level_rank"); + + entity.Property("CreationUtc") + .HasConversion(utcDateTimeConverter) + .HasColumnType("TEXT") + .HasColumnName("creation_utc"); + + entity.Property("DataSourceId") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("data_source_id"); + + entity.Property("EmbeddedAtUtc") + .HasConversion(utcDateTimeConverter) + .HasColumnType("TEXT") + .HasColumnName("embedded_at_utc"); + + entity.Property("FileName") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("file_name"); + + entity.Property("FileSize") + .HasColumnType("INTEGER") + .HasColumnName("file_size"); + + entity.Property("FileType") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("file_type"); + + entity.Property("Fingerprint") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("fingerprint"); + + entity.Property("LastWriteUtc") + .HasConversion(utcDateTimeConverter) + .HasColumnType("TEXT") + .HasColumnName("last_write_utc"); + + entity.Property("RelativePath") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("relative_path"); + + entity.HasKey("ParentFileId"); + + entity.HasIndex("AbsolutePath") + .HasDatabaseName("idx_embedded_files_absolute_path"); + + entity.HasIndex("ComplianceLevelRank") + .HasDatabaseName("idx_embedded_files_compliance"); + + entity.HasIndex("DataSourceId") + .HasDatabaseName("idx_embedded_files_data_source"); + + entity.HasIndex("DataSourceId", "AbsolutePath") + .IsUnique() + .HasDatabaseName("idx_embedded_files_data_source_absolute_path"); + + entity.HasIndex("FileType") + .HasDatabaseName("idx_embedded_files_file_type"); + + entity.ToTable("embedded_files"); + }); + + modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateChunkEntity", entity => + { + entity.Property("Id") + .ValueGeneratedOnAdd() + .HasColumnType("INTEGER") + .HasColumnName("id") + .HasAnnotation("Sqlite:Autoincrement", true); + + entity.Property("ChunkId") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("chunk_id"); + + entity.Property("ChunkIndex") + .HasColumnType("INTEGER") + .HasColumnName("chunk_index"); + + entity.Property("ChunkText") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("chunk_text"); + + entity.Property("EmbeddedAtUtc") + .HasConversion(utcDateTimeConverter) + .HasColumnType("TEXT") + .HasColumnName("embedded_at_utc"); + + entity.Property("PageNumber") + .HasColumnType("INTEGER") + .HasColumnName("page_number"); + + entity.Property("ParentFileId") + .IsRequired() + .HasColumnType("TEXT") + .HasColumnName("parent_file_id"); + + entity.HasKey("Id"); + + entity.HasIndex("ChunkId") + .IsUnique() + .HasDatabaseName("idx_embedding_chunks_chunk_id"); + + entity.HasIndex("PageNumber") + .HasDatabaseName("idx_embedding_chunks_page"); + + entity.HasIndex("ParentFileId") + .HasDatabaseName("idx_embedding_chunks_parent_file"); + + entity.HasIndex("ParentFileId", "ChunkIndex") + .IsUnique() + .HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index"); + + entity.ToTable("embedding_chunks"); + }); + + modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateSearchResultEntity", entity => + { + entity.Property("AbsolutePath") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("ChunkCount") + .HasColumnType("INTEGER"); + + entity.Property("ChunkId") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("ChunkIndex") + .HasColumnType("INTEGER"); + + entity.Property("ChunkText") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("ComplianceLevel") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("ComplianceLevelRank") + .HasColumnType("INTEGER"); + + entity.Property("CreationUtc") + .HasConversion(utcDateTimeConverter) + .HasColumnType("TEXT"); + + entity.Property("DataSourceId") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("DataSourceName") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("DataSourceType") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("EmbeddedAtUtc") + .HasConversion(utcDateTimeConverter) + .HasColumnType("TEXT"); + + entity.Property("FileName") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("FileSize") + .HasColumnType("INTEGER"); + + entity.Property("FileType") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("Fingerprint") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("LastWriteUtc") + .HasConversion(utcDateTimeConverter) + .HasColumnType("TEXT"); + + entity.Property("PageNumber") + .HasColumnType("INTEGER"); + + entity.Property("ParentFileId") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("RelativePath") + .IsRequired() + .HasColumnType("TEXT"); + + entity.Property("Score") + .HasColumnType("REAL"); + + entity.HasNoKey(); + + entity.ToView("embedding_chunk_search_results"); + }); + + modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity => + { + entity.HasOne("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", "DataSource") + .WithMany("Files") + .HasForeignKey("DataSourceId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + entity.Navigation("DataSource"); + }); + + modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateChunkEntity", entity => + { + entity.HasOne("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", "File") + .WithMany("Chunks") + .HasForeignKey("ParentFileId") + .OnDelete(DeleteBehavior.Cascade) + .IsRequired(); + + entity.Navigation("File"); + }); + + modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity => + { + entity.Navigation("Files"); + }); + + modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateFileEntity", entity => + { + entity.Navigation("Chunks"); + }); +#pragma warning restore 612, 618 + } +} diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs index 905d38e6..93657a1c 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs @@ -5,7 +5,7 @@ using AIStudio.Settings; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Services; -using Microsoft.Data.Sqlite; +using Microsoft.EntityFrameworkCore; namespace AIStudio.Tools.Databases.EmbeddingState; @@ -18,18 +18,12 @@ public sealed class SqliteEmbeddingStateClientImplementation( private const string DATABASE_NAME = "Local RAG Index"; private const string DATABASE_FILENAME = "rag-index.sqlite3"; private const int MAX_FTS_QUERY_TERMS = 32; + private const int CHUNK_UPSERT_BATCH_SIZE = 500; private static readonly Regex FTS_TOKEN_REGEX = new(@"[\p{L}\p{Nd}_]+", RegexOptions.Compiled | RegexOptions.CultureInvariant); private readonly string databasePath = databasePath; - private readonly string connectionString = new SqliteConnectionStringBuilder - { - DataSource = databasePath, - Mode = SqliteOpenMode.ReadWriteCreate, - Cache = SqliteCacheMode.Shared, - ForeignKeys = true, - DefaultTimeout = 30, - }.ToString(); + private readonly DbContextOptions dbContextOptions = EmbeddingStateDbContext.CreateOptions(databasePath); private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SqliteEmbeddingStateClientImplementation).Namespace, nameof(SqliteEmbeddingStateClientImplementation)); @@ -68,54 +62,45 @@ public sealed class SqliteEmbeddingStateClientImplementation( public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo() { + await using var context = this.CreateContext(); + yield return (TB("Reported version"), version); yield return (TB("Database path"), this.databasePath); yield return (TB("Storage size"), this.GetStorageSize()); - yield return (TB("Indexed data sources"), (await this.CountAsync("data_sources", CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); - yield return (TB("Indexed files"), (await this.CountAsync("embedded_files", CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); - yield return (TB("Search chunks"), (await this.CountAsync("embedding_chunks", CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); + yield return (TB("Indexed data sources"), (await context.DataSources.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); + yield return (TB("Indexed files"), (await context.EmbeddedFiles.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); + yield return (TB("Search chunks"), (await context.EmbeddingChunks.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); } public override async Task GetManifestAsync(string dataSourceId, CancellationToken token) { - await using var connection = await this.OpenConnectionAsync(token); + await using var context = this.CreateContext(); var manifest = new DataSourceEmbeddingManifest(); - await using (var command = CreateCommand(connection, """ - SELECT embedding_provider_id, embedding_signature, source_hash, vector_size - FROM data_sources - WHERE data_source_id = $dataSourceId - """)) - { - command.Parameters.AddWithValue("$dataSourceId", dataSourceId); - await using var reader = await command.ExecuteReaderAsync(token); - if (await reader.ReadAsync(token)) - { - manifest.EmbeddingProviderId = reader.GetString(0); - manifest.EmbeddingSignature = reader.GetString(1); - manifest.SourceHash = reader.GetString(2); - manifest.VectorSize = reader.GetInt32(3); - } - } + var dataSource = await context.DataSources + .AsNoTracking() + .FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token); - await using (var command = CreateCommand(connection, """ - SELECT absolute_path, fingerprint, file_size, last_write_utc, embedded_at_utc, chunk_count - FROM embedded_files - WHERE data_source_id = $dataSourceId - AND chunk_count > 0 - """)) + if (dataSource is null) + return manifest; + + manifest.EmbeddingProviderId = dataSource.EmbeddingProviderId; + manifest.EmbeddingSignature = dataSource.EmbeddingSignature; + manifest.SourceHash = dataSource.SourceHash; + manifest.VectorSize = dataSource.VectorSize; + + var files = await context.EmbeddedFiles + .AsNoTracking() + .Where(file => file.DataSourceId == dataSourceId && file.ChunkCount > 0) + .ToListAsync(token); + foreach (var file in files) { - command.Parameters.AddWithValue("$dataSourceId", dataSourceId); - await using var reader = await command.ExecuteReaderAsync(token); - while (await reader.ReadAsync(token)) - { - manifest.Files[reader.GetString(0)] = new EmbeddedFileRecord( - reader.GetString(1), - reader.GetInt64(2), - ParseUtc(reader.GetString(3)), - ParseUtc(reader.GetString(4)), - reader.GetInt32(5)); - } + manifest.Files[file.AbsolutePath] = new EmbeddedFileRecord( + file.Fingerprint, + file.FileSize, + file.LastWriteUtc, + file.EmbeddedAtUtc, + file.ChunkCount); } return manifest; @@ -131,148 +116,82 @@ public sealed class SqliteEmbeddingStateClientImplementation( int vectorSize, CancellationToken token) { - await using var connection = await this.OpenConnectionAsync(token); - await ExecuteNonQueryAsync(connection, """ - INSERT INTO data_sources ( - data_source_id, - data_source_name, - data_source_type, - embedding_provider_id, - embedding_signature, - source_hash, - vector_size, - updated_at_utc) - VALUES ( - $dataSourceId, - $dataSourceName, - $dataSourceType, - $embeddingProviderId, - $embeddingSignature, - $sourceHash, - $vectorSize, - $updatedAtUtc) - ON CONFLICT(data_source_id) DO UPDATE SET - data_source_name = excluded.data_source_name, - data_source_type = excluded.data_source_type, - embedding_provider_id = excluded.embedding_provider_id, - embedding_signature = excluded.embedding_signature, - source_hash = excluded.source_hash, - vector_size = excluded.vector_size, - updated_at_utc = excluded.updated_at_utc - """, token, - ("$dataSourceId", dataSourceId), - ("$dataSourceName", dataSourceName), - ("$dataSourceType", dataSourceType), - ("$embeddingProviderId", embeddingProviderId), - ("$embeddingSignature", embeddingSignature), - ("$sourceHash", sourceHash), - ("$vectorSize", vectorSize), - ("$updatedAtUtc", ToUtcText(DateTime.UtcNow))); + await using var context = this.CreateContext(); + var dataSource = await context.DataSources.FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token); + if (dataSource is null) + { + dataSource = new EmbeddingStateDataSourceEntity + { + DataSourceId = dataSourceId, + }; + context.DataSources.Add(dataSource); + } + + ApplyDataSource(dataSource, dataSourceName, dataSourceType, embeddingProviderId, embeddingSignature, sourceHash, vectorSize); + await context.SaveChangesAsync(token); } public override async Task UpdateVectorSizeAsync(string dataSourceId, int vectorSize, CancellationToken token) { - await using var connection = await this.OpenConnectionAsync(token); - await ExecuteNonQueryAsync(connection, """ - UPDATE data_sources - SET vector_size = $vectorSize, - updated_at_utc = $updatedAtUtc - WHERE data_source_id = $dataSourceId - """, token, - ("$dataSourceId", dataSourceId), - ("$vectorSize", vectorSize), - ("$updatedAtUtc", ToUtcText(DateTime.UtcNow))); + await using var context = this.CreateContext(); + var dataSource = await context.DataSources.FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token); + if (dataSource is null) + return; + + dataSource.VectorSize = vectorSize; + dataSource.UpdatedAtUtc = DateTime.UtcNow; + await context.SaveChangesAsync(token); } public override async Task UpdateDataSourceHashAsync(string dataSourceId, string sourceHash, CancellationToken token) { - await using var connection = await this.OpenConnectionAsync(token); - await ExecuteNonQueryAsync(connection, """ - UPDATE data_sources - SET source_hash = $sourceHash, - updated_at_utc = $updatedAtUtc - WHERE data_source_id = $dataSourceId - """, token, - ("$dataSourceId", dataSourceId), - ("$sourceHash", sourceHash), - ("$updatedAtUtc", ToUtcText(DateTime.UtcNow))); + await using var context = this.CreateContext(); + var dataSource = await context.DataSources.FirstOrDefaultAsync(source => source.DataSourceId == dataSourceId, token); + if (dataSource is null) + return; + + dataSource.SourceHash = sourceHash; + dataSource.UpdatedAtUtc = DateTime.UtcNow; + await context.SaveChangesAsync(token); } public override async Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token) { - await using var connection = await this.OpenConnectionAsync(token); - await ExecuteNonQueryAsync(connection, """ - INSERT INTO embedded_files ( - parent_file_id, - data_source_id, - absolute_path, - file_name, - relative_path, - file_type, - fingerprint, - file_size, - creation_utc, - last_write_utc, - embedded_at_utc, - chunk_count, - compliance_level, - compliance_level_rank) - VALUES ( - $parentFileId, - $dataSourceId, - $absolutePath, - $fileName, - $relativePath, - $fileType, - $fingerprint, - $fileSize, - $creationUtc, - $lastWriteUtc, - $embeddedAtUtc, - $chunkCount, - $complianceLevel, - $complianceLevelRank) - ON CONFLICT(parent_file_id) DO UPDATE SET - data_source_id = excluded.data_source_id, - absolute_path = excluded.absolute_path, - file_name = excluded.file_name, - relative_path = excluded.relative_path, - file_type = excluded.file_type, - fingerprint = excluded.fingerprint, - file_size = excluded.file_size, - creation_utc = excluded.creation_utc, - last_write_utc = excluded.last_write_utc, - embedded_at_utc = excluded.embedded_at_utc, - chunk_count = excluded.chunk_count, - compliance_level = excluded.compliance_level, - compliance_level_rank = excluded.compliance_level_rank - """, token, - ("$parentFileId", file.ParentFileId), - ("$dataSourceId", dataSourceId), - ("$absolutePath", file.AbsolutePath), - ("$fileName", file.FileName), - ("$relativePath", file.RelativePath), - ("$fileType", file.FileType), - ("$fingerprint", file.Fingerprint), - ("$fileSize", file.FileSize), - ("$creationUtc", ToUtcText(file.CreationUtc)), - ("$lastWriteUtc", ToUtcText(file.LastWriteUtc)), - ("$embeddedAtUtc", ToUtcText(file.EmbeddedAtUtc)), - ("$chunkCount", file.ChunkCount), - ("$complianceLevel", file.ComplianceLevel), - ("$complianceLevelRank", file.ComplianceLevelRank)); + await using var context = this.CreateContext(); + var fileEntity = await context.EmbeddedFiles.FirstOrDefaultAsync(entity => entity.ParentFileId == file.ParentFileId, token); + if (fileEntity is null) + { + fileEntity = new EmbeddingStateFileEntity + { + ParentFileId = file.ParentFileId, + }; + context.EmbeddedFiles.Add(fileEntity); + } + + ApplyFile(fileEntity, dataSourceId, file); + await context.SaveChangesAsync(token); } public override async Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) { - await using var connection = await this.OpenConnectionAsync(token); - await ExecuteNonQueryAsync(connection, """ - DELETE FROM embedded_files - WHERE data_source_id = $dataSourceId - AND absolute_path = $filePath - """, token, - ("$dataSourceId", dataSourceId), - ("$filePath", filePath)); + await using var context = this.CreateContext(); + await using var transaction = await context.Database.BeginTransactionAsync(token); + + var parentFileIds = await context.EmbeddedFiles + .Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath) + .Select(file => file.ParentFileId) + .ToListAsync(token); + + foreach (var parentFileIdBatch in parentFileIds.Chunk(CHUNK_UPSERT_BATCH_SIZE)) + await context.EmbeddingChunks + .Where(chunk => parentFileIdBatch.Contains(chunk.ParentFileId)) + .ExecuteDeleteAsync(token); + + await context.EmbeddedFiles + .Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath) + .ExecuteDeleteAsync(token); + + await transaction.CommitAsync(token); } public override async Task UpsertChunksAsync(string dataSourceId, IReadOnlyList chunks, CancellationToken token) @@ -280,41 +199,38 @@ public sealed class SqliteEmbeddingStateClientImplementation( if (chunks.Count == 0) return; - await using var connection = await this.OpenConnectionAsync(token); - await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(token); + await using var context = this.CreateContext(); + await using var transaction = await context.Database.BeginTransactionAsync(token); - foreach (var chunk in chunks) + foreach (var chunkBatch in chunks.Chunk(CHUNK_UPSERT_BATCH_SIZE)) { token.ThrowIfCancellationRequested(); - await ExecuteNonQueryAsync(connection, transaction, """ - INSERT INTO embedding_chunks ( - chunk_id, - parent_file_id, - page_number, - chunk_index, - chunk_text, - embedded_at_utc) - VALUES ( - $chunkId, - $parentFileId, - $pageNumber, - $chunkIndex, - $chunkText, - $embeddedAtUtc) - ON CONFLICT(chunk_id) DO UPDATE SET - parent_file_id = excluded.parent_file_id, - page_number = excluded.page_number, - chunk_index = excluded.chunk_index, - chunk_text = excluded.chunk_text, - embedded_at_utc = excluded.embedded_at_utc - """, token, - ("$chunkId", chunk.ChunkId), - ("$parentFileId", chunk.ParentFileId), - ("$pageNumber", chunk.PageNumber), - ("$chunkIndex", chunk.ChunkIndex), - ("$chunkText", chunk.ChunkText), - ("$embeddedAtUtc", ToUtcText(chunk.EmbeddedAtUtc))); + var chunkIds = chunkBatch + .Select(chunk => chunk.ChunkId) + .Distinct(StringComparer.Ordinal) + .ToArray(); + var existingChunks = await context.EmbeddingChunks + .Where(chunk => chunkIds.Contains(chunk.ChunkId)) + .ToDictionaryAsync(chunk => chunk.ChunkId, StringComparer.Ordinal, token); + + foreach (var chunk in chunkBatch) + { + if (!existingChunks.TryGetValue(chunk.ChunkId, out var chunkEntity)) + { + chunkEntity = new EmbeddingStateChunkEntity + { + ChunkId = chunk.ChunkId, + }; + context.EmbeddingChunks.Add(chunkEntity); + existingChunks[chunk.ChunkId] = chunkEntity; + } + + ApplyChunk(chunkEntity, chunk); + } + + await context.SaveChangesAsync(token); + context.ChangeTracker.Clear(); } await transaction.CommitAsync(token); @@ -329,83 +245,66 @@ public sealed class SqliteEmbeddingStateClientImplementation( if (string.IsNullOrWhiteSpace(ftsQuery)) return []; - var results = new List(maxMatches); - await using var connection = await this.OpenConnectionAsync(token); - await using var command = CreateCommand(connection, """ - SELECT - c.chunk_id, - c.parent_file_id, - ds.data_source_id, - ds.data_source_name, - ds.data_source_type, - f.absolute_path, - f.file_name, - f.relative_path, - f.file_type, - c.page_number, - c.chunk_index, - c.chunk_text, - bm25(embedding_chunks_fts) AS score, - f.fingerprint, - f.file_size, - f.creation_utc, - f.last_write_utc, - c.embedded_at_utc, - f.chunk_count, - f.compliance_level, - f.compliance_level_rank - FROM embedding_chunks_fts - JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid - JOIN embedded_files f ON f.parent_file_id = c.parent_file_id - JOIN data_sources ds ON ds.data_source_id = f.data_source_id - WHERE ds.data_source_id = $dataSourceId - AND embedding_chunks_fts MATCH $query - ORDER BY score - LIMIT $maxMatches - """); + await using var context = this.CreateContext(); + var results = await context.SearchResults + .FromSqlInterpolated($""" + SELECT + c.chunk_id AS ChunkId, + c.parent_file_id AS ParentFileId, + ds.data_source_id AS DataSourceId, + ds.data_source_name AS DataSourceName, + ds.data_source_type AS DataSourceType, + f.absolute_path AS AbsolutePath, + f.file_name AS FileName, + f.relative_path AS RelativePath, + f.file_type AS FileType, + c.page_number AS PageNumber, + c.chunk_index AS ChunkIndex, + c.chunk_text AS ChunkText, + bm25(embedding_chunks_fts) AS Score, + f.fingerprint AS Fingerprint, + f.file_size AS FileSize, + f.creation_utc AS CreationUtc, + f.last_write_utc AS LastWriteUtc, + c.embedded_at_utc AS EmbeddedAtUtc, + f.chunk_count AS ChunkCount, + f.compliance_level AS ComplianceLevel, + f.compliance_level_rank AS ComplianceLevelRank + FROM embedding_chunks_fts + JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid + JOIN embedded_files f ON f.parent_file_id = c.parent_file_id + JOIN data_sources ds ON ds.data_source_id = f.data_source_id + WHERE ds.data_source_id = {dataSourceId} + AND embedding_chunks_fts MATCH {ftsQuery} + ORDER BY Score + LIMIT {maxMatches} + """) + .AsNoTracking() + .ToListAsync(token); - command.Parameters.AddWithValue("$dataSourceId", dataSourceId); - command.Parameters.AddWithValue("$query", ftsQuery); - command.Parameters.AddWithValue("$maxMatches", maxMatches); - - await using var reader = await command.ExecuteReaderAsync(token); - while (await reader.ReadAsync(token)) - { - results.Add(new EmbeddingStateSearchResult( - reader.GetString(0), - reader.GetString(1), - reader.GetString(2), - reader.GetString(3), - reader.GetString(4), - reader.GetString(5), - reader.GetString(6), - reader.GetString(7), - reader.GetString(8), - reader.IsDBNull(9) ? null : reader.GetInt32(9), - reader.GetInt32(10), - reader.GetString(11), - reader.GetDouble(12), - reader.GetString(13), - reader.GetInt64(14), - ParseUtc(reader.GetString(15)), - ParseUtc(reader.GetString(16)), - ParseUtc(reader.GetString(17)), - reader.GetInt32(18), - reader.GetString(19), - reader.GetInt32(20))); - } - - return results; + return results.Select(ToSearchResult).ToList(); } public override async Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) { - await using var connection = await this.OpenConnectionAsync(token); - await ExecuteNonQueryAsync(connection, """ - DELETE FROM data_sources - WHERE data_source_id = $dataSourceId - """, token, - ("$dataSourceId", dataSourceId)); + await using var context = this.CreateContext(); + await using var transaction = await context.Database.BeginTransactionAsync(token); + + var parentFileIds = await context.EmbeddedFiles + .Where(file => file.DataSourceId == dataSourceId) + .Select(file => file.ParentFileId) + .ToListAsync(token); + + foreach (var parentFileIdBatch in parentFileIds.Chunk(CHUNK_UPSERT_BATCH_SIZE)) + await context.EmbeddingChunks + .Where(chunk => parentFileIdBatch.Contains(chunk.ParentFileId)) + .ExecuteDeleteAsync(token); + + await context.DataSources + .Where(source => source.DataSourceId == dataSourceId) + .ExecuteDeleteAsync(token); + + await transaction.CommitAsync(token); } public override void Dispose() @@ -414,198 +313,88 @@ public sealed class SqliteEmbeddingStateClientImplementation( private async Task InitializeAsync(CancellationToken token) { - await using var connection = await this.OpenConnectionAsync(token); - await ExecuteNonQueryAsync(connection, """ - PRAGMA journal_mode=WAL; - PRAGMA foreign_keys=ON; - - CREATE TABLE IF NOT EXISTS data_sources ( - data_source_id TEXT PRIMARY KEY, - data_source_name TEXT NOT NULL, - data_source_type TEXT NOT NULL, - embedding_provider_id TEXT NOT NULL, - embedding_signature TEXT NOT NULL, - source_hash TEXT NOT NULL DEFAULT '', - vector_size INTEGER NOT NULL DEFAULT 0, - updated_at_utc TEXT NOT NULL - ); - - CREATE TABLE IF NOT EXISTS embedded_files ( - parent_file_id TEXT PRIMARY KEY, - data_source_id TEXT NOT NULL, - absolute_path TEXT COLLATE NOCASE NOT NULL, - file_name TEXT NOT NULL, - relative_path TEXT NOT NULL, - file_type TEXT NOT NULL, - fingerprint TEXT NOT NULL, - file_size INTEGER NOT NULL, - creation_utc TEXT NOT NULL, - last_write_utc TEXT NOT NULL, - embedded_at_utc TEXT NOT NULL, - chunk_count INTEGER NOT NULL, - compliance_level TEXT NOT NULL, - compliance_level_rank INTEGER NOT NULL, - FOREIGN KEY (data_source_id) - REFERENCES data_sources(data_source_id) - ON DELETE CASCADE, - UNIQUE(data_source_id, absolute_path) - ); - - CREATE TABLE IF NOT EXISTS embedding_chunks ( - id INTEGER PRIMARY KEY AUTOINCREMENT, - chunk_id TEXT NOT NULL UNIQUE, - parent_file_id TEXT NOT NULL, - page_number INTEGER NULL, - chunk_index INTEGER NOT NULL, - chunk_text TEXT NOT NULL, - embedded_at_utc TEXT NOT NULL, - FOREIGN KEY (parent_file_id) - REFERENCES embedded_files(parent_file_id) - ON DELETE CASCADE, - UNIQUE(parent_file_id, chunk_index) - ); - - CREATE INDEX IF NOT EXISTS idx_embedded_files_data_source - ON embedded_files(data_source_id); - - CREATE INDEX IF NOT EXISTS idx_embedded_files_absolute_path - ON embedded_files(absolute_path); - - CREATE INDEX IF NOT EXISTS idx_embedded_files_file_type - ON embedded_files(file_type); - - CREATE INDEX IF NOT EXISTS idx_embedded_files_compliance - ON embedded_files(compliance_level_rank); - - CREATE INDEX IF NOT EXISTS idx_embedding_chunks_parent_file - ON embedding_chunks(parent_file_id); - - CREATE INDEX IF NOT EXISTS idx_embedding_chunks_page - ON embedding_chunks(page_number); - - CREATE VIRTUAL TABLE IF NOT EXISTS embedding_chunks_fts - USING fts5(chunk_id UNINDEXED, file_name, chunk_text); - - CREATE TRIGGER IF NOT EXISTS embedding_chunks_ai - AFTER INSERT ON embedding_chunks - BEGIN - INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text) - VALUES ( - new.id, - new.chunk_id, - (SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id), - new.chunk_text); - END; - - CREATE TRIGGER IF NOT EXISTS embedding_chunks_ad - AFTER DELETE ON embedding_chunks - BEGIN - DELETE FROM embedding_chunks_fts - WHERE rowid = old.id; - END; - - CREATE TRIGGER IF NOT EXISTS embedding_chunks_au - AFTER UPDATE ON embedding_chunks - BEGIN - DELETE FROM embedding_chunks_fts - WHERE rowid = old.id; - - INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text) - VALUES ( - new.id, - new.chunk_id, - (SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id), - new.chunk_text); - END; - - CREATE TRIGGER IF NOT EXISTS embedded_files_file_name_au - AFTER UPDATE OF file_name ON embedded_files - BEGIN - DELETE FROM embedding_chunks_fts - WHERE rowid IN ( - SELECT id - FROM embedding_chunks - WHERE parent_file_id = new.parent_file_id - ); - - INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text) - SELECT id, chunk_id, new.file_name, chunk_text - FROM embedding_chunks - WHERE parent_file_id = new.parent_file_id; - END; - """, token); + await using var context = this.CreateContext(); + await EmbeddingStateSchemaMigrator.MigrateAsync(context, token); } private async Task GetSqliteVersionAsync(CancellationToken token) { - await using var connection = await this.OpenConnectionAsync(token); - await using var command = CreateCommand(connection, "SELECT sqlite_version()"); - var versionObject = await command.ExecuteScalarAsync(token); - return Convert.ToString(versionObject, CultureInfo.InvariantCulture) ?? string.Empty; + await using var context = this.CreateContext(); + var versions = await context.Database + .SqlQueryRaw("SELECT sqlite_version() AS Value") + .ToListAsync(token); + return versions.FirstOrDefault() ?? string.Empty; } - private async Task CountAsync(string tableName, CancellationToken token) + private EmbeddingStateDbContext CreateContext() => new(this.dbContextOptions); + + private static void ApplyDataSource( + EmbeddingStateDataSourceEntity dataSource, + string dataSourceName, + string dataSourceType, + string embeddingProviderId, + string embeddingSignature, + string sourceHash, + int vectorSize) { - await using var connection = await this.OpenConnectionAsync(token); - await using var command = CreateCommand(connection, $"SELECT COUNT(*) FROM {tableName}"); - var countObject = await command.ExecuteScalarAsync(token); - return Convert.ToInt64(countObject, CultureInfo.InvariantCulture); + dataSource.DataSourceName = dataSourceName; + dataSource.DataSourceType = dataSourceType; + dataSource.EmbeddingProviderId = embeddingProviderId; + dataSource.EmbeddingSignature = embeddingSignature; + dataSource.SourceHash = sourceHash; + dataSource.VectorSize = vectorSize; + dataSource.UpdatedAtUtc = DateTime.UtcNow; } - private async Task OpenConnectionAsync(CancellationToken token) + private static void ApplyFile(EmbeddingStateFileEntity fileEntity, string dataSourceId, EmbeddingStateFile file) { - var connection = new SqliteConnection(this.connectionString); - await connection.OpenAsync(token); - return connection; + fileEntity.DataSourceId = dataSourceId; + fileEntity.AbsolutePath = file.AbsolutePath; + fileEntity.FileName = file.FileName; + fileEntity.RelativePath = file.RelativePath; + fileEntity.FileType = file.FileType; + fileEntity.Fingerprint = file.Fingerprint; + fileEntity.FileSize = file.FileSize; + fileEntity.CreationUtc = file.CreationUtc; + fileEntity.LastWriteUtc = file.LastWriteUtc; + fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc; + fileEntity.ChunkCount = file.ChunkCount; + fileEntity.ComplianceLevel = file.ComplianceLevel; + fileEntity.ComplianceLevelRank = file.ComplianceLevelRank; } - private static SqliteCommand CreateCommand(SqliteConnection connection, string commandText) + private static void ApplyChunk(EmbeddingStateChunkEntity chunkEntity, EmbeddingStateChunk chunk) { - var command = connection.CreateCommand(); - command.CommandText = commandText; - return command; + chunkEntity.ChunkId = chunk.ChunkId; + chunkEntity.ParentFileId = chunk.ParentFileId; + chunkEntity.PageNumber = chunk.PageNumber; + chunkEntity.ChunkIndex = chunk.ChunkIndex; + chunkEntity.ChunkText = chunk.ChunkText; + chunkEntity.EmbeddedAtUtc = chunk.EmbeddedAtUtc; } - private static async Task ExecuteNonQueryAsync( - SqliteConnection connection, - string commandText, - CancellationToken token, - params (string Name, object? Value)[] parameters) - { - await using var command = CreateCommand(connection, commandText); - foreach (var (name, value) in parameters) - command.Parameters.AddWithValue(name, value ?? DBNull.Value); - - await command.ExecuteNonQueryAsync(token); - } - - private static async Task ExecuteNonQueryAsync( - SqliteConnection connection, - SqliteTransaction transaction, - string commandText, - CancellationToken token, - params (string Name, object? Value)[] parameters) - { - await using var command = CreateCommand(connection, commandText); - command.Transaction = transaction; - foreach (var (name, value) in parameters) - command.Parameters.AddWithValue(name, value ?? DBNull.Value); - - await command.ExecuteNonQueryAsync(token); - } - - private static string ToUtcText(DateTime dateTime) - { - var utc = dateTime.Kind is DateTimeKind.Utc ? dateTime : dateTime.ToUniversalTime(); - return utc.ToString("O", CultureInfo.InvariantCulture); - } - - private static DateTime ParseUtc(string value) - { - return DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var dateTime) - ? dateTime.ToUniversalTime() - : DateTime.UnixEpoch; - } + private static EmbeddingStateSearchResult ToSearchResult(EmbeddingStateSearchResultEntity result) => new( + result.ChunkId, + result.ParentFileId, + result.DataSourceId, + result.DataSourceName, + result.DataSourceType, + result.AbsolutePath, + result.FileName, + result.RelativePath, + result.FileType, + result.PageNumber, + result.ChunkIndex, + result.ChunkText, + result.Score, + result.Fingerprint, + result.FileSize, + result.CreationUtc, + result.LastWriteUtc, + result.EmbeddedAtUtc, + result.ChunkCount, + result.ComplianceLevel, + result.ComplianceLevelRank); private static string BuildFtsQuery(string query) { diff --git a/app/MindWork AI Studio/packages.lock.json b/app/MindWork AI Studio/packages.lock.json index 75d27fc2..3c428c94 100644 --- a/app/MindWork AI Studio/packages.lock.json +++ b/app/MindWork AI Studio/packages.lock.json @@ -32,27 +32,43 @@ }, "Microsoft.Data.Sqlite.Core": { "type": "Direct", - "requested": "[9.0.9, )", - "resolved": "9.0.9", - "contentHash": "DjxZRueHp0qvZxhvW+H1IWYkSofZI8Chg710KYJjNP/6S4q3rt97pvR8AHOompkSwaN92VLKz5uw01iUt85cMg==", + "requested": "[9.0.18, )", + "resolved": "9.0.18", + "contentHash": "2ME/X/d9EG1rIuA9lpCJeGpIydXBgdRSCg2jfzYc4REfUqBwGAfdn8vVEFddGvYMHrp/2LdzIradPYhXBkoXAA==", "dependencies": { "SQLitePCLRaw.core": "2.1.10" } }, + "Microsoft.EntityFrameworkCore.Sqlite": { + "type": "Direct", + "requested": "[9.0.18, )", + "resolved": "9.0.18", + "contentHash": "EXKh713SdpKsJAd0PCNWfcZ23Dh8AKOy4PCNK29FXhC4Ag8t/h1e4ZoaoQyzWO17jRiSiddXXZIoPzw6z/gf+Q==", + "dependencies": { + "Microsoft.EntityFrameworkCore.Sqlite.Core": "9.0.18", + "Microsoft.Extensions.Caching.Memory": "9.0.18", + "Microsoft.Extensions.Configuration.Abstractions": "9.0.18", + "Microsoft.Extensions.DependencyModel": "9.0.18", + "Microsoft.Extensions.Logging": "9.0.18", + "SQLitePCLRaw.bundle_e_sqlite3": "2.1.10", + "SQLitePCLRaw.core": "2.1.10", + "System.Text.Json": "9.0.18" + } + }, "Microsoft.Extensions.FileProviders.Embedded": { "type": "Direct", - "requested": "[9.0.17, )", - "resolved": "9.0.17", - "contentHash": "ItYX3BajZhWwq1wmvUnYA1jahNi9jyy2BMGzyWPTgdSuay8FfMF0gAfNe8mVE6F+GJaQWymElj8hKimRmGxOzw==", + "requested": "[9.0.18, )", + "resolved": "9.0.18", + "contentHash": "+t0Bq5qZZ/zbmO4X70nDMC+anTsNSCxNvjtqXmRiUwh53cNfMoXkB/R95rUO9+yFYhsTR7B302ys9LqXDdIt6g==", "dependencies": { - "Microsoft.Extensions.FileProviders.Abstractions": "9.0.17" + "Microsoft.Extensions.FileProviders.Abstractions": "9.0.18" } }, "Microsoft.NET.ILLink.Tasks": { "type": "Direct", - "requested": "[9.0.17, )", - "resolved": "9.0.17", - "contentHash": "P5qY/hIYMlo0+QRM0W3Gd/SRf20TX+z5W5NwpdzkOk0FtgcbSTNwNcYBRNDgfThFcLpcDFslz65RcGqWOq00/w==" + "requested": "[9.0.18, )", + "resolved": "9.0.18", + "contentHash": "ztGVXB28bi8SeplFmAx+4MkqP1ieA4UNzj/M3qyyz5tLa37Ln8x8LuaXdxzzoOdaucjQBKXSdCMFSbpQaNGIEg==" }, "MudBlazor": { "type": "Direct", @@ -86,12 +102,12 @@ }, "SQLitePCLRaw.bundle_e_sqlite3": { "type": "Direct", - "requested": "[2.1.10, )", - "resolved": "2.1.10", - "contentHash": "UxWuisvZ3uVcVOLJQv7urM/JiQH+v3TmaJc1BLKl5Dxfm/nTzTUrqswCqg/INiYLi61AXnHo1M1JPmPqqLnAdg==", + "requested": "[3.0.5, )", + "resolved": "3.0.5", + "contentHash": "SW8iASIyWMrLzqabUHYQRvALhvD4ylSBsj4PgEVGwc36kQjc9xT5kSV/XQ9rU7nIpBWD+LPyX3Hlw5FzyUzGeQ==", "dependencies": { - "SQLitePCLRaw.lib.e_sqlite3": "2.1.10", - "SQLitePCLRaw.provider.e_sqlite3": "2.1.10" + "SQLite": "3.53.4", + "SQLitePCLRaw.config.e_sqlite3": "3.0.5" } }, "BuildBundlerMinifier": { @@ -163,25 +179,105 @@ "resolved": "9.0.11", "contentHash": "O0HzG5utNH6ihO632k0nHFZa8iNDmGphdgWWqeDSdN/T9n0ZOXlA5+q77DxY3nHTjNfA0KMfpykIhEI+Wmzosg==" }, + "Microsoft.EntityFrameworkCore": { + "type": "Transitive", + "resolved": "9.0.18", + "contentHash": "ByxJvHZwP1QWLXmPLNClJ4mVAgzAjzdKlK/pb2E9qbyuQdhwGTy/yfTDsPlwYtXlwnDes9/2p4mOP/+Cbo4pFA==", + "dependencies": { + "Microsoft.EntityFrameworkCore.Abstractions": "9.0.18", + "Microsoft.EntityFrameworkCore.Analyzers": "9.0.18", + "Microsoft.Extensions.Caching.Memory": "9.0.18", + "Microsoft.Extensions.Logging": "9.0.18" + } + }, + "Microsoft.EntityFrameworkCore.Abstractions": { + "type": "Transitive", + "resolved": "9.0.18", + "contentHash": "4nD2ZtEG/fNyHT4bo0MovOw6wDVv2xunhmRm448AKbX3y2ohEZkkxI4Nm+JHBZmluzsV6HHWFRWawFj8VuNltg==" + }, + "Microsoft.EntityFrameworkCore.Analyzers": { + "type": "Transitive", + "resolved": "9.0.18", + "contentHash": "KI+qKZHU1OfEUjQEpQEh/EXo9wN8DP+Jt1aReNJe/MezR/cVTr79c4VeMv4cyizki7YWW3DDDWZCcy/Eiay6dg==" + }, + "Microsoft.EntityFrameworkCore.Relational": { + "type": "Transitive", + "resolved": "9.0.18", + "contentHash": "kn6DongUAVevGsydWJdJUB0N6QamtwyDjhwrbV8Yrfs84i5rcEj5SY+3eyPzAInzd1icrZBd7A+r0yEpULkjEg==", + "dependencies": { + "Microsoft.EntityFrameworkCore": "9.0.18", + "Microsoft.Extensions.Caching.Memory": "9.0.18", + "Microsoft.Extensions.Configuration.Abstractions": "9.0.18", + "Microsoft.Extensions.Logging": "9.0.18" + } + }, + "Microsoft.EntityFrameworkCore.Sqlite.Core": { + "type": "Transitive", + "resolved": "9.0.18", + "contentHash": "MK3PMGOmNbhUJSh5tewYuDa0w0zz2TX/naZoyasTae14C5XxhzuKA6rDBeKCct0lHIPMEGnCcCk2Alf7nmS3ag==", + "dependencies": { + "Microsoft.Data.Sqlite.Core": "9.0.18", + "Microsoft.EntityFrameworkCore.Relational": "9.0.18", + "Microsoft.Extensions.Caching.Memory": "9.0.18", + "Microsoft.Extensions.Configuration.Abstractions": "9.0.18", + "Microsoft.Extensions.DependencyModel": "9.0.18", + "Microsoft.Extensions.Logging": "9.0.18", + "SQLitePCLRaw.core": "2.1.10", + "System.Text.Json": "9.0.18" + } + }, + "Microsoft.Extensions.Caching.Abstractions": { + "type": "Transitive", + "resolved": "9.0.18", + "contentHash": "Y8PnPKq+ASBazn0QA5d98/dWT0MAfv9CJYV6zPZ/TisZpAUe5zWgKldJLXkrmUuIzvUpQS7/SF7+1dld3OP/EQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "9.0.18" + } + }, + "Microsoft.Extensions.Caching.Memory": { + "type": "Transitive", + "resolved": "9.0.18", + "contentHash": "ZQ+x93CRR8DOGT+YPP4YXvwTAJ0MfvAghs5cwCSvtZFiORSfprAw9f67d0QDhf6PCIYyFsMKFB+PI3p9lpiMUQ==", + "dependencies": { + "Microsoft.Extensions.Caching.Abstractions": "9.0.18", + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.18", + "Microsoft.Extensions.Logging.Abstractions": "9.0.18", + "Microsoft.Extensions.Options": "9.0.18", + "Microsoft.Extensions.Primitives": "9.0.18" + } + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "9.0.18", + "contentHash": "jgC5SZeK94t8DXl836ifIM5gScNqknkOwISrY503eWg7AHBrv+fnwty+C9n35H4WvBjs8OY21jPlRRf7ESBK/g==", + "dependencies": { + "Microsoft.Extensions.Primitives": "9.0.18" + } + }, "Microsoft.Extensions.DependencyInjection": { "type": "Transitive", - "resolved": "9.0.11", - "contentHash": "UquyDzvz0EneIQrrU67GJkIgynS+VD7t+RDtNv6VgKMOFrLBjldn6hzlXppGGecFMvAkMTqn4T8RYvzw7j7fQA==", + "resolved": "9.0.18", + "contentHash": "xPoYhpBypsweLpZ29GPp6Kv6g1fZMUwJ800D/xO7ASuNn5zx0BNIP600wXXhbPNeO8kKngFRtoWplRffxBUDgw==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.11" + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.18" } }, "Microsoft.Extensions.DependencyInjection.Abstractions": { "type": "Transitive", - "resolved": "9.0.11", - "contentHash": "+ZxxZzcVU+IEzq12GItUzf/V3mEc5nSLiXijwvDc4zyhbjvSZZ043giSZqGnhakrjwRWjkerIHPrRwm9okEIpw==" + "resolved": "9.0.18", + "contentHash": "eSUdaLUP+pj1J7SpCsUYQP6dNk0w9g0KGu4ovPYBnlfqomW1u+1fF8ZIjMVuHY6Z3p30yL0SxND9JTS6i3NolQ==" + }, + "Microsoft.Extensions.DependencyModel": { + "type": "Transitive", + "resolved": "9.0.18", + "contentHash": "uv+aDeo/9tDFxb5ykdU9WTPYDOzeFK53m51QbYHaWe7IvPRMSyMDUlDDXSRYJ2aPNWAE5ISPjvvHQL9UlQS+vw==" }, "Microsoft.Extensions.FileProviders.Abstractions": { "type": "Transitive", - "resolved": "9.0.17", - "contentHash": "uTkT+/Km0tEPOw9kiLTXJwXlEVQZ5IBxRQm2EvIAwebfKqqaVY/ClkgcZ7FyzzwqFkFmhklWet4Ju4yWRy5jPg==", + "resolved": "9.0.18", + "contentHash": "YqkFlTwnVSMuunsf8IT9b+KySfm6vnMBBM+CKYCfXfjRMQ62uFggVOEu4C2cgR4fXpEO1rZ6utUZC1KoYKgiSg==", "dependencies": { - "Microsoft.Extensions.Primitives": "9.0.17" + "Microsoft.Extensions.Primitives": "9.0.18" } }, "Microsoft.Extensions.Localization": { @@ -200,68 +296,83 @@ "resolved": "9.0.1", "contentHash": "CABog43lyaZQMjmlktuImCy6zmAzRBaXqN81uPaMQjlp//ISDVYItZPh6KWpWRF4MY/B67X5oDc3JTUpfdocZw==" }, + "Microsoft.Extensions.Logging": { + "type": "Transitive", + "resolved": "9.0.18", + "contentHash": "LuOCZhcJ4TeCLE99TRqI1FHReB29G7bAPInrAAG3E85/S+E4mcrmaoeoXLnOUi6DEosx87pauyFNICdiFIFqkA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "9.0.18", + "Microsoft.Extensions.Logging.Abstractions": "9.0.18", + "Microsoft.Extensions.Options": "9.0.18" + } + }, "Microsoft.Extensions.Logging.Abstractions": { "type": "Transitive", - "resolved": "9.0.11", - "contentHash": "UKWFTDwtZQIoypyt1YPVsxTnDK+0sKn26+UeSGeNlkRQddrkt9EC6kP4g94rgO/WOZkz94bKNlF1dVZN3QfPFQ==", + "resolved": "9.0.18", + "contentHash": "uzNHNdNZJTHACwFkWclx8s5GXLjGacJj4QwBQZ/6BeixTJjlHmNESIK738iPFLPkO8jWsENLUcUW/kAPe0b8ew==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.11" + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.18" } }, "Microsoft.Extensions.Options": { "type": "Transitive", - "resolved": "9.0.11", - "contentHash": "HX4M3BLkW1dtByMKHDVq6r7Jy6e4hf8NDzHpIgz7C8BtYk9JQHhfYX5c1UheQTD5Veg1yBhz/cD9C8vtrGrk9w==", + "resolved": "9.0.18", + "contentHash": "HiA/R0jGR10TGuR6coM8KMUdJ5YlP3bhGxqEdGLkxFdBV1EFNdZMAajSEZHwkex4X/iMQwRCQVotqpElcxOImg==", "dependencies": { - "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.11", - "Microsoft.Extensions.Primitives": "9.0.11" + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.18", + "Microsoft.Extensions.Primitives": "9.0.18" } }, "Microsoft.Extensions.Primitives": { "type": "Transitive", - "resolved": "9.0.17", - "contentHash": "WBjZ/zeb6PyCLT6lpGSzNtdMyRDloFSPqjY9kIGb5rdSng03rd0+ix/jDEYU6DUjE7JVLuhggXeMONVBxBHEXg==" + "resolved": "9.0.18", + "contentHash": "hfHudMC5zDlwMrC0HiHOJesSHMvM+CdqjomjcV/YVzFq5dfSpBRvyRLm1n1Bfh41ZpQnyJzqX+YEo95BAmcDAQ==" }, "Microsoft.JSInterop": { "type": "Transitive", "resolved": "9.0.11", "contentHash": "5w/W57cXjt8Ugp5COQCsv1R/wt7KzZXjbTqK4AFvgsxqmv1DFJ6OzagzJmwgp6unczFuff6t8wNi+URePV6PYQ==" }, - "SQLitePCLRaw.core": { + "SQLite": { "type": "Transitive", - "resolved": "2.1.10", - "contentHash": "Ii8JCbC7oiVclaE/mbDEK000EFIJ+ShRPwAvvV89GOZhQ+ZLtlnSWl6ksCNMKu/VGXA4Nfi2B7LhN/QFN9oBcw==", + "resolved": "3.53.4", + "contentHash": "KN7jeWqgUPeBRe1FlcpZURzxomuKKEKHmBBQfg+Nx7NkY1LjKhzHvH+3ASkNvhayESE34nMBinL9CV21JfPRJw==" + }, + "SQLitePCLRaw.config.e_sqlite3": { + "type": "Transitive", + "resolved": "3.0.5", + "contentHash": "aSk8WE5tF2MybESMgtZAEyMVCAWA0nBqOMc48HFoa9UoSdtm3goDXVzNRnefeKIwE6bV9NaNXptn1F9ReMQI0Q==", "dependencies": { - "System.Memory": "4.5.3" + "SQLitePCLRaw.provider.e_sqlite3": "3.0.5" } }, - "SQLitePCLRaw.lib.e_sqlite3": { + "SQLitePCLRaw.core": { "type": "Transitive", - "resolved": "2.1.10", - "contentHash": "mAr69tDbnf3QJpRy2nJz8Qdpebdil00fvycyByR58Cn9eARvR+UiG2Vzsp+4q1tV3ikwiYIjlXCQFc12GfebbA==" + "resolved": "3.0.5", + "contentHash": "k81AYXXRCw3Zj8rOhyBoCsx/U97KYDFI2CSKr/ijl5BwpsW/hX/4kBiDmerFaoust8nxBwa0IHQFw8MmSHRtnQ==" }, "SQLitePCLRaw.provider.e_sqlite3": { "type": "Transitive", - "resolved": "2.1.10", - "contentHash": "uZVTi02C1SxqzgT0HqTWatIbWGb40iIkfc3FpFCpE/r7g6K0PqzDUeefL6P6HPhDtc6BacN3yQysfzP7ks+wSQ==", + "resolved": "3.0.5", + "contentHash": "um8YSWduhhuskTG2bHfZrBqMNQOydfU8pcseT+cu5RivOGyoUbCrGXxgoRNTmRYw2VbMWnEZVVFcneZT/5dsBg==", "dependencies": { - "SQLitePCLRaw.core": "2.1.10" + "SQLitePCLRaw.core": "3.0.5" } }, - "System.Memory": { + "System.Text.Json": { "type": "Transitive", - "resolved": "4.5.3", - "contentHash": "3oDzvc/zzetpTKWMShs1AADwZjQ/36HnsufHRPcOjyRAAMLDlu2iD33MBI2opxnezcVUtXyqDXXjoFMOU9c7SA==" + "resolved": "9.0.18", + "contentHash": "/xX8vajlGUp1FoGSNUJBJS+E/2TQjmcMoUzHePa+coPdLnFGPcolRHlybSZhKyAKj+sNDSFlLNYYLe8aEOElfA==" }, "sharedtools": { "type": "Project" } }, "net9.0/win-x64": { - "SQLitePCLRaw.lib.e_sqlite3": { + "SQLite": { "type": "Transitive", - "resolved": "2.1.10", - "contentHash": "mAr69tDbnf3QJpRy2nJz8Qdpebdil00fvycyByR58Cn9eARvR+UiG2Vzsp+4q1tV3ikwiYIjlXCQFc12GfebbA==" + "resolved": "3.53.4", + "contentHash": "KN7jeWqgUPeBRe1FlcpZURzxomuKKEKHmBBQfg+Nx7NkY1LjKhzHvH+3ASkNvhayESE34nMBinL9CV21JfPRJw==" } } }