replaced embedding_state with index_store as a sqlite name

2026-10-17 19:38:00 +00:00 · 2026-08-14 12:37:50 +02:00
1 parent 50aa5c485d
commit 1a263127f5
@@ -0,0 +1,202 @@
+#nullable disable
+
+using Microsoft.EntityFrameworkCore.Infrastructure;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+namespace AIStudio.Tools.Databases.IndexStore.Migrations;
+
+[DbContext(typeof(IndexStoreDbContext))]
+[Migration("20260804000000_InitialRagIndex")]
+public partial class InitialRagIndex : Migration
+{
+    protected override void Up(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.CreateTable(
+            name: "data_sources",
+            columns: table => new
+            {
+                data_source_id = table.Column<string>(type: "TEXT", nullable: false),
+                data_source_name = table.Column<string>(type: "TEXT", nullable: false),
+                data_source_type = table.Column<string>(type: "TEXT", nullable: false),
+                embedding_provider_id = table.Column<string>(type: "TEXT", nullable: false),
+                embedding_signature = table.Column<string>(type: "TEXT", nullable: false),
+                source_hash = table.Column<string>(type: "TEXT", nullable: false, defaultValue: string.Empty),
+                vector_size = table.Column<int>(type: "INTEGER", nullable: false, defaultValue: 0),
+                updated_at_utc = table.Column<string>(type: "TEXT", nullable: false),
+            },
+            constraints: table =>
+            {
+                table.PrimaryKey("PK_data_sources", source => source.data_source_id);
+            });
+
+        migrationBuilder.CreateTable(
+            name: "embedded_files",
+            columns: table => new
+            {
+                parent_file_id = table.Column<string>(type: "TEXT", nullable: false),
+                data_source_id = table.Column<string>(type: "TEXT", nullable: false),
+                absolute_path = table.Column<string>(type: "TEXT", nullable: false, collation: "NOCASE"),
+                file_name = table.Column<string>(type: "TEXT", nullable: false),
+                relative_path = table.Column<string>(type: "TEXT", nullable: false),
+                file_type = table.Column<string>(type: "TEXT", nullable: false),
+                fingerprint = table.Column<string>(type: "TEXT", nullable: false),
+                file_size = table.Column<long>(type: "INTEGER", nullable: false),
+                creation_utc = table.Column<string>(type: "TEXT", nullable: false),
+                last_write_utc = table.Column<string>(type: "TEXT", nullable: false),
+                embedded_at_utc = table.Column<string>(type: "TEXT", nullable: false),
+                chunk_count = table.Column<int>(type: "INTEGER", nullable: false),
+                confidence_level = table.Column<string>(type: "TEXT", nullable: false),
+                confidence_level_rank = table.Column<int>(type: "INTEGER", nullable: false),
+            },
+            constraints: table =>
+            {
+                table.PrimaryKey("PK_embedded_files", file => file.parent_file_id);
+                table.ForeignKey(
+                    name: "FK_embedded_files_data_sources_data_source_id",
+                    column: file => file.data_source_id,
+                    principalTable: "data_sources",
+                    principalColumn: "data_source_id",
+                    onDelete: ReferentialAction.Cascade);
+            });
+
+        migrationBuilder.CreateTable(
+            name: "embedding_chunks",
+            columns: table => new
+            {
+                id = table.Column<int>(type: "INTEGER", nullable: false)
+                    .Annotation("Sqlite:Autoincrement", true),
+                chunk_id = table.Column<string>(type: "TEXT", nullable: false),
+                parent_file_id = table.Column<string>(type: "TEXT", nullable: false),
+                page_number = table.Column<int>(type: "INTEGER", nullable: true),
+                chunk_index = table.Column<int>(type: "INTEGER", nullable: false),
+                chunk_text = table.Column<string>(type: "TEXT", nullable: false),
+                embedded_at_utc = table.Column<string>(type: "TEXT", nullable: false),
+            },
+            constraints: table =>
+            {
+                table.PrimaryKey("PK_embedding_chunks", chunk => chunk.id);
+                table.ForeignKey(
+                    name: "FK_embedding_chunks_embedded_files_parent_file_id",
+                    column: chunk => chunk.parent_file_id,
+                    principalTable: "embedded_files",
+                    principalColumn: "parent_file_id",
+                    onDelete: ReferentialAction.Cascade);
+            });
+
+        migrationBuilder.CreateIndex(
+            name: "idx_embedded_files_absolute_path",
+            table: "embedded_files",
+            column: "absolute_path");
+
+        migrationBuilder.CreateIndex(
+            name: "idx_embedded_files_confidence",
+            table: "embedded_files",
+            column: "confidence_level_rank");
+
+        migrationBuilder.CreateIndex(
+            name: "idx_embedded_files_data_source",
+            table: "embedded_files",
+            column: "data_source_id");
+
+        migrationBuilder.CreateIndex(
+            name: "idx_embedded_files_data_source_absolute_path",
+            table: "embedded_files",
+            columns: ["data_source_id", "absolute_path"],
+            unique: true);
+
+        migrationBuilder.CreateIndex(
+            name: "idx_embedded_files_file_type",
+            table: "embedded_files",
+            column: "file_type");
+
+        migrationBuilder.CreateIndex(
+            name: "idx_embedding_chunks_chunk_id",
+            table: "embedding_chunks",
+            column: "chunk_id",
+            unique: true);
+
+        migrationBuilder.CreateIndex(
+            name: "idx_embedding_chunks_page",
+            table: "embedding_chunks",
+            column: "page_number");
+
+        migrationBuilder.CreateIndex(
+            name: "idx_embedding_chunks_parent_file",
+            table: "embedding_chunks",
+            column: "parent_file_id");
+
+        migrationBuilder.CreateIndex(
+            name: "idx_embedding_chunks_parent_file_chunk_index",
+            table: "embedding_chunks",
+            columns: ["parent_file_id", "chunk_index"],
+            unique: true);
+
+        migrationBuilder.Sql("""
+                             CREATE VIRTUAL TABLE IF NOT EXISTS embedding_chunks_fts
+                                 USING fts5(chunk_id UNINDEXED, file_name, chunk_text);
+
+                             CREATE TRIGGER IF NOT EXISTS embedding_chunks_ai
+                                 AFTER INSERT ON embedding_chunks
+                             BEGIN
+                                 INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
+                                 VALUES (
+                                     new.id,
+                                     new.chunk_id,
+                                     (SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id),
+                                     new.chunk_text);
+                             END;
+
+                             CREATE TRIGGER IF NOT EXISTS embedding_chunks_ad
+                                 AFTER DELETE ON embedding_chunks
+                             BEGIN
+                                 DELETE FROM embedding_chunks_fts
+                                 WHERE rowid = old.id;
+                             END;
+
+                             CREATE TRIGGER IF NOT EXISTS embedding_chunks_au
+                                 AFTER UPDATE ON embedding_chunks
+                             BEGIN
+                                 DELETE FROM embedding_chunks_fts
+                                 WHERE rowid = old.id;
+
+                                 INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
+                                 VALUES (
+                                     new.id,
+                                     new.chunk_id,
+                                     (SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id),
+                                     new.chunk_text);
+                             END;
+
+                             CREATE TRIGGER IF NOT EXISTS embedded_files_file_name_au
+                                 AFTER UPDATE OF file_name ON embedded_files
+                             BEGIN
+                                 DELETE FROM embedding_chunks_fts
+                                 WHERE rowid IN (
+                                     SELECT id
+                                     FROM embedding_chunks
+                                     WHERE parent_file_id = new.parent_file_id
+                                 );
+
+                                 INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
+                                 SELECT id, chunk_id, new.file_name, chunk_text
+                                 FROM embedding_chunks
+                                 WHERE parent_file_id = new.parent_file_id;
+                             END;
+                             """);
+    }
+
+    protected override void Down(MigrationBuilder migrationBuilder)
+    {
+        migrationBuilder.Sql("""
+                             DROP TRIGGER IF EXISTS embedded_files_file_name_au;
+                             DROP TRIGGER IF EXISTS embedding_chunks_au;
+                             DROP TRIGGER IF EXISTS embedding_chunks_ad;
+                             DROP TRIGGER IF EXISTS embedding_chunks_ai;
+                             DROP TABLE IF EXISTS embedding_chunks_fts;
+                             """);
+
+        migrationBuilder.DropTable(name: "embedding_chunks");
+        migrationBuilder.DropTable(name: "embedded_files");
+        migrationBuilder.DropTable(name: "data_sources");
+    }
+}
@@ -0,0 +1,327 @@
+#nullable disable
+
+using Microsoft.EntityFrameworkCore;
+using Microsoft.EntityFrameworkCore.Infrastructure;
+
+namespace AIStudio.Tools.Databases.IndexStore.Migrations;
+
+[DbContext(typeof(IndexStoreDbContext))]
+partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
+{
+    protected override void BuildModel(ModelBuilder modelBuilder)
+    {
+#pragma warning disable 612, 618
+        modelBuilder.HasAnnotation("ProductVersion", "9.0.18");
+        var utcDateTimeOffsetConverter = new IndexStoreDateTimeOffsetConverter();
+
+        modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity =>
+        {
+            entity.Property<string>("DataSourceId")
+                .HasColumnType("TEXT")
+                .HasColumnName("data_source_id");
+
+            entity.Property<string>("DataSourceName")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("data_source_name");
+
+            entity.Property<string>("DataSourceType")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("data_source_type");
+
+            entity.Property<string>("EmbeddingProviderId")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("embedding_provider_id");
+
+            entity.Property<string>("EmbeddingSignature")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("embedding_signature");
+
+            entity.Property<string>("SourceHash")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("source_hash")
+                .HasDefaultValue(string.Empty);
+
+            entity.Property<DateTimeOffset>("UpdatedAtUtc")
+                .HasConversion(utcDateTimeOffsetConverter)
+                .HasColumnType("TEXT")
+                .HasColumnName("updated_at_utc");
+
+            entity.Property<int>("VectorSize")
+                .HasColumnType("INTEGER")
+                .HasColumnName("vector_size")
+                .HasDefaultValue(0);
+
+            entity.HasKey("DataSourceId");
+
+            entity.ToTable("data_sources");
+        });
+
+        modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
+        {
+            entity.Property<string>("ParentFileId")
+                .HasColumnType("TEXT")
+                .HasColumnName("parent_file_id");
+
+            entity.Property<string>("AbsolutePath")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("absolute_path")
+                .UseCollation("NOCASE");
+
+            entity.Property<int>("ChunkCount")
+                .HasColumnType("INTEGER")
+                .HasColumnName("chunk_count");
+
+            entity.Property<string>("ConfidenceLevel")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("confidence_level");
+
+            entity.Property<int>("ConfidenceLevelRank")
+                .HasColumnType("INTEGER")
+                .HasColumnName("confidence_level_rank");
+
+            entity.Property<DateTimeOffset>("CreationUtc")
+                .HasConversion(utcDateTimeOffsetConverter)
+                .HasColumnType("TEXT")
+                .HasColumnName("creation_utc");
+
+            entity.Property<string>("DataSourceId")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("data_source_id");
+
+            entity.Property<DateTimeOffset>("EmbeddedAtUtc")
+                .HasConversion(utcDateTimeOffsetConverter)
+                .HasColumnType("TEXT")
+                .HasColumnName("embedded_at_utc");
+
+            entity.Property<string>("FileName")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("file_name");
+
+            entity.Property<long>("FileSize")
+                .HasColumnType("INTEGER")
+                .HasColumnName("file_size");
+
+            entity.Property<string>("FileType")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("file_type");
+
+            entity.Property<string>("Fingerprint")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("fingerprint");
+
+            entity.Property<DateTimeOffset>("LastWriteUtc")
+                .HasConversion(utcDateTimeOffsetConverter)
+                .HasColumnType("TEXT")
+                .HasColumnName("last_write_utc");
+
+            entity.Property<string>("RelativePath")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("relative_path");
+
+            entity.HasKey("ParentFileId");
+
+            entity.HasIndex("AbsolutePath")
+                .HasDatabaseName("idx_embedded_files_absolute_path");
+
+            entity.HasIndex("ConfidenceLevelRank")
+                .HasDatabaseName("idx_embedded_files_confidence");
+
+            entity.HasIndex("DataSourceId")
+                .HasDatabaseName("idx_embedded_files_data_source");
+
+            entity.HasIndex("DataSourceId", "AbsolutePath")
+                .IsUnique()
+                .HasDatabaseName("idx_embedded_files_data_source_absolute_path");
+
+            entity.HasIndex("FileType")
+                .HasDatabaseName("idx_embedded_files_file_type");
+
+            entity.ToTable("embedded_files");
+        });
+
+        modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateChunkEntity", entity =>
+        {
+            entity.Property<int>("Id")
+                .ValueGeneratedOnAdd()
+                .HasColumnType("INTEGER")
+                .HasColumnName("id")
+                .HasAnnotation("Sqlite:Autoincrement", true);
+
+            entity.Property<string>("ChunkId")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("chunk_id");
+
+            entity.Property<int>("ChunkIndex")
+                .HasColumnType("INTEGER")
+                .HasColumnName("chunk_index");
+
+            entity.Property<string>("ChunkText")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("chunk_text");
+
+            entity.Property<DateTimeOffset>("EmbeddedAtUtc")
+                .HasConversion(utcDateTimeOffsetConverter)
+                .HasColumnType("TEXT")
+                .HasColumnName("embedded_at_utc");
+
+            entity.Property<int?>("PageNumber")
+                .HasColumnType("INTEGER")
+                .HasColumnName("page_number");
+
+            entity.Property<string>("ParentFileId")
+                .IsRequired()
+                .HasColumnType("TEXT")
+                .HasColumnName("parent_file_id");
+
+            entity.HasKey("Id");
+
+            entity.HasIndex("ChunkId")
+                .IsUnique()
+                .HasDatabaseName("idx_embedding_chunks_chunk_id");
+
+            entity.HasIndex("PageNumber")
+                .HasDatabaseName("idx_embedding_chunks_page");
+
+            entity.HasIndex("ParentFileId")
+                .HasDatabaseName("idx_embedding_chunks_parent_file");
+
+            entity.HasIndex("ParentFileId", "ChunkIndex")
+                .IsUnique()
+                .HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index");
+
+            entity.ToTable("embedding_chunks");
+        });
+
+        modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.IndexStoreSearchResultEntity", entity =>
+        {
+            entity.Property<string>("AbsolutePath")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<int>("ChunkCount")
+                .HasColumnType("INTEGER");
+
+            entity.Property<string>("ChunkId")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<int>("ChunkIndex")
+                .HasColumnType("INTEGER");
+
+            entity.Property<string>("ChunkText")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<string>("ConfidenceLevel")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<int>("ConfidenceLevelRank")
+                .HasColumnType("INTEGER");
+
+            entity.Property<DateTimeOffset>("CreationUtc")
+                .HasConversion(utcDateTimeOffsetConverter)
+                .HasColumnType("TEXT");
+
+            entity.Property<string>("DataSourceId")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<string>("DataSourceName")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<string>("DataSourceType")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<DateTimeOffset>("EmbeddedAtUtc")
+                .HasConversion(utcDateTimeOffsetConverter)
+                .HasColumnType("TEXT");
+
+            entity.Property<string>("FileName")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<long>("FileSize")
+                .HasColumnType("INTEGER");
+
+            entity.Property<string>("FileType")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<string>("Fingerprint")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<DateTimeOffset>("LastWriteUtc")
+                .HasConversion(utcDateTimeOffsetConverter)
+                .HasColumnType("TEXT");
+
+            entity.Property<int?>("PageNumber")
+                .HasColumnType("INTEGER");
+
+            entity.Property<string>("ParentFileId")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<string>("RelativePath")
+                .IsRequired()
+                .HasColumnType("TEXT");
+
+            entity.Property<double>("Score")
+                .HasColumnType("REAL");
+
+            entity.HasNoKey();
+
+            entity.ToView("embedding_chunk_search_results");
+        });
+
+        modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
+        {
+            entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", "DataSource")
+                .WithMany("Files")
+                .HasForeignKey("DataSourceId")
+                .OnDelete(DeleteBehavior.Cascade)
+                .IsRequired();
+
+            entity.Navigation("DataSource");
+        });
+
+        modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateChunkEntity", entity =>
+        {
+            entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", "File")
+                .WithMany("Chunks")
+                .HasForeignKey("ParentFileId")
+                .OnDelete(DeleteBehavior.Cascade)
+                .IsRequired();
+
+            entity.Navigation("File");
+        });
+
+        modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity =>
+        {
+            entity.Navigation("Files");
+        });
+
+        modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
+        {
+            entity.Navigation("Chunks");
+        });
+#pragma warning restore 612, 618
+    }
+}