MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-11-05 11:28:36 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added a way to read what the AI received from your sources (#1100)

Browse Source 
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Paul KoudelkaandThorsten Sommer authored and GitHub committed 2026-10-26 10:00:01 +01:00
1 parent e0d8e18f34
commit cbc4ce5413
229 files changed
+12739 -1269
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20261022000000_ChunkMetadata.cs
+83
View File
@@ -0,0 +1,83 @@
#nullable disable
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
/// <summary>
/// Adds what a chunk and its file know beyond their text: where a chunk is, the hash of its text,
/// its generation, and the hash of the content of its file.
/// </summary>
/// <remarks>
/// Each chunk gains:
/// <list type="bullet">
/// <item>the page it ends on, next to the page it starts on, and its sheet and rows, its lines, and
/// the attachment of a mail it is from, cf. SourceLocation;</item>
/// <item>the hash of its text, so a document read again keeps the vector of every chunk whose text
/// did not change, cf. ChunkTextHash;</item>
/// <item>its generation, so a document read again gets its new chunks next to the old ones and
/// keeps the old ones until the new ones are complete, cf. IndexedDocumentIds.CreateChunkIdSource.
/// Two generations may hold the same position, so the unique index over parent and position
/// gains the generation as well.</item>
/// </list>
/// Each file gains the hash of its content, so a file saved again without changes needs neither
/// extraction nor embedding, and the generation it is searched by.<br/><br/>
/// Chunks and files stored before keep NULL, which reads as unknown, and generation 0, which is
/// the generation every one of them has: an index built before has one generation per document.
/// They gain the rest once their document is indexed again, so no index has to be rebuilt, cf.
/// DataSourceEmbeddingService.CHUNK_METADATA_VERSION.
/// </remarks>
[DbContext(typeof(IndexStoreDbContext))]
[Migration("20261022000000_ChunkMetadata")]
public partial class ChunkMetadata : Migration
{
/// <remarks>
/// Both directions go through raw SQL, like MediaTranscriptKey: the SQLite provider answers
/// DropColumn by rebuilding the table, which would drop the triggers that keep the full-text
/// index of the chunks in step, and dropping the old embedded_files table would let the cascade
/// of the foreign key in embedding_chunks take every chunk with it. Adding a column and
/// replacing an index are native in SQLite and leave the rows alone.
/// </remarks>
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN end_page_number INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN sheet_name TEXT NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN start_row INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN end_row INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN start_line INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN end_line INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN attachment_position INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN text_hash TEXT NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN generation INTEGER NOT NULL DEFAULT 0;");
migrationBuilder.Sql("ALTER TABLE embedded_files ADD COLUMN content_hash TEXT NULL;");
migrationBuilder.Sql("ALTER TABLE embedded_files ADD COLUMN generation INTEGER NOT NULL DEFAULT 0;");
migrationBuilder.Sql("DROP INDEX IF EXISTS idx_embedding_chunks_parent_file_chunk_index;");
migrationBuilder.Sql("CREATE UNIQUE INDEX idx_embedding_chunks_parent_file_generation_chunk_index ON embedding_chunks (parent_file_id, generation, chunk_index);");
}
/// <remarks>
/// Going back needs every document to have a single generation again, as it has whenever no
/// indexing run is under way and no failed attempt is kept.
/// </remarks>
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql("DROP INDEX IF EXISTS idx_embedding_chunks_parent_file_generation_chunk_index;");
migrationBuilder.Sql("CREATE UNIQUE INDEX idx_embedding_chunks_parent_file_chunk_index ON embedding_chunks (parent_file_id, chunk_index);");
migrationBuilder.Sql("ALTER TABLE embedded_files DROP COLUMN generation;");
migrationBuilder.Sql("ALTER TABLE embedded_files DROP COLUMN content_hash;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN generation;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN text_hash;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN attachment_position;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN end_line;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN start_line;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN end_row;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN start_row;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN sheet_name;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN end_page_number;");
}
}
app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs
+67 -2
View File
@@ -75,6 +75,10 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnType("INTEGER")
.HasColumnName("chunk_count");
entity.Property<string>("ContentHash")
.HasColumnType("TEXT")
.HasColumnName("content_hash");
entity.Property<DateTimeOffset>("CreationUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
@@ -109,6 +113,10 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnType("TEXT")
.HasColumnName("fingerprint");
entity.Property<int>("Generation")
.HasColumnType("INTEGER")
.HasColumnName("generation");
entity.Property<DateTimeOffset>("LastWriteUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
@@ -149,6 +157,10 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnName("id")
.HasAnnotation("Sqlite:Autoincrement", true);
entity.Property<int?>("AttachmentPosition")
.HasColumnType("INTEGER")
.HasColumnName("attachment_position");
entity.Property<string>("ChunkId")
.IsRequired()
.HasColumnType("TEXT")
@@ -168,6 +180,22 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnType("TEXT")
.HasColumnName("embedded_at_utc");
entity.Property<int?>("EndLine")
.HasColumnType("INTEGER")
.HasColumnName("end_line");
entity.Property<int?>("EndPageNumber")
.HasColumnType("INTEGER")
.HasColumnName("end_page_number");
entity.Property<int?>("EndRow")
.HasColumnType("INTEGER")
.HasColumnName("end_row");
entity.Property<int>("Generation")
.HasColumnType("INTEGER")
.HasColumnName("generation");
entity.Property<int?>("PageNumber")
.HasColumnType("INTEGER")
.HasColumnName("page_number");
@@ -177,6 +205,22 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnType("TEXT")
.HasColumnName("parent_file_id");
entity.Property<string>("SheetName")
.HasColumnType("TEXT")
.HasColumnName("sheet_name");
entity.Property<int?>("StartLine")
.HasColumnType("INTEGER")
.HasColumnName("start_line");
entity.Property<int?>("StartRow")
.HasColumnType("INTEGER")
.HasColumnName("start_row");
entity.Property<string>("TextHash")
.HasColumnType("TEXT")
.HasColumnName("text_hash");
entity.HasKey("Id");
entity.HasIndex("ChunkId")
@@ -189,9 +233,9 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
entity.HasIndex("ParentFileId")
.HasDatabaseName("idx_embedding_chunks_parent_file");
entity.HasIndex("ParentFileId", "ChunkIndex")
entity.HasIndex("ParentFileId", "Generation", "ChunkIndex")
.IsUnique()
.HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index");
.HasDatabaseName("idx_embedding_chunks_parent_file_generation_chunk_index");
entity.ToTable("embedding_chunks");
});
@@ -251,6 +295,9 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.IsRequired()
.HasColumnType("TEXT");
entity.Property<int?>("AttachmentPosition")
.HasColumnType("INTEGER");
entity.Property<int>("ChunkCount")
.HasColumnType("INTEGER");
@@ -281,6 +328,15 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT");
entity.Property<int?>("EndLine")
.HasColumnType("INTEGER");
entity.Property<int?>("EndPageNumber")
.HasColumnType("INTEGER");
entity.Property<int?>("EndRow")
.HasColumnType("INTEGER");
entity.Property<string>("FileName")
.IsRequired()
.HasColumnType("TEXT");
@@ -314,6 +370,15 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
entity.Property<double>("Score")
.HasColumnType("REAL");
entity.Property<string>("SheetName")
.HasColumnType("TEXT");
entity.Property<int?>("StartLine")
.HasColumnType("INTEGER");
entity.Property<int?>("StartRow")
.HasColumnType("INTEGER");
entity.HasNoKey();
entity.ToView("embedding_chunk_search_results");