From 186f10cee226528b854331cec06473c55a490863 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Wed, 16 Sep 2026 10:20:28 +0200 Subject: [PATCH] Fixed your documents being indexed again after a confidence level change (#976) --- .../IndexStore/EmbeddingStateFile.cs | 4 +- .../IndexStore/EmbeddingStateFileEntity.cs | 4 -- .../IndexStore/IndexStoreDbContext.cs | 3 - .../IndexStore/IndexStoreSchemaMigrator.cs | 1 + .../IndexStore/IndexStoreSearchResult.cs | 4 +- .../IndexStoreSearchResultEntity.cs | 4 -- .../20260915000000_DropFileConfidenceLevel.cs | 51 +++++++++++++ .../IndexStoreDbContextModelSnapshot.cs | 19 ----- .../SqliteIndexStoreClientImplementation.cs | 10 +-- .../VectorStore/VectorSearchResult.cs | 4 +- .../VectorStore/VectorStoragePoint.cs | 4 +- .../DataSourceEmbeddingService.Files.cs | 31 ++++---- .../Services/DataSourceEmbeddingService.cs | 6 +- .../DataSourceLocalRetrievalService.cs | 12 +--- app/Tests/Tools/EmbeddingSignatureTests.cs | 72 +++++++++++++++++++ runtime/src/qdrant_edge_database.rs | 8 --- 16 files changed, 154 insertions(+), 83 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs create mode 100644 app/Tests/Tools/EmbeddingSignatureTests.cs diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs index cbf268b4..8089ea18 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs @@ -11,6 +11,4 @@ public sealed record EmbeddingStateFile( DateTimeOffset CreationUtc, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, - int ChunkCount, - string ConfidenceLevel, - int ConfidenceLevelRank); + int ChunkCount); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs index bd116ba2..1f33c2e8 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs @@ -26,10 +26,6 @@ internal sealed class EmbeddingStateFileEntity public int ChunkCount { get; set; } - public string ConfidenceLevel { get; set; } = string.Empty; - - public int ConfidenceLevelRank { get; set; } - public EmbeddingStateDataSourceEntity? DataSource { get; set; } public List Chunks { get; set; } = []; diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs index d07b977b..870837d5 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs @@ -67,13 +67,10 @@ internal sealed class IndexStoreDbContext(DbContextOptions entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired(); entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired(); entity.Property(file => file.ChunkCount).HasColumnName("chunk_count"); - entity.Property(file => file.ConfidenceLevel).HasColumnName("confidence_level").IsRequired(); - entity.Property(file => file.ConfidenceLevelRank).HasColumnName("confidence_level_rank"); entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source"); entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path"); entity.HasIndex(file => file.FileType).HasDatabaseName("idx_embedded_files_file_type"); - entity.HasIndex(file => file.ConfidenceLevelRank).HasDatabaseName("idx_embedded_files_confidence"); entity.HasIndex(file => new { file.DataSourceId, file.AbsolutePath }).HasDatabaseName("idx_embedded_files_data_source_absolute_path").IsUnique(); entity diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs index f684e6db..7ae6e4f2 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs @@ -8,6 +8,7 @@ internal static class IndexStoreSchemaMigrator { [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialRagIndex))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))] + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))] public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token) { await context.Database.MigrateAsync(token); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs index 0ae8d884..58d1d767 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs @@ -19,6 +19,4 @@ public sealed record IndexStoreSearchResult( DateTimeOffset CreationUtc, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, - int ChunkCount, - string ConfidenceLevel, - int ConfidenceLevelRank); + int ChunkCount); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs index 1385c9f5..51ea7796 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs @@ -39,8 +39,4 @@ internal sealed class IndexStoreSearchResultEntity public DateTimeOffset EmbeddedAtUtc { get; set; } public int ChunkCount { get; set; } - - public string ConfidenceLevel { get; set; } = string.Empty; - - public int ConfidenceLevelRank { get; set; } } diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs new file mode 100644 index 00000000..84fa066d --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20260915000000_DropFileConfidenceLevel.cs @@ -0,0 +1,51 @@ +#nullable disable + +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace AIStudio.Tools.Databases.IndexStore.Migrations; + +/// +/// Drops the copy of the data source confidence level which every indexed file carried. +/// +/// +/// The confidence level is what a data source asks of a provider. It is a property of the data +/// source, it is enforced live before anything is indexed or answered, and it changes no vector. +/// Keeping a copy per file only meant the index had to be thrown away whenever the setting changed. +/// +[DbContext(typeof(IndexStoreDbContext))] +[Migration("20260915000000_DropFileConfidenceLevel")] +public partial class DropFileConfidenceLevel : Migration +{ + /// + /// The columns go through raw SQL instead of DropColumn on purpose. The SQLite provider answers + /// DropColumn by rebuilding the table, and a rebuild drops the table the trigger + /// embedded_files_file_name_au hangs on, which would silently stop the full-text index from + /// following a renamed file. A native ALTER TABLE ... DROP COLUMN leaves the table itself alone. + /// It does refuse a column an index names, so the index has to go first. + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.DropIndex( + name: "idx_embedded_files_confidence", + table: "embedded_files"); + + migrationBuilder.Sql(""" + ALTER TABLE embedded_files DROP COLUMN confidence_level; + ALTER TABLE embedded_files DROP COLUMN confidence_level_rank; + """); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql(""" + ALTER TABLE embedded_files ADD COLUMN confidence_level TEXT NOT NULL DEFAULT ''; + ALTER TABLE embedded_files ADD COLUMN confidence_level_rank INTEGER NOT NULL DEFAULT 0; + """); + + migrationBuilder.CreateIndex( + name: "idx_embedded_files_confidence", + table: "embedded_files", + column: "confidence_level_rank"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs index cf9015d6..3d8336c1 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs @@ -80,15 +80,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot .HasColumnType("INTEGER") .HasColumnName("chunk_count"); - entity.Property("ConfidenceLevel") - .IsRequired() - .HasColumnType("TEXT") - .HasColumnName("confidence_level"); - - entity.Property("ConfidenceLevelRank") - .HasColumnType("INTEGER") - .HasColumnName("confidence_level_rank"); - entity.Property("CreationUtc") .HasConversion(utcDateTimeOffsetConverter) .HasColumnType("TEXT") @@ -138,9 +129,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot entity.HasIndex("AbsolutePath") .HasDatabaseName("idx_embedded_files_absolute_path"); - entity.HasIndex("ConfidenceLevelRank") - .HasDatabaseName("idx_embedded_files_confidence"); - entity.HasIndex("DataSourceId") .HasDatabaseName("idx_embedded_files_data_source"); @@ -278,13 +266,6 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot .IsRequired() .HasColumnType("TEXT"); - entity.Property("ConfidenceLevel") - .IsRequired() - .HasColumnType("TEXT"); - - entity.Property("ConfidenceLevelRank") - .HasColumnType("INTEGER"); - entity.Property("CreationUtc") .HasConversion(utcDateTimeOffsetConverter) .HasColumnType("TEXT"); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs index 1e8d44cc..513bc969 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs @@ -302,9 +302,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat f.creation_utc AS CreationUtc, f.last_write_utc AS LastWriteUtc, c.embedded_at_utc AS EmbeddedAtUtc, - f.chunk_count AS ChunkCount, - f.confidence_level AS ConfidenceLevel, - f.confidence_level_rank AS ConfidenceLevelRank + f.chunk_count AS ChunkCount FROM embedding_chunks_fts JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid JOIN embedded_files f ON f.parent_file_id = c.parent_file_id @@ -394,8 +392,6 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat fileEntity.LastWriteUtc = file.LastWriteUtc; fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc; fileEntity.ChunkCount = file.ChunkCount; - fileEntity.ConfidenceLevel = file.ConfidenceLevel; - fileEntity.ConfidenceLevelRank = file.ConfidenceLevelRank; } private static void ApplyPermanentFailure(IndexingFailureEntity failureEntity, string dataSourceId, PermanentIndexingFailure failure) @@ -445,9 +441,7 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat result.CreationUtc, result.LastWriteUtc, result.EmbeddedAtUtc, - result.ChunkCount, - result.ConfidenceLevel, - result.ConfidenceLevelRank); + result.ChunkCount); private static string BuildFtsQuery(string query) { diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs index 693b2527..c0b127de 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorSearchResult.cs @@ -19,6 +19,4 @@ public sealed record VectorSearchResult( string Fingerprint, string CreationUtc, string LastWriteUtc, - string EmbeddedAtUtc, - string ConfidenceLevel, - int ConfidenceLevelRank); + string EmbeddedAtUtc); diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs index 042824d9..87a6f334 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs @@ -19,6 +19,4 @@ public sealed record VectorStoragePoint( string Fingerprint, DateTimeOffset CreationUtc, DateTimeOffset LastWriteUtc, - DateTimeOffset EmbeddedAtUtc, - string ConfidenceLevel, - int ConfidenceLevelRank); + DateTimeOffset EmbeddedAtUtc); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 0810f1b4..6b735487 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -58,7 +58,7 @@ public sealed partial class DataSourceEmbeddingService private sealed record EmbeddingChunkDraft(string ChunkId, string Text, int ChunkIndex, int? PageNumber); - private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength); + internal sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength); private sealed record ChunkingStrategy(string Name, IReadOnlyList Rules); @@ -986,7 +986,23 @@ public sealed partial class DataSourceEmbeddingService } } - private string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions) + /// + /// Describes how the vectors of a data source were made. + /// + /// + /// What appears here decides when stored embeddings are thrown away: a signature differing from + /// the persisted one drops the whole index and builds it again. So it names the embedding model, + /// where it runs, how the text was cut for it, and the chunk metadata version — the things a + /// vector actually depends on. + /// + /// The confidence level a data source asks of a provider is deliberately not among them. It + /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks + /// it against the participating chat providers and against the embedding provider, and this + /// service checks it again before each indexing run. It was part of this signature once, which + /// re-embedded every file of a data source whenever somebody raised or lowered it — real money + /// at a cloud embedding provider, for nothing. + /// + internal static string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions) { return string.Join('|', CHUNK_METADATA_VERSION, @@ -997,7 +1013,6 @@ public sealed partial class DataSourceEmbeddingService embeddingProvider.Hostname, embeddingProvider.TokenizerPath, embeddingProvider.EffectiveTokenLimit, - GetDataSourceConfidenceLevel(dataSource).ToString(), dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0, dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH, chunkingOptions.MaxChunkTokenLength, @@ -1101,7 +1116,6 @@ public sealed partial class DataSourceEmbeddingService { file.Refresh(); var absolutePath = Path.GetFullPath(file.FullName); - var confidenceLevel = GetDataSourceConfidenceLevel(dataSource); return new( this.CreateParentFileId(dataSource.Id, absolutePath), absolutePath, @@ -1113,9 +1127,7 @@ public sealed partial class DataSourceEmbeddingService file.Exists ? new DateTimeOffset(file.CreationTimeUtc) : DateTimeOffset.UnixEpoch, file.Exists ? new DateTimeOffset(file.LastWriteTimeUtc) : DateTimeOffset.UnixEpoch, embeddedAtUtc, - chunkCount, - confidenceLevel.ToString(), - (int)confidenceLevel); + chunkCount); } private IReadOnlyList CreateEmbeddingStateChunks(EmbeddingStateFile parentFile, IReadOnlyList batch, DateTimeOffset embeddedAtUtc) @@ -1131,11 +1143,6 @@ public sealed partial class DataSourceEmbeddingService .ToList(); } - private static ConfidenceLevel GetDataSourceConfidenceLevel(IDataSource dataSource) => - dataSource is not IInternalDataSource internalDataSource || internalDataSource.ConfidenceLevel is ConfidenceLevel.NONE - ? ConfidenceLevel.UNKNOWN - : internalDataSource.ConfidenceLevel; - private static string GetFileType(FileInfo file) { var extension = file.Extension.TrimStart('.').ToLowerInvariant(); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 90fe8df1..db4ca76f 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -1030,9 +1030,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM fingerprint, parentFile.CreationUtc, parentFile.LastWriteUtc, - embeddedAtUtc, - parentFile.ConfidenceLevel, - parentFile.ConfidenceLevelRank)).ToList(); + embeddedAtUtc)).ToList(); await vectorStore.InsertEmbedding(collectionName, points, token); } @@ -1237,7 +1235,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM CancellationToken token) { var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider); - var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); + var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); var manifest = await indexStore.GetManifestAsync(dataSource.Id, token); logger.LogInformation( diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index c8d12b68..7e5f2645 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -52,9 +52,7 @@ public sealed class DataSourceLocalRetrievalService( int ChunkIndex, string Text, double Score, - int Rank, - string ConfidenceLevel, - int ConfidenceLevelRank); + int Rank); // ReSharper restore NotAccessedPositionalProperty.Local public Task> RetrieveDataAsync(DataSourceLocalFile dataSource, IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) => @@ -354,9 +352,7 @@ public sealed class DataSourceLocalRetrievalService( result.ChunkIndex, result.Text, result.Score, - rank, - result.ConfidenceLevel, - result.ConfidenceLevelRank); + rank); private static LocalRetrievalHit FromBm25Result(IndexStoreSearchResult result, int rank) => new( @@ -374,9 +370,7 @@ public sealed class DataSourceLocalRetrievalService( result.ChunkIndex, result.ChunkText, result.Score, - rank, - result.ConfidenceLevel, - result.ConfidenceLevelRank); + rank); private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit) { diff --git a/app/Tests/Tools/EmbeddingSignatureTests.cs b/app/Tests/Tools/EmbeddingSignatureTests.cs new file mode 100644 index 00000000..4a0c6eac --- /dev/null +++ b/app/Tests/Tools/EmbeddingSignatureTests.cs @@ -0,0 +1,72 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.Services; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks what makes the stored embeddings of a data source invalid. +/// +/// +/// The embedding signature decides whether an index survives: when it differs from the one persisted +/// for a data source, everything stored is thrown away and embedded again. That is the right answer +/// for anything a vector depends on, and an expensive mistake for everything else. The confidence +/// level a data source asks of a provider used to be part of it, so changing that one setting +/// re-embedded every file of the source — at a cloud embedding provider, for real money and no gain. +/// +[TestFixture] +public sealed class EmbeddingSignatureTests +{ + [Test] + public void ChangingTheConfidenceLevelKeepsTheStoredEmbeddings() + { + var low = DataSource(ConfidenceLevel.LOW); + var high = DataSource(ConfidenceLevel.HIGH); + + Assert.That(Signature(high), Is.EqualTo(Signature(low)), "The confidence level changes no vector, so the stored index stays valid and nothing is embedded again."); + } + + [Test] + public void ChangingTheChunkSizeDropsTheStoredEmbeddings() + { + var small = DataSource(ConfidenceLevel.LOW) with { MaxChunkTokenLength = 512 }; + var large = DataSource(ConfidenceLevel.LOW) with { MaxChunkTokenLength = 1024 }; + + Assert.That(Signature(large), Is.Not.EqualTo(Signature(small)), "Other chunk boundaries mean other vectors, so the index has to be built again."); + } + + [Test] + public void ChangingTheEmbeddingModelDropsTheStoredEmbeddings() + { + var dataSource = DataSource(ConfidenceLevel.LOW); + + Assert.That( + Signature(dataSource, EmbeddingProviderFor("text-embedding-3-large")), + Is.Not.EqualTo(Signature(dataSource, EmbeddingProviderFor("text-embedding-3-small"))), + "Another model means another vector space, so nothing stored may be kept."); + } + + private static string Signature(DataSourceLocalDirectory dataSource, EmbeddingProvider? embeddingProvider = null) => + DataSourceEmbeddingService.BuildEmbeddingSignature( + dataSource, + embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"), + new(512, 100)); + + private static DataSourceLocalDirectory DataSource(ConfidenceLevel confidenceLevel) => new() + { + Num = 1, + Id = "6f1d6a4e-6a5e-4c62-9a4f-0f2d2c8b7a11", + Name = "Test data", + Description = "Documents used by the tests.", + Type = DataSourceType.LOCAL_DIRECTORY, + EmbeddingId = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", + MaxChunkTokenLength = 512, + ChunkOverlapTokenLength = 100, + ConfidenceLevel = confidenceLevel, + Path = "/tmp/test-data", + }; + + private static EmbeddingProvider EmbeddingProviderFor(string modelId) => + new(1, "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "Test embeddings", LLMProviders.OPEN_AI, new(modelId, modelId)); +} \ No newline at end of file diff --git a/runtime/src/qdrant_edge_database.rs b/runtime/src/qdrant_edge_database.rs index 659b72ec..8b54c9c2 100644 --- a/runtime/src/qdrant_edge_database.rs +++ b/runtime/src/qdrant_edge_database.rs @@ -86,8 +86,6 @@ pub struct QdrantEdgeStoragePoint { pub creation_utc: String, pub last_write_utc: String, pub embedded_at_utc: String, - pub confidence_level: String, - pub confidence_level_rank: i32, } #[derive(Deserialize)] @@ -159,8 +157,6 @@ pub struct QdrantEdgeSearchResult { pub creation_utc: String, pub last_write_utc: String, pub embedded_at_utc: String, - pub confidence_level: String, - pub confidence_level_rank: i32, } #[derive(Clone, Serialize)] @@ -758,8 +754,6 @@ fn to_qdrant_edge_point(point: QdrantEdgeStoragePoint) -> QdrantEdgeResult QdrantEdgeSearchResult { creation_utc: payload_string(&payload, "creation_utc"), last_write_utc: payload_string(&payload, "last_write_utc"), embedded_at_utc: payload_string(&payload, "embedded_at_utc"), - confidence_level: payload_string(&payload, "confidence_level"), - confidence_level_rank: payload_i32(&payload, "confidence_level_rank").unwrap_or_default(), } }