From 14ef111a7945207c0d4e148044cd3892090fe0bf Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Tue, 15 Sep 2026 21:02:48 +0200 Subject: [PATCH] Fixed data sources being re-indexed after a confidence level change --- .../DataSourceEmbeddingService.Files.cs | 21 +++++- .../Services/DataSourceEmbeddingService.cs | 2 +- .../DataSourceLocalRetrievalService.cs | 12 +--- app/Tests/Tools/EmbeddingSignatureTests.cs | 72 +++++++++++++++++++ 4 files changed, 94 insertions(+), 13 deletions(-) create mode 100644 app/Tests/Tools/EmbeddingSignatureTests.cs diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 0810f1b4..6033a283 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -58,7 +58,7 @@ public sealed partial class DataSourceEmbeddingService private sealed record EmbeddingChunkDraft(string ChunkId, string Text, int ChunkIndex, int? PageNumber); - private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength); + internal sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength); private sealed record ChunkingStrategy(string Name, IReadOnlyList Rules); @@ -986,7 +986,23 @@ public sealed partial class DataSourceEmbeddingService } } - private string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions) + /// + /// Describes how the vectors of a data source were made. + /// + /// + /// What appears here decides when stored embeddings are thrown away: a signature differing from + /// the persisted one drops the whole index and builds it again. So it names the embedding model, + /// where it runs, how the text was cut for it, and the chunk metadata version — the things a + /// vector actually depends on. + /// + /// The confidence level a data source asks of a provider is deliberately not among them. It + /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks + /// it against the participating chat providers and against the embedding provider, and this + /// service checks it again before each indexing run. It was part of this signature once, which + /// re-embedded every file of a data source whenever somebody raised or lowered it — real money + /// at a cloud embedding provider, for nothing. + /// + internal static string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions) { return string.Join('|', CHUNK_METADATA_VERSION, @@ -997,7 +1013,6 @@ public sealed partial class DataSourceEmbeddingService embeddingProvider.Hostname, embeddingProvider.TokenizerPath, embeddingProvider.EffectiveTokenLimit, - GetDataSourceConfidenceLevel(dataSource).ToString(), dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0, dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH, chunkingOptions.MaxChunkTokenLength, diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 90fe8df1..bfe3bc5d 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -1237,7 +1237,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM CancellationToken token) { var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider); - var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); + var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); var manifest = await indexStore.GetManifestAsync(dataSource.Id, token); logger.LogInformation( diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index c8d12b68..7e5f2645 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -52,9 +52,7 @@ public sealed class DataSourceLocalRetrievalService( int ChunkIndex, string Text, double Score, - int Rank, - string ConfidenceLevel, - int ConfidenceLevelRank); + int Rank); // ReSharper restore NotAccessedPositionalProperty.Local public Task> RetrieveDataAsync(DataSourceLocalFile dataSource, IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) => @@ -354,9 +352,7 @@ public sealed class DataSourceLocalRetrievalService( result.ChunkIndex, result.Text, result.Score, - rank, - result.ConfidenceLevel, - result.ConfidenceLevelRank); + rank); private static LocalRetrievalHit FromBm25Result(IndexStoreSearchResult result, int rank) => new( @@ -374,9 +370,7 @@ public sealed class DataSourceLocalRetrievalService( result.ChunkIndex, result.ChunkText, result.Score, - rank, - result.ConfidenceLevel, - result.ConfidenceLevelRank); + rank); private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit) { diff --git a/app/Tests/Tools/EmbeddingSignatureTests.cs b/app/Tests/Tools/EmbeddingSignatureTests.cs new file mode 100644 index 00000000..4a0c6eac --- /dev/null +++ b/app/Tests/Tools/EmbeddingSignatureTests.cs @@ -0,0 +1,72 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.Services; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks what makes the stored embeddings of a data source invalid. +/// +/// +/// The embedding signature decides whether an index survives: when it differs from the one persisted +/// for a data source, everything stored is thrown away and embedded again. That is the right answer +/// for anything a vector depends on, and an expensive mistake for everything else. The confidence +/// level a data source asks of a provider used to be part of it, so changing that one setting +/// re-embedded every file of the source — at a cloud embedding provider, for real money and no gain. +/// +[TestFixture] +public sealed class EmbeddingSignatureTests +{ + [Test] + public void ChangingTheConfidenceLevelKeepsTheStoredEmbeddings() + { + var low = DataSource(ConfidenceLevel.LOW); + var high = DataSource(ConfidenceLevel.HIGH); + + Assert.That(Signature(high), Is.EqualTo(Signature(low)), "The confidence level changes no vector, so the stored index stays valid and nothing is embedded again."); + } + + [Test] + public void ChangingTheChunkSizeDropsTheStoredEmbeddings() + { + var small = DataSource(ConfidenceLevel.LOW) with { MaxChunkTokenLength = 512 }; + var large = DataSource(ConfidenceLevel.LOW) with { MaxChunkTokenLength = 1024 }; + + Assert.That(Signature(large), Is.Not.EqualTo(Signature(small)), "Other chunk boundaries mean other vectors, so the index has to be built again."); + } + + [Test] + public void ChangingTheEmbeddingModelDropsTheStoredEmbeddings() + { + var dataSource = DataSource(ConfidenceLevel.LOW); + + Assert.That( + Signature(dataSource, EmbeddingProviderFor("text-embedding-3-large")), + Is.Not.EqualTo(Signature(dataSource, EmbeddingProviderFor("text-embedding-3-small"))), + "Another model means another vector space, so nothing stored may be kept."); + } + + private static string Signature(DataSourceLocalDirectory dataSource, EmbeddingProvider? embeddingProvider = null) => + DataSourceEmbeddingService.BuildEmbeddingSignature( + dataSource, + embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"), + new(512, 100)); + + private static DataSourceLocalDirectory DataSource(ConfidenceLevel confidenceLevel) => new() + { + Num = 1, + Id = "6f1d6a4e-6a5e-4c62-9a4f-0f2d2c8b7a11", + Name = "Test data", + Description = "Documents used by the tests.", + Type = DataSourceType.LOCAL_DIRECTORY, + EmbeddingId = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", + MaxChunkTokenLength = 512, + ChunkOverlapTokenLength = 100, + ConfidenceLevel = confidenceLevel, + Path = "/tmp/test-data", + }; + + private static EmbeddingProvider EmbeddingProviderFor(string modelId) => + new(1, "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "Test embeddings", LLMProviders.OPEN_AI, new(modelId, modelId)); +} \ No newline at end of file