diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 6cc91238..e6dc8450 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -1012,6 +1012,12 @@ public sealed partial class DataSourceEmbeddingService /// another lands on the identical path, while moving the data directory changes every path /// without changing a single tokenizer. /// + /// The chunk settings enter only as what they amount to, never as what somebody typed. A data + /// source storing 0 means "follow the embedding provider", and writing that provider's own limit + /// into the field changes nothing about how the text is cut. Carrying the typed numbers as well + /// made that a different signature, so opening the expert settings of a data source — which + /// fills an empty limit with the provider's — threw the whole index away for nothing. + /// /// The confidence level a data source asks of a provider is deliberately not among them. It /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks /// it against the participating chat providers and against the embedding provider, and this @@ -1031,8 +1037,6 @@ public sealed partial class DataSourceEmbeddingService embeddingProvider.HFInferenceProvider, embeddingProvider.TokenizerFingerprint, embeddingProvider.EffectiveTokenLimit, - dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0, - dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH, chunkingOptions.MaxChunkTokenLength, chunkingOptions.OverlapTokenLength); } diff --git a/app/Tests/Tools/EmbeddingChangeImpactTests.cs b/app/Tests/Tools/EmbeddingChangeImpactTests.cs index 673c7083..e9301df1 100644 --- a/app/Tests/Tools/EmbeddingChangeImpactTests.cs +++ b/app/Tests/Tools/EmbeddingChangeImpactTests.cs @@ -99,6 +99,54 @@ public sealed class EmbeddingChangeImpactTests }); } + /// + /// Writing out what a data source already follows must not cost it its index. + /// + /// + /// A token limit of 0 means "follow the embedding provider", and opening the expert settings of a + /// data source fills that empty field with exactly the provider's limit. Both are the same cut, + /// so nobody may be asked about it -- and above all, nothing may be re-embedded for it. Somebody + /// who only wanted to change how many matches an answer may use paid for a full rebuild. + /// + [Test] + public void SpellingOutWhatTheProviderAlreadyDictatesKeepsTheStoredIndex() + { + var embeddingProvider = StoredEmbeddingProvider() with { TokenLimit = 8192 }; + var followingTheProvider = StoredDataSource() with { MaxChunkTokenLength = 0 }; + + Assert.Multiple(() => + { + Assert.That( + EmbeddingChangeImpact.AffectsStoredIndex(embeddingProvider, followingTheProvider, followingTheProvider with { MaxChunkTokenLength = 8192 }), + Is.False, + "The provider limit typed into the field is the cut the data source already had."); + + Assert.That( + EmbeddingChangeImpact.AffectsStoredIndex(embeddingProvider, followingTheProvider, followingTheProvider with { MaxChunkTokenLength = 4096 }), + Is.True, + "Anything below the provider limit really does cut the text elsewhere."); + }); + } + + /// + /// An overlap larger than the chunk is capped, so several of them are the same cut. + /// + /// + /// The same reasoning as for the token limit: what counts is where the text is cut, not what + /// somebody typed into the field. + /// + [Test] + public void AnOverlapWhichIsCappedAnywayKeepsTheStoredIndex() + { + var embeddingProvider = StoredEmbeddingProvider(); + var stored = StoredDataSource() with { MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 600 }; + + Assert.That( + EmbeddingChangeImpact.AffectsStoredIndex(embeddingProvider, stored, stored with { ChunkOverlapTokenLength = 700 }), + Is.False, + "Both overlaps are capped to the chunk size, so the text is cut identically."); + } + [Test] public void HarmlessDataSourceEditsKeepTheStoredIndex() { diff --git a/app/Tests/Tools/EmbeddingSignatureTests.cs b/app/Tests/Tools/EmbeddingSignatureTests.cs index 804eba41..94513cfa 100644 --- a/app/Tests/Tools/EmbeddingSignatureTests.cs +++ b/app/Tests/Tools/EmbeddingSignatureTests.cs @@ -92,15 +92,24 @@ public sealed class EmbeddingSignatureTests { Assert.That( Signature(DataSource(ConfidenceLevel.LOW)), - Is.EqualTo("2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100|512|100"), + Is.EqualTo("2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100"), "Reordering or extending the signature throws away every index anybody has. This test makes that a decision somebody takes rather than something which happens on the way past."); } + /// + /// Builds the signature the way an indexing run does, working the chunking out along the way. + /// + /// + /// Handing in fixed chunking options instead would hide exactly what these tests are here for: + /// the signature would then no longer notice a data source being cut differently. + /// + /// The data source to build the signature for. + /// The embedding provider, or the test default. + /// The signature of that pairing. private static string Signature(DataSourceLocalDirectory dataSource, EmbeddingProvider? embeddingProvider = null) => DataSourceEmbeddingService.BuildEmbeddingSignature( dataSource, - embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"), - new(512, 100)); + embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small")); private static DataSourceLocalDirectory DataSource(ConfidenceLevel confidenceLevel) => new() {