diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
index 6cc91238..e6dc8450 100644
--- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
+++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
@@ -1012,6 +1012,12 @@ public sealed partial class DataSourceEmbeddingService
/// another lands on the identical path, while moving the data directory changes every path
/// without changing a single tokenizer.
///
+ /// The chunk settings enter only as what they amount to, never as what somebody typed. A data
+ /// source storing 0 means "follow the embedding provider", and writing that provider's own limit
+ /// into the field changes nothing about how the text is cut. Carrying the typed numbers as well
+ /// made that a different signature, so opening the expert settings of a data source — which
+ /// fills an empty limit with the provider's — threw the whole index away for nothing.
+ ///
/// The confidence level a data source asks of a provider is deliberately not among them. It
/// changes no vector, and it is enforced live on every request anyway: DataSourceService checks
/// it against the participating chat providers and against the embedding provider, and this
@@ -1031,8 +1037,6 @@ public sealed partial class DataSourceEmbeddingService
embeddingProvider.HFInferenceProvider,
embeddingProvider.TokenizerFingerprint,
embeddingProvider.EffectiveTokenLimit,
- dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
- dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH,
chunkingOptions.MaxChunkTokenLength,
chunkingOptions.OverlapTokenLength);
}
diff --git a/app/Tests/Tools/EmbeddingChangeImpactTests.cs b/app/Tests/Tools/EmbeddingChangeImpactTests.cs
index 673c7083..e9301df1 100644
--- a/app/Tests/Tools/EmbeddingChangeImpactTests.cs
+++ b/app/Tests/Tools/EmbeddingChangeImpactTests.cs
@@ -99,6 +99,54 @@ public sealed class EmbeddingChangeImpactTests
});
}
+ ///
+ /// Writing out what a data source already follows must not cost it its index.
+ ///
+ ///
+ /// A token limit of 0 means "follow the embedding provider", and opening the expert settings of a
+ /// data source fills that empty field with exactly the provider's limit. Both are the same cut,
+ /// so nobody may be asked about it -- and above all, nothing may be re-embedded for it. Somebody
+ /// who only wanted to change how many matches an answer may use paid for a full rebuild.
+ ///
+ [Test]
+ public void SpellingOutWhatTheProviderAlreadyDictatesKeepsTheStoredIndex()
+ {
+ var embeddingProvider = StoredEmbeddingProvider() with { TokenLimit = 8192 };
+ var followingTheProvider = StoredDataSource() with { MaxChunkTokenLength = 0 };
+
+ Assert.Multiple(() =>
+ {
+ Assert.That(
+ EmbeddingChangeImpact.AffectsStoredIndex(embeddingProvider, followingTheProvider, followingTheProvider with { MaxChunkTokenLength = 8192 }),
+ Is.False,
+ "The provider limit typed into the field is the cut the data source already had.");
+
+ Assert.That(
+ EmbeddingChangeImpact.AffectsStoredIndex(embeddingProvider, followingTheProvider, followingTheProvider with { MaxChunkTokenLength = 4096 }),
+ Is.True,
+ "Anything below the provider limit really does cut the text elsewhere.");
+ });
+ }
+
+ ///
+ /// An overlap larger than the chunk is capped, so several of them are the same cut.
+ ///
+ ///
+ /// The same reasoning as for the token limit: what counts is where the text is cut, not what
+ /// somebody typed into the field.
+ ///
+ [Test]
+ public void AnOverlapWhichIsCappedAnywayKeepsTheStoredIndex()
+ {
+ var embeddingProvider = StoredEmbeddingProvider();
+ var stored = StoredDataSource() with { MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 600 };
+
+ Assert.That(
+ EmbeddingChangeImpact.AffectsStoredIndex(embeddingProvider, stored, stored with { ChunkOverlapTokenLength = 700 }),
+ Is.False,
+ "Both overlaps are capped to the chunk size, so the text is cut identically.");
+ }
+
[Test]
public void HarmlessDataSourceEditsKeepTheStoredIndex()
{
diff --git a/app/Tests/Tools/EmbeddingSignatureTests.cs b/app/Tests/Tools/EmbeddingSignatureTests.cs
index 804eba41..94513cfa 100644
--- a/app/Tests/Tools/EmbeddingSignatureTests.cs
+++ b/app/Tests/Tools/EmbeddingSignatureTests.cs
@@ -92,15 +92,24 @@ public sealed class EmbeddingSignatureTests
{
Assert.That(
Signature(DataSource(ConfidenceLevel.LOW)),
- Is.EqualTo("2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100|512|100"),
+ Is.EqualTo("2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100"),
"Reordering or extending the signature throws away every index anybody has. This test makes that a decision somebody takes rather than something which happens on the way past.");
}
+ ///
+ /// Builds the signature the way an indexing run does, working the chunking out along the way.
+ ///
+ ///
+ /// Handing in fixed chunking options instead would hide exactly what these tests are here for:
+ /// the signature would then no longer notice a data source being cut differently.
+ ///
+ /// The data source to build the signature for.
+ /// The embedding provider, or the test default.
+ /// The signature of that pairing.
private static string Signature(DataSourceLocalDirectory dataSource, EmbeddingProvider? embeddingProvider = null) =>
DataSourceEmbeddingService.BuildEmbeddingSignature(
dataSource,
- embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"),
- new(512, 100));
+ embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"));
private static DataSourceLocalDirectory DataSource(ConfidenceLevel confidenceLevel) => new()
{