Keep the embedding signature to what actually cuts the text

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs

@@ -1012,6 +1012,12 @@ public sealed partial class DataSourceEmbeddingService
     /// another lands on the identical path, while moving the data directory changes every path
     /// without changing a single tokenizer.
     ///
+    /// The chunk settings enter only as what they amount to, never as what somebody typed. A data
+    /// source storing 0 means "follow the embedding provider", and writing that provider's own limit
+    /// into the field changes nothing about how the text is cut. Carrying the typed numbers as well
+    /// made that a different signature, so opening the expert settings of a data source — which
+    /// fills an empty limit with the provider's — threw the whole index away for nothing.
+    ///
     /// The confidence level a data source asks of a provider is deliberately not among them. It
     /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks
     /// it against the participating chat providers and against the embedding provider, and this
@@ -1031,8 +1037,6 @@ public sealed partial class DataSourceEmbeddingService
             embeddingProvider.HFInferenceProvider,
             embeddingProvider.TokenizerFingerprint,
             embeddingProvider.EffectiveTokenLimit,
-            dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
-            dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH,
             chunkingOptions.MaxChunkTokenLength,
             chunkingOptions.OverlapTokenLength);
     }

app/Tests/Tools/EmbeddingChangeImpactTests.cs

@@ -99,6 +99,54 @@ public sealed class EmbeddingChangeImpactTests
         });
     }
 
+    /// <summary>
+    /// Writing out what a data source already follows must not cost it its index.
+    /// </summary>
+    /// <remarks>
+    /// A token limit of 0 means "follow the embedding provider", and opening the expert settings of a
+    /// data source fills that empty field with exactly the provider's limit. Both are the same cut,
+    /// so nobody may be asked about it -- and above all, nothing may be re-embedded for it. Somebody
+    /// who only wanted to change how many matches an answer may use paid for a full rebuild.
+    /// </remarks>
+    [Test]
+    public void SpellingOutWhatTheProviderAlreadyDictatesKeepsTheStoredIndex()
+    {
+        var embeddingProvider = StoredEmbeddingProvider() with { TokenLimit = 8192 };
+        var followingTheProvider = StoredDataSource() with { MaxChunkTokenLength = 0 };
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(
+                EmbeddingChangeImpact.AffectsStoredIndex(embeddingProvider, followingTheProvider, followingTheProvider with { MaxChunkTokenLength = 8192 }),
+                Is.False,
+                "The provider limit typed into the field is the cut the data source already had.");
+
+            Assert.That(
+                EmbeddingChangeImpact.AffectsStoredIndex(embeddingProvider, followingTheProvider, followingTheProvider with { MaxChunkTokenLength = 4096 }),
+                Is.True,
+                "Anything below the provider limit really does cut the text elsewhere.");
+        });
+    }
+
+    /// <summary>
+    /// An overlap larger than the chunk is capped, so several of them are the same cut.
+    /// </summary>
+    /// <remarks>
+    /// The same reasoning as for the token limit: what counts is where the text is cut, not what
+    /// somebody typed into the field.
+    /// </remarks>
+    [Test]
+    public void AnOverlapWhichIsCappedAnywayKeepsTheStoredIndex()
+    {
+        var embeddingProvider = StoredEmbeddingProvider();
+        var stored = StoredDataSource() with { MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 600 };
+
+        Assert.That(
+            EmbeddingChangeImpact.AffectsStoredIndex(embeddingProvider, stored, stored with { ChunkOverlapTokenLength = 700 }),
+            Is.False,
+            "Both overlaps are capped to the chunk size, so the text is cut identically.");
+    }
+
     [Test]
     public void HarmlessDataSourceEditsKeepTheStoredIndex()
     {

app/Tests/Tools/EmbeddingSignatureTests.cs

@@ -92,15 +92,24 @@ public sealed class EmbeddingSignatureTests
     {
         Assert.That(
             Signature(DataSource(ConfidenceLevel.LOW)),
-            Is.EqualTo("2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100|512|100"),
+            Is.EqualTo("2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100"),
             "Reordering or extending the signature throws away every index anybody has. This test makes that a decision somebody takes rather than something which happens on the way past.");
     }
 
+    /// <summary>
+    /// Builds the signature the way an indexing run does, working the chunking out along the way.
+    /// </summary>
+    /// <remarks>
+    /// Handing in fixed chunking options instead would hide exactly what these tests are here for:
+    /// the signature would then no longer notice a data source being cut differently.
+    /// </remarks>
+    /// <param name="dataSource">The data source to build the signature for.</param>
+    /// <param name="embeddingProvider">The embedding provider, or the test default.</param>
+    /// <returns>The signature of that pairing.</returns>
     private static string Signature(DataSourceLocalDirectory dataSource, EmbeddingProvider? embeddingProvider = null) =>
         DataSourceEmbeddingService.BuildEmbeddingSignature(
             dataSource,
-            embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"),
-            new(512, 100));
+            embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"));
 
     private static DataSourceLocalDirectory DataSource(ConfidenceLevel confidenceLevel) => new()
     {