Fixed data sources being re-indexed after a confidence level change

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs

@@ -58,7 +58,7 @@ public sealed partial class DataSourceEmbeddingService
 
     private sealed record EmbeddingChunkDraft(string ChunkId, string Text, int ChunkIndex, int? PageNumber);
 
-    private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength);
+    internal sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength);
 
     private sealed record ChunkingStrategy(string Name, IReadOnlyList<ChunkingRule> Rules);
 
@@ -986,7 +986,23 @@ public sealed partial class DataSourceEmbeddingService
         }
     }
 
-    private string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions)
+    /// <summary>
+    /// Describes how the vectors of a data source were made.
+    /// </summary>
+    /// <remarks>
+    /// What appears here decides when stored embeddings are thrown away: a signature differing from
+    /// the persisted one drops the whole index and builds it again. So it names the embedding model,
+    /// where it runs, how the text was cut for it, and the chunk metadata version — the things a
+    /// vector actually depends on.
+    ///
+    /// The confidence level a data source asks of a provider is deliberately not among them. It
+    /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks
+    /// it against the participating chat providers and against the embedding provider, and this
+    /// service checks it again before each indexing run. It was part of this signature once, which
+    /// re-embedded every file of a data source whenever somebody raised or lowered it — real money
+    /// at a cloud embedding provider, for nothing.
+    /// </remarks>
+    internal static string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions)
     {
         return string.Join('|',
             CHUNK_METADATA_VERSION,
@@ -997,7 +1013,6 @@ public sealed partial class DataSourceEmbeddingService
             embeddingProvider.Hostname,
             embeddingProvider.TokenizerPath,
             embeddingProvider.EffectiveTokenLimit,
-            GetDataSourceConfidenceLevel(dataSource).ToString(),
             dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
             dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH,
             chunkingOptions.MaxChunkTokenLength,

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -1237,7 +1237,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         CancellationToken token)
     {
         var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
-        var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
+        var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
         var manifest = await indexStore.GetManifestAsync(dataSource.Id, token);
 
         logger.LogInformation(

app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs

@@ -52,9 +52,7 @@ public sealed class DataSourceLocalRetrievalService(
         int ChunkIndex,
         string Text,
         double Score,
-        int Rank,
+        int Rank);
-        string ConfidenceLevel,
-        int ConfidenceLevelRank);
     // ReSharper restore NotAccessedPositionalProperty.Local
 
     public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(DataSourceLocalFile dataSource, IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
@@ -354,9 +352,7 @@ public sealed class DataSourceLocalRetrievalService(
             result.ChunkIndex,
             result.Text,
             result.Score,
-            rank,
+            rank);
-            result.ConfidenceLevel,
-            result.ConfidenceLevelRank);
 
     private static LocalRetrievalHit FromBm25Result(IndexStoreSearchResult result, int rank) =>
         new(
@@ -374,9 +370,7 @@ public sealed class DataSourceLocalRetrievalService(
             result.ChunkIndex,
             result.ChunkText,
             result.Score,
-            rank,
+            rank);
-            result.ConfidenceLevel,
-            result.ConfidenceLevelRank);
 
     private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit)
     {

app/Tests/Tools/EmbeddingSignatureTests.cs

@@ -0,0 +1,72 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks what makes the stored embeddings of a data source invalid.
+/// </summary>
+/// <remarks>
+/// The embedding signature decides whether an index survives: when it differs from the one persisted
+/// for a data source, everything stored is thrown away and embedded again. That is the right answer
+/// for anything a vector depends on, and an expensive mistake for everything else. The confidence
+/// level a data source asks of a provider used to be part of it, so changing that one setting
+/// re-embedded every file of the source — at a cloud embedding provider, for real money and no gain.
+/// </remarks>
+[TestFixture]
+public sealed class EmbeddingSignatureTests
+{
+    [Test]
+    public void ChangingTheConfidenceLevelKeepsTheStoredEmbeddings()
+    {
+        var low = DataSource(ConfidenceLevel.LOW);
+        var high = DataSource(ConfidenceLevel.HIGH);
+
+        Assert.That(Signature(high), Is.EqualTo(Signature(low)), "The confidence level changes no vector, so the stored index stays valid and nothing is embedded again.");
+    }
+
+    [Test]
+    public void ChangingTheChunkSizeDropsTheStoredEmbeddings()
+    {
+        var small = DataSource(ConfidenceLevel.LOW) with { MaxChunkTokenLength = 512 };
+        var large = DataSource(ConfidenceLevel.LOW) with { MaxChunkTokenLength = 1024 };
+
+        Assert.That(Signature(large), Is.Not.EqualTo(Signature(small)), "Other chunk boundaries mean other vectors, so the index has to be built again.");
+    }
+
+    [Test]
+    public void ChangingTheEmbeddingModelDropsTheStoredEmbeddings()
+    {
+        var dataSource = DataSource(ConfidenceLevel.LOW);
+
+        Assert.That(
+            Signature(dataSource, EmbeddingProviderFor("text-embedding-3-large")),
+            Is.Not.EqualTo(Signature(dataSource, EmbeddingProviderFor("text-embedding-3-small"))),
+            "Another model means another vector space, so nothing stored may be kept.");
+    }
+
+    private static string Signature(DataSourceLocalDirectory dataSource, EmbeddingProvider? embeddingProvider = null) =>
+        DataSourceEmbeddingService.BuildEmbeddingSignature(
+            dataSource,
+            embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"),
+            new(512, 100));
+
+    private static DataSourceLocalDirectory DataSource(ConfidenceLevel confidenceLevel) => new()
+    {
+        Num = 1,
+        Id = "6f1d6a4e-6a5e-4c62-9a4f-0f2d2c8b7a11",
+        Name = "Test data",
+        Description = "Documents used by the tests.",
+        Type = DataSourceType.LOCAL_DIRECTORY,
+        EmbeddingId = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01",
+        MaxChunkTokenLength = 512,
+        ChunkOverlapTokenLength = 100,
+        ConfidenceLevel = confidenceLevel,
+        Path = "/tmp/test-data",
+    };
+
+    private static EmbeddingProvider EmbeddingProviderFor(string modelId) =>
+        new(1, "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "Test embeddings", LLMProviders.OPEN_AI, new(modelId, modelId));
+}