Fixed data sources being re-indexed after a confidence level change

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs

@@ -58,7 +58,7 @@ public sealed partial class DataSourceEmbeddingService
 
     private sealed record EmbeddingChunkDraft(string ChunkId, string Text, int ChunkIndex, int? PageNumber);
 
-    private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength);
+    internal sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength);
 
     private sealed record ChunkingStrategy(string Name, IReadOnlyList<ChunkingRule> Rules);
 
@@ -986,7 +986,23 @@ public sealed partial class DataSourceEmbeddingService
         }
     }
 
-    private string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions)
+    /// <summary>
+    /// Describes how the vectors of a data source were made.
+    /// </summary>
+    /// <remarks>
+    /// What appears here decides when stored embeddings are thrown away: a signature differing from
+    /// the persisted one drops the whole index and builds it again. So it names the embedding model,
+    /// where it runs, how the text was cut for it, and the chunk metadata version — the things a
+    /// vector actually depends on.
+    ///
+    /// The confidence level a data source asks of a provider is deliberately not among them. It
+    /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks
+    /// it against the participating chat providers and against the embedding provider, and this
+    /// service checks it again before each indexing run. It was part of this signature once, which
+    /// re-embedded every file of a data source whenever somebody raised or lowered it — real money
+    /// at a cloud embedding provider, for nothing.
+    /// </remarks>
+    internal static string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions)
     {
         return string.Join('|',
             CHUNK_METADATA_VERSION,
@@ -997,7 +1013,6 @@ public sealed partial class DataSourceEmbeddingService
             embeddingProvider.Hostname,
             embeddingProvider.TokenizerPath,
             embeddingProvider.EffectiveTokenLimit,
-            GetDataSourceConfidenceLevel(dataSource).ToString(),
             dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
             dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH,
             chunkingOptions.MaxChunkTokenLength,

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -1237,7 +1237,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         CancellationToken token)
     {
         var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider);
-        var embeddingSignature = this.BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
+        var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions);
         var manifest = await indexStore.GetManifestAsync(dataSource.Id, token);
 
         logger.LogInformation(

app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs

@@ -52,9 +52,7 @@ public sealed class DataSourceLocalRetrievalService(
         int ChunkIndex,
         string Text,
         double Score,
-        int Rank,
-        string ConfidenceLevel,
-        int ConfidenceLevelRank);
+        int Rank);
     // ReSharper restore NotAccessedPositionalProperty.Local
 
     public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(DataSourceLocalFile dataSource, IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
@@ -354,9 +352,7 @@ public sealed class DataSourceLocalRetrievalService(
             result.ChunkIndex,
             result.Text,
             result.Score,
-            rank,
-            result.ConfidenceLevel,
-            result.ConfidenceLevelRank);
+            rank);
 
     private static LocalRetrievalHit FromBm25Result(IndexStoreSearchResult result, int rank) =>
         new(
@@ -374,9 +370,7 @@ public sealed class DataSourceLocalRetrievalService(
             result.ChunkIndex,
             result.ChunkText,
             result.Score,
-            rank,
-            result.ConfidenceLevel,
-            result.ConfidenceLevelRank);
+            rank);
 
     private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit)
     {