Added local RAG (#756)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Tools/Services/ArbitraryFileDataSegment.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Tools.Services;
+
+public sealed record ArbitraryFileDataSegment(string Content, int TokenCount);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs

@@ -0,0 +1,24 @@
+using System.Net;
+
+using AIStudio.Provider;
+
+namespace AIStudio.Tools.Services;
+
+/// <summary>
+/// One input which could not be embedded, together with everything known about why.
+/// </summary>
+/// <remarks>
+/// The reason alone used to be all we kept, which made every failure look alike in the UI: a
+/// rejected API key, an unreachable provider, and a file nobody may read were one and the same
+/// list entry. The surrounding fields are what lets the UI offer the matching way out.
+/// </remarks>
+/// <param name="FilePath">The file that failed or the name of the data source when the failure was not about one file.</param>
+/// <param name="Reason">What to tell the user about it, ready to show.</param>
+/// <param name="OccurredAtUtc">When it happened, so the list still makes sense when the user looks at it later.</param>
+/// <param name="FailureReason">What kind of failure it was. Everything that did not come from a provider stays at NONE.</param>
+/// <param name="StatusCode">What the provider answered, where it answered at all.</param>
+/// <param name="EmbeddingProviderName">The embedding provider that was asked.</param>
+/// <param name="ExtractionCode">Why reading the file failed, where the failure was about reading it at all.</param>
+/// <param name="IsPermanent">Whether the file stays out of the index until it changes.</param>
+public sealed record DataSourceEmbeddingFailure(string FilePath, string Reason, DateTimeOffset OccurredAtUtc, ProviderRequestFailureReason FailureReason = ProviderRequestFailureReason.NONE,
+    HttpStatusCode? StatusCode = null, string EmbeddingProviderName = "", FileExtractionErrorCode ExtractionCode = FileExtractionErrorCode.NONE, bool IsPermanent = false);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs

@@ -0,0 +1,25 @@
+namespace AIStudio.Tools.Services;
+
+public sealed class DataSourceEmbeddingManifest
+{
+    public string EmbeddingProviderId { get; set; } = string.Empty;
+
+    public string EmbeddingSignature { get; set; } = string.Empty;
+
+    public string SourceHash { get; set; } = string.Empty;
+
+    public int VectorSize { get; set; }
+
+    public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);
+
+    /// <summary>
+    /// The files whose indexing failed for a reason which lies in the file itself, keyed by their
+    /// absolute path.
+    /// </summary>
+    /// <remarks>
+    /// These files are not read again as long as their fingerprint stays the same. Without this,
+    /// a folder holding hundreds of scanned documents without a text layer would be read again on
+    /// every single run, with the outcome known in advance.
+    /// </remarks>
+    public Dictionary<string, PermanentIndexingFailureRecord> PermanentFailures { get; init; } = new(StringComparer.OrdinalIgnoreCase);
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingNames.cs

@@ -0,0 +1,12 @@
+namespace AIStudio.Tools.Services;
+
+internal static class DataSourceEmbeddingNames
+{
+    public static string GetCollectionName(string dataSourceId)
+    {
+        if (!Guid.TryParse(dataSourceId, out var parsedDataSourceId))
+            throw new ArgumentException("Data source ID must be a valid GUID.", nameof(dataSourceId));
+
+        return $"rag_{parsedDataSourceId:N}";
+    }
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingOverview.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Tools.Services;
+
+public sealed record DataSourceEmbeddingOverview(DataSourceEmbeddingState State, int IndexedFiles, int TotalFiles, int FailedFiles);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingProviders.cs

@@ -0,0 +1,18 @@
+using System.Diagnostics.CodeAnalysis;
+
+using AIStudio.Provider;
+using AIStudio.Settings;
+
+namespace AIStudio.Tools.Services;
+
+internal static class DataSourceEmbeddingProviders
+{
+    public static bool TryResolve(SettingsManager settingsManager, IDataSource dataSource, [NotNullWhen(true)] out EmbeddingProvider? embeddingProvider)
+    {
+        embeddingProvider = settingsManager.ConfigurationData.EmbeddingProviders.FirstOrDefault(provider =>
+            dataSource is IInternalDataSource internalDataSource &&
+            provider.Id.Equals(internalDataSource.EmbeddingId, StringComparison.OrdinalIgnoreCase));
+
+        return embeddingProvider != default && embeddingProvider.UsedLLMProvider is not LLMProviders.NONE;
+    }
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs

@@ -0,0 +1,26 @@
+using AIStudio.Tools.Databases.IndexStore;
+using AIStudio.Tools.Databases.VectorStore;
+
+namespace AIStudio.Tools.Services;
+
+public sealed partial class DataSourceEmbeddingService
+{
+    private async Task ResetPersistedStateAsync(
+        string dataSourceId,
+        VectorStoreClient? vectorStore,
+        IndexStoreClient? indexStore,
+        CancellationToken token)
+    {
+        await this.DeleteCollectionAsync(DataSourceEmbeddingNames.GetCollectionName(dataSourceId), vectorStore, token);
+
+        indexStore ??= await databaseClientProvider.GetIndexStoreAsync(token);
+        if (!indexStore.IsAvailable)
+        {
+            logger.LogWarning("Could not delete local RAG embedding state for data source '{DataSourceId}' because the database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name);
+            return;
+        }
+
+        await indexStore.DeleteDataSourceAsync(dataSourceId, token);
+        logger.LogInformation("Reset persisted local RAG embedding state for data source '{DataSourceId}'.", dataSourceId);
+    }
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Watchers.cs

@@ -0,0 +1,289 @@
+using System.Collections.Concurrent;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+
+namespace AIStudio.Tools.Services;
+
+public sealed partial class DataSourceEmbeddingService
+{
+    private const int WATCHER_DEBOUNCE_SECONDS = 2;
+
+    private readonly ConcurrentDictionary<string, DataSourceWatcherRegistration> watchers = new(StringComparer.OrdinalIgnoreCase);
+    private readonly Dictionary<string, CancellationTokenSource> watcherDebounceTokens = new(StringComparer.OrdinalIgnoreCase);
+    private readonly object watcherDebounceLock = new();
+    
+    private void RefreshWatchers()
+    {
+        if (!settingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh)
+        {
+            this.RemoveAllWatchers();
+            return;
+        }
+
+        if (Volatile.Read(ref this.startupHashCheckCompleted) == 0)
+        {
+            logger.LogDebug("File watchers are not activated yet because the startup persisted hash check has not completed.");
+            this.RemoveAllWatchers();
+            return;
+        }
+
+        var supportedSources = settingsManager.ConfigurationData.DataSources
+            .Where(this.IsSupportedInternalDataSource)
+            .ToDictionary(source => source.Id, StringComparer.OrdinalIgnoreCase);
+
+        foreach (var existingWatcherId in this.watchers.Keys.Except(supportedSources.Keys, StringComparer.OrdinalIgnoreCase).ToList())
+            this.RemoveWatcher(existingWatcherId);
+
+        foreach (var dataSource in supportedSources.Values)
+            this.EnsureWatcher(dataSource);
+    }
+
+    private void EnsureWatcher(IDataSource dataSource)
+    {
+        if (!settingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh)
+            return;
+
+        var configuration = GetWatchConfiguration(dataSource);
+        if (configuration is null)
+            return;
+
+        if (this.watchers.TryGetValue(dataSource.Id, out var existingRegistration))
+        {
+            if (IsSameWatchConfiguration(existingRegistration.Configuration, configuration))
+                return;
+
+            this.RemoveWatcher(dataSource.Id);
+        }
+
+        var watcher = this.CreateWatcher(dataSource.Id, configuration);
+        if (watcher is null)
+            return;
+
+        if (!this.watchers.TryAdd(dataSource.Id, new DataSourceWatcherRegistration(watcher, configuration)))
+            watcher.Dispose();
+    }
+
+    private FileSystemWatcher? CreateWatcher(string dataSourceId, DataSourceWatcherConfiguration configuration)
+    {
+        try
+        {
+            var watcher = new FileSystemWatcher(configuration.RootPath)
+            {
+                Filter = configuration.Filter,
+                IncludeSubdirectories = configuration.IncludeSubdirectories,
+                NotifyFilter = NotifyFilters.FileName | NotifyFilters.DirectoryName | NotifyFilters.LastWrite | NotifyFilters.CreationTime | NotifyFilters.Size,
+            };
+
+            watcher.Changed += (_, args) => this.OnWatchedDataSourceChanged(dataSourceId, configuration, args);
+            watcher.Deleted += (_, args) => this.OnWatchedDataSourceChanged(dataSourceId, configuration, args);
+            watcher.Created += (_, args) => this.OnWatchedDataSourceChanged(dataSourceId, configuration, args);
+            watcher.Renamed += (_, args) => this.OnWatchedDataSourceChanged(dataSourceId, configuration, args);
+            watcher.Error += (_, args) =>
+            {
+                logger.LogWarning(args.GetException(), "The file watcher for data source '{DataSourceId}' failed. Recreating it.", dataSourceId);
+                this.RemoveWatcher(dataSourceId);
+                this.EnsureWatcher(dataSourceId);
+                this.ScheduleWatchedDataSourceRefresh(dataSourceId);
+            };
+            watcher.EnableRaisingEvents = true;
+            return watcher;
+        }
+        catch (Exception exception)
+        {
+            logger.LogWarning(exception, "Failed to create file watcher for data source '{DataSourceId}' at '{RootPath}'.", dataSourceId, configuration.RootPath);
+            return null;
+        }
+    }
+
+    private void RemoveWatcher(string dataSourceId)
+    {
+        this.CancelPendingWatcherRefresh(dataSourceId);
+
+        if (this.watchers.TryRemove(dataSourceId, out var registration))
+            registration.Watcher.Dispose();
+    }
+    
+    private void RemoveAllWatchers()
+    {
+        foreach (var watcherId in this.watchers.Keys.ToList())
+            this.RemoveWatcher(watcherId);
+    }
+
+    private void DisposeWatchers()
+    {
+        this.CancelAllPendingWatcherRefreshes();
+
+        foreach (var registration in this.watchers.Values)
+            registration.Watcher.Dispose();
+
+        this.watchers.Clear();
+    }
+
+    private void OnWatchedDataSourceChanged(string dataSourceId, DataSourceWatcherConfiguration configuration, FileSystemEventArgs args)
+    {
+        if (!this.IsRelevantWatcherEvent(configuration, args))
+        {
+            logger.LogDebug(
+                "Ignoring file system change for data source '{DataSourceId}' at '{Path}' (event={ChangeType}) because the path is not part of the RAG index.",
+                dataSourceId,
+                args.FullPath,
+                args.ChangeType);
+            return;
+        }
+
+        logger.LogDebug(
+            "Detected relevant file system change for data source '{DataSourceId}' at '{Path}' (event={ChangeType}). Scheduling a debounced embedding run.",
+            dataSourceId,
+            args.FullPath,
+            args.ChangeType);
+
+        this.ScheduleWatchedDataSourceRefresh(dataSourceId);
+    }
+
+    private void ScheduleWatchedDataSourceRefresh(string dataSourceId)
+    {
+        if (!settingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh)
+            return;
+
+        var debounceToken = new CancellationTokenSource();
+
+        lock (this.watcherDebounceLock)
+        {
+            if (this.watcherDebounceTokens.Remove(dataSourceId, out var existingToken))
+                existingToken.Cancel();
+
+            this.watcherDebounceTokens[dataSourceId] = debounceToken;
+        }
+
+        _ = Task.Run(async () =>
+        {
+            try
+            {
+                await Task.Delay(TimeSpan.FromSeconds(WATCHER_DEBOUNCE_SECONDS), debounceToken.Token);
+                if (!this.TryCompletePendingWatcherRefresh(dataSourceId, debounceToken))
+                    return;
+
+                var dataSource = settingsManager.ConfigurationData.DataSources
+                    .FirstOrDefault(source => source.Id.Equals(dataSourceId, StringComparison.OrdinalIgnoreCase));
+
+                if (dataSource is not null)
+                {
+                    logger.LogInformation("Queueing data source '{DataSourceName}' ({DataSourceId}) after file system changes settled. The hash pipeline will reindex only changed files.", dataSource.Name, dataSource.Id);
+                    await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.WATCHER_HASH_CHECK);
+                }
+            }
+            catch (OperationCanceledException)
+            {
+            }
+            catch (Exception exception)
+            {
+                logger.LogWarning(exception, "Failed to queue watched data source '{DataSourceId}' after a file system change.", dataSourceId);
+            }
+            finally
+            {
+                debounceToken.Dispose();
+            }
+        });
+    }
+
+    private void EnsureWatcher(string dataSourceId)
+    {
+        var dataSource = settingsManager.ConfigurationData.DataSources
+            .FirstOrDefault(source => source.Id.Equals(dataSourceId, StringComparison.OrdinalIgnoreCase));
+
+        if (dataSource is not null)
+            this.EnsureWatcher(dataSource);
+    }
+
+    private void CancelPendingWatcherRefresh(string dataSourceId)
+    {
+        lock (this.watcherDebounceLock)
+        {
+            if (this.watcherDebounceTokens.Remove(dataSourceId, out var token))
+                token.Cancel();
+        }
+    }
+
+    private void CancelAllPendingWatcherRefreshes()
+    {
+        lock (this.watcherDebounceLock)
+        {
+            foreach (var token in this.watcherDebounceTokens.Values)
+                token.Cancel();
+
+            this.watcherDebounceTokens.Clear();
+        }
+    }
+
+    private bool TryCompletePendingWatcherRefresh(string dataSourceId, CancellationTokenSource debounceToken)
+    {
+        lock (this.watcherDebounceLock)
+        {
+            if (!this.watcherDebounceTokens.TryGetValue(dataSourceId, out var currentToken) || !ReferenceEquals(currentToken, debounceToken))
+                return false;
+
+            this.watcherDebounceTokens.Remove(dataSourceId);
+            return true;
+        }
+    }
+
+    private bool IsRelevantWatcherEvent(DataSourceWatcherConfiguration configuration, FileSystemEventArgs args)
+    {
+        if (args is RenamedEventArgs renamedArgs)
+        {
+            return this.IsRelevantWatcherPath(configuration, renamedArgs.FullPath, args.ChangeType)
+                   || this.IsRelevantWatcherPath(configuration, renamedArgs.OldFullPath, args.ChangeType);
+        }
+
+        return this.IsRelevantWatcherPath(configuration, args.FullPath, args.ChangeType);
+    }
+
+    private bool IsRelevantWatcherPath(DataSourceWatcherConfiguration configuration, string path, WatcherChangeTypes changeType)
+    {
+        if (string.IsNullOrWhiteSpace(path))
+            return false;
+
+        var fileName = Path.GetFileName(path);
+        if (string.IsNullOrWhiteSpace(fileName))
+            return true;
+
+        if (!configuration.IncludeSubdirectories && !string.Equals(fileName, configuration.Filter, StringComparison.OrdinalIgnoreCase))
+            return false;
+
+        if (Directory.Exists(path))
+            return true;
+
+        if (IsSkippedRagFileName(fileName))
+            return false;
+
+        if (this.IsSupportedRagFilePath(path))
+            return true;
+
+        return changeType is WatcherChangeTypes.Deleted or WatcherChangeTypes.Renamed
+               && string.IsNullOrWhiteSpace(Path.GetExtension(path));
+    }
+
+    private static DataSourceWatcherConfiguration? GetWatchConfiguration(IDataSource dataSource) => dataSource switch
+    {
+        DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path) => new DataSourceWatcherConfiguration(
+            localDirectory.Path,
+            "*.*",
+            true),
+        DataSourceLocalFile localFile when File.Exists(localFile.FilePath) && !string.IsNullOrWhiteSpace(Path.GetDirectoryName(localFile.FilePath)) => new DataSourceWatcherConfiguration(
+            Path.GetDirectoryName(localFile.FilePath)!,
+            Path.GetFileName(localFile.FilePath),
+            false),
+        _ => null,
+    };
+
+    private static bool IsSameWatchConfiguration(DataSourceWatcherConfiguration left, DataSourceWatcherConfiguration right)
+    {
+        return left.IncludeSubdirectories == right.IncludeSubdirectories
+               && string.Equals(left.RootPath, right.RootPath, StringComparison.OrdinalIgnoreCase)
+               && string.Equals(left.Filter, right.Filter, StringComparison.OrdinalIgnoreCase);
+    }
+
+    private sealed record DataSourceWatcherConfiguration(string RootPath, string Filter, bool IncludeSubdirectories);
+
+    private sealed record DataSourceWatcherRegistration(FileSystemWatcher Watcher, DataSourceWatcherConfiguration Configuration);
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingState.cs

@@ -0,0 +1,10 @@
+namespace AIStudio.Tools.Services;
+
+public enum DataSourceEmbeddingState
+{
+    IDLE,
+    QUEUED,
+    RUNNING,
+    COMPLETED,
+    FAILED,
+}

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs

@@ -0,0 +1,45 @@
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.PluginSystem;
+
+namespace AIStudio.Tools.Services;
+
+public sealed record DataSourceEmbeddingStatus(
+    string DataSourceId,
+    string DataSourceName,
+    DataSourceType DataSourceType,
+    DataSourceEmbeddingState State,
+    int TotalFiles,
+    int IndexedFiles,
+    int FailedFiles,
+    string CurrentFile,
+    string LastError,
+    IReadOnlyList<DataSourceEmbeddingFailure> Failures,
+    int PermanentlySkippedFiles = 0)
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus));
+
+    /// <remarks>
+    /// Files which were skipped for good are done, even though nothing was indexed of them.
+    /// Leaving them out would keep the bar short of the end for a data source which has nothing
+    /// left to do.
+    /// </remarks>
+    public int ProgressPercent => this.TotalFiles <= 0 ? 0 : Math.Clamp((int)Math.Round((this.IndexedFiles + this.PermanentlySkippedFiles) * 100d / this.TotalFiles), 0, 100);
+
+    public string StateLabel => this.State switch
+    {
+        DataSourceEmbeddingState.QUEUED => TB("Queued"),
+        DataSourceEmbeddingState.RUNNING => TB("Running"),
+        DataSourceEmbeddingState.COMPLETED => TB("Completed"),
+        DataSourceEmbeddingState.FAILED => TB("Needs attention"),
+        _ => TB("Idle")
+    };
+
+    public int SortOrder => this.State switch
+    {
+        DataSourceEmbeddingState.RUNNING => 0,
+        DataSourceEmbeddingState.QUEUED => 1,
+        DataSourceEmbeddingState.FAILED => 2,
+        DataSourceEmbeddingState.COMPLETED => 3,
+        _ => 4,
+    };
+}

app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs

@@ -0,0 +1,506 @@
+using AIStudio.Chat;
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Databases;
+using AIStudio.Tools.Databases.IndexStore;
+using AIStudio.Tools.Databases.VectorStore;
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.RAG;
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tools.Services;
+
+public sealed class DataSourceLocalRetrievalService(
+    SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider,
+    ILogger<DataSourceLocalRetrievalService> logger)
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceLocalRetrievalService).Namespace, nameof(DataSourceLocalRetrievalService));
+
+    //
+    // Which gaps the user was already told about in this session. Retrieval runs for every single
+    // message, so without this one broken embedding provider would put a warning on every prompt.
+    //
+    private readonly HashSet<string> reportedRetrievalGaps = new(StringComparer.Ordinal);
+    private readonly Lock retrievalGapLock = new();
+
+    private enum RetrievalChannel
+    {
+        VECTOR,
+        BM25,
+    }
+
+    //
+    // A hit keeps the complete shape both retrieval channels deliver, even where nothing reads a
+    // value yet. Merging is deterministic on purpose for now, so Channel, Score and Rank have no
+    // consumer until reranking arrives. Naming them still beats handing an unlabelled tuple of
+    // strings and numbers through the service.
+    //
+    // ReSharper disable NotAccessedPositionalProperty.Local
+    private sealed record LocalRetrievalHit(
+        RetrievalChannel Channel,
+        string ChunkId,
+        string ParentFileId,
+        string DataSourceId,
+        string DataSourceName,
+        string DataSourceType,
+        string AbsolutePath,
+        string FileName,
+        string RelativePath,
+        string FileType,
+        int? PageNumber,
+        int ChunkIndex,
+        string Text,
+        double Score,
+        int Rank,
+        string ConfidenceLevel,
+        int ConfidenceLevelRank);
+    // ReSharper restore NotAccessedPositionalProperty.Local
+
+    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(DataSourceLocalFile dataSource, IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
+        this.RetrieveDataAsync(dataSource, lastUserPrompt, token);
+
+    public Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(DataSourceLocalDirectory dataSource, IContent lastUserPrompt, ChatThread thread, CancellationToken token = default) =>
+        this.RetrieveDataAsync(dataSource, lastUserPrompt, token);
+
+    private async Task<IReadOnlyList<IRetrievalContext>> RetrieveDataAsync(IInternalDataSource dataSource, IContent lastUserPrompt, CancellationToken token)
+    {
+        var query = GetQueryText(lastUserPrompt);
+        if (string.IsNullOrWhiteSpace(query))
+        {
+            logger.LogDebug("Skipping local retrieval for data source '{DataSourceName}' ({DataSourceId}) because the latest prompt does not contain text.", dataSource.Name, dataSource.Id);
+            return [];
+        }
+
+        var maxMatches = (int)dataSource.MaxMatches;
+        if (maxMatches == 0)
+            return [];
+
+        var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
+        var vectorTask = this.SearchVectorAsync(dataSource, query, maxMatches, collectionName, token);
+        var bm25Task = this.SearchBm25Async(dataSource, query, maxMatches, token);
+
+        await Task.WhenAll(vectorTask, bm25Task);
+        token.ThrowIfCancellationRequested();
+
+        var hits = MergeResults(vectorTask.Result, bm25Task.Result, maxMatches);
+        logger.LogInformation(
+            "Retrieved {MergedHits} local RAG hits for data source '{DataSourceName}' ({DataSourceId}). VectorCandidates={VectorHits}, BM25Candidates={BM25Hits}, RequestedPerChannel={RequestedPerChannel}.",
+            hits.Count,
+            dataSource.Name,
+            dataSource.Id,
+            vectorTask.Result.Count,
+            bm25Task.Result.Count,
+            maxMatches);
+
+        return hits
+            .Where(hit => !string.IsNullOrWhiteSpace(hit.Text))
+            .Select(ToRetrievalContext)
+            .ToList();
+    }
+
+    private async Task<IReadOnlyList<VectorSearchResult>> SearchVectorAsync(
+        IInternalDataSource dataSource,
+        string query,
+        int maxMatches,
+        string collectionName,
+        CancellationToken token)
+    {
+        try
+        {
+            var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token);
+            if (!vectorStore.IsAvailable)
+            {
+                logger.LogWarning(
+                    "Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because vector store '{VectorStoreName}' is unavailable.",
+                    dataSource.Name,
+                    dataSource.Id,
+                    vectorStore.Name);
+                await this.ReportRetrievalGapAsync(dataSource, "no-vector-store", string.Format(TB("The data source '{0}' was left out of the answer: its local index is not available."), dataSource.Name));
+                return [];
+            }
+
+            if (!DataSourceEmbeddingProviders.TryResolve(settingsManager, dataSource, out var embeddingProvider))
+            {
+                logger.LogWarning("Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the selected embedding provider is not available.", dataSource.Name, dataSource.Id);
+                await this.ReportRetrievalGapAsync(dataSource, "no-embedding-provider", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider is not available. Please check it in the settings."), dataSource.Name));
+                return [];
+            }
+
+            if (!await this.QueryFitsEmbeddingProviderAsync(dataSource, embeddingProvider, query, token))
+                return [];
+
+            var provider = embeddingProvider.CreateProvider();
+            var vectors = await provider.EmbedTextAsync(embeddingProvider.Model, settingsManager, token, [query]);
+            token.ThrowIfCancellationRequested();
+            var vector = vectors.FirstOrDefault();
+            if (vector is null || vector.Count == 0)
+            {
+                logger.LogWarning("Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because query embedding returned no vector.", dataSource.Name, dataSource.Id);
+                await this.ReportRetrievalGapAsync(dataSource, "no-query-vector", string.Format(TB("The data source '{0}' was left out of the answer: its embedding provider '{1}' did not return a vector for your message."), dataSource.Name, embeddingProvider.Name));
+                return [];
+            }
+
+            var results = this.LimitSearchResults(
+                dataSource,
+                "vector",
+                await vectorStore.SearchEmbeddingAsync(collectionName, vector, maxMatches, token),
+                maxMatches);
+            this.LogVectorResults(dataSource, results);
+            return results;
+        }
+        catch (OperationCanceledException) when (token.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (ProviderRequestException exception)
+        {
+            //
+            // The embedding provider named the cause and what to do about it. That sentence is
+            // worth far more to the user than the fact that a search came back empty:
+            //
+            logger.LogWarning(
+                exception,
+                "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}) because the embedding provider failed. FailureReason={FailureReason}, StatusCode={StatusCode}.",
+                dataSource.Name, dataSource.Id, exception.FailureReason, exception.StatusCode);
+            await this.ReportRetrievalGapAsync(dataSource, $"provider-{exception.FailureReason}", string.Format(TB("The data source '{0}' was left out of the answer. {1}"), dataSource.Name, exception.UserMessage));
+            return [];
+        }
+        catch (Exception exception)
+        {
+            logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
+            await this.ReportRetrievalGapAsync(dataSource, "vector-search-failed", string.Format(TB("The data source '{0}' was left out of the answer because searching it failed."), dataSource.Name));
+            return [];
+        }
+    }
+
+    /// <summary>
+    /// Tells the user once that a data source cannot take part in answering.
+    /// </summary>
+    /// <remarks>
+    /// A failed search is not an error of the chat: the model still answers, only without what
+    /// this data source knows. Saying so once is what keeps somebody from trusting an answer
+    /// which was put together without half of its sources. Saying it with every prompt would be
+    /// worse than saying nothing, which is why every gap is reported once per session.
+    /// </remarks>
+    /// <param name="dataSource">The data source which could not be searched.</param>
+    /// <param name="gapKey">What kind of gap this is, so a different problem is reported again.</param>
+    /// <param name="userMessage">What to tell the user.</param>
+    private async Task ReportRetrievalGapAsync(IInternalDataSource dataSource, string gapKey, string userMessage)
+    {
+        lock (this.retrievalGapLock)
+        {
+            if (!this.reportedRetrievalGaps.Add($"{dataSource.Id}::{gapKey}"))
+                return;
+        }
+
+        await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.SearchOff, userMessage));
+    }
+
+    private async Task<bool> QueryFitsEmbeddingProviderAsync(
+        IInternalDataSource dataSource,
+        EmbeddingProvider embeddingProvider,
+        string query,
+        CancellationToken token)
+    {
+        var providerTokenLimit = Math.Max(1, embeddingProvider.EffectiveTokenLimit);
+        if (query.Length > RustService.MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH)
+        {
+            logger.LogWarning(
+                "Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the latest prompt has {CharacterCount} characters and exceeds the safe tokenizer request length of {MaxCharacterCount}. ProviderTokenLimit={ProviderTokenLimit}.",
+                dataSource.Name,
+                dataSource.Id,
+                query.Length,
+                RustService.MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH,
+                providerTokenLimit);
+            await this.ReportRetrievalGapAsync(dataSource, "query-too-long", string.Format(TB("The data source '{0}' was left out of the answer because your message is too long to search with."), dataSource.Name));
+            return false;
+        }
+
+        var tokenCountResponse = await rustService.GetTokenCount(embeddingProvider, query, token);
+        if (tokenCountResponse is not { Success: true })
+        {
+            logger.LogWarning(
+                "Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the token count for embedding provider '{EmbeddingProviderName}' could not be determined. Reason='{Reason}'.",
+                dataSource.Name,
+                dataSource.Id,
+                embeddingProvider.Name,
+                tokenCountResponse?.Message ?? "No response was returned by the tokenizer service.");
+            await this.ReportRetrievalGapAsync(dataSource, "no-token-count", string.Format(TB("The data source '{0}' was left out of the answer: the tokenizer of its embedding provider '{1}' is not available."), dataSource.Name, embeddingProvider.Name));
+            return false;
+        }
+
+        var queryTokenCount = tokenCountResponse.Value.TokenCount;
+        if (queryTokenCount > providerTokenLimit)
+        {
+            logger.LogWarning(
+                "Skipping vector retrieval for data source '{DataSourceName}' ({DataSourceId}) because the latest prompt has {QueryTokenCount} tokens, exceeding embedding provider '{EmbeddingProviderName}' limit of {ProviderTokenLimit} tokens.",
+                dataSource.Name,
+                dataSource.Id,
+                queryTokenCount,
+                embeddingProvider.Name,
+                providerTokenLimit);
+            await this.ReportRetrievalGapAsync(dataSource, "query-over-token-limit", string.Format(TB("The data source '{0}' was left out of the answer because your message is longer than its embedding provider '{1}' accepts."), dataSource.Name, embeddingProvider.Name));
+            return false;
+        }
+
+        return true;
+    }
+
+    private async Task<IReadOnlyList<IndexStoreSearchResult>> SearchBm25Async(IInternalDataSource dataSource, string query, int maxMatches, CancellationToken token)
+    {
+        try
+        {
+            var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
+            if (!indexStore.IsAvailable)
+            {
+                logger.LogWarning(
+                    "Skipping BM25 retrieval for data source '{DataSourceName}' ({DataSourceId}) because local RAG index '{DatabaseName}' is unavailable.",
+                    dataSource.Name,
+                    dataSource.Id,
+                    indexStore.Name);
+                return [];
+            }
+
+            var results = this.LimitSearchResults(
+                dataSource,
+                "BM25",
+                await indexStore.SearchChunksAsync(dataSource.Id, query, maxMatches, token),
+                maxMatches);
+            this.LogBm25Results(dataSource, results);
+            return results;
+        }
+        catch (OperationCanceledException) when (token.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (Exception exception)
+        {
+            logger.LogWarning(exception, "BM25 retrieval failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
+            return [];
+        }
+    }
+
+    private IReadOnlyList<T> LimitSearchResults<T>(IInternalDataSource dataSource, string searchName, IReadOnlyList<T> results, int maxMatches)
+    {
+        if (results.Count <= maxMatches)
+            return results;
+
+        logger.LogWarning(
+            "Local RAG {SearchName} search returned {ReturnedHits} chunks for data source '{DataSourceName}' ({DataSourceId}), which exceeds the configured maximum {MaxMatches}. Truncating to the datasource limit.",
+            searchName,
+            results.Count,
+            dataSource.Name,
+            dataSource.Id,
+            maxMatches);
+
+        return results.Take(maxMatches).ToList();
+    }
+
+    private static IReadOnlyList<LocalRetrievalHit> MergeResults(
+        IReadOnlyList<VectorSearchResult> vectorResults,
+        IReadOnlyList<IndexStoreSearchResult> bm25Results,
+        int maxMatches)
+    {
+        // Future reranking should replace this deterministic channel merge.
+        var merged = new List<LocalRetrievalHit>(maxMatches * 2);
+        var seenChunkIds = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
+
+        AppendHits(
+            merged,
+            seenChunkIds,
+            vectorResults
+                .Select((result, index) => FromVectorResult(result, index + 1)),
+            maxMatches);
+
+        AppendHits(
+            merged,
+            seenChunkIds,
+            bm25Results
+                .Select((result, index) => FromBm25Result(result, index + 1)),
+            maxMatches);
+
+        return merged;
+    }
+
+    private static void AppendHits(List<LocalRetrievalHit> merged, HashSet<string> seenChunkIds, IEnumerable<LocalRetrievalHit> hits, int maxNewHits)
+    {
+        var added = 0;
+        foreach (var hit in hits)
+        {
+            if (!string.IsNullOrWhiteSpace(hit.ChunkId) && !seenChunkIds.Add(hit.ChunkId))
+                continue;
+
+            merged.Add(hit);
+            added++;
+            if (added >= maxNewHits)
+                return;
+        }
+    }
+
+    private static LocalRetrievalHit FromVectorResult(VectorSearchResult result, int rank) =>
+        new(
+            RetrievalChannel.VECTOR,
+            result.ChunkId,
+            result.ParentFileId,
+            result.DataSourceId,
+            result.DataSourceName,
+            result.DataSourceType,
+            FirstNonEmpty(result.AbsolutePath, result.FilePath),
+            result.FileName,
+            result.RelativePath,
+            result.FileType,
+            result.PageNumber,
+            result.ChunkIndex,
+            result.Text,
+            result.Score,
+            rank,
+            result.ConfidenceLevel,
+            result.ConfidenceLevelRank);
+
+    private static LocalRetrievalHit FromBm25Result(IndexStoreSearchResult result, int rank) =>
+        new(
+            RetrievalChannel.BM25,
+            result.ChunkId,
+            result.ParentFileId,
+            result.DataSourceId,
+            result.DataSourceName,
+            result.DataSourceType,
+            result.AbsolutePath,
+            result.FileName,
+            result.RelativePath,
+            result.FileType,
+            result.PageNumber,
+            result.ChunkIndex,
+            result.ChunkText,
+            result.Score,
+            rank,
+            result.ConfidenceLevel,
+            result.ConfidenceLevelRank);
+
+    private static RetrievalTextContext ToRetrievalContext(LocalRetrievalHit hit)
+    {
+        var sourceName = FirstNonEmpty(hit.FileName, hit.DataSourceName);
+        var path = FirstNonEmpty(hit.AbsolutePath, hit.RelativePath);
+        var referenceLink = string.IsNullOrWhiteSpace(path) ? string.Empty : BuildReferenceLink(path, hit);
+
+        return new RetrievalTextContext
+        {
+            DataSourceName = sourceName,
+            Category = RetrievalContentCategory.TEXT,
+            Type = GetRetrievalContentType(hit.FileType),
+            Path = path,
+            Links = [],
+            MatchedText = hit.Text,
+            SurroundingContent = [],
+            ReferenceTitle = BuildReferenceTitle(hit),
+            ReferenceLink = referenceLink,
+        };
+    }
+
+    private static string BuildReferenceTitle(LocalRetrievalHit hit)
+    {
+        var sourceName = FirstNonEmpty(hit.FileName, hit.DataSourceName);
+        return BuildLocatedReferenceTitle(sourceName, hit.ChunkIndex, hit.PageNumber);
+    }
+
+    private static string BuildLocatedReferenceTitle(string sourceName, int chunkIndex, int? pageNumber)
+    {
+        var location = pageNumber is > 0
+            ? string.Format(TB("Page {0}"), pageNumber)
+            : string.Format(TB("Chunk {0}"), chunkIndex + 1);
+
+        return $"{sourceName} ({location})";
+    }
+
+    private static string BuildReferenceLink(string path, LocalRetrievalHit hit)
+    {
+        var link = NormalizeLocalReferencePath(path);
+        var separator = link.Contains('#', StringComparison.Ordinal) ? "&" : "#";
+        return $"{link}{separator}chunk={hit.ChunkIndex}";
+    }
+
+    private static string NormalizeLocalReferencePath(string path)
+    {
+        try
+        {
+            return Path.IsPathRooted(path)
+                ? new Uri(Path.GetFullPath(path)).AbsoluteUri
+                : path;
+        }
+        catch
+        {
+            return path;
+        }
+    }
+
+    private static RetrievalContentType GetRetrievalContentType(string fileType)
+    {
+        if (FileTypes.IsAllowedExtension(fileType, FileTypes.TABULAR, FileTypes.SPREADSHEET))
+            return RetrievalContentType.TEXT_SPREADSHEET;
+
+        if (FileTypes.IsAllowedExtension(fileType, FileTypes.POWER_POINT))
+            return RetrievalContentType.TEXT_PRESENTATION;
+
+        return FileTypes.IsAllowedExtension(fileType, FileTypes.HTML)
+            ? RetrievalContentType.TEXT_WEBSITE
+            : RetrievalContentType.TEXT_DOCUMENT;
+    }
+
+    private static string GetQueryText(IContent lastUserPrompt) => lastUserPrompt switch
+    {
+        ContentText text => text.Text,
+        _ => string.Empty
+    };
+
+    private static string FirstNonEmpty(params string[] values) =>
+        values.FirstOrDefault(value => !string.IsNullOrWhiteSpace(value)) ?? string.Empty;
+
+    private void LogVectorResults(IInternalDataSource dataSource, IReadOnlyList<VectorSearchResult> results)
+    {
+        if (results.Count == 0)
+        {
+            logger.LogInformation("Local RAG vector search found no chunks for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
+            return;
+        }
+
+        foreach (var result in results.Select((result, index) => (Result: result, Rank: index + 1)))
+        {
+            logger.LogInformation(
+                "Local RAG vector search found chunk for data source '{DataSourceName}' ({DataSourceId}). Rank={Rank}, Score={Score}, ChunkId='{ChunkId}', ParentFileId='{ParentFileId}', File='{FileName}', Path='{Path}', Title='{Title}'.",
+                dataSource.Name,
+                dataSource.Id,
+                result.Rank,
+                result.Result.Score,
+                result.Result.ChunkId,
+                result.Result.ParentFileId,
+                result.Result.FileName,
+                FirstNonEmpty(result.Result.AbsolutePath, result.Result.FilePath),
+                BuildLocatedReferenceTitle(FirstNonEmpty(result.Result.FileName, dataSource.Name), result.Result.ChunkIndex, result.Result.PageNumber));
+        }
+    }
+
+    private void LogBm25Results(IInternalDataSource dataSource, IReadOnlyList<IndexStoreSearchResult> results)
+    {
+        if (results.Count == 0)
+        {
+            logger.LogInformation("Local RAG BM25 search found no chunks for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
+            return;
+        }
+
+        foreach (var result in results.Select((result, index) => (Result: result, Rank: index + 1)))
+        {
+            logger.LogInformation(
+                "Local RAG BM25 search found chunk for data source '{DataSourceName}' ({DataSourceId}). Rank={Rank}, Score={Score}, ChunkId='{ChunkId}', ParentFileId='{ParentFileId}', File='{FileName}', Path='{Path}', Title='{Title}'.",
+                dataSource.Name,
+                dataSource.Id,
+                result.Rank,
+                result.Result.Score,
+                result.Result.ChunkId,
+                result.Result.ParentFileId,
+                result.Result.FileName,
+                result.Result.AbsolutePath,
+                BuildLocatedReferenceTitle(FirstNonEmpty(result.Result.FileName, dataSource.Name), result.Result.ChunkIndex, result.Result.PageNumber));
+        }
+    }
+}

app/MindWork AI Studio/Tools/Services/DataSourceService.cs

@@ -9,6 +9,15 @@ namespace AIStudio.Tools.Services;
 
 public sealed class DataSourceService
 {
+    //
+    // Trust is recorded for every participating provider, while only the chat provider's trust
+    // decides about external data sources below. The agent which validates retrieval contexts
+    // checks its own provider before it runs, so nothing slips through today. Keeping the value
+    // named here is what makes that asymmetry visible.
+    //
+    // ReSharper disable once NotAccessedPositionalProperty.Local
+    private readonly record struct ParticipatingProvider(string Role, bool IsTrusted, ConfidenceLevel ConfidenceLevel);
+
     private readonly RustService rustService;
     private readonly SettingsManager settingsManager;
     private readonly ILogger<DataSourceService> logger;
@@ -27,9 +36,10 @@ public sealed class DataSourceService
     /// It also returns the data sources selected before when they are still allowed.
     /// </summary>
     /// <param name="selectedLLMProvider">The selected LLM provider.</param>
+    /// <param name="dataSourceOptions">The active data source options, which determine which agent providers participate.</param>
     /// <param name="previousSelectedDataSources">The data sources selected before.</param>
     /// <returns>The allowed data sources and the data sources selected before -- when they are still allowed.</returns>
-    public async Task<AllowedSelectedDataSources> GetDataSources(AIStudio.Settings.Provider selectedLLMProvider, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
+    public async Task<AllowedSelectedDataSources> GetDataSources(AIStudio.Settings.Provider selectedLLMProvider, DataSourceOptions dataSourceOptions, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
     {
         //
         // Case: Somehow the selected LLM provider was not set. The default provider
@@ -42,7 +52,10 @@ public sealed class DataSourceService
             return new([], []);
         }
         
-        return await this.GetDataSources(selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), previousSelectedDataSources);
+        var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
+        var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.Id, dataSourceOptions,
+            new("chat provider", usingTrustedProvider, selectedLLMProvider.GetConfidenceLevel(this.settingsManager)));
+        return await this.GetDataSources(usingTrustedProvider, participatingProviders, previousSelectedDataSources);
     }
 
     /// <summary>
@@ -51,9 +64,10 @@ public sealed class DataSourceService
     /// this method checks only the supplied data sources.
     /// </summary>
     /// <param name="selectedLLMProvider">The selected LLM provider.</param>
+    /// <param name="dataSourceOptions">The active data source options, which determine which agent providers participate.</param>
     /// <param name="requestedDataSources">The data sources to check.</param>
     /// <returns>The requested data sources that are allowed for the provider.</returns>
-    public async Task<IReadOnlyList<IDataSource>> GetAllowedDataSources(AIStudio.Settings.Provider selectedLLMProvider, IReadOnlyCollection<IDataSource> requestedDataSources)
+    public async Task<IReadOnlyList<IDataSource>> GetAllowedDataSources(AIStudio.Settings.Provider selectedLLMProvider, DataSourceOptions dataSourceOptions, IReadOnlyCollection<IDataSource> requestedDataSources)
     {
         if (selectedLLMProvider == Settings.Provider.NONE)
         {
@@ -61,7 +75,10 @@ public sealed class DataSourceService
             return [];
         }
 
-        return await this.GetAllowedDataSources(selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), requestedDataSources);
+        var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
+        var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.Id, dataSourceOptions,
+            new("chat provider", usingTrustedProvider, selectedLLMProvider.GetConfidenceLevel(this.settingsManager)));
+        return await this.GetAllowedDataSources(usingTrustedProvider, participatingProviders, requestedDataSources);
     }
     
     /// <summary>
@@ -69,9 +86,10 @@ public sealed class DataSourceService
     /// It also returns the data sources selected before when they are still allowed.
     /// </summary>
     /// <param name="selectedLLMProvider">The selected LLM provider.</param>
+    /// <param name="dataSourceOptions">The active data source options, which determine which agent providers participate.</param>
     /// <param name="previousSelectedDataSources">The data sources selected before.</param>
     /// <returns>The allowed data sources and the data sources selected before -- when they are still allowed.</returns>
-    public async Task<AllowedSelectedDataSources> GetDataSources(IProvider selectedLLMProvider, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
+    public async Task<AllowedSelectedDataSources> GetDataSources(IProvider selectedLLMProvider, DataSourceOptions dataSourceOptions, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
     {
         //
         // Case: Somehow the selected LLM provider was not set. The default provider
@@ -84,27 +102,60 @@ public sealed class DataSourceService
             return new([], []);
         }
         
-        return await this.GetDataSources(selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), previousSelectedDataSources);
+        var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
+        var participatingProviders = this.GetParticipatingProviders(selectedLLMProvider.ConfiguredProviderId, dataSourceOptions,
+            new("chat provider", usingTrustedProvider, selectedLLMProvider.GetConfidenceLevel(this.settingsManager)));
+        return await this.GetDataSources(usingTrustedProvider, participatingProviders, previousSelectedDataSources);
     }
     
-    private async Task<AllowedSelectedDataSources> GetDataSources(bool usingTrustedProvider, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
+    private IReadOnlyList<ParticipatingProvider> GetParticipatingProviders(string currentProviderId, DataSourceOptions dataSourceOptions, ParticipatingProvider currentProvider)
+    {
+        var providers = new List<ParticipatingProvider> { currentProvider };
+
+        if (dataSourceOptions.AutomaticDataSourceSelection)
+            this.AddAgentProvider(providers, Components.AGENT_DATA_SOURCE_SELECTION, currentProviderId, "data source selection agent");
+
+        if (dataSourceOptions.AutomaticValidation && this.settingsManager.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation)
+            this.AddAgentProvider(providers, Components.AGENT_RETRIEVAL_CONTEXT_VALIDATION, currentProviderId, "retrieval context validation agent");
+
+        return providers;
+    }
+
+    private void AddAgentProvider(List<ParticipatingProvider> providers, Components component, string currentProviderId, string role)
+    {
+        var provider = this.settingsManager.GetPreselectedProvider(component, currentProviderId, true);
+        if (provider == Settings.Provider.NONE)
+        {
+            this.logger.LogWarning($"No provider is available for the {role}. Data sources cannot be made available while this agent is enabled.");
+            providers.Add(new(role, false, ConfidenceLevel.NONE));
+            return;
+        }
+
+        providers.Add(new(
+            role,
+            provider.IsTrustedForDataSourceSecurityChecks(this.settingsManager),
+            provider.GetConfidenceLevel(this.settingsManager)));
+    }
+
+    private async Task<AllowedSelectedDataSources> GetDataSources(bool usingTrustedProvider, IReadOnlyList<ParticipatingProvider> participatingProviders, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
     {
         var allDataSources = this.settingsManager.ConfigurationData.DataSources.ToList();
         var previousSelectedDataSourceIds = previousSelectedDataSources?.Select(source => source.Id).ToHashSet(StringComparer.Ordinal) ?? [];
-        var filteredDataSources = await this.GetAllowedDataSources(usingTrustedProvider, allDataSources);
+        var filteredDataSources = await this.GetAllowedDataSources(usingTrustedProvider, participatingProviders, allDataSources);
         var filteredSelectedDataSources = filteredDataSources.Where(source => previousSelectedDataSourceIds.Contains(source.Id)).ToList();
         
         return new(filteredDataSources, filteredSelectedDataSources);
     }
 
-    private async Task<IReadOnlyList<IDataSource>> GetAllowedDataSources(bool usingTrustedProvider, IReadOnlyCollection<IDataSource> requestedDataSources)
+    private async Task<IReadOnlyList<IDataSource>> GetAllowedDataSources(bool usingTrustedProvider, IReadOnlyList<ParticipatingProvider> participatingProviders, IReadOnlyCollection<IDataSource> requestedDataSources)
     {
         var filteredDataSources = new List<IDataSource>(requestedDataSources.Count);
         var tasks = new List<Task<IDataSource?>>(requestedDataSources.Count);
 
         // Start all checks in parallel:
         foreach (var source in requestedDataSources)
-            tasks.Add(this.CheckOneDataSource(source, usingTrustedProvider));
+            tasks.Add(this.CheckOneDataSource(source, usingTrustedProvider, participatingProviders));
+
 
         // Wait for all checks and collect the results:
         foreach (var task in tasks)
@@ -117,8 +168,35 @@ public sealed class DataSourceService
         return filteredDataSources;
     }
     
-    private async Task<IDataSource?> CheckOneDataSource(IDataSource source, bool usingTrustedProvider)
+    private async Task<IDataSource?> CheckOneDataSource(IDataSource source, bool usingTrustedProvider, IReadOnlyList<ParticipatingProvider> participatingProviders)
     {
+        if (source is IInternalDataSource internalSource)
+        {
+            foreach (var provider in participatingProviders)
+            {
+                if (!provider.ConfidenceLevel.AllowsDataSourceConfidenceLevel(internalSource.ConfidenceLevel))
+                {
+                    this.logger.LogWarning($"The internal data source '{source.Name}' (id={source.Id}) requires provider confidence '{internalSource.ConfidenceLevel.GetName()}'. The {provider.Role} only has confidence '{provider.ConfidenceLevel.GetName()}'. We skip this source.");
+                    return null;
+                }
+            }
+
+            if (!DataSourceEmbeddingProviders.TryResolve(this.settingsManager, source, out var embeddingProvider))
+            {
+                this.logger.LogWarning($"The internal data source '{source.Name}' (id={source.Id}) has no usable embedding provider. We skip this source.");
+                return null;
+            }
+
+            var embeddingProviderConfidence = embeddingProvider.GetConfidenceLevel(this.settingsManager);
+            if (!embeddingProviderConfidence.AllowsDataSourceConfidenceLevel(internalSource.ConfidenceLevel))
+            {
+                this.logger.LogWarning($"The internal data source '{source.Name}' (id={source.Id}) requires provider confidence '{internalSource.ConfidenceLevel.GetName()}'. Its embedding provider '{embeddingProvider.Name}' only has confidence '{embeddingProviderConfidence.GetName()}'. We skip this source.");
+                return null;
+            }
+
+            return source;
+        }
+
         //
         // Unfortunately, we have to live-check any ERI source for its security requirements.
         // Because the ERI server operator might change the security requirements at any time.
@@ -153,8 +231,11 @@ public sealed class DataSourceService
             eriSourceRequirements = securityRequest.Data;
             this.logger.LogInformation($"Security requirements for ERI source '{source.Name}' (id={source.Id}) retrieved successfully.");
         }
-        
-        switch (source.SecurityPolicy)
+
+        if (source is not IExternalDataSource externalSource)
+            return source;
+
+        switch (externalSource.SecurityPolicy)
         {
             case DataSourceSecurity.ALLOW_ANY:
                 
@@ -228,4 +309,4 @@ public sealed class DataSourceService
                 return null;
         }
     }
-}
+}

app/MindWork AI Studio/Tools/Services/DirectChatService.cs

@@ -187,10 +187,24 @@ public sealed class DirectChatService(SettingsManager settingsManager, DataSourc
             requestedDataSources.Add(dataSource);
         }
 
+        //
+        // The options the launched chat will run under. We build them here already, because the
+        // data-source check depends on them: they decide which agent providers take part, and an
+        // agent with too little confidence makes a data source unavailable.
+        //
+        var standardOptions = settingsManager.ConfigurationData.Chat.PreselectedDataSourceOptions;
+        var launchedDataSourceOptions = new DataSourceOptions
+        {
+            DisableDataSources = false,
+            AutomaticDataSourceSelection = false,
+            AutomaticValidation = standardOptions.AutomaticValidation,
+            PreselectedDataSourceIds = requestedDataSources.Select(source => source.Id).ToList(),
+        };
+
         IReadOnlyList<IDataSource> availableDataSources;
         try
         {
-            availableDataSources = await dataSourceService.GetAllowedDataSources(provider, requestedDataSources);
+            availableDataSources = await dataSourceService.GetAllowedDataSources(provider, launchedDataSourceOptions, requestedDataSources);
         }
         catch (Exception exception)
         {
@@ -203,13 +217,6 @@ public sealed class DirectChatService(SettingsManager settingsManager, DataSourc
         if (unavailableDataSources.Count > 0)
             return new(null, string.Format(TB("The following data sources selected by the assistant chat launcher are currently unavailable or not permitted for the selected provider: {0}"), string.Join(", ", unavailableDataSources)));
 
-        var standardOptions = settingsManager.ConfigurationData.Chat.PreselectedDataSourceOptions;
-        return new(new()
-        {
-            DisableDataSources = false,
-            AutomaticDataSourceSelection = false,
-            AutomaticValidation = standardOptions.AutomaticValidation,
-            PreselectedDataSourceIds = requestedDataSources.Select(source => source.Id).ToList(),
-        }, string.Empty);
+        return new(launchedDataSourceOptions, string.Empty);
     }
 }

app/MindWork AI Studio/Tools/Services/EmbeddedFileRecord.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Tools.Services;
+
+public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount);

app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs

@@ -0,0 +1,19 @@
+namespace AIStudio.Tools.Services;
+
+public sealed class FileEnumerationResult
+{
+    public List<FileInfo> Files { get; } = [];
+
+    public List<DataSourceEmbeddingFailure> Failures { get; } = [];
+
+    public int FailedFiles { get; set; }
+
+    public string LastError { get; set; } = string.Empty;
+
+    public void AddFailure(string filePath, string reason)
+    {
+        this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason, DateTimeOffset.UtcNow));
+        this.FailedFiles = this.Failures.Count;
+        this.LastError = reason;
+    }
+}

app/MindWork AI Studio/Tools/Services/PermanentIndexingFailureRecord.cs

@@ -0,0 +1,3 @@
+namespace AIStudio.Tools.Services;
+
+public sealed record PermanentIndexingFailureRecord(string Fingerprint, FileExtractionErrorCode Code, string Message, DateTimeOffset OccurredAtUtc);

app/MindWork AI Studio/Tools/Services/RustService.Databases.cs

@@ -49,5 +49,22 @@ public sealed partial class RustService
             throw new InvalidOperationException(operation?.Issue ?? $"The {databaseName} operation failed.");
     }
 
+    public async Task<TResult?> ExecuteDatabaseQuery<TRequest, TResult>(string databaseName, string path, TRequest request, CancellationToken cancellationToken = default)
+    {
+        using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+        cts.CancelAfter(TimeSpan.FromMinutes(5));
+
+        using var response = await this.http.PostAsJsonAsync(path, request, this.jsonRustSerializerOptions, cts.Token);
+        response.EnsureSuccessStatusCode();
+
+        var operation = await response.Content.ReadFromJsonAsync<DatabaseQueryResponse<TResult>>(this.jsonRustSerializerOptions, cts.Token);
+        if (operation is not { Success: true })
+            throw new InvalidOperationException(operation?.Issue ?? $"The {databaseName} query failed.");
+
+        return operation.Data;
+    }
+
     private sealed record DatabaseOperationResponse(bool Success, string Issue);
+
+    private sealed record DatabaseQueryResponse<TResult>(bool Success, string Issue, TResult? Data);
 }

app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs

@@ -1,5 +1,8 @@
 using System.Text;
 using System.Text.Json;
+using System.Runtime.CompilerServices;
+
+using AIStudio.Settings;
 using AIStudio.Tools.Security;
 
 namespace AIStudio.Tools.Services;
@@ -38,7 +41,7 @@ public sealed partial class RustService
         var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
 
         var streamId = Guid.NewGuid().ToString();
-        var requestUri = $"/retrieval/fs/extract?path={Uri.EscapeDataString(path)}&stream_id={streamId}&extract_images={extractImages}";
+        var requestUri = $"/retrieval/fs/extract?path={Uri.EscapeDataString(path)}&stream_id={streamId}&extract_images={extractImages}&include_token_count=false";
 
         //
         // Both reasons to stop end the same read, so we combine them: our own timeout bounds the
@@ -169,7 +172,11 @@ public sealed partial class RustService
                 }
                 catch (JsonException e)
                 {
-                    this.logger?.LogError(e, "Failed to deserialize SSE event while reading '{Path}': {JsonContent}", path, jsonContent);
+                    // The runtime may report a failure as a bare JSON string instead of a chunk.
+                    // That form still carries a readable reason, so we log it as such -- but it
+                    // remains a failure and must reach the caller like any other:
+                    if (!this.TryLogSseErrorMessage(jsonContent, path))
+                        this.logger?.LogError(e, "Failed to deserialize SSE event while reading '{Path}': {JsonContent}", path, jsonContent);
 
                     if (failureCode is FileExtractionErrorCode.NONE)
                     {
@@ -201,9 +208,9 @@ public sealed partial class RustService
         }
         finally
         {
-            var finalContentChunk = ContentStreamSseHandler.Clear(streamId);
-            if (!string.IsNullOrWhiteSpace(finalContentChunk))
-                resultBuilder.AppendLine(finalContentChunk);
+            // Reading the whole file at once needs no token counts, so only the content is used here:
+            if (ContentStreamSseHandler.Clear(streamId) is { } finalContentChunk && !string.IsNullOrWhiteSpace(finalContentChunk.Content))
+                resultBuilder.AppendLine(finalContentChunk.Content);
         }
 
         if (failureCode is not FileExtractionErrorCode.NONE)
@@ -246,4 +253,207 @@ public sealed partial class RustService
             PromptInjectionRedactedCount = promptInjectionRedactedCount,
         };
     }
-}
+
+    public async IAsyncEnumerable<string> StreamArbitraryFileData(string path, bool extractImages = false, [EnumeratorCancellation] CancellationToken token = default)
+    {
+        await foreach (var segment in this.StreamArbitraryFileDataCore(path, extractImages, false, string.Empty, token))
+            yield return segment.Content;
+    }
+
+    public async IAsyncEnumerable<ArbitraryFileDataSegment> StreamArbitraryFileDataWithTokenCounts(
+        string path,
+        EmbeddingProvider embeddingProvider,
+        [EnumeratorCancellation] CancellationToken token = default)
+    {
+        await foreach (var segment in this.StreamArbitraryFileDataCore(path, false, true, embeddingProvider.TokenizerPath, token))
+        {
+            if (segment.TokenCount is { } tokenCount)
+            {
+                yield return new(segment.Content, tokenCount);
+                continue;
+            }
+
+            //
+            // A segment the runtime did not count, e.g. a page which carries an embedded image on
+            // top of its text. The runtime leaves such a count out on purpose instead of failing
+            // the extraction, because we can count the segment ourselves. Without this, a document
+            // would be dropped over a number we are able to produce.
+            //
+            var countedSegment = await this.GetTokenCount(embeddingProvider, segment.Content, token);
+            if (countedSegment is { Success: true } counted)
+            {
+                yield return new(segment.Content, counted.TokenCount);
+                continue;
+            }
+
+            //
+            // Carries a code so callers can classify it: the file itself is fine, the answer of
+            // the runtime was not, which makes this worth another attempt.
+            //
+            throw new FileExtractionException(FileExtractionErrorCode.INVALID_RESPONSE, $"Rust did not return a token count for an extracted segment from '{path}' using provider '{embeddingProvider.Name}', and counting it afterwards failed as well: {countedSegment?.Message}");
+        }
+    }
+
+    private async IAsyncEnumerable<(string Content, int? TokenCount)> StreamArbitraryFileDataCore(
+        string path,
+        bool extractImages,
+        bool includeTokenCount,
+        string tokenizerPath,
+        [EnumeratorCancellation] CancellationToken token)
+    {
+        var streamId = Guid.NewGuid().ToString();
+        var requestUri = $"/retrieval/fs/extract?path={Uri.EscapeDataString(path)}&stream_id={streamId}&extract_images={extractImages}&include_token_count={includeTokenCount}&tokenizer_path={Uri.EscapeDataString(tokenizerPath)}";
+        using var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
+        using var response = await this.http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
+
+        if (!response.IsSuccessStatusCode)
+        {
+            var responseBody = await response.Content.ReadAsStringAsync(token);
+            this.logger?.LogError(
+                "Failed to stream arbitrary file data from Rust runtime. Status: {StatusCode}, reason: '{ReasonPhrase}', path: '{Path}', body: '{Body}'",
+                response.StatusCode,
+                response.ReasonPhrase,
+                path,
+                responseBody);
+
+            if (includeTokenCount)
+                throw new InvalidOperationException($"Rust could not extract and count '{path}'. HTTP {(int)response.StatusCode} ({response.ReasonPhrase}): {responseBody}");
+
+            yield break;
+        }
+
+        var promptInjectionFindings = new List<PromptInjectionFinding>();
+        var promptInjectionRedactedCount = 0;
+
+        ContentStreamPendingContent? finalContentChunk;
+        try
+        {
+            await using var stream = await response.Content.ReadAsStreamAsync(token);
+            using var reader = new StreamReader(stream);
+
+            while (!reader.EndOfStream && !token.IsCancellationRequested)
+            {
+                var line = await reader.ReadLineAsync(token);
+                if (string.IsNullOrWhiteSpace(line))
+                    continue;
+
+                if (!line.StartsWith("data:", StringComparison.InvariantCulture))
+                    continue;
+
+                var jsonContent = line[5..];
+                ContentStreamSseEvent? sseEvent = null;
+                try
+                {
+                    sseEvent = JsonSerializer.Deserialize<ContentStreamSseEvent>(jsonContent);
+                }
+                catch (JsonException)
+                {
+                    if (this.TryLogSseErrorMessage(jsonContent, path))
+                    {
+                        if (includeTokenCount)
+                            throw new InvalidOperationException($"Rust could not extract and count a segment from '{path}'. See the runtime log for details.");
+
+                        continue;
+                    }
+
+                    this.logger?.LogError("Failed to deserialize SSE event: {JsonContent}", jsonContent);
+                }
+
+                if (sseEvent is null)
+                    continue;
+
+                var processedEvent = ContentStreamSseHandler.ProcessEvent(sseEvent, extractImages);
+                if (processedEvent.Error is { } error)
+                {
+                    // A notice says something about the file without failing the read, so the
+                    // remaining content still belongs into the index:
+                    if (error.IsNotice)
+                    {
+                        this.logger?.LogInformation(
+                            "The runtime reported a notice while reading '{Path}' for embedding: code={ErrorCode}, detectedFormat='{DetectedFormat}', message='{Message}'",
+                            path,
+                            error.ParsedCode,
+                            error.DetectedFormat,
+                            error.Message);
+
+                        continue;
+                    }
+
+                    //
+                    // Everything else stops the read. Embedding a document which was only read in
+                    // part would put a silently incomplete text into the index, and nothing after
+                    // this point would reveal the gap:
+                    //
+                    this.logger?.LogError(
+                        "The runtime reported a failure while reading '{Path}' for embedding: code={ErrorCode}, page={PageNumber}, detectedFormat='{DetectedFormat}', message='{Message}'",
+                        path,
+                        error.ParsedCode,
+                        error.PageNumber,
+                        error.DetectedFormat,
+                        error.Message);
+
+                    throw new FileExtractionException(error.ParsedCode, $"Rust could not extract '{path}': {error.Message}", error.PageNumber, error.DetectedFormat);
+                }
+
+                if (processedEvent.PromptInjection is { } promptInjection)
+                {
+                    //
+                    // Not a failure: the passages were removed and the document around them is
+                    // intact, so what remains still belongs into the index. It only has to reach
+                    // the user, because from here on the indexed document is no longer the one
+                    // sitting on their disk.
+                    //
+                    promptInjectionRedactedCount += promptInjection.RedactedCount;
+                    if (promptInjection.Findings is { } findings)
+                        promptInjectionFindings.AddRange(findings);
+
+                    continue;
+                }
+
+                //
+                // The count comes from the processed event, not from the event which was just read:
+                // a reader may hold content back across several events, and the count of the content
+                // it releases is the count of that content, not of the event that released it.
+                //
+                if (!string.IsNullOrWhiteSpace(processedEvent.Content))
+                    yield return (processedEvent.Content, processedEvent.TokenCount);
+            }
+        }
+        finally
+        {
+            finalContentChunk = ContentStreamSseHandler.Clear(streamId);
+        }
+
+        if (finalContentChunk is { } pendingContent && !string.IsNullOrWhiteSpace(pendingContent.Content))
+            yield return (pendingContent.Content, pendingContent.TokenCount);
+
+        if (promptInjectionRedactedCount is 0)
+            yield break;
+
+        //
+        // Reported from here for the same reason as in ReadArbitraryFileData above: these two
+        // methods together are every way of reading a file, so they are the only two places
+        // where no caller can forget the report. Here it was missing, which is why a whole
+        // indexing run could filter documents without ever saying so.
+        //
+        var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
+        await guardService.ReportAsync(new(PromptInjectionSource.FileContent(path), promptInjectionFindings, promptInjectionRedactedCount));
+    }
+
+    private bool TryLogSseErrorMessage(string jsonContent, string path)
+    {
+        try
+        {
+            var errorMessage = JsonSerializer.Deserialize<string>(jsonContent);
+            if (string.IsNullOrWhiteSpace(errorMessage))
+                return false;
+
+            this.logger?.LogError("Rust retrieval stream error for '{Path}': {ErrorMessage}", path, errorMessage);
+            return true;
+        }
+        catch (JsonException)
+        {
+            return false;
+        }
+    }
+}

app/MindWork AI Studio/Tools/Services/RustService.Tokenizer.cs

@@ -0,0 +1,87 @@
+using AIStudio.Settings;
+using AIStudio.Tools.Rust;
+
+namespace AIStudio.Tools.Services;
+
+public sealed partial class RustService
+{
+    internal const int MAX_TOKEN_COUNT_REQUEST_TEXT_LENGTH = 200_000;
+
+    private static TokenizerResponse CreateUnavailableTokenizerResponse(string message) => new(
+        false,
+        0,
+        message,
+        string.Empty);
+
+    public async Task<TokenizerResponse> ValidateTokenizer(string filePath)
+    {
+        var result = await this.http.PostAsJsonAsync("/tokenizer/validate", new {
+            file_path = filePath,
+        }, this.jsonRustSerializerOptions);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError($"Failed to validate the tokenizer '{result.StatusCode}'");
+            return CreateUnavailableTokenizerResponse("An error occured while sending the path to the Rust framework for validation: "+result.StatusCode);
+        }
+
+        var response = await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
+
+        return response;
+    }
+    
+    public async Task<TokenizerResponse> StoreTokenizer(string modelId, string filePath)
+    {
+        this.logger!.LogInformation($"Storing tokenizer for model '{modelId}' from file '{filePath}'");
+        var result = await this.http.PostAsJsonAsync("/tokenizer/store", new {
+            model_id = modelId,
+            file_path = filePath,
+        }, this.jsonRustSerializerOptions);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError($"Failed to store the tokenizer '{result.StatusCode}'");
+            return CreateUnavailableTokenizerResponse("An error occured while sending the path to the Rust framework for storing: "+result.StatusCode);
+        }
+
+        return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
+    }
+
+    public async Task<TokenizerResponse> DeleteTokenizer(string modelId)
+    {
+        this.logger!.LogInformation($"Deleting tokenizer for model '{modelId}'");
+        var result = await this.http.PostAsJsonAsync("/tokenizer/delete", new {
+            model_id = modelId,
+        }, this.jsonRustSerializerOptions);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError($"Failed to delete the tokenizer '{result.StatusCode}'");
+            return CreateUnavailableTokenizerResponse("An error occured while sending the tokenizer delete request to the Rust framework: "+result.StatusCode);
+        }
+
+        return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions);
+    }
+    
+    public Task<TokenizerResponse?> GetTokenCount(AIStudio.Settings.Provider provider, string text, CancellationToken cancellationToken = default) =>
+        this.GetTokenCount(provider.InstanceName, provider.TokenizerPath, text, cancellationToken);
+
+    public Task<TokenizerResponse?> GetTokenCount(EmbeddingProvider provider, string text, CancellationToken cancellationToken = default) =>
+        this.GetTokenCount(provider.Name, provider.TokenizerPath, text, cancellationToken);
+
+    private async Task<TokenizerResponse?> GetTokenCount(string providerName, string tokenizerPath, string text, CancellationToken cancellationToken)
+    {
+        var result = await this.http.PostAsJsonAsync("/tokenizer/count", new {
+            text,
+            tokenizer_path = tokenizerPath,
+        }, this.jsonRustSerializerOptions, cancellationToken);
+
+        if (!result.IsSuccessStatusCode)
+        {
+            this.logger!.LogError("Failed to get the token count for provider '{ProviderName}': {StatusCode}", providerName, result.StatusCode);
+            return CreateUnavailableTokenizerResponse("Error while getting token count from Rust service: "+result.StatusCode);
+        }
+
+        return await result.Content.ReadFromJsonAsync<TokenizerResponse>(this.jsonRustSerializerOptions, cancellationToken);
+    }
+}

app/MindWork AI Studio/Tools/Services/TokenizerModelId.cs

@@ -0,0 +1,20 @@
+namespace AIStudio.Tools.Services;
+
+public static class TokenizerModelId
+{
+    public static string ForProvider(Settings.Provider provider) => ForProviderId(provider.Id);
+
+    public static string ForProviderId(string guid) => "chat_" + NormalizeGuid(guid);
+
+    public static string ForEmbeddingProvider(Settings.EmbeddingProvider provider) => ForEmbeddingProviderId(provider.Id);
+
+    public static string ForEmbeddingProviderId(string guid) => "embedding_" + NormalizeGuid(guid);
+
+    private static string NormalizeGuid(string guid)
+    {
+        if (Guid.TryParse(guid, out var parsedGuid))
+            return parsedGuid.ToString("D");
+
+        return guid.Trim();
+    }
+}