Merge branch 'main' into chunk-data

app/MindWork AI Studio/Tools/Services/EnterpriseEnvironmentService.cs

@@ -200,7 +200,7 @@ public sealed class EnterpriseEnvironmentService(ILogger<EnterpriseEnvironmentSe
             {
                 logger.LogInformation("The enterprise encryption secret changed. Refreshing the enterprise encryption service and reloading plugins.");
                 PluginFactory.InitializeEnterpriseEncryption(enterpriseEncryptionSecret);
-                await this.RemoveEnterpriseManagedApiKeysAsync();
+                await this.RemoveEnterpriseManagedSecretsAsync();
                 await PluginFactory.LoadAll();
             }
 
@@ -249,34 +249,36 @@ public sealed class EnterpriseEnvironmentService(ILogger<EnterpriseEnvironmentSe
         return serverUrl.Trim().TrimEnd('/');
     }
 
-    private async Task RemoveEnterpriseManagedApiKeysAsync()
+    private async Task RemoveEnterpriseManagedSecretsAsync()
     {
         var secretTargets = GetEnterpriseManagedSecretTargets();
         if (secretTargets.Count == 0)
         {
-            logger.LogInformation("No enterprise-managed API keys are currently known in the settings. No keyring cleanup is required.");
+            logger.LogInformation("No enterprise-managed secrets are currently known in the settings. No keyring cleanup is required.");
             return;
         }
 
-        logger.LogInformation("Removing {SecretCount} enterprise-managed API key(s) from the OS keyring after an enterprise encryption secret change.", secretTargets.Count);
+        logger.LogInformation("Removing {SecretCount} enterprise-managed secret(s) from the OS keyring after an enterprise encryption secret change.", secretTargets.Count);
         foreach (var target in secretTargets)
         {
             try
             {
-                var deleteResult = await rustService.DeleteAPIKey(target, target.StoreType);
+                var deleteResult = target.StoreType is SecretStoreType.DATA_SOURCE
+                    ? await rustService.DeleteSecret(target, target.StoreType)
+                    : await rustService.DeleteAPIKey(target, target.StoreType);
                 if (deleteResult.Success)
                 {
                     if (deleteResult.WasEntryFound)
-                        logger.LogInformation("Successfully deleted enterprise-managed API key '{SecretName}' from the OS keyring.", target.SecretName);
+                        logger.LogInformation("Successfully deleted enterprise-managed secret '{SecretName}' from the OS keyring.", target.SecretName);
                     else
-                        logger.LogInformation("Enterprise-managed API key '{SecretName}' was already absent from the OS keyring.", target.SecretName);
+                        logger.LogInformation("Enterprise-managed secret '{SecretName}' was already absent from the OS keyring.", target.SecretName);
                 }
                 else
-                    logger.LogWarning("Failed to delete enterprise-managed API key '{SecretName}' from the OS keyring: {Issue}", target.SecretName, deleteResult.Issue);
+                    logger.LogWarning("Failed to delete enterprise-managed secret '{SecretName}' from the OS keyring: {Issue}", target.SecretName, deleteResult.Issue);
             }
             catch (Exception e)
             {
-                logger.LogWarning(e, "Failed to delete enterprise-managed API key '{SecretName}' from the OS keyring.", target.SecretName);
+                logger.LogWarning(e, "Failed to delete enterprise-managed secret '{SecretName}' from the OS keyring.", target.SecretName);
             }
         }
     }
@@ -289,6 +291,7 @@ public sealed class EnterpriseEnvironmentService(ILogger<EnterpriseEnvironmentSe
         AddEnterpriseManagedSecretTargets(configurationData.Providers, SecretStoreType.LLM_PROVIDER, secretTargets);
         AddEnterpriseManagedSecretTargets(configurationData.EmbeddingProviders, SecretStoreType.EMBEDDING_PROVIDER, secretTargets);
         AddEnterpriseManagedSecretTargets(configurationData.TranscriptionProviders, SecretStoreType.TRANSCRIPTION_PROVIDER, secretTargets);
+        AddEnterpriseManagedSecretTargets(configurationData.DataSources.OfType<IExternalDataSource>(), SecretStoreType.DATA_SOURCE, secretTargets);
 
         return secretTargets.ToList();
     }

app/MindWork AI Studio/Tools/Services/GlobalShortcutService.cs

@@ -185,9 +185,7 @@ public sealed class GlobalShortcutService : BackgroundService, IMessageBusReceiv
             return new(shortcut, isEnabled, false);
 
         var fallbackShortcut = settingsSnapshot.App.ShortcutVoiceRecording;
-        var fallbackEnabled =
-            settingsSnapshot.App.EnabledPreviewFeatures.Contains(PreviewFeatures.PRE_SPEECH_TO_TEXT_2026) &&
-            !string.IsNullOrWhiteSpace(settingsSnapshot.App.UseTranscriptionProvider);
+        var fallbackEnabled = !string.IsNullOrWhiteSpace(settingsSnapshot.App.UseTranscriptionProvider);
 
         if (!fallbackEnabled || string.IsNullOrWhiteSpace(fallbackShortcut))
             return new(shortcut, isEnabled, false);

app/MindWork AI Studio/Tools/Services/RustService.Databases.cs

@@ -5,49 +5,27 @@ namespace AIStudio.Tools.Services;
 
 public sealed partial class RustService
 {
-    public async Task<EmbeddingStoreConfiguration> GetEmbeddingStoreConfiguration(EmbeddingStoreKind kind)
-    {
-        switch (kind)
-        {
-            case EmbeddingStoreKind.QDRANT_REMOTE:
-            {
-                var qdrantInfo = await this.GetQdrantInfo();
-                var invalidFields = new List<string>();
-                if (!qdrantInfo.IsAvailable)
-                    invalidFields.Add(qdrantInfo.UnavailableReason ?? "unknown");
-                if (string.IsNullOrWhiteSpace(qdrantInfo.Path))
-                    invalidFields.Add("Path");
-                if (qdrantInfo.PortHttp == 0)
-                    invalidFields.Add("HttpPort");
-                if (qdrantInfo.PortGrpc == 0)
-                    invalidFields.Add("GrpcPort");
-                if (string.IsNullOrWhiteSpace(qdrantInfo.Fingerprint))
-                    invalidFields.Add("Fingerprint");
-                if (string.IsNullOrWhiteSpace(qdrantInfo.ApiToken))
-                    invalidFields.Add("ApiToken");
-                if (invalidFields.Count <= 0) return new EmbeddingStoreConfiguration(kind, "Qdrant", new RemoteLocation(qdrantInfo.Path, qdrantInfo.PortHttp, qdrantInfo.PortGrpc, qdrantInfo.Fingerprint, qdrantInfo.ApiToken), null);
-                var reason = string.Join(", ", invalidFields);
-                Console.WriteLine($"Warning: Qdrant is not available. Starting without vector database. Reason: '{reason}'.");
-
-                return new EmbeddingStoreConfiguration(
-                    EmbeddingStoreKind.NONE,
-                    "Qdrant",
-                    null,
-                    reason);
-
-            }
-            default:
-                return new EmbeddingStoreConfiguration(kind, kind.ToString(), null, $"No configuration available for {kind}");
-        }
-    }
-    
-    public async Task<QdrantInfo> GetQdrantInfo()
+    public async Task<QdrantInfo> GetQdrantInfo(CancellationToken cancellationToken = default)
     {
         try
         {
-            var cts = new CancellationTokenSource(TimeSpan.FromSeconds(45));
-            var response = await this.http.GetFromJsonAsync<QdrantInfo>("/system/qdrant/info", this.jsonRustSerializerOptions, cts.Token);
-            return response;
+            using var cts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
+            cts.CancelAfter(TimeSpan.FromSeconds(45));
+            
+            return await this.http.GetFromJsonAsync<QdrantInfo>("/system/qdrant/info", this.jsonRustSerializerOptions, cts.Token);
+        }
+        catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested)
+        {
+            if(this.logger is not null)
+                this.logger.LogWarning("Fetching Qdrant info from Rust service was cancelled by caller.");
+            else
+                Console.WriteLine("Fetching Qdrant info from Rust service was cancelled by caller.");
+            
+            return new QdrantInfo
+            {
+                Status = QdrantStatus.UNAVAILABLE,
+                UnavailableReason = "Operation cancelled by caller."
+            };
         }
         catch (Exception e)
         {
@@ -56,7 +34,11 @@ public sealed partial class RustService
             else
                 Console.WriteLine($"Error while fetching Qdrant info from Rust service: '{e}'.");
             
-            return default;
+            return new QdrantInfo
+            {
+                Status = QdrantStatus.UNAVAILABLE,
+                UnavailableReason = e.Message
+            };
         }
     }
 }

app/MindWork AI Studio/Tools/Services/RustService.FileSystem.cs

@@ -6,8 +6,11 @@ public sealed partial class RustService
 {
     public async Task<DirectorySelectionResponse> SelectDirectory(string title, string? initialDirectory = null)
     {
-        PreviousDirectory? previousDirectory = initialDirectory is null ? null : new (initialDirectory);
-        var result = await this.http.PostAsJsonAsync($"/select/directory?title={title}", previousDirectory, this.jsonRustSerializerOptions);
+        var encodedTitle = Uri.EscapeDataString(title);
+        var result = initialDirectory is null
+            ? await this.http.PostAsync($"/select/directory?title={encodedTitle}", null)
+            : await this.http.PostAsJsonAsync($"/select/directory?title={encodedTitle}", new PreviousDirectory(initialDirectory), this.jsonRustSerializerOptions);
+        
         if (!result.IsSuccessStatusCode)
         {
             this.logger!.LogError($"Failed to select a directory: '{result.StatusCode}'");

app/MindWork AI Studio/Tools/Services/RustService.OS.cs

@@ -32,4 +32,35 @@ public sealed partial class RustService
             this.userLanguageLock.Release();
         }
     }
+
+    public async Task<string> ReadUserName(bool forceRequest = false)
+    {
+        if (!forceRequest && !string.IsNullOrWhiteSpace(this.cachedUserName))
+            return this.cachedUserName;
+
+        await this.userNameLock.WaitAsync();
+        try
+        {
+            if (!forceRequest && !string.IsNullOrWhiteSpace(this.cachedUserName))
+                return this.cachedUserName;
+
+            var response = await this.http.GetAsync("/system/username");
+            if (!response.IsSuccessStatusCode)
+            {
+                this.logger!.LogError($"Failed to read the user name from Rust: '{response.StatusCode}'");
+                return string.Empty;
+            }
+
+            var userName = (await response.Content.ReadAsStringAsync()).Trim();
+            if (string.IsNullOrWhiteSpace(userName))
+                return string.Empty;
+
+            this.cachedUserName = userName;
+            return userName;
+        }
+        finally
+        {
+            this.userNameLock.Release();
+        }
+    }
 }

app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs

@@ -14,7 +14,16 @@ public sealed partial class RustService
         var response = await this.http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
 
         if (!response.IsSuccessStatusCode)
+        {
+            var responseBody = await response.Content.ReadAsStringAsync();
+            this.logger?.LogError(
+                "Failed to read arbitrary file data from Rust runtime. Status: {StatusCode}, reason: '{ReasonPhrase}', path: '{Path}', body: '{Body}'",
+                response.StatusCode,
+                response.ReasonPhrase,
+                path,
+                responseBody);
             return string.Empty;
+        }
 
         var resultBuilder = new StringBuilder();
 

app/MindWork AI Studio/Tools/Services/RustService.Secrets.cs

@@ -4,26 +4,34 @@ namespace AIStudio.Tools.Services;
 
 public sealed partial class RustService
 {
+    private static string SecretKey(ISecretId secretId, SecretStoreType storeType) => $"{storeType.Prefix()}::{secretId.SecretId}::{secretId.SecretName}";
+
+    private static string LegacySecretKey(ISecretId secretId) => $"secret::{secretId.SecretId}::{secretId.SecretName}";
+
     /// <summary>
     /// Try to get the secret data for the given secret ID.
     /// </summary>
     /// <param name="secretId">The secret ID to get the data for.</param>
+    /// <param name="storeType">The secret store type.</param>
     /// <param name="isTrying">Indicates if we are trying to get the data. In that case, we don't log errors.</param>
     /// <returns>The requested secret.</returns>
-    public async Task<RequestedSecret> GetSecret(ISecretId secretId, bool isTrying = false)
+    public async Task<RequestedSecret> GetSecret(ISecretId secretId, SecretStoreType storeType, bool isTrying = false)
     {
-        var secretRequest = new SelectSecretRequest($"secret::{secretId.SecretId}::{secretId.SecretName}", Environment.UserName, isTrying);
-        var result = await this.http.PostAsJsonAsync("/secrets/get", secretRequest, this.jsonRustSerializerOptions);
-        if (!result.IsSuccessStatusCode)
+        var secretKey = SecretKey(secretId, storeType);
+        var secret = await this.GetSecretByKey(secretKey, isTrying || storeType is SecretStoreType.DATA_SOURCE);
+        if (secret.Success || storeType is not SecretStoreType.DATA_SOURCE)
+            return secret;
+
+        var legacySecretKey = LegacySecretKey(secretId);
+        var legacySecret = await this.GetSecretByKey(legacySecretKey, isTrying: true);
+        if (legacySecret.Success)
         {
-            if(!isTrying)
-                this.logger!.LogError($"Failed to get the secret data for secret ID '{secretId.SecretId}' due to an API issue: '{result.StatusCode}'");
-            return new RequestedSecret(false, new EncryptedText(string.Empty), TB("Failed to get the secret data due to an API issue."));
+            this.logger!.LogDebug($"Successfully retrieved the legacy data source secret for '{legacySecretKey}'.");
+            return legacySecret;
         }
-        
-        var secret = await result.Content.ReadFromJsonAsync<RequestedSecret>(this.jsonRustSerializerOptions);
+
         if (!secret.Success && !isTrying)
-            this.logger!.LogError($"Failed to get the secret data for secret ID '{secretId.SecretId}': '{secret.Issue}'");
+            this.logger!.LogError($"Failed to get the secret data for '{secretKey}': '{secret.Issue}'");
         
         return secret;
     }
@@ -33,21 +41,26 @@ public sealed partial class RustService
     /// </summary>
     /// <param name="secretId">The secret ID to store the data for.</param>
     /// <param name="secretData">The data to store.</param>
+    /// <param name="storeType">The secret store type.</param>
     /// <returns>The store secret response.</returns>
-    public async Task<StoreSecretResponse> SetSecret(ISecretId secretId, string secretData)
+    public async Task<StoreSecretResponse> SetSecret(ISecretId secretId, string secretData, SecretStoreType storeType)
     {
+        var secretKey = SecretKey(secretId, storeType);
         var encryptedSecret = await this.encryptor!.Encrypt(secretData);
-        var request = new StoreSecretRequest($"secret::{secretId.SecretId}::{secretId.SecretName}", Environment.UserName, encryptedSecret);
+        var request = new StoreSecretRequest(secretKey, Environment.UserName, encryptedSecret);
         var result = await this.http.PostAsJsonAsync("/secrets/store", request, this.jsonRustSerializerOptions);
         if (!result.IsSuccessStatusCode)
         {
-            this.logger!.LogError($"Failed to store the secret data for secret ID '{secretId.SecretId}' due to an API issue: '{result.StatusCode}'");
-            return new StoreSecretResponse(false, TB("Failed to get the secret data due to an API issue."));
+            this.logger!.LogError($"Failed to store the secret data for '{secretKey}' due to an API issue: '{result.StatusCode}'");
+            return new StoreSecretResponse(false, TB("Failed to store the secret data due to an API issue."));
         }
         
         var state = await result.Content.ReadFromJsonAsync<StoreSecretResponse>(this.jsonRustSerializerOptions);
         if (!state.Success)
-            this.logger!.LogError($"Failed to store the secret data for secret ID '{secretId.SecretId}': '{state.Issue}'");
+            this.logger!.LogError($"Failed to store the secret data for '{secretKey}': '{state.Issue}'");
+
+        if (state.Success && storeType is SecretStoreType.DATA_SOURCE)
+            await this.DeleteSecretByKey(LegacySecretKey(secretId));
         
         return state;
     }
@@ -56,20 +69,48 @@ public sealed partial class RustService
     /// Tries to delete the secret data for the given secret ID.
     /// </summary>
     /// <param name="secretId">The secret ID to delete the data for.</param>
+    /// <param name="storeType">The secret store type.</param>
     /// <returns>The delete secret response.</returns>
-    public async Task<DeleteSecretResponse> DeleteSecret(ISecretId secretId)
+    public async Task<DeleteSecretResponse> DeleteSecret(ISecretId secretId, SecretStoreType storeType)
     {
-        var request = new SelectSecretRequest($"secret::{secretId.SecretId}::{secretId.SecretName}", Environment.UserName, false);
+        var deleteResult = await this.DeleteSecretByKey(SecretKey(secretId, storeType));
+        if (storeType is not SecretStoreType.DATA_SOURCE || !deleteResult.Success)
+            return deleteResult;
+
+        var legacyDeleteResult = await this.DeleteSecretByKey(LegacySecretKey(secretId));
+        if (!legacyDeleteResult.Success)
+            return legacyDeleteResult;
+
+        return deleteResult with { WasEntryFound = deleteResult.WasEntryFound || legacyDeleteResult.WasEntryFound };
+    }
+
+    private async Task<RequestedSecret> GetSecretByKey(string secretKey, bool isTrying)
+    {
+        var secretRequest = new SelectSecretRequest(secretKey, Environment.UserName, isTrying);
+        var result = await this.http.PostAsJsonAsync("/secrets/get", secretRequest, this.jsonRustSerializerOptions);
+        if (!result.IsSuccessStatusCode)
+        {
+            if(!isTrying)
+                this.logger!.LogError($"Failed to get the secret data for '{secretKey}' due to an API issue: '{result.StatusCode}'");
+            return new RequestedSecret(false, new EncryptedText(string.Empty), TB("Failed to get the secret data due to an API issue."));
+        }
+
+        return await result.Content.ReadFromJsonAsync<RequestedSecret>(this.jsonRustSerializerOptions);
+    }
+
+    private async Task<DeleteSecretResponse> DeleteSecretByKey(string secretKey)
+    {
+        var request = new SelectSecretRequest(secretKey, Environment.UserName, false);
         var result = await this.http.PostAsJsonAsync("/secrets/delete", request, this.jsonRustSerializerOptions);
         if (!result.IsSuccessStatusCode)
         {
-            this.logger!.LogError($"Failed to delete the secret data for secret ID '{secretId.SecretId}' due to an API issue: '{result.StatusCode}'");
+            this.logger!.LogError($"Failed to delete the secret data for '{secretKey}' due to an API issue: '{result.StatusCode}'");
             return new DeleteSecretResponse{Success = false, WasEntryFound = false, Issue = TB("Failed to delete the secret data due to an API issue.")};
         }
         
         var state = await result.Content.ReadFromJsonAsync<DeleteSecretResponse>(this.jsonRustSerializerOptions);
         if (!state.Success)
-            this.logger!.LogError($"Failed to delete the secret data for secret ID '{secretId.SecretId}': '{state.Issue}'");
+            this.logger!.LogError($"Failed to delete the secret data for '{secretKey}': '{state.Issue}'");
         
         return state;
     }

app/MindWork AI Studio/Tools/Services/RustService.cs

@@ -18,6 +18,7 @@ public sealed partial class RustService : BackgroundService
     
     private readonly HttpClient http;
     private readonly SemaphoreSlim userLanguageLock = new(1, 1);
+    private readonly SemaphoreSlim userNameLock = new(1, 1);
 
     private readonly JsonSerializerOptions jsonRustSerializerOptions = new()
     {
@@ -31,6 +32,7 @@ public sealed partial class RustService : BackgroundService
     private ILogger<RustService>? logger;
     private Encryption? encryptor;
     private string? cachedUserLanguage;
+    private string? cachedUserName;
     
     private readonly string apiPort;
     private readonly string certificateFingerprint;
@@ -92,6 +94,7 @@ public sealed partial class RustService : BackgroundService
         this.http.Dispose();
         this.userLanguageLock.Dispose();
         this.tokenizerLock.Dispose();
+        this.userNameLock.Dispose();
         base.Dispose();
     }