diff --git a/app/MindWork AI Studio/Chat/IImageSourceExtensions.cs b/app/MindWork AI Studio/Chat/IImageSourceExtensions.cs index c6461643..6c3f204f 100644 --- a/app/MindWork AI Studio/Chat/IImageSourceExtensions.cs +++ b/app/MindWork AI Studio/Chat/IImageSourceExtensions.cs @@ -89,8 +89,10 @@ public static class IImageSourceExtensions case ContentImageSource.URL: { - using var httpClient = new HttpClient(); - using var response = await httpClient.GetAsync(image.Source, HttpCompletionOption.ResponseHeadersRead, token); + using var httpClient = ExternalHttpClientTimeout.CreateHttpClient(); + using var timeoutTokenSource = ExternalHttpClientTimeout.CreateTimeoutTokenSource(token); + var timeoutToken = timeoutTokenSource.Token; + using var response = await httpClient.GetAsync(image.Source, HttpCompletionOption.ResponseHeadersRead, timeoutToken); if(response.IsSuccessStatusCode) { // Read the length of the content: @@ -101,7 +103,7 @@ public static class IImageSourceExtensions return (success: false, string.Empty); } - var bytes = await response.Content.ReadAsByteArrayAsync(token); + var bytes = await response.Content.ReadAsByteArrayAsync(timeoutToken); return (success: true, Convert.ToBase64String(bytes)); } diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index 928627b9..ef98a1e6 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -260,9 +260,9 @@ CONFIG["SETTINGS"] = {} -- Examples are: "CmdOrControl+Shift+D", "Alt+F9", "F8" -- CONFIG["SETTINGS"]["DataApp.ShortcutVoiceRecording"] = "CmdOrControl+1" --- Configure the HTTP timeout for requests to LLM providers, in seconds. +-- Configure the HTTP timeout for external requests, in seconds. -- The default is 3600 (1 hour). --- CONFIG["SETTINGS"]["DataApp.ProviderHttpTimeoutSeconds"] = 3600 +-- CONFIG["SETTINGS"]["DataApp.HttpClientTimeoutSeconds"] = 3600 -- Example chat templates for this configuration: CONFIG["CHAT_TEMPLATES"] = {} diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index ebb0feda..123c4a86 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -24,15 +24,12 @@ namespace AIStudio.Provider; /// public abstract class BaseProvider : IProvider, ISecretId { - private const int DEFAULT_HTTP_TIMEOUT_SECONDS = 3600; - private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(BaseProvider).Namespace, nameof(BaseProvider)); - private static readonly SettingsManager SETTINGS_MANAGER = Program.SERVICE_PROVIDER.GetRequiredService(); /// /// The HTTP client to use it for all requests. /// - protected readonly HttpClient HttpClient = new(); + protected readonly HttpClient HttpClient = ExternalHttpClientTimeout.CreateHttpClient(); /// /// The logger to use. @@ -77,7 +74,6 @@ public abstract class BaseProvider : IProvider, ISecretId // Set the base URL: this.HttpClient.BaseAddress = new(url); - this.HttpClient.Timeout = GetProviderHttpTimeout(); } #region Handling of IProvider, which all providers must implement @@ -145,13 +141,7 @@ public abstract class BaseProvider : IProvider, ISecretId if (token.IsCancellationRequested) return false; - if (exception is TimeoutException) - return true; - - if (exception is OperationCanceledException) - return true; - - return exception.InnerException is not null && this.IsTimeoutException(exception.InnerException, token); + return ExternalHttpClientTimeout.IsTimeoutException(exception, token); } protected Task SendTimeoutError(string action) => MessageBus.INSTANCE.SendError(new( @@ -160,38 +150,9 @@ public abstract class BaseProvider : IProvider, ISecretId TB("The request to the LLM provider '{0}' (type={1}) timed out after {2} while {3}. Please try again or check whether the provider is still responding."), this.InstanceName, this.Provider, - GetProviderHttpTimeoutDescription(), + ExternalHttpClientTimeout.GetTimeoutDescription(), action))); - private static TimeSpan GetProviderHttpTimeout() - { - var seconds = SETTINGS_MANAGER.ConfigurationData.App.ProviderHttpTimeoutSeconds; - if (seconds <= 0) - seconds = DEFAULT_HTTP_TIMEOUT_SECONDS; - - return TimeSpan.FromSeconds(seconds); - } - - private static string GetProviderHttpTimeoutDescription() - { - var timeout = GetProviderHttpTimeout(); - - if (timeout.TotalHours >= 1 && timeout.TotalMinutes % 60 == 0) - { - var hours = (int)timeout.TotalHours; - return hours == 1 ? "1 hour" : $"{hours} hours"; - } - - if (timeout.TotalMinutes >= 1 && timeout.TotalSeconds % 60 == 0) - { - var minutes = (int)timeout.TotalMinutes; - return minutes == 1 ? "1 minute" : $"{minutes} minutes"; - } - - var seconds = (int)timeout.TotalSeconds; - return seconds == 1 ? "1 second" : $"{seconds} seconds"; - } - protected async Task GetModelLoadingSecretKey(SecretStoreType storeType, string? apiKeyProvisional = null, bool isTryingSecret = false) => apiKeyProvisional switch { not null => apiKeyProvisional, @@ -266,12 +227,14 @@ public abstract class BaseProvider : IProvider, ISecretId /// Sends a request and handles rate limiting by exponential backoff. /// /// A function that builds the request. - /// The cancellation token. + /// The user cancellation token. + /// The token to use for the HTTP request. /// The status object of the request. - private async Task SendRequest(Func> requestBuilder, CancellationToken token = default) + private async Task SendRequest(Func> requestBuilder, CancellationToken userCancellationToken = default, CancellationToken requestCancellationToken = default) { const int MAX_RETRIES = 6; const double RETRY_DELAY_SECONDS = 4; + var effectiveCancellationToken = requestCancellationToken.CanBeCanceled ? requestCancellationToken : userCancellationToken; var retry = 0; var response = default(HttpResponseMessage); @@ -291,9 +254,9 @@ public abstract class BaseProvider : IProvider, ISecretId HttpResponseMessage nextResponse; try { - nextResponse = await this.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token); + nextResponse = await this.HttpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, effectiveCancellationToken); } - catch (Exception e) when (this.IsTimeoutException(e, token)) + catch (Exception e) when (this.IsTimeoutException(e, userCancellationToken)) { await this.SendTimeoutError("waiting for the chat response"); this.logger.LogError(e, "Timed out while sending a streaming request to provider '{ProviderInstanceName}' (provider={ProviderType}).", this.InstanceName, this.Provider); @@ -306,7 +269,7 @@ public abstract class BaseProvider : IProvider, ISecretId break; } - var errorBody = await nextResponse.Content.ReadAsStringAsync(token); + var errorBody = await nextResponse.Content.ReadAsStringAsync(effectiveCancellationToken); if (nextResponse.StatusCode is HttpStatusCode.Forbidden) { await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Block, string.Format(TB("We tried to communicate with the LLM provider '{0}' (type={1}). You might not be able to use this provider from your location. The provider message is: '{2}'"), this.InstanceName, this.Provider, nextResponse.ReasonPhrase))); @@ -372,7 +335,7 @@ public abstract class BaseProvider : IProvider, ISecretId timeSeconds = 90; this.logger.LogDebug("Failed request with status code {ResponseStatusCode} (message = '{ErrorMessage}'). Retrying in {TimeSeconds:0.00} seconds.", nextResponse.StatusCode, errorMessage, timeSeconds); - await Task.Delay(TimeSpan.FromSeconds(timeSeconds), token); + await Task.Delay(TimeSpan.FromSeconds(timeSeconds), effectiveCancellationToken); } if(retry >= MAX_RETRIES || !string.IsNullOrWhiteSpace(errorMessage)) @@ -399,10 +362,12 @@ public abstract class BaseProvider : IProvider, ISecretId var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine); StreamReader? streamReader = null; + using var timeoutTokenSource = ExternalHttpClientTimeout.CreateTimeoutTokenSource(token); + var timeoutToken = timeoutTokenSource.Token; try { // Send the request using exponential backoff: - var responseData = await this.SendRequest(requestBuilder, token); + var responseData = await this.SendRequest(requestBuilder, token, timeoutToken); if(responseData.IsFailedAfterAllRetries) { this.logger.LogError($"The {providerName} chat completion failed: {responseData.ErrorMessage}"); @@ -410,7 +375,7 @@ public abstract class BaseProvider : IProvider, ISecretId } // Open the response stream: - var providerStream = await responseData.Response!.Content.ReadAsStreamAsync(token); + var providerStream = await responseData.Response!.Content.ReadAsStreamAsync(timeoutToken); // Add a stream reader to read the stream, line by line: streamReader = new StreamReader(providerStream); @@ -441,18 +406,6 @@ public abstract class BaseProvider : IProvider, ISecretId // while (true) { - try - { - if(streamReader.EndOfStream) - break; - } - catch (Exception e) - { - await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Stream, string.Format(TB("Tried to stream the LLM provider '{0}' answer. There were some problems with the stream. The message is: '{1}'"), this.InstanceName, e.Message))); - this.logger.LogWarning($"Failed to read the end-of-stream state from {providerName} '{this.InstanceName}': {e.Message}"); - break; - } - // Check if the token is canceled: if (token.IsCancellationRequested) { @@ -467,7 +420,7 @@ public abstract class BaseProvider : IProvider, ISecretId string? line; try { - line = await streamReader.ReadLineAsync(token); + line = await streamReader.ReadLineAsync(timeoutToken); } catch (Exception e) { @@ -489,6 +442,9 @@ public abstract class BaseProvider : IProvider, ISecretId break; } + if (line is null) + break; + // Skip empty lines: if (string.IsNullOrWhiteSpace(line)) continue; @@ -588,10 +544,12 @@ public abstract class BaseProvider : IProvider, ISecretId var annotationSupported = typeof(TAnnotation) != typeof(NoResponsesAnnotationStreamLine) && typeof(TAnnotation) != typeof(NoChatCompletionAnnotationStreamLine); StreamReader? streamReader = null; + using var timeoutTokenSource = ExternalHttpClientTimeout.CreateTimeoutTokenSource(token); + var timeoutToken = timeoutTokenSource.Token; try { // Send the request using exponential backoff: - var responseData = await this.SendRequest(requestBuilder, token); + var responseData = await this.SendRequest(requestBuilder, token, timeoutToken); if(responseData.IsFailedAfterAllRetries) { this.logger.LogError($"The {providerName} responses call failed: {responseData.ErrorMessage}"); @@ -599,7 +557,7 @@ public abstract class BaseProvider : IProvider, ISecretId } // Open the response stream: - var providerStream = await responseData.Response!.Content.ReadAsStreamAsync(token); + var providerStream = await responseData.Response!.Content.ReadAsStreamAsync(timeoutToken); // Add a stream reader to read the stream, line by line: streamReader = new StreamReader(providerStream); @@ -630,18 +588,6 @@ public abstract class BaseProvider : IProvider, ISecretId // while (true) { - try - { - if(streamReader.EndOfStream) - break; - } - catch (Exception e) - { - await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Stream, string.Format(TB("Tried to stream the LLM provider '{0}' answer. There were some problems with the stream. The message is: '{1}'"), this.InstanceName, e.Message))); - this.logger.LogWarning($"Failed to read the end-of-stream state from {providerName} '{this.InstanceName}': {e.Message}"); - break; - } - // Check if the token is canceled: if (token.IsCancellationRequested) { @@ -656,7 +602,7 @@ public abstract class BaseProvider : IProvider, ISecretId string? line; try { - line = await streamReader.ReadLineAsync(token); + line = await streamReader.ReadLineAsync(timeoutToken); } catch (Exception e) { @@ -678,6 +624,9 @@ public abstract class BaseProvider : IProvider, ISecretId break; } + if (line is null) + break; + // Skip empty lines: if (string.IsNullOrWhiteSpace(line)) continue; diff --git a/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs b/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs index 562d0f21..13273018 100644 --- a/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs +++ b/app/MindWork AI Studio/Provider/Helmholtz/ProviderHelmholtz.cs @@ -161,4 +161,4 @@ public sealed class ProviderHelmholtz() : BaseProvider(LLMProviders.HELMHOLTZ, " return FailedModelLoadResult(ModelLoadFailureReason.PROVIDER_UNAVAILABLE, e.Message); } } -} +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs index e366657e..598cb2f3 100644 --- a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs +++ b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs @@ -196,4 +196,4 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide return FailedModelLoadResult(ModelLoadFailureReason.PROVIDER_UNAVAILABLE, e.Message); } } -} +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/DataModel/DataApp.cs b/app/MindWork AI Studio/Settings/DataModel/DataApp.cs index 1b25ba88..ad027064 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataApp.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataApp.cs @@ -95,9 +95,9 @@ public sealed class DataApp(Expression>? configSelection = n public string ShortcutVoiceRecording { get; set; } = ManagedConfiguration.Register(configSelection, n => n.ShortcutVoiceRecording, string.Empty); /// - /// The HTTP timeout in seconds for requests to LLM providers. + /// The HTTP timeout in seconds for external HTTP clients. /// - public int ProviderHttpTimeoutSeconds { get; set; } = ManagedConfiguration.Register(configSelection, n => n.ProviderHttpTimeoutSeconds, 3600); + public int HttpClientTimeoutSeconds { get; set; } = ManagedConfiguration.Register(configSelection, n => n.HttpClientTimeoutSeconds, ExternalHttpClientTimeout.DEFAULT_HTTP_CLIENT_TIMEOUT_SECONDS); /// /// Should the user be allowed to add providers? diff --git a/app/MindWork AI Studio/Tools/ERIClient/ERIClientBase.cs b/app/MindWork AI Studio/Tools/ERIClient/ERIClientBase.cs index 338401e3..389a90e3 100644 --- a/app/MindWork AI Studio/Tools/ERIClient/ERIClientBase.cs +++ b/app/MindWork AI Studio/Tools/ERIClient/ERIClientBase.cs @@ -23,10 +23,7 @@ public abstract class ERIClientBase(IERIDataSource dataSource) : IDisposable } }; - protected readonly HttpClient HttpClient = new() - { - BaseAddress = new Uri($"{dataSource.Hostname}:{dataSource.Port}"), - }; + protected readonly HttpClient HttpClient = ExternalHttpClientTimeout.CreateHttpClient(new Uri($"{dataSource.Hostname}:{dataSource.Port}")); protected string SecurityToken = string.Empty; diff --git a/app/MindWork AI Studio/Tools/ExternalHttpClientTimeout.cs b/app/MindWork AI Studio/Tools/ExternalHttpClientTimeout.cs new file mode 100644 index 00000000..b467f2c5 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ExternalHttpClientTimeout.cs @@ -0,0 +1,77 @@ +using AIStudio.Settings; + +namespace AIStudio.Tools; + +/// +/// Provides utility methods to standardize the management of HTTP client timeouts +/// across various components in the application. +/// +public static class ExternalHttpClientTimeout +{ + public const int DEFAULT_HTTP_CLIENT_TIMEOUT_SECONDS = 3600; + + public static HttpClient CreateHttpClient(Uri? baseAddress = null) + { + var httpClient = new HttpClient(); + Configure(httpClient, baseAddress); + return httpClient; + } + + public static string GetTimeoutDescription() + { + var timeout = GetTimeout(); + + if (timeout.TotalHours >= 1 && timeout.TotalMinutes % 60 == 0) + { + var hours = (int)timeout.TotalHours; + return hours == 1 ? "1 hour" : $"{hours} hours"; + } + + if (timeout.TotalMinutes >= 1 && timeout.TotalSeconds % 60 == 0) + { + var minutes = (int)timeout.TotalMinutes; + return minutes == 1 ? "1 minute" : $"{minutes} minutes"; + } + + var seconds = (int)timeout.TotalSeconds; + return seconds == 1 ? "1 second" : $"{seconds} seconds"; + } + + public static CancellationTokenSource CreateTimeoutTokenSource(CancellationToken cancellationToken) + { + var timeoutTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + timeoutTokenSource.CancelAfter(GetTimeout()); + return timeoutTokenSource; + } + + public static bool IsTimeoutException(Exception exception, CancellationToken userCancellationToken = default) + { + if (userCancellationToken.IsCancellationRequested) + return false; + + if (exception is TimeoutException) + return true; + + if (exception is OperationCanceledException) + return true; + + return exception.InnerException is not null && IsTimeoutException(exception.InnerException, userCancellationToken); + } + + private static TimeSpan GetTimeout() + { + var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService(); + var seconds = settingsManager.ConfigurationData.App.HttpClientTimeoutSeconds; + if (seconds <= 0) + seconds = DEFAULT_HTTP_CLIENT_TIMEOUT_SECONDS; + + return TimeSpan.FromSeconds(seconds); + } + + private static void Configure(HttpClient httpClient, Uri? baseAddress = null) + { + httpClient.Timeout = GetTimeout(); + if (baseAddress is not null) + httpClient.BaseAddress = baseAddress; + } +} diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs index 337526fc..dd422c06 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs @@ -172,8 +172,8 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT // Config: global voice recording shortcut ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.ShortcutVoiceRecording, this.Id, settingsTable, dryRun); - // Config: timeout for HTTP requests to providers - ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.ProviderHttpTimeoutSeconds, this.Id, settingsTable, dryRun); + // Config: timeout for external HTTP requests + ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.HttpClientTimeoutSeconds, this.Id, settingsTable, dryRun); // Handle configured LLM providers: PluginConfigurationObject.TryParse(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, x => x.NextProviderNum, mainTable, this.Id, ref this.configObjects, dryRun); diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Download.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Download.cs index 9b56e3af..daf77fb0 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Download.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Download.cs @@ -15,7 +15,7 @@ public static partial class PluginFactory var serverUrl = configServerUrl.EndsWith('/') ? configServerUrl[..^1] : configServerUrl; var downloadUrl = $"{serverUrl}/{configPlugId}.zip"; - using var http = new HttpClient(); + using var http = ExternalHttpClientTimeout.CreateHttpClient(); using var request = new HttpRequestMessage(HttpMethod.Get, downloadUrl); var response = await http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken); if (!response.IsSuccessStatusCode) @@ -52,7 +52,7 @@ public static partial class PluginFactory try { await LockHotReloadAsync(); - using var httpClient = new HttpClient(); + using var httpClient = ExternalHttpClientTimeout.CreateHttpClient(); var response = await httpClient.GetAsync(downloadUrl, cancellationToken); if (!response.IsSuccessStatusCode) { diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs index f97f6a9b..b0dfd89d 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs @@ -245,8 +245,8 @@ public static partial class PluginFactory if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ShortcutVoiceRecording, AVAILABLE_PLUGINS)) wasConfigurationChanged = true; - // Check for the provider HTTP timeout: - if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ProviderHttpTimeoutSeconds, AVAILABLE_PLUGINS)) + // Check for the external HTTP client timeout: + if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.HttpClientTimeoutSeconds, AVAILABLE_PLUGINS)) wasConfigurationChanged = true; // Check if audit is required before it can be activated diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.5.5.md b/app/MindWork AI Studio/wwwroot/changelog/v26.5.5.md index b8d5f9a1..67c080e9 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.5.5.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.5.5.md @@ -2,6 +2,7 @@ - Released the voice recording and transcription for all users. You no longer need to enable a preview feature to configure transcription providers, select a transcription provider, or use dictation. - Added support for organization-managed ERI servers in configuration plugins, so admins can preconfigure external data sources for users. - Added an export option for ERI server data sources, so admins can create configuration plugin snippets without writing the Lua code manually. +- Added an option to configure the timeout setting for all requests. This is useful when you have a slow network connection, or you have to work with slow AI servers. It is also possible to configure this timeout for an entire organization using configuration plugins. - Added the username to the information page to make organization support easier when users share their screen. - Improved the app's security foundation with major modernization of the native runtime and its internal communication layer. This work is mostly invisible during everyday use, but it replaces older components that no longer received the security updates we require. We also continued updating security-sensitive dependencies so AI Studio stays on a healthier, better maintained base. - Improved the Pandoc management and detection process to make it more reliable.