Fixed an incomplete plugin start with extensive enterprise configurations (#1033)

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2026-10-09 20:11:15 +02:00
1 parent bdf828fa4f
commit 56215aa6dd
16 files changed
+493 -70

No files matched your search

@@ -50,8 +50,12 @@ public record FileAttachment(FileAttachmentType Type, string FileName, string Fi
/// Gets a value indicating whether the file still exists on the file system.
/// </summary>
/// <remarks>
/// This property checks the file system each time it is accessed.
/// This property checks the file system each time it is accessed. It is therefore excluded from
/// serialization: a path on a network share which is out of reach blocks until the SMB timeout,
/// and the settings carry such attachments in chat templates. Every store of the settings would
/// wait that long. Reading never used the value, so older files with the field still load.
/// </remarks>
[JsonIgnore]
public bool Exists => File.Exists(this.FilePath);
/// <summary>
@@ -338,13 +338,9 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
// Initialize the enterprise encryption service for decrypting API keys:
await PluginFactory.InitializeEnterpriseEncryption(this.RustService);
// Load (but not start) all plugins without waiting for them:
#if DEBUG
var pluginLoadingTimeout = new CancellationTokenSource();
#else
var pluginLoadingTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(5));
#endif
await PluginFactory.LoadAll(pluginLoadingTimeout.Token);
// Load and start all plugins. Each plugin has a time limit of its own,
// so a slow one cannot keep the others from starting:
await PluginFactory.LoadAll();
// Set up hot reloading for plugins:
PluginFactory.SetUpHotReloading();
@@ -186,7 +186,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
return null;
}
public static bool TryParseConfiguration(int idx, LuaTable table, Guid configPluginId, out DataSourceERI_V1 dataSource)
public static bool TryParseConfiguration(int idx, LuaTable table, Guid configPluginId, bool dryRun, out DataSourceERI_V1 dataSource)
{
dataSource = default;
if (!table.TryGetValue("Id", out var idValue) || !idValue.TryRead<string>(out var idText) || !Guid.TryParse(idText, out var id))
@@ -301,7 +301,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
EnterpriseConfigurationPluginId = configPluginId,
};
return TryQueueEnterpriseSecret(idx, table, configPluginId, dataSource);
return TryQueueEnterpriseSecret(idx, table, configPluginId, dataSource, dryRun);
}
/// <summary>
@@ -360,7 +360,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
""";
}
private static bool TryQueueEnterpriseSecret(int idx, LuaTable table, Guid configPluginId, DataSourceERI_V1 dataSource)
private static bool TryQueueEnterpriseSecret(int idx, LuaTable table, Guid configPluginId, DataSourceERI_V1 dataSource, bool dryRun)
{
var secretFieldName = dataSource.AuthMethod switch
{
@@ -397,6 +397,13 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
return false;
}
//
// A dry run only checks the configuration, so it queues nothing: the start which follows it
// would otherwise store the secret once more for every dry run before it:
//
if (dryRun)
return true;
PendingEnterpriseSecrets.Add(new(
$"{ISecretId.ENTERPRISE_KEY_PREFIX}::{dataSource.Id}",
dataSource.Name,
@@ -61,7 +61,7 @@ public sealed record EmbeddingProvider(
#endregion
public static bool TryParseEmbeddingProviderTable(int idx, LuaTable table, Guid configPluginId, string pluginPath, out ConfigurationBaseObject provider)
public static bool TryParseEmbeddingProviderTable(int idx, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject provider)
{
provider = NONE;
if (!table.TryGetValue("Id", out var idValue) || !idValue.TryRead<string>(out var idText) || !Guid.TryParse(idText, out var id))
@@ -189,13 +189,20 @@ public sealed record EmbeddingProvider(
{
if (encryption.TryDecrypt(apiKeyText, out var decryptedApiKey))
{
// Queue the API key for storage in the OS keyring:
PendingEnterpriseApiKeys.Add(new(
$"{ISecretId.ENTERPRISE_KEY_PREFIX}::{usedLLMProvider.ToSecretId()}",
name,
decryptedApiKey,
SecretStoreType.EMBEDDING_PROVIDER));
LOGGER.LogDebug($"Successfully decrypted API key for embedding provider {idx}. It will be stored in the OS keyring. (Plugin ID: {configPluginId})");
//
// Queue the API key for storage in the OS keyring. A dry run only checks the
// configuration, so it queues nothing: the start which follows it would
// otherwise store the key once more for every dry run before it:
//
if (!dryRun)
{
PendingEnterpriseApiKeys.Add(new(
$"{ISecretId.ENTERPRISE_KEY_PREFIX}::{usedLLMProvider.ToSecretId()}",
name,
decryptedApiKey,
SecretStoreType.EMBEDDING_PROVIDER));
LOGGER.LogDebug($"Successfully decrypted API key for embedding provider {idx}. It will be stored in the OS keyring. (Plugin ID: {configPluginId})");
}
}
else
LOGGER.LogWarning($"Failed to decrypt API key for embedding provider {idx}. The encryption secret may be incorrect. (Plugin ID: {configPluginId})");
+15 -8
View File
@@ -88,7 +88,7 @@ public sealed record Provider(
#endregion
public static bool TryParseProviderTable(int idx, LuaTable table, Guid configPluginId, string pluginPath, out ConfigurationBaseObject provider)
public static bool TryParseProviderTable(int idx, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject provider)
{
provider = NONE;
if (!table.TryGetValue("Id", out var idValue) || !idValue.TryRead<string>(out var idText) || !Guid.TryParse(idText, out var id))
@@ -210,13 +210,20 @@ public sealed record Provider(
{
if (encryption.TryDecrypt(apiKeyText, out var decryptedApiKey))
{
// Queue the API key for storage in the OS keyring:
PendingEnterpriseApiKeys.Add(new(
$"{ISecretId.ENTERPRISE_KEY_PREFIX}::{usedLLMProvider.ToSecretId()}",
instanceName,
decryptedApiKey,
SecretStoreType.LLM_PROVIDER));
LOGGER.LogDebug($"Successfully decrypted API key for provider {idx}. It will be stored in the OS keyring. (Plugin ID: {configPluginId})");
//
// Queue the API key for storage in the OS keyring. A dry run only checks the
// configuration, so it queues nothing: the start which follows it would
// otherwise store the key once more for every dry run before it:
//
if (!dryRun)
{
PendingEnterpriseApiKeys.Add(new(
$"{ISecretId.ENTERPRISE_KEY_PREFIX}::{usedLLMProvider.ToSecretId()}",
instanceName,
decryptedApiKey,
SecretStoreType.LLM_PROVIDER));
LOGGER.LogDebug($"Successfully decrypted API key for provider {idx}. It will be stored in the OS keyring. (Plugin ID: {configPluginId})");
}
}
else
LOGGER.LogWarning($"Failed to decrypt API key for provider {idx}. The encryption secret may be incorrect. (Plugin ID: {configPluginId})");
@@ -1,4 +1,6 @@
using System.Diagnostics;
using System.Linq.Expressions;
using System.Text;
using System.Text.Json;
using AIStudio.Provider;
@@ -30,6 +32,16 @@ public sealed class SettingsManager
Converters = { new TolerantEnumConverter() },
};
/// <summary>
/// From this duration on, storing the settings is reported as slow.
/// </summary>
/// <remarks>
/// The settings file is a few hundred kilobytes at most, so storing it usually takes some
/// milliseconds. Far beyond that, something is blocking: on one machine, every store took 18
/// seconds, and each one held up the start of AI Studio.
/// </remarks>
private static readonly TimeSpan SLOW_STORE_THRESHOLD = TimeSpan.FromSeconds(1);
private readonly ILogger<SettingsManager> logger;
private readonly RustService rustService;
@@ -337,10 +349,24 @@ public sealed class SettingsManager
return;
}
//
// Serializing and writing are measured on their own, because a slow store has two
// different causes: a property which asks the file system or the network while it is
// serialized, or a file system which is slow to write, e.g., because of a virus scanner
// or a synchronized profile folder. Only the two durations tell them apart:
//
var serializingStartedAt = Stopwatch.GetTimestamp();
var settingsJson = JsonSerializer.Serialize(this.ConfigurationData, JSON_OPTIONS);
var serializingDuration = Stopwatch.GetElapsedTime(serializingStartedAt);
var writingStartedAt = Stopwatch.GetTimestamp();
var settingsPath = Path.Combine(ConfigDirectory!, SETTINGS_FILENAME);
await this.StoreSerializedSettings(settingsJson, settingsPath);
await this.StoreSerializedVersionBackup(this.ConfigurationData.Version, settingsJson);
var writingDuration = Stopwatch.GetElapsedTime(writingStartedAt);
if (serializingDuration + writingDuration >= SLOW_STORE_THRESHOLD)
this.logger.LogWarning($"Storing the settings took {(serializingDuration + writingDuration).TotalMilliseconds:F0} ms, which is unusually long: serializing took {serializingDuration.TotalMilliseconds:F0} ms, writing took {writingDuration.TotalMilliseconds:F0} ms. The settings are {Encoding.UTF8.GetByteCount(settingsJson) / 1024} KB. A slow serialization points to a setting which reaches out to the file system or the network while it is serialized; slow writing points to the file system itself, e.g., a virus scanner or a synchronized profile folder.");
}
finally
{
@@ -56,7 +56,7 @@ public sealed record TranscriptionProvider(
#endregion
public static bool TryParseTranscriptionProviderTable(int idx, LuaTable table, Guid configPluginId, string pluginPath, out ConfigurationBaseObject provider)
public static bool TryParseTranscriptionProviderTable(int idx, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject provider)
{
provider = NONE;
if (!table.TryGetValue("Id", out var idValue) || !idValue.TryRead<string>(out var idText) || !Guid.TryParse(idText, out var id))
@@ -160,13 +160,20 @@ public sealed record TranscriptionProvider(
{
if (encryption.TryDecrypt(apiKeyText, out var decryptedApiKey))
{
// Queue the API key for storage in the OS keyring:
PendingEnterpriseApiKeys.Add(new(
$"{ISecretId.ENTERPRISE_KEY_PREFIX}::{usedLLMProvider.ToSecretId()}",
name,
decryptedApiKey,
SecretStoreType.TRANSCRIPTION_PROVIDER));
LOGGER.LogDebug($"Successfully decrypted API key for transcription provider {idx}. It will be stored in the OS keyring. (Plugin ID: {configPluginId})");
//
// Queue the API key for storage in the OS keyring. A dry run only checks the
// configuration, so it queues nothing: the start which follows it would
// otherwise store the key once more for every dry run before it:
//
if (!dryRun)
{
PendingEnterpriseApiKeys.Add(new(
$"{ISecretId.ENTERPRISE_KEY_PREFIX}::{usedLLMProvider.ToSecretId()}",
name,
decryptedApiKey,
SecretStoreType.TRANSCRIPTION_PROVIDER));
LOGGER.LogDebug($"Successfully decrypted API key for transcription provider {idx}. It will be stored in the OS keyring. (Plugin ID: {configPluginId})");
}
}
else
LOGGER.LogWarning($"Failed to decrypt API key for transcription provider {idx}. The encryption secret may be incorrect. (Plugin ID: {configPluginId})");
@@ -12,7 +12,6 @@ namespace AIStudio.Tools.PluginSystem;
public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginType type) : PluginBase(isInternal, state, type), ILivePluginContentSource
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PluginConfiguration).Namespace, nameof(PluginConfiguration));
private static SettingsManager SettingsManagerAccess => Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger(nameof(PluginConfiguration));
private List<PluginConfigurationObject> configObjects = [];
@@ -83,8 +82,11 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
await StoreEnterpriseApiKeysAsync();
await StoreEnterpriseSecretsAsync();
await SettingsManagerAccess.StoreSettings();
await MessageBus.INSTANCE.SendMessage<bool>(null, Event.CONFIGURATION_CHANGED);
//
// The settings are not stored here: PluginFactory.LoadAll does that once, after all
// plugins have started. Storing them per configuration plugin wrote the whole settings
// file again for each one, which made the start of every further plugin wait for it.
//
}
}
@@ -143,11 +143,11 @@ public sealed record PluginConfigurationObject
var (wasParsingSuccessful, configObject) = configObjectType switch
{
PluginConfigurationObjectType.LLM_PROVIDER => (Settings.Provider.TryParseProviderTable(i, luaObjectTable, configPluginId, pluginPath, out var configurationObject) && configurationObject != Settings.Provider.NONE, configurationObject),
PluginConfigurationObjectType.LLM_PROVIDER => (Settings.Provider.TryParseProviderTable(i, luaObjectTable, configPluginId, pluginPath, dryRun, out var configurationObject) && configurationObject != Settings.Provider.NONE, configurationObject),
PluginConfigurationObjectType.CHAT_TEMPLATE => (ChatTemplate.TryParseChatTemplateTable(i, luaObjectTable, configPluginId, pluginPath, out var configurationObject) && configurationObject != ChatTemplate.NO_CHAT_TEMPLATE, configurationObject),
PluginConfigurationObjectType.PROFILE => (Profile.TryParseProfileTable(i, luaObjectTable, configPluginId, out var configurationObject) && configurationObject != Profile.NO_PROFILE, configurationObject),
PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER => (TranscriptionProvider.TryParseTranscriptionProviderTable(i, luaObjectTable, configPluginId, pluginPath, out var configurationObject) && configurationObject != TranscriptionProvider.NONE, configurationObject),
PluginConfigurationObjectType.EMBEDDING_PROVIDER => (EmbeddingProvider.TryParseEmbeddingProviderTable(i, luaObjectTable, configPluginId, pluginPath, out var configurationObject) && configurationObject != EmbeddingProvider.NONE, configurationObject),
PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER => (TranscriptionProvider.TryParseTranscriptionProviderTable(i, luaObjectTable, configPluginId, pluginPath, dryRun, out var configurationObject) && configurationObject != TranscriptionProvider.NONE, configurationObject),
PluginConfigurationObjectType.EMBEDDING_PROVIDER => (EmbeddingProvider.TryParseEmbeddingProviderTable(i, luaObjectTable, configPluginId, pluginPath, dryRun, out var configurationObject) && configurationObject != EmbeddingProvider.NONE, configurationObject),
PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY => (DataDocumentAnalysisPolicy.TryProcessConfiguration(i, luaObjectTable, configPluginId, out var configurationObject) && configurationObject is DataDocumentAnalysisPolicy, configurationObject),
_ => (false, NoConfigurationObject.INSTANCE)
@@ -278,7 +278,7 @@ public sealed record PluginConfigurationObject
continue;
}
if (!DataSourceERI_V1.TryParseConfiguration(i, luaObjectTable, configPluginId, out var configObject))
if (!DataSourceERI_V1.TryParseConfiguration(i, luaObjectTable, configPluginId, dryRun, out var configObject))
{
LOG.LogWarning("The table '{LuaTableName}' entry at index {Index} does not contain a valid data source (config plugin id: {ConfigPluginId}).", LUA_TABLE_NAME, i, configPluginId);
continue;
@@ -372,6 +372,11 @@ public sealed record PluginConfigurationObject
/// those which could not be loaded. Objects of a deployed plugin are never removed, because the
/// plugin was not removed either.
/// </param>
/// <param name="notStartedConfigPluginIds">
/// The IDs of the configuration plugins which were loaded, but did not start. Their objects are
/// never removed either: such a plugin contributed nothing to the list of configuration objects,
/// so all of its objects would look as if the plugin had dropped them.
/// </param>
/// <param name="configObjectList">A list of all existing configuration objects.</param>
/// <param name="secretStoreType">An optional parameter specifying the type of secret store to use for deleting associated API keys from the OS keyring, if applicable.</param>
/// <param name="deleteSecret">When true, delete the associated non-API-key secret from the OS keyring.</param>
@@ -381,6 +386,7 @@ public sealed record PluginConfigurationObject
Expression<Func<Data, List<TClass>>> configObjectSelection,
IList<IAvailablePlugin> availablePlugins,
IReadOnlySet<Guid> deployedEnterpriseConfigPluginIds,
IReadOnlySet<Guid> notStartedConfigPluginIds,
IList<PluginConfigurationObject> configObjectList,
SecretStoreType? secretStoreType = null,
bool deleteSecret = false) where TClass : IConfigurationObject
@@ -410,6 +416,15 @@ public sealed record PluginConfigurationObject
if(deployedEnterpriseConfigPluginIds.Contains(configObjectSourcePluginId) && availablePlugins.All(plugin => plugin.Id != configObjectSourcePluginId))
continue;
//
// Was the source plugin loaded, but did not start? Then it is not broken, it could not
// run this time, e.g., because it ran out of time on a slow machine. It contributed no
// objects, so every one of its objects would look removed from the plugin. They stay
// until the plugin starts again and tells us which ones it still defines:
//
if(notStartedConfigPluginIds.Contains(configObjectSourcePluginId))
continue;
// Is the source plugin still available? If not, we can be pretty sure that this configuration object is left
// over and should be removed:
var templateSourcePlugin = availablePlugins.FirstOrDefault(plugin => plugin.Id == configObjectSourcePluginId);
@@ -1,3 +1,4 @@
using System.Diagnostics;
using System.Linq.Expressions;
using System.Text;
using AIStudio.Settings;
@@ -13,6 +14,18 @@ public static partial class PluginFactory
private static readonly List<IAvailablePlugin> AVAILABLE_PLUGINS = [];
private static readonly SemaphoreSlim PLUGIN_LOAD_SEMAPHORE = new(1, 1);
/// <summary>
/// How long a single plugin may take to load, and again to start.
/// </summary>
/// <remarks>
/// The limit applies to each plugin on its own, never to all plugins together. A shared budget
/// let a slow machine spend it on the first plugins, so the ones at the end of the line never
/// started: the language plugin among them, and configuration plugins of the organization,
/// whose objects the clean-up then removed. The limit holds in every build, so a developer sees
/// the same start as a user does.
/// </remarks>
private static readonly TimeSpan PLUGIN_TIMEOUT = TimeSpan.FromSeconds(10);
/// <summary>
/// A list of all available plugins.
/// </summary>
@@ -42,6 +55,8 @@ public static partial class PluginFactory
await PLUGIN_LOAD_SEMAPHORE.WaitAsync(cancellationToken);
var configObjectList = new List<PluginConfigurationObject>();
var hasStartedConfigurationPlugins = false;
var notStartedConfigPluginIds = new HashSet<Guid>();
try
{
@@ -65,11 +80,13 @@ public static partial class PluginFactory
IEnumerable<string> pluginMainFiles = pluginsDirectoryExists ? Directory.EnumerateFiles(PLUGINS_ROOT, "plugin.lua", SearchOption.AllDirectories) : [];
foreach (var pluginMainFile in pluginMainFiles)
{
using var pluginTimeout = CreatePluginTimeout(cancellationToken);
var loadingStartedAt = Stopwatch.GetTimestamp();
try
{
if (cancellationToken.IsCancellationRequested)
{
LOG.LogWarning("Was not able to load all plugins, because the operation was cancelled. It seems to be a timeout.");
LOG.LogWarning("Was not able to load all plugins, because the operation was cancelled.");
break;
}
@@ -79,11 +96,11 @@ public static partial class PluginFactory
await using(var fileStream = fileInfo.Open(FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
{
using var reader = new StreamReader(fileStream, Encoding.UTF8);
code = await reader.ReadToEndAsync(cancellationToken);
code = await reader.ReadToEndAsync(pluginTimeout.Token);
}
var pluginPath = Path.GetDirectoryName(pluginMainFile)!;
var plugin = await Load(pluginPath, code, cancellationToken: cancellationToken);
var plugin = await Load(pluginPath, code, cancellationToken: pluginTimeout.Token);
switch (plugin)
{
@@ -189,6 +206,10 @@ public static partial class PluginFactory
AVAILABLE_PLUGINS.Add(new PluginMetadata(plugin, pluginPath, isManagedByConfigServer, managedConfigurationId, configurationPriority));
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
LOG.LogError($"Was not able to load plugin '{pluginMainFile}'. Reason: It did not finish within the limit of {PLUGIN_TIMEOUT.TotalSeconds:0} seconds and was stopped after {Stopwatch.GetElapsedTime(loadingStartedAt).TotalSeconds:0.0} seconds. The remaining plugins load regardless.");
}
catch (Exception e)
{
LOG.LogError($"Was not able to load plugin '{pluginMainFile}'. Issue: {e.Message}");
@@ -201,6 +222,25 @@ public static partial class PluginFactory
{
var configObjects = await RestartAllPlugins(cancellationToken);
configObjectList.AddRange(configObjects);
//
// Each configuration plugin which started has changed the settings, but none of
// them stored the change. We store all of it once, at the end of this method:
//
hasStartedConfigurationPlugins = RUNNING_PLUGINS.OfType<PluginConfiguration>().Any();
//
// A configuration plugin which loaded but did not start has contributed nothing to
// the configuration objects. To the clean-up below, it would look as if the plugin
// had dropped all of its objects, and they would be deleted, secrets included. That
// is what a slow machine did to the configuration of an organization, once the
// time ran out before its plugin got its turn:
//
foreach (var notStartedConfigPlugin in AVAILABLE_PLUGINS.Where(plugin => plugin.Type is PluginType.CONFIGURATION && RUNNING_PLUGINS.All(runningPlugin => runningPlugin.Id != plugin.Id)))
{
notStartedConfigPluginIds.Add(notStartedConfigPlugin.Id);
LOG.LogWarning($"The configuration plugin '{notStartedConfigPlugin.Name}' (Id='{notStartedConfigPlugin.Id}') was loaded, but did not start. Everything it manages stays unchanged until it starts again. Please check the errors above.");
}
}
}
finally
@@ -255,35 +295,42 @@ public static partial class PluginFactory
LOG.LogWarning($"The configuration plugin '{unloadedEnterpriseConfigPluginId}' is deployed, but was not loaded. Everything it manages stays unchanged, because the plugin was not removed. Please check the errors above and fix the plugin.");
}
var wasConfigurationChanged = hasStartedConfigurationPlugins;
// Check LLM providers:
var wasConfigurationChanged = await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, configObjectList, SecretStoreType.LLM_PROVIDER);
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.LLM_PROVIDER))
wasConfigurationChanged = true;
// Check transcription providers:
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER, x => x.TranscriptionProviders, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, configObjectList, SecretStoreType.TRANSCRIPTION_PROVIDER))
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER, x => x.TranscriptionProviders, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.TRANSCRIPTION_PROVIDER))
wasConfigurationChanged = true;
// Check embedding providers:
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, configObjectList, SecretStoreType.EMBEDDING_PROVIDER))
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.EMBEDDING_PROVIDER))
wasConfigurationChanged = true;
// Check data sources:
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.DATA_SOURCE, x => x.DataSources, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, configObjectList, SecretStoreType.DATA_SOURCE, deleteSecret: true))
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.DATA_SOURCE, x => x.DataSources, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.DATA_SOURCE, deleteSecret: true))
wasConfigurationChanged = true;
// Check chat templates:
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, configObjectList))
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList))
wasConfigurationChanged = true;
// Check profiles:
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.PROFILE, x => x.Profiles, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, configObjectList))
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.PROFILE, x => x.Profiles, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList))
wasConfigurationChanged = true;
// Check document analysis policies:
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY, x => x.DocumentAnalysis.Policies, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, configObjectList))
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY, x => x.DocumentAnalysis.Policies, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList))
wasConfigurationChanged = true;
// Check left-over mandatory info acceptances:
if (SettingsManagerAccess.ConfigurationData.MandatoryInformation.RemoveLeftOverAcceptances(GetMandatoryInfos()))
//
// Check left-over mandatory info acceptances. The mandatory infos come from the running
// configuration plugins only, so we skip this while one of them did not start: the user
// would have to accept its infos again, although nothing about them changed:
//
if (notStartedConfigPluginIds.Count == 0 && SettingsManagerAccess.ConfigurationData.MandatoryInformation.RemoveLeftOverAcceptances(GetMandatoryInfos()))
wasConfigurationChanged = true;
// Check all managed settings, i.e. settings which a configuration plugin can lock,
@@ -294,11 +341,12 @@ public static partial class PluginFactory
//
// The enterprise approvals of all configuration plugins add up. Now that every plugin has
// contributed and the clean-up above has dropped the removed ones, we rebuild the effective
// list. We skip that while a configuration plugin is deployed but could not be loaded: its
// approvals are missing from the contributions, and withdrawing them would demand a new
// security audit for assistant plugins the organization has approved:
// list. We skip that while a configuration plugin is deployed but could not be loaded, or
// was loaded but did not start: its approvals are missing from the contributions, and
// withdrawing them would demand a new security audit for assistant plugins the
// organization has approved:
//
if(unloadedEnterpriseConfigPluginIds.Count == 0 && PluginConfiguration.RefreshEnterpriseApprovedAssistantPlugins())
if(unloadedEnterpriseConfigPluginIds.Count == 0 && notStartedConfigPluginIds.Count == 0 && PluginConfiguration.RefreshEnterpriseApprovedAssistantPlugins())
wasConfigurationChanged = true;
//
@@ -310,7 +358,7 @@ public static partial class PluginFactory
wasConfigurationChanged = true;
// Compatibility shim, see documentation/compatibility-shims/2026-08-orphaned-config-locks.md (remove after 2027-08-06):
if (RepairLegacyConfigOnlySettings(unloadedEnterpriseConfigPluginIds.Count > 0))
if (RepairLegacyConfigOnlySettings(unloadedEnterpriseConfigPluginIds.Count > 0 || notStartedConfigPluginIds.Count > 0))
wasConfigurationChanged = true;
if (wasConfigurationChanged)
@@ -354,6 +402,23 @@ public static partial class PluginFactory
return deployedEnterpriseConfigPluginIds;
}
/// <summary>
/// Creates the time limit for loading or starting a single plugin.
/// </summary>
/// <remarks>
/// The limit is linked to the token of the caller, so cancelling the whole operation still stops
/// the plugin at hand. Tell the two apart by that token: when it is not cancelled, the plugin ran
/// out of time, and only this plugin is affected.
/// </remarks>
/// <param name="cancellationToken">The token of the whole operation.</param>
/// <returns>A token source that cancels after PLUGIN_TIMEOUT or together with the caller's token.</returns>
private static CancellationTokenSource CreatePluginTimeout(CancellationToken cancellationToken)
{
var pluginTimeout = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
pluginTimeout.CancelAfter(PLUGIN_TIMEOUT);
return pluginTimeout;
}
/// <param name="pluginPath">The directory the plugin is located in, or null when the code has no directory yet.</param>
/// <param name="code">The Lua code of the plugin's main file.</param>
/// <param name="allowedBaseDirectory">
@@ -463,17 +528,17 @@ public static partial class PluginFactory
/// This is only valid as long as none of these settings gets a user interface. When you add
/// one, remove the setting from this method and from the shim's document.
/// </remarks>
/// <param name="hasUnloadedConfigPlugins" >
/// True when at least one configuration plugin is deployed but could not be loaded. In that case,
/// we cannot tell whether a value comes from that plugin or from a removed one, so we repair
/// nothing at all.
/// <param name="hasMissingConfigPlugins" >
/// True when at least one configuration plugin is deployed but could not be loaded, or was loaded
/// but did not start. In that case, we cannot tell whether a value comes from that plugin or from
/// a removed one, so we repair nothing at all.
/// </param>
/// <returns>True when at least one setting was repaired, otherwise false.</returns>
private static bool RepairLegacyConfigOnlySettings(bool hasUnloadedConfigPlugins)
private static bool RepairLegacyConfigOnlySettings(bool hasMissingConfigPlugins)
{
if (hasUnloadedConfigPlugins)
if (hasMissingConfigPlugins)
{
LOG.LogWarning("Skipping the repair of configuration-only settings: at least one configuration plugin is deployed, but could not be loaded. We try again the next time AI Studio starts.");
LOG.LogWarning("Skipping the repair of configuration-only settings: at least one configuration plugin could not be loaded or did not start. We try again the next time AI Studio starts.");
return false;
}
@@ -1,3 +1,4 @@
using System.Diagnostics;
using System.Text;
using AIStudio.Models.Registry;
using AIStudio.Settings;
@@ -10,6 +11,18 @@ public static partial class PluginFactory
{
private static readonly List<PluginBase> RUNNING_PLUGINS = [];
/// <summary>
/// From this duration on, the start of a single plugin is reported as slow.
/// </summary>
/// <remarks>
/// The plugins start one after another, so every plugin after a slow one has to wait. A plugin
/// usually starts within milliseconds. A configuration plugin takes longer, because it applies its
/// configuration, stores its secrets in the OS keyring, and loads tokenizers, but seconds are
/// still far beyond that. The time limit per plugin covers only running its Lua code, so this
/// warning is the only sign of a plugin which spends its time elsewhere.
/// </remarks>
private static readonly TimeSpan SLOW_PLUGIN_START_THRESHOLD = TimeSpan.FromSeconds(2);
/// <summary>
/// A list of all running plugins.
/// </summary>
@@ -41,7 +54,7 @@ public static partial class PluginFactory
{
try
{
var startedBasePlugin = await Start(baseLanguagePluginMetaData, cancellationToken);
var startedBasePlugin = await StartWithTimeout(baseLanguagePluginMetaData, cancellationToken);
if (startedBasePlugin is NoPlugin noPlugin)
LOG.LogError($"Was not able to start the base language plugin: Id='{baseLanguagePluginId}'. Reason: {noPlugin.Issues.First()}");
@@ -84,7 +97,7 @@ public static partial class PluginFactory
{
if(cancellationToken.IsCancellationRequested)
{
LOG.LogWarning("Cancellation requested while starting plugins. Stopping the plugin startup process. Probably due to a timeout.");
LOG.LogWarning("Cancellation requested while starting plugins. Stopping the plugin startup process.");
break;
}
@@ -100,7 +113,7 @@ public static partial class PluginFactory
// differently from the other half for no reason anyone could see.
//
if (availablePlugin.IsInternal || SettingsManagerAccess.IsPluginEnabled(availablePlugin) || availablePlugin.Type is PluginType.CONFIGURATION or PluginType.ASSISTANT or PluginType.MODEL)
if(await Start(availablePlugin, cancellationToken) is { IsValid: true } plugin)
if(await StartWithTimeout(availablePlugin, cancellationToken) is { IsValid: true } plugin)
{
if (plugin is PluginConfiguration configPlugin)
configObjects.AddRange(configPlugin.ConfigObjects);
@@ -184,6 +197,37 @@ public static partial class PluginFactory
}
}
/// <summary>
/// Starts a plugin within the time limit each plugin gets on its own.
/// </summary>
/// <remarks>
/// A plugin which runs out of time is not started, and the next plugin gets the full limit
/// again. Only the caller's token stops the start of all plugins.
/// </remarks>
/// <param name="meta">The plugin to start.</param>
/// <param name="cancellationToken">The token of the whole start.</param>
/// <returns>The started plugin, or a NoPlugin when it could not be started in time.</returns>
private static async Task<PluginBase> StartWithTimeout(IAvailablePlugin meta, CancellationToken cancellationToken)
{
using var pluginTimeout = CreatePluginTimeout(cancellationToken);
var startingStartedAt = Stopwatch.GetTimestamp();
try
{
var plugin = await Start(meta, pluginTimeout.Token);
var startingDuration = Stopwatch.GetElapsedTime(startingStartedAt);
if (startingDuration >= SLOW_PLUGIN_START_THRESHOLD)
LOG.LogWarning($"Starting the plugin took {startingDuration.TotalMilliseconds:F0} ms, which is unusually long: Id='{meta.Id}', Type='{meta.Type}', Name='{meta.Name}', Version='{meta.Version}'. All plugins after it had to wait for it.");
return plugin;
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
var reason = $"It did not finish within the limit of {PLUGIN_TIMEOUT.TotalSeconds:0} seconds and was stopped after {Stopwatch.GetElapsedTime(startingStartedAt).TotalSeconds:0.0} seconds.";
LOG.LogError($"Was not able to start plugin: Id='{meta.Id}', Type='{meta.Type}', Name='{meta.Name}', Version='{meta.Version}'. Reason: {reason} The remaining plugins start regardless.");
return new NoPlugin(reason);
}
}
private static async Task<PluginBase> Start(IAvailablePlugin meta, CancellationToken cancellationToken = default)
{
var pluginMainFile = Path.Join(meta.LocalPath, "plugin.lua");
@@ -15,5 +15,8 @@
- Fixed two confidence levels sharing the same name in the German user interface. Moderate is called "Mäßig" again, and Medium stays "Mittel".
- Fixed renaming an external data source (ERI server) leaving its secret behind under the old name in the keychain of your operating system.
- Fixed the Search Confluence tool missing pages that the search of your Confluence wiki finds. The tool now searches the same way as the search box of your wiki.
- Fixed AI Studio starting only partly when the configuration of your organization took a while to apply. The plugins late in line never started, so the app stayed in English instead of your language, and the configuration of your organization was missing, together with its models and its welcome text. Now every plugin gets its own time to start, and a slow one no longer holds up the others.
- Fixed AI Studio deleting the providers, chat templates, and profiles of your organization, including their API keys, when the configuration of your organization did not start in time. They now stay in place until the configuration starts again.
- Fixed saving the settings taking about 20 seconds each time when a chat template had an attachment on a network drive that was out of reach. Since AI Studio saves its settings while starting, this also slowed down every start.
- Upgraded several libraries to improve security.
- Upgraded to Rust v1.99.0
@@ -0,0 +1,77 @@
using System.Text.Json;
using AIStudio.Chat;
using AIStudio.Settings;
namespace AIStudio.Tests.Chat;
/// <summary>
/// Checks that storing a file attachment never asks the file system about it.
/// </summary>
/// <remarks>
/// Whether the file still exists is a question to the file system, and on a network share which is
/// out of reach it takes until the SMB timeout to get an answer -- about 20 seconds. Chat templates
/// carry attachments, so every store of the settings would wait that long, and the plugins starting
/// during that time ran out of their budget. The answer is not worth storing anyway: it is stale the
/// moment it is written.
/// </remarks>
[TestFixture]
public sealed class FileAttachmentSerializationTests
{
private const string FILE_PATH = @"\\10.255.255.1\share\report.pdf";
[Test]
public void AChatTemplateIsStoredWithoutCheckingItsAttachments()
{
var template = new ChatTemplate
{
FileAttachments = [new FileAttachment(FileAttachmentType.DOCUMENT, "report.pdf", FILE_PATH, 1024)],
ExampleConversation =
[
new ContentBlock
{
ContentType = ContentType.TEXT,
Role = ChatRole.USER,
Content = new ContentText
{
Text = "Summarize the report.",
FileAttachments = [new FileAttachmentImage("chart.png", @"\\10.255.255.1\share\chart.png", 2048)],
},
},
],
};
var json = JsonSerializer.Serialize(template, SettingsManager.JSON_OPTIONS);
Assert.Multiple(() =>
{
Assert.That(json, Does.Contain(FILE_PATH.Replace(@"\", @"\\")), "The attachment has to reach the JSON, or this test checks nothing.");
Assert.That(json, Does.Contain("chart.png"), "The attachment of the example conversation has to reach the JSON, or this test checks nothing.");
Assert.That(json, Does.Not.Contain("\"Exists\""));
});
}
[Test]
public void AnAttachmentStoredWithTheFormerFieldStillLoads()
{
const string LEGACY_JSON = """
{
"$type": "file",
"Type": "DOCUMENT",
"FileName": "report.pdf",
"FilePath": "/tmp/report.pdf",
"FileSizeBytes": 1024,
"Exists": true
}
""";
var read = JsonSerializer.Deserialize<FileAttachment>(LEGACY_JSON, SettingsManager.JSON_OPTIONS)!;
Assert.Multiple(() =>
{
Assert.That(read.FileName, Is.EqualTo("report.pdf"));
Assert.That(read.FilePath, Is.EqualTo("/tmp/report.pdf"));
Assert.That(read.FileSizeBytes, Is.EqualTo(1024));
});
}
}
@@ -0,0 +1,125 @@
using AIStudio.Settings;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Services;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.Tools.PluginSystem;
/// <summary>
/// Checks which objects of a configuration plugin the clean-up removes from the settings.
/// </summary>
/// <remarks>
/// After every start of the plugins, the clean-up compares the objects in the settings with the ones
/// the configuration plugins defined, and removes what no plugin defines anymore. A plugin which was
/// loaded but did not start has defined nothing at all. Its objects must not count as dropped: on a
/// slow machine, where the time ran out before the plugin got its turn, that deleted the providers,
/// chat templates, and profiles of an organization on every start, their API keys included.<br/><br/>
/// The settings are reached through Program.SERVICE_PROVIDER, which is why this fixture does not run
/// alongside others.
/// </remarks>
[TestFixture]
[NonParallelizable]
public sealed class LeftOverConfigurationObjectsTests
{
private static readonly Guid PLUGIN_ID = Guid.Parse("5c1f0e7a-2b9d-4c3e-8f6a-1d2e3f4a5b6c");
private RustService rustService = null!;
private ServiceProvider serviceProvider = null!;
private IServiceProvider previousServiceProvider = null!;
private SettingsManager settingsManager = null!;
[SetUp]
public void CreateSettingsWithATemplateOfThePlugin()
{
// Only builds its HTTP clients. Nothing connects, because a chat template has no secret:
this.rustService = new RustService("1", "unused");
this.settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, this.rustService);
this.settingsManager.ConfigurationData.ChatTemplates.Add(new ChatTemplate
{
Id = Guid.NewGuid().ToString(),
Name = "Template of the organization",
IsEnterpriseConfiguration = true,
EnterpriseConfigurationPluginId = PLUGIN_ID,
});
this.previousServiceProvider = Program.SERVICE_PROVIDER;
this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).AddSingleton(this.rustService).BuildServiceProvider();
Program.SERVICE_PROVIDER = this.serviceProvider;
}
[TearDown]
public void RestoreApplicationState()
{
Program.SERVICE_PROVIDER = this.previousServiceProvider;
this.serviceProvider.Dispose();
this.rustService.Dispose();
}
[Test]
public async Task TheObjectsOfAPluginWhichDidNotStartStay()
{
var wasChanged = await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, [new AvailablePlugin()], new HashSet<Guid>(), new HashSet<Guid> { PLUGIN_ID }, []);
Assert.Multiple(() =>
{
Assert.That(wasChanged, Is.False);
Assert.That(this.settingsManager.ConfigurationData.ChatTemplates, Has.Count.EqualTo(1));
});
}
[Test]
public async Task TheObjectsOfAPluginWhichStartedWithoutThemAreRemoved()
{
var wasChanged = await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, [new AvailablePlugin()], new HashSet<Guid>(), new HashSet<Guid>(), []);
Assert.Multiple(() =>
{
Assert.That(wasChanged, Is.True, "A plugin which started and no longer defines the template has dropped it, or the test above checks nothing.");
Assert.That(this.settingsManager.ConfigurationData.ChatTemplates, Is.Empty);
});
}
/// <summary>
/// The configuration plugin as the plugin factory lists it after loading it.
/// </summary>
private sealed class AvailablePlugin : IAvailablePlugin
{
public string IconDataUrl => string.Empty;
public PluginType Type => PluginType.CONFIGURATION;
public Guid Id => PLUGIN_ID;
public string Name => "Configuration of the organization";
public string Description => string.Empty;
public PluginVersion Version => new(1, 0, 0);
public string[] Authors => [];
public string SupportContact => string.Empty;
public string SourceURL => string.Empty;
public PluginCategory[] Categories => [];
public PluginTargetGroup[] TargetGroups => [];
public bool IsMaintained => true;
public string DeprecationMessage => string.Empty;
public bool IsInternal => false;
public string LocalPath => string.Empty;
public bool IsManagedByConfigServer => false;
public Guid? ManagedConfigurationId => null;
public int ConfigurationPriority => 0;
}
}
@@ -0,0 +1,38 @@
using AIStudio.Tools.PluginSystem;
namespace AIStudio.Tests.Tools.PluginSystem;
/// <summary>
/// Checks that a plugin whose code never finishes can be stopped.
/// </summary>
/// <remarks>
/// Every plugin gets a time limit of its own while it is loaded and started, so that one plugin
/// cannot keep all the others from starting. That limit is only worth something when the Lua
/// runtime gives up on a cancelled token, even in the middle of a loop which never yields. If it
/// did not, a single plugin like that would hang the start of AI Studio for good.
/// </remarks>
[TestFixture]
public sealed class PluginTimeoutTests
{
private const string ENDLESS_PLUGIN = """
ID = "0b7e5c3a-4f1d-4e8b-9a6c-2d3f4e5a6b7c"
while true do end
""";
[Test]
public async Task ALoopWhichNeverEndsIsStopped()
{
//
// The loop runs on its own, and when the runtime ignores the token, it outlives this test.
// It therefore gets the token alone, never the source, which is disposed at the end:
//
using var timeout = new CancellationTokenSource(TimeSpan.FromMilliseconds(200));
var token = timeout.Token;
var loading = Task.Run(() => PluginFactory.Load(null, ENDLESS_PLUGIN, cancellationToken: token));
var finishedInTime = await Task.WhenAny(loading, Task.Delay(TimeSpan.FromSeconds(10))) == loading;
Assert.That(finishedInTime, Is.True, "The Lua runtime ignored the cancelled token.");
Assert.That(async () => await loading, Throws.InstanceOf<OperationCanceledException>(), "Only a cancellation tells a time limit apart from a broken plugin.");
}
}
@@ -23,7 +23,7 @@ At the end of `PluginFactory.LoadAll`, AI Studio checks a fixed list of settings
Repairing means restoring the default value. Each repair is logged as a warning.
Nothing is repaired at all while a configuration plugin is deployed but could not be loaded, e.g. because of invalid Lua code. In that situation, we cannot tell whether a value comes from that plugin or from a removed one, so the repair is postponed to the next start.
Nothing is repaired at all while a configuration plugin is deployed but could not be loaded, e.g. because of invalid Lua code, or was loaded but did not start, e.g. because it ran out of time. In that situation, we cannot tell whether a value comes from that plugin or from a removed one, so the repair is postponed to the next start.
The check runs on every start, not once. This is safe because none of these settings has a user interface that writes to it, so a non-default value can only originate from a configuration plugin. This is the load-bearing assumption of the whole shim: as soon as one of these settings gets a user interface, the shim would overwrite the user's choice on every start. In that case, remove the setting from `RepairLegacyConfigOnlySettings` and from the list above.