Merge branch 'main' into chunk-data

This commit is contained in:
Paul Koudelka committed 2026-07-01 09:54:12 +02:00
commit 2e06af13e0
133 files changed
+3704 -968

No files matched your search

@@ -169,7 +169,7 @@ public static class ComponentsExtensions
public static ChatTemplate PreselectedChatTemplate(this Components component, SettingsManager settingsManager) => component switch
{
Components.CHAT => settingsManager.ConfigurationData.Chat.PreselectOptions ? settingsManager.ConfigurationData.ChatTemplates.FirstOrDefault(x => x.Id == settingsManager.ConfigurationData.Chat.PreselectedChatTemplate) ?? ChatTemplate.NO_CHAT_TEMPLATE : ChatTemplate.NO_CHAT_TEMPLATE,
Components.CHAT => settingsManager.ConfigurationData.Chat.PreselectOptions ? settingsManager.GetChatTemplateById(settingsManager.ConfigurationData.Chat.PreselectedChatTemplate) : ChatTemplate.NO_CHAT_TEMPLATE,
_ => ChatTemplate.NO_CHAT_TEMPLATE,
};
@@ -19,6 +19,10 @@ public static class ExternalHttpClientTimeout
private const string ENV_CUSTOM_ROOT_CERTIFICATES_ENABLED = "MINDWORK_AI_STUDIO_EXTERNAL_HTTP_CUSTOM_ROOT_CERTIFICATES_ENABLED";
private const string ENV_CUSTOM_ROOT_CERTIFICATE_BUNDLE_PATH = "MINDWORK_AI_STUDIO_EXTERNAL_HTTP_CUSTOM_ROOT_CERTIFICATE_BUNDLE_PATH";
private const string ENV_CUSTOM_ROOT_CERTIFICATE_ALLOWED_HOSTS = "MINDWORK_AI_STUDIO_EXTERNAL_HTTP_CUSTOM_ROOT_CERTIFICATE_ALLOWED_HOSTS";
private const string ENV_POLICY_CUSTOM_ROOT_CERTIFICATES_CONFIGURED = "AI_STUDIO_EXTERNAL_HTTP_CUSTOM_ROOT_CERTIFICATES_POLICY_CONFIGURED";
private const string ENV_POLICY_CUSTOM_ROOT_CERTIFICATES_ENABLED = "AI_STUDIO_EXTERNAL_HTTP_CUSTOM_ROOT_CERTIFICATES_ENABLED";
private const string ENV_POLICY_CUSTOM_ROOT_CERTIFICATE_BUNDLE_PATH = "AI_STUDIO_EXTERNAL_HTTP_CUSTOM_ROOT_CERTIFICATE_BUNDLE_PATH";
private const string ENV_POLICY_CUSTOM_ROOT_CERTIFICATE_ALLOWED_HOSTS = "AI_STUDIO_EXTERNAL_HTTP_CUSTOM_ROOT_CERTIFICATE_ALLOWED_HOSTS";
// id-kp-serverAuth: Extended Key Usage for TLS server authentication.
// See RFC 5280, section 4.2.1.12: https://www.rfc-editor.org/rfc/rfc5280#section-4.2.1.12
@@ -26,7 +30,7 @@ public static class ExternalHttpClientTimeout
private static string TB(string fallbackEN) => PluginSystem.I18N.I.T(fallbackEN, typeof(ExternalHttpClientTimeout).Namespace, nameof(ExternalHttpClientTimeout));
private static readonly Lazy<ILogger> LOGGER = new(() => Program.LOGGER_FACTORY.CreateLogger(nameof(ExternalHttpClientTimeout)));
private static readonly Lazy<SettingsManager> SETTINGS_MANAGER = new(() => Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>());
private static SettingsManager SettingsManagerAccess => Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
private static readonly Lock CUSTOM_ROOT_CERTIFICATE_LOCK = new();
private static CustomRootCertificateCache? CUSTOM_ROOT_CERTIFICATE_CACHE;
@@ -91,7 +95,7 @@ public static class ExternalHttpClientTimeout
private static TimeSpan GetTimeout()
{
var seconds = SETTINGS_MANAGER.Value.ConfigurationData.App.HttpClientTimeoutSeconds;
var seconds = SettingsManagerAccess.ConfigurationData.App.HttpClientTimeoutSeconds;
if (seconds <= 0)
seconds = DEFAULT_HTTP_CLIENT_TIMEOUT_SECONDS;
@@ -123,29 +127,43 @@ public static class ExternalHttpClientTimeout
private static CustomRootCertificateConfiguration ReadCustomRootCertificateConfiguration()
{
if (TryParseBooleanEnvironmentValue(Environment.GetEnvironmentVariable(ENV_POLICY_CUSTOM_ROOT_CERTIFICATES_CONFIGURED), out var policyConfigured) && policyConfigured)
{
var policyEnabled = TryParseBooleanEnvironmentValue(Environment.GetEnvironmentVariable(ENV_POLICY_CUSTOM_ROOT_CERTIFICATES_ENABLED), out var parsedPolicyEnabled) && parsedPolicyEnabled;
var policyBundlePath = Environment.GetEnvironmentVariable(ENV_POLICY_CUSTOM_ROOT_CERTIFICATE_BUNDLE_PATH)?.Trim() ?? string.Empty;
var policyAllowedHosts = Environment.GetEnvironmentVariable(ENV_POLICY_CUSTOM_ROOT_CERTIFICATE_ALLOWED_HOSTS) ?? string.Empty;
return new(policyEnabled, policyBundlePath, ReadAllowedHostPatternsFromDelimitedValue(policyAllowedHosts), TB("policy files"));
}
var envEnabled = Environment.GetEnvironmentVariable(ENV_CUSTOM_ROOT_CERTIFICATES_ENABLED);
var envBundlePath = Environment.GetEnvironmentVariable(ENV_CUSTOM_ROOT_CERTIFICATE_BUNDLE_PATH);
var envAllowedHosts = Environment.GetEnvironmentVariable(ENV_CUSTOM_ROOT_CERTIFICATE_ALLOWED_HOSTS);
if (!string.IsNullOrWhiteSpace(envEnabled) || !string.IsNullOrWhiteSpace(envBundlePath) || !string.IsNullOrWhiteSpace(envAllowedHosts))
{
var enabled = TryParseBooleanEnvironmentValue(envEnabled, out var parsedEnvEnabled)
? parsedEnvEnabled
: SettingsManagerAccess.ConfigurationData.App.ExternalHttpCustomRootCertificatesEnabled;
var enabled = TryParseBooleanEnvironmentValue(envEnabled, out var parsedEnvEnabled)
? parsedEnvEnabled
: SETTINGS_MANAGER.Value.ConfigurationData.App.ExternalHttpCustomRootCertificatesEnabled;
var bundlePath = !string.IsNullOrWhiteSpace(envBundlePath)
? envBundlePath.Trim()
: SettingsManagerAccess.ConfigurationData.App.ExternalHttpCustomRootCertificateBundlePath.Trim();
var bundlePath = !string.IsNullOrWhiteSpace(envBundlePath)
? envBundlePath.Trim()
: SETTINGS_MANAGER.Value.ConfigurationData.App.ExternalHttpCustomRootCertificateBundlePath.Trim();
var allowedHostPatterns = !string.IsNullOrWhiteSpace(envAllowedHosts)
? ReadAllowedHostPatternsFromDelimitedValue(envAllowedHosts)
: ReadAllowedHostPatterns(SettingsManagerAccess.ConfigurationData.App.ExternalHttpCustomRootCertificateAllowedHosts);
var allowedHostPatterns = ReadAllowedHostPatterns(envAllowedHosts);
var source = ReadCustomRootCertificateConfigurationSource(envEnabled, envBundlePath, envAllowedHosts);
return new(enabled, bundlePath, allowedHostPatterns, TB("environment variables"));
}
return new(enabled, bundlePath, allowedHostPatterns, source);
return new(
SettingsManagerAccess.ConfigurationData.App.ExternalHttpCustomRootCertificatesEnabled,
SettingsManagerAccess.ConfigurationData.App.ExternalHttpCustomRootCertificateBundlePath.Trim(),
ReadAllowedHostPatterns(SettingsManagerAccess.ConfigurationData.App.ExternalHttpCustomRootCertificateAllowedHosts),
ReadCustomRootCertificateSettingsSource());
}
private static string ReadCustomRootCertificateConfigurationSource(string? envEnabled, string? envBundlePath, string? envAllowedHosts)
private static string ReadCustomRootCertificateSettingsSource()
{
if (!string.IsNullOrWhiteSpace(envEnabled) || !string.IsNullOrWhiteSpace(envBundlePath) || !string.IsNullOrWhiteSpace(envAllowedHosts))
return TB("environment variables");
var enabledIsManaged = ManagedConfiguration.TryGet(x => x.App, x => x.ExternalHttpCustomRootCertificatesEnabled, out var enabledMeta) && enabledMeta.IsLocked;
var bundlePathIsManaged = ManagedConfiguration.TryGet(x => x.App, x => x.ExternalHttpCustomRootCertificateBundlePath, out var bundlePathMeta) && bundlePathMeta.IsLocked;
var allowedHostsIsManaged = ManagedConfiguration.TryGet(x => x.App, x => x.ExternalHttpCustomRootCertificateAllowedHosts, out var allowedHostsMeta) && allowedHostsMeta.IsLocked;
@@ -154,12 +172,13 @@ public static class ExternalHttpClientTimeout
: TB("app settings");
}
private static IReadOnlyList<string> ReadAllowedHostPatterns(string? envAllowedHosts)
private static IReadOnlyList<string> ReadAllowedHostPatternsFromDelimitedValue(string allowedHosts)
{
IEnumerable<string> rawPatterns = !string.IsNullOrWhiteSpace(envAllowedHosts)
? envAllowedHosts.Split([';', ','], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
: SETTINGS_MANAGER.Value.ConfigurationData.App.ExternalHttpCustomRootCertificateAllowedHosts;
return ReadAllowedHostPatterns(allowedHosts.Split([';', ','], StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries));
}
private static IReadOnlyList<string> ReadAllowedHostPatterns(IEnumerable<string> rawPatterns)
{
var patterns = new HashSet<string>(StringComparer.OrdinalIgnoreCase);
foreach (var rawPattern in rawPatterns)
{
@@ -17,6 +17,7 @@ public sealed class PandocProcessBuilder
private static readonly RID CPU_ARCHITECTURE = RIDExtensions.GetCurrentRID();
private static readonly RID METADATA_ARCHITECTURE = META_DATA_ARCH.Architecture.ToRID();
private static readonly ILogger LOGGER = Program.LOGGER_FACTORY.CreateLogger(nameof(PandocProcessBuilder));
private const string FLATPAK_PANDOC_PLUGIN_BIN_DIRECTORY = "/app/plugins/pandoc/bin";
// Tracks whether the first log has been written to avoid log spam on repeated calls:
private static bool HAS_LOGGED_ONCE;
@@ -220,7 +221,8 @@ public sealed class PandocProcessBuilder
}
}
foreach (var candidate in SystemPandocExecutableCandidates(PandocExecutableName))
var runtimeInfo = await rustService.GetRuntimeInfo();
foreach (var candidate in SystemPandocExecutableCandidates(PandocExecutableName, runtimeInfo.LinuxPackageType))
{
if (!File.Exists(candidate))
continue;
@@ -250,7 +252,7 @@ public sealed class PandocProcessBuilder
/// </summary>
public static string PandocExecutableName => CPU_ARCHITECTURE is RID.WIN_ARM64 or RID.WIN_X64 ? "pandoc.exe" : "pandoc";
private static IEnumerable<string> SystemPandocExecutableCandidates(string executableName)
private static IEnumerable<string> SystemPandocExecutableCandidates(string executableName, string linuxPackageType)
{
var candidates = new List<string>();
@@ -269,6 +271,9 @@ public sealed class PandocProcessBuilder
break;
case RID.LINUX_X64 or RID.LINUX_ARM64:
if (string.Equals(linuxPackageType, "flatpak", StringComparison.OrdinalIgnoreCase))
AddCandidate(candidates, FLATPAK_PANDOC_PLUGIN_BIN_DIRECTORY, executableName);
AddCandidate(candidates, "/usr/local/bin", executableName);
AddCandidate(candidates, "/usr/bin", executableName);
AddCandidate(candidates, "/snap/bin", executableName);
@@ -0,0 +1,18 @@
namespace AIStudio.Tools.PluginSystem;
/// <summary>
/// Represents complex content from a configuration plugin that is read live from
/// running plugins and is not persisted to the settings data model.
/// </summary>
public interface ILivePluginContent
{
/// <summary>
/// The stable ID of the live plugin content.
/// </summary>
public string Id { get; }
/// <summary>
/// The ID of the enterprise configuration plugin that provides this content.
/// </summary>
public Guid EnterpriseConfigurationPluginId { get; }
}
@@ -9,11 +9,12 @@ namespace AIStudio.Tools.PluginSystem;
public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginType type) : PluginBase(isInternal, state, type)
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PluginConfiguration).Namespace, nameof(PluginConfiguration));
private static readonly SettingsManager SETTINGS_MANAGER = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
private static SettingsManager SettingsManagerAccess => Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger(nameof(PluginConfiguration));
private List<PluginConfigurationObject> configObjects = [];
private List<DataMandatoryInfo> mandatoryInfos = [];
private List<DataIntroduction> introductions = [];
/// <summary>
/// The list of configuration objects. Configuration objects are, e.g., providers or chat templates.
@@ -22,9 +23,16 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
/// <summary>
/// The list of mandatory infos provided by this configuration plugin.
/// Mandatory infos are live plugin content and are not persisted to ConfigurationData.
/// </summary>
public IReadOnlyList<DataMandatoryInfo> MandatoryInfos => this.mandatoryInfos;
/// <summary>
/// The list of introductions provided by this configuration plugin.
/// Introductions are live plugin content and are not persisted to ConfigurationData.
/// </summary>
public IReadOnlyList<DataIntroduction> Introductions => this.introductions;
/// <summary>
/// True/false when explicitly configured in the plugin, otherwise null.
/// </summary>
@@ -43,7 +51,7 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
await StoreEnterpriseApiKeysAsync();
await StoreEnterpriseSecretsAsync();
await SETTINGS_MANAGER.StoreSettings();
await SettingsManagerAccess.StoreSettings();
await MessageBus.INSTANCE.SendMessage<bool>(null, Event.CONFIGURATION_CHANGED);
}
}
@@ -132,6 +140,7 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
{
this.configObjects.Clear();
this.mandatoryInfos.Clear();
this.introductions.Clear();
// Ensure that the main CONFIG table exists and is a valid Lua table:
if (!this.State.Environment["CONFIG"].TryRead<LuaTable>(out var mainTable))
@@ -156,6 +165,9 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
// Config: what should be the start page?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.StartPage, this.Id, settingsTable, dryRun);
// Config: show built-in introduction on the home page?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.ShowIntroduction, this.Id, settingsTable, dryRun);
// Config: show quick start guide on the home page?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.ShowQuickStartGuide, this.Id, settingsTable, dryRun);
@@ -184,6 +196,16 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.ExternalHttpCustomRootCertificatesEnabled, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.ExternalHttpCustomRootCertificateBundlePath, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.ExternalHttpCustomRootCertificateAllowedHosts, this.Id, settingsTable, dryRun);
// Config: provider confidence settings
ManagedConfiguration.TryProcessConfiguration(x => x.Confidence, x => x.EnforceGlobalMinimumConfidence, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Confidence, x => x.GlobalMinimumConfidence, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Confidence, x => x.ShowProviderConfidence, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Confidence, x => x.ConfidenceScheme, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Confidence, x => x.CustomConfidenceScheme, this.Id, settingsTable, dryRun);
// Config: data source security settings
ManagedConfiguration.TryProcessConfiguration(x => x.DataSourceSecurity, x => x.TrustedProviderIds, this.Id, settingsTable, dryRun);
// Handle configured LLM providers:
PluginConfigurationObject.TryParse(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, x => x.NextProviderNum, mainTable, this.Id, ref this.configObjects, dryRun);
@@ -208,6 +230,9 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
// Handle configured mandatory infos:
this.TryReadMandatoryInfos(mainTable);
// Handle configured introductions:
this.TryReadIntroductions(mainTable);
// Config: preselected provider?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.PreselectedProvider, Guid.Empty, this.Id, settingsTable, dryRun);
@@ -215,6 +240,12 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
// Config: preselected profile?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.PreselectedProfile, Guid.Empty, this.Id, settingsTable, dryRun);
// Config: preselected chat options?
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectOptions, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectedProvider, Guid.Empty, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectedProfile, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectedChatTemplate, this.Id, settingsTable, dryRun);
// Config: transcription provider?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.UseTranscriptionProvider, Guid.Empty, this.Id, settingsTable, dryRun);
@@ -242,4 +273,25 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
LOG.LogWarning("The table 'MANDATORY_INFOS' entry at index {Index} does not contain a valid mandatory info (config plugin id: {ConfigPluginId}).", i, this.Id);
}
}
private void TryReadIntroductions(LuaTable mainTable)
{
if (!mainTable.TryGetValue("INTRODUCTIONS", out var introductionsValue) || !introductionsValue.TryRead<LuaTable>(out var introductionsTable))
return;
for (var i = 1; i <= introductionsTable.ArrayLength; i++)
{
var luaIntroductionValue = introductionsTable[i];
if (!luaIntroductionValue.TryRead<LuaTable>(out var luaIntroductionTable))
{
LOG.LogWarning("The table 'INTRODUCTIONS' entry at index {Index} is not a valid table (config plugin id: {ConfigPluginId}).", i, this.Id);
continue;
}
if (DataIntroduction.TryParseConfiguration(i, luaIntroductionTable, this.Id, out var introduction))
this.introductions.Add(introduction);
else
LOG.LogWarning("The table 'INTRODUCTIONS' entry at index {Index} does not contain a valid introduction (config plugin id: {ConfigPluginId}).", i, this.Id);
}
}
}
@@ -15,8 +15,9 @@ namespace AIStudio.Tools.PluginSystem;
/// </summary>
public sealed record PluginConfigurationObject
{
private static readonly RustService RUST_SERVICE = Program.SERVICE_PROVIDER.GetRequiredService<RustService>();
private static readonly SettingsManager SETTINGS_MANAGER = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
private static RustService RustService => Program.SERVICE_PROVIDER.GetRequiredService<RustService>();
private static SettingsManager SettingsManagerAccess => Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
private static ThreadSafeRandom Rng => Program.SERVICE_PROVIDER.GetRequiredService<ThreadSafeRandom>();
private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger<PluginConfigurationObject>();
/// <summary>
@@ -92,7 +93,8 @@ public sealed record PluginConfigurationObject
return false;
}
var storedObjects = configObjectSelection.Compile()(SETTINGS_MANAGER.ConfigurationData);
var localSettingsManager = SettingsManagerAccess;
var storedObjects = configObjectSelection.Compile()(localSettingsManager.ConfigurationData);
var numberObjects = luaTable.ArrayLength;
ThreadSafeRandom? random = null;
for (var i = 1; i <= numberObjects; i++)
@@ -142,7 +144,7 @@ public sealed record PluginConfigurationObject
// Case: The object does not exist, we have to add it
else
{
if (nextConfigObjectNumSelection.TryIncrement(SETTINGS_MANAGER.ConfigurationData, IncrementType.POST) is { Success: true, UpdatedValue: var nextNum })
if (nextConfigObjectNumSelection.TryIncrement(localSettingsManager.ConfigurationData, IncrementType.POST) is { Success: true, UpdatedValue: var nextNum })
{
// Case: Increment the next number was successful
configObject = configObject with { Num = nextNum };
@@ -151,7 +153,7 @@ public sealed record PluginConfigurationObject
else
{
// Case: The next number could not be incremented, we use a random number
random ??= new ThreadSafeRandom();
random ??= Rng;
configObject = configObject with { Num = (uint)random.Next(500_000, 1_000_000) };
storedObjects.Add((TClass)configObject);
LOG.LogWarning("The next number for the configuration object '{ConfigObjectName}' (id={ConfigObjectId}) could not be incremented. Using a random number instead (config plugin id: {ConfigPluginId}).", configObject.Name, configObject.Id, configPluginId);
@@ -222,7 +224,8 @@ public sealed record PluginConfigurationObject
return false;
}
var storedObjects = SETTINGS_MANAGER.ConfigurationData.DataSources;
var localSettingsManager = SettingsManagerAccess;
var storedObjects = localSettingsManager.ConfigurationData.DataSources;
var numberObjects = luaTable.ArrayLength;
ThreadSafeRandom? random = null;
for (var i = 1; i <= numberObjects; i++)
@@ -259,14 +262,14 @@ public sealed record PluginConfigurationObject
}
else
{
if (IncrementDataSourceNum() is { Success: true, UpdatedValue: var nextNum })
if (IncrementDataSourceNum(localSettingsManager.ConfigurationData) is { Success: true, UpdatedValue: var nextNum })
{
configObject = configObject with { Num = nextNum };
storedObjects.Add(configObject);
}
else
{
random ??= new ThreadSafeRandom();
random ??= Rng;
configObject = configObject with { Num = (uint)random.Next(500_000, 1_000_000) };
storedObjects.Add(configObject);
LOG.LogWarning("The next number for the data source '{ConfigObjectName}' (id={ConfigObjectId}) could not be incremented. Using a random number instead (config plugin id: {ConfigPluginId}).", configObject.Name, configObject.Id, configPluginId);
@@ -276,9 +279,9 @@ public sealed record PluginConfigurationObject
return true;
static IncrementResult<uint> IncrementDataSourceNum()
static IncrementResult<uint> IncrementDataSourceNum(Data data)
{
return ((Expression<Func<Data, uint>>)(x => x.NextDataSourceNum)).TryIncrement(SETTINGS_MANAGER.ConfigurationData, IncrementType.POST);
return ((Expression<Func<Data, uint>>)(x => x.NextDataSourceNum)).TryIncrement(data, IncrementType.POST);
}
}
@@ -301,7 +304,8 @@ public sealed record PluginConfigurationObject
SecretStoreType? secretStoreType = null,
bool deleteSecret = false) where TClass : IConfigurationObject
{
var configuredObjects = configObjectSelection.Compile()(SETTINGS_MANAGER.ConfigurationData);
var localSettingsManager = SettingsManagerAccess;
var configuredObjects = configObjectSelection.Compile()(localSettingsManager.ConfigurationData);
var leftOverObjects = new List<TClass>();
foreach (var configuredObject in configuredObjects)
{
@@ -357,7 +361,7 @@ public sealed record PluginConfigurationObject
// Delete the API key from the OS keyring if the removed object has one:
if(deleteSecret && item is ISecretId regularSecretId)
{
var deleteResult = await RUST_SERVICE.DeleteSecret(regularSecretId, secretStoreType ?? SecretStoreType.DATA_SOURCE);
var deleteResult = await RustService.DeleteSecret(regularSecretId, secretStoreType ?? SecretStoreType.DATA_SOURCE);
if (deleteResult.Success)
LOG.LogInformation($"Successfully deleted secret for removed enterprise object '{item.Name}' from the OS keyring.");
else
@@ -365,7 +369,7 @@ public sealed record PluginConfigurationObject
}
else if(secretStoreType is not null && item is ISecretId secretId)
{
var deleteResult = await RUST_SERVICE.DeleteAPIKey(secretId, secretStoreType.Value);
var deleteResult = await RustService.DeleteAPIKey(secretId, secretStoreType.Value);
if (deleteResult.Success)
LOG.LogInformation($"Successfully deleted API key for removed enterprise provider '{item.Name}' from the OS keyring.");
else
@@ -191,7 +191,7 @@ public static partial class PluginFactory
wasConfigurationChanged = true;
// Check left-over mandatory info acceptances:
if (SETTINGS_MANAGER.ConfigurationData.MandatoryInformation.RemoveLeftOverAcceptances(GetMandatoryInfos()))
if (SettingsManagerAccess.ConfigurationData.MandatoryInformation.RemoveLeftOverAcceptances(GetMandatoryInfos()))
wasConfigurationChanged = true;
// Check for a preselected provider:
@@ -201,6 +201,19 @@ public static partial class PluginFactory
// Check for a preselected profile:
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.PreselectedProfile, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
// Check for preselected chat options:
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectOptions, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectedProvider, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectedProfile, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.PreselectedChatTemplate, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
// Check for the update interval:
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.UpdateInterval, AVAILABLE_PLUGINS))
@@ -214,6 +227,10 @@ public static partial class PluginFactory
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.StartPage, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
// Check for the built-in introduction visibility:
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ShowIntroduction, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
// Check for the quick start guide visibility:
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ShowQuickStartGuide, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
@@ -263,6 +280,26 @@ public static partial class PluginFactory
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.ExternalHttpCustomRootCertificateAllowedHosts, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
// Check provider confidence settings:
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Confidence, x => x.EnforceGlobalMinimumConfidence, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Confidence, x => x.GlobalMinimumConfidence, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Confidence, x => x.ShowProviderConfidence, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Confidence, x => x.ConfidenceScheme, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Confidence, x => x.CustomConfidenceScheme, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
// Check data source security settings:
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.DataSourceSecurity, x => x.TrustedProviderIds, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
// Check if audit is required before it can be activated
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.AssistantPluginAudit, x => x.RequireAuditBeforeActivation, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
@@ -285,7 +322,7 @@ public static partial class PluginFactory
if (wasConfigurationChanged)
{
await SETTINGS_MANAGER.StoreSettings();
await SettingsManagerAccess.StoreSettings();
await MessageBus.INSTANCE.SendMessage<bool>(null, Event.CONFIGURATION_CHANGED);
}
}
@@ -64,7 +64,7 @@ public static partial class PluginFactory
try
{
if (availablePlugin.IsInternal || SETTINGS_MANAGER.IsPluginEnabled(availablePlugin) || availablePlugin.Type == PluginType.CONFIGURATION || availablePlugin.Type == PluginType.ASSISTANT)
if (availablePlugin.IsInternal || SettingsManagerAccess.IsPluginEnabled(availablePlugin) || availablePlugin.Type == PluginType.CONFIGURATION || availablePlugin.Type == PluginType.ASSISTANT)
if(await Start(availablePlugin, cancellationToken) is { IsValid: true } plugin)
{
if (plugin is PluginConfiguration configPlugin)
@@ -6,7 +6,7 @@ namespace AIStudio.Tools.PluginSystem;
public static partial class PluginFactory
{
private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger(nameof(PluginFactory));
private static readonly SettingsManager SETTINGS_MANAGER = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
private static SettingsManager SettingsManagerAccess => Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
private static string DATA_DIR = string.Empty;
private static string PLUGINS_ROOT = string.Empty;
@@ -136,4 +136,13 @@ public static partial class PluginFactory
.SelectMany(plugin => plugin.MandatoryInfos)
.ToList();
}
public static IReadOnlyList<DataIntroduction> GetIntroductions()
{
return RUNNING_PLUGINS
.OfType<PluginConfiguration>()
.SelectMany(plugin => plugin.Introductions)
.OrderBy(introduction => introduction.Index)
.ToList();
}
}
@@ -6,7 +6,7 @@ namespace AIStudio.Tools.RAG;
public static class IRetrievalContextExtensions
{
private static readonly ILogger<IRetrievalContext> LOGGER = Program.SERVICE_PROVIDER.GetService<ILogger<IRetrievalContext>>()!;
private static readonly ILogger<IRetrievalContext> LOGGER = Program.LOGGER_FACTORY.CreateLogger<IRetrievalContext>();
public static async Task<string> AsMarkdown(this IReadOnlyList<IRetrievalContext> retrievalContexts, StringBuilder? sb = null, CancellationToken token = default)
{
@@ -1,6 +1,5 @@
using AIStudio.Assistants.ERI;
using AIStudio.Provider;
using AIStudio.Provider.SelfHosted;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ERIClient;
@@ -43,7 +42,7 @@ public sealed class DataSourceService
return new([], []);
}
return await this.GetDataSources(selectedLLMProvider.IsSelfHosted, previousSelectedDataSources);
return await this.GetDataSources(selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), previousSelectedDataSources);
}
/// <summary>
@@ -66,10 +65,10 @@ public sealed class DataSourceService
return new([], []);
}
return await this.GetDataSources(selectedLLMProvider is ProviderSelfHosted, previousSelectedDataSources);
return await this.GetDataSources(selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), previousSelectedDataSources);
}
private async Task<AllowedSelectedDataSources> GetDataSources(bool usingSelfHostedProvider, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
private async Task<AllowedSelectedDataSources> GetDataSources(bool usingTrustedProvider, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
{
var allDataSources = this.settingsManager.ConfigurationData.DataSources;
var filteredDataSources = new List<IDataSource>(allDataSources.Count);
@@ -78,7 +77,7 @@ public sealed class DataSourceService
// Start all checks in parallel:
foreach (var source in allDataSources)
tasks.Add(this.CheckOneDataSource(source, usingSelfHostedProvider));
tasks.Add(this.CheckOneDataSource(source, usingTrustedProvider));
// Wait for all checks and collect the results:
foreach (var task in tasks)
@@ -95,7 +94,7 @@ public sealed class DataSourceService
return new(filteredDataSources, filteredSelectedDataSources);
}
private async Task<IDataSource?> CheckOneDataSource(IDataSource source, bool usingSelfHostedProvider)
private async Task<IDataSource?> CheckOneDataSource(IDataSource source, bool usingTrustedProvider)
{
//
// Unfortunately, we have to live-check any ERI source for its security requirements.
@@ -137,10 +136,10 @@ public sealed class DataSourceService
case DataSourceSecurity.ALLOW_ANY:
//
// Case: The data source allows any provider type. We want to use a self-hosted provider.
// Case: The data source allows any provider type. We want to use a trusted provider.
// There is no issue with this source. Accept it.
//
if(usingSelfHostedProvider)
if(usingTrustedProvider)
return source;
//
@@ -151,13 +150,13 @@ public sealed class DataSourceService
return source;
//
// Case: The ERI source requires a self-hosted provider. This misconfiguration happens
// Case: The ERI source requires a self-hosted or organization-trusted provider. This misconfiguration happens
// when the ERI server operator changes the security requirements. The ERI server
// operator owns the data -- we have to respect their rules. We skip this source.
//
if (eriSourceRequirements is { AllowedProviderType: ProviderType.SELF_HOSTED })
{
this.logger.LogWarning($"The ERI source '{source.Name}' (id={source.Id}) requires a self-hosted provider. We skip this source.");
this.logger.LogWarning($"The ERI source '{source.Name}' (id={source.Id}) requires a self-hosted or organization-trusted provider. We skip this source.");
return null;
}
@@ -171,22 +170,22 @@ public sealed class DataSourceService
//
// Case: Missing rules. We skip this source. Better safe than sorry.
//
this.logger.LogDebug($"The ERI source '{source.Name}' (id={source.Id}) was filtered out due to missing rules.");
this.logger.LogWarning($"The ERI source '{source.Name}' (id={source.Id}) was filtered out due to missing rules.");
return null;
//
// Case: The data source requires a self-hosted provider. We want to use a self-hosted provider.
// Case: The data source requires a trusted provider. We want to use a trusted provider.
// There is no issue with this source. Accept it.
//
case DataSourceSecurity.SELF_HOSTED when usingSelfHostedProvider:
case DataSourceSecurity.SELF_HOSTED when usingTrustedProvider:
return source;
//
// Case: The data source requires a self-hosted provider. We want to use a cloud provider.
// Case: The data source requires a trusted provider. We want to use an untrusted provider.
// We skip this source.
//
case DataSourceSecurity.SELF_HOSTED when !usingSelfHostedProvider:
this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) requires a self-hosted provider. We skip this source.");
case DataSourceSecurity.SELF_HOSTED when !usingTrustedProvider:
this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) requires a self-hosted or organization-trusted provider. We skip this source.");
return null;
//
@@ -4,6 +4,23 @@ namespace AIStudio.Tools.Services;
public sealed partial class RustService
{
private const string SELF_HOSTED_SECRET_ID = "Self-hosted";
// Temporary compatibility shim until 2026-12-19:
// documentation/compatibility-shims/2026-06-self-hosted-secret-id.md
private const string LEGACY_SELF_HOSTED_SECRET_ID_DE = "Selbst gehostet";
private static string APIKey(SecretStoreType storeType, ISecretId secretId) => $"{storeType.Prefix()}::{secretId.SecretId}::{secretId.SecretName}::api_key";
private static IEnumerable<string> LegacySelfHostedAPIKeys(ISecretId secretId, SecretStoreType storeType)
{
if (secretId.SecretId == SELF_HOSTED_SECRET_ID)
yield return $"{storeType.Prefix()}::{LEGACY_SELF_HOSTED_SECRET_ID_DE}::{secretId.SecretName}::api_key";
if (secretId.SecretId == $"{ISecretId.ENTERPRISE_KEY_PREFIX}::{SELF_HOSTED_SECRET_ID}")
yield return $"{storeType.Prefix()}::{ISecretId.ENTERPRISE_KEY_PREFIX}::{LEGACY_SELF_HOSTED_SECRET_ID_DE}::{secretId.SecretName}::api_key";
}
/// <summary>
/// Try to get the API key for the given secret ID.
/// </summary>
@@ -13,24 +30,55 @@ public sealed partial class RustService
/// <returns>The requested secret.</returns>
public async Task<RequestedSecret> GetAPIKey(ISecretId secretId, SecretStoreType storeType, bool isTrying = false)
{
var prefix = storeType.Prefix();
var secretRequest = new SelectSecretRequest($"{prefix}::{secretId.SecretId}::{secretId.SecretName}::api_key", Environment.UserName, isTrying);
var secretKey = APIKey(storeType, secretId);
var legacySecretKeys = LegacySelfHostedAPIKeys(secretId, storeType).ToList();
var secret = await this.GetAPIKeyByKey(secretKey, isTrying || legacySecretKeys.Count > 0);
if (secret.Success)
{
foreach (var legacySecretKey in legacySecretKeys)
await this.DeleteAPIKeyByKey(legacySecretKey, isTrying: true);
return secret;
}
foreach (var legacySecretKey in legacySecretKeys)
{
var legacySecret = await this.GetAPIKeyByKey(legacySecretKey, isTrying: true);
if (!legacySecret.Success)
continue;
this.logger!.LogInformation($"Migrating legacy self-hosted API key namespace '{legacySecretKey}' to '{secretKey}'.");
var migrationResult = await this.StoreEncryptedAPIKeyByKey(secretKey, legacySecret.Secret);
if (migrationResult.Success)
await this.DeleteAPIKeyByKey(legacySecretKey, isTrying: true);
else
this.logger!.LogWarning($"Failed to migrate legacy self-hosted API key namespace '{legacySecretKey}' to '{secretKey}': '{migrationResult.Issue}'");
return legacySecret;
}
if (!isTrying)
this.logger!.LogError($"Failed to get the API key for '{secretKey}': '{secret.Issue}'");
return secret;
}
private async Task<RequestedSecret> GetAPIKeyByKey(string secretKey, bool isTrying)
{
var secretRequest = new SelectSecretRequest(secretKey, Environment.UserName, isTrying);
var result = await this.http.PostAsJsonAsync("/secrets/get", secretRequest, this.jsonRustSerializerOptions);
if (!result.IsSuccessStatusCode)
{
if(!isTrying)
this.logger!.LogError($"Failed to get the API key for '{prefix}::{secretId.SecretId}::{secretId.SecretName}::api_key' due to an API issue: '{result.StatusCode}'");
this.logger!.LogError($"Failed to get the API key for '{secretKey}' due to an API issue: '{result.StatusCode}'");
return new RequestedSecret(false, new EncryptedText(string.Empty), TB("Failed to get the API key due to an API issue."));
}
var secret = await result.Content.ReadFromJsonAsync<RequestedSecret>(this.jsonRustSerializerOptions);
if (!secret.Success && !isTrying)
this.logger!.LogError($"Failed to get the API key for '{prefix}::{secretId.SecretId}::{secretId.SecretName}::api_key': '{secret.Issue}'");
if (secret.Success)
this.logger!.LogDebug($"Successfully retrieved the API key for '{prefix}::{secretId.SecretId}::{secretId.SecretName}::api_key'.");
this.logger!.LogDebug($"Successfully retrieved the API key for '{secretKey}'.");
else if (isTrying)
this.logger!.LogDebug($"No API key configured for '{prefix}::{secretId.SecretId}::{secretId.SecretName}::api_key' (try mode): '{secret.Issue}'");
this.logger!.LogDebug($"No API key configured for '{secretKey}' (try mode): '{secret.Issue}'");
return secret;
}
@@ -44,21 +92,34 @@ public sealed partial class RustService
/// <returns>The store secret response.</returns>
public async Task<StoreSecretResponse> SetAPIKey(ISecretId secretId, string key, SecretStoreType storeType)
{
var prefix = storeType.Prefix();
var encryptedKey = await this.encryptor!.Encrypt(key);
var request = new StoreSecretRequest($"{prefix}::{secretId.SecretId}::{secretId.SecretName}::api_key", Environment.UserName, encryptedKey);
var secretKey = APIKey(storeType, secretId);
var state = await this.StoreEncryptedAPIKeyByKey(secretKey, encryptedKey);
if (state.Success)
{
foreach (var legacySecretKey in LegacySelfHostedAPIKeys(secretId, storeType))
await this.DeleteAPIKeyByKey(legacySecretKey, isTrying: true);
}
return state;
}
private async Task<StoreSecretResponse> StoreEncryptedAPIKeyByKey(string secretKey, EncryptedText encryptedKey)
{
var request = new StoreSecretRequest(secretKey, Environment.UserName, encryptedKey);
var result = await this.http.PostAsJsonAsync("/secrets/store", request, this.jsonRustSerializerOptions);
if (!result.IsSuccessStatusCode)
{
this.logger!.LogError($"Failed to store the API key for '{prefix}::{secretId.SecretId}::{secretId.SecretName}::api_key' due to an API issue: '{result.StatusCode}'");
return new StoreSecretResponse(false, TB("Failed to get the API key due to an API issue."));
this.logger!.LogError($"Failed to store the API key for '{secretKey}' due to an API issue: '{result.StatusCode}'");
return new StoreSecretResponse(false, TB("Failed to store the API key due to an API issue."));
}
var state = await result.Content.ReadFromJsonAsync<StoreSecretResponse>(this.jsonRustSerializerOptions);
if (!state.Success)
this.logger!.LogError($"Failed to store the API key for '{prefix}::{secretId.SecretId}::{secretId.SecretName}::api_key': '{state.Issue}'");
this.logger!.LogError($"Failed to store the API key for '{secretKey}': '{state.Issue}'");
else
this.logger!.LogDebug($"Successfully stored the API key for '{secretKey}'.");
this.logger!.LogDebug($"Successfully stored the API key for '{prefix}::{secretId.SecretId}::{secretId.SecretName}::api_key'.");
return state;
}
@@ -70,18 +131,35 @@ public sealed partial class RustService
/// <returns>The delete secret response.</returns>
public async Task<DeleteSecretResponse> DeleteAPIKey(ISecretId secretId, SecretStoreType storeType)
{
var prefix = storeType.Prefix();
var request = new SelectSecretRequest($"{prefix}::{secretId.SecretId}::{secretId.SecretName}::api_key", Environment.UserName, false);
var deleteResult = await this.DeleteAPIKeyByKey(APIKey(storeType, secretId));
if (!deleteResult.Success)
return deleteResult;
foreach (var legacySecretKey in LegacySelfHostedAPIKeys(secretId, storeType))
{
var legacyDeleteResult = await this.DeleteAPIKeyByKey(legacySecretKey, isTrying: true);
if (!legacyDeleteResult.Success)
return legacyDeleteResult;
deleteResult = deleteResult with { WasEntryFound = deleteResult.WasEntryFound || legacyDeleteResult.WasEntryFound };
}
return deleteResult;
}
private async Task<DeleteSecretResponse> DeleteAPIKeyByKey(string secretKey, bool isTrying = false)
{
var request = new SelectSecretRequest(secretKey, Environment.UserName, false);
var result = await this.http.PostAsJsonAsync("/secrets/delete", request, this.jsonRustSerializerOptions);
if (!result.IsSuccessStatusCode)
{
this.logger!.LogError($"Failed to delete the API key for secret ID '{secretId.SecretId}' due to an API issue: '{result.StatusCode}'");
this.logger!.LogError($"Failed to delete the API key for '{secretKey}' due to an API issue: '{result.StatusCode}'");
return new DeleteSecretResponse{Success = false, WasEntryFound = false, Issue = TB("Failed to delete the API key due to an API issue.")};
}
var state = await result.Content.ReadFromJsonAsync<DeleteSecretResponse>(this.jsonRustSerializerOptions);
if (!state.Success)
this.logger!.LogError($"Failed to delete the API key for secret ID '{secretId.SecretId}': '{state.Issue}'");
if (!state.Success && !isTrying)
this.logger!.LogError($"Failed to delete the API key for '{secretKey}': '{state.Issue}'");
return state;
}
@@ -17,7 +17,6 @@ public sealed class TemporaryChatService(ILogger<TemporaryChatService> logger, S
logger.LogInformation("The temporary chat maintenance service was initialized.");
await settingsManager.LoadSettings();
if(settingsManager.ConfigurationData.Workspace.StorageTemporaryMaintenancePolicy is WorkspaceStorageTemporaryMaintenancePolicy.NO_AUTOMATIC_MAINTENANCE)
{
logger.LogWarning("Automatic maintenance of temporary chat storage is disabled. Exiting maintenance service.");
@@ -23,6 +23,7 @@ public sealed class ProviderValidation
public Func<bool> IsModelProvidedManually { get; init; } = () => false;
public Func<string> GetCustomTokenizerValidationIssue { get; init; } = () => string.Empty;
public Func<bool> IsModelSelectionHidden { get; init; } = () => false;
public string? ValidatingHostname(string hostname)
{
@@ -78,9 +79,13 @@ public sealed class ProviderValidation
if (this.GetProvider() is LLMProviders.NONE)
return null;
// For self-hosted llama.cpp or whisper.cpp, no model selection needed
// For self-hosted whisper.cpp, no model selection needed
// (model is loaded at startup):
if (this.GetProvider() is LLMProviders.SELF_HOSTED && this.GetHost() is Host.LLAMA_CPP or Host.WHISPER_CPP)
if (this.GetProvider() is LLMProviders.SELF_HOSTED && this.GetHost() is Host.WHISPER_CPP)
return null;
// For legacy hosts without model selection, no selection validation is needed:
if (this.IsModelSelectionHidden())
return null;
// For manually entered models, this validation doesn't apply: