Added tool calling support (#731)

Co-authored-by: krut_ni <nils.kruthoff@dlr.de>
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Settings/ConfigurationSelectData.cs

@@ -0,0 +1,9 @@
+namespace AIStudio.Settings;
+
+/// <summary>
+/// A data structure to map a name to a value.
+/// </summary>
+/// <param name="Name">The name of the value, to be displayed in the UI.</param>
+/// <param name="Value">The value to be stored.</param>
+/// <typeparam name="T">The type of the value to store.</typeparam>
+public readonly record struct ConfigurationSelectData<T>(string Name, T Value);

app/MindWork AI Studio/Settings/ConfigurationSelectDataFactory.cs

@@ -15,14 +15,6 @@ using WritingStylesEMail = AIStudio.Assistants.EMail.WritingStyles;
 
 namespace AIStudio.Settings;
 
-/// <summary>
-/// A data structure to map a name to a value.
-/// </summary>
-/// <param name="Name">The name of the value, to be displayed in the UI.</param>
-/// <param name="Value">The value to be stored.</param>
-/// <typeparam name="T">The type of the value to store.</typeparam>
-public readonly record struct ConfigurationSelectData<T>(string Name, T Value);
-
 /// <summary>
 /// A static factory class to get the lists of selectable values.
 /// </summary>

app/MindWork AI Studio/Settings/DataModel/Data.cs

@@ -194,4 +194,6 @@ public sealed class Data
     public DataBiasOfTheDay BiasOfTheDay { get; init; } = new();
     
     public DataI18N I18N { get; init; } = new();
-}   
+
+    public DataTools Tools { get; init; } = new(x => x.Tools);
+}

app/MindWork AI Studio/Settings/DataModel/DataDocumentAnalysisPolicy.cs

@@ -57,6 +57,19 @@ public sealed record DataDocumentAnalysisPolicy : ConfigurationBaseObject
     /// The minimum confidence level required for a provider to be considered.
     /// </summary>
     public ConfidenceLevel MinimumProviderConfidence { get; set; } = ConfidenceLevel.NONE;
+
+    /// <summary>
+    /// The tools this policy permits the model to use.
+    /// </summary>
+    /// <remarks>
+    /// A limit, not a preselection: a tool absent from this list cannot be chosen for a run of this
+    /// policy. Empty therefore means no tools at all, which is what a policy written before this
+    /// field existed gets — an analysis keeps working exactly as its author wrote it.<br/><br/>
+    /// This narrows what the user may pick; it never widens what a tool is allowed to do. Every
+    /// permitted tool still has to pass the provider confidence checks, so a tool demanding High
+    /// confidence stays out of reach of a weaker provider whether a policy lists it or not.
+    /// </remarks>
+    public HashSet<string> AllowedToolIds { get; set; } = [];
     
     /// <summary>
     /// Which LLM provider should be preselected?
@@ -130,6 +143,23 @@ public sealed record DataDocumentAnalysisPolicy : ConfigurationBaseObject
         if (table.TryGetValue("HidePolicyDefinition", out var hideValue) && hideValue.TryRead<bool>(out var hide))
             hidePolicyDefinition = hide;
 
+        //
+        // Unknown tool IDs are kept rather than rejected: an organization may roll out a policy
+        // before the plugin providing that tool reaches every workstation. A tool that does not
+        // exist simply never shows up, and the policy starts working once it does.
+        //
+        var allowedToolIds = new HashSet<string>(StringComparer.Ordinal);
+        if (table.TryGetValue("AllowedToolIds", out var toolIdsValue) && toolIdsValue.TryRead<LuaTable>(out var toolIdsTable))
+        {
+            for (var toolIdx = 1; toolIdx <= toolIdsTable.ArrayLength; toolIdx++)
+            {
+                if (toolIdsTable[toolIdx].TryRead<string>(out var toolId) && !string.IsNullOrWhiteSpace(toolId))
+                    allowedToolIds.Add(toolId.Trim());
+                else
+                    LOG.LogWarning("The configured document analysis policy {PolicyIndex} contains an invalid entry in its AllowedToolIds list.", idx);
+            }
+        }
+
         policy = new DataDocumentAnalysisPolicy
         {
             Id = id.ToString(),
@@ -139,6 +169,7 @@ public sealed record DataDocumentAnalysisPolicy : ConfigurationBaseObject
             AnalysisRules = analysisRules,
             OutputRules = outputRules,
             MinimumProviderConfidence = minimumConfidence,
+            AllowedToolIds = allowedToolIds,
             PreselectedProvider = preselectedProvider,
             PreselectedProfile = preselectedProfile,
             HidePolicyDefinition = hidePolicyDefinition,

app/MindWork AI Studio/Settings/DataModel/DataTools.cs

@@ -0,0 +1,64 @@
+using System.Linq.Expressions;
+
+namespace AIStudio.Settings.DataModel;
+
+public sealed class DataTools(Expression<Func<Data, DataTools>>? configSelection = null)
+{
+    public DataTools() : this(null)
+    {
+    }
+
+    /// <summary>
+    /// The settings the user entered per tool: tool ID, then field name.
+    /// </summary>
+    public Dictionary<string, Dictionary<string, string>> Settings { get; set; } = [];
+
+    public Dictionary<string, HashSet<string>> DefaultToolIdsByComponent { get; set; } = [];
+
+    public HashSet<string> VisibleToolSelectionComponents { get; set; } = [];
+
+    public bool EnableTools { get; set; } = ManagedConfiguration.Register(
+        configSelection,
+        x => x.EnableTools,
+        true);
+
+    public HashSet<string> DisabledToolIds { get; set; } = ManagedConfiguration.Register(
+        configSelection,
+        x => x.DisabledToolIds,
+        []);
+
+    public Dictionary<string, string> MinimumProviderConfidenceByToolId { get; set; } = ManagedConfiguration.Register(
+        configSelection,
+        x => x.MinimumProviderConfidenceByToolId,
+        new Dictionary<string, string>(StringComparer.Ordinal));
+
+    /// <summary>
+    /// Tool settings an organization fixed, which the user cannot change. Keys are
+    /// "toolId.fieldName".
+    /// </summary>
+    /// <remarks>
+    /// Keyed by tool and field rather than held in a property per setting, because a property per
+    /// setting only works for the tools AI Studio ships. Tools defined by plugin authors are not
+    /// known at compile time, yet an organization has to be able to configure them the same way.
+    /// <br/><br/>
+    /// Secrets never travel this way. They belong in the operating system's keyring, which a
+    /// configuration file cannot reach.
+    /// </remarks>
+    public Dictionary<string, string> LockedToolSettings { get; set; } = ManagedConfiguration.Register(
+        configSelection,
+        x => x.LockedToolSettings,
+        new Dictionary<string, string>(StringComparer.Ordinal));
+
+    /// <summary>
+    /// Tool settings an organization pre-filled but left changeable. Keys are "toolId.fieldName".
+    /// </summary>
+    /// <remarks>
+    /// Applies until the user saves a value of their own, which then wins. That is the difference
+    /// to the locked settings above, and the reason both exist: an organization can fix the search
+    /// instance while leaving the timeouts to the user.
+    /// </remarks>
+    public Dictionary<string, string> DefaultToolSettings { get; set; } = ManagedConfiguration.Register(
+        configSelection,
+        x => x.DefaultToolSettings,
+        new Dictionary<string, string>(StringComparer.Ordinal));
+}

app/MindWork AI Studio/Settings/DataSourceSecurityTrustExtensions.cs

@@ -42,6 +42,8 @@ public static class DataSourceSecurityTrustExtensions
 
     public static bool IsTrustedByConfiguration(this TranscriptionProvider provider, SettingsManager settingsManager) => IsTrustedProviderId(provider.Id, settingsManager);
 
+    public static bool IsTrustedByConfiguration(this IProvider provider, SettingsManager settingsManager) => IsTrustedProviderId(provider.ConfiguredProviderId, settingsManager);
+
     private static bool IsTrustedProviderId(string providerId, SettingsManager settingsManager)
     {
         if (string.IsNullOrWhiteSpace(providerId))
@@ -49,4 +51,4 @@ public static class DataSourceSecurityTrustExtensions
 
         return settingsManager.ConfigurationData.DataSourceSecurity.TrustedProviderIds.Any(id => string.Equals(id, providerId, StringComparison.OrdinalIgnoreCase));
     }
-}
+}

app/MindWork AI Studio/Settings/ManagedConfiguration.Parsing.cs

@@ -775,7 +775,7 @@ public static partial class ManagedConfiguration
             return false;
 
         var successful = false;
-        var configuredValue = configMeta.Default;
+        var configuredValue = CloneStringDictionary(configMeta.Default);
 
         // Step 1 -- try to read the Lua value (we expect a table) out of the Lua table:
         if (settings.TryGetValue(SettingsManager.ToSettingName(propertyExpression), out var configuredLuaList) &&
@@ -812,7 +812,9 @@ public static partial class ManagedConfiguration
         if(dryRun)
             return successful;
 
-        return HandleParsedValue(configPluginId, dryRun, successful, configMeta, configuredValue);
+        var settingName = SettingName(propertyExpression);
+        var managedMode = ReadManagedConfigurationMode(propertyExpression, settings);
+        return HandleParsedDictionaryValue(configPluginId, dryRun, successful, configMeta, configuredValue, managedMode, settingName);
     }
 
     /// <summary>
@@ -1036,6 +1038,67 @@ public static partial class ManagedConfiguration
         return successful;
     }
 
+    private static bool HandleParsedDictionaryValue<TClass>(
+        Guid configPluginId,
+        bool dryRun,
+        bool successful,
+        ConfigMeta<TClass, IDictionary<string, string>> configMeta,
+        IDictionary<string, string> configuredValue,
+        ManagedConfigurationMode managedMode,
+        string settingName)
+    {
+        if (dryRun)
+            return successful;
+
+        switch (successful)
+        {
+            case true when managedMode is ManagedConfigurationMode.LOCKED:
+                ClearEditableDefaultState(settingName);
+                configMeta.ClearEditableDefaultConfiguration();
+                configMeta.SetValue(CloneStringDictionary(configuredValue));
+                configMeta.LockConfiguration(configPluginId);
+                break;
+
+            case true when managedMode is ManagedConfigurationMode.EDITABLE_DEFAULT:
+                var currentValueSerialized = SerializeManagedStringDictionaryValue(configMeta.GetValue());
+                var configuredValueSerialized = SerializeManagedStringDictionaryValue(configuredValue);
+
+                string lastAppliedValue;
+                if (!TryGetEditableDefaultState(settingName, out var editableDefaultState))
+                {
+                    configMeta.SetValue(CloneStringDictionary(configuredValue));
+                    lastAppliedValue = configuredValueSerialized;
+                }
+                else
+                {
+                    lastAppliedValue = editableDefaultState.LastAppliedValue;
+                    if (string.Equals(currentValueSerialized, lastAppliedValue, StringComparison.Ordinal))
+                    {
+                        configMeta.SetValue(CloneStringDictionary(configuredValue));
+                        lastAppliedValue = configuredValueSerialized;
+                    }
+                }
+
+                SetEditableDefaultState(settingName, configPluginId, lastAppliedValue);
+                configMeta.UnlockConfiguration();
+                configMeta.SetEditableDefaultConfiguration(configPluginId);
+                break;
+
+            case false when configMeta.IsLocked && configMeta.LockedByConfigPluginId == configPluginId:
+                configMeta.ResetLockedConfiguration();
+                break;
+
+            case false when configMeta.ManagedMode is ManagedConfigurationMode.EDITABLE_DEFAULT
+                            && TryGetEditableDefaultState(settingName, out var editableDefaultStateToRemove)
+                            && editableDefaultStateToRemove.ConfigPluginId == configPluginId:
+                configMeta.ClearEditableDefaultConfiguration();
+                ClearEditableDefaultState(settingName);
+                break;
+        }
+
+        return successful;
+    }
+
     private static ManagedConfigurationMode ReadManagedConfigurationMode<TClass, TValue>(
         Expression<Func<TClass, TValue>> propertyExpression,
         LuaTable settings)
@@ -1069,4 +1132,12 @@ public static partial class ManagedConfiguration
 
         _ => value.ToString() ?? string.Empty,
     };
-}
+
+    private static Dictionary<string, string> CloneStringDictionary(IDictionary<string, string> values) => new(values, StringComparer.Ordinal);
+
+    private static string SerializeManagedStringDictionaryValue(IDictionary<string, string> values) => string.Join(
+        "\n",
+        values
+            .OrderBy(pair => pair.Key, StringComparer.Ordinal)
+            .Select(pair => $"{pair.Key}={pair.Value}"));
+}

app/MindWork AI Studio/Settings/ProviderCapabilityOverrides.cs

@@ -18,6 +18,7 @@ public sealed record ProviderCapabilityOverrides
     private static readonly IReadOnlyList<Capability> SUPPORTED_CAPABILITIES =
     [
         Capability.AUDIO_INPUT,
+        Capability.FUNCTION_CALLING,
         Capability.MULTIPLE_IMAGE_INPUT,
         Capability.SPEECH_INPUT,
         Capability.VIDEO_INPUT,
@@ -30,6 +31,10 @@ public sealed record ProviderCapabilityOverrides
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public bool? AudioInput { get; init; }
 
+    [JsonPropertyName("FUNCTION_CALLING")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public bool? FunctionCalling { get; init; }
+
     [JsonPropertyName("MULTIPLE_IMAGE_INPUT")]
     [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
     public bool? MultipleImageInput { get; init; }
@@ -57,6 +62,7 @@ public sealed record ProviderCapabilityOverrides
     [JsonIgnore]
     public bool HasOverrides =>
         this.AudioInput is not null ||
+        this.FunctionCalling is not null ||
         this.MultipleImageInput is not null ||
         this.SpeechInput is not null ||
         this.VideoInput is not null ||
@@ -67,6 +73,7 @@ public sealed record ProviderCapabilityOverrides
     public bool? GetOverride(Capability capability) => capability switch
     {
         Capability.AUDIO_INPUT => this.AudioInput,
+        Capability.FUNCTION_CALLING => this.FunctionCalling,
         Capability.MULTIPLE_IMAGE_INPUT => this.MultipleImageInput,
         Capability.SPEECH_INPUT => this.SpeechInput,
         Capability.VIDEO_INPUT => this.VideoInput,
@@ -79,6 +86,7 @@ public sealed record ProviderCapabilityOverrides
     public ProviderCapabilityOverrides SetOverride(Capability capability, bool? value) => capability switch
     {
         Capability.AUDIO_INPUT => this with { AudioInput = value },
+        Capability.FUNCTION_CALLING => this with { FunctionCalling = value },
         Capability.MULTIPLE_IMAGE_INPUT => this with { MultipleImageInput = value },
         Capability.SPEECH_INPUT => this with { SpeechInput = value },
         Capability.VIDEO_INPUT => this with { VideoInput = value },

app/MindWork AI Studio/Settings/SettingsManager.cs

@@ -3,6 +3,7 @@ using System.Text.Json;
 
 using AIStudio.Provider;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ToolCallingSystem;
 using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.Services;
 
@@ -15,6 +16,8 @@ namespace AIStudio.Settings;
 /// </summary>
 public sealed class SettingsManager
 {
+    public readonly record struct ToolMinimumProviderConfidenceResolution(ConfidenceLevel ConfidenceLevel, string Source);
+
     private const string SETTINGS_FILENAME = "settings.json";
     private const Version CURRENT_SETTINGS_VERSION = Version.V6;
     
@@ -784,6 +787,112 @@ public sealed class SettingsManager
         return this.ConfigurationData.ChatTemplates.FirstOrDefault(x => x.Id.Equals(chatTemplateId, StringComparison.OrdinalIgnoreCase)) ?? ChatTemplate.NO_CHAT_TEMPLATE;
     }
 
+    public HashSet<string> GetDefaultToolIds(AIStudio.Tools.Components component)
+    {
+        var key = component.ToString();
+        if (this.ConfigurationData.Tools.DefaultToolIdsByComponent.TryGetValue(key, out var toolIds))
+            return ToolSelectionRules.NormalizeSelection(toolIds);
+
+        return [];
+    }
+
+
+    public bool AreToolsEnabled() => this.ConfigurationData.Tools.EnableTools;
+
+    public bool IsToolActive(string toolId) =>
+        this.AreToolsEnabled() &&
+        !this.ConfigurationData.Tools.DisabledToolIds.Contains(toolId);
+
+    /// <remarks>
+    /// The document analysis is deliberately absent: there its policy names the tools, so the user
+    /// has nothing to select.
+    /// </remarks>
+    public bool IsToolSelectionVisible(AIStudio.Tools.Components component) => component switch
+    {
+        AIStudio.Tools.Components.CHAT or
+        AIStudio.Tools.Components.CODING_ASSISTANT or
+        AIStudio.Tools.Components.SLIDE_BUILDER_ASSISTANT => true,
+        _ => this.ConfigurationData.Tools.VisibleToolSelectionComponents.Contains(component.ToString()),
+    };
+
+    public void SetToolSelectionVisibility(AIStudio.Tools.Components component, bool isVisible)
+    {
+        if (component is
+            AIStudio.Tools.Components.CHAT or
+            AIStudio.Tools.Components.CODING_ASSISTANT or
+            AIStudio.Tools.Components.SLIDE_BUILDER_ASSISTANT)
+            return;
+
+        var key = component.ToString();
+        if (isVisible)
+            this.ConfigurationData.Tools.VisibleToolSelectionComponents.Add(key);
+        else
+            this.ConfigurationData.Tools.VisibleToolSelectionComponents.Remove(key);
+    }
+
+    /// <summary>
+    /// Resolves which provider confidence a tool needs, and where that value came from.
+    /// </summary>
+    /// <remarks>
+    /// The default is passed in rather than looked up here. It belongs to the tool definition,
+    /// and the definitions live in the tool registry — which already depends on this class, so
+    /// asking it back would be a circle. Every caller has the definition at hand anyway.
+    /// </remarks>
+    /// <param name="toolId">The tool to resolve the confidence for.</param>
+    /// <param name="defaultLevel">The tool's own minimum, used when nothing overrides it.</param>
+    public ToolMinimumProviderConfidenceResolution GetMinimumProviderConfidenceResolutionForTool(string toolId, ConfidenceLevel defaultLevel)
+    {
+        if (ManagedConfiguration.TryGet(x => x.Tools, x => x.MinimumProviderConfidenceByToolId, out var configMeta) && configMeta.IsLocked)
+        {
+            var managedValues = configMeta.GetValue();
+            if (managedValues.TryGetValue(toolId, out var configuredManagedLevel) &&
+                Enum.TryParse<ConfidenceLevel>(configuredManagedLevel, true, out var managedConfidenceLevel) &&
+                Enum.IsDefined(managedConfidenceLevel) &&
+                managedConfidenceLevel is not ConfidenceLevel.UNKNOWN)
+            {
+                return new(managedConfidenceLevel, "managed config");
+            }
+
+            if (managedValues.ContainsKey(toolId))
+            {
+                this.logger.LogError(
+                    "Managed minimum provider confidence '{ConfiguredLevel}' for tool '{ToolId}' is invalid. Requiring HIGH as a safe fallback.",
+                    configuredManagedLevel,
+                    toolId);
+                return new(ConfidenceLevel.HIGH, "invalid managed config; safe fallback");
+            }
+        }
+
+        if (this.ConfigurationData.Tools.MinimumProviderConfidenceByToolId.TryGetValue(toolId, out var configuredLevel) &&
+            Enum.TryParse<ConfidenceLevel>(configuredLevel, true, out var confidenceLevel) &&
+            Enum.IsDefined(confidenceLevel) &&
+            confidenceLevel is not ConfidenceLevel.UNKNOWN)
+        {
+            return new(confidenceLevel, "stored override");
+        }
+
+        return new(defaultLevel, "default fallback");
+    }
+
+    public ConfidenceLevel GetMinimumProviderConfidenceForTool(string toolId, ConfidenceLevel defaultLevel) => this.GetMinimumProviderConfidenceResolutionForTool(toolId, defaultLevel).ConfidenceLevel;
+
+    /// <summary>
+    /// Stores which provider confidence a tool needs.
+    /// </summary>
+    /// <param name="toolId">The tool to store the confidence for.</param>
+    /// <param name="confidenceLevel">The level the user chose.</param>
+    /// <param name="defaultLevel">The tool's own minimum. Choosing it again removes the override.</param>
+    public void SetMinimumProviderConfidenceForTool(string toolId, ConfidenceLevel confidenceLevel, ConfidenceLevel defaultLevel)
+    {
+        if (confidenceLevel == defaultLevel)
+        {
+            this.ConfigurationData.Tools.MinimumProviderConfidenceByToolId.Remove(toolId);
+            return;
+        }
+
+        this.ConfigurationData.Tools.MinimumProviderConfidenceByToolId[toolId] = confidenceLevel.ToString();
+    }
+
     public ConfidenceLevel GetConfiguredConfidenceLevel(LLMProviders llmProvider)
     {
         if(llmProvider is LLMProviders.NONE)
@@ -879,4 +988,4 @@ public sealed class SettingsManager
         // Return the full name of the property, including the class name:
         return $"{typeof(TIn).Name}.{memberExpr.Member.Name}";
     }
-}
+}