Added tool calling support (#731)

Co-authored-by: krut_ni <nils.kruthoff@dlr.de>
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Assistants/Builder/AssistantBuilder.razor

@@ -39,6 +39,7 @@
                                     @bind-ProfileId="@this.launcherProfileId"
                                     @bind-ChatTemplateId="@this.launcherChatTemplateId"
                                     @bind-DataSourceIds="@this.launcherDataSourceIds"
+                                    @bind-ToolIds="@this.launcherToolIds"
                                     ValidateWorkspaceName="@this.ValidateLauncherWorkspaceName"/>
         </MudPaper>
     }

app/MindWork AI Studio/Assistants/Builder/AssistantBuilder.razor.cs

@@ -6,6 +6,7 @@ using AIStudio.Tools.PluginSystem;
 using AIStudio.Tools.PluginSystem.Assistants;
 using AIStudio.Tools.PluginSystem.Assistants.DataModel;
 using AIStudio.Tools.Services;
+using AIStudio.Tools.ToolCallingSystem;
 using Microsoft.AspNetCore.Components;
 using DialogOptions = AIStudio.Dialogs.DialogOptions;
 
@@ -101,6 +102,7 @@ public partial class AssistantBuilder : AssistantBaseCore<NoSettingsPanel>
     private string launcherProfileId = string.Empty;
     private string launcherChatTemplateId = string.Empty;
     private IEnumerable<string> launcherDataSourceIds = [];
+    private HashSet<string> launcherToolIds = [];
     private IEnumerable<AssistantComponentType> selectedAssistantComponents = [];
     private CommonLanguages selectedOutputLanguage = CommonLanguages.AS_IS;
     private string customOutputLanguage = string.Empty;
@@ -138,6 +140,7 @@ public partial class AssistantBuilder : AssistantBaseCore<NoSettingsPanel>
     private static readonly AssistantSessionStateKey<string> LAUNCHER_PROFILE_ID_STATE_KEY = new(nameof(launcherProfileId));
     private static readonly AssistantSessionStateKey<string> LAUNCHER_CHAT_TEMPLATE_ID_STATE_KEY = new(nameof(launcherChatTemplateId));
     private static readonly AssistantSessionStateKey<List<string>> LAUNCHER_DATA_SOURCE_IDS_STATE_KEY = new(nameof(launcherDataSourceIds));
+    private static readonly AssistantSessionStateKey<HashSet<string>> LAUNCHER_TOOL_IDS_STATE_KEY = new(nameof(launcherToolIds));
     private static readonly AssistantSessionStateKey<List<AssistantComponentType>> SELECTED_ASSISTANT_COMPONENTS_STATE_KEY = new(nameof(selectedAssistantComponents));
     private static readonly AssistantSessionStateKey<CommonLanguages> SELECTED_OUTPUT_LANGUAGE_STATE_KEY = new(nameof(selectedOutputLanguage));
     private static readonly AssistantSessionStateKey<string> CUSTOM_OUTPUT_LANGUAGE_STATE_KEY = new(nameof(customOutputLanguage));
@@ -243,6 +246,7 @@ public partial class AssistantBuilder : AssistantBaseCore<NoSettingsPanel>
         this.launcherProfileId = string.Empty;
         this.launcherChatTemplateId = string.Empty;
         this.launcherDataSourceIds = [];
+        this.launcherToolIds = [];
         this.selectedAssistantComponents = [];
         this.selectedOutputLanguage = CommonLanguages.AS_IS;
         this.customOutputLanguage = string.Empty;
@@ -279,6 +283,7 @@ public partial class AssistantBuilder : AssistantBaseCore<NoSettingsPanel>
         state.Set(LAUNCHER_PROFILE_ID_STATE_KEY, this.launcherProfileId);
         state.Set(LAUNCHER_CHAT_TEMPLATE_ID_STATE_KEY, this.launcherChatTemplateId);
         state.SetList(LAUNCHER_DATA_SOURCE_IDS_STATE_KEY, this.launcherDataSourceIds);
+        state.SetHashSet(LAUNCHER_TOOL_IDS_STATE_KEY, this.launcherToolIds);
         state.SetList(SELECTED_ASSISTANT_COMPONENTS_STATE_KEY, this.selectedAssistantComponents);
         state.Set(SELECTED_OUTPUT_LANGUAGE_STATE_KEY, this.selectedOutputLanguage);
         state.Set(CUSTOM_OUTPUT_LANGUAGE_STATE_KEY, this.customOutputLanguage);
@@ -320,6 +325,7 @@ public partial class AssistantBuilder : AssistantBaseCore<NoSettingsPanel>
         state.Restore(LAUNCHER_PROFILE_ID_STATE_KEY, value => this.launcherProfileId = value);
         state.Restore(LAUNCHER_CHAT_TEMPLATE_ID_STATE_KEY, value => this.launcherChatTemplateId = value);
         state.Restore(LAUNCHER_DATA_SOURCE_IDS_STATE_KEY, value => this.launcherDataSourceIds = value);
+        state.Restore(LAUNCHER_TOOL_IDS_STATE_KEY, value => this.launcherToolIds = ToolSelectionRules.NormalizeSelection(value));
         state.Restore(SELECTED_ASSISTANT_COMPONENTS_STATE_KEY, value => this.selectedAssistantComponents = value);
         state.Restore(SELECTED_OUTPUT_LANGUAGE_STATE_KEY, value => this.selectedOutputLanguage = value);
         state.Restore(CUSTOM_OUTPUT_LANGUAGE_STATE_KEY, value => this.customOutputLanguage = value);
@@ -543,12 +549,14 @@ public partial class AssistantBuilder : AssistantBaseCore<NoSettingsPanel>
             return null;
 
         var dataSourceIds = this.launcherDataSourceIds.Distinct(StringComparer.OrdinalIgnoreCase).ToArray();
+        var toolIds = ToolSelectionRules.NormalizeSelection(this.launcherToolIds).ToArray();
         return new(
             this.launcherWorkspaceName.Trim(),
             NullIfEmpty(this.launcherProviderId),
             NullIfEmpty(this.launcherProfileId),
             NullIfEmpty(this.launcherChatTemplateId),
-            dataSourceIds.Length == 0 ? null : dataSourceIds);
+            dataSourceIds.Length == 0 ? null : dataSourceIds,
+            toolIds.Length == 0 ? null : toolIds);
     }
 
     private void CreateChatLauncherChanged(bool createLauncher)

app/MindWork AI Studio/Assistants/Builder/AssistantBuilderAssistantMetadata.cs

@@ -8,5 +8,6 @@ internal sealed class AssistantBuilderAssistantMetadata
     public string? SystemPrompt { get; init; }
     public string? SubmitText { get; init; }
     public bool? AllowAiStudioProfiles { get; init; }
+    public string[]? ToolIds { get; init; }
     public AssistantBuilderChatLaunchMetadata? Launch { get; init; }
 }

app/MindWork AI Studio/Assistants/Builder/AssistantBuilderChatLaunchMetadata.cs

@@ -7,4 +7,5 @@ internal sealed class AssistantBuilderChatLaunchMetadata
     public string? ProfileId { get; init; }
     public string? ChatTemplateId { get; init; }
     public string[]? DataSourceIds { get; init; }
+    public string[]? ToolIds { get; init; }
 }

app/MindWork AI Studio/Assistants/Builder/AssistantBuilderLuaResponse.schema.json

@@ -90,6 +90,15 @@
         },
         "allow_ai_studio_profiles": {
           "type": "boolean"
+        },
+        "tool_ids": {
+          "type": "array",
+          "minItems": 1,
+          "uniqueItems": true,
+          "items": {
+            "type": "string",
+            "minLength": 1
+          }
         }
       }
     },
@@ -156,6 +165,15 @@
               "const": "00000000-0000-0000-0000-000000000000"
             }
           }
+        },
+        "tool_ids": {
+          "type": "array",
+          "minItems": 1,
+          "uniqueItems": true,
+          "items": {
+            "type": "string",
+            "minLength": 1
+          }
         }
       }
     }

app/MindWork AI Studio/Assistants/Builder/LauncherTextsResponse.cs

@@ -0,0 +1,90 @@
+using System.Text.Json;
+
+namespace AIStudio.Assistants.Builder;
+
+/// <summary>
+/// The three texts a model writes for a direct chat launcher.
+/// </summary>
+/// <remarks>
+/// A launcher has no system prompt, no form, and no prompt builder, and its chat settings come
+/// straight from the Builder form. That leaves nothing for a model to write except the names a
+/// person reads, so it is asked for those alone and AI Studio writes the plugin.lua itself.
+/// </remarks>
+internal sealed class LauncherTextsResponse
+{
+    public const string SCHEMA_VERSION_VALUE = "assistant_builder_launcher_texts_v1";
+
+    private static readonly JsonSerializerOptions JSON_OPTIONS = new()
+    {
+        PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
+        AllowTrailingCommas = false,
+        ReadCommentHandling = JsonCommentHandling.Disallow,
+        MaxDepth = 8,
+    };
+
+    public string SchemaVersion { get; init; } = string.Empty;
+
+    /// <summary>
+    /// The plugin name, shown on the plugins page.
+    /// </summary>
+    public string PluginName { get; init; } = string.Empty;
+
+    /// <summary>
+    /// The title on the tile.
+    /// </summary>
+    public string Title { get; init; } = string.Empty;
+
+    /// <summary>
+    /// The short description, used for both the plugin and the tile.
+    /// </summary>
+    public string Description { get; init; } = string.Empty;
+
+    public static bool TryParse(string modelResponse, out LauncherTextsResponse response, out LuaResponseParseError error, out string technicalDetails)
+    {
+        response = new();
+        error = LuaResponseParseError.NONE;
+        technicalDetails = string.Empty;
+
+        var json = LuaResponse.ExtractJson(modelResponse);
+        if (string.IsNullOrWhiteSpace(json))
+        {
+            error = LuaResponseParseError.MISSING_JSON_OBJECT;
+            return false;
+        }
+
+        LauncherTextsResponse? parsed;
+        try
+        {
+            parsed = JsonSerializer.Deserialize<LauncherTextsResponse>(json, JSON_OPTIONS);
+        }
+        catch (JsonException e)
+        {
+            error = LuaResponseParseError.INVALID_JSON;
+            technicalDetails = e.Message;
+            return false;
+        }
+
+        if (parsed is null)
+        {
+            error = LuaResponseParseError.EMPTY_JSON_OBJECT;
+            return false;
+        }
+
+        if (!string.Equals(parsed.SchemaVersion, SCHEMA_VERSION_VALUE, StringComparison.Ordinal))
+        {
+            error = LuaResponseParseError.UNSUPPORTED_SCHEMA_VERSION;
+            return false;
+        }
+
+        if (string.IsNullOrWhiteSpace(parsed.PluginName) ||
+            string.IsNullOrWhiteSpace(parsed.Title) ||
+            string.IsNullOrWhiteSpace(parsed.Description))
+        {
+            error = LuaResponseParseError.INCOMPLETE_ASSISTANT_METADATA;
+            return false;
+        }
+
+        response = parsed;
+        return true;
+    }
+}

app/MindWork AI Studio/Assistants/Builder/LuaResponse.Parse.cs

@@ -109,10 +109,15 @@ internal sealed partial class LuaResponse
         "FORM" => !string.IsNullOrWhiteSpace(assistant.SystemPrompt) &&
                   !string.IsNullOrWhiteSpace(assistant.SubmitText) &&
                   assistant.AllowAiStudioProfiles.HasValue &&
+                  IsValidToolIds(assistant.ToolIds) &&
                   assistant.Launch is null,
+
+        // A launcher names its tools inside launch, so the same field one level up would be a
+        // second, competing selection:
         "CHAT_LAUNCHER" => assistant.SystemPrompt is null &&
                            assistant.SubmitText is null &&
                            assistant.AllowAiStudioProfiles is null &&
+                           assistant.ToolIds is null &&
                            IsValidChatLaunchMetadata(assistant.Launch),
         _ => false,
     };
@@ -127,16 +132,36 @@ internal sealed partial class LuaResponse
             !IsOptionalGuid(launch.ChatTemplateId, allowEmpty: true))
             return false;
 
-        return launch.DataSourceIds is null ||
-               launch.DataSourceIds.Length > 0 &&
-               launch.DataSourceIds.All(id => Guid.TryParse(id, out var parsed) && parsed != Guid.Empty) &&
-               launch.DataSourceIds.Distinct(StringComparer.OrdinalIgnoreCase).Count() == launch.DataSourceIds.Length;
+        if (launch.DataSourceIds is not null &&
+            (launch.DataSourceIds.Length == 0 ||
+             !launch.DataSourceIds.All(id => Guid.TryParse(id, out var parsed) && parsed != Guid.Empty) ||
+             launch.DataSourceIds.Distinct(StringComparer.OrdinalIgnoreCase).Count() != launch.DataSourceIds.Length))
+            return false;
+
+        return IsValidToolIds(launch.ToolIds);
     }
 
+    /// <remarks>
+    /// Tool IDs are plain names, so only their shape can be checked here. Whether the named tools
+    /// exist is decided later, against the tools this AI Studio actually has.
+    /// </remarks>
+    private static bool IsValidToolIds(string[]? toolIds) =>
+        toolIds is null ||
+        toolIds.Length > 0 &&
+        toolIds.All(id => !string.IsNullOrWhiteSpace(id)) &&
+        toolIds.Distinct(StringComparer.Ordinal).Count() == toolIds.Length;
+
     private static bool IsOptionalGuid(string? value, bool allowEmpty) => value is null ||
         Guid.TryParse(value, out var parsed) && (allowEmpty || parsed != Guid.Empty);
 
-    private static string ExtractJson(string input)
+    /// <summary>
+    /// Reads the first complete JSON object out of a model answer that may carry text around it.
+    /// </summary>
+    /// <remarks>
+    /// Shared with the launcher texts response, which is a different shape but arrives the same
+    /// way, wrapped in whatever prose the model felt like adding.
+    /// </remarks>
+    internal static string ExtractJson(string input)
     {
         var start = input.IndexOf('{');
         if (start < 0)

app/MindWork AI Studio/Assistants/Builder/LuaResponseParseError.cs

@@ -13,26 +13,4 @@ public enum LuaResponseParseError
     INCOMPLETE_ASSISTANT_METADATA,
     MISSING_LUA,
     LUA_MISSING_ID,
-}
-
-public static class LuaResponseParseErrorExtension
-{
-    private static string TB(string fallbackEN) => Tools.PluginSystem.I18N.I.T(fallbackEN, typeof(LuaResponseParseErrorExtension).Namespace, nameof(LuaResponseParseErrorExtension));
-
-    public static string GetMessage(this LuaResponseParseError parseError, string technicalDetails) => parseError switch
-    {
-        LuaResponseParseError.MISSING_JSON_OBJECT => TB("The model response is missing or unreadable."),
-        LuaResponseParseError.INVALID_JSON => string.IsNullOrWhiteSpace(technicalDetails)
-            ? TB("The model returned an invalid response.")
-            : string.Format(TB("The model returned an invalid response: {0}"), technicalDetails),
-        LuaResponseParseError.EMPTY_JSON_OBJECT => TB("The model returned an empty JSON object."),
-        LuaResponseParseError.UNSUPPORTED_SCHEMA_VERSION => TB("The model responded with an unsupported or deprecated JSON schema."),
-        LuaResponseParseError.MISSING_PLUGIN_METADATA => TB("The model's answer is missing the plugin metadata."),
-        LuaResponseParseError.MISSING_ASSISTANT_METADATA => TB("The model's answer is missing the assistant metadata."),
-        LuaResponseParseError.INCOMPLETE_PLUGIN_METADATA => TB("The model's answer contains incomplete plugin metadata."),
-        LuaResponseParseError.INCOMPLETE_ASSISTANT_METADATA => TB("The model's answer contains incomplete assistant metadata."),
-        LuaResponseParseError.MISSING_LUA => TB("The model response does not contain the generated Lua plugin code."),
-        LuaResponseParseError.LUA_MISSING_ID => TB("The generated Lua plugin code does not contain a readable plugin ID."),
-        _ => TB("The model returned an unusable JSON response."),
-    };
-}
+}

app/MindWork AI Studio/Assistants/Builder/LuaResponseParseErrorExtension.cs

@@ -0,0 +1,23 @@
+namespace AIStudio.Assistants.Builder;
+
+public static class LuaResponseParseErrorExtension
+{
+    private static string TB(string fallbackEN) => Tools.PluginSystem.I18N.I.T(fallbackEN, typeof(LuaResponseParseError).Namespace, nameof(LuaResponseParseError));
+
+    public static string GetMessage(this LuaResponseParseError parseError, string technicalDetails) => parseError switch
+    {
+        LuaResponseParseError.MISSING_JSON_OBJECT => TB("The model response is missing or unreadable."),
+        LuaResponseParseError.INVALID_JSON => string.IsNullOrWhiteSpace(technicalDetails)
+            ? TB("The model returned an invalid response.")
+            : string.Format(TB("The model returned an invalid response: {0}"), technicalDetails),
+        LuaResponseParseError.EMPTY_JSON_OBJECT => TB("The model returned an empty JSON object."),
+        LuaResponseParseError.UNSUPPORTED_SCHEMA_VERSION => TB("The model responded with an unsupported or deprecated JSON schema."),
+        LuaResponseParseError.MISSING_PLUGIN_METADATA => TB("The model's answer is missing the plugin metadata."),
+        LuaResponseParseError.MISSING_ASSISTANT_METADATA => TB("The model's answer is missing the assistant metadata."),
+        LuaResponseParseError.INCOMPLETE_PLUGIN_METADATA => TB("The model's answer contains incomplete plugin metadata."),
+        LuaResponseParseError.INCOMPLETE_ASSISTANT_METADATA => TB("The model's answer contains incomplete assistant metadata."),
+        LuaResponseParseError.MISSING_LUA => TB("The model response does not contain the generated Lua plugin code."),
+        LuaResponseParseError.LUA_MISSING_ID => TB("The generated Lua plugin code does not contain a readable plugin ID."),
+        _ => TB("The model returned an unusable JSON response."),
+    };
+}