Added tool calling support (#731)

Co-authored-by: krut_ni <nils.kruthoff@dlr.de>
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor

@@ -47,6 +47,8 @@
     <ReadFileContent Text="@T("Load prompt from file")" @bind-FileContent="@this.freePrompt" EnableDragDrop="true" Layer="@DropLayers.ASSISTANTS" CatchAllDocuments="true" Disabled="@this.isProcessingBatch"/>
 
     <MudTextField T="string" @bind-Text="@this.freePrompt" Validation="@this.ValidateFreePrompt" Immediate="@true" Disabled="@this.isProcessingBatch" Label="@T("What should the AI do with each document?")" HelperText="@T("These instructions are applied to every single document of the batch run.")" Variant="Variant.Outlined" Margin="Margin.Normal" Lines="5" AutoGrow="@true" MaxLines="26" Class="mb-3" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
+
+    <ToolSelectionField Component="@this.Component" SelectedToolIds="@this.SelectedToolIds" SelectedToolIdsChanged="@this.SelectedToolIdsChanged" Disabled="@this.isProcessingBatch" Label="@T("Tools for this batch run")" Help="@T("The AI may use these tools while working on each document. Every tool has to meet the confidence requirements of the selected provider, so a tool may stay unavailable even when it is selected here.")"/>
 }
 else if (this.promptSource is BatchProcessingPromptSource.FILE_IMPORT)
 {
@@ -65,6 +67,8 @@ else if (this.promptSource is BatchProcessingPromptSource.FILE_IMPORT)
     <MudJustifiedText Typo="Typo.body2" Class="mb-3">
         @T("The content of the selected file is used as the instructions for every single document of the batch run.")
     </MudJustifiedText>
+
+    <ToolSelectionField Component="@this.Component" SelectedToolIds="@this.SelectedToolIds" SelectedToolIdsChanged="@this.SelectedToolIdsChanged" Disabled="@this.isProcessingBatch" Label="@T("Tools for this batch run")" Help="@T("The AI may use these tools while working on each document. Every tool has to meet the confidence requirements of the selected provider, so a tool may stay unavailable even when it is selected here.")"/>
 }
 else
 {
@@ -99,6 +103,12 @@ else
                 @this.selectedPolicy.PolicyDescription
             </MudJustifiedText>
         }
+
+        @* Read-only: the policy decides its tools, and this run follows the policy. *@
+        @if (this.selectedPolicy is not null)
+        {
+            <ToolSelectionField Component="@this.Component" SelectedToolIds="@this.PolicyToolIds" ReadOnly="@true" Label="@T("Tools of this policy")" Help="@T("These tools are part of the selected policy and cannot be changed here. Every tool has to meet the confidence requirements of the selected provider, so a tool may stay unavailable even when the policy permits it.")"/>
+        }
     }
 }
 
@@ -185,6 +195,15 @@ else
     </MudAlert>
 }
 
+@*
+    Only for a policy run: where the user picks the tools themselves, the selection field already
+    shows what is locked, and they can simply switch a blocked tool off.
+*@
+@if (this.promptSource is BatchProcessingPromptSource.POLICY)
+{
+    <ManagedToolsWarning Component="@this.Component" ToolIds="@this.PolicyToolIds" ProviderSettings="@this.ProviderSettings"/>
+}
+
 <ProviderSelection @bind-ProviderSettings="@this.ProviderSettings" ValidateProvider="@this.ValidatingProviderWithBatchState" Disabled="@this.isProcessingBatch" ExplicitMinimumConfidence="@this.GetMinimumConfidenceLevel()"/>
 
 @if (this.fileResults.Count > 0)

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Persistence.cs

@@ -106,9 +106,9 @@ public partial class AssistantBatchProcessing
     private async Task WriteLogAsync(string resolvedOutputDirectory)
     {
         var sb = new StringBuilder();
-        sb.AppendLine(CsvWriter.ToRow(LOG_SEPARATOR, T("File"), T("Time"), T("Model"), T("Status"), T("Details")));
+        sb.AppendLine(CsvWriter.ToRow(LOG_SEPARATOR, T("File"), T("Time"), T("Model"), T("Status"), T("Details"), T("Tools used")));
         foreach (var fileResult in this.fileResults.Where(x => x.Status is not BatchProcessingFileStatus.QUEUED and not BatchProcessingFileStatus.PROCESSING))
-            sb.AppendLine(CsvWriter.ToRow(LOG_SEPARATOR, fileResult.RelativePath, fileResult.ProcessedAt.ToString(TIME_FORMAT, CultureInfo.InvariantCulture), fileResult.ModelName, fileResult.Status.ToString(), fileResult.Message));
+            sb.AppendLine(CsvWriter.ToRow(LOG_SEPARATOR, fileResult.RelativePath, fileResult.ProcessedAt.ToString(TIME_FORMAT, CultureInfo.InvariantCulture), fileResult.ModelName, fileResult.Status.ToString(), fileResult.Message, fileResult.UsedTools));
 
         await this.WriteCsvFileAsync(Path.Join(resolvedOutputDirectory, LOG_FILENAME), sb.ToString());
     }
@@ -176,7 +176,9 @@ public partial class AssistantBatchProcessing
         try
         {
             var content = await File.ReadAllTextAsync(logFilePath);
-            var rows = BatchProcessingCsv.ParseWithDetectedSeparator(content, 5, LOG_SEPARATOR, '|');
+            // A log written before the tools column existed has five fields. It stays
+            // readable, so that a run started with an earlier version can be continued:
+            var rows = BatchProcessingCsv.ParseWithDetectedSeparator(content, [6, 5], LOG_SEPARATOR, '|');
 
             // The first row is the header, which we skip:
             foreach (var row in rows.Skip(1))
@@ -184,7 +186,7 @@ public partial class AssistantBatchProcessing
                 if (row.Count < 5 || string.IsNullOrWhiteSpace(row[0]))
                     continue;
 
-                entries[row[0]] = new BatchProcessingLogEntry(row[0], row[1], row[2], row[3], row[4]);
+                entries[row[0]] = new BatchProcessingLogEntry(row[0], row[1], row[2], row[3], row[4], row.Count > 5 ? row[5] : string.Empty);
             }
         }
         catch (Exception e)
@@ -213,7 +215,7 @@ public partial class AssistantBatchProcessing
 
             var content = await File.ReadAllTextAsync(resultsFilePath);
             var configuredSeparator = this.csvSeparator.Character(this.customCsvSeparator);
-            var rows = BatchProcessingCsv.ParseWithDetectedSeparator(content, 2, configuredSeparator, ';', '|', ',', '\t');
+            var rows = BatchProcessingCsv.ParseWithDetectedSeparator(content, [2], configuredSeparator, ';', '|', ',', '\t');
             foreach (var row in rows.Skip(1))
             {
                 if (row.Count < 2 || string.IsNullOrWhiteSpace(row[0]))

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Prompts.cs

@@ -1,6 +1,7 @@
 using AIStudio.Chat;
 using AIStudio.Provider;
 using AIStudio.Settings;
+using AIStudio.Tools.ToolCallingSystem;
 
 namespace AIStudio.Assistants.BatchProcessing;
 
@@ -86,18 +87,34 @@ public partial class AssistantBatchProcessing
                 """;
     }
 
-    private async Task<string> CallAIAsync(string fileName, string fileContent, CancellationToken token)
+    /// <param name="fileName">The name of the document being processed.</param>
+    /// <param name="fileContent">The content handed to the model.</param>
+    /// <param name="token">The cancellation token.</param>
+    /// <returns>The answer of the model, and which tools it used to get there.</returns>
+    private async Task<(string Answer, string UsedTools)> CallAIAsync(string fileName, string fileContent, CancellationToken token)
     {
+        //
+        // Every file of the batch gets the tools the user picked for the job. The batch builds its
+        // own throwaway thread per file instead of going through the assistant's own thread, so it
+        // has to hand the tools over itself.
+        //
         var chatThread = new ChatThread
         {
             IncludeDateTime = false,
             SelectedProvider = this.ProviderSettings.Id,
             SelectedProfile = Profile.NO_PROFILE.Id,
+            SelectedToolIds = [..this.SelectedToolIds],
             SystemPrompt = this.SystemPrompt,
             WorkspaceId = Guid.Empty,
             ChatId = Guid.NewGuid(),
             Name = this.Title,
             Blocks = [],
+            RuntimeComponent = this.Component,
+            RuntimeSelectedToolIds = this.GetRunnableToolIds(),
+
+            // Always true here, unlike in the assistant base: a batch run takes its tools from the
+            // selected policy or from its own field, never from the tool selection in the footer.
+            RuntimeToolsAreAssistantManaged = true,
         };
 
         var userPrompt = new ContentText
@@ -123,6 +140,32 @@ public partial class AssistantBatchProcessing
         });
 
         await aiText.CreateFromProviderAsync(this.ProviderSettings.CreateProvider(), this.ProviderSettings.Model, userPrompt, chatThread, token);
-        return aiText.Text.RemoveThinkTags().Trim();
+        return (aiText.Text.RemoveThinkTags().Trim(), this.SummarizeToolUsage(aiText));
+    }
+
+    /// <summary>
+    /// Sums up the tool calls of one document for the log.
+    /// </summary>
+    /// <remarks>
+    /// Names each tool once with how often it ran, because a model may search
+    /// several times for the same document. A call that failed or was blocked
+    /// is named with its outcome: for judging an answer it matters whether a
+    /// tool delivered or came back empty-handed.
+    /// </remarks>
+    private string SummarizeToolUsage(ContentText aiText) => string.Join(", ", aiText.ToolInvocations
+        .GroupBy(invocation => (invocation.ToolName, invocation.Status))
+        .OrderBy(group => group.Key.ToolName, StringComparer.OrdinalIgnoreCase)
+        .Select(group => this.FormatToolUsage(group.Key.ToolName, group.Key.Status, group.Count())));
+
+    private string FormatToolUsage(string toolName, ToolInvocationTraceStatus status, int count)
+    {
+        var nameWithCount = count > 1 ? $"{toolName} ({count}x)" : toolName;
+        return status switch
+        {
+            ToolInvocationTraceStatus.ERROR => $"{nameWithCount} [{this.T("failed")}]",
+            ToolInvocationTraceStatus.BLOCKED => $"{nameWithCount} [{this.T("blocked")}]",
+
+            _ => nameWithCount,
+        };
     }
 }

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Run.cs

@@ -70,6 +70,7 @@ public partial class AssistantBatchProcessing
                 fileResult.Status = BatchProcessingFileStatus.DONE;
                 fileResult.Message = logEntry.Details;
                 fileResult.ModelName = logEntry.Model;
+                fileResult.UsedTools = logEntry.UsedTools;
                 fileResult.ResultText = previousResults.GetValueOrDefault(relativePath, string.Empty);
 
                 if (DateTimeOffset.TryParseExact(logEntry.Time, TIME_FORMAT, CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var processedAt))
@@ -194,7 +195,7 @@ public partial class AssistantBatchProcessing
         string aiAnswer;
         try
         {
-            aiAnswer = await this.CallAIAsync(fileResult.FileName, fileContent, token);
+            (aiAnswer, fileResult.UsedTools) = await this.CallAIAsync(fileResult.FileName, fileContent, token);
         }
         catch (OperationCanceledException)
         {

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs

@@ -31,6 +31,24 @@ public partial class AssistantBatchProcessing : AssistantBaseCore<SettingsDialog
 
     protected override Tools.Components Component => Tools.Components.BATCH_PROCESSING_ASSISTANT;
 
+    /// <summary>
+    /// The tools a run uses, taken from wherever the instructions come from.
+    /// </summary>
+    /// <remarks>
+    /// Never from the footer: the tools belong to the instructions, and that is where they are
+    /// chosen. Working from a document analysis policy means following it, tools included, so
+    /// there is nothing left to pick. With instructions of one's own, the field next to them
+    /// decides.
+    /// </remarks>
+    protected override IReadOnlySet<string> AssistantManagedToolIds => this.promptSource is BatchProcessingPromptSource.POLICY
+        ? this.PolicyToolIds
+        : this.SelectedToolIds;
+
+    /// <summary>
+    /// The tools of the selected policy, or none while no policy is selected.
+    /// </summary>
+    private HashSet<string> PolicyToolIds => this.selectedPolicy is null ? [] : [..this.selectedPolicy.AllowedToolIds];
+
     protected override string Title => T("Batch Processing Assistant");
 
     protected override string Description => T("Process all documents and media files of a folder in one batch run: documents are converted to Markdown, while audio and video files are transcribed automatically, before their content is sent to the AI along with your instructions. You choose whether each answer is stored as its own Markdown file or whether all answers are collected in one CSV results table. A log records what happened to every file, so a run which was interrupted or produced errors can be continued later. A single failing file never stops the entire run.");

app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingCsv.cs

@@ -99,7 +99,12 @@ public static class BatchProcessingCsv
     /// content with it. Preferred separators are used as fallbacks for files
     /// whose first record does not reveal a valid separator.
     /// </summary>
-    public static List<List<string>> ParseWithDetectedSeparator(string content, int expectedNumFields, params char[] preferredSeparators)
+    /// <remarks>
+    /// Several accepted field counts allow a file written by an earlier version
+    /// to be read as well. The log gained a column, and a run started with the
+    /// previous version must still be continuable.
+    /// </remarks>
+    public static List<List<string>> ParseWithDetectedSeparator(string content, IReadOnlyList<int> acceptedNumFields, params char[] preferredSeparators)
     {
         var firstRecord = ReadFirstRecord(content);
         var candidates = new List<char>();
@@ -135,7 +140,7 @@ public static class BatchProcessingCsv
         foreach (var separator in candidates)
         {
             var header = Parse(firstRecord, separator);
-            if (header.Count is 1 && header[0].Count == expectedNumFields)
+            if (header.Count is 1 && acceptedNumFields.Contains(header[0].Count))
                 return Parse(content, separator);
         }
 

app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingFileResult.cs

@@ -56,4 +56,14 @@ public sealed class BatchProcessingFileResult
     /// The time when the processing of this file finished.
     /// </summary>
     public DateTimeOffset ProcessedAt { get; set; }
+
+    /// <summary>
+    /// The tools the model used for this file, ready to be read in the log.
+    /// </summary>
+    /// <remarks>
+    /// Recorded per file, because the model decides per document whether it
+    /// needs a tool at all. Without this, a batch run gives no clue why one
+    /// answer is better informed than the next.
+    /// </remarks>
+    public string UsedTools { get; set; } = string.Empty;
 }

app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingLogEntry.cs

@@ -3,7 +3,11 @@ namespace AIStudio.Assistants.BatchProcessing;
 /// <summary>
 /// One row of the log of a previous batch run.
 /// </summary>
-public sealed record BatchProcessingLogEntry(string RelativePath, string Time, string Model, string Status, string Details)
+/// <remarks>
+/// The tools column arrived later than the rest. A log written before it existed
+/// leaves it empty, which is also what a run without any tool call looks like.
+/// </remarks>
+public sealed record BatchProcessingLogEntry(string RelativePath, string Time, string Model, string Status, string Details, string UsedTools = "")
 {
     public bool WasSuccessful => string.Equals(this.Status, nameof(BatchProcessingFileStatus.DONE), StringComparison.OrdinalIgnoreCase);
 }