Rework of the file export (#894)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor

@@ -115,10 +115,19 @@ else
     }
 </MudSelect>
 
-@if (this.outputMode is BatchProcessingOutputMode.MARKDOWN_FILES)
+@if (this.outputMode is BatchProcessingOutputMode.INDIVIDUAL_FILES)
 {
+    <MudSelect T="FileExportFormat" @bind-Value="@this.resultFileFormat" Disabled="@this.isProcessingBatch" AdornmentIcon="@Icons.Material.Filled.Description" Adornment="Adornment.Start" Label="@T("File format")" HelperText="@T("Choose the format of the result files. Everything except Markdown is converted by Pandoc, which AI Studio offers to install when it is missing.")" Variant="Variant.Outlined" Margin="Margin.Dense" Class="mb-3">
+        @foreach (var format in FileExportFormatExtensions.ANSWER_FORMATS)
+        {
+            <MudSelectItem Value="@format">
+                @format.ToName()
+            </MudSelectItem>
+        }
+    </MudSelect>
+
     <MudJustifiedText Typo="Typo.body2" Class="mb-3">
-        @T("Each answer is stored as its own Markdown file. Those files are named after the document, e.g., the answer for report.pdf is stored as report_result.md.")
+        @(string.Format(T("Each answer is stored as its own file. Those files are named after the document, e.g., the answer for report.pdf is stored as report_result{0}."), this.resultFileFormat.ToFileExtension()))
     </MudJustifiedText>
 }
 else

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Persistence.cs

@@ -71,8 +71,8 @@ public partial class AssistantBatchProcessing
     /// <summary>
     /// Checks whether a document can be restored from the previous run. Beyond
     /// the log entry, the result of the previous run must still exist: in the
-    /// table mode the answer within the results table, in the Markdown mode the
-    /// result file. Without the result, restoring would mark the document as
+    /// table mode the answer within the results table, in the individual file
+    /// mode the result file. Without the result, restoring would mark the document as
     /// done while its answer is lost, so we process it again instead.
     /// </summary>
     private bool CanRestoreFromPreviousRun(string relativePath, string resolvedOutputDirectory, Dictionary<string, BatchProcessingLogEntry> previousLog, Dictionary<string, string> previousResults, out BatchProcessingLogEntry? logEntry)
@@ -106,9 +106,9 @@ public partial class AssistantBatchProcessing
     private async Task WriteLogAsync(string resolvedOutputDirectory)
     {
         var sb = new StringBuilder();
-        sb.AppendLine(BatchProcessingCsv.ToCsvRow(LOG_SEPARATOR, T("File"), T("Time"), T("Model"), T("Status"), T("Details")));
+        sb.AppendLine(CsvWriter.ToRow(LOG_SEPARATOR, T("File"), T("Time"), T("Model"), T("Status"), T("Details")));
         foreach (var fileResult in this.fileResults.Where(x => x.Status is not BatchProcessingFileStatus.QUEUED and not BatchProcessingFileStatus.PROCESSING))
-            sb.AppendLine(BatchProcessingCsv.ToCsvRow(LOG_SEPARATOR, fileResult.RelativePath, fileResult.ProcessedAt.ToString(TIME_FORMAT, CultureInfo.InvariantCulture), fileResult.ModelName, fileResult.Status.ToString(), fileResult.Message));
+            sb.AppendLine(CsvWriter.ToRow(LOG_SEPARATOR, fileResult.RelativePath, fileResult.ProcessedAt.ToString(TIME_FORMAT, CultureInfo.InvariantCulture), fileResult.ModelName, fileResult.Status.ToString(), fileResult.Message));
 
         await this.WriteCsvFileAsync(Path.Join(resolvedOutputDirectory, LOG_FILENAME), sb.ToString());
     }
@@ -120,9 +120,9 @@ public partial class AssistantBatchProcessing
     {
         var separator = this.csvSeparator.Character(this.customCsvSeparator);
         var sb = new StringBuilder();
-        sb.AppendLine(BatchProcessingCsv.ToCsvRow(separator, T("File"), this.ResultColumnHeader));
+        sb.AppendLine(CsvWriter.ToRow(separator, T("File"), this.ResultColumnHeader));
         foreach (var fileResult in this.fileResults.Where(x => x.Status is BatchProcessingFileStatus.DONE))
-            sb.AppendLine(BatchProcessingCsv.ToCsvRow(separator, fileResult.RelativePath, fileResult.ResultText));
+            sb.AppendLine(CsvWriter.ToRow(separator, fileResult.RelativePath, fileResult.ResultText));
 
         await this.WriteCsvFileAsync(Path.Join(resolvedOutputDirectory, this.ResolveResultsFileName()), sb.ToString());
     }
@@ -232,7 +232,7 @@ public partial class AssistantBatchProcessing
     }
 
     /// <summary>
-    /// Creates the name of the Markdown result file for one document.
+    /// Creates the name of the result file for one document, in the chosen file format.
     /// </summary>
     /// <remarks>
     /// Two documents of the same run may share their name and differ only in
@@ -242,13 +242,14 @@ public partial class AssistantBatchProcessing
     /// </remarks>
     private string CreateResultFileName(string sourceFileName)
     {
+        var extension = this.resultFileFormat.ToFileExtension();
         var stem = Path.GetFileNameWithoutExtension(sourceFileName);
-        var candidate = $"{stem}{RESULT_FILE_SUFFIX}";
+        var candidate = $"{stem}{RESULT_FILE_SUFFIX}{extension}";
 
         var counter = 2;
         while (!this.usedResultFileNames.Add(candidate))
         {
-            candidate = $"{stem}_result_{counter}.md";
+            candidate = $"{stem}{RESULT_FILE_SUFFIX}_{counter}{extension}";
             counter++;
         }
 

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Run.cs

@@ -1,6 +1,5 @@
 using System.Diagnostics;
 using System.Globalization;
-using System.Text;
 
 namespace AIStudio.Assistants.BatchProcessing;
 
@@ -14,6 +13,19 @@ public partial class AssistantBatchProcessing
 
         var (resolvedOutputDirectory, files) = runPreparation.Value;
 
+        //
+        // Every format but Markdown is written by Pandoc, so it has to be there before the first
+        // document. Asking per document would put the installation dialog in front of the user
+        // hundreds of times, and starting without it would spend time and tokens on answers we
+        // cannot write anywhere:
+        //
+        if (this.outputMode is BatchProcessingOutputMode.INDIVIDUAL_FILES && this.resultFileFormat.UsesPandoc())
+        {
+            var pandocState = await this.PandocAvailability.EnsureAvailabilityAsync(showSuccessMessage: false, showDialog: true);
+            if (!pandocState.IsAvailable)
+                return;
+        }
+
         //
         // When the output folder already contains a log, a previous run was
         // interrupted or produced errors. Let the user decide what to do:
@@ -63,7 +75,7 @@ public partial class AssistantBatchProcessing
                 if (DateTimeOffset.TryParseExact(logEntry.Time, TIME_FORMAT, CultureInfo.InvariantCulture, DateTimeStyles.AssumeLocal, out var processedAt))
                     fileResult.ProcessedAt = processedAt;
 
-                // Reserve the Markdown file name of the previous run, so that a
+                // Reserve the result file name of the previous run, so that a
                 // document processed now cannot overwrite that earlier result:
                 if (!string.IsNullOrWhiteSpace(logEntry.Details))
                     this.usedResultFileNames.Add(logEntry.Details);
@@ -211,12 +223,26 @@ public partial class AssistantBatchProcessing
         }
 
         fileResult.ResultText = aiAnswer;
-        if (this.outputMode is BatchProcessingOutputMode.MARKDOWN_FILES)
+        if (this.outputMode is BatchProcessingOutputMode.INDIVIDUAL_FILES)
         {
             try
             {
                 var resultFilePath = Path.Join(resolvedOutputDirectory, this.CreateResultFileName(fileResult.FileName));
-                await File.WriteAllTextAsync(resultFilePath, aiAnswer, Encoding.UTF8, CancellationToken.None);
+                if (this.resultFileFormat.UsesPandoc())
+                {
+                    //
+                    // Pandoc reports a failure instead of throwing, because one document which
+                    // cannot be converted must not end a run over hundreds of them:
+                    //
+                    if (!await PandocExport.ConvertAsync(this.RustService, aiAnswer, resultFilePath, this.resultFileFormat, token))
+                    {
+                        this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("Was not able to convert the answer into the chosen file format."));
+                        return;
+                    }
+                }
+                else
+                    await File.WriteAllTextAsync(resultFilePath, aiAnswer, this.resultFileFormat.ToFileEncoding(), CancellationToken.None);
+
                 this.FinishFileResult(fileResult, BatchProcessingFileStatus.DONE, Path.GetFileName(resultFilePath));
             }
             catch (Exception e)

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Session.cs

@@ -16,6 +16,7 @@ public partial class AssistantBatchProcessing
     private static readonly AssistantSessionStateKey<string> PROMPT_FILE_LOAD_ISSUE_STATE_KEY = new(nameof(promptFileLoadIssue));
     private static readonly AssistantSessionStateKey<DataDocumentAnalysisPolicy?> SELECTED_POLICY_STATE_KEY = new(nameof(selectedPolicy));
     private static readonly AssistantSessionStateKey<BatchProcessingOutputMode> OUTPUT_MODE_STATE_KEY = new(nameof(outputMode));
+    private static readonly AssistantSessionStateKey<FileExportFormat> RESULT_FILE_FORMAT_STATE_KEY = new(nameof(resultFileFormat));
     private static readonly AssistantSessionStateKey<string> RESULT_COLUMN_HEADER_STATE_KEY = new(nameof(resultColumnHeader));
     private static readonly AssistantSessionStateKey<string> CSV_FILE_NAME_STATE_KEY = new(nameof(csvFileName));
     private static readonly AssistantSessionStateKey<BatchProcessingCsvSeparator> CSV_SEPARATOR_STATE_KEY = new(nameof(csvSeparator));
@@ -43,6 +44,7 @@ public partial class AssistantBatchProcessing
         state.Set(PROMPT_FILE_LOAD_ISSUE_STATE_KEY, this.promptFileLoadIssue);
         state.Set(SELECTED_POLICY_STATE_KEY, this.selectedPolicy);
         state.Set(OUTPUT_MODE_STATE_KEY, this.outputMode);
+        state.Set(RESULT_FILE_FORMAT_STATE_KEY, this.resultFileFormat);
         state.Set(RESULT_COLUMN_HEADER_STATE_KEY, this.resultColumnHeader);
         state.Set(CSV_FILE_NAME_STATE_KEY, this.csvFileName);
         state.Set(CSV_SEPARATOR_STATE_KEY, this.csvSeparator);
@@ -71,6 +73,7 @@ public partial class AssistantBatchProcessing
         state.Restore(PROMPT_FILE_LOAD_ISSUE_STATE_KEY, value => this.promptFileLoadIssue = value);
         state.Restore(SELECTED_POLICY_STATE_KEY, value => this.selectedPolicy = value);
         state.Restore(OUTPUT_MODE_STATE_KEY, value => this.outputMode = value);
+        state.Restore(RESULT_FILE_FORMAT_STATE_KEY, value => this.resultFileFormat = value);
         state.Restore(RESULT_COLUMN_HEADER_STATE_KEY, value => this.resultColumnHeader = value);
         state.Restore(CSV_FILE_NAME_STATE_KEY, value => this.csvFileName = value);
         state.Restore(CSV_SEPARATOR_STATE_KEY, value => this.csvSeparator = value);

app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs

@@ -1,6 +1,7 @@
 using AIStudio.Dialogs.Settings;
 using AIStudio.Provider;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
 
 using Microsoft.AspNetCore.Components;
 
@@ -11,10 +12,13 @@ public partial class AssistantBatchProcessing : AssistantBaseCore<SettingsDialog
     [Inject]
     private IDialogService DialogService { get; init; } = null!;
 
+    [Inject]
+    private PandocAvailabilityService PandocAvailability { get; init; } = null!;
+
     private const string DEFAULT_OUTPUT_DIRECTORY_NAME = "ai-results";
     private const string DEFAULT_RESULTS_FILENAME = "batch-results.csv";
     private const string CSV_EXTENSION = ".csv";
-    private const string RESULT_FILE_SUFFIX = "_result.md";
+    private const string RESULT_FILE_SUFFIX = "_result";
     private const string TRANSCRIPT_FILE_SUFFIX = ".transcript.md";
     private const string TIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
     private const char LOG_SEPARATOR = ';';
@@ -87,7 +91,8 @@ public partial class AssistantBatchProcessing : AssistantBaseCore<SettingsDialog
     private string promptFilePath = string.Empty;
     private string promptFileLoadIssue = string.Empty;
     private DataDocumentAnalysisPolicy? selectedPolicy;
-    private BatchProcessingOutputMode outputMode = BatchProcessingOutputMode.MARKDOWN_FILES;
+    private BatchProcessingOutputMode outputMode = BatchProcessingOutputMode.INDIVIDUAL_FILES;
+    private FileExportFormat resultFileFormat = FileExportFormat.MARKDOWN;
     private string resultColumnHeader = string.Empty;
     private string csvFileName = string.Empty;
     private BatchProcessingCsvSeparator csvSeparator = BatchProcessingCsvSeparator.SEMICOLON;
@@ -160,7 +165,8 @@ public partial class AssistantBatchProcessing : AssistantBaseCore<SettingsDialog
             this.freePrompt = string.Empty;
             this.promptFilePath = string.Empty;
             this.selectedPolicy = null;
-            this.outputMode = BatchProcessingOutputMode.MARKDOWN_FILES;
+            this.outputMode = BatchProcessingOutputMode.INDIVIDUAL_FILES;
+            this.resultFileFormat = FileExportFormat.MARKDOWN;
             this.resultColumnHeader = string.Empty;
             this.csvFileName = string.Empty;
             this.csvSeparator = BatchProcessingCsvSeparator.SEMICOLON;
@@ -180,6 +186,7 @@ public partial class AssistantBatchProcessing : AssistantBaseCore<SettingsDialog
         this.selectedPolicy = this.SettingsManager.ConfigurationData.DocumentAnalysis.Policies
             .FirstOrDefault(policy => policy.Id == settings.PreselectedPolicyId);
         this.outputMode = settings.OutputMode;
+        this.resultFileFormat = settings.ResultFileFormat;
         this.resultColumnHeader = settings.ResultColumnHeader;
         this.csvFileName = settings.CsvFileName;
         this.csvSeparator = settings.CsvSeparator;

app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingCsv.cs

@@ -3,35 +3,13 @@ using System.Text;
 namespace AIStudio.Assistants.BatchProcessing;
 
 /// <summary>
-/// Reads and writes the CSV files of the batch processing assistant. Fields
-/// are quoted according to RFC 4180 using the separator selected for the
-/// respective file.
+/// Reads the CSV files of the batch processing assistant. Writing them is the job of CsvWriter,
+/// which quotes fields according to RFC 4180 using the separator selected for the respective file.
 /// </summary>
 public static class BatchProcessingCsv
 {
-    public static string ToCsvRow(char separator, params string[] fields) => string.Join(separator, fields.Select(field => ToCsvField(field, separator)));
-
     /// <summary>
-    /// Quotes one CSV field according to RFC 4180.
-    /// </summary>
-    private static string ToCsvField(string text, char separator)
-    {
-        if (string.IsNullOrEmpty(text))
-            return string.Empty;
-
-        // Quoting the complete field is important for long and multi-line AI
-        // answers: neither separators nor line breaks within an answer may
-        // create another column or row.
-        if (!text.Contains(separator) && !text.Contains('"') && !text.Contains('\n') && !text.Contains('\r'))
-            return text;
-
-        return $"""
-                "{text.Replace("\"", "\"\"")}"
-                """;
-    }
-
-    /// <summary>
-    /// Parses a CSV text which was written by <see cref="ToCsvRow"/>.
+    /// Parses a CSV text which was written by CsvWriter.ToRow.
     /// </summary>
     /// <remarks>
     /// We parse the file ourselves instead of splitting lines, because quoted

app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingOutputMode.cs

@@ -6,9 +6,14 @@ namespace AIStudio.Assistants.BatchProcessing;
 public enum BatchProcessingOutputMode
 {
     /// <summary>
-    /// One Markdown result file per processed document.
+    /// One result file per processed document, written in the chosen file format.
     /// </summary>
-    MARKDOWN_FILES,
+    /// <remarks>
+    /// This must stay the first member. Enums are persisted under their name, and an unknown name
+    /// falls back to the default value of the enum, which is the member with the value zero. That
+    /// is what lets settings written before this member was renamed still land here.
+    /// </remarks>
+    INDIVIDUAL_FILES,
 
     /// <summary>
     /// A CSV results table, where each AI answer becomes one row. The content of

app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingOutputModeExtensions.cs

@@ -6,7 +6,7 @@ public static class BatchProcessingOutputModeExtensions
 
     public static string Name(this BatchProcessingOutputMode outputMode) => outputMode switch
     {
-        BatchProcessingOutputMode.MARKDOWN_FILES => TB("One Markdown file per document"),
+        BatchProcessingOutputMode.INDIVIDUAL_FILES => TB("One file per document"),
         BatchProcessingOutputMode.TABLE_ONLY => TB("One CSV results table, where each answer becomes one row"),
 
         _ => TB("Unknown output mode"),