From 9e07155e3d8231529981486305c85562cc3d112d Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Tue, 11 Aug 2026 18:59:57 +0200 Subject: [PATCH] Make batch CSV separators configurable --- .../AssistantBatchProcessing.razor | 16 +++- ...istantBatchProcessing.razor.Persistence.cs | 15 +-- .../AssistantBatchProcessing.razor.Session.cs | 6 ++ ...sistantBatchProcessing.razor.Validation.cs | 12 +++ .../AssistantBatchProcessing.razor.cs | 7 ++ .../BatchProcessing/BatchProcessingCsv.cs | 94 ++++++++++++++++--- .../BatchProcessingCsvSeparator.cs | 13 +++ .../BatchProcessingCsvSeparatorExtensions.cs | 41 ++++++++ .../Assistants/I18N/allTexts.lua | 54 ++++++++++- .../Components/ConfigurationText.razor | 4 +- .../Components/ConfigurationText.razor.cs | 13 ++- .../SettingsDialogBatchProcessing.razor | 5 + .../SettingsDialogBatchProcessing.razor.cs | 15 +++ .../Plugins/configuration/plugin.lua | 6 ++ .../Settings/DataModel/DataBatchProcessing.cs | 4 + .../Tools/PluginSystem/PluginConfiguration.cs | 2 + 16 files changed, 281 insertions(+), 26 deletions(-) create mode 100644 app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingCsvSeparator.cs create mode 100644 app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingCsvSeparatorExtensions.cs diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor index 83efbef2..0ba9d801 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor @@ -126,12 +126,26 @@ else + + + @foreach (var separator in Enum.GetValues()) + { + + @separator.Name() + + } + + + @if (this.csvSeparator is BatchProcessingCsvSeparator.CUSTOM) + { + + } } - @T("We always write a log named log.csv, which lists every document with its processing time, the model, the status, and the details of any error. Its columns are separated by a vertical bar, so you can open it with any spreadsheet application. When you start another run on the same output folder, we ask you whether to continue that run: documents which failed or are missing in the log are then processed again. When no output folder is selected, everything is written to the subfolder 'ai-results' within the input folder.") + @T("We always write a semicolon-separated log named log.csv, which lists every document with its processing time, the model, the status, and the details of any error. When you start another run on the same output folder, we ask you whether to continue that run: documents which failed or are missing in the log are then processed again. When no output folder is selected, everything is written to the subfolder 'ai-results' within the input folder.") diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Persistence.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Persistence.cs index b9337923..23103e82 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Persistence.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Persistence.cs @@ -106,9 +106,9 @@ public partial class AssistantBatchProcessing private async Task WriteLogAsync(string resolvedOutputDirectory) { var sb = new StringBuilder(); - sb.AppendLine(BatchProcessingCsv.ToCsvRow(T("File"), T("Time"), T("Model"), T("Status"), T("Details"))); + sb.AppendLine(BatchProcessingCsv.ToCsvRow(LOG_SEPARATOR, T("File"), T("Time"), T("Model"), T("Status"), T("Details"))); foreach (var fileResult in this.fileResults.Where(x => x.Status is not BatchProcessingFileStatus.QUEUED and not BatchProcessingFileStatus.PROCESSING)) - sb.AppendLine(BatchProcessingCsv.ToCsvRow(fileResult.RelativePath, fileResult.ProcessedAt.ToString(TIME_FORMAT, CultureInfo.InvariantCulture), fileResult.ModelName, fileResult.Status.ToString(), fileResult.Message)); + sb.AppendLine(BatchProcessingCsv.ToCsvRow(LOG_SEPARATOR, fileResult.RelativePath, fileResult.ProcessedAt.ToString(TIME_FORMAT, CultureInfo.InvariantCulture), fileResult.ModelName, fileResult.Status.ToString(), fileResult.Message)); await this.WriteCsvFileAsync(Path.Join(resolvedOutputDirectory, LOG_FILENAME), sb.ToString()); } @@ -118,10 +118,11 @@ public partial class AssistantBatchProcessing /// private async Task WriteResultsTableAsync(string resolvedOutputDirectory) { + var separator = this.csvSeparator.Character(this.customCsvSeparator); var sb = new StringBuilder(); - sb.AppendLine(BatchProcessingCsv.ToCsvRow(T("File"), this.ResultColumnHeader)); + sb.AppendLine(BatchProcessingCsv.ToCsvRow(separator, T("File"), this.ResultColumnHeader)); foreach (var fileResult in this.fileResults.Where(x => x.Status is BatchProcessingFileStatus.DONE)) - sb.AppendLine(BatchProcessingCsv.ToCsvRow(fileResult.RelativePath, fileResult.ResultText)); + sb.AppendLine(BatchProcessingCsv.ToCsvRow(separator, fileResult.RelativePath, fileResult.ResultText)); await this.WriteCsvFileAsync(Path.Join(resolvedOutputDirectory, this.ResolveResultsFileName()), sb.ToString()); } @@ -175,7 +176,7 @@ public partial class AssistantBatchProcessing try { var content = await File.ReadAllTextAsync(logFilePath); - var rows = BatchProcessingCsv.Parse(content); + var rows = BatchProcessingCsv.ParseWithDetectedSeparator(content, 5, LOG_SEPARATOR, '|'); // The first row is the header, which we skip: foreach (var row in rows.Skip(1)) @@ -211,7 +212,9 @@ public partial class AssistantBatchProcessing return results; var content = await File.ReadAllTextAsync(resultsFilePath); - foreach (var row in BatchProcessingCsv.Parse(content).Skip(1)) + var configuredSeparator = this.csvSeparator.Character(this.customCsvSeparator); + var rows = BatchProcessingCsv.ParseWithDetectedSeparator(content, 2, configuredSeparator, ';', '|', ',', '\t'); + foreach (var row in rows.Skip(1)) { if (row.Count < 2 || string.IsNullOrWhiteSpace(row[0])) continue; diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Session.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Session.cs index 182f17ed..a7d00e40 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Session.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Session.cs @@ -18,6 +18,8 @@ public partial class AssistantBatchProcessing private static readonly AssistantSessionStateKey OUTPUT_MODE_STATE_KEY = new(nameof(outputMode)); private static readonly AssistantSessionStateKey RESULT_COLUMN_HEADER_STATE_KEY = new(nameof(resultColumnHeader)); private static readonly AssistantSessionStateKey CSV_FILE_NAME_STATE_KEY = new(nameof(csvFileName)); + private static readonly AssistantSessionStateKey CSV_SEPARATOR_STATE_KEY = new(nameof(csvSeparator)); + private static readonly AssistantSessionStateKey CUSTOM_CSV_SEPARATOR_STATE_KEY = new(nameof(customCsvSeparator)); private static readonly AssistantSessionStateKey> FILE_RESULTS_STATE_KEY = new(nameof(fileResults)); private static readonly AssistantSessionStateKey> USED_RESULT_FILE_NAMES_STATE_KEY = new(nameof(usedResultFileNames)); private static readonly AssistantSessionStateKey IS_PROCESSING_BATCH_STATE_KEY = new(nameof(isProcessingBatch)); @@ -40,6 +42,8 @@ public partial class AssistantBatchProcessing state.Set(OUTPUT_MODE_STATE_KEY, this.outputMode); state.Set(RESULT_COLUMN_HEADER_STATE_KEY, this.resultColumnHeader); state.Set(CSV_FILE_NAME_STATE_KEY, this.csvFileName); + state.Set(CSV_SEPARATOR_STATE_KEY, this.csvSeparator); + state.Set(CUSTOM_CSV_SEPARATOR_STATE_KEY, this.customCsvSeparator); state.SetList(FILE_RESULTS_STATE_KEY, this.fileResults.Select(CloneFileResult)); state.SetHashSet(USED_RESULT_FILE_NAMES_STATE_KEY, this.usedResultFileNames); state.Set(IS_PROCESSING_BATCH_STATE_KEY, this.isProcessingBatch); @@ -63,6 +67,8 @@ public partial class AssistantBatchProcessing state.Restore(OUTPUT_MODE_STATE_KEY, value => this.outputMode = value); state.Restore(RESULT_COLUMN_HEADER_STATE_KEY, value => this.resultColumnHeader = value); state.Restore(CSV_FILE_NAME_STATE_KEY, value => this.csvFileName = value); + state.Restore(CSV_SEPARATOR_STATE_KEY, value => this.csvSeparator = value); + state.Restore(CUSTOM_CSV_SEPARATOR_STATE_KEY, value => this.customCsvSeparator = value); state.Restore(FILE_RESULTS_STATE_KEY, values => { this.fileResults.Clear(); diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs index d1b6cba4..bae6eb5e 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs @@ -56,6 +56,18 @@ public partial class AssistantBatchProcessing return null; } + private string? ValidateCustomCsvSeparator(string separator) + { + if (this.outputMode is not BatchProcessingOutputMode.TABLE_ONLY + || this.csvSeparator is not BatchProcessingCsvSeparator.CUSTOM) + return null; + + if (!BatchProcessingCsvSeparatorExtensions.IsValidCustomSeparator(separator)) + return T("Please enter exactly one punctuation or symbol character. Letters, numbers, spaces, quotation marks, and line breaks cannot be used as CSV separators."); + + return null; + } + private string? ValidateFreePrompt(string prompt) { if (this.promptSource is BatchProcessingPromptSource.FREE_PROMPT && string.IsNullOrWhiteSpace(prompt)) diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs index f13065d8..9c6d031a 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs @@ -17,6 +17,7 @@ public partial class AssistantBatchProcessing : AssistantBaseCore /// The name of the log file. It is fixed, so that a later batch run finds @@ -88,6 +89,8 @@ public partial class AssistantBatchProcessing : AssistantBaseCore fileResults = []; private readonly HashSet usedResultFileNames = new(StringComparer.OrdinalIgnoreCase); @@ -156,6 +159,8 @@ public partial class AssistantBatchProcessing : AssistantBaseCore /// Reads and writes the CSV files of the batch processing assistant. Fields -/// are quoted according to RFC 4180, but the separator is a vertical bar, so -/// that the files open nicely in spreadsheet applications regardless of the -/// list separator of the user's locale. +/// are quoted according to RFC 4180 using the separator selected for the +/// respective file. /// public static class BatchProcessingCsv { - private const char SEPARATOR = '|'; - - public static string ToCsvRow(params string[] fields) => string.Join(SEPARATOR, fields.Select(ToCsvField)); + public static string ToCsvRow(char separator, params string[] fields) => string.Join(separator, fields.Select(field => ToCsvField(field, separator))); /// /// Quotes one CSV field according to RFC 4180. /// - private static string ToCsvField(string text) + private static string ToCsvField(string text, char separator) { if (string.IsNullOrEmpty(text)) return string.Empty; - if (!text.Contains(SEPARATOR) && !text.Contains('"') && !text.Contains('\n') && !text.Contains('\r')) + // Quoting the complete field is important for long and multi-line AI + // answers: neither separators nor line breaks within an answer may + // create another column or row. + if (!text.Contains(separator) && !text.Contains('"') && !text.Contains('\n') && !text.Contains('\r')) return text; - return $"\"{text.Replace("\"", "\"\"")}\""; + return $""" + "{text.Replace("\"", "\"\"")}" + """; } /// @@ -35,7 +37,7 @@ public static class BatchProcessingCsv /// We parse the file ourselves instead of splitting lines, because quoted /// fields may contain the separator and line breaks. /// - public static List> Parse(string content) + private static List> Parse(string content, char separator) { var rows = new List>(); var fields = new List(); @@ -73,7 +75,7 @@ public static class BatchProcessingCsv hasContent = true; break; - case SEPARATOR: + case var _ when character == separator: hasContent = true; EndField(); break; @@ -113,4 +115,74 @@ public static class BatchProcessingCsv hasContent = false; } } + + /// + /// Detects the separator from the first CSV record and parses the complete + /// content with it. Preferred separators are used as fallbacks for files + /// whose first record does not reveal a valid separator. + /// + public static List> ParseWithDetectedSeparator(string content, int expectedNumFields, params char[] preferredSeparators) + { + var firstRecord = ReadFirstRecord(content); + var candidates = new List(); + var isQuoted = false; + for (var index = 0; index < firstRecord.Length; index++) + { + var character = firstRecord[index]; + if (character is '"') + { + if (isQuoted && index + 1 < firstRecord.Length && firstRecord[index + 1] is '"') + { + index++; + continue; + } + + isQuoted = !isQuoted; + continue; + } + + if (!isQuoted + && character is not '\r' and not '\n' + && (char.IsPunctuation(character) || char.IsSymbol(character) || character is '\t') + && !candidates.Contains(character)) + candidates.Add(character); + } + + foreach (var separator in preferredSeparators) + { + if (!candidates.Contains(separator)) + candidates.Add(separator); + } + + foreach (var separator in candidates) + { + var header = Parse(firstRecord, separator); + if (header.Count is 1 && header[0].Count == expectedNumFields) + return Parse(content, separator); + } + + throw new InvalidDataException("Was not able to detect the CSV separator."); + } + + private static string ReadFirstRecord(string content) + { + var isQuoted = false; + for (var index = 0; index < content.Length; index++) + { + if (content[index] is '"') + { + if (isQuoted && index + 1 < content.Length && content[index + 1] is '"') + { + index++; + continue; + } + + isQuoted = !isQuoted; + } + else if (content[index] is '\n' && !isQuoted) + return content[..(index + 1)]; + } + + return content; + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingCsvSeparator.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingCsvSeparator.cs new file mode 100644 index 00000000..9ef9cc02 --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingCsvSeparator.cs @@ -0,0 +1,13 @@ +namespace AIStudio.Assistants.BatchProcessing; + +/// +/// Defines the separators available for Batch Processing result tables. +/// +public enum BatchProcessingCsvSeparator +{ + COMMA, + SEMICOLON, + PIPE, + TAB, + CUSTOM, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingCsvSeparatorExtensions.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingCsvSeparatorExtensions.cs new file mode 100644 index 00000000..d6f08f06 --- /dev/null +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/BatchProcessingCsvSeparatorExtensions.cs @@ -0,0 +1,41 @@ +namespace AIStudio.Assistants.BatchProcessing; + +public static class BatchProcessingCsvSeparatorExtensions +{ + private const char DEFAULT_SEPARATOR = ';'; + + private static string TB(string fallbackEN) => Tools.PluginSystem.I18N.I.T(fallbackEN, typeof(BatchProcessingCsvSeparatorExtensions).Namespace, nameof(BatchProcessingCsvSeparatorExtensions)); + + public static string Name(this BatchProcessingCsvSeparator separator) => separator switch + { + BatchProcessingCsvSeparator.COMMA => TB("Comma (,)"), + BatchProcessingCsvSeparator.SEMICOLON => TB("Semicolon (;)"), + BatchProcessingCsvSeparator.PIPE => TB("Vertical bar (|)"), + BatchProcessingCsvSeparator.TAB => TB("Tab"), + BatchProcessingCsvSeparator.CUSTOM => TB("Custom character"), + + _ => TB("Unknown"), + }; + + public static char Character(this BatchProcessingCsvSeparator separator, string customSeparator) => separator switch + { + BatchProcessingCsvSeparator.COMMA => ',', + BatchProcessingCsvSeparator.SEMICOLON => ';', + BatchProcessingCsvSeparator.PIPE => '|', + BatchProcessingCsvSeparator.TAB => '\t', + BatchProcessingCsvSeparator.CUSTOM when IsValidCustomSeparator(customSeparator) => customSeparator[0], + + _ => DEFAULT_SEPARATOR, + }; + + internal static bool IsValidCustomSeparator(string separator) + { + if (string.IsNullOrEmpty(separator) || separator.Length is not 1) + return false; + + var character = separator[0]; + return !char.IsLetterOrDigit(character) + && !char.IsWhiteSpace(character) + && character is not '"' and not '\r' and not '\n'; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 01f36b7c..1df997e9 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -334,6 +334,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T83711157"] = "Copy result -- The transcription provider returned an empty transcript. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1080540822"] = "The transcription provider returned an empty transcript." +-- We always write a semicolon-separated log named log.csv, which lists every document with its processing time, the model, the status, and the details of any error. When you start another run on the same output folder, we ask you whether to continue that run: documents which failed or are missing in the log are then processed again. When no output folder is selected, everything is written to the subfolder 'ai-results' within the input folder. +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1124333059"] = "We always write a semicolon-separated log named log.csv, which lists every document with its processing time, the model, the status, and the details of any error. When you start another run on the same output folder, we ask you whether to continue that run: documents which failed or are missing in the log are then processed again. When no output folder is selected, everything is written to the subfolder 'ai-results' within the input folder." + -- Name of the results table (optional) UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1127380661"] = "Name of the results table (optional)" @@ -382,6 +385,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING -- No matching files were found in the selected folder. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1528532808"] = "No matching files were found in the selected folder." +-- Custom column separator +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1547654319"] = "Custom column separator" + -- Select the output folder UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1598970341"] = "Select the output folder" @@ -421,9 +427,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING -- Configured instructions file: {0} UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2215428124"] = "Configured instructions file: {0}" --- We always write a log named log.csv, which lists every document with its processing time, the model, the status, and the details of any error. Its columns are separated by a vertical bar, so you can open it with any spreadsheet application. When you start another run on the same output folder, we ask you whether to continue that run: documents which failed or are missing in the log are then processed again. When no output folder is selected, everything is written to the subfolder 'ai-results' within the input folder. -UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T225540173"] = "We always write a log named log.csv, which lists every document with its processing time, the model, the status, and the details of any error. Its columns are separated by a vertical bar, so you can open it with any spreadsheet application. When you start another run on the same output folder, we ask you whether to continue that run: documents which failed or are missing in the log are then processed again. When no output folder is selected, everything is written to the subfolder 'ai-results' within the input folder." - -- No usable transcription provider is configured. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2282521655"] = "No usable transcription provider is configured." @@ -466,6 +469,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING -- The batch run was canceled. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2641642683"] = "The batch run was canceled." +-- Choose which character separates the columns of the results table. +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2642486086"] = "Choose which character separates the columns of the results table." + -- The configured instructions file no longer exists. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2652734495"] = "The configured instructions file no longer exists." @@ -481,6 +487,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING -- Each answer is stored as its own Markdown file. Those files are named after the document, e.g., the answer for report.pdf is stored as report_result.md. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2724126639"] = "Each answer is stored as its own Markdown file. Those files are named after the document, e.g., the answer for report.pdf is stored as report_result.md." +-- Please enter exactly one punctuation or symbol character. Letters, numbers, spaces, quotation marks, and line breaks cannot be used as CSV separators. +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2908365499"] = "Please enter exactly one punctuation or symbol character. Letters, numbers, spaces, quotation marks, and line breaks cannot be used as CSV separators." + -- Was not able to write the result file: {0} UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2991295581"] = "Was not able to write the result file: {0}" @@ -547,12 +556,18 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING -- Was not able to extract any text from this file. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T4175885324"] = "Was not able to extract any text from this file." +-- Column separator +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T423947932"] = "Column separator" + -- The configured instructions file could not be read. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T4274794480"] = "The configured instructions file could not be read." -- Progress UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T439787878"] = "Progress" +-- Enter one punctuation or symbol character. +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T469253621"] = "Enter one punctuation or symbol character." + -- No, only process files in the selected folder UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T49675965"] = "No, only process files in the selected folder" @@ -583,6 +598,24 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING -- A separate output folder is excluded from document discovery. This includes the default 'ai-results' folder, so results from an earlier run are not processed again. If the input folder itself is used for output, known batch result files are excluded instead. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T822136905"] = "A separate output folder is excluded from document discovery. This includes the default 'ai-results' folder, so results from an earlier run are not processed again. If the input folder itself is used for output, known batch result files are excluded instead." +-- Comma (,) +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::BATCHPROCESSINGCSVSEPARATOREXTENSIONS::T1676507543"] = "Comma (,)" + +-- Semicolon (;) +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::BATCHPROCESSINGCSVSEPARATOREXTENSIONS::T3267990938"] = "Semicolon (;)" + +-- Unknown +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::BATCHPROCESSINGCSVSEPARATOREXTENSIONS::T3424652889"] = "Unknown" + +-- Tab +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::BATCHPROCESSINGCSVSEPARATOREXTENSIONS::T4219689196"] = "Tab" + +-- Vertical bar (|) +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::BATCHPROCESSINGCSVSEPARATOREXTENSIONS::T4252399493"] = "Vertical bar (|)" + +-- Custom character +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::BATCHPROCESSINGCSVSEPARATOREXTENSIONS::T719177757"] = "Custom character" + -- One CSV results table, where each answer becomes one row UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::BATCHPROCESSINGOUTPUTMODEEXTENSIONS::T1515293131"] = "One CSV results table, where each answer becomes one row" @@ -6523,6 +6556,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T17 -- Batch processing options are preselected UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1893713430"] = "Batch processing options are preselected" +-- Default custom column separator +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T19367494"] = "Default custom column separator" + -- Default document analysis policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2015391667"] = "Default document analysis policy" @@ -6547,9 +6583,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T26 -- Input UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2677268763"] = "Input" +-- Default column separator +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2745158463"] = "Default column separator" + -- Preselect batch processing options? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2849251744"] = "Preselect batch processing options?" +-- Please enter exactly one punctuation or symbol character. Letters, numbers, spaces, quotation marks, and line breaks cannot be used as CSV separators. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2908365499"] = "Please enter exactly one punctuation or symbol character. Letters, numbers, spaces, quotation marks, and line breaks cannot be used as CSV separators." + -- Default file patterns UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2909693903"] = "Default file patterns" @@ -6592,12 +6634,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T40 -- The configured default policy no longer exists. Select another policy before starting a policy-based batch run. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T438852523"] = "The configured default policy no longer exists. Select another policy before starting a policy-based batch run." +-- Enter one punctuation or symbol character. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T469253621"] = "Enter one punctuation or symbol character." + -- Select the default Markdown instructions file UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T470691525"] = "Select the default Markdown instructions file" -- Assistant: Batch Processing defaults UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T481452904"] = "Assistant: Batch Processing defaults" +-- Choose which character separates the columns of new results tables. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T573962596"] = "Choose which character separates the columns of new results tables." + -- Default output mode UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T601648878"] = "Default output mode" diff --git a/app/MindWork AI Studio/Components/ConfigurationText.razor b/app/MindWork AI Studio/Components/ConfigurationText.razor index a5d52346..feede5e2 100644 --- a/app/MindWork AI Studio/Components/ConfigurationText.razor +++ b/app/MindWork AI Studio/Components/ConfigurationText.razor @@ -15,6 +15,7 @@ AutoGrow="@this.AutoGrow" MaxLines="@this.GetMaxLines" Immediate="@true" + Validation="@this.Validation" Underline="false" /> } @@ -34,6 +35,7 @@ else AutoGrow="@this.AutoGrow" MaxLines="@this.GetMaxLines" Immediate="@true" + Validation="@this.Validation" Underline="false" Class="flex-grow-1" /> @@ -41,4 +43,4 @@ else @this.ResetButtonText -} +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ConfigurationText.razor.cs b/app/MindWork AI Studio/Components/ConfigurationText.razor.cs index a1b1f393..9610731e 100644 --- a/app/MindWork AI Studio/Components/ConfigurationText.razor.cs +++ b/app/MindWork AI Studio/Components/ConfigurationText.razor.cs @@ -53,6 +53,12 @@ public partial class ConfigurationText : ConfigurationBaseCore /// [Parameter] public string ResetButtonText { get; set; } = string.Empty; + + /// + /// Validates the configured text before it is stored. + /// + [Parameter] + public Func? Validation { get; set; } private string internalText = string.Empty; private readonly Timer timer = new(TimeSpan.FromMilliseconds(500)) @@ -69,10 +75,6 @@ public partial class ConfigurationText : ConfigurationBaseCore protected override string Label => this.OptionDescription; - #endregion - - #region Overrides of ConfigurationBase - protected override async Task OnInitializedAsync() { this.timer.Elapsed += async (_, _) => await this.InvokeAsync(async () => await this.OptionChanged(this.internalText)); @@ -110,6 +112,9 @@ public partial class ConfigurationText : ConfigurationBaseCore private async Task OptionChanged(string updatedText) { + if (this.Validation?.Invoke(updatedText) is not null) + return; + this.TextUpdate(updatedText); await this.SettingsManager.StoreSettings(); await this.InformAboutChange(); diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor index 80351ae4..bb9226d9 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor @@ -47,6 +47,11 @@ { + + @if (this.SettingsManager.ConfigurationData.BatchProcessing.CsvSeparator is BatchProcessingCsvSeparator.CUSTOM) + { + + } } diff --git a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor.cs b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor.cs index f295ed8a..34d1f600 100644 --- a/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor.cs +++ b/app/MindWork AI Studio/Dialogs/Settings/SettingsDialogBatchProcessing.razor.cs @@ -45,6 +45,21 @@ public partial class SettingsDialogBatchProcessing : SettingsDialogBase .Select(value => new ConfigurationSelectData(value.Name(), value)) ]; + private IReadOnlyList> CsvSeparatorData => + [ + .. Enum + .GetValues() + .Select(value => new ConfigurationSelectData(value.Name(), value)) + ]; + + private string? ValidateCustomCsvSeparator(string separator) + { + if (!BatchProcessingCsvSeparatorExtensions.IsValidCustomSeparator(separator)) + return T("Please enter exactly one punctuation or symbol character. Letters, numbers, spaces, quotation marks, and line breaks cannot be used as CSV separators."); + + return null; + } + private IReadOnlyList> PolicyData { get diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index 3fd0a22d..2361ee52 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -424,6 +424,10 @@ CONFIG["SETTINGS"] = {} -- CONFIG["SETTINGS"]["DataBatchProcessing.OutputMode"] = "MARKDOWN_FILES" -- CONFIG["SETTINGS"]["DataBatchProcessing.CsvFileName"] = "batch-results.csv" -- CONFIG["SETTINGS"]["DataBatchProcessing.ResultColumnHeader"] = "Result" +-- Allowed CSV separator values are: COMMA, SEMICOLON, PIPE, TAB, CUSTOM +-- A custom separator must be exactly one punctuation or symbol character. +-- CONFIG["SETTINGS"]["DataBatchProcessing.CsvSeparator"] = "SEMICOLON" +-- CONFIG["SETTINGS"]["DataBatchProcessing.CustomCsvSeparator"] = "^" -- -- Configure the minimum provider confidence and the default provider. -- Allowed confidence values are: NONE, UNTRUSTED, UNKNOWN, VERY_LOW, LOW, MODERATE, MEDIUM, HIGH @@ -444,6 +448,8 @@ CONFIG["SETTINGS"] = {} -- CONFIG["SETTINGS"]["DataBatchProcessing.OutputMode.AllowUserOverride"] = true -- CONFIG["SETTINGS"]["DataBatchProcessing.CsvFileName.AllowUserOverride"] = true -- CONFIG["SETTINGS"]["DataBatchProcessing.ResultColumnHeader.AllowUserOverride"] = true +-- CONFIG["SETTINGS"]["DataBatchProcessing.CsvSeparator.AllowUserOverride"] = true +-- CONFIG["SETTINGS"]["DataBatchProcessing.CustomCsvSeparator.AllowUserOverride"] = true -- CONFIG["SETTINGS"]["DataBatchProcessing.MinimumProviderConfidence.AllowUserOverride"] = true -- CONFIG["SETTINGS"]["DataBatchProcessing.PreselectedProvider.AllowUserOverride"] = true diff --git a/app/MindWork AI Studio/Settings/DataModel/DataBatchProcessing.cs b/app/MindWork AI Studio/Settings/DataModel/DataBatchProcessing.cs index 48268019..e256c1e3 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataBatchProcessing.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataBatchProcessing.cs @@ -44,6 +44,10 @@ public sealed class DataBatchProcessing(Expression value.ResultColumnHeader, string.Empty); + public BatchProcessingCsvSeparator CsvSeparator { get; set; } = ManagedConfiguration.Register(configSelection, value => value.CsvSeparator, BatchProcessingCsvSeparator.SEMICOLON); + + public string CustomCsvSeparator { get; set; } = ManagedConfiguration.Register(configSelection, value => value.CustomCsvSeparator, string.Empty); + public ConfidenceLevel MinimumProviderConfidence { get; set; } = ManagedConfiguration.Register(configSelection, value => value.MinimumProviderConfidence, ConfidenceLevel.NONE); public string PreselectedProvider { get; set; } = ManagedConfiguration.Register(configSelection, value => value.PreselectedProvider, string.Empty); diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs index 4f2ba2ca..0d083e13 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs @@ -350,6 +350,8 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.OutputMode, this.Id, settingsTable, dryRun); ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.CsvFileName, this.Id, settingsTable, dryRun); ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.ResultColumnHeader, this.Id, settingsTable, dryRun); + ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.CsvSeparator, this.Id, settingsTable, dryRun); + ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.CustomCsvSeparator, this.Id, settingsTable, dryRun); ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.MinimumProviderConfidence, this.Id, settingsTable, dryRun); ManagedConfiguration.TryProcessConfiguration(x => x.BatchProcessing, x => x.PreselectedProvider, Guid.Empty, this.Id, settingsTable, dryRun);