mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-08-21 01:02:11 +00:00
05790e8af4
The assistant processes all documents of a folder in one batch run. Each document is extracted to Markdown by the Rust runtime and sent to the selected provider together with the user's instructions. The instructions come from one of three sources: a free prompt, one of the existing document analysis policies including its minimum provider confidence, or a file the user imports. The output is either one Markdown file per document, or one CSV results table in which each answer becomes a row. The user can name the results table; its columns are the document and the answer. Every run writes a log named log.csv with the document, time, model, status, and the reason for any error. When a later run finds a log in the output folder, the assistant asks whether to continue it. Continuing processes only the documents that failed or whose results no longer exist, which recovers runs interrupted by a crash or by documents exceeding the context window of the model. A single failing document never stops the run, and the run can be canceled at any time. Columns are separated by a vertical bar and quoted per RFC 4180, so that the files open in spreadsheet applications regardless of the list separator of the user. Documents are identified by their path relative to the input folder, because two subfolders may contain a document of the same name. Includes the English and German localization. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
116 lines
3.1 KiB
C#
116 lines
3.1 KiB
C#
using System.Text;
|
|
|
|
namespace AIStudio.Assistants.BatchProcessing;
|
|
|
|
/// <summary>
|
|
/// Reads and writes the CSV files of the batch processing assistant. Fields
|
|
/// are quoted according to RFC 4180, but the separator is a vertical bar, so
|
|
/// that the files open nicely in spreadsheet applications regardless of the
|
|
/// list separator of the user's locale.
|
|
/// </summary>
|
|
public static class BatchProcessingCsv
|
|
{
|
|
public const char SEPARATOR = '|';
|
|
|
|
public static string ToCsvRow(params string[] fields) => string.Join(SEPARATOR, fields.Select(ToCsvField));
|
|
|
|
/// <summary>
|
|
/// Quotes one CSV field according to RFC 4180.
|
|
/// </summary>
|
|
private static string ToCsvField(string text)
|
|
{
|
|
if (string.IsNullOrEmpty(text))
|
|
return string.Empty;
|
|
|
|
if (!text.Contains(SEPARATOR) && !text.Contains('"') && !text.Contains('\n') && !text.Contains('\r'))
|
|
return text;
|
|
|
|
return $"\"{text.Replace("\"", "\"\"")}\"";
|
|
}
|
|
|
|
/// <summary>
|
|
/// Parses a CSV text which was written by <see cref="ToCsvRow"/>.
|
|
/// </summary>
|
|
/// <remarks>
|
|
/// We parse the file ourselves instead of splitting lines, because quoted
|
|
/// fields may contain the separator and line breaks.
|
|
/// </remarks>
|
|
public static List<List<string>> Parse(string content)
|
|
{
|
|
var rows = new List<List<string>>();
|
|
var fields = new List<string>();
|
|
var field = new StringBuilder();
|
|
var isQuoted = false;
|
|
var hasContent = false;
|
|
|
|
void EndField()
|
|
{
|
|
fields.Add(field.ToString());
|
|
field.Clear();
|
|
}
|
|
|
|
void EndRow()
|
|
{
|
|
EndField();
|
|
if (hasContent)
|
|
rows.Add([..fields]);
|
|
|
|
fields.Clear();
|
|
hasContent = false;
|
|
}
|
|
|
|
for (var index = 0; index < content.Length; index++)
|
|
{
|
|
var character = content[index];
|
|
if (isQuoted)
|
|
{
|
|
if (character is not '"')
|
|
{
|
|
field.Append(character);
|
|
continue;
|
|
}
|
|
|
|
// A doubled quote is an escaped quote, everything else ends the quoted field:
|
|
if (index + 1 < content.Length && content[index + 1] is '"')
|
|
{
|
|
field.Append('"');
|
|
index++;
|
|
continue;
|
|
}
|
|
|
|
isQuoted = false;
|
|
continue;
|
|
}
|
|
|
|
switch (character)
|
|
{
|
|
case '"':
|
|
isQuoted = true;
|
|
hasContent = true;
|
|
break;
|
|
|
|
case SEPARATOR:
|
|
hasContent = true;
|
|
EndField();
|
|
break;
|
|
|
|
case '\r':
|
|
break;
|
|
|
|
case '\n':
|
|
EndRow();
|
|
break;
|
|
|
|
default:
|
|
hasContent = true;
|
|
field.Append(character);
|
|
break;
|
|
}
|
|
}
|
|
|
|
if (hasContent || field.Length > 0)
|
|
EndRow();
|
|
|
|
return rows;
|
|
}
|
|
} |