using System.Text; namespace AIStudio.Assistants.BatchProcessing; /// /// Reads and writes the CSV files of the batch processing assistant. Fields /// are quoted according to RFC 4180 using the separator selected for the /// respective file. /// public static class BatchProcessingCsv { public static string ToCsvRow(char separator, params string[] fields) => string.Join(separator, fields.Select(field => ToCsvField(field, separator))); /// /// Quotes one CSV field according to RFC 4180. /// private static string ToCsvField(string text, char separator) { if (string.IsNullOrEmpty(text)) return string.Empty; // Quoting the complete field is important for long and multi-line AI // answers: neither separators nor line breaks within an answer may // create another column or row. if (!text.Contains(separator) && !text.Contains('"') && !text.Contains('\n') && !text.Contains('\r')) return text; return $""" "{text.Replace("\"", "\"\"")}" """; } /// /// Parses a CSV text which was written by . /// /// /// We parse the file ourselves instead of splitting lines, because quoted /// fields may contain the separator and line breaks. /// private static List> Parse(string content, char separator) { var rows = new List>(); var fields = new List(); var field = new StringBuilder(); var isQuoted = false; var hasContent = false; for (var index = 0; index < content.Length; index++) { var character = content[index]; if (isQuoted) { if (character is not '"') { field.Append(character); continue; } // A doubled quote is an escaped quote, everything else ends the quoted field: if (index + 1 < content.Length && content[index + 1] is '"') { field.Append('"'); index++; continue; } isQuoted = false; continue; } switch (character) { case '"': isQuoted = true; hasContent = true; break; case var _ when character == separator: hasContent = true; EndField(); break; case '\r': break; case '\n': EndRow(); break; default: hasContent = true; field.Append(character); break; } } if (hasContent || field.Length > 0) EndRow(); return rows; void EndField() { fields.Add(field.ToString()); field.Clear(); } void EndRow() { EndField(); if (hasContent) rows.Add([..fields]); fields.Clear(); hasContent = false; } } /// /// Detects the separator from the first CSV record and parses the complete /// content with it. Preferred separators are used as fallbacks for files /// whose first record does not reveal a valid separator. /// public static List> ParseWithDetectedSeparator(string content, int expectedNumFields, params char[] preferredSeparators) { var firstRecord = ReadFirstRecord(content); var candidates = new List(); var isQuoted = false; for (var index = 0; index < firstRecord.Length; index++) { var character = firstRecord[index]; if (character is '"') { if (isQuoted && index + 1 < firstRecord.Length && firstRecord[index + 1] is '"') { index++; continue; } isQuoted = !isQuoted; continue; } if (!isQuoted && character is not '\r' and not '\n' && (char.IsPunctuation(character) || char.IsSymbol(character) || character is '\t') && !candidates.Contains(character)) candidates.Add(character); } foreach (var separator in preferredSeparators) { if (!candidates.Contains(separator)) candidates.Add(separator); } foreach (var separator in candidates) { var header = Parse(firstRecord, separator); if (header.Count is 1 && header[0].Count == expectedNumFields) return Parse(content, separator); } throw new InvalidDataException("Was not able to detect the CSV separator."); } private static string ReadFirstRecord(string content) { var isQuoted = false; for (var index = 0; index < content.Length; index++) { if (content[index] is '"') { if (isQuoted && index + 1 < content.Length && content[index + 1] is '"') { index++; continue; } isQuoted = !isQuoted; } else if (content[index] is '\n' && !isQuoted) return content[..(index + 1)]; } return content; } }