Fixed text files not being read when they are not encoded in UTF-8

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -7708,6 +7708,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1290340974"] = "Unknown configur
 -- Copies the configuration slot to the clipboard
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1347508205"] = "Copies the configuration slot to the clipboard"
 
+-- Once the encoding of a text file is known, encoding_rs turns its content into the text AI Studio works with. Together with chardetng, this lets AI Studio read text, CSV, and similar files no matter which encoding they were saved in.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1378412877"] = "Once the encoding of a text file is known, encoding_rs turns its content into the text AI Studio works with. Together with chardetng, this lets AI Studio read text, CSV, and similar files no matter which encoding they were saved in."
+
 -- This library is used to read PDF files. This is necessary, e.g., for using PDFs as a data source for a chat.
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1388816916"] = "This library is used to read PDF files. This is necessary, e.g., for using PDFs as a data source for a chat."
 
@@ -7819,6 +7822,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T234598990"] = "Linux AppImages b
 -- Used PDFium version
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2368247719"] = "Used PDFium version"
 
+-- Text files are not always saved in the same encoding: files written on Windows often use a legacy one. chardetng recognizes which encoding a text file uses, so AI Studio can read it instead of rejecting it.
+UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T236832881"] = "Text files are not always saved in the same encoding: files written on Windows often use a legacy one. chardetng recognizes which encoding a text file uses, so AI Studio can read it instead of rejecting it."
+
 -- installation provided by the system
 UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2371107659"] = "installation provided by the system"
 
@@ -9082,9 +9088,6 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::EXTERNALHTTPCLIENTTIMEOUT::T599774443"] = "The
 -- policy files
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::EXTERNALHTTPCLIENTTIMEOUT::T632340680"] = "policy files"
 
--- No text could be read from the file '{0}', so it was not sent. The file might consist of scanned images without a text layer.
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T1268231046"] = "No text could be read from the file '{0}', so it was not sent. The file might consist of scanned images without a text layer."
-
 -- The file type of '{0}' could not be determined, so the file was not sent.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T1459702734"] = "The file type of '{0}' could not be determined, so the file was not sent."
 
@@ -9115,6 +9118,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T2793077828"]
 -- The file '{0}' is not a readable PDF and was not sent. It might be damaged or transferred incompletely.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T2891768359"] = "The file '{0}' is not a readable PDF and was not sent. It might be damaged or transferred incompletely."
 
+-- No text could be read from the file '{0}', so it was not sent. It might contain images only, such as a scanned PDF without a text layer, or no readable text at all.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T2897122009"] = "No text could be read from the file '{0}', so it was not sent. It might contain images only, such as a scanned PDF without a text layer, or no readable text at all."
+
 -- The file '{0}' is a {1}, which AI Studio cannot read, so it was not sent.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T3262447403"] = "The file '{0}' is a {1}, which AI Studio cannot read, so it was not sent."
 

app/MindWork AI Studio/Pages/Information.razor

@@ -340,6 +340,8 @@
                     <ThirdPartyComponent Name="windows-rs" Developer="Microsoft, Kenny Kerr, Ryan Levick, Rafael Rivera, sivadeilra, Marijn Suijten & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/microsoft/windows-rs/blob/master/license-mit" RepositoryUrl="https://github.com/microsoft/windows-rs" UseCase="@T("The windows-rs project provides access to Windows APIs from Rust. We use several libraries from this project: windows-registry is used to read the desired configuration in Windows enterprise environments. The windows and windows-collections libraries are used to open the native Windows share dialog, e.g., when you share a plugin with others.")"/>
                     <ThirdPartyComponent Name="objc2" Developer="Steven Sheldon, Mads Marquart, silvanshade, Dzmitry Malyshau, Felix Nemo Kaaman, adamnemecek, Samuel Sleight, Paul Mabileau & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/madsmtm/objc2/blob/main/LICENSE-MIT.txt" RepositoryUrl="https://github.com/madsmtm/objc2" UseCase="@T("The objc2 project provides access to Apple's Objective-C frameworks from Rust. On macOS, we use the libraries objc2, objc2-app-kit, and objc2-foundation to open the native macOS share sheet, e.g., when you share a plugin with others.")"/>
                     <ThirdPartyComponent Name="file-format" Developer="Mickaël Malécot & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/mmalecot/file-format/blob/main/LICENSE-MIT" RepositoryUrl="https://github.com/mmalecot/file-format" UseCase="@T("This library identifies files by their content. It is used for document streaming and as the first safety and media classification step before local audio processing.")"/>
+                    <ThirdPartyComponent Name="chardetng" Developer="Henri Sivonen & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/hsivonen/chardetng/blob/main/LICENSE-MIT" RepositoryUrl="https://github.com/hsivonen/chardetng" UseCase="@T("Text files are not always saved in the same encoding: files written on Windows often use a legacy one. chardetng recognizes which encoding a text file uses, so AI Studio can read it instead of rejecting it.")"/>
+                    <ThirdPartyComponent Name="encoding_rs" Developer="Henri Sivonen, M. Larsen, kornelski, Manish Goregaokar & Open Source Community" LicenseName="MIT & BSD-3-Clause" LicenseUrl="https://github.com/hsivonen/encoding_rs/blob/main/COPYRIGHT" RepositoryUrl="https://github.com/hsivonen/encoding_rs" UseCase="@T("Once the encoding of a text file is known, encoding_rs turns its content into the text AI Studio works with. Together with chardetng, this lets AI Studio read text, CSV, and similar files no matter which encoding they were saved in.")"/>
                     <ThirdPartyComponent Name="Symphonia" Developer="Philip Deljanov & Open Source Community" LicenseName="MPL-2.0" LicenseUrl="https://github.com/pdeljanov/Symphonia/blob/v0.6.0/LICENSE" RepositoryUrl="https://github.com/pdeljanov/Symphonia" UseCase="@T("Symphonia is used for media container demuxing and audio decoding. The exact MPL-covered source is available from the repository linked and is identified in the offline notices bundled with AI Studio.")"/>
                     <ThirdPartyComponent Name="Ropus" Developer="0x4D44, Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo, Mozilla, Amazon & Open Source Community" LicenseName="BSD-3-Clause" LicenseUrl="https://github.com/0x4D44/ropus/blob/main/LICENSE" RepositoryUrl="https://github.com/0x4d44/ropus" UseCase="@T("Ropus provides the Opus encoder and decoder used by the media pipeline.")"/>
                     <ThirdPartyComponent Name="Rubato" Developer="Henrik Enquist & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/HEnquist/rubato/blob/master/LICENSE-MIT" RepositoryUrl="https://github.com/HEnquist/rubato" UseCase="@T("We use Rubato to resample the decoded audio to 48 kHz before the Opus encoding.")"/>

app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua

@@ -9084,9 +9084,6 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::EXTERNALHTTPCLIENTTIMEOUT::T599774443"] = "Das
 -- policy files
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::EXTERNALHTTPCLIENTTIMEOUT::T632340680"] = "Richtliniendateien"
 
--- No text could be read from the file '{0}', so it was not sent. The file might consist of scanned images without a text layer.
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T1268231046"] = "Aus der Datei „{0}“ konnte kein Text gelesen werden, daher wurde sie nicht gesendet. Die Datei könnte aus gescannten Bildern ohne Textebene bestehen."
-
 -- The file type of '{0}' could not be determined, so the file was not sent.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T1459702734"] = "Der Dateityp von „{0}“ konnte nicht bestimmt werden. Daher wurde die Datei nicht gesendet."
 
@@ -9117,6 +9114,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T2793077828"]
 -- The file '{0}' is not a readable PDF and was not sent. It might be damaged or transferred incompletely.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T2891768359"] = "Die Datei „{0}“ ist keine lesbare PDF-Datei und wurde nicht gesendet. Sie ist möglicherweise beschädigt oder wurde unvollständig übertragen."
 
+-- No text could be read from the file '{0}', so it was not sent. It might contain images only, such as a scanned PDF without a text layer, or no readable text at all.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T2897122009"] = "Aus der Datei „{0}“ konnte kein Text gelesen werden, daher wurde sie nicht gesendet. Möglicherweise enthält sie nur Bilder, etwa ein gescanntes PDF ohne Textebene, oder gar keinen lesbaren Text."
+
 -- The file '{0}' is a {1}, which AI Studio cannot read, so it was not sent.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T3262447403"] = "Die Datei „{0}“ ist eine {1}, die AI Studio nicht lesen kann. Daher wurde sie nicht gesendet."
 

app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua

@@ -9084,9 +9084,6 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::EXTERNALHTTPCLIENTTIMEOUT::T599774443"] = "The
 -- policy files
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::EXTERNALHTTPCLIENTTIMEOUT::T632340680"] = "policy files"
 
--- No text could be read from the file '{0}', so it was not sent. The file might consist of scanned images without a text layer.
-UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T1268231046"] = "No text could be read from the file '{0}', so it was not sent. The file might consist of scanned images without a text layer."
-
 -- The file type of '{0}' could not be determined, so the file was not sent.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T1459702734"] = "The file type of '{0}' could not be determined, so the file was not sent."
 
@@ -9117,6 +9114,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T2793077828"]
 -- The file '{0}' is not a readable PDF and was not sent. It might be damaged or transferred incompletely.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T2891768359"] = "The file '{0}' is not a readable PDF and was not sent. It might be damaged or transferred incompletely."
 
+-- No text could be read from the file '{0}', so it was not sent. It might contain images only, such as a scanned PDF without a text layer, or no readable text at all.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T2897122009"] = "No text could be read from the file '{0}', so it was not sent. It might contain images only, such as a scanned PDF without a text layer, or no readable text at all."
+
 -- The file '{0}' is a {1}, which AI Studio cannot read, so it was not sent.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONRESULTEXTENSIONS::T3262447403"] = "The file '{0}' is a {1}, which AI Studio cannot read, so it was not sent."
 

app/MindWork AI Studio/Tools/FileExtractionResultExtensions.cs

@@ -81,7 +81,7 @@ internal static class FileExtractionResultExtensions
         FileExtractionErrorCode.PDF_ENCRYPTED => TB("The file '{0}' is protected and could not be opened, so it was not sent."),
         FileExtractionErrorCode.PDFIUM_UNAVAILABLE => TB("AI Studio was not able to start its PDF engine, so the file '{0}' was not sent."),
         FileExtractionErrorCode.PANDOC_UNAVAILABLE => TB("Reading the file '{0}' needs Pandoc, which is not available, so the file was not sent."),
-        FileExtractionErrorCode.NO_TEXT_EXTRACTED => TB("No text could be read from the file '{0}', so it was not sent. The file might consist of scanned images without a text layer."),
+        FileExtractionErrorCode.NO_TEXT_EXTRACTED => TB("No text could be read from the file '{0}', so it was not sent. It might contain images only, such as a scanned PDF without a text layer, or no readable text at all."),
         FileExtractionErrorCode.NO_CONTENT => TB("The file '{0}' did not provide any content and was not sent."),
 
         FileExtractionErrorCode.NOT_TEXT_CONTENT => TB("The file '{0}' is not a text file and was not sent. Its content could not be read as text, so it might have a wrong file extension."),

runtime/Cargo.lock

@@ -1251,6 +1251,17 @@ dependencies = [
  "whatlang",
 ]
 
+[[package]]
+name = "chardetng"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "13de944a44b5064ee5d3a5ceccc49a41bfec50f2580e66f82e87703acdb88b53"
+dependencies = [
+ "cfg-if",
+ "encoding_rs",
+ "memchr",
+]
+
 [[package]]
 name = "chrono"
 version = "0.4.44"
@@ -2160,9 +2171,9 @@ checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
 
 [[package]]
 name = "encoding_rs"
-version = "0.8.34"
+version = "0.8.35"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b45de904aa0b010bce2ab45264d0631681847fa7b6f2eaa7dab7619943bc4f59"
+checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3"
 dependencies = [
  "cfg-if",
 ]
@@ -4062,7 +4073,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
 dependencies = [
  "cfg-if",
- "windows-targets 0.48.5",
+ "windows-targets 0.52.6",
 ]
 
 [[package]]
@@ -4249,9 +4260,11 @@ dependencies = [
  "calamine",
  "cbc 0.2.1",
  "cfg-if",
+ "chardetng",
  "dbus-secret-service",
  "dbus-secret-service-keyring-store",
  "dirs",
+ "encoding_rs",
  "file-format",
  "flexi_logger",
  "futures",

runtime/Cargo.toml

@@ -46,6 +46,12 @@ rcgen = { version = "0.14.8", features = ["pem"] }
 # is only detected as a plain archive.
 file-format = { version = "0.29.0", features = ["reader-zip", "reader-cfb", "reader-txt", "reader-exe"] }
 
+# Text files are not always UTF-8: on Windows they are frequently encoded in Windows-1252, whose
+# umlauts are single bytes and therefore invalid UTF-8. chardetng guesses the encoding, encoding_rs
+# decodes it.
+chardetng = "1.0.0"
+encoding_rs = "0.8.35"
+
 symphonia = { version = "0.6", default-features = false, features = ["aac", "aiff", "alac", "caf", "flac", "isomp4", "mkv", "mp1", "mp2", "mp3", "ogg", "pcm", "vorbis", "wav"] }
 ropus = "=0.12.18"
 rubato = { version = "4", default-features = false, features = ["fft_resampler"] }

runtime/src/file_data.rs

@@ -9,6 +9,8 @@ use axum::extract::rejection::QueryRejection;
 use axum::response::sse::{Event, Sse};
 use base64::{engine::general_purpose, Engine as _};
 use calamine::{open_workbook_auto, Error as CalamineError, Reader};
+use chardetng::EncodingDetector;
+use encoding_rs::Encoding;
 use file_format::{FileFormat, Kind};
 use futures::{Stream, StreamExt};
 use pdfium_render::prelude::{Pdfium, PdfiumError, PdfiumInternalError};
@@ -19,7 +21,7 @@ use std::path::Path;
 use std::pin::Pin;
 use std::fmt;
 use log::{debug, error, warn};
-use tokio::io::{AsyncBufReadExt, AsyncReadExt};
+use tokio::io::AsyncReadExt;
 use tokio::sync::mpsc;
 use tokio_stream::wrappers::ReceiverStream;
 
@@ -554,14 +556,59 @@ async fn stream_data(file_path: &str, extract_images: bool) -> Result<ChunkStrea
     Ok(Box::pin(stream))
 }
 
-async fn stream_text_file(file_path: &str, use_md_fences: bool, fence_language: Option<String>) -> Result<ChunkStream> {
+/// How many bytes we inspect for NUL bytes to tell binary content from text.
-    let file = tokio::fs::File::open(file_path).await?;
+const BINARY_PROBE_SIZE: usize = 8_192;
-    let reader = tokio::io::BufReader::new(file);
-    let mut lines = reader.lines();
-    let mut line_number = 0;
 
-    // The stream outlives this call, so it needs its own copy of the path for diagnostics:
+/// Reads a text file and decodes it, no matter which encoding it uses.
-    let path = file_path.to_owned();
+///
+/// Insisting on UTF-8 is not enough in practice: text files written on Windows are frequently
+/// encoded in Windows-1252, where umlauts are single bytes which UTF-8 rejects. Such a file used
+/// to look like it was not text at all.
+async fn read_text_file(file_path: &str) -> Result<String> {
+    let bytes = tokio::fs::read(file_path).await.map_err(|error| ExtractionError::new(
+        classify_io_error(&error),
+        format!("The file could not be read: {error}"),
+    ))?;
+
+    //
+    // A byte order mark is authoritative and also covers UTF-16, which the detector below does not
+    // recognize. We therefore check it first and let `decode` act on it.
+    //
+    if let Some((encoding, _)) = Encoding::for_bom(&bytes) {
+        let (text, _, _) = encoding.decode(&bytes);
+        debug!("Decoded '{file_path}' as {name}, chosen by its byte order mark.", name = encoding.name());
+        return Ok(text.into_owned());
+    }
+
+    //
+    // Without a byte order mark, every byte sequence decodes into *something*, so the decoder can
+    // no longer tell us that a file is binary. NUL bytes do: they do not occur in text, and after
+    // the check above no UTF-16 file can reach this point.
+    //
+    let probe_length = min(bytes.len(), BINARY_PROBE_SIZE);
+    if bytes[..probe_length].contains(&0) {
+        return Err(ExtractionError::new(
+            ExtractionErrorCode::NotTextContent,
+            "The file contains binary data and is not a text file.",
+        ).into());
+    }
+
+    let mut detector = EncodingDetector::new();
+    detector.feed(&bytes, true);
+
+    let (text, encoding, had_errors) = detector.guess(None, true).decode(&bytes);
+    if had_errors {
+        warn!("Decoding '{file_path}' as {name} replaced malformed sequences.", name = encoding.name());
+    } else {
+        debug!("Decoded '{file_path}' as {name}.", name = encoding.name());
+    }
+
+    Ok(text.into_owned())
+}
+
+async fn stream_text_file(file_path: &str, use_md_fences: bool, fence_language: Option<String>) -> Result<ChunkStream> {
+    let text = read_text_file(file_path).await?;
+    let mut line_number = 0;
 
     let stream = stream! {
 
@@ -581,35 +628,12 @@ async fn stream_text_file(file_path: &str, use_md_fences: bool, fence_language:
             };
         }
 
-        loop {
+        for line in text.lines() {
-            match lines.next_line().await {
+            line_number += 1;
-                Ok(Some(line)) => {
+            yield Ok(Chunk::new(
-                    line_number += 1;
+                line.to_string(),
-                    yield Ok(Chunk::new(
+                Metadata::Text { line_number }
-                        line,
+            ));
-                        Metadata::Text { line_number }
-                    ));
-                },
-
-                Ok(None) => break,
-
-                //
-                // Reading a line fails when the bytes are not valid UTF-8, which means the file is
-                // not text at all. Treating that like the end of the file, as this loop did before,
-                // turned every binary file into an empty document without a word.
-                //
-                Err(e) => {
-                    let code = if e.kind() == std::io::ErrorKind::InvalidData {
-                        ExtractionErrorCode::NotTextContent
-                    } else {
-                        classify_io_error(&e)
-                    };
-
-                    error!("Reading '{path}' as text failed after {line_number} line(s) ({code:?}): {e}");
-                    yield Err(ExtractionError::new(code, format!("The file could not be read as text: {e}")).into());
-                    break;
-                },
-            }
         }
 
         if use_md_fences {
@@ -864,21 +888,44 @@ async fn convert_with_pandoc(
         .with_output_format(to)
         .build()
         .command.output().await?;
-    
+
+    let exit_code = output.status.code();
+    let stderr_text = String::from_utf8_lossy(&output.stderr).trim().to_string();
+    debug!("Pandoc converted '{file_path}' from '{from}' to '{to}': exit={exit_code:?}, {stdout_length} byte(s) of output.", stdout_length = output.stdout.len());
+
+    if !stderr_text.is_empty() {
+        warn!("Pandoc reported while converting '{file_path}': {stderr_text}");
+    }
+
     let stream = stream! {
-        if output.status.success() {
+        if !output.status.success() {
-            match String::from_utf8(output.stdout.clone()) {
+            yield Err(ExtractionError::new(
+                ExtractionErrorCode::Internal,
+                format!("Pandoc failed with exit code {exit_code:?}: {stderr_text}"),
+            ).into());
+        } else {
+            match String::from_utf8(output.stdout) {
+                //
+                // Pandoc succeeded, yet nothing came out. Passing that on as content would hand an
+                // empty document to the AI, which is exactly what this whole path must not do.
+                //
+                Ok(content) if content.trim().is_empty() => {
+                    yield Err(ExtractionError::new(
+                        ExtractionErrorCode::NoTextExtracted,
+                        format!("Pandoc read the file without finding any text{separator}{stderr_text}", separator = if stderr_text.is_empty() { "." } else { ": " }),
+                    ).into());
+                },
+
                 Ok(content) => yield Ok(Chunk::new(
                     content,
                     Metadata::Document {}
                 )),
-                Err(e) => yield Err(e.into()),
+
+                Err(e) => yield Err(ExtractionError::new(
+                    ExtractionErrorCode::Internal,
+                    format!("The output of Pandoc was not valid UTF-8: {e}"),
+                ).into()),
             }
-        } else {
-            yield Err(format!(
-                "Pandoc error: {}",
-                String::from_utf8_lossy(&output.stderr)
-            ).into());
         }
     };