From 54562095918582f51d4031700153f1730b015461 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Mon, 10 Aug 2026 16:12:33 +0200 Subject: [PATCH] Fixed the encoding detection to use the API of chardetng 1.0 --- runtime/src/file_data.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/runtime/src/file_data.rs b/runtime/src/file_data.rs index e0a59549..ff086643 100644 --- a/runtime/src/file_data.rs +++ b/runtime/src/file_data.rs @@ -9,7 +9,7 @@ use axum::extract::rejection::QueryRejection; use axum::response::sse::{Event, Sse}; use base64::{engine::general_purpose, Engine as _}; use calamine::{open_workbook_auto, Error as CalamineError, Reader}; -use chardetng::EncodingDetector; +use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection}; use encoding_rs::Encoding; use file_format::{FileFormat, Kind}; use futures::{Stream, StreamExt}; @@ -593,10 +593,15 @@ async fn read_text_file(file_path: &str) -> Result { ).into()); } - let mut detector = EncodingDetector::new(); + // + // Both options are about untrusted web content which may run scripts, which is not what we + // read here: these are local files the user picked, so allowing both guesses gives the better + // detection. + // + let mut detector = EncodingDetector::new(Iso2022JpDetection::Allow); detector.feed(&bytes, true); - let (text, encoding, had_errors) = detector.guess(None, true).decode(&bytes); + let (text, encoding, had_errors) = detector.guess(None, Utf8Detection::Allow).decode(&bytes); if had_errors { warn!("Decoding '{file_path}' as {name} replaced malformed sequences.", name = encoding.name()); } else {