MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-08-11 19:52:10 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Fixed the encoding detection to use the API of chardetng 1.0

Browse Source
This commit is contained in:
Thorsten Sommer 2026-08-10 16:12:33 +02:00
parent 0efbbc9f22
commit 5456209591
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
1 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
runtime/src/file_data.rs
View File

@ -9,7 +9,7 @@ use axum::extract::rejection::QueryRejection;
use axum::response::sse::{Event, Sse}; use axum::response::sse::{Event, Sse};
use base64::{engine::general_purpose, Engine as _}; use base64::{engine::general_purpose, Engine as _};
use calamine::{open_workbook_auto, Error as CalamineError, Reader}; use calamine::{open_workbook_auto, Error as CalamineError, Reader};
use chardetng::EncodingDetector; use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection};
use encoding_rs::Encoding; use encoding_rs::Encoding;
use file_format::{FileFormat, Kind}; use file_format::{FileFormat, Kind};
use futures::{Stream, StreamExt}; use futures::{Stream, StreamExt};
@ -593,10 +593,15 @@ async fn read_text_file(file_path: &str) -> Result<String> {
).into()); ).into());
} }
let mut detector = EncodingDetector::new(); //
// Both options are about untrusted web content which may run scripts, which is not what we
// read here: these are local files the user picked, so allowing both guesses gives the better
// detection.
//
let mut detector = EncodingDetector::new(Iso2022JpDetection::Allow);
detector.feed(&bytes, true); detector.feed(&bytes, true);
let (text, encoding, had_errors) = detector.guess(None, true).decode(&bytes); let (text, encoding, had_errors) = detector.guess(None, Utf8Detection::Allow).decode(&bytes);
if had_errors { if had_errors {
warn!("Decoding '{file_path}' as {name} replaced malformed sequences.", name = encoding.name()); warn!("Decoding '{file_path}' as {name} replaced malformed sequences.", name = encoding.name());
} else { } else {