current main is now successfully merged into branch

runtime/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "mindwork-ai-studio"
-version = "26.4.1"
+version = "26.5.5"
 edition = "2024"
 description = "MindWork AI Studio"
 authors = ["Thorsten Sommer"]
@@ -48,6 +48,7 @@ tempfile = "3.27.0"
 strum_macros = "0.28.0"
 sysinfo = "0.39.1"
 bytes = "1.11.1"
+tokenizers = "0.23.1"
 
 [target.'cfg(target_os = "windows")'.dependencies]
 windows-registry = "0.6.1"

runtime/src/app_window.rs

@@ -28,7 +28,7 @@ use crate::pdfium::PDFIUM_LIB_PATH;
 use crate::qdrant::{start_qdrant_server, stop_qdrant_server};
 #[cfg(debug_assertions)]
 use crate::dotnet::create_startup_env_file;
-use crate::tokenizer::set_path_resolver;
+use crate::tokenizer::set_default_tokenizer_path;
 
 /// The Tauri main window.
 pub static MAIN_WINDOW: Lazy<Mutex<Option<WebviewWindow>>> = Lazy::new(|| Mutex::new(None));
@@ -151,7 +151,7 @@ pub fn start_tauri() {
 
             start_qdrant_server(app.handle().clone());
 
-            set_path_resolver(app.path_resolver());
+            set_default_tokenizer_path(app.handle().clone());
 
             info!(Source = "Bootloader Tauri"; "Reconfigure the file logger to use the app data directory {data_path:?}");
             switch_to_file_logging(data_path).map_err(|e| error!("Failed to switch logging to file: {e}")).unwrap();

runtime/src/runtime_api.rs

@@ -33,6 +33,7 @@ pub fn start_runtime_api() {
         .route("/system/dotnet/port", get(crate::dotnet::dotnet_port))
         .route("/system/dotnet/ready", get(crate::dotnet::dotnet_ready))
         .route("/system/qdrant/info", get(crate::qdrant::qdrant_port))
+        .route("/system/tokenizer/info", get(crate::tokenizer::tokenizer_info))
         .route("/clipboard/set", post(crate::clipboard::set_clipboard))
         .route("/events", get(crate::app_window::get_event_stream))
         .route("/updates/check", get(crate::app_window::check_for_update))
@@ -56,6 +57,11 @@ pub fn start_runtime_api() {
         .route("/retrieval/fs/extract", get(crate::file_data::extract_data))
         .route("/log/paths", get(crate::log::get_log_paths))
         .route("/log/event", post(crate::log::log_event))
+        .route("/tokenizer/count", post(crate::tokenizer::token_count))
+        .route("/tokenizer/validate", post(crate::tokenizer::validate_tokenizer))
+        .route("/tokenizer/store", post(crate::tokenizer::store_tokenizer))
+        .route("/tokenizer/delete", post(crate::tokenizer::delete_tokenizer))
+        .route("/tokenizer/set", post(crate::tokenizer::set_tokenizer))
         .route("/shortcuts/register", post(crate::app_window::register_shortcut))
         .route("/shortcuts/validate", post(crate::app_window::validate_shortcut))
         .route("/shortcuts/suspend", post(crate::app_window::suspend_shortcuts))

runtime/src/tokenizer.rs

@@ -1,24 +1,43 @@
 use std::fs;
 use std::path::PathBuf;
-use std::sync::{OnceLock, RwLock};
-use log::warn;
-use rocket::post;
-use rocket::serde::json::Json;
-use rocket::serde::Serialize;
-use serde::Deserialize;
-use tauri::PathResolver;
-use tokenizers::Error;
-use tokenizers::tokenizer::{Tokenizer, Error as TokenizerError};
+use std::sync::{Mutex, OnceLock, RwLock};
+
+use axum::Json;
+use log::{error, warn};
+use once_cell::sync::Lazy;
+use serde::{Deserialize, Serialize};
+use tauri::path::BaseDirectory;
+use tauri::Manager;
+use tokenizers::tokenizer::Tokenizer;
+
 use crate::api_token::APIToken;
 use crate::environment::DATA_DIRECTORY;
 
-static TOKENIZER: OnceLock<RwLock<Option<Tokenizer>>> = OnceLock::new();
+const DEFAULT_TOKENIZER_RESOURCE_PATH: &str = "resources/tokenizers/tokenizer.json";
+const NO_TOKENIZER_LOADED_MESSAGE: &str = "Tokenizer must be set before counting tokens.";
 
-static TOKENIZER_PATH_RESOLVER: OnceLock<PathResolver> = OnceLock::new();
+static TOKENIZER: OnceLock<RwLock<Option<Tokenizer>>> = OnceLock::new();
+static DEFAULT_TOKENIZER_PATH: OnceLock<PathBuf> = OnceLock::new();
+static TOKENIZER_STATUS: Lazy<Mutex<TokenizerStatusInfo>> = Lazy::new(|| Mutex::new(TokenizerStatusInfo::default()));
+static TOKENIZER_OPERATION_LOCK: Lazy<Mutex<()>> = Lazy::new(|| Mutex::new(()));
+
+#[derive(Clone, Copy, Default, Serialize, PartialEq, Eq)]
+pub enum TokenizerStatus {
+    #[default]
+    Unavailable,
+    Running,
+    Available,
+}
+
+#[derive(Default)]
+struct TokenizerStatusInfo {
+    status: TokenizerStatus,
+    unavailable_reason: Option<String>,
+}
 
 #[derive(Deserialize)]
 pub struct SetTokenText {
-    pub text: String,
+    text: String,
 }
 
 #[derive(Clone, Deserialize)]
@@ -42,94 +61,205 @@ pub struct TokenizerResponse {
     success: bool,
     token_count: usize,
     message: String,
+    status: TokenizerStatus,
+    stored_path: String,
 }
 
-impl From<Result<usize, TokenizerError>> for TokenizerResponse {
-    fn from(result: Result<usize, TokenizerError>) -> Self {
-        match result {
-            Ok(count) => TokenizerResponse {
-                success: true,
-                token_count: count,
-                message: "Success".to_string(),
-            },
-            Err(e) => TokenizerResponse {
-                success: false,
-                token_count: 0,
-                message: e.to_string(),
-            },
+impl TokenizerResponse {
+    fn available(token_count: usize) -> Self {
+        TokenizerResponse {
+            success: true,
+            token_count,
+            message: String::new(),
+            status: TokenizerStatus::Available,
+            stored_path: String::new(),
+        }
+    }
+
+    fn stored(stored_path: String) -> Self {
+        TokenizerResponse {
+            success: true,
+            token_count: 0,
+            message: String::new(),
+            status: TokenizerStatus::Available,
+            stored_path,
+        }
+    }
+
+    fn unavailable(reason: String) -> Self {
+        TokenizerResponse {
+            success: false,
+            token_count: 0,
+            message: reason,
+            status: TokenizerStatus::Unavailable,
+            stored_path: String::new(),
         }
     }
 }
 
-pub fn set_path_resolver(path_resolver: PathResolver) {
-    match TOKENIZER_PATH_RESOLVER.set(path_resolver) {
+pub fn set_default_tokenizer_path(app_handle: tauri::AppHandle) {
+    let tokenizer_path = match app_handle
+        .path()
+        .resolve(DEFAULT_TOKENIZER_RESOURCE_PATH, BaseDirectory::Resource)
+    {
+        Ok(path) => path,
+        Err(e) => {
+            let reason = format!("The default tokenizer file '{DEFAULT_TOKENIZER_RESOURCE_PATH}' could not be resolved: {e}");
+            error!(Source = "Tokenizer"; "{reason}");
+            set_tokenizer_unavailable(reason);
+            return;
+        }
+    };
+
+    if !tokenizer_path.is_file() {
+        let reason = format!("The default tokenizer file was not found: {}", tokenizer_path.display());
+        error!(Source = "Tokenizer"; "{reason}");
+        set_tokenizer_unavailable(reason);
+        return;
+    }
+
+    match DEFAULT_TOKENIZER_PATH.set(tokenizer_path) {
         Ok(_) => (),
-        Err(e) => warn!(Source = "Tokenizer"; "Could not set the path resolver: {:?}", e),
+        Err(e) => warn!(Source = "Tokenizer"; "Could not set the default tokenizer path: {:?}", e),
     }
 }
 
-fn tokenizer_state() -> &'static RwLock<Option<Tokenizer>> {
-    TOKENIZER.get_or_init(|| RwLock::new(None))
+pub async fn tokenizer_info(_token: APIToken) -> Json<TokenizerResponse> {
+    let status = TOKENIZER_STATUS.lock().unwrap();
+    match status.status {
+        TokenizerStatus::Available => Json(TokenizerResponse::available(0)),
+        TokenizerStatus::Running => Json(TokenizerResponse {
+            success: false,
+            token_count: 0,
+            message: String::new(),
+            status: TokenizerStatus::Running,
+            stored_path: String::new(),
+        }),
+        TokenizerStatus::Unavailable => Json(TokenizerResponse::unavailable(status.unavailable_reason.clone().unwrap_or_default())),
+    }
 }
 
-pub fn handle_tokenizer_set(path: &str) -> Result<(), Error> {
-    let tokenizer_path = if path.trim().is_empty() {
-        let relative_source_path = String::from("resources/tokenizers/tokenizer.json");
-        let path_resolver = TOKENIZER_PATH_RESOLVER
-            .get()
-            .ok_or_else(|| Error::from("Tokenizer path resolver is not initialized"))?;
-        path_resolver
-            .resolve_resource(relative_source_path)
-            .ok_or_else(|| Error::from("Failed to resolve default tokenizer resource path"))?
-    } else {
-        PathBuf::from(path)
-    };
+pub async fn token_count(_token: APIToken, req: Json<SetTokenText>) -> Json<TokenizerResponse> {
+    match get_token_count(&req.text) {
+        Ok(count) => Json(TokenizerResponse::available(count)),
+        Err(e) => Json(TokenizerResponse::unavailable(e)),
+    }
+}
 
-    let tokenizer = Tokenizer::from_file(tokenizer_path)?;
-    let mut tokenizer_guard = tokenizer_state()
-        .write()
-        .map_err(|_| Error::from("Tokenizer state lock is poisoned"))?;
-    *tokenizer_guard = Some(tokenizer);
+pub async fn validate_tokenizer(_token: APIToken, payload: Json<TokenizerPath>) -> Json<TokenizerResponse> {
+    match handle_tokenizer_validate(&PathBuf::from(payload.file_path.clone())) {
+        Ok(count) => Json(TokenizerResponse::available(count)),
+        Err(e) => Json(TokenizerResponse::unavailable(e)),
+    }
+}
 
+pub async fn store_tokenizer(_token: APIToken, payload: Json<TokenizerStorage>) -> Json<TokenizerResponse> {
+    match handle_tokenizer_store(&payload) {
+        Ok(dest_path) => Json(TokenizerResponse::stored(dest_path)),
+        Err(e) => Json(TokenizerResponse::unavailable(e.to_string())),
+    }
+}
+
+pub async fn delete_tokenizer(_token: APIToken, payload: Json<TokenizerDelete>) -> Json<TokenizerResponse> {
+    match handle_tokenizer_delete(&payload) {
+        Ok(_) => Json(TokenizerResponse::stored(String::new())),
+        Err(e) => Json(TokenizerResponse::unavailable(e.to_string())),
+    }
+}
+
+pub async fn set_tokenizer(_token: APIToken, payload: Json<TokenizerPath>) -> Json<TokenizerResponse> {
+    match handle_tokenizer_set(&payload.file_path) {
+        Ok(_) => Json(TokenizerResponse::available(0)),
+        Err(e) => Json(TokenizerResponse::unavailable(e)),
+    }
+}
+
+pub fn handle_tokenizer_set(path: &str) -> Result<(), String> {
+    let _operation_guard = begin_tokenizer_operation()?;
+    set_tokenizer_running();
+
+    let tokenizer_path = resolve_tokenizer_path(path).map_err(|e| {
+        error!(Source = "Tokenizer"; "{e} Starting the app without a tokenizer.");
+        unavailable_with_status_update(&e)
+    })?;
+
+    let tokenizer = load_tokenizer_from_file(&tokenizer_path).map_err(|e| {
+        error!(Source = "Tokenizer"; "{e}");
+        unavailable_with_status_update(&e)
+    })?;
+
+    match tokenizer_state().write() {
+        Ok(mut tokenizer_guard) => *tokenizer_guard = Some(tokenizer),
+        Err(_) => return Err(unavailable_with_status_update("Tokenizer state lock is poisoned.")),
+    }
+
+    set_tokenizer_available();
     Ok(())
 }
 
-fn handle_tokenizer_validate(path: &PathBuf) -> Result<usize, TokenizerError> {
-    if !path.is_file() {
-        return Err(TokenizerError::from(format!(
-            "Tokenizer file was not found: {}",
-            path.display()
-        )));
+fn handle_tokenizer_validate(path: &PathBuf) -> Result<usize, String> {
+    let _operation_guard = begin_tokenizer_operation()?;
+    set_tokenizer_running();
+
+    let result = validate_tokenizer_file(path);
+    match tokenizer_state().read() {
+        Ok(tokenizer_guard) if tokenizer_guard.is_some() => set_tokenizer_available(),
+        Ok(_) => set_tokenizer_unavailable(NO_TOKENIZER_LOADED_MESSAGE.to_string()),
+        Err(_) => set_tokenizer_unavailable("Tokenizer state lock is poisoned.".to_string()),
     }
 
-    let tokenizer = Tokenizer::from_file(path).map_err(|e| {
-        TokenizerError::from(format!(
-            "Failed to load tokenizer from '{}': {}",
-            path.display(),
-            e
-        ))
-    })?;
+    result
+}
 
+pub fn get_token_count(text: &str) -> Result<usize, String> {
+    if text.trim().is_empty() {
+        return Ok(0);
+    }
+
+    let _operation_guard = begin_tokenizer_operation()?;
+    {
+        let status = TOKENIZER_STATUS.lock().unwrap();
+        if status.status != TokenizerStatus::Available {
+            return Err(status.unavailable_reason.clone().unwrap_or_else(|| NO_TOKENIZER_LOADED_MESSAGE.to_string()));
+        }
+    }
+
+    let tokenizer_guard = tokenizer_state()
+        .read()
+        .map_err(|_| unavailable_with_status_update("Tokenizer state lock is poisoned."))?;
+    let tokenizer = match tokenizer_guard.as_ref() {
+        Some(tokenizer) => tokenizer,
+        None => {
+            drop(tokenizer_guard);
+            return Err(unavailable_with_status_update("Tokenizer not initialized."));
+        }
+    };
+    let token_count = match tokenizer.encode(text, true) {
+        Ok(enc) => enc.len(),
+        Err(e) => {
+            let reason = format!("Failed to tokenize text: {e}");
+            drop(tokenizer_guard);
+            return Err(unavailable_with_status_update(&reason));
+        }
+    };
+
+    Ok(token_count)
+}
+
+fn validate_tokenizer_file(path: &PathBuf) -> Result<usize, String> {
+    let tokenizer = load_tokenizer_from_file(path)?;
     let test_string = "Hello, world! This is a test string for tokenizer validation.";
-
-    let encoding = tokenizer.encode(test_string, true).map_err(|e| {
-        TokenizerError::from(format!(
-            "Tokenizer failed to encode validation string: {}",
-            e
-        ))
-    })?;
+    let encoding = tokenizer
+        .encode(test_string, true)
+        .map_err(|e| format!("Tokenizer failed to encode validation string: {e}"))?;
     let token_count = encoding.len();
 
     if token_count == 0 {
-        return Err(TokenizerError::from(
-            "Tokenizer produced 0 tokens for test string. The tokenizer is likely invalid or misconfigured.",
-        ));
+        return Err("Tokenizer produced 0 tokens for test string. The tokenizer is likely invalid or misconfigured.".to_string());
     }
 
     if encoding.get_tokens().iter().any(|t| t.is_empty()) {
-        return Err(TokenizerError::from(
-            "Tokenizer produced empty tokens. The tokenizer is invalid.",
-        ));
+        return Err("Tokenizer produced empty tokens. The tokenizer is invalid.".to_string());
     }
 
     Ok(token_count)
@@ -147,26 +277,25 @@ fn handle_tokenizer_store(payload: &TokenizerStorage) -> Result<String, std::io:
         .file_name()
         .and_then(|n| n.to_str())
         .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid tokenizer file path"))?;
-    let model_path = &base_path.join(&payload.model_id);
-    let destination_path = &model_path.join(source_name);
+    let model_path = base_path.join(&payload.model_id);
+    let destination_path = model_path.join(source_name);
 
-    if source_path.eq(destination_path) {
-        return Ok(destination_path.to_str().unwrap().to_string());
+    if source_path.eq(&destination_path) {
+        return Ok(destination_path.to_string_lossy().to_string());
     }
 
-    match model_path.try_exists()? {
-        true => fs::remove_dir_all(model_path.clone())?,
-        false => (),
+    if model_path.try_exists()? {
+        fs::remove_dir_all(&model_path)?;
     }
 
     if payload.file_path.trim().is_empty() {
-        return Ok(String::from(""));
+        return Ok(String::new());
     }
-    fs::create_dir_all(model_path)?;
 
+    fs::create_dir_all(&model_path)?;
     fs::copy(&source_path, &destination_path)?;
 
-    Ok(destination_path.to_str().unwrap().to_string())
+    Ok(destination_path.to_string_lossy().to_string())
 }
 
 fn handle_tokenizer_delete(payload: &TokenizerDelete) -> Result<(), std::io::Error> {
@@ -189,74 +318,61 @@ fn handle_tokenizer_delete(payload: &TokenizerDelete) -> Result<(), std::io::Err
     Ok(())
 }
 
-pub fn get_token_count(text: &str) -> Result<usize, TokenizerError> {
-    if text.trim().is_empty() {
-        return Err(TokenizerError::from("Input text is empty"));
+fn tokenizer_state() -> &'static RwLock<Option<Tokenizer>> {
+    TOKENIZER.get_or_init(|| RwLock::new(None))
+}
+
+fn begin_tokenizer_operation() -> Result<std::sync::MutexGuard<'static, ()>, String> {
+    TOKENIZER_OPERATION_LOCK
+        .lock()
+        .map_err(|_| unavailable_with_status_update("Tokenizer operation lock is poisoned."))
+}
+
+fn set_tokenizer_available() {
+    let mut status = TOKENIZER_STATUS.lock().unwrap();
+    status.status = TokenizerStatus::Available;
+    status.unavailable_reason = None;
+}
+
+fn set_tokenizer_running() {
+    let mut status = TOKENIZER_STATUS.lock().unwrap();
+    status.status = TokenizerStatus::Running;
+    status.unavailable_reason = None;
+}
+
+fn set_tokenizer_unavailable(reason: String) {
+    let mut status = TOKENIZER_STATUS.lock().unwrap();
+    status.status = TokenizerStatus::Unavailable;
+    status.unavailable_reason = Some(reason);
+}
+
+fn unavailable_with_status_update(reason: &str) -> String {
+    let reason = reason.to_string();
+    match tokenizer_state().write() {
+        Ok(mut tokenizer_guard) => *tokenizer_guard = None,
+        Err(_) => set_tokenizer_unavailable("Tokenizer state lock is poisoned.".to_string()),
     }
 
-    let tokenizer = tokenizer_state()
-        .read()
-        .map_err(|_| TokenizerError::from("Tokenizer state lock is poisoned"))?
-        .clone()
-        .ok_or_else(|| TokenizerError::from("Tokenizer not initialized"))?;
-    let enc = tokenizer.encode(text, true)?;
-    Ok(enc.len())
+    set_tokenizer_unavailable(reason.clone());
+    reason
 }
 
-#[post("/tokenizer/count", data = "<req>")]
-pub fn token_count(_token: APIToken, req: Json<SetTokenText>) -> Json<TokenizerResponse> {
-    Json(get_token_count(&req.text).into())
-}
-
-#[post("/tokenizer/validate", data = "<payload>")]
-pub fn validate_tokenizer(_token: APIToken, payload: Json<TokenizerPath>) -> Json<TokenizerResponse> {
-    Json(handle_tokenizer_validate(&PathBuf::from(payload.file_path.clone())).into())
-}
-
-#[post("/tokenizer/store", data = "<payload>")]
-pub fn store_tokenizer(_token: APIToken, payload: Json<TokenizerStorage>) -> Json<TokenizerResponse> {
-    match handle_tokenizer_store(&payload) {
-        Ok(dest_path) => Json(TokenizerResponse {
-            success: true,
-            token_count: 0,
-            message: dest_path,
-        }),
-        Err(e) => Json(TokenizerResponse {
-            success: false,
-            token_count: 0,
-            message: e.to_string(),
-        }),
+fn resolve_tokenizer_path(path: &str) -> Result<PathBuf, String> {
+    if !path.trim().is_empty() {
+        return Ok(PathBuf::from(path));
     }
+
+    DEFAULT_TOKENIZER_PATH
+        .get()
+        .cloned()
+        .ok_or_else(|| "Default tokenizer path is not initialized.".to_string())
 }
 
-#[post("/tokenizer/delete", data = "<payload>")]
-pub fn delete_tokenizer(_token: APIToken, payload: Json<TokenizerDelete>) -> Json<TokenizerResponse> {
-    match handle_tokenizer_delete(&payload) {
-        Ok(_) => Json(TokenizerResponse {
-            success: true,
-            token_count: 0,
-            message: "Success".to_string(),
-        }),
-        Err(e) => Json(TokenizerResponse {
-            success: false,
-            token_count: 0,
-            message: e.to_string(),
-        }),
+fn load_tokenizer_from_file(path: &PathBuf) -> Result<Tokenizer, String> {
+    if !path.is_file() {
+        return Err(format!("Tokenizer file was not found: {}", path.display()));
     }
-}
 
-#[post("/tokenizer/set", data = "<payload>")]
-pub fn set_tokenizer(_token: APIToken, payload: Json<TokenizerPath>) -> Json<TokenizerResponse> {
-    match handle_tokenizer_set(&payload.file_path) {
-        Ok(_) => Json(TokenizerResponse {
-            success: true,
-            token_count: 0,
-            message: "Success".to_string(),
-        }),
-        Err(e) => Json(TokenizerResponse {
-            success: false,
-            token_count: 0,
-            message: e.to_string(),
-        }),
-    }
+    Tokenizer::from_file(path)
+        .map_err(|e| format!("Failed to load tokenizer from '{}': {e}", path.display()))
 }