added functionality to save/move tokenizer file

runtime/src/app_window.rs

@@ -11,7 +11,6 @@ use serde::Deserialize;
 use strum_macros::Display;
 use tauri::updater::UpdateResponse;
 use tauri::{FileDropEvent, GlobalShortcutManager, UpdaterEvent, RunEvent, Manager, PathResolver, Window, WindowEvent, generate_context};
-use tauri::api::dialog::blocking::FileDialogBuilder;
 use tokio::sync::broadcast;
 use tokio::time;
 use crate::api_token::APIToken;
@@ -474,241 +473,6 @@ pub async fn install_update(_token: APIToken) {
     }
 }
 
-/// Let the user select a directory.
-#[post("/select/directory?<title>", data = "<previous_directory>")]
-pub fn select_directory(_token: APIToken, title: &str, previous_directory: Option<Json<PreviousDirectory>>) -> Json<DirectorySelectionResponse> {
-    let folder_path = match previous_directory {
-        Some(previous) => {
-            let previous_path = previous.path.as_str();
-            FileDialogBuilder::new()
-                .set_title(title)
-                .set_directory(previous_path)
-                .pick_folder()
-        },
-
-        None => {
-            FileDialogBuilder::new()
-                .set_title(title)
-                .pick_folder()
-        },
-    };
-    
-    match folder_path {
-        Some(path) => {
-            info!("User selected directory: {path:?}");
-            Json(DirectorySelectionResponse {
-                user_cancelled: false,
-                selected_directory: path.to_str().unwrap().to_string(),
-            })
-        },
-
-        None => {
-            info!("User cancelled directory selection.");
-            Json(DirectorySelectionResponse {
-                user_cancelled: true,
-                selected_directory: String::from(""),
-            })
-        },
-    }
-}
-
-#[derive(Clone, Deserialize)]
-pub struct PreviousDirectory {
-    path: String,
-}
-
-#[derive(Clone, Deserialize)]
-pub struct FileTypeFilter {
-    filter_name: String,
-    filter_extensions: Vec<String>,
-}
-
-#[derive(Clone, Deserialize)]
-pub struct SelectFileOptions {
-    title: String,
-    previous_file: Option<PreviousFile>,
-    filter: Option<FileTypeFilter>,
-}
-
-#[derive(Clone, Deserialize)]
-pub struct SaveFileOptions {
-    title: String,
-    name_file: Option<PreviousFile>,
-    filter: Option<FileTypeFilter>,
-}
-
-#[derive(Serialize)]
-pub struct DirectorySelectionResponse {
-    user_cancelled: bool,
-    selected_directory: String,
-}
-
-/// Let the user select a file.
-#[post("/select/file", data = "<payload>")]
-pub fn select_file(_token: APIToken, payload: Json<SelectFileOptions>) -> Json<FileSelectionResponse> {
-
-    // Create a new file dialog builder:
-    let file_dialog = FileDialogBuilder::new();
-
-    // Set the title of the file dialog:
-    let file_dialog = file_dialog.set_title(&payload.title);
-
-    // Set the file type filter if provided:
-    let file_dialog = apply_filter(file_dialog, &payload.filter);
-
-    // Set the previous file path if provided:
-    let file_dialog = match &payload.previous_file {
-        Some(previous) => {
-            let previous_path = previous.file_path.as_str();
-            file_dialog.set_directory(previous_path)
-        },
-
-        None => file_dialog,
-    };
-
-    // Show the file dialog and get the selected file path:
-    let file_path = file_dialog.pick_file();
-    match file_path {
-        Some(path) => {
-            info!("User selected file: {path:?}");
-            Json(FileSelectionResponse {
-                user_cancelled: false,
-                selected_file_path: path.to_str().unwrap().to_string(),
-            })
-        },
-
-        None => {
-            info!("User cancelled file selection.");
-            Json(FileSelectionResponse {
-                user_cancelled: true,
-                selected_file_path: String::from(""),
-            })
-        },
-    }
-}
-
-/// Let the user select some files.
-#[post("/select/files", data = "<payload>")]
-pub fn select_files(_token: APIToken, payload: Json<SelectFileOptions>) -> Json<FilesSelectionResponse> {
-
-    // Create a new file dialog builder:
-    let file_dialog = FileDialogBuilder::new();
-
-    // Set the title of the file dialog:
-    let file_dialog = file_dialog.set_title(&payload.title);
-
-    // Set the file type filter if provided:
-    let file_dialog = apply_filter(file_dialog, &payload.filter);
-
-    // Set the previous file path if provided:
-    let file_dialog = match &payload.previous_file {
-        Some(previous) => {
-            let previous_path = previous.file_path.as_str();
-            file_dialog.set_directory(previous_path)
-        },
-
-        None => file_dialog,
-    };
-
-    // Show the file dialog and get the selected file path:
-    let file_paths = file_dialog.pick_files();
-    match file_paths {
-        Some(paths) => {
-            info!("User selected {} files.", paths.len());
-            Json(FilesSelectionResponse {
-                user_cancelled: false,
-                selected_file_paths: paths.iter().map(|p| p.to_str().unwrap().to_string()).collect(),
-            })
-        }
-
-        None => {
-            info!("User cancelled file selection.");
-            Json(FilesSelectionResponse {
-                user_cancelled: true,
-                selected_file_paths: Vec::new(),
-            })
-        },
-    }
-}
-
-#[post("/save/file", data = "<payload>")]
-pub fn save_file(_token: APIToken, payload: Json<SaveFileOptions>) -> Json<FileSaveResponse> {
-
-    // Create a new file dialog builder:
-    let file_dialog = FileDialogBuilder::new();
-
-    // Set the title of the file dialog:
-    let file_dialog = file_dialog.set_title(&payload.title);
-
-    // Set the file type filter if provided:
-    let file_dialog = apply_filter(file_dialog, &payload.filter);
-
-    // Set the previous file path if provided:
-    let file_dialog = match &payload.name_file {
-        Some(previous) => {
-            let previous_path = previous.file_path.as_str();
-            file_dialog.set_directory(previous_path)
-        },
-
-        None => file_dialog,
-    };
-
-    // Displays the file dialogue box and select the file:
-    let file_path = file_dialog.save_file();
-    match file_path {
-        Some(path) => {
-            info!("User selected file for writing operation: {path:?}");
-            Json(FileSaveResponse {
-                user_cancelled: false,
-                save_file_path: path.to_str().unwrap().to_string(),
-            })
-        },
-
-        None => {
-            info!("User cancelled file selection.");
-            Json(FileSaveResponse {
-                user_cancelled: true,
-                save_file_path: String::from(""),
-            })
-        },
-    }
-}
-
-#[derive(Clone, Deserialize)]
-pub struct PreviousFile {
-    file_path: String,
-}
-
-/// Applies an optional file type filter to a FileDialogBuilder.
-fn apply_filter(file_dialog: FileDialogBuilder, filter: &Option<FileTypeFilter>) -> FileDialogBuilder {
-    match filter {
-        Some(f) => file_dialog.add_filter(
-            &f.filter_name,
-            &f.filter_extensions.iter().map(|s| s.as_str()).collect::<Vec<&str>>(),
-        ),
-
-        None => file_dialog,
-    }
-}
-
-#[derive(Serialize)]
-pub struct FileSelectionResponse {
-    user_cancelled: bool,
-    selected_file_path: String,
-}
-
-#[derive(Serialize)]
-pub struct FilesSelectionResponse {
-    user_cancelled: bool,
-    selected_file_paths: Vec<String>,
-}
-
-#[derive(Serialize)]
-pub struct FileSaveResponse {
-    user_cancelled: bool,
-    save_file_path: String,
-}
-
 /// Request payload for registering a global shortcut.
 #[derive(Clone, Deserialize)]
 pub struct RegisterShortcutRequest {

runtime/src/file_actions.rs

@@ -0,0 +1,241 @@
+use log::info;
+use rocket::post;
+use rocket::serde::{Deserialize, Serialize};
+use rocket::serde::json::Json;
+use tauri::api::dialog::blocking::FileDialogBuilder;
+use crate::api_token::APIToken;
+
+#[derive(Clone, Deserialize)]
+pub struct PreviousDirectory {
+    path: String,
+}
+
+#[derive(Clone, Deserialize)]
+pub struct FileTypeFilter {
+    filter_name: String,
+    filter_extensions: Vec<String>,
+}
+
+#[derive(Clone, Deserialize)]
+pub struct SelectFileOptions {
+    title: String,
+    previous_file: Option<PreviousFile>,
+    filter: Option<FileTypeFilter>,
+}
+
+#[derive(Clone, Deserialize)]
+pub struct SaveFileOptions {
+    title: String,
+    name_file: Option<PreviousFile>,
+    filter: Option<FileTypeFilter>,
+}
+
+#[derive(Serialize)]
+pub struct DirectorySelectionResponse {
+    user_cancelled: bool,
+    selected_directory: String,
+}
+
+#[derive(Serialize)]
+pub struct FileSelectionResponse {
+    user_cancelled: bool,
+    selected_file_path: String,
+}
+
+#[derive(Serialize)]
+pub struct FilesSelectionResponse {
+    user_cancelled: bool,
+    selected_file_paths: Vec<String>,
+}
+
+#[derive(Serialize)]
+pub struct FileSaveResponse {
+    user_cancelled: bool,
+    save_file_path: String,
+}
+
+#[derive(Clone, Deserialize)]
+pub struct PreviousFile {
+    file_path: String,
+}
+
+/// Let the user select a directory.
+#[post("/select/directory?<title>", data = "<previous_directory>")]
+pub fn select_directory(_token: APIToken, title: &str, previous_directory: Option<Json<PreviousDirectory>>) -> Json<DirectorySelectionResponse> {
+    let folder_path = match previous_directory {
+        Some(previous) => {
+            let previous_path = previous.path.as_str();
+            FileDialogBuilder::new()
+                .set_title(title)
+                .set_directory(previous_path)
+                .pick_folder()
+        },
+
+        None => {
+            FileDialogBuilder::new()
+                .set_title(title)
+                .pick_folder()
+        },
+    };
+
+    match folder_path {
+        Some(path) => {
+            info!("User selected directory: {path:?}");
+            Json(DirectorySelectionResponse {
+                user_cancelled: false,
+                selected_directory: path.to_str().unwrap().to_string(),
+            })
+        },
+
+        None => {
+            info!("User cancelled directory selection.");
+            Json(DirectorySelectionResponse {
+                user_cancelled: true,
+                selected_directory: String::from(""),
+            })
+        },
+    }
+}
+
+/// Let the user select a file.
+#[post("/select/file", data = "<payload>")]
+pub fn select_file(_token: APIToken, payload: Json<SelectFileOptions>) -> Json<FileSelectionResponse> {
+
+    // Create a new file dialog builder:
+    let file_dialog = FileDialogBuilder::new();
+
+    // Set the title of the file dialog:
+    let file_dialog = file_dialog.set_title(&payload.title);
+
+    // Set the file type filter if provided:
+    let file_dialog = apply_filter(file_dialog, &payload.filter);
+
+    // Set the previous file path if provided:
+    let file_dialog = match &payload.previous_file {
+        Some(previous) => {
+            let previous_path = previous.file_path.as_str();
+            file_dialog.set_directory(previous_path)
+        },
+
+        None => file_dialog,
+    };
+
+    // Show the file dialog and get the selected file path:
+    let file_path = file_dialog.pick_file();
+    match file_path {
+        Some(path) => {
+            info!("User selected file: {path:?}");
+            Json(FileSelectionResponse {
+                user_cancelled: false,
+                selected_file_path: path.to_str().unwrap().to_string(),
+            })
+        },
+
+        None => {
+            info!("User cancelled file selection.");
+            Json(FileSelectionResponse {
+                user_cancelled: true,
+                selected_file_path: String::from(""),
+            })
+        },
+    }
+}
+
+/// Let the user select some files.
+#[post("/select/files", data = "<payload>")]
+pub fn select_files(_token: APIToken, payload: Json<SelectFileOptions>) -> Json<FilesSelectionResponse> {
+
+    // Create a new file dialog builder:
+    let file_dialog = FileDialogBuilder::new();
+
+    // Set the title of the file dialog:
+    let file_dialog = file_dialog.set_title(&payload.title);
+
+    // Set the file type filter if provided:
+    let file_dialog = apply_filter(file_dialog, &payload.filter);
+
+    // Set the previous file path if provided:
+    let file_dialog = match &payload.previous_file {
+        Some(previous) => {
+            let previous_path = previous.file_path.as_str();
+            file_dialog.set_directory(previous_path)
+        },
+
+        None => file_dialog,
+    };
+
+    // Show the file dialog and get the selected file path:
+    let file_paths = file_dialog.pick_files();
+    match file_paths {
+        Some(paths) => {
+            info!("User selected {} files.", paths.len());
+            Json(FilesSelectionResponse {
+                user_cancelled: false,
+                selected_file_paths: paths.iter().map(|p| p.to_str().unwrap().to_string()).collect(),
+            })
+        }
+
+        None => {
+            info!("User cancelled file selection.");
+            Json(FilesSelectionResponse {
+                user_cancelled: true,
+                selected_file_paths: Vec::new(),
+            })
+        },
+    }
+}
+
+#[post("/save/file", data = "<payload>")]
+pub fn save_file(_token: APIToken, payload: Json<SaveFileOptions>) -> Json<FileSaveResponse> {
+
+    // Create a new file dialog builder:
+    let file_dialog = FileDialogBuilder::new();
+
+    // Set the title of the file dialog:
+    let file_dialog = file_dialog.set_title(&payload.title);
+
+    // Set the file type filter if provided:
+    let file_dialog = apply_filter(file_dialog, &payload.filter);
+
+    // Set the previous file path if provided:
+    let file_dialog = match &payload.name_file {
+        Some(previous) => {
+            let previous_path = previous.file_path.as_str();
+            file_dialog.set_directory(previous_path)
+        },
+
+        None => file_dialog,
+    };
+
+    // Displays the file dialogue box and select the file:
+    let file_path = file_dialog.save_file();
+    match file_path {
+        Some(path) => {
+            info!("User selected file for writing operation: {path:?}");
+            Json(FileSaveResponse {
+                user_cancelled: false,
+                save_file_path: path.to_str().unwrap().to_string(),
+            })
+        },
+
+        None => {
+            info!("User cancelled file selection.");
+            Json(FileSaveResponse {
+                user_cancelled: true,
+                save_file_path: String::from(""),
+            })
+        },
+    }
+}
+
+/// Applies an optional file type filter to a FileDialogBuilder.
+fn apply_filter(file_dialog: FileDialogBuilder, filter: &Option<FileTypeFilter>) -> FileDialogBuilder {
+    match filter {
+        Some(f) => file_dialog.add_filter(
+            &f.filter_name,
+            &f.filter_extensions.iter().map(|s| s.as_str()).collect::<Vec<&str>>(),
+        ),
+
+        None => file_dialog,
+    }
+}

runtime/src/lib.rs

@@ -17,4 +17,6 @@ pub mod qdrant;
 pub mod certificate_factory;
 pub mod runtime_api_token;
 pub mod stale_process_cleanup;
-mod sidecar_types;
+mod sidecar_types;
+pub mod tokenizer;
+pub mod file_actions;

runtime/src/main.rs

@@ -11,7 +11,7 @@ use mindwork_ai_studio::environment::is_dev;
 use mindwork_ai_studio::log::init_logging;
 use mindwork_ai_studio::metadata::MetaData;
 use mindwork_ai_studio::runtime_api::start_runtime_api;
-
+use mindwork_ai_studio::tokenizer::{init_tokenizer};
 
 #[tokio::main]
 async fn main() {
@@ -43,8 +43,12 @@ async fn main() {
         info!("Running in production mode.");
     }
 
+    if let Err(e) = init_tokenizer() {
+        warn!(Source = "Tokenizer"; "Error during the initialisation of the tokenizer: {}", e);
+    }
+
     generate_runtime_certificate();
     start_runtime_api();
     
     start_tauri();
-}
+}

runtime/src/runtime_api.rs

@@ -72,10 +72,10 @@ pub fn start_runtime_api() {
                 crate::app_window::get_event_stream,
                 crate::app_window::check_for_update,
                 crate::app_window::install_update,
-                crate::app_window::select_directory,
-                crate::app_window::select_file,
-                crate::app_window::select_files,
-                crate::app_window::save_file,
+                crate::file_actions::select_directory,
+                crate::file_actions::select_file,
+                crate::file_actions::select_files,
+                crate::file_actions::save_file,
                 crate::secret::get_secret,
                 crate::secret::store_secret,
                 crate::secret::delete_secret,
@@ -89,6 +89,9 @@ pub fn start_runtime_api() {
                 crate::file_data::extract_data,
                 crate::log::get_log_paths,
                 crate::log::log_event,
+                crate::tokenizer::token_count,
+                crate::tokenizer::validate_tokenizer,
+                crate::tokenizer::store_tokenizer,
                 crate::app_window::register_shortcut,
                 crate::app_window::validate_shortcut,
                 crate::app_window::suspend_shortcuts,

runtime/src/tokenizer.rs

@@ -0,0 +1,194 @@
+use rocket::yansi::Paint;
+use std::fs;
+use std::path::{PathBuf};
+use std::sync::OnceLock;
+use rocket::{post};
+use rocket::serde::json::Json;
+use rocket::serde::Serialize;
+use serde::Deserialize;
+use tokenizers::Error;
+use tokenizers::tokenizer::{Tokenizer, Error as TokenizerError};
+use crate::api_token::APIToken;
+use crate::environment::{DATA_DIRECTORY};
+
+static TOKENIZER: OnceLock<Tokenizer> = OnceLock::new();
+
+#[derive(Deserialize)]
+pub struct SetTokenText {
+    pub text: String,
+}
+
+#[derive(Clone, Deserialize)]
+pub struct TokenizerStorage {
+    model_id: String,
+    previous_model_id: String,
+    file_path: String,
+}
+
+#[derive(Clone, Deserialize)]
+pub struct TokenizerValidation {
+    file_path: String,
+}
+
+#[derive(Serialize)]
+pub struct TokenizerResponse {
+    success: bool,
+    token_count: usize,
+    message: String,
+}
+
+impl From<Result<usize, TokenizerError>> for TokenizerResponse {
+    fn from(result: Result<usize, TokenizerError>) -> Self {
+        match result {
+            Ok(count) => TokenizerResponse {
+                success: true,
+                token_count: count,
+                message: "Success".to_string(),
+            },
+            Err(e) => TokenizerResponse {
+                success: false,
+                token_count: 0,
+                message: e.to_string(),
+            },
+        }
+    }
+}
+
+pub fn init_tokenizer() -> Result<(), Error>{
+    let mut target_dir = PathBuf::from("target");
+    target_dir.push("tokenizers");
+    fs::create_dir_all(&target_dir)?;
+
+    let mut local_tokenizer_path = target_dir.clone();
+    local_tokenizer_path.push("tokenizer.json");
+
+    TOKENIZER.set(Tokenizer::from_file(local_tokenizer_path)?).expect("Could not set the tokenizer.");
+    Ok(())
+}
+
+fn validate_tokenizer_at_path(path: &PathBuf) -> Result<usize, TokenizerError> {
+    if !path.is_file() {
+        return Err(TokenizerError::from(format!(
+            "Tokenizer file was not found: {}",
+            path.display()
+        )));
+    }
+
+    let tokenizer = Tokenizer::from_file(path).map_err(|e| {
+        println!("Failed to load tokenizer from {}: {}", Paint::red(&path.display()), e);
+        TokenizerError::from(format!(
+            "Failed to load tokenizer from '{}': {}",
+            path.display(),
+            e
+        ))
+    })?;
+    println!("Loaded tokenizer from {}", Paint::green(&path.display()));
+
+    let test_string = "Hello, world! This is a test string for tokenizer validation.";
+
+    let encoding = tokenizer.encode(test_string, true).map_err(|e| {
+        println!(
+            "Tokenizer failed to encode validation string for {}: {}",
+            Paint::red(&path.display()),
+            e
+        );
+        TokenizerError::from(format!(
+            "Tokenizer failed to encode validation string: {}",
+            e
+        ))
+    })?;
+    let token_count = encoding.len();
+
+    if token_count == 0 {
+        return Err(TokenizerError::from(
+            "Tokenizer produced 0 tokens for test string. The tokenizer is likely invalid or misconfigured."
+        ));
+    }
+
+    if encoding.get_tokens().iter().any(|t| t.is_empty()) {
+        return Err(TokenizerError::from(
+            "Tokenizer produced empty tokens. The tokenizer is invalid."
+        ));
+    }
+
+    Ok(token_count)
+}
+
+fn handle_tokenizer_store(payload: &TokenizerStorage) -> Result<(), std::io::Error> {
+    let data_dir = DATA_DIRECTORY
+        .get()
+        .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::Other, "DATA_DIRECTORY not initialized"))?;
+
+    let base_path = PathBuf::from(data_dir).join("tokenizers");
+
+    // Delete previous model if file_path is empty
+    if payload.file_path.trim().is_empty() {
+        if payload.previous_model_id.trim().is_empty() {
+            return Ok(()); // Nothing to delete
+        }
+        let previous_path = base_path.join(&payload.previous_model_id);
+        fs::remove_dir_all(previous_path)?;
+        return Ok(());
+    }
+
+    // Copy file
+    let source_path = PathBuf::from(&payload.file_path);
+    let source_name = source_path.file_name()
+        .and_then(|n| n.to_str())
+        .ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid tokenizer file path"))?;
+    fs::create_dir_all(&base_path.join(&payload.model_id))?;
+    let destination_path = base_path.join(&payload.model_id).join(source_name);
+    println!("Moving tokenizer file from {} to {}", source_path.display(), destination_path.display());
+
+    let previous_path = base_path.join(&payload.previous_model_id);
+
+    // Delete previous tokenizer folder if specified
+    if !payload.previous_model_id.trim().is_empty() && source_path.starts_with(&previous_path){
+        fs::rename(&source_path, &destination_path)?;
+        if previous_path.exists() {
+            fs::remove_dir_all(previous_path)?;
+        }
+    }else{
+        fs::copy( & source_path, & destination_path)?;
+    }
+    Ok(())
+}
+
+pub fn get_token_count(text: &str) -> Result<usize, TokenizerError> {
+    if text.trim().is_empty() {
+        return Err(TokenizerError::from("Input text is empty"));
+    }
+
+    let tokenizer = TOKENIZER.get().cloned().ok_or_else(|| TokenizerError::from("Tokenizer not initialized"))?;
+    let enc = tokenizer.encode(text, true)?;
+    Ok(enc.len())
+}
+
+#[post("/tokenizer/count", data = "<req>")]
+pub fn token_count(_token: APIToken, req: Json<SetTokenText>) -> Json<TokenizerResponse> {
+    Json(get_token_count(&req.text).into())
+}
+
+#[post("/tokenizer/validate", data = "<payload>")]
+pub fn validate_tokenizer(_token: APIToken, payload: Json<TokenizerValidation>) -> Json<TokenizerResponse>{
+    println!("Received tokenizer validation request: {}", payload.file_path);
+    Json(validate_tokenizer_at_path(&PathBuf::from(payload.file_path.clone())).into())
+}
+
+#[post("/tokenizer/store", data = "<payload>")]
+pub fn store_tokenizer(_token: APIToken, payload: Json<TokenizerStorage>) -> Json<TokenizerResponse>{
+    println!("Received tokenizer store request: {}, {}, {}", payload.model_id, payload.previous_model_id, payload.file_path);
+    match handle_tokenizer_store(&payload) {
+        Ok(()) => Json(TokenizerResponse {
+            success: true,
+            token_count: 0,
+            message: "Success".to_string(),
+        }),
+        Err(e) => Json(TokenizerResponse {
+            success: false,
+            token_count: 0,
+            message: e.to_string(),
+        }),
+    }
+
+}