diff --git a/app/AGENTS.md b/app/AGENTS.md index fef391084..d798ccfef 100644 --- a/app/AGENTS.md +++ b/app/AGENTS.md @@ -30,6 +30,8 @@ Key structure: - **Naming conventions** - Constants, enum members, and `static readonly` fields use `UPPER_SNAKE_CASE` such as `MY_CONSTANT`. - **Message Bus** - Singleton event bus for cross-component communication inside the .NET app - **Prompt injection reports** - The user hears once per action about what the filter removed, as soon as that action ends. Content which belongs to an action carries its scope from `PromptInjectionGuardService.BeginAction` down to where it is read or filtered; whatever is filtered without a scope is reported right away. Pass the scope as a parameter, never through a field or an `AsyncLocal`: the service serves the whole app, and the message bus delivers to other receivers in the execution context of whoever sends. +- **Paths in stored data** - A stored path to a file which AI Studio keeps below the data directory, such as a transcript or a tokenizer copy, is relative to that directory, so that a backup or a copy of the data directory works on another computer, under another user name, or on another OS. Mark such a property with `[JsonConverter(typeof(DataDirectoryRelativePathConverter))]`, for a positional record parameter with `[property: ...]`; in memory, the path stays absolute. Better still, derive the path from IDs and store none, as `VisualBriefingStore` does. Paths to files of the user stay absolute. Chats stored before v26.10.1 are migrated by a compatibility shim, see `documentation/compatibility-shims/2026-10-relative-data-paths.md`. +- **New files and folders in the data or configuration directory** - Whoever adds a file or folder directly to the data or configuration directory, or below `plugins/` or `databases/` one which needs a decision of its own, lists it in `DataBackupContents.ENTRIES`: as part of a backup, or with the reason a backup leaves it out. A backup takes along only what is listed as included, and a restore replaces only that. An entry included in a backup also goes into `REPLACEABLE_ELEMENTS` in `runtime/src/data_restore.rs`; otherwise the runtime refuses to restore it. The folder panel on the information page shows the list to users who back up these folders with a tool of their own. - **Encryption** - Initialized before Rust service is marked ready - **Debug environment** - Reads `startup.env` file with IPC credentials - **Production environment** - Runtime launches .NET sidecar with environment variables diff --git a/app/MindWork AI Studio/Assistants/ERI/AssistantERI.razor.cs b/app/MindWork AI Studio/Assistants/ERI/AssistantERI.razor.cs index c6725c33b..47a5220c7 100644 --- a/app/MindWork AI Studio/Assistants/ERI/AssistantERI.razor.cs +++ b/app/MindWork AI Studio/Assistants/ERI/AssistantERI.razor.cs @@ -1159,11 +1159,22 @@ public partial class AssistantERI : AssistantBaseCore """, true); var fileListAnswer = await this.AddAIResponseAsync(time, true); - // Is this an update of the ERI server? If so, we need to delete the previously generated files: + // + // Is this an update of the ERI server? If so, we need to delete the previously generated files. + // Only those within the current base directory, though: the list names files by their full path, + // and it outlives a change of the base directory, as well as a restore on another computer. + // There, the same path may belong to files of somebody else: + // if (writeToFilesystemSnapshot && previouslyGeneratedFilesSnapshot.Length > 0 && !string.IsNullOrWhiteSpace(fileListAnswer)) { foreach (var file in previouslyGeneratedFilesSnapshot) { + if (!PathTools.IsInsideDirectory(baseDirectorySnapshot, file)) + { + this.Logger.LogWarning($"The previously created file '{file}' is not inside the base directory '{baseDirectorySnapshot}' and was kept."); + continue; + } + try { if (File.Exists(file)) @@ -1212,7 +1223,7 @@ public partial class AssistantERI : AssistantBaseCore // Security check: ensure that the desired file path is inside the base directory. // We cannot trust the beginning of the file path because it would be possible // to escape by using `..` in the file path. - if (!desiredFilePath.StartsWith(baseDirectorySnapshot, StringComparison.InvariantCultureIgnoreCase) || desiredFilePath.Contains("..")) + if (!PathTools.IsInsideDirectory(baseDirectorySnapshot, desiredFilePath)) this.Logger.LogWarning($"The file path '{desiredFilePath}' is may not inside the base directory '{baseDirectorySnapshot}'."); else diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 21ffa78c2..32a444ced 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -3772,6 +3772,237 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T4081853237"] = "C -- Configure Keyboard Shortcut UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T636303786"] = "Configure Keyboard Shortcut" +-- Keep the existing chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1274903302"] = "Keep the existing chat" + +-- The backup goes on when you leave this page. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1298413158"] = "The backup goes on when you leave this page." + +-- Unpacking file {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1496386567"] = "Unpacking file {0} of {1}" + +-- Created on {0} with AI Studio {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1503285113"] = "Created on {0} with AI Studio {1}" + +-- AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1555678613"] = "AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first." + +-- Replace All Data +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1670691112"] = "Replace All Data" + +-- Reading the backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1731783566"] = "Reading the backup" + +-- Please stay on this page until this is done. Leaving the page stops adding the chats. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1758163038"] = "Please stay on this page until this is done. Leaving the page stops adding the chats." + +-- Back up +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1823019579"] = "Back up" + +-- When a chat already exists +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2117204104"] = "When a chat already exists" + +-- Restoring this backup replaces your settings, your chats, your plugins, and the indexes of your data sources with those of the backup. Whatever you changed since the backup was made is lost. Afterward, AI Studio restarts. Do you want to restore this backup? +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2176422849"] = "Restoring this backup replaces your settings, your chats, your plugins, and the indexes of your data sources with those of the backup. Whatever you changed since the backup was made is lost. Afterward, AI Studio restarts. Do you want to restore this backup?" + +-- 1 chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2301695346"] = "1 chat" + +-- Writing file {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2416389933"] = "Writing file {0} of {1}" + +-- Temporary chats +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2532369939"] = "Temporary chats" + +-- Adding the chats was stopped. The chats added until then were kept. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2586532370"] = "Adding the chats was stopped. The chats added until then were kept." + +-- Restore +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T267943793"] = "Restore" + +-- Replaces your settings, chats, plugins, and the indexes of your data sources with those of the backup. AI Studio restarts to do so. The plugins and settings of your organization stay as they are. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2693525709"] = "Replaces your settings, chats, plugins, and the indexes of your data sources with those of the backup. AI Studio restarts to do so. The plugins and settings of your organization stay as they are." + +-- Chats which already existed and stayed as they are: {0}. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T29276052"] = "Chats which already existed and stayed as they are: {0}." + +-- The indexes of your data sources are not part of this backup, because they were not available. After a restore, AI Studio indexes your data sources anew. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3006171489"] = "The indexes of your data sources are not part of this backup, because they were not available. After a restore, AI Studio indexes your data sources anew." + +-- Preparing the backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3082169282"] = "Preparing the backup" + +-- Back up all your data into one file: your settings, chats, plugins, and the indexes of your data sources. Your API keys and passwords stay out of it. AI Studio keeps them in the secure storage of your OS. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3087579229"] = "Back up all your data into one file: your settings, chats, plugins, and the indexes of your data sources. Your API keys and passwords stay out of it. AI Studio keeps them in the secure storage of your OS." + +-- Choose backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3114985646"] = "Choose backup" + +-- Select a backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T311951952"] = "Select a backup" + +-- Add chats only +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3238328507"] = "Add chats only" + +-- Chat {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T355446621"] = "Chat {0} of {1}" + +-- {0} chats +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3562573794"] = "{0} chats" + +-- Add {0} chats +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3585479613"] = "Add {0} chats" + +-- Backed up {0} files into: {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3630348902"] = "Backed up {0} files into: {1}" + +-- Restore a backup: replace all your data with those of the backup, e.g., on a new computer, or add only its chats to the chats you have. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3657899737"] = "Restore a backup: replace all your data with those of the backup, e.g., on a new computer, or add only its chats to the chats you have." + +-- Add it as an additional chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3966117123"] = "Add it as an additional chat" + +-- Chats added: {0}. Workspaces they went into: {1}. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3972560517"] = "Chats added: {0}. Workspaces they went into: {1}." + +-- The backup was stopped. No file was left behind. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T431814261"] = "The backup was stopped. No file was left behind." + +-- Adds the chats of the backup to the chats you have. Nothing else changes. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T457177394"] = "Adds the chats of the backup to the chats you have. Nothing else changes." + +-- Chats which could not be added: {0}. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T611768714"] = "Chats which could not be added: {0}." + +-- The restore was stopped. All your data stay as they are. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T749188343"] = "The restore was stopped. All your data stay as they are." + +-- Back up data +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T796613169"] = "Back up data" + +-- Replace all data +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T819832488"] = "Replace all data" + +-- Cancel +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T900713019"] = "Cancel" + +-- Copying the indexes of your data sources: {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T907141309"] = "Copying the indexes of your data sources: {0} of {1}" + +-- Add 1 chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T938996955"] = "Add 1 chat" + +-- Folder +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T104168189"] = "Folder" + +-- Reason +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1093747001"] = "Reason" + +-- Configuration folder +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1191404447"] = "Configuration folder" + +-- Could not open the folder. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1215994612"] = "Could not open the folder." + +-- Your projects in the Visual Briefing Assistant. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1259707710"] = "Your projects in the Visual Briefing Assistant." + +-- Your temporary chats, along with their transcripts. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1293614512"] = "Your temporary chats, along with their transcripts." + +-- What belongs into a backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1682175486"] = "What belongs into a backup" + +-- The index of your local data sources: which files AI Studio embedded, and their text. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T170332953"] = "The index of your local data sources: which files AI Studio embedded, and their text." + +-- Keys with which AI Studio protects data in its user interface. Only the Flatpak keeps them here, and on another computer, AI Studio creates new ones. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1981169629"] = "Keys with which AI Studio protects data in its user interface. Only the Flatpak keeps them here, and on another computer, AI Studio creates new ones." + +-- Tools which AI Studio downloaded for this OS, such as Pandoc. On another computer, AI Studio downloads them again. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1992121437"] = "Tools which AI Studio downloaded for this OS, such as Pandoc. On another computer, AI Studio downloads them again." + +-- What a backup can leave out +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2120934651"] = "What a backup can leave out" + +-- Good to know +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2373663376"] = "Good to know" + +-- Data folder +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2422278827"] = "Data folder" + +-- Explanation +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2840582448"] = "Explanation" + +-- Copies of your settings, which every version of AI Studio keeps for itself. Your current settings are all a backup needs. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2851981238"] = "Copies of your settings, which every version of AI Studio keeps for itself. Your current settings are all a backup needs." + +-- The tokenizers you chose for your providers. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2892362083"] = "The tokenizers you chose for your providers." + +-- To back up your data with AI Studio itself, or to move it to another computer, use Data Backup & Restore on this page or in the settings instead. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2906063884"] = "To back up your data with AI Studio itself, or to move it to another computer, use Data Backup & Restore on this page or in the settings instead." + +-- Your plugins, such as the assistants you built, apart from the folders listed below. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2979110360"] = "Your plugins, such as the assistants you built, apart from the folders listed below." + +-- Open in your file manager +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2986518596"] = "Open in your file manager" + +-- Your organization deployed some plugins among your own ones, for example, with a device management. Their folders do not show this, so the list above names them one by one. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3156743605"] = "Your organization deployed some plugins among your own ones, for example, with a device management. Their folders do not show this, so the list above names them one by one." + +-- Content +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3244793698"] = "Content" + +-- The embeddings of your local data sources. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3301102729"] = "The embeddings of your local data sources." + +-- The folder is not known yet. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3310368654"] = "The folder is not known yet." + +-- Could not open the folder: {0} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3380661810"] = "Could not open the folder: {0}" + +-- Unknown error +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3461425987"] = "Unknown error" + +-- Plugins of your organization. Your organization brings them back. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T346886000"] = "Plugins of your organization. Your organization brings them back." + +-- Back up the indexes of your data sources only while AI Studio is closed. AI Studio writes to them while it runs, so a copy made meanwhile may not fit together, and AI Studio would have to embed your data sources anew. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3655837348"] = "Back up the indexes of your data sources only while AI Studio is closed. AI Studio writes to them while it runs, so a copy made meanwhile may not fit together, and AI Studio would have to embed your data sources anew." + +-- Files which AI Studio needs only for a moment and deletes again. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T366353671"] = "Files which AI Studio needs only for a moment and deletes again." + +-- Plugins which come with AI Studio, such as the language packs. AI Studio brings them back by itself. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3877928356"] = "Plugins which come with AI Studio, such as the language packs. AI Studio brings them back by itself." + +-- Path +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3949388886"] = "Path" + +-- Your API keys, tokens, and passwords are not in these folders. AI Studio keeps them in the secure storage of your OS. After you restored your data on another computer, enter them there once more. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T4093107108"] = "Your API keys, tokens, and passwords are not in these folders. AI Studio keeps them in the secure storage of your OS. After you restored your data on another computer, enter them there once more." + +-- Your chats in workspaces, along with their transcripts. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T509585668"] = "Your chats in workspaces, along with their transcripts." + +-- Log files. They help when something goes wrong, but they hold none of your data. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T53143530"] = "Log files. They help when something goes wrong, but they hold none of your data." + +-- AI Studio keeps your data in two folders on this computer. When you back up your computer with a tool of your own, such as [Restic](https://restic.net), [Borg](https://www.borgbackup.org), or [Time Machine](https://support.apple.com/en-us/104984), this shows which of their contents belong into the backup and which it can leave out. Click a path to copy it to the clipboard, or use the folder button to open it in your file manager. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T69622543"] = "AI Studio keeps your data in two folders on this computer. When you back up your computer with a tool of your own, such as [Restic](https://restic.net), [Borg](https://www.borgbackup.org), or [Time Machine](https://support.apple.com/en-us/104984), this shows which of their contents belong into the backup and which it can leave out. Click a path to copy it to the clipboard, or use the folder button to open it in your file manager." + +-- Your settings. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T790727583"] = "Your settings." + +-- The following is meant for experts who back up their computer with a tool of their own. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T813925103"] = "The following is meant for experts who back up their computer with a tool of their own." + +-- On this computer, the data folder lies inside the configuration folder. A backup of the configuration folder therefore takes the data folder along. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T91194331"] = "On this computer, the data folder lies inside the configuration folder. A backup of the configuration folder therefore takes the data folder along." + -- Yes, please send my data to the external embedding provider UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T1159107763"] = "Yes, please send my data to the external embedding provider" @@ -3808,6 +4039,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T6617545 -- All files in this folder and its subfolders UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T916879200"] = "All files in this folder and its subfolders" +-- AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1006756723"] = "AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done." + -- You might configure different data sources. A data source can include one file, all files in a directory, or data from your company. Later, you can incorporate these data sources as needed when the AI requires this data to complete a certain task. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "You might configure different data sources. A data source can include one file, all files in a directory, or data from your company. Later, you can incorporate these data sources as needed when the AI requires this data to complete a certain task." @@ -6085,6 +6319,84 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::CONFIRMDIALOG::T1642511898"] = "No" -- Yes UI_TEXT_CONTENT["AISTUDIO::DIALOGS::CONFIRMDIALOG::T3013883440"] = "Yes" +-- Restart now +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T1409292924"] = "Restart now" + +-- Discard the restore +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T3017406920"] = "Discard the restore" + +-- Your backup is ready. AI Studio restarts to put it in place. Whatever you changed until then would be lost, so please restart now. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T495853773"] = "Your backup is ready. AI Studio restarts to put it in place. Whatever you changed until then would be lost, so please restart now." + +-- AI Studio could not restart by itself. Please quit AI Studio and start it again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T573887863"] = "AI Studio could not restart by itself. Please quit AI Studio and start it again." + +-- Restoring your backup failed. AI Studio put all your data back the way they were before. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1057857006"] = "Restoring your backup failed. AI Studio put all your data back the way they were before." + +-- Your backup was not restored +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1193456690"] = "Your backup was not restored" + +-- Restoring your backup failed, and AI Studio could not yet put all your data back the way they were. It tries again at the next start, so please restart AI Studio. If this message appears again, please contact the support, and keep the folders '.restore-previous' and '.restore-pending' in your data folder. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T123391329"] = "Restoring your backup failed, and AI Studio could not yet put all your data back the way they were. It tries again at the next start, so please restart AI Studio. If this message appears again, please contact the support, and keep the folders '.restore-previous' and '.restore-pending' in your data folder." + +-- The batch processing uses '{0}', which does not exist on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1314976065"] = "The batch processing uses '{0}', which does not exist on this computer." + +-- The backup is from {0} and was made with AI Studio {1}. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1363238678"] = "The backup is from {0} and was made with AI Studio {1}." + +-- A backup holds no API keys or passwords, and some files and folders are not on this computer. Please check the following: +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1633897333"] = "A backup holds no API keys or passwords, and some files and folders are not on this computer. Please check the following:" + +-- Enter the API key of the provider '{0}' again, if it needs one. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1645175373"] = "Enter the API key of the provider '{0}' again, if it needs one." + +-- Restoring your backup is not finished +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2415452768"] = "Restoring your backup is not finished" + +-- Enter the API key of the embedding provider '{0}' again, if it needs one. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2492749778"] = "Enter the API key of the embedding provider '{0}' again, if it needs one." + +-- Enter the API key of the transcription provider '{0}' again, if it needs one. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2568556297"] = "Enter the API key of the transcription provider '{0}' again, if it needs one." + +-- Nothing else to do: every provider and every data source has what it needs on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2671436044"] = "Nothing else to do: every provider and every data source has what it needs on this computer." + +-- The data source '{0}' names a file or folder which does not exist on this computer. Choose it anew. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T270687216"] = "The data source '{0}' names a file or folder which does not exist on this computer. Choose it anew." + +-- The file with your own root certificates, '{0}', does not exist on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2844911500"] = "The file with your own root certificates, '{0}', does not exist on this computer." + +-- AI Studio could not start restoring your backup, so all your data stayed as they were. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3044053464"] = "AI Studio could not start restoring your backup, so all your data stayed as they were." + +-- Enter the password of the mailbox '{0}' again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3083231765"] = "Enter the password of the mailbox '{0}' again." + +-- Enter the password or token of the data source '{0}' again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3430781520"] = "Enter the password or token of the data source '{0}' again." + +-- Technical details: {0} +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3436390766"] = "Technical details: {0}" + +-- Close +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3448155331"] = "Close" + +-- Enter the setting '{1}' of the tool '{0}' again, if you used it. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3658964195"] = "Enter the setting '{1}' of the tool '{0}' again, if you used it." + +-- AI Studio replaced your data with those of the backup. The plugins and settings of your organization stayed as they were. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3896631626"] = "AI Studio replaced your data with those of the backup. The plugins and settings of your organization stayed as they were." + +-- Your backup was restored +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3980336073"] = "Your backup was restored" + +-- The ERI server '{0}' writes into a folder which does not exist on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T472632172"] = "The ERI server '{0}' writes into a folder which does not exist on this computer." + -- How many matches do you want at most per query? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1DIALOG::T1827669611"] = "How many matches do you want at most per query?" @@ -9385,6 +9697,9 @@ UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2800137365"] = "An update to ver -- Please wait for the update to complete... UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2864211629"] = "Please wait for the update to complete..." +-- Restore of a Backup +UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2912921801"] = "Restore of a Backup" + -- Supporters UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2929332068"] = "Supporters" @@ -10189,6 +10504,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3294830584"] = "Discover MindWor -- External HTTPS custom root certificates UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3315279770"] = "External HTTPS custom root certificates" +-- Data Backup & Restore +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3317700277"] = "Data Backup & Restore" + -- User-language provided by the OS UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3334355246"] = "User-language provided by the OS" @@ -10345,6 +10663,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T551035563"] = "Open Log Viewer" -- This is a library providing the foundations for asynchronous programming in Rust. It includes key trait definitions like Stream, as well as utilities like join!, select!, and various futures combinator methods which enable expressive asynchronous control flow. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "This is a library providing the foundations for asynchronous programming in Rust. It includes key trait definitions like Stream, as well as utilities like join!, select!, and various futures combinator methods which enable expressive asynchronous control flow." +-- Your Data Folders +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T570573187"] = "Your Data Folders" + -- Used .NET SDK UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Used .NET SDK" @@ -10528,15 +10849,15 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::PLUGINS::T837269472"] = "The plugin could not -- Settings UI_TEXT_CONTENT["AISTUDIO::PAGES::SETTINGS::T1258653480"] = "Settings" +-- Data Backup & Restore +UI_TEXT_CONTENT["AISTUDIO::PAGES::SETTINGS::T3317700277"] = "Data Backup & Restore" + -- Thank you very much, Kerstin, for creating the Wiki, for advising us on ERI with real use cases, and for your work on version 2 of the ERI protocol. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1367356224"] = "Thank you very much, Kerstin, for creating the Wiki, for advising us on ERI with real use cases, and for your work on version 2 of the ERI protocol." -- Thank you for being the first to contribute a one-time donation. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1470916504"] = "Thank you for being the first to contribute a one-time donation." --- Thank you, Jens, for the Batch Processing Assistant, the exact token count in the chat, and everything else you are building for AI Studio with so much dedication. -UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1701321325"] = "Thank you, Jens, for the Batch Processing Assistant, the exact token count in the chat, and everything else you are building for AI Studio with so much dedication." - -- Thank you, Peer, for your courage in being the second person to support the project financially. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1714878838"] = "Thank you, Peer, for your courage in being the second person to support the project financially." @@ -10624,6 +10945,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T686206269"] = "Thanks for your bu -- Business Contributors UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T838479287"] = "Business Contributors" +-- Thank you, Jens, for the Batch Processing Assistant, the exact token count in the chat, the backups of all data, and everything else you are building for AI Studio with so much dedication. +UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T882777973"] = "Thank you, Jens, for the Batch Processing Assistant, the exact token count in the chat, the backups of all data, and everything else you are building for AI Studio with so much dedication." + -- Thanks, Oliver, for making sure no spaces get lost in answers from OpenAI. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T916186976"] = "Thanks, Oliver, for making sure no spaces get lost in answers from OpenAI." @@ -11266,6 +11590,15 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::AUTHMETHODSV1EXTENSIONS::T810547195"] = "None" -- Username & Password UI_TEXT_CONTENT["AISTUDIO::TOOLS::AUTHMETHODSV1EXTENSIONS::T863358904"] = "Username & Password" +-- Unexpected error: {0} +UI_TEXT_CONTENT["AISTUDIO::TOOLS::CHATARCHIVE::CHATARCHIVEIMPORTER::T2350673880"] = "Unexpected error: {0}" + +-- This file is no backup of AI Studio, or it was not written completely. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::CHATARCHIVE::CHATARCHIVEIMPORTER::T3123318432"] = "This file is no backup of AI Studio, or it was not written completely." + +-- Imported workspace +UI_TEXT_CONTENT["AISTUDIO::TOOLS::CHATARCHIVE::CHATARCHIVEIMPORTER::T722364324"] = "Imported workspace" + -- Spanish (Spain) UI_TEXT_CONTENT["AISTUDIO::TOOLS::COMMONLANGUAGEEXTENSIONS::T1269145091"] = "Spanish (Spain)" @@ -11401,6 +11734,24 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T3893997203"] = " -- Trust all LLM providers UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T4107860491"] = "Trust all LLM providers" +-- This file is no backup of AI Studio, or it was not written completely. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATABACKUPREADER::T3123318432"] = "This file is no backup of AI Studio, or it was not written completely." + +-- This backup was made by a newer version of AI Studio. Please update AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATABACKUPREADER::T3812849494"] = "This backup was made by a newer version of AI Studio. Please update AI Studio first." + +-- The content list of this backup could not be read. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATABACKUPREADER::T3954689017"] = "The content list of this backup could not be read." + +-- This file is no backup of AI Studio, or it was not written completely. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATARESTOREPREPARER::T3123318432"] = "This file is no backup of AI Studio, or it was not written completely." + +-- The backup is damaged: the file '{0}' does not match its content list. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATARESTOREPREPARER::T3493230602"] = "The backup is damaged: the file '{0}' does not match its content list." + +-- This backup was made by a newer version of AI Studio. Please update AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATARESTOREPREPARER::T3812849494"] = "This backup was made by a newer version of AI Studio. Please update AI Studio first." + -- Reason UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1093747001"] = "Reason" @@ -11551,6 +11902,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T858000918"] = "Thes -- Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while. UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T874850580"] = "Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while." +-- AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T1006756723"] = "AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done." + +-- Close +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T3448155331"] = "Close" + -- Repair Data Source UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Repair Data Source" @@ -12481,6 +12838,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T377 -- Text UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text" +-- AI Studio backup +UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1046878067"] = "AI Studio backup" + -- Office Files UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office Files" @@ -12742,9 +13102,45 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGINGENERATIONSERVICE::T5 -- Workspace UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGINGENERATIONSERVICE::T658612054"] = "Workspace" +-- A backup is running already. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T1319749333"] = "A backup is running already." + +-- The backup is too large for the chosen drive, which takes no files beyond 4 GB. USB sticks formatted with FAT32 are such drives. Please choose another drive. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T1869616367"] = "The backup is too large for the chosen drive, which takes no files beyond 4 GB. USB sticks formatted with FAT32 are such drives. Please choose another drive." + +-- The backup failed: {0} +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T1915001713"] = "The backup failed: {0}" + +-- Your organization does not allow backups. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T2011318013"] = "Your organization does not allow backups." + +-- AI Studio is still writing an answer or a transcript. Please start the backup once it is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T4161531565"] = "AI Studio is still writing an answer or a transcript. Please start the backup once it is done." + +-- Your organization does not allow restoring backups. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T1032491606"] = "Your organization does not allow restoring backups." + +-- AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T1555678613"] = "AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first." + +-- AI Studio is still writing an answer or a transcript. Please restore once it is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T1847516630"] = "AI Studio is still writing an answer or a transcript. Please restore once it is done." + +-- The restore could not be prepared: {0} +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T2653345012"] = "The restore could not be prepared: {0}" + +-- A restore is being prepared already. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T4157700208"] = "A restore is being prepared already." + +-- A backup is running. Please restore once it is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T553866901"] = "A backup is running. Please restore once it is done." + -- The local RAG index database is not available. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1738200026"] = "The local RAG index database is not available." +-- The index of the data source '{0}' is missing. The data source stays out of your chats until its index was built anew. Use the repair action to start that. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T204783307"] = "The index of the data source '{0}' is missing. The data source stays out of your chats until its index was built anew. Use the repair action to start that." + -- The selected embedding provider is not allowed to index this data source. The data source asks for the confidence level '{0}', while the embedding provider has '{1}'. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2186533187"] = "The selected embedding provider is not allowed to index this data source. The data source asks for the confidence level '{0}', while the embedding provider has '{1}'." @@ -12850,6 +13246,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T2360 -- The mail '{0}' could not be indexed. AI Studio tries again during the next sync. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T804280374"] = "The mail '{0}' could not be indexed. AI Studio tries again during the next sync." +-- The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1214987337"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index." + +-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1393648320"] = "The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source." + -- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1935191670"] = "The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run." @@ -12859,15 +13261,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T242460 -- The folder '{0}' could not be opened. Please check whether you are allowed to read it. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3230000698"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it." +-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3722338131"] = "The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source." + -- The file '{0}' could not be read. Please check whether you are allowed to read it. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3924882233"] = "The file '{0}' could not be read. Please check whether you are allowed to read it." --- The file '{0}' does not exist. -UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T451561215"] = "The file '{0}' does not exist." - --- The folder '{0}' does not exist. -UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T871336081"] = "The folder '{0}' does not exist." - -- The local index '{0}' could not be created again. Please restart AI Studio and try once more. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::INDEXEDRUNCONTEXT::T1394295123"] = "The local index '{0}' could not be created again. Please restart AI Studio and try once more." diff --git a/app/MindWork AI Studio/Chat/ChatThread.cs b/app/MindWork AI Studio/Chat/ChatThread.cs index 276f8ee27..e19af0c5e 100644 --- a/app/MindWork AI Studio/Chat/ChatThread.cs +++ b/app/MindWork AI Studio/Chat/ChatThread.cs @@ -555,6 +555,53 @@ public sealed record ChatThread return true; } + /// + /// Replaces every attachment of the chat, in its messages as well as in its draft, with what the given function makes of it. + /// + /// + /// The one place which knows where a chat keeps attachments. Moving a chat, copying it, and + /// writing it into a backup or reading it back from one all go through here, so none of them + /// forgets the draft or a kind of message. + /// + /// Gives the attachment to keep in place of the given one, or the given one itself when nothing changes. + public void ReplaceAttachments(Func replace) + { + foreach (var content in this.Blocks.Select(block => block.Content).OfType()) + { + for (var index = 0; index < content.FileAttachments.Count; index++) + content.FileAttachments[index] = replace(content.FileAttachments[index]); + } + + this.Draft.ReplaceAttachments(replace); + } + + /// + /// Replaces every file path the chat stores with what the given function makes of it. + /// + /// + /// Covers the attachments, cf. ReplaceAttachments, and the images of the messages which are held + /// by a local path. An attachment whose path the function returns unchanged stays as it is, and + /// so does one without a path. + /// + /// Maps a path to its replacement, or returns it unchanged. + public void RewriteFilePaths(Func rewrite) + { + this.ReplaceAttachments(attachment => + { + if (string.IsNullOrWhiteSpace(attachment.FilePath)) + return attachment; + + var rewrittenPath = rewrite(attachment.FilePath); + return rewrittenPath == attachment.FilePath ? attachment : attachment.WithFilePath(rewrittenPath); + }); + + foreach (var image in this.Blocks.Select(block => block.Content).OfType()) + { + if (image.SourceType is ContentImageSource.LOCAL_PATH && !string.IsNullOrWhiteSpace(image.Source)) + image.Source = rewrite(image.Source); + } + } + /// /// Finds what a provider reported for this conversation, as long as the report still describes it. /// diff --git a/app/MindWork AI Studio/Chat/FileAttachment.cs b/app/MindWork AI Studio/Chat/FileAttachment.cs index 7d29f73be..9c68b5e37 100644 --- a/app/MindWork AI Studio/Chat/FileAttachment.cs +++ b/app/MindWork AI Studio/Chat/FileAttachment.cs @@ -9,13 +9,13 @@ namespace AIStudio.Chat; /// /// The type of the file attachment. /// The name of the file, including extension. -/// The full path to the file, including the filename and extension. +/// The full path to the file, including the filename and extension. A file below the data directory, such as a transcript, is stored with a path relative to it. /// The size of the file in bytes. [JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] [JsonDerivedType(typeof(FileAttachment), typeDiscriminator: "file")] [JsonDerivedType(typeof(FileAttachmentImage), typeDiscriminator: "image")] [JsonDerivedType(typeof(ManagedTranscriptAttachment), typeDiscriminator: "managed_transcript")] -public record FileAttachment(FileAttachmentType Type, string FileName, string FilePath, long FileSizeBytes) +public record FileAttachment(FileAttachmentType Type, string FileName, [property: JsonConverter(typeof(DataDirectoryRelativePathConverter))] string FilePath, long FileSizeBytes) { /// /// Gets a value indicating whether the file type is forbidden and should not be attached. @@ -63,6 +63,17 @@ public record FileAttachment(FileAttachmentType Type, string FileName, string Fi /// public virtual FileAttachment Normalize() => FromPath(this.FilePath); + /// + /// Makes the same attachment for its file at another place, e.g. after the chat holding the file moved. + /// + /// + /// Deliberately no with expression: that one keeps whatever was derived from the old path when + /// the attachment was made, such as FilePathAsUrl, and the source of an image. + /// + /// The path of the file now. + /// An attachment of the same kind, with the new path. + public virtual FileAttachment WithFilePath(string filePath) => new(this.Type, this.FileName, filePath, this.FileSizeBytes); + /// /// Creates a FileAttachment from a file path by automatically determining the type, /// extracting the filename, and reading the file size. diff --git a/app/MindWork AI Studio/Chat/FileAttachmentImage.cs b/app/MindWork AI Studio/Chat/FileAttachmentImage.cs index 81b46f202..9bdf97333 100644 --- a/app/MindWork AI Studio/Chat/FileAttachmentImage.cs +++ b/app/MindWork AI Studio/Chat/FileAttachmentImage.cs @@ -1,3 +1,5 @@ +using System.Text.Json.Serialization; + namespace AIStudio.Chat; public record FileAttachmentImage(string FileName, string FilePath, long FileSizeBytes) : FileAttachment(FileAttachmentType.IMAGE, FileName, FilePath, FileSizeBytes), IImageSource @@ -13,5 +15,18 @@ public record FileAttachmentImage(string FileName, string FilePath, long FileSiz /// /// The image source. /// + /// + /// An attached image is a file, so its source is the path of that file. It is not stored a second + /// time: the stored file path is relative when the image lies below the data directory, and the + /// source has to follow it when the chat is read again, on whatever computer that happens. + /// + [JsonIgnore] public string Source { get; set; } = FilePath; + + /// + public override FileAttachment WithFilePath(string filePath) => new FileAttachmentImage(this.FileName, filePath, this.FileSizeBytes) + { + SourceType = this.SourceType, + Source = this.SourceType is ContentImageSource.LOCAL_PATH ? filePath : this.Source, + }; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs b/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs index 4b8117342..5cc360f20 100644 --- a/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs +++ b/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs @@ -15,6 +15,17 @@ namespace AIStudio.Chat; public sealed record ManagedTranscriptAttachment(string FileName, string FilePath, long FileSizeBytes, string OriginalFileName, bool IsStaged) : FileAttachment(FileAttachmentType.DOCUMENT, FileName, FilePath, FileSizeBytes) { + /// The folder within a chat directory which holds the attachments AI Studio made for the chat. + private const string ATTACHMENTS_FOLDER = "attachments"; + + /// The folder within the attachments folder which holds the transcripts. + private const string TRANSCRIPTS_FOLDER = "transcripts"; + + /// Gets the directory which holds the transcripts of a chat. + /// The directory of the chat. + /// The transcript directory, which may not exist yet. + public static string GetTranscriptDirectory(string chatDirectory) => Path.Combine(chatDirectory, ATTACHMENTS_FOLDER, TRANSCRIPTS_FOLDER); + /// Refreshes the path-derived name and current file size. public override FileAttachment Normalize() { @@ -22,6 +33,9 @@ public sealed record ManagedTranscriptAttachment(string FileName, string FilePat return this with { FileName = Path.GetFileName(this.FilePath), FileSizeBytes = size }; } + /// + public override FileAttachment WithFilePath(string filePath) => new ManagedTranscriptAttachment(this.FileName, filePath, this.FileSizeBytes, this.OriginalFileName, this.IsStaged); + /// Creates a transcript in an operation-specific staging directory. /// Original media path. /// Provider transcript. @@ -71,7 +85,7 @@ public sealed record ManagedTranscriptAttachment(string FileName, string FilePat var fullDataRoot = Path.GetFullPath(SettingsManager.DataDirectory!); var relative = Path.GetRelativePath(fullDataRoot, fullFilePath); - if (Path.IsPathRooted(relative) || relative == ".." || relative.StartsWith($"..{Path.DirectorySeparatorChar}", PathComparison)) + if (Path.IsPathRooted(relative) || relative == ".." || relative.StartsWith($"..{Path.DirectorySeparatorChar}", PathTools.COMPARISON)) return false; if (fileInfo.LinkTarget is not null || HasLinkedDirectory(fileInfo.Directory, fullDataRoot)) @@ -82,16 +96,12 @@ public sealed record ManagedTranscriptAttachment(string FileName, string FilePat && Guid.TryParseExact(segments[1], "N", out _) && !string.IsNullOrWhiteSpace(segments[2]); - var isTemporaryChatTranscript = segments is ["tempChats", _, _, _, _] + var isTemporaryChatTranscript = segments is ["tempChats", _, ATTACHMENTS_FOLDER, TRANSCRIPTS_FOLDER, _] + && Guid.TryParse(segments[1], out _); + + var isWorkspaceChatTranscript = segments is ["workspaces", _, _, ATTACHMENTS_FOLDER, TRANSCRIPTS_FOLDER, _] && Guid.TryParse(segments[1], out _) - && segments[2] == "attachments" - && segments[3] == "transcripts"; - - var isWorkspaceChatTranscript = segments is ["workspaces", _, _, _, _, _] - && Guid.TryParse(segments[1], out _) - && Guid.TryParse(segments[2], out _) - && segments[3] == "attachments" - && segments[4] == "transcripts"; + && Guid.TryParse(segments[2], out _); if (!isStaging && !isTemporaryChatTranscript && !isWorkspaceChatTranscript) return false; @@ -107,7 +117,7 @@ public sealed record ManagedTranscriptAttachment(string FileName, string FilePat /// Rejects paths traversing any symbolic-link or junction directory below the data root. private static bool HasLinkedDirectory(DirectoryInfo? directory, string fullDataRoot) { - while (directory is not null && !string.Equals(Path.GetFullPath(directory.FullName), fullDataRoot, PathComparison)) + while (directory is not null && !string.Equals(Path.GetFullPath(directory.FullName), fullDataRoot, PathTools.COMPARISON)) { if (directory.LinkTarget is not null) return true; @@ -163,7 +173,4 @@ public sealed record ManagedTranscriptAttachment(string FileName, string FilePat await File.WriteAllTextAsync(path, markdown, new UTF8Encoding(false)); } - - /// Gets the platform path comparison used for canonical containment checks. - private static StringComparison PathComparison => OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/DataBackupPanel.razor b/app/MindWork AI Studio/Components/DataBackupPanel.razor new file mode 100644 index 000000000..6942679de --- /dev/null +++ b/app/MindWork AI Studio/Components/DataBackupPanel.razor @@ -0,0 +1,225 @@ +@using AIStudio.Tools.ChatArchive +@inherits MSGComponentBase + + + + + + @T("Back up all your data into one file: your settings, chats, plugins, and the indexes of your data sources. Your API keys and passwords stay out of it. AI Studio keeps them in the secure storage of your OS.") + + + @if (this.DataBackupService.IsRunning) + { + + + + @this.BackupProgressText + + + @T("Cancel") + + + + @T("The backup goes on when you leave this page.") + + } + else + { + + + @if (this.DataBackupService.LastResult is { } backupResult) + { + @if (backupResult.Success) + { + + + @string.Format(T("Backed up {0} files into: {1}"), backupResult.FileCount, backupResult.ArchivePath) + + @if (!backupResult.IncludesDataSourceIndexes) + { + + @T("The indexes of your data sources are not part of this backup, because they were not available. After a restore, AI Studio indexes your data sources anew.") + + } + + } + else if (backupResult.Cancelled) + { + + @T("The backup was stopped. No file was left behind.") + + } + else + { + + @backupResult.Issue + + } + } + } + + + + + + @T("Restore a backup: replace all your data with those of the backup, e.g., on a new computer, or add only its chats to the chats you have.") + + + @if (this.DataRestoreService.IsRestoreUnfinished) + { + + @T("AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first.") + + } + + + + @if (!string.IsNullOrWhiteSpace(this.selectedArchivePath)) + { + + @this.selectedArchivePath + + } + + + @if (this.preview is { } backupPreview) + { + @if (!backupPreview.Success) + { + + @backupPreview.Issue + + } + else + { + + + @string.Format(T("Created on {0} with AI Studio {1}"), CommonTools.FormatTimestampToGeneral(backupPreview.Manifest.CreatedAt.ToLocalTime().DateTime, this.currentCultureInfo), backupPreview.Manifest.AppVersion) + + + @foreach (var workspace in backupPreview.Manifest.Workspaces) + { + + @($"{workspace.Name} ({this.ChatCount(workspace.Chats.Count)})") + + } + + @if (backupPreview.Manifest.TemporaryChats.Count > 0) + { + + @($"{T("Temporary chats")} ({this.ChatCount(backupPreview.Manifest.TemporaryChats.Count)})") + + } + + + + @T("Replace all data") + + + @T("Replaces your settings, chats, plugins, and the indexes of your data sources with those of the backup. AI Studio restarts to do so. The plugins and settings of your organization stay as they are.") + + + @if (this.DataRestoreService.IsPreparing) + { + + + + @this.RestoreProgressText + + + @T("Cancel") + + + } + else + { + + @T("Replace all data") + + } + + @if (this.DataRestoreService.LastResult is { Success: false } restoreResult) + { + + @(restoreResult.Cancelled ? T("The restore was stopped. All your data stay as they are.") : restoreResult.Issue) + + } + + + @T("Add chats only") + + + @T("Adds the chats of the backup to the chats you have. Nothing else changes.") + + + + + @T("Keep the existing chat") + + + @T("Add it as an additional chat") + + + + @if (this.isImporting) + { + + + + @string.Format(T("Chat {0} of {1}"), this.importProgress.ProcessedChats + 1, this.importProgress.TotalChats) + + + @T("Cancel") + + + + @T("Please stay on this page until this is done. Leaving the page stops adding the chats.") + + } + else + { + + @this.AddChatsText(backupPreview.Manifest.TotalChatCount) + + } + } + } + + @if (this.importResult is { } importOutcome) + { + @if (importOutcome.Success) + { + + + @string.Format(T("Chats added: {0}. Workspaces they went into: {1}."), importOutcome.ImportedChats, importOutcome.ImportedWorkspaces) + + @if (importOutcome.Cancelled) + { + + @T("Adding the chats was stopped. The chats added until then were kept.") + + } + @if (importOutcome.SkippedChats > 0) + { + + @string.Format(T("Chats which already existed and stayed as they are: {0}."), importOutcome.SkippedChats) + + } + @if (importOutcome.FailedChats > 0) + { + + @string.Format(T("Chats which could not be added: {0}."), importOutcome.FailedChats) + + } + + } + else + { + + @importOutcome.Issue + + } + } + + + \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/DataBackupPanel.razor.cs b/app/MindWork AI Studio/Components/DataBackupPanel.razor.cs new file mode 100644 index 000000000..ab2b8972c --- /dev/null +++ b/app/MindWork AI Studio/Components/DataBackupPanel.razor.cs @@ -0,0 +1,308 @@ +using System.Globalization; + +using AIStudio.Dialogs; +using AIStudio.Tools.ChatArchive; +using AIStudio.Tools.DataBackup; +using AIStudio.Tools.Rust; +using AIStudio.Tools.Services; + +using Microsoft.AspNetCore.Components; + +using DialogOptions = AIStudio.Dialogs.DialogOptions; + +namespace AIStudio.Components; + +/// +/// Backs up all data into one file, and restores such a backup: as a whole, or only its chats. +/// +/// +/// It appears on the information page and in the settings. A backup and the preparation of a restore +/// run in their services, so leaving the page does not stop them; this component only shows them. +/// Adding chats runs here, and leaving the page stops it. +/// +public partial class DataBackupPanel : MSGComponentBase +{ + /// + /// The shortest time between two progress renders. Without this, a large backup would re-render + /// the whole page once per file or chat. + /// + private static readonly TimeSpan PROGRESS_RENDER_INTERVAL = TimeSpan.FromMilliseconds(100); + + [Inject] + private ILogger Logger { get; init; } = null!; + + [Inject] + private RustService RustService { get; init; } = null!; + + [Inject] + private IDialogService DialogService { get; init; } = null!; + + [Inject] + private DataBackupService DataBackupService { get; init; } = null!; + + [Inject] + private DataRestoreService DataRestoreService { get; init; } = null!; + + private ChatArchiveCollisionBehavior collisionBehavior = ChatArchiveCollisionBehavior.SKIP; + private CancellationTokenSource? importCancellation; + private ChatArchiveProgress importProgress; + private ChatArchiveImportPreview? preview; + private ChatArchiveImportResult? importResult; + private DateTimeOffset lastImportRender; + private DateTimeOffset lastServiceRender; + private string selectedArchivePath = string.Empty; + + /// + /// The culture of the active language plugin, which dates are shown in. + /// + private CultureInfo currentCultureInfo = CultureInfo.InvariantCulture; + + /// + /// Blocks any further action, starting with the click that opens a file dialog. Without this, a + /// second click during the open dialog would start a second run. + /// + private bool isBusy; + + /// + /// Whether chats are being added, which shows their progress. + /// + private bool isImporting; + + private bool AllowBackup => this.SettingsManager.ConfigurationData.App.AllowUserToBackUpData; + + private bool AllowRestore => this.SettingsManager.ConfigurationData.App.AllowUserToRestoreData; + + /// + /// Whether the chosen backup can replace all data now. + /// + /// + /// The permission is asked here again and not only when choosing the backup: a configuration + /// plugin may take it away while a chosen backup waits. + /// + private bool CanRestore => this.preview is { Success: true } && this.AllowRestore && !this.DataRestoreService.IsRestoreUnfinished; + + /// + /// Whether the chats of the chosen backup can be added now. + /// + private bool CanImport => this.preview is { Success: true, Manifest.TotalChatCount: > 0 } && this.AllowRestore && !this.DataRestoreService.IsPreparing; + + private double ImportProgressPercentage => this.importProgress.TotalChats is 0 ? 0d : this.importProgress.ProcessedChats * 100d / this.importProgress.TotalChats; + + private double BackupProgressPercentage => this.DataBackupService.Progress.Total is 0 ? 0d : this.DataBackupService.Progress.Processed * 100d / this.DataBackupService.Progress.Total; + + private double RestoreProgressPercentage => this.DataRestoreService.Progress.Total is 0 ? 0d : this.DataRestoreService.Progress.Processed * 100d / this.DataRestoreService.Progress.Total; + + private string BackupProgressText => this.DataBackupService.Progress switch + { + { Phase: DataBackupPhase.COPYING_INDEXES } current => string.Format(T("Copying the indexes of your data sources: {0} of {1}"), current.Processed + 1, current.Total), + { Phase: DataBackupPhase.WRITING, Total: > 0 } current => string.Format(T("Writing file {0} of {1}"), current.Processed, current.Total), + _ => T("Preparing the backup"), + }; + + private string RestoreProgressText => this.DataRestoreService.Progress is { Total: > 0 } current + ? string.Format(T("Unpacking file {0} of {1}"), current.Processed, current.Total) + : T("Reading the backup"); + + #region Overrides of MSGComponentBase + + protected override async Task OnInitializedAsync() + { + this.DataBackupService.StateChanged += this.OnServiceStateChanged; + this.DataRestoreService.StateChanged += this.OnServiceStateChanged; + + var activeLanguagePlugin = await this.SettingsManager.GetActiveLanguagePlugin(); + this.currentCultureInfo = CommonTools.DeriveActiveCultureOrInvariant(activeLanguagePlugin.IETFTag); + + await base.OnInitializedAsync(); + } + + protected override void DisposeResources() + { + this.DataBackupService.StateChanged -= this.OnServiceStateChanged; + this.DataRestoreService.StateChanged -= this.OnServiceStateChanged; + this.importCancellation?.Cancel(); + this.DisposeImportCancellation(); + } + + #endregion + + /// + /// The number of chats, in the singular for one of them. + /// + private string ChatCount(int count) => count is 1 ? T("1 chat") : string.Format(T("{0} chats"), count); + + private string AddChatsText(int count) => count is 1 ? T("Add 1 chat") : string.Format(T("Add {0} chats"), count); + + /// + /// Shows what a backup or the preparation of a restore does, which runs in its service, not here. + /// + private void OnServiceStateChanged() + { + // Render at most every PROGRESS_RENDER_INTERVAL, but never skip the end of a run: + var now = DateTimeOffset.UtcNow; + var isRunning = this.DataBackupService.IsRunning || this.DataRestoreService.IsPreparing; + if (isRunning && now - this.lastServiceRender < PROGRESS_RENDER_INTERVAL) + return; + + this.lastServiceRender = now; + _ = this.InvokeAsync(this.StateHasChanged); + } + + private async Task StartBackupAsync() + { + if (this.isBusy || this.DataBackupService.IsRunning || !this.AllowBackup) + return; + + string archivePath; + this.isBusy = true; + try + { + var suggestedFileName = $"ai-studio-backup-{DateTime.Now:yyyy-MM-dd}{DataBackupFormat.FILE_EXTENSION}"; + var saveResponse = await this.RustService.SaveFile(T("Back up data"), [FileTypes.BACKUP_ARCHIVE], suggestedFileName); + if (saveResponse.UserCancelled) + return; + + archivePath = saveResponse.SaveFilePath; + } + finally + { + this.isBusy = false; + } + + // + // The service runs the backup and reports every step through StateChanged, so leaving the + // page does not stop it: + // + await this.DataBackupService.CreateBackupAsync(archivePath); + } + + private async Task ChooseArchiveAsync() + { + if (this.isBusy || !this.AllowRestore) + return; + + this.isBusy = true; + try + { + var selectionResponse = await this.RustService.SelectFile(T("Select a backup"), [FileTypes.BACKUP_ARCHIVE]); + if (selectionResponse.UserCancelled) + return; + + this.selectedArchivePath = selectionResponse.SelectedFilePath; + this.importResult = null; + this.preview = await ChatArchiveImporter.ReadPreviewAsync(this.selectedArchivePath, CancellationToken.None); + } + finally + { + this.isBusy = false; + + // The lockable button calls this as a plain function, not as an event of this + // component, so nothing renders the preview unless we do: + await this.InvokeAsync(this.StateHasChanged); + } + } + + /// + /// Asks whether to replace all data with those of the chosen backup, and prepares the restore. + /// + /// + /// Once the restore is prepared, the main layout asks for the restart and blocks everything else + /// until then: whatever changes meanwhile would be lost. + /// + private async Task StartRestoreAsync() + { + if (this.isBusy || !this.CanRestore) + return; + + this.isBusy = true; + try + { + var dialogParameters = new DialogParameters + { + { x => x.Message, T("Restoring this backup replaces your settings, your chats, your plugins, and the indexes of your data sources with those of the backup. Whatever you changed since the backup was made is lost. Afterward, AI Studio restarts. Do you want to restore this backup?") }, + }; + + var dialogReference = await this.DialogService.ShowAsync(T("Replace All Data"), dialogParameters, DialogOptions.FULLSCREEN); + var dialogResult = await dialogReference.Result; + if (dialogResult is null || dialogResult.Canceled) + return; + } + finally + { + this.isBusy = false; + } + + await this.DataRestoreService.PrepareRestoreAsync(this.selectedArchivePath); + } + + private async Task StartImportAsync() + { + if (this.isBusy || !this.CanImport) + return; + + this.isBusy = true; + try + { + this.importResult = null; + this.isImporting = true; + this.importProgress = new(0, this.preview!.Manifest.TotalChatCount); + this.importCancellation = new(); + + var archivePath = this.selectedArchivePath; + var token = this.importCancellation.Token; + var reporter = this.CreateImportProgressReporter(); + this.importResult = await Task.Run(() => ChatArchiveImporter.ImportAsync(archivePath, this.collisionBehavior, reporter, token), token); + + // + // Drop the preview once every chat was added. Otherwise, another click on the button + // would silently add all of them a second time: + // + if (this.importResult is { Success: true, Cancelled: false }) + { + this.preview = null; + this.selectedArchivePath = string.Empty; + } + } + catch (OperationCanceledException) + { + this.Logger.LogInformation("The user stopped adding the chats of a backup."); + } + finally + { + this.isImporting = false; + this.DisposeImportCancellation(); + this.isBusy = false; + } + } + + private void CancelImport() => this.importCancellation?.Cancel(); + + /// + /// Detaches the cancellation source before disposing it, so that a late click on the cancel + /// button cannot reach a disposed source. + /// + private void DisposeImportCancellation() + { + var cancellation = this.importCancellation; + this.importCancellation = null; + + cancellation?.Dispose(); + } + + private IProgress CreateImportProgressReporter() + { + this.lastImportRender = DateTimeOffset.MinValue; + return new Progress(value => + { + this.importProgress = value; + + // Render at most every PROGRESS_RENDER_INTERVAL, but never skip the final state: + var now = DateTimeOffset.UtcNow; + if (value.ProcessedChats < value.TotalChats && now - this.lastImportRender < PROGRESS_RENDER_INTERVAL) + return; + + this.lastImportRender = now; + this.InvokeAsync(this.StateHasChanged); + }); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/DataFolders.razor b/app/MindWork AI Studio/Components/DataFolders.razor new file mode 100644 index 000000000..bfdd04b96 --- /dev/null +++ b/app/MindWork AI Studio/Components/DataFolders.razor @@ -0,0 +1,115 @@ +@using AIStudio.Tools.DataBackup +@inherits MSGComponentBase + +@{ + var organizationPluginFolders = OrganizationPluginFolders; +} + + + @this.ExpertNotice + + + + @T("Explanation") + + + +@foreach (var location in LOCATIONS) +{ + var folder = FolderPath(location); + + @this.FolderName(location) + + + + + + @folder + + + + + + + +} + +@if (IsDataFolderInsideConfigFolder) +{ + + @T("On this computer, the data folder lies inside the configuration folder. A backup of the configuration folder therefore takes the data folder along.") + +} + + + @T("What belongs into a backup") + + + + + @T("Folder") + @T("Path") + @T("Content") + + + + @foreach (var entry in INCLUDED_ENTRIES) + { + + @this.FolderName(entry.Location) + @entry.RelativePath + @this.Describe(entry.Kind) + + } + + + + + @T("What a backup can leave out") + + + + + @T("Folder") + @T("Path") + @T("Reason") + + + + @foreach (var entry in EXCLUDED_ENTRIES) + { + + @this.FolderName(entry.Location) + @entry.RelativePath + @this.Describe(entry.Kind) + + } + @foreach (var pluginFolder in organizationPluginFolders) + { + + @this.FolderName(DataBackupLocation.DATA_DIRECTORY) + @pluginFolder + @this.Describe(DataBackupEntryKind.ORGANIZATION_PLUGINS) + + } + + + + + @T("Good to know") + + + @T("Back up the indexes of your data sources only while AI Studio is closed. AI Studio writes to them while it runs, so a copy made meanwhile may not fit together, and AI Studio would have to embed your data sources anew.") + + + @T("Your API keys, tokens, and passwords are not in these folders. AI Studio keeps them in the secure storage of your OS. After you restored your data on another computer, enter them there once more.") + +@if (organizationPluginFolders.Count > 0) +{ + + @T("Your organization deployed some plugins among your own ones, for example, with a device management. Their folders do not show this, so the list above names them one by one.") + +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/DataFolders.razor.cs b/app/MindWork AI Studio/Components/DataFolders.razor.cs new file mode 100644 index 000000000..15aaee28b --- /dev/null +++ b/app/MindWork AI Studio/Components/DataFolders.razor.cs @@ -0,0 +1,126 @@ +using AIStudio.Settings; +using AIStudio.Tools.DataBackup; +using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.Rust; +using AIStudio.Tools.Services; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +/// +/// Shows the data and configuration folders of AI Studio, and which of their contents belong into a backup. +/// +/// +/// For users who back up their computer with a tool of their own. Everything shown here comes from +/// the same list the backups of AI Studio follow. It stays complete even when the organization does not +/// allow backups within AI Studio: that decides about the backups AI Studio makes, not about which +/// folders a user may copy. +/// +public partial class DataFolders : MSGComponentBase +{ + private static readonly DataBackupLocation[] LOCATIONS = [DataBackupLocation.DATA_DIRECTORY, DataBackupLocation.CONFIG_DIRECTORY]; + + private static readonly IReadOnlyList INCLUDED_ENTRIES = DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded).ToList(); + + private static readonly IReadOnlyList EXCLUDED_ENTRIES = DataBackupContents.ENTRIES.Where(entry => !entry.IsIncluded).ToList(); + + [Inject] + private RustService RustService { get; init; } = null!; + + [Inject] + private ILogger Logger { get; init; } = null!; + + /// + /// Tells everybody but experts where to go instead. + /// + /// + /// Pointing to the backups of AI Studio makes sense only while the organization allows them. + /// + private string ExpertNotice + { + get + { + var notice = T("The following is meant for experts who back up their computer with a tool of their own."); + return this.SettingsManager.ConfigurationData.App.AllowUserToBackUpData + ? $"{notice} {T("To back up your data with AI Studio itself, or to move it to another computer, use Data Backup & Restore on this page or in the settings instead.")}" + : notice; + } + } + + private static bool IsDataFolderInsideConfigFolder =>!string.IsNullOrWhiteSpace(SettingsManager.ConfigDirectory) + && !string.IsNullOrWhiteSpace(SettingsManager.DataDirectory) + && PathTools.IsInsideDirectory(SettingsManager.ConfigDirectory, SettingsManager.DataDirectory); + + /// + /// The plugins which the organization deployed among the user's own ones, as AI Studio found them when it loaded the plugins last. + /// + private static IReadOnlyList OrganizationPluginFolders => string.IsNullOrWhiteSpace(SettingsManager.DataDirectory) + ? [] + : DataBackupContents.FindOrganizationPluginsAmongUserPlugins(PluginFactory.AvailablePlugins.OfType(), SettingsManager.DataDirectory); + + private static string FolderPath(DataBackupLocation location) => location switch + { + DataBackupLocation.DATA_DIRECTORY => SettingsManager.DataDirectory ?? string.Empty, + DataBackupLocation.CONFIG_DIRECTORY => SettingsManager.ConfigDirectory ?? string.Empty, + _ => string.Empty, + }; + + private string FolderName(DataBackupLocation location) => location switch + { + DataBackupLocation.DATA_DIRECTORY => T("Data folder"), + DataBackupLocation.CONFIG_DIRECTORY => T("Configuration folder"), + _ => string.Empty, + }; + + private string Describe(DataBackupEntryKind kind) => kind switch + { + DataBackupEntryKind.SETTINGS => T("Your settings."), + DataBackupEntryKind.WORKSPACES => T("Your chats in workspaces, along with their transcripts."), + DataBackupEntryKind.TEMPORARY_CHATS => T("Your temporary chats, along with their transcripts."), + DataBackupEntryKind.PLUGINS => T("Your plugins, such as the assistants you built, apart from the folders listed below."), + DataBackupEntryKind.TOKENIZERS => T("The tokenizers you chose for your providers."), + DataBackupEntryKind.VISUAL_BRIEFINGS => T("Your projects in the Visual Briefing Assistant."), + DataBackupEntryKind.DATA_SOURCE_INDEX => T("The index of your local data sources: which files AI Studio embedded, and their text."), + DataBackupEntryKind.VECTOR_STORES => T("The embeddings of your local data sources."), + + DataBackupEntryKind.SETTINGS_OF_OTHER_VERSIONS => T("Copies of your settings, which every version of AI Studio keeps for itself. Your current settings are all a backup needs."), + DataBackupEntryKind.BUILT_IN_PLUGINS => T("Plugins which come with AI Studio, such as the language packs. AI Studio brings them back by itself."), + DataBackupEntryKind.ORGANIZATION_PLUGINS => T("Plugins of your organization. Your organization brings them back."), + DataBackupEntryKind.TEMPORARY_FILES => T("Files which AI Studio needs only for a moment and deletes again."), + DataBackupEntryKind.DOWNLOADED_TOOLS => T("Tools which AI Studio downloaded for this OS, such as Pandoc. On another computer, AI Studio downloads them again."), + DataBackupEntryKind.INTERFACE_KEYS => T("Keys with which AI Studio protects data in its user interface. Only the Flatpak keeps them here, and on another computer, AI Studio creates new ones."), + DataBackupEntryKind.LOGS => T("Log files. They help when something goes wrong, but they hold none of your data."), + + _ => string.Empty, + }; + + private async Task CopyPath(string path) => await this.RustService.CopyText2Clipboard(path); + + private async Task OpenInFileManager(string path) + { + if (string.IsNullOrWhiteSpace(path)) + { + await this.MessageBus.SendWarning(new(Icons.Material.Filled.Folder, T("The folder is not known yet."))); + return; + } + + OpenPathResponse response; + try + { + response = await this.RustService.TryOpenPathInRuntimeFileManager(path); + } + catch (Exception e) + { + this.Logger.LogWarning(e, "Could not open the folder '{Path}' in the file manager.", path); + await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, T("Could not open the folder."))); + return; + } + + if (response.Success) + return; + + var issue = string.IsNullOrWhiteSpace(response.Issue) ? T("Unknown error") : response.Issue; + await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, string.Format(T("Could not open the folder: {0}"), issue))); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs index 1eee3b8bc..f3d607f24 100644 --- a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs +++ b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs @@ -419,6 +419,9 @@ public partial class DataSourceManagement : MSGComponentBase if (dataSource.IsEnterpriseConfiguration) return; + if (await this.RefuseWhileIndexingPaused(T("Delete Data Source"))) + return; + var dialogParameters = new DialogParameters { { x => x.Message, string.Format(T("Are you sure you want to delete the data source '{0}' of type '{1}'?"), dataSource.Name, dataSource.Type.GetDisplayName()) }, @@ -457,6 +460,21 @@ public partial class DataSourceManagement : MSGComponentBase } } + /// + /// Tells the user to wait while a backup copies the indexes or a restore waits for the restart: + /// deleting a data source discards its index, which waits for the end of the pause. + /// + /// The title of the action which waits. + /// True when the action has to wait. + private async Task RefuseWhileIndexingPaused(string title) + { + if (!this.DataSourceEmbeddingService.IsIndexingPaused) + return false; + + await this.DialogService.ShowMessageBox(title, T("AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done."), T("Close")); + return true; + } + /// /// Edits a row of the table, which is either a data source or a mailbox. /// @@ -533,6 +551,9 @@ public partial class DataSourceManagement : MSGComponentBase if (mailbox.IsEnterpriseConfiguration) return; + if (await this.RefuseWhileIndexingPaused(T("Delete Mailbox"))) + return; + var dialogParameters = new DialogParameters { { x => x.Message, string.Format(T("Are you sure you want to delete the mailbox '{0}'? Your mails stay on the server as they are. AI Studio only deletes its index of them and the stored password."), mailbox.Name) }, diff --git a/app/MindWork AI Studio/Dialogs/DataRestorePendingDialog.razor b/app/MindWork AI Studio/Dialogs/DataRestorePendingDialog.razor new file mode 100644 index 000000000..0a7588990 --- /dev/null +++ b/app/MindWork AI Studio/Dialogs/DataRestorePendingDialog.razor @@ -0,0 +1,25 @@ +@inherits MSGComponentBase + + + + + + @T("Your backup is ready. AI Studio restarts to put it in place. Whatever you changed until then would be lost, so please restart now.") + + + @if (this.restartFailed) + { + + @T("AI Studio could not restart by itself. Please quit AI Studio and start it again.") + + } + + + + @T("Discard the restore") + + + @T("Restart now") + + + \ No newline at end of file diff --git a/app/MindWork AI Studio/Dialogs/DataRestorePendingDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataRestorePendingDialog.razor.cs new file mode 100644 index 000000000..30ff7125e --- /dev/null +++ b/app/MindWork AI Studio/Dialogs/DataRestorePendingDialog.razor.cs @@ -0,0 +1,60 @@ +using AIStudio.Components; +using AIStudio.Tools.Services; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Dialogs; + +/// +/// Blocks AI Studio while a prepared restore waits for the restart which applies it: whatever the user +/// changed until then would be lost. The user restarts, or discards the restore. +/// +public partial class DataRestorePendingDialog : MSGComponentBase +{ + [CascadingParameter] + private IMudDialogInstance MudDialog { get; set; } = null!; + + [Inject] + private DataRestoreService DataRestoreService { get; init; } = null!; + + [Inject] + private RustService RustService { get; init; } = null!; + + private bool isWorking; + + /// + /// Whether the runtime could not restart, e.g., in the development environment, where it did not + /// start the .NET server. + /// + private bool restartFailed; + + private async Task RestartAsync() + { + this.isWorking = true; + try + { + // On success, AI Studio ends right away, and this dialog with it: + this.restartFailed = !await this.RustService.RestartApplication(); + } + finally + { + this.isWorking = false; + } + } + + private async Task DiscardAsync() + { + this.isWorking = true; + try + { + await this.DataRestoreService.DiscardPendingRestoreAsync(); + } + finally + { + this.isWorking = false; + } + + if (!this.DataRestoreService.IsRestorePending) + this.MudDialog.Close(DialogResult.Ok(false)); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Dialogs/DataRestoreReportDialog.razor b/app/MindWork AI Studio/Dialogs/DataRestoreReportDialog.razor new file mode 100644 index 000000000..0ce16942c --- /dev/null +++ b/app/MindWork AI Studio/Dialogs/DataRestoreReportDialog.razor @@ -0,0 +1,56 @@ +@using AIStudio.Tools.DataBackup +@inherits MSGComponentBase + + + + + + + @this.Headline + + + @this.Explanation + + @if (!string.IsNullOrWhiteSpace(this.BackupDescription)) + { + + @this.BackupDescription + + } + + + + @if (this.Report.Outcome is DataRestoreOutcome.APPLIED) + { + @if (this.OpenPoints.Count == 0) + { + + @T("Nothing else to do: every provider and every data source has what it needs on this computer.") + + } + else + { + + @T("A backup holds no API keys or passwords, and some files and folders are not on this computer. Please check the following:") + + + @foreach (var openPoint in this.OpenPoints) + { + @this.Describe(openPoint) + } + + } + } + else if (!string.IsNullOrWhiteSpace(this.Report.Detail)) + { + + @string.Format(T("Technical details: {0}"), this.Report.Detail) + + } + + + + @T("Close") + + + \ No newline at end of file diff --git a/app/MindWork AI Studio/Dialogs/DataRestoreReportDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataRestoreReportDialog.razor.cs new file mode 100644 index 000000000..94d5a36f9 --- /dev/null +++ b/app/MindWork AI Studio/Dialogs/DataRestoreReportDialog.razor.cs @@ -0,0 +1,102 @@ +using System.Globalization; + +using AIStudio.Components; +using AIStudio.Tools.DataBackup; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Dialogs; + +/// +/// Tells the user, at the start after a restore, how it went, and what remains to do: entering the API +/// keys and passwords again, for example, since a backup holds none. +/// +public partial class DataRestoreReportDialog : MSGComponentBase +{ + [CascadingParameter] + private IMudDialogInstance MudDialog { get; set; } = null!; + + /// + /// How the runtime applied the restore. + /// + [Parameter] + public DataRestoreReport Report { get; set; } = null!; + + /// + /// What remains to do after a restore which was applied. + /// + [Parameter] + public IReadOnlyList OpenPoints { get; set; } = []; + + private CultureInfo currentCultureInfo = CultureInfo.InvariantCulture; + + #region Overrides of MSGComponentBase + + protected override async Task OnInitializedAsync() + { + var activeLanguagePlugin = await this.SettingsManager.GetActiveLanguagePlugin(); + this.currentCultureInfo = CommonTools.DeriveActiveCultureOrInvariant(activeLanguagePlugin.IETFTag); + + await base.OnInitializedAsync(); + } + + #endregion + + private string Icon => this.Report.Outcome switch + { + DataRestoreOutcome.APPLIED => Icons.Material.Filled.CheckCircle, + DataRestoreOutcome.ROLLBACK_FAILED => Icons.Material.Filled.Error, + _ => Icons.Material.Filled.WarningAmber, + }; + + private Color IconColor => this.Report.Outcome switch + { + DataRestoreOutcome.APPLIED => Color.Success, + DataRestoreOutcome.ROLLBACK_FAILED => Color.Error, + _ => Color.Warning, + }; + + private string Headline => this.Report.Outcome switch + { + DataRestoreOutcome.APPLIED => T("Your backup was restored"), + DataRestoreOutcome.ROLLBACK_FAILED => T("Restoring your backup is not finished"), + _ => T("Your backup was not restored"), + }; + + private string Explanation => this.Report.Outcome switch + { + DataRestoreOutcome.APPLIED => T("AI Studio replaced your data with those of the backup. The plugins and settings of your organization stayed as they were."), + DataRestoreOutcome.REFUSED => T("AI Studio could not start restoring your backup, so all your data stayed as they were."), + DataRestoreOutcome.ROLLED_BACK => T("Restoring your backup failed. AI Studio put all your data back the way they were before."), + DataRestoreOutcome.ROLLBACK_FAILED => T("Restoring your backup failed, and AI Studio could not yet put all your data back the way they were. It tries again at the next start, so please restart AI Studio. If this message appears again, please contact the support, and keep the folders '.restore-previous' and '.restore-pending' in your data folder."), + _ => string.Empty, + }; + + private string BackupDescription + { + get + { + if (!DateTimeOffset.TryParse(this.Report.BackupCreatedAtUtc, CultureInfo.InvariantCulture, DateTimeStyles.None, out var createdAt)) + return string.Empty; + + return string.Format(T("The backup is from {0} and was made with AI Studio {1}."), CommonTools.FormatTimestampToGeneral(createdAt.ToLocalTime().DateTime, this.currentCultureInfo), this.Report.BackupAppVersion); + } + } + + private string Describe(DataRestoreOpenPoint openPoint) => openPoint.Kind switch + { + DataRestoreOpenPointKind.PROVIDER_WITHOUT_API_KEY => string.Format(T("Enter the API key of the provider '{0}' again, if it needs one."), openPoint.Name), + DataRestoreOpenPointKind.EMBEDDING_PROVIDER_WITHOUT_API_KEY => string.Format(T("Enter the API key of the embedding provider '{0}' again, if it needs one."), openPoint.Name), + DataRestoreOpenPointKind.TRANSCRIPTION_PROVIDER_WITHOUT_API_KEY => string.Format(T("Enter the API key of the transcription provider '{0}' again, if it needs one."), openPoint.Name), + DataRestoreOpenPointKind.DATA_SOURCE_WITHOUT_SECRET => string.Format(T("Enter the password or token of the data source '{0}' again."), openPoint.Name), + DataRestoreOpenPointKind.MAILBOX_WITHOUT_PASSWORD => string.Format(T("Enter the password of the mailbox '{0}' again."), openPoint.Name), + DataRestoreOpenPointKind.TOOL_WITHOUT_SECRET => string.Format(T("Enter the setting '{1}' of the tool '{0}' again, if you used it."), openPoint.Name, openPoint.Setting), + DataRestoreOpenPointKind.DATA_SOURCE_NOT_FOUND => string.Format(T("The data source '{0}' names a file or folder which does not exist on this computer. Choose it anew."), openPoint.Name), + DataRestoreOpenPointKind.BATCH_PROCESSING_PATH_NOT_FOUND => string.Format(T("The batch processing uses '{0}', which does not exist on this computer."), openPoint.Name), + DataRestoreOpenPointKind.CERTIFICATE_BUNDLE_NOT_FOUND => string.Format(T("The file with your own root certificates, '{0}', does not exist on this computer."), openPoint.Name), + DataRestoreOpenPointKind.ERI_SERVER_DIRECTORY_NOT_FOUND => string.Format(T("The ERI server '{0}' writes into a folder which does not exist on this computer."), openPoint.Name), + _ => openPoint.Name, + }; + + private void Close() => this.MudDialog.Close(DialogResult.Ok(true)); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Layout/MainLayout.razor.cs b/app/MindWork AI Studio/Layout/MainLayout.razor.cs index c9371c647..410623a47 100644 --- a/app/MindWork AI Studio/Layout/MainLayout.razor.cs +++ b/app/MindWork AI Studio/Layout/MainLayout.razor.cs @@ -58,6 +58,9 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan [Inject] private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!; + [Inject] + private DataRestoreService DataRestoreService { get; init; } = null!; + [Inject] private CircuitStateService CircuitState { get; init; } = null!; @@ -82,6 +85,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan private bool settingsWriteProtectionWarningShown; private readonly SemaphoreSlim mandatoryInfoDialogSemaphore = new(1, 1); private readonly SemaphoreSlim promptInjectionDialogSemaphore = new(1, 1); + private readonly SemaphoreSlim dataRestorePendingDialogSemaphore = new(1, 1); private DataSourceEmbeddingOverview embeddingOverview = new(DataSourceEmbeddingState.COMPLETED, 0, 0, 0); private IReadOnlyCollection navItems = []; @@ -94,6 +98,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan { this.NavigationManager.RegisterLocationChangingHandler(this.OnLocationChanging); this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; + this.DataRestoreService.StateChanged += this.OnDataRestoreStateChanged; // // We use the Tauri API (Rust) to get the data and config directories @@ -391,7 +396,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan case Event.STARTUP_COMPLETED: this.startupCompleted = true; - this.EnsureMandatoryInfosAcceptedAsync().Observe($"{nameof(MainLayout)}: mandatory infos after the startup"); + this.ShowStartupDialogsAsync().Observe($"{nameof(MainLayout)}: dialogs after the startup"); break; case Event.RAG_EMBEDDING_STATUS_CHANGED: @@ -640,6 +645,70 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan this.StateHasChanged(); } + /// + /// Shows what the user has to see once the plugins started: the mandatory infos first, then how a + /// restore of a backup went. + /// + private async Task ShowStartupDialogsAsync() + { + await this.EnsureMandatoryInfosAcceptedAsync(); + await this.ShowDataRestoreReportAsync(); + await this.ShowDataRestorePendingDialogAsync(); + } + + /// + /// Asks for the restart as soon as a restore is prepared. + /// + private void OnDataRestoreStateChanged() + { + if (this.DataRestoreService.IsRestorePending) + this.InvokeAsync(this.ShowDataRestorePendingDialogAsync).Observe($"{nameof(MainLayout)}: asking for the restart which applies a restore"); + } + + /// + /// Blocks AI Studio while a prepared restore waits for the restart, since whatever changes until + /// then would be lost. + /// + private async Task ShowDataRestorePendingDialogAsync() + { + if (!await this.dataRestorePendingDialogSemaphore.WaitAsync(0)) + return; + + try + { + await this.DataRestoreService.StartupStateLoaded; + if (!this.DataRestoreService.IsRestorePending) + return; + + var dialogReference = await this.DialogService.ShowAsync(T("Restore of a Backup"), DialogOptions.BLOCKING_FULLSCREEN); + await dialogReference.Result; + } + finally + { + this.dataRestorePendingDialogSemaphore.Release(); + } + } + + /// + /// Tells how the runtime applied a restore at this start, and what remains to do. + /// + private async Task ShowDataRestoreReportAsync() + { + await this.DataRestoreService.StartupStateLoaded; + if (this.DataRestoreService.Report is not { } report) + return; + + var dialogParameters = new DialogParameters + { + { x => x.Report, report }, + { x => x.OpenPoints, await this.DataRestoreService.CollectOpenPointsAsync() }, + }; + + var dialogReference = await this.DialogService.ShowAsync(T("Restore of a Backup"), dialogParameters, DialogOptions.FULLSCREEN); + await dialogReference.Result; + this.DataRestoreService.ForgetReport(); + } + private async Task EnsureMandatoryInfosAcceptedAsync() { if (!await this.mandatoryInfoDialogSemaphore.WaitAsync(0)) @@ -722,9 +791,11 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan public void Dispose() { this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; + this.DataRestoreService.StateChanged -= this.OnDataRestoreStateChanged; this.CircuitState.ConnectionRestored -= this.OnConnectionRestored; this.MessageBus.Unregister(this); this.mandatoryInfoDialogSemaphore.Dispose(); + this.dataRestorePendingDialogSemaphore.Dispose(); } #endregion diff --git a/app/MindWork AI Studio/Pages/Information.razor b/app/MindWork AI Studio/Pages/Information.razor index 5a3533a9d..878d299bb 100644 --- a/app/MindWork AI Studio/Pages/Information.razor +++ b/app/MindWork AI Studio/Pages/Information.razor @@ -340,6 +340,14 @@ + + + + + + + + diff --git a/app/MindWork AI Studio/Pages/Settings.razor b/app/MindWork AI Studio/Pages/Settings.razor index 33102d8c9..c41344e46 100644 --- a/app/MindWork AI Studio/Pages/Settings.razor +++ b/app/MindWork AI Studio/Pages/Settings.razor @@ -37,6 +37,10 @@ + + + + \ No newline at end of file diff --git a/app/MindWork AI Studio/Pages/Supporters.razor b/app/MindWork AI Studio/Pages/Supporters.razor index 227bb5a44..4248108d8 100644 --- a/app/MindWork AI Studio/Pages/Supporters.razor +++ b/app/MindWork AI Studio/Pages/Supporters.razor @@ -94,7 +94,7 @@ - + diff --git a/app/MindWork AI Studio/Plugins/assistants/README.md b/app/MindWork AI Studio/Plugins/assistants/README.md index 6a1903a9f..cd267985b 100644 --- a/app/MindWork AI Studio/Plugins/assistants/README.md +++ b/app/MindWork AI Studio/Plugins/assistants/README.md @@ -64,19 +64,18 @@ This example shows: Treat the example as the recommended minimum viable pattern for assistant plugins, not as a feature-by-feature clone of `AssistantTranslation.razor`. ## Directory Structure -Each assistant plugin lives in its own directory under the assistants plugin root. In practice, you usually keep the manifest in `plugin.lua`, optional icon rendering in `icon.lua`, and any bundled media in `assets/`. +Each assistant plugin lives in its own directory under the assistants plugin root, `plugins/assistants/` in the data directory of AI Studio. The panel **Your Data Folders** on the Information page shows where the data directory lies on your computer. In practice, you usually keep the manifest in `plugin.lua`, optional icon rendering in `icon.lua`, and any bundled media in `assets/`. ``` . -└── com.github.mindwork-ai.ai-studio/ - └── data/ - └── plugins/ - └── assistants/ - └── your-assistant-directory/ - ├── assets/ - │ └── your-media-files.jpg - ├── icon.lua - └── plugin.lua +└── / + └── plugins/ + └── assistants/ + └── your-assistant-directory/ + ├── assets/ + │ └── your-media-files.jpg + ├── icon.lua + └── plugin.lua ``` A plugin may split its Lua code into several files and load them with `require`. Name the file relative to the plugin directory and without `.lua`, and separate subdirectories with a slash: `require("icon")` loads `icon.lua`, and `require("lib/helpers")` loads `lib/helpers.lua`. A plugin can only load files from its own directory, and `dofile` and `loadfile` are not available. diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index 77218cda9..9baf0bdee 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -452,6 +452,18 @@ CONFIG["SETTINGS"] = {} -- On Linux, this button exports the plugin archive instead of using a native share sheet. -- CONFIG["SETTINGS"]["DataApp.AllowUserToSharePlugins"] = false +-- Configure the user permission to back up the data of AI Studio into a file, e.g. to take it +-- to a new computer. When set to false, the backup button stays visible but is disabled. +-- A backup holds chats and other confidential content in a single file, which can leave the +-- computer on any stick or cloud drive. Secrets such as API keys never go into a backup. +-- CONFIG["SETTINGS"]["DataApp.AllowUserToBackUpData"] = false + +-- Configure the user permission to restore the data of AI Studio from a backup file, or to add +-- the chats of a backup. When set to false, the restore button stays visible but is disabled. +-- A backup brings along the locations of documents and local data sources, and those are read +-- again afterward. +-- CONFIG["SETTINGS"]["DataApp.AllowUserToRestoreData"] = false + -- Configure whether administration settings are visible in the UI: -- CONFIG["SETTINGS"]["DataApp.ShowAdminSettings"] = true diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 583390ca2..2be934879 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -3774,6 +3774,237 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T4081853237"] = "T -- Configure Keyboard Shortcut UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T636303786"] = "Tastaturkurzbefehl konfigurieren" +-- Keep the existing chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1274903302"] = "Vorhandenen Chat behalten" + +-- The backup goes on when you leave this page. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1298413158"] = "Die Sicherung läuft weiter, wenn Sie diese Seite verlassen." + +-- Unpacking file {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1496386567"] = "Datei {0} von {1} wird entpackt" + +-- Created on {0} with AI Studio {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1503285113"] = "Erstellt am {0} mit AI Studio {1}" + +-- AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1555678613"] = "AI Studio hat die Wiederherstellung einer früheren Sicherung noch nicht abgeschlossen. Bitte starten Sie AI Studio zuerst neu." + +-- Replace All Data +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1670691112"] = "Alle Daten ersetzen" + +-- Reading the backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1731783566"] = "Sicherung wird gelesen" + +-- Please stay on this page until this is done. Leaving the page stops adding the chats. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1758163038"] = "Bitte bleiben Sie auf dieser Seite, bis der Vorgang abgeschlossen ist. Wenn Sie die Seite verlassen, werden keine weiteren Chats hinzugefügt." + +-- Back up +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1823019579"] = "Sichern" + +-- When a chat already exists +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2117204104"] = "Wenn ein Chat bereits vorhanden ist" + +-- Restoring this backup replaces your settings, your chats, your plugins, and the indexes of your data sources with those of the backup. Whatever you changed since the backup was made is lost. Afterward, AI Studio restarts. Do you want to restore this backup? +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2176422849"] = "Beim Wiederherstellen dieser Sicherung werden Ihre Einstellungen, Chats, Plugins und die Indexe Ihrer Datenquellen durch die der Sicherung ersetzt. Alles, was Sie seit dem Erstellen der Sicherung geändert haben, geht verloren. Anschließend startet AI Studio neu. Möchten Sie diese Sicherung wiederherstellen?" + +-- 1 chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2301695346"] = "1 Chat" + +-- Writing file {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2416389933"] = "Datei {0} von {1} wird geschrieben" + +-- Temporary chats +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2532369939"] = "Temporäre Chats" + +-- Adding the chats was stopped. The chats added until then were kept. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2586532370"] = "Das Hinzufügen der Chats wurde abgebrochen. Die bis dahin hinzugefügten Chats bleiben erhalten." + +-- Restore +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T267943793"] = "Wiederherstellen" + +-- Replaces your settings, chats, plugins, and the indexes of your data sources with those of the backup. AI Studio restarts to do so. The plugins and settings of your organization stay as they are. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2693525709"] = "Ersetzt Ihre Einstellungen, Chats, Plugins und die Indexe Ihrer Datenquellen durch die der Sicherung. AI Studio startet dafür neu. Die Plugins und Einstellungen Ihrer Organisation bleiben unverändert." + +-- Chats which already existed and stayed as they are: {0}. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T29276052"] = "Chats, die bereits vorhanden waren und unverändert geblieben sind: {0}." + +-- The indexes of your data sources are not part of this backup, because they were not available. After a restore, AI Studio indexes your data sources anew. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3006171489"] = "Die Indexe Ihrer Datenquellen sind nicht in dieser Sicherung enthalten, weil sie nicht verfügbar waren. Nach einer Wiederherstellung indexiert AI Studio Ihre Datenquellen neu." + +-- Preparing the backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3082169282"] = "Sicherung wird vorbereitet" + +-- Back up all your data into one file: your settings, chats, plugins, and the indexes of your data sources. Your API keys and passwords stay out of it. AI Studio keeps them in the secure storage of your OS. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3087579229"] = "Sichern Sie alle Ihre Daten in einer einzigen Datei: Ihre Einstellungen, Chats, Plugins und die Indexe Ihrer Datenquellen. Ihre API-Schlüssel und Passwörter werden nicht mitgesichert. AI Studio bewahrt sie im sicheren Speicher Ihres Betriebssystems auf." + +-- Choose backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3114985646"] = "Sicherung auswählen" + +-- Select a backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T311951952"] = "Sicherung auswählen" + +-- Add chats only +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3238328507"] = "Nur Chats hinzufügen" + +-- Chat {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T355446621"] = "Chat {0} von {1}" + +-- {0} chats +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3562573794"] = "{0} Chats" + +-- Add {0} chats +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3585479613"] = "{0} Chats hinzufügen" + +-- Backed up {0} files into: {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3630348902"] = "{0} Dateien wurden gesichert in: {1}" + +-- Restore a backup: replace all your data with those of the backup, e.g., on a new computer, or add only its chats to the chats you have. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3657899737"] = "Stellen Sie eine Sicherung wieder her: Ersetzen Sie alle Ihre Daten durch die der Sicherung, z. B. auf einem neuen Rechner, oder fügen Sie nur ihre Chats zu Ihren vorhandenen Chats hinzu." + +-- Add it as an additional chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3966117123"] = "Als zusätzlichen Chat hinzufügen" + +-- Chats added: {0}. Workspaces they went into: {1}. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3972560517"] = "Hinzugefügte Chats: {0}. Betroffene Arbeitsbereiche: {1}." + +-- The backup was stopped. No file was left behind. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T431814261"] = "Die Sicherung wurde abgebrochen. Es ist keine Datei zurückgeblieben." + +-- Adds the chats of the backup to the chats you have. Nothing else changes. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T457177394"] = "Fügt die Chats der Sicherung zu Ihren vorhandenen Chats hinzu. Alles andere bleibt unverändert." + +-- Chats which could not be added: {0}. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T611768714"] = "Chats, die nicht hinzugefügt werden konnten: {0}." + +-- The restore was stopped. All your data stay as they are. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T749188343"] = "Die Wiederherstellung wurde abgebrochen. Alle Ihre Daten bleiben unverändert." + +-- Back up data +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T796613169"] = "Daten sichern" + +-- Replace all data +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T819832488"] = "Alle Daten ersetzen" + +-- Cancel +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T900713019"] = "Abbrechen" + +-- Copying the indexes of your data sources: {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T907141309"] = "Die Indexe Ihrer Datenquellen werden kopiert: {0} von {1}" + +-- Add 1 chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T938996955"] = "1 Chat hinzufügen" + +-- Folder +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T104168189"] = "Ordner" + +-- Reason +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1093747001"] = "Grund" + +-- Configuration folder +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1191404447"] = "Konfigurationsordner" + +-- Could not open the folder. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1215994612"] = "Der Ordner konnte nicht geöffnet werden." + +-- Your projects in the Visual Briefing Assistant. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1259707710"] = "Ihre Projekte im Assistenten für visuelle Briefings." + +-- Your temporary chats, along with their transcripts. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1293614512"] = "Ihre temporären Chats mit ihren Transkripten." + +-- What belongs into a backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1682175486"] = "Was in eine Sicherung gehört" + +-- The index of your local data sources: which files AI Studio embedded, and their text. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T170332953"] = "Der Index Ihrer lokalen Datenquellen: welche Dateien AI Studio eingebettet hat und deren Text." + +-- Keys with which AI Studio protects data in its user interface. Only the Flatpak keeps them here, and on another computer, AI Studio creates new ones. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1981169629"] = "Schlüssel, mit denen AI Studio Daten in seiner Benutzeroberfläche schützt. Nur die Flatpak-Version legt sie hier ab, und auf einem anderen Rechner erstellt AI Studio neue." + +-- Tools which AI Studio downloaded for this OS, such as Pandoc. On another computer, AI Studio downloads them again. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1992121437"] = "Werkzeuge, die AI Studio für dieses Betriebssystem heruntergeladen hat, z. B. Pandoc. Auf einem anderen Rechner lädt AI Studio sie erneut herunter." + +-- What a backup can leave out +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2120934651"] = "Was eine Sicherung auslassen kann" + +-- Good to know +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2373663376"] = "Gut zu wissen" + +-- Data folder +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2422278827"] = "Datenordner" + +-- Explanation +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2840582448"] = "Erklärung" + +-- Copies of your settings, which every version of AI Studio keeps for itself. Your current settings are all a backup needs. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2851981238"] = "Kopien Ihrer Einstellungen, die jede Version von AI Studio für sich anlegt. Für eine Sicherung genügen Ihre aktuellen Einstellungen." + +-- The tokenizers you chose for your providers. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2892362083"] = "Die Tokenizer, die Sie für Ihre Anbieter ausgewählt haben." + +-- To back up your data with AI Studio itself, or to move it to another computer, use Data Backup & Restore on this page or in the settings instead. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2906063884"] = "Um Ihre Daten mit AI Studio selbst zu sichern oder auf einen anderen Rechner zu übertragen, verwenden Sie „Datensicherung & Wiederherstellung“ auf dieser Seite oder in den Einstellungen." + +-- Your plugins, such as the assistants you built, apart from the folders listed below. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2979110360"] = "Ihre Plugins, z. B. selbst erstellte Assistenten, mit Ausnahme der unten aufgeführten Ordner." + +-- Open in your file manager +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2986518596"] = "Im Dateimanager öffnen" + +-- Your organization deployed some plugins among your own ones, for example, with a device management. Their folders do not show this, so the list above names them one by one. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3156743605"] = "Ihre Organisation hat einige Plugins neben Ihren eigenen bereitgestellt, zum Beispiel über eine Geräteverwaltung. In den Ordnern ist nicht ersichtlich, welche Plugins von Ihrer Organisation stammen. Deshalb werden sie oben einzeln aufgeführt." + +-- Content +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3244793698"] = "Inhalt" + +-- The embeddings of your local data sources. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3301102729"] = "Die Einbettungen Ihrer lokalen Datenquellen." + +-- The folder is not known yet. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3310368654"] = "Der Ordner ist noch nicht bekannt." + +-- Could not open the folder: {0} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3380661810"] = "Der Ordner konnte nicht geöffnet werden: {0}" + +-- Unknown error +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3461425987"] = "Unbekannter Fehler" + +-- Plugins of your organization. Your organization brings them back. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T346886000"] = "Plugins Ihrer Organisation. Ihre Organisation stellt sie wieder bereit." + +-- Back up the indexes of your data sources only while AI Studio is closed. AI Studio writes to them while it runs, so a copy made meanwhile may not fit together, and AI Studio would have to embed your data sources anew. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3655837348"] = "Sichern Sie die Indexe Ihrer Datenquellen nur, wenn AI Studio geschlossen ist. Während AI Studio läuft, schreibt es in diese Indexe. Eine Kopie, die währenddessen entsteht, passt deshalb womöglich nicht zusammen, und AI Studio müsste Ihre Datenquellen neu einbetten." + +-- Files which AI Studio needs only for a moment and deletes again. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T366353671"] = "Dateien, die AI Studio nur kurz benötigt und anschließend wieder löscht." + +-- Plugins which come with AI Studio, such as the language packs. AI Studio brings them back by itself. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3877928356"] = "Plugins, die mit AI Studio mitgeliefert werden, etwa Sprachpakete. AI Studio stellt sie automatisch wieder her." + +-- Path +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3949388886"] = "Pfad" + +-- Your API keys, tokens, and passwords are not in these folders. AI Studio keeps them in the secure storage of your OS. After you restored your data on another computer, enter them there once more. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T4093107108"] = "Ihre API-Schlüssel, Token und Passwörter befinden sich nicht in diesen Ordnern. AI Studio bewahrt sie im sicheren Speicher Ihres Betriebssystems auf. Nachdem Sie Ihre Daten auf einem anderen Rechner wiederhergestellt haben, geben Sie sie dort erneut ein." + +-- Your chats in workspaces, along with their transcripts. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T509585668"] = "Ihre Chats in Arbeitsbereichen mit ihren Transkripten." + +-- Log files. They help when something goes wrong, but they hold none of your data. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T53143530"] = "Protokolldateien. Sie helfen, wenn etwas schiefgeht, enthalten aber keine Ihrer Daten." + +-- AI Studio keeps your data in two folders on this computer. When you back up your computer with a tool of your own, such as [Restic](https://restic.net), [Borg](https://www.borgbackup.org), or [Time Machine](https://support.apple.com/en-us/104984), this shows which of their contents belong into the backup and which it can leave out. Click a path to copy it to the clipboard, or use the folder button to open it in your file manager. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T69622543"] = "AI Studio speichert Ihre Daten in zwei Ordnern auf diesem Rechner. Wenn Sie Ihren Rechner mit einem eigenen Programm sichern, etwa mit [Restic](https://restic.net), [Borg](https://www.borgbackup.org) oder [Time Machine](https://support.apple.com/de-de/104984), sehen Sie hier, welche Inhalte dieser Ordner in die Sicherung gehören und welche sie auslassen kann. Klicken Sie auf einen Pfad, um ihn in die Zwischenablage zu kopieren, oder auf das Ordnersymbol, um ihn in Ihrem Dateimanager zu öffnen." + +-- Your settings. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T790727583"] = "Ihre Einstellungen." + +-- The following is meant for experts who back up their computer with a tool of their own. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T813925103"] = "Das Folgende richtet sich an Experten, die ihren Rechner mit einem eigenen Programm sichern." + +-- On this computer, the data folder lies inside the configuration folder. A backup of the configuration folder therefore takes the data folder along. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T91194331"] = "Auf diesem Rechner befindet sich der Datenordner im Konfigurationsordner. Bei einer Sicherung des Konfigurationsordners wird daher auch der Datenordner mitgesichert." + -- Yes, please send my data to the external embedding provider UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T1159107763"] = "Ja, bitte senden Sie meine Daten an den externen Einbettungsanbieter" @@ -3810,6 +4041,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T6617545 -- All files in this folder and its subfolders UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T916879200"] = "Alle Dateien in diesem Ordner und seinen Unterordnern" +-- AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1006756723"] = "AI Studio sichert gerade die Indexe Ihrer Datenquellen oder stellt sie wieder her. Bitte versuchen Sie es erneut, sobald der Vorgang abgeschlossen ist." + -- You might configure different data sources. A data source can include one file, all files in a directory, or data from your company. Later, you can incorporate these data sources as needed when the AI requires this data to complete a certain task. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "Sie können verschiedene Datenquellen konfigurieren. Eine Datenquelle kann eine einzelne Datei, alle Dateien in einem Ordner oder Daten aus Ihrem Unternehmen enthalten. Später können Sie diese Datenquellen bei Bedarf einbinden, wenn die KI diese Daten zur Erledigung einer bestimmten Aufgabe benötigt." @@ -6087,6 +6321,84 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::CONFIRMDIALOG::T1642511898"] = "Nein" -- Yes UI_TEXT_CONTENT["AISTUDIO::DIALOGS::CONFIRMDIALOG::T3013883440"] = "Ja" +-- Restart now +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T1409292924"] = "Jetzt neu starten" + +-- Discard the restore +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T3017406920"] = "Wiederherstellung verwerfen" + +-- Your backup is ready. AI Studio restarts to put it in place. Whatever you changed until then would be lost, so please restart now. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T495853773"] = "Ihre Sicherung ist bereit. AI Studio startet neu, um sie einzuspielen. Alles, was Sie bis dahin ändern, ginge verloren. Bitte starten Sie AI Studio daher jetzt neu." + +-- AI Studio could not restart by itself. Please quit AI Studio and start it again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T573887863"] = "AI Studio konnte sich nicht selbst neu starten. Bitte beenden Sie AI Studio und starten Sie es erneut." + +-- Restoring your backup failed. AI Studio put all your data back the way they were before. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1057857006"] = "Die Wiederherstellung Ihrer Sicherung ist fehlgeschlagen. AI Studio hat alle Ihre Daten wieder in den Zustand von vorher versetzt." + +-- Your backup was not restored +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1193456690"] = "Ihre Sicherung wurde nicht wiederhergestellt" + +-- Restoring your backup failed, and AI Studio could not yet put all your data back the way they were. It tries again at the next start, so please restart AI Studio. If this message appears again, please contact the support, and keep the folders '.restore-previous' and '.restore-pending' in your data folder. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T123391329"] = "Die Wiederherstellung Ihrer Sicherung ist fehlgeschlagen, und AI Studio konnte Ihre Daten noch nicht vollständig in den Zustand von vorher versetzen. Beim nächsten Start versucht AI Studio es erneut, bitte starten Sie AI Studio daher neu. Wenn diese Meldung erneut erscheint, wenden Sie sich bitte an den Support und löschen Sie die Ordner „.restore-previous“ und „.restore-pending“ in Ihrem Datenordner nicht." + +-- The batch processing uses '{0}', which does not exist on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1314976065"] = "Die Stapelverarbeitung verwendet „{0}“, das auf diesem Rechner nicht vorhanden ist." + +-- The backup is from {0} and was made with AI Studio {1}. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1363238678"] = "Die Sicherung stammt vom {0} und wurde mit AI Studio {1} erstellt." + +-- A backup holds no API keys or passwords, and some files and folders are not on this computer. Please check the following: +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1633897333"] = "Eine Sicherung enthält keine API-Schlüssel oder Passwörter, und einige Dateien und Ordner befinden sich nicht auf diesem Rechner. Bitte prüfen Sie Folgendes:" + +-- Enter the API key of the provider '{0}' again, if it needs one. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1645175373"] = "Geben Sie den API-Schlüssel des Anbieters „{0}“ erneut ein, falls er einen benötigt." + +-- Restoring your backup is not finished +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2415452768"] = "Die Wiederherstellung Ihrer Sicherung ist noch nicht abgeschlossen" + +-- Enter the API key of the embedding provider '{0}' again, if it needs one. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2492749778"] = "Geben Sie den API-Schlüssel des Einbettungsanbieters „{0}“ erneut ein, falls er einen benötigt." + +-- Enter the API key of the transcription provider '{0}' again, if it needs one. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2568556297"] = "Geben Sie den API-Schlüssel des Transkriptionsanbieters „{0}“ erneut ein, falls er einen benötigt." + +-- Nothing else to do: every provider and every data source has what it needs on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2671436044"] = "Es ist nichts weiter zu tun: Alle Anbieter und Datenquellen haben auf diesem Rechner, was sie benötigen." + +-- The data source '{0}' names a file or folder which does not exist on this computer. Choose it anew. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T270687216"] = "Die Datenquelle „{0}“ verweist auf einen Pfad, den es auf diesem Rechner nicht gibt. Wählen Sie die Datei oder den Ordner neu aus." + +-- The file with your own root certificates, '{0}', does not exist on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2844911500"] = "Die Datei mit Ihren eigenen Stammzertifikaten, „{0}“, existiert auf diesem Rechner nicht." + +-- AI Studio could not start restoring your backup, so all your data stayed as they were. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3044053464"] = "AI Studio konnte die Wiederherstellung Ihrer Sicherung nicht beginnen. Daher sind alle Ihre Daten unverändert geblieben." + +-- Enter the password of the mailbox '{0}' again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3083231765"] = "Geben Sie das Passwort für das Postfach „{0}“ erneut ein." + +-- Enter the password or token of the data source '{0}' again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3430781520"] = "Geben Sie das Passwort oder das Token der Datenquelle „{0}“ erneut ein." + +-- Technical details: {0} +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3436390766"] = "Technische Details: {0}" + +-- Close +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3448155331"] = "Schließen" + +-- Enter the setting '{1}' of the tool '{0}' again, if you used it. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3658964195"] = "Geben Sie die Einstellung „{1}“ des Werkzeugs „{0}“ erneut ein, falls Sie sie verwendet haben." + +-- AI Studio replaced your data with those of the backup. The plugins and settings of your organization stayed as they were. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3896631626"] = "AI Studio hat Ihre Daten durch die der Sicherung ersetzt. Die Plugins und Einstellungen Ihrer Organisation sind unverändert geblieben." + +-- Your backup was restored +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3980336073"] = "Ihre Sicherung wurde wiederhergestellt" + +-- The ERI server '{0}' writes into a folder which does not exist on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T472632172"] = "Der ERI-Server „{0}“ schreibt in einen Ordner, der auf diesem Rechner nicht existiert." + -- How many matches do you want at most per query? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1DIALOG::T1827669611"] = "Wie viele Treffer möchten Sie maximal pro Abfrage erhalten?" @@ -9387,6 +9699,9 @@ UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2800137365"] = "Ein Update auf V -- Please wait for the update to complete... UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2864211629"] = "Bitte warten Sie, bis das Update abgeschlossen ist ..." +-- Restore of a Backup +UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2912921801"] = "Wiederherstellung einer Sicherung" + -- Supporters UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2929332068"] = "Unterstützer" @@ -10191,6 +10506,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3294830584"] = "Entdecken Sie di -- External HTTPS custom root certificates UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3315279770"] = "Externe HTTPS-Stammzertifikate für benutzerdefinierte Zertifizierungsstellen" +-- Data Backup & Restore +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3317700277"] = "Datensicherung & Wiederherstellung" + -- User-language provided by the OS UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3334355246"] = "Vom Betriebssystem bereitgestellte Sprache" @@ -10347,6 +10665,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T551035563"] = "Protokollanzeige -- This is a library providing the foundations for asynchronous programming in Rust. It includes key trait definitions like Stream, as well as utilities like join!, select!, and various futures combinator methods which enable expressive asynchronous control flow. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "Dies ist eine Bibliothek, die die Grundlagen für asynchrones Programmieren in Rust bereitstellt. Sie enthält zentrale Trait-Definitionen wie Stream sowie Hilfsfunktionen wie join!, select! und verschiedene Methoden zur Kombination von Futures, die einen ausdrucksstarken asynchronen Kontrollfluss ermöglichen." +-- Your Data Folders +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T570573187"] = "Ihre Datenordner" + -- Used .NET SDK UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Verwendetes .NET SDK" @@ -10530,15 +10851,15 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::PLUGINS::T837269472"] = "Das Plugin konnte nic -- Settings UI_TEXT_CONTENT["AISTUDIO::PAGES::SETTINGS::T1258653480"] = "Einstellungen" +-- Data Backup & Restore +UI_TEXT_CONTENT["AISTUDIO::PAGES::SETTINGS::T3317700277"] = "Datensicherung & Wiederherstellung" + -- Thank you very much, Kerstin, for creating the Wiki, for advising us on ERI with real use cases, and for your work on version 2 of the ERI protocol. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1367356224"] = "Vielen herzlichen Dank, Kerstin, für die Erstellung des Wikis, deine Beratung zu ERI anhand realer Anwendungsfälle und deine Arbeit an Version 2 des ERI-Protokolls." -- Thank you for being the first to contribute a one-time donation. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1470916504"] = "Vielen Dank, dass du die erste einmalige Spende geleistet hast." --- Thank you, Jens, for the Batch Processing Assistant, the exact token count in the chat, and everything else you are building for AI Studio with so much dedication. -UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1701321325"] = "Vielen Dank, Jens, für den Assistenten für die Stapelverarbeitung, die exakte Tokenanzahl im Chat und alles Weitere, was du mit so viel Hingabe für AI Studio entwickelst." - -- Thank you, Peer, for your courage in being the second person to support the project financially. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1714878838"] = "Vielen Dank, Peer, für deinen Mut, als zweite Person das Projekt finanziell zu unterstützen." @@ -10626,6 +10947,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T686206269"] = "Danke für deinen -- Business Contributors UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T838479287"] = "Spenden von Unternehmen" +-- Thank you, Jens, for the Batch Processing Assistant, the exact token count in the chat, the backups of all data, and everything else you are building for AI Studio with so much dedication. +UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T882777973"] = "Vielen Dank, Jens, für den Assistenten für die Stapelverarbeitung, die exakte Tokenanzahl im Chat, die Sicherung aller Daten und alles Weitere, was du mit so viel Hingabe für AI Studio entwickelst." + -- Thanks, Oliver, for making sure no spaces get lost in answers from OpenAI. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T916186976"] = "Danke, Oliver, dass du dafür gesorgt hast, dass in Antworten von OpenAI keine Leerzeichen mehr verloren gehen." @@ -11268,6 +11592,15 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::AUTHMETHODSV1EXTENSIONS::T810547195"] = "Keine -- Username & Password UI_TEXT_CONTENT["AISTUDIO::TOOLS::AUTHMETHODSV1EXTENSIONS::T863358904"] = "Benutzername & Passwort" +-- Unexpected error: {0} +UI_TEXT_CONTENT["AISTUDIO::TOOLS::CHATARCHIVE::CHATARCHIVEIMPORTER::T2350673880"] = "Unerwarteter Fehler: {0}" + +-- This file is no backup of AI Studio, or it was not written completely. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::CHATARCHIVE::CHATARCHIVEIMPORTER::T3123318432"] = "Diese Datei ist keine Sicherung von AI Studio oder wurde nicht vollständig geschrieben." + +-- Imported workspace +UI_TEXT_CONTENT["AISTUDIO::TOOLS::CHATARCHIVE::CHATARCHIVEIMPORTER::T722364324"] = "Importierter Arbeitsbereich" + -- Spanish (Spain) UI_TEXT_CONTENT["AISTUDIO::TOOLS::COMMONLANGUAGEEXTENSIONS::T1269145091"] = "Spanisch (Spanien)" @@ -11403,6 +11736,24 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T3893997203"] = " -- Trust all LLM providers UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T4107860491"] = "Allen LLM-Anbietern vertrauen" +-- This file is no backup of AI Studio, or it was not written completely. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATABACKUPREADER::T3123318432"] = "Diese Datei ist keine Sicherung von AI Studio oder wurde nicht vollständig geschrieben." + +-- This backup was made by a newer version of AI Studio. Please update AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATABACKUPREADER::T3812849494"] = "Diese Sicherung wurde mit einer neueren Version von AI Studio erstellt. Bitte aktualisieren Sie AI Studio zuerst." + +-- The content list of this backup could not be read. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATABACKUPREADER::T3954689017"] = "Das Inhaltsverzeichnis dieser Sicherung konnte nicht gelesen werden." + +-- This file is no backup of AI Studio, or it was not written completely. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATARESTOREPREPARER::T3123318432"] = "Diese Datei ist keine Sicherung von AI Studio oder wurde nicht vollständig geschrieben." + +-- The backup is damaged: the file '{0}' does not match its content list. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATARESTOREPREPARER::T3493230602"] = "Die Sicherung ist beschädigt: Die Datei „{0}“ stimmt nicht mit dem Inhaltsverzeichnis überein." + +-- This backup was made by a newer version of AI Studio. Please update AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATARESTOREPREPARER::T3812849494"] = "Diese Sicherung wurde mit einer neueren Version von AI Studio erstellt. Bitte aktualisieren Sie AI Studio zuerst." + -- Reason UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1093747001"] = "Grund" @@ -11553,6 +11904,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T858000918"] = "Dies -- Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while. UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T874850580"] = "Alles, was für sie vorbereitet wurde, wird verworfen, und jedes ihrer Dokumente wird erneut an Ihren Einbettungsanbieter gesendet. Bei einer großen Datenquelle dauert dies eine Weile." +-- AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T1006756723"] = "AI Studio sichert gerade die Indexe Ihrer Datenquellen oder stellt sie wieder her. Bitte versuchen Sie es erneut, sobald der Vorgang abgeschlossen ist." + +-- Close +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T3448155331"] = "Schließen" + -- Repair Data Source UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Datenquelle reparieren" @@ -12483,6 +12840,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T377 -- Text UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text" +-- AI Studio backup +UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1046878067"] = "AI-Studio-Sicherung" + -- Office Files UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office-Dateien" @@ -12744,9 +13104,45 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGINGENERATIONSERVICE::T5 -- Workspace UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGINGENERATIONSERVICE::T658612054"] = "Arbeitsbereich" +-- A backup is running already. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T1319749333"] = "Eine Sicherung läuft bereits." + +-- The backup is too large for the chosen drive, which takes no files beyond 4 GB. USB sticks formatted with FAT32 are such drives. Please choose another drive. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T1869616367"] = "Die Sicherung ist zu groß für das ausgewählte Laufwerk, da darauf keine Dateien mit mehr als 4 GB gespeichert werden können. USB-Sticks mit FAT32-Formatierung haben diese Einschränkung. Bitte wählen Sie ein anderes Laufwerk." + +-- The backup failed: {0} +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T1915001713"] = "Die Sicherung ist fehlgeschlagen: {0}" + +-- Your organization does not allow backups. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T2011318013"] = "Ihre Organisation erlaubt keine Sicherungen." + +-- AI Studio is still writing an answer or a transcript. Please start the backup once it is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T4161531565"] = "AI Studio schreibt noch eine Antwort oder ein Transkript. Bitte starten Sie die Sicherung, sobald das erledigt ist." + +-- Your organization does not allow restoring backups. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T1032491606"] = "Ihre Organisation erlaubt keine Wiederherstellung von Sicherungen." + +-- AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T1555678613"] = "AI Studio hat die Wiederherstellung einer früheren Sicherung noch nicht abgeschlossen. Bitte starten Sie AI Studio zuerst neu." + +-- AI Studio is still writing an answer or a transcript. Please restore once it is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T1847516630"] = "AI Studio schreibt noch eine Antwort oder ein Transkript. Bitte starten Sie die Wiederherstellung, sobald das erledigt ist." + +-- The restore could not be prepared: {0} +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T2653345012"] = "Die Wiederherstellung konnte nicht vorbereitet werden: {0}" + +-- A restore is being prepared already. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T4157700208"] = "Eine Wiederherstellung wird bereits vorbereitet." + +-- A backup is running. Please restore once it is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T553866901"] = "Gerade läuft eine Sicherung. Bitte starten Sie die Wiederherstellung, sobald sie abgeschlossen ist." + -- The local RAG index database is not available. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1738200026"] = "Die lokale RAG-Indexdatenbank ist nicht verfügbar." +-- The index of the data source '{0}' is missing. The data source stays out of your chats until its index was built anew. Use the repair action to start that. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T204783307"] = "Der Index der Datenquelle „{0}“ fehlt. Die Datenquelle wird in Ihren Chats nicht verwendet, bis ihr Index neu erstellt wurde. Verwenden Sie die Aktion „Reparieren“, um dies zu starten." + -- The selected embedding provider is not allowed to index this data source. The data source asks for the confidence level '{0}', while the embedding provider has '{1}'. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2186533187"] = "Der ausgewählte Einbettungsanbieter darf diese Datenquelle nicht indexieren. Die Datenquelle erfordert das Vertrauensniveau „{0}“, während der Einbettungsanbieter „{1}“ hat." @@ -12852,6 +13248,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T2360 -- The mail '{0}' could not be indexed. AI Studio tries again during the next sync. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T804280374"] = "Die E-Mail „{0}“ konnte nicht indexiert werden. AI Studio versucht es bei der nächsten Synchronisierung erneut." +-- The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1214987337"] = "Der Ordner „{0}“ konnte nicht geöffnet werden. Bitte prüfen Sie, ob Sie die erforderlichen Leserechte haben. Aus dem Index wurde nichts entfernt." + +-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1393648320"] = "Die Datei „{0}“ ist nicht erreichbar. Möglicherweise ist das Laufwerk, auf dem sie sich befindet, nicht verbunden, oder die Datei wurde verschoben oder umbenannt. Aus dem Index wurde nichts entfernt. Sobald die Datei wieder verfügbar ist, aktualisieren Sie diese Datenquelle. Wenn sie dauerhaft nicht mehr vorhanden ist, entfernen Sie diese Datenquelle." + -- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1935191670"] = "Die Datei „{0}“ wurde während der Indexierung geändert. Die bereits indexierten Inhalte werden verworfen, und die Datei wird beim nächsten Durchlauf erneut verarbeitet." @@ -12861,15 +13263,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T242460 -- The folder '{0}' could not be opened. Please check whether you are allowed to read it. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3230000698"] = "Der Ordner „{0}“ konnte nicht geöffnet werden. Bitte prüfen Sie, ob Sie ihn lesen dürfen." +-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3722338131"] = "Der Ordner „{0}“ ist nicht erreichbar. Möglicherweise ist das Laufwerk, auf dem er sich befindet, nicht verbunden, oder der Ordner wurde verschoben oder umbenannt. Aus dem Index wurde nichts entfernt. Sobald der Ordner wieder verfügbar ist, aktualisieren Sie diese Datenquelle." + -- The file '{0}' could not be read. Please check whether you are allowed to read it. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3924882233"] = "Die Datei „{0}“ konnte nicht gelesen werden. Bitte prüfen Sie, ob Sie berechtigt sind, sie zu lesen." --- The file '{0}' does not exist. -UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T451561215"] = "Die Datei „{0}“ existiert nicht." - --- The folder '{0}' does not exist. -UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T871336081"] = "Der Ordner „{0}“ existiert nicht." - -- The local index '{0}' could not be created again. Please restart AI Studio and try once more. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::INDEXEDRUNCONTEXT::T1394295123"] = "Der lokale Index „{0}“ konnte nicht erneut erstellt werden. Bitte starten Sie AI Studio neu und versuchen Sie es noch einmal." diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 0c7043755..9d67e6a87 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -3774,6 +3774,237 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T4081853237"] = "C -- Configure Keyboard Shortcut UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T636303786"] = "Configure Keyboard Shortcut" +-- Keep the existing chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1274903302"] = "Keep the existing chat" + +-- The backup goes on when you leave this page. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1298413158"] = "The backup goes on when you leave this page." + +-- Unpacking file {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1496386567"] = "Unpacking file {0} of {1}" + +-- Created on {0} with AI Studio {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1503285113"] = "Created on {0} with AI Studio {1}" + +-- AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1555678613"] = "AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first." + +-- Replace All Data +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1670691112"] = "Replace All Data" + +-- Reading the backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1731783566"] = "Reading the backup" + +-- Please stay on this page until this is done. Leaving the page stops adding the chats. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1758163038"] = "Please stay on this page until this is done. Leaving the page stops adding the chats." + +-- Back up +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1823019579"] = "Back up" + +-- When a chat already exists +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2117204104"] = "When a chat already exists" + +-- Restoring this backup replaces your settings, your chats, your plugins, and the indexes of your data sources with those of the backup. Whatever you changed since the backup was made is lost. Afterward, AI Studio restarts. Do you want to restore this backup? +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2176422849"] = "Restoring this backup replaces your settings, your chats, your plugins, and the indexes of your data sources with those of the backup. Whatever you changed since the backup was made is lost. Afterward, AI Studio restarts. Do you want to restore this backup?" + +-- 1 chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2301695346"] = "1 chat" + +-- Writing file {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2416389933"] = "Writing file {0} of {1}" + +-- Temporary chats +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2532369939"] = "Temporary chats" + +-- Adding the chats was stopped. The chats added until then were kept. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2586532370"] = "Adding the chats was stopped. The chats added until then were kept." + +-- Restore +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T267943793"] = "Restore" + +-- Replaces your settings, chats, plugins, and the indexes of your data sources with those of the backup. AI Studio restarts to do so. The plugins and settings of your organization stay as they are. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T2693525709"] = "Replaces your settings, chats, plugins, and the indexes of your data sources with those of the backup. AI Studio restarts to do so. The plugins and settings of your organization stay as they are." + +-- Chats which already existed and stayed as they are: {0}. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T29276052"] = "Chats which already existed and stayed as they are: {0}." + +-- The indexes of your data sources are not part of this backup, because they were not available. After a restore, AI Studio indexes your data sources anew. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3006171489"] = "The indexes of your data sources are not part of this backup, because they were not available. After a restore, AI Studio indexes your data sources anew." + +-- Preparing the backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3082169282"] = "Preparing the backup" + +-- Back up all your data into one file: your settings, chats, plugins, and the indexes of your data sources. Your API keys and passwords stay out of it. AI Studio keeps them in the secure storage of your OS. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3087579229"] = "Back up all your data into one file: your settings, chats, plugins, and the indexes of your data sources. Your API keys and passwords stay out of it. AI Studio keeps them in the secure storage of your OS." + +-- Choose backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3114985646"] = "Choose backup" + +-- Select a backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T311951952"] = "Select a backup" + +-- Add chats only +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3238328507"] = "Add chats only" + +-- Chat {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T355446621"] = "Chat {0} of {1}" + +-- {0} chats +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3562573794"] = "{0} chats" + +-- Add {0} chats +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3585479613"] = "Add {0} chats" + +-- Backed up {0} files into: {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3630348902"] = "Backed up {0} files into: {1}" + +-- Restore a backup: replace all your data with those of the backup, e.g., on a new computer, or add only its chats to the chats you have. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3657899737"] = "Restore a backup: replace all your data with those of the backup, e.g., on a new computer, or add only its chats to the chats you have." + +-- Add it as an additional chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3966117123"] = "Add it as an additional chat" + +-- Chats added: {0}. Workspaces they went into: {1}. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T3972560517"] = "Chats added: {0}. Workspaces they went into: {1}." + +-- The backup was stopped. No file was left behind. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T431814261"] = "The backup was stopped. No file was left behind." + +-- Adds the chats of the backup to the chats you have. Nothing else changes. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T457177394"] = "Adds the chats of the backup to the chats you have. Nothing else changes." + +-- Chats which could not be added: {0}. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T611768714"] = "Chats which could not be added: {0}." + +-- The restore was stopped. All your data stay as they are. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T749188343"] = "The restore was stopped. All your data stay as they are." + +-- Back up data +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T796613169"] = "Back up data" + +-- Replace all data +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T819832488"] = "Replace all data" + +-- Cancel +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T900713019"] = "Cancel" + +-- Copying the indexes of your data sources: {0} of {1} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T907141309"] = "Copying the indexes of your data sources: {0} of {1}" + +-- Add 1 chat +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T938996955"] = "Add 1 chat" + +-- Folder +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T104168189"] = "Folder" + +-- Reason +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1093747001"] = "Reason" + +-- Configuration folder +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1191404447"] = "Configuration folder" + +-- Could not open the folder. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1215994612"] = "Could not open the folder." + +-- Your projects in the Visual Briefing Assistant. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1259707710"] = "Your projects in the Visual Briefing Assistant." + +-- Your temporary chats, along with their transcripts. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1293614512"] = "Your temporary chats, along with their transcripts." + +-- What belongs into a backup +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1682175486"] = "What belongs into a backup" + +-- The index of your local data sources: which files AI Studio embedded, and their text. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T170332953"] = "The index of your local data sources: which files AI Studio embedded, and their text." + +-- Keys with which AI Studio protects data in its user interface. Only the Flatpak keeps them here, and on another computer, AI Studio creates new ones. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1981169629"] = "Keys with which AI Studio protects data in its user interface. Only the Flatpak keeps them here, and on another computer, AI Studio creates new ones." + +-- Tools which AI Studio downloaded for this OS, such as Pandoc. On another computer, AI Studio downloads them again. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T1992121437"] = "Tools which AI Studio downloaded for this OS, such as Pandoc. On another computer, AI Studio downloads them again." + +-- What a backup can leave out +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2120934651"] = "What a backup can leave out" + +-- Good to know +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2373663376"] = "Good to know" + +-- Data folder +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2422278827"] = "Data folder" + +-- Explanation +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2840582448"] = "Explanation" + +-- Copies of your settings, which every version of AI Studio keeps for itself. Your current settings are all a backup needs. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2851981238"] = "Copies of your settings, which every version of AI Studio keeps for itself. Your current settings are all a backup needs." + +-- The tokenizers you chose for your providers. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2892362083"] = "The tokenizers you chose for your providers." + +-- To back up your data with AI Studio itself, or to move it to another computer, use Data Backup & Restore on this page or in the settings instead. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2906063884"] = "To back up your data with AI Studio itself, or to move it to another computer, use Data Backup & Restore on this page or in the settings instead." + +-- Your plugins, such as the assistants you built, apart from the folders listed below. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2979110360"] = "Your plugins, such as the assistants you built, apart from the folders listed below." + +-- Open in your file manager +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2986518596"] = "Open in your file manager" + +-- Your organization deployed some plugins among your own ones, for example, with a device management. Their folders do not show this, so the list above names them one by one. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3156743605"] = "Your organization deployed some plugins among your own ones, for example, with a device management. Their folders do not show this, so the list above names them one by one." + +-- Content +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3244793698"] = "Content" + +-- The embeddings of your local data sources. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3301102729"] = "The embeddings of your local data sources." + +-- The folder is not known yet. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3310368654"] = "The folder is not known yet." + +-- Could not open the folder: {0} +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3380661810"] = "Could not open the folder: {0}" + +-- Unknown error +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3461425987"] = "Unknown error" + +-- Plugins of your organization. Your organization brings them back. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T346886000"] = "Plugins of your organization. Your organization brings them back." + +-- Back up the indexes of your data sources only while AI Studio is closed. AI Studio writes to them while it runs, so a copy made meanwhile may not fit together, and AI Studio would have to embed your data sources anew. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3655837348"] = "Back up the indexes of your data sources only while AI Studio is closed. AI Studio writes to them while it runs, so a copy made meanwhile may not fit together, and AI Studio would have to embed your data sources anew." + +-- Files which AI Studio needs only for a moment and deletes again. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T366353671"] = "Files which AI Studio needs only for a moment and deletes again." + +-- Plugins which come with AI Studio, such as the language packs. AI Studio brings them back by itself. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3877928356"] = "Plugins which come with AI Studio, such as the language packs. AI Studio brings them back by itself." + +-- Path +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3949388886"] = "Path" + +-- Your API keys, tokens, and passwords are not in these folders. AI Studio keeps them in the secure storage of your OS. After you restored your data on another computer, enter them there once more. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T4093107108"] = "Your API keys, tokens, and passwords are not in these folders. AI Studio keeps them in the secure storage of your OS. After you restored your data on another computer, enter them there once more." + +-- Your chats in workspaces, along with their transcripts. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T509585668"] = "Your chats in workspaces, along with their transcripts." + +-- Log files. They help when something goes wrong, but they hold none of your data. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T53143530"] = "Log files. They help when something goes wrong, but they hold none of your data." + +-- AI Studio keeps your data in two folders on this computer. When you back up your computer with a tool of your own, such as [Restic](https://restic.net), [Borg](https://www.borgbackup.org), or [Time Machine](https://support.apple.com/en-us/104984), this shows which of their contents belong into the backup and which it can leave out. Click a path to copy it to the clipboard, or use the folder button to open it in your file manager. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T69622543"] = "AI Studio keeps your data in two folders on this computer. When you back up your computer with a tool of your own, such as [Restic](https://restic.net), [Borg](https://www.borgbackup.org), or [Time Machine](https://support.apple.com/en-us/104984), this shows which of their contents belong into the backup and which it can leave out. Click a path to copy it to the clipboard, or use the folder button to open it in your file manager." + +-- Your settings. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T790727583"] = "Your settings." + +-- The following is meant for experts who back up their computer with a tool of their own. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T813925103"] = "The following is meant for experts who back up their computer with a tool of their own." + +-- On this computer, the data folder lies inside the configuration folder. A backup of the configuration folder therefore takes the data folder along. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T91194331"] = "On this computer, the data folder lies inside the configuration folder. A backup of the configuration folder therefore takes the data folder along." + -- Yes, please send my data to the external embedding provider UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T1159107763"] = "Yes, please send my data to the external embedding provider" @@ -3810,6 +4041,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T6617545 -- All files in this folder and its subfolders UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T916879200"] = "All files in this folder and its subfolders" +-- AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1006756723"] = "AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done." + -- You might configure different data sources. A data source can include one file, all files in a directory, or data from your company. Later, you can incorporate these data sources as needed when the AI requires this data to complete a certain task. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "You might configure different data sources. A data source can include one file, all files in a directory, or data from your company. Later, you can incorporate these data sources as needed when the AI requires this data to complete a certain task." @@ -6087,6 +6321,84 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::CONFIRMDIALOG::T1642511898"] = "No" -- Yes UI_TEXT_CONTENT["AISTUDIO::DIALOGS::CONFIRMDIALOG::T3013883440"] = "Yes" +-- Restart now +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T1409292924"] = "Restart now" + +-- Discard the restore +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T3017406920"] = "Discard the restore" + +-- Your backup is ready. AI Studio restarts to put it in place. Whatever you changed until then would be lost, so please restart now. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T495853773"] = "Your backup is ready. AI Studio restarts to put it in place. Whatever you changed until then would be lost, so please restart now." + +-- AI Studio could not restart by itself. Please quit AI Studio and start it again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREPENDINGDIALOG::T573887863"] = "AI Studio could not restart by itself. Please quit AI Studio and start it again." + +-- Restoring your backup failed. AI Studio put all your data back the way they were before. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1057857006"] = "Restoring your backup failed. AI Studio put all your data back the way they were before." + +-- Your backup was not restored +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1193456690"] = "Your backup was not restored" + +-- Restoring your backup failed, and AI Studio could not yet put all your data back the way they were. It tries again at the next start, so please restart AI Studio. If this message appears again, please contact the support, and keep the folders '.restore-previous' and '.restore-pending' in your data folder. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T123391329"] = "Restoring your backup failed, and AI Studio could not yet put all your data back the way they were. It tries again at the next start, so please restart AI Studio. If this message appears again, please contact the support, and keep the folders '.restore-previous' and '.restore-pending' in your data folder." + +-- The batch processing uses '{0}', which does not exist on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1314976065"] = "The batch processing uses '{0}', which does not exist on this computer." + +-- The backup is from {0} and was made with AI Studio {1}. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1363238678"] = "The backup is from {0} and was made with AI Studio {1}." + +-- A backup holds no API keys or passwords, and some files and folders are not on this computer. Please check the following: +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1633897333"] = "A backup holds no API keys or passwords, and some files and folders are not on this computer. Please check the following:" + +-- Enter the API key of the provider '{0}' again, if it needs one. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T1645175373"] = "Enter the API key of the provider '{0}' again, if it needs one." + +-- Restoring your backup is not finished +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2415452768"] = "Restoring your backup is not finished" + +-- Enter the API key of the embedding provider '{0}' again, if it needs one. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2492749778"] = "Enter the API key of the embedding provider '{0}' again, if it needs one." + +-- Enter the API key of the transcription provider '{0}' again, if it needs one. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2568556297"] = "Enter the API key of the transcription provider '{0}' again, if it needs one." + +-- Nothing else to do: every provider and every data source has what it needs on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2671436044"] = "Nothing else to do: every provider and every data source has what it needs on this computer." + +-- The data source '{0}' names a file or folder which does not exist on this computer. Choose it anew. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T270687216"] = "The data source '{0}' names a file or folder which does not exist on this computer. Choose it anew." + +-- The file with your own root certificates, '{0}', does not exist on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T2844911500"] = "The file with your own root certificates, '{0}', does not exist on this computer." + +-- AI Studio could not start restoring your backup, so all your data stayed as they were. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3044053464"] = "AI Studio could not start restoring your backup, so all your data stayed as they were." + +-- Enter the password of the mailbox '{0}' again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3083231765"] = "Enter the password of the mailbox '{0}' again." + +-- Enter the password or token of the data source '{0}' again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3430781520"] = "Enter the password or token of the data source '{0}' again." + +-- Technical details: {0} +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3436390766"] = "Technical details: {0}" + +-- Close +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3448155331"] = "Close" + +-- Enter the setting '{1}' of the tool '{0}' again, if you used it. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3658964195"] = "Enter the setting '{1}' of the tool '{0}' again, if you used it." + +-- AI Studio replaced your data with those of the backup. The plugins and settings of your organization stayed as they were. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3896631626"] = "AI Studio replaced your data with those of the backup. The plugins and settings of your organization stayed as they were." + +-- Your backup was restored +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T3980336073"] = "Your backup was restored" + +-- The ERI server '{0}' writes into a folder which does not exist on this computer. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATARESTOREREPORTDIALOG::T472632172"] = "The ERI server '{0}' writes into a folder which does not exist on this computer." + -- How many matches do you want at most per query? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1DIALOG::T1827669611"] = "How many matches do you want at most per query?" @@ -9387,6 +9699,9 @@ UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2800137365"] = "An update to ver -- Please wait for the update to complete... UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2864211629"] = "Please wait for the update to complete..." +-- Restore of a Backup +UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2912921801"] = "Restore of a Backup" + -- Supporters UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2929332068"] = "Supporters" @@ -10191,6 +10506,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3294830584"] = "Discover MindWor -- External HTTPS custom root certificates UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3315279770"] = "External HTTPS custom root certificates" +-- Data Backup & Restore +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3317700277"] = "Data Backup & Restore" + -- User-language provided by the OS UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3334355246"] = "User-language provided by the OS" @@ -10347,6 +10665,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T551035563"] = "Open Log Viewer" -- This is a library providing the foundations for asynchronous programming in Rust. It includes key trait definitions like Stream, as well as utilities like join!, select!, and various futures combinator methods which enable expressive asynchronous control flow. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T566998575"] = "This is a library providing the foundations for asynchronous programming in Rust. It includes key trait definitions like Stream, as well as utilities like join!, select!, and various futures combinator methods which enable expressive asynchronous control flow." +-- Your Data Folders +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T570573187"] = "Your Data Folders" + -- Used .NET SDK UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T585329785"] = "Used .NET SDK" @@ -10530,15 +10851,15 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::PLUGINS::T837269472"] = "The plugin could not -- Settings UI_TEXT_CONTENT["AISTUDIO::PAGES::SETTINGS::T1258653480"] = "Settings" +-- Data Backup & Restore +UI_TEXT_CONTENT["AISTUDIO::PAGES::SETTINGS::T3317700277"] = "Data Backup & Restore" + -- Thank you very much, Kerstin, for creating the Wiki, for advising us on ERI with real use cases, and for your work on version 2 of the ERI protocol. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1367356224"] = "Thank you very much, Kerstin, for creating the Wiki, for advising us on ERI with real use cases, and for your work on version 2 of the ERI protocol." -- Thank you for being the first to contribute a one-time donation. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1470916504"] = "Thank you for being the first to contribute a one-time donation." --- Thank you, Jens, for the Batch Processing Assistant, the exact token count in the chat, and everything else you are building for AI Studio with so much dedication. -UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1701321325"] = "Thank you, Jens, for the Batch Processing Assistant, the exact token count in the chat, and everything else you are building for AI Studio with so much dedication." - -- Thank you, Peer, for your courage in being the second person to support the project financially. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T1714878838"] = "Thank you, Peer, for your courage in being the second person to support the project financially." @@ -10626,6 +10947,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T686206269"] = "Thanks for your bu -- Business Contributors UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T838479287"] = "Business Contributors" +-- Thank you, Jens, for the Batch Processing Assistant, the exact token count in the chat, the backups of all data, and everything else you are building for AI Studio with so much dedication. +UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T882777973"] = "Thank you, Jens, for the Batch Processing Assistant, the exact token count in the chat, the backups of all data, and everything else you are building for AI Studio with so much dedication." + -- Thanks, Oliver, for making sure no spaces get lost in answers from OpenAI. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T916186976"] = "Thanks, Oliver, for making sure no spaces get lost in answers from OpenAI." @@ -11268,6 +11592,15 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::AUTHMETHODSV1EXTENSIONS::T810547195"] = "No au -- Username & Password UI_TEXT_CONTENT["AISTUDIO::TOOLS::AUTHMETHODSV1EXTENSIONS::T863358904"] = "Username & Password" +-- Unexpected error: {0} +UI_TEXT_CONTENT["AISTUDIO::TOOLS::CHATARCHIVE::CHATARCHIVEIMPORTER::T2350673880"] = "Unexpected error: {0}" + +-- This file is no backup of AI Studio, or it was not written completely. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::CHATARCHIVE::CHATARCHIVEIMPORTER::T3123318432"] = "This file is no backup of AI Studio, or it was not written completely." + +-- Imported workspace +UI_TEXT_CONTENT["AISTUDIO::TOOLS::CHATARCHIVE::CHATARCHIVEIMPORTER::T722364324"] = "Imported workspace" + -- Spanish (Spain) UI_TEXT_CONTENT["AISTUDIO::TOOLS::COMMONLANGUAGEEXTENSIONS::T1269145091"] = "Spanish (Spain)" @@ -11403,6 +11736,24 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T3893997203"] = " -- Trust all LLM providers UI_TEXT_CONTENT["AISTUDIO::TOOLS::CONFIDENCESCHEMESEXTENSIONS::T4107860491"] = "Trust all LLM providers" +-- This file is no backup of AI Studio, or it was not written completely. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATABACKUPREADER::T3123318432"] = "This file is no backup of AI Studio, or it was not written completely." + +-- This backup was made by a newer version of AI Studio. Please update AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATABACKUPREADER::T3812849494"] = "This backup was made by a newer version of AI Studio. Please update AI Studio first." + +-- The content list of this backup could not be read. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATABACKUPREADER::T3954689017"] = "The content list of this backup could not be read." + +-- This file is no backup of AI Studio, or it was not written completely. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATARESTOREPREPARER::T3123318432"] = "This file is no backup of AI Studio, or it was not written completely." + +-- The backup is damaged: the file '{0}' does not match its content list. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATARESTOREPREPARER::T3493230602"] = "The backup is damaged: the file '{0}' does not match its content list." + +-- This backup was made by a newer version of AI Studio. Please update AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATARESTOREPREPARER::T3812849494"] = "This backup was made by a newer version of AI Studio. Please update AI Studio first." + -- Reason UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1093747001"] = "Reason" @@ -11553,6 +11904,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T858000918"] = "Thes -- Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while. UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T874850580"] = "Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while." +-- AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T1006756723"] = "AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done." + +-- Close +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T3448155331"] = "Close" + -- Repair Data Source UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Repair Data Source" @@ -12483,6 +12840,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T377 -- Text UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text" +-- AI Studio backup +UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1046878067"] = "AI Studio backup" + -- Office Files UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office Files" @@ -12744,9 +13104,45 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGINGENERATIONSERVICE::T5 -- Workspace UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGINGENERATIONSERVICE::T658612054"] = "Workspace" +-- A backup is running already. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T1319749333"] = "A backup is running already." + +-- The backup is too large for the chosen drive, which takes no files beyond 4 GB. USB sticks formatted with FAT32 are such drives. Please choose another drive. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T1869616367"] = "The backup is too large for the chosen drive, which takes no files beyond 4 GB. USB sticks formatted with FAT32 are such drives. Please choose another drive." + +-- The backup failed: {0} +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T1915001713"] = "The backup failed: {0}" + +-- Your organization does not allow backups. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T2011318013"] = "Your organization does not allow backups." + +-- AI Studio is still writing an answer or a transcript. Please start the backup once it is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATABACKUPSERVICE::T4161531565"] = "AI Studio is still writing an answer or a transcript. Please start the backup once it is done." + +-- Your organization does not allow restoring backups. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T1032491606"] = "Your organization does not allow restoring backups." + +-- AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T1555678613"] = "AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first." + +-- AI Studio is still writing an answer or a transcript. Please restore once it is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T1847516630"] = "AI Studio is still writing an answer or a transcript. Please restore once it is done." + +-- The restore could not be prepared: {0} +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T2653345012"] = "The restore could not be prepared: {0}" + +-- A restore is being prepared already. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T4157700208"] = "A restore is being prepared already." + +-- A backup is running. Please restore once it is done. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATARESTORESERVICE::T553866901"] = "A backup is running. Please restore once it is done." + -- The local RAG index database is not available. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T1738200026"] = "The local RAG index database is not available." +-- The index of the data source '{0}' is missing. The data source stays out of your chats until its index was built anew. Use the repair action to start that. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T204783307"] = "The index of the data source '{0}' is missing. The data source stays out of your chats until its index was built anew. Use the repair action to start that." + -- The selected embedding provider is not allowed to index this data source. The data source asks for the confidence level '{0}', while the embedding provider has '{1}'. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T2186533187"] = "The selected embedding provider is not allowed to index this data source. The data source asks for the confidence level '{0}', while the embedding provider has '{1}'." @@ -12852,6 +13248,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T2360 -- The mail '{0}' could not be indexed. AI Studio tries again during the next sync. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T804280374"] = "The mail '{0}' could not be indexed. AI Studio tries again during the next sync." +-- The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1214987337"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index." + +-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1393648320"] = "The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source." + -- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1935191670"] = "The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run." @@ -12861,15 +13263,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T242460 -- The folder '{0}' could not be opened. Please check whether you are allowed to read it. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3230000698"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it." +-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3722338131"] = "The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source." + -- The file '{0}' could not be read. Please check whether you are allowed to read it. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3924882233"] = "The file '{0}' could not be read. Please check whether you are allowed to read it." --- The file '{0}' does not exist. -UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T451561215"] = "The file '{0}' does not exist." - --- The folder '{0}' does not exist. -UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T871336081"] = "The folder '{0}' does not exist." - -- The local index '{0}' could not be created again. Please restart AI Studio and try once more. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::INDEXEDRUNCONTEXT::T1394295123"] = "The local index '{0}' could not be created again. Please restart AI Studio and try once more." diff --git a/app/MindWork AI Studio/Program.cs b/app/MindWork AI Studio/Program.cs index 5af2adf97..7052b9590 100644 --- a/app/MindWork AI Studio/Program.cs +++ b/app/MindWork AI Studio/Program.cs @@ -210,6 +210,8 @@ internal sealed class Program builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -226,6 +228,11 @@ internal sealed class Program builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); + builder.Services.AddHostedService(); + + // Compatibility shim, see documentation/compatibility-shims/2026-10-relative-data-paths.md. Remove after 2027-04-14: + builder.Services.AddHostedService(); + builder.Services.AddHostedService(); builder.Services.AddHostedService(sp => sp.GetRequiredService()); builder.Services.AddSingleton(); diff --git a/app/MindWork AI Studio/Settings/ChatTemplate.cs b/app/MindWork AI Studio/Settings/ChatTemplate.cs index 22f69c3ed..ebe52bec8 100644 --- a/app/MindWork AI Studio/Settings/ChatTemplate.cs +++ b/app/MindWork AI Studio/Settings/ChatTemplate.cs @@ -421,11 +421,7 @@ public record ChatTemplate( } var combinedPath = Path.GetFullPath(Path.Combine(pluginRoot, relativePath)); - var pluginRootWithSeparator = pluginRoot.EndsWith(Path.DirectorySeparatorChar) - ? pluginRoot - : pluginRoot + Path.DirectorySeparatorChar; - var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - if (!combinedPath.StartsWith(pluginRootWithSeparator, comparison)) + if (!PathTools.IsInsideDirectory(pluginRoot, combinedPath)) { LOGGER.LogWarning("The relative FileAttachments entry {AttachmentNum} in chat template {IdxChatTemplate} points outside of the plugin folder and will be ignored.", attachmentNum, idx); return false; diff --git a/app/MindWork AI Studio/Settings/DataModel/Data.cs b/app/MindWork AI Studio/Settings/DataModel/Data.cs index f72aa878d..9a4120a2e 100644 --- a/app/MindWork AI Studio/Settings/DataModel/Data.cs +++ b/app/MindWork AI Studio/Settings/DataModel/Data.cs @@ -116,6 +116,16 @@ public sealed class Data /// public List AppliedEnterpriseAssistantActivations { get; set; } = []; + /// + /// Whether AI Studio already stored the chats and the settings once more, so that every path below + /// the data directory in them is relative. + /// + /// + /// Compatibility shim, see documentation/compatibility-shims/2026-10-relative-data-paths.md. + /// Remove after 2027-04-14. + /// + public bool StoredDataPathsAreRelative { get; set; } + /// /// The next provider number to use. /// diff --git a/app/MindWork AI Studio/Settings/DataModel/DataApp.cs b/app/MindWork AI Studio/Settings/DataModel/DataApp.cs index 6918c4b10..252da172a 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataApp.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataApp.cs @@ -219,6 +219,24 @@ public sealed class DataApp(Expression>? configSelection = n /// public bool AllowUserToSharePlugins { get; set; } = ManagedConfiguration.Register(configSelection, n => n.AllowUserToSharePlugins, true); + /// + /// Should the user be allowed to back up the data of AI Studio into a file? + /// + /// + /// A backup holds chats and other confidential content in a single file, which can leave the + /// computer on any stick or cloud drive. Secrets never go into it. + /// + public bool AllowUserToBackUpData { get; set; } = ManagedConfiguration.Register(configSelection, n => n.AllowUserToBackUpData, true); + + /// + /// Should the user be allowed to restore the data of AI Studio from a backup file, or to add the chats of one? + /// + /// + /// A backup brings along everything its chats and settings name, including the locations of + /// attached documents and of local data sources, which are read again afterward. + /// + public bool AllowUserToRestoreData { get; set; } = ManagedConfiguration.Register(configSelection, n => n.AllowUserToRestoreData, true); + /// /// Should administration settings be visible in the UI? /// diff --git a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs index c84a110ce..01842831e 100644 --- a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs +++ b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs @@ -22,7 +22,7 @@ public sealed record EmbeddingProvider( Guid EnterpriseConfigurationPluginId = default, string Hostname = "http://localhost:1234", Host Host = Host.NONE, - string TokenizerPath = "", + [property: JsonConverter(typeof(DataDirectoryRelativePathConverter))] string TokenizerPath = "", string TokenizerFingerprint = "", int EmbeddingBatchSize = 0, int TokenLimit = 0, diff --git a/app/MindWork AI Studio/Settings/PluginIconFile.cs b/app/MindWork AI Studio/Settings/PluginIconFile.cs index 224b1a175..78b47fdc0 100644 --- a/app/MindWork AI Studio/Settings/PluginIconFile.cs +++ b/app/MindWork AI Studio/Settings/PluginIconFile.cs @@ -56,7 +56,7 @@ internal static class PluginIconFile return false; } - if (!IsInsideDirectory(pluginRoot, resolvedPath)) + if (!PathTools.IsInsideDirectory(pluginRoot, resolvedPath)) { issue = "The icon path points outside of the configuration plugin directory."; return false; @@ -108,18 +108,10 @@ internal static class PluginIconFile currentPath = Path.Combine(currentPath, segment); FileSystemInfo pathInfo = Directory.Exists(currentPath) ? new DirectoryInfo(currentPath) : new FileInfo(currentPath); var finalTarget = pathInfo.ResolveLinkTarget(true); - if (finalTarget is not null && !IsInsideDirectory(rootDirectory, finalTarget.FullName)) + if (finalTarget is not null && !PathTools.IsInsideDirectory(rootDirectory, finalTarget.FullName)) return false; } return true; } - - private static bool IsInsideDirectory(string rootDirectory, string path) - { - var root = Path.GetFullPath(rootDirectory).TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; - var target = Path.GetFullPath(path); - var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - return target.StartsWith(root, comparison); - } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Settings/Provider.cs b/app/MindWork AI Studio/Settings/Provider.cs index fd954bd64..77ef5c77f 100644 --- a/app/MindWork AI Studio/Settings/Provider.cs +++ b/app/MindWork AI Studio/Settings/Provider.cs @@ -36,7 +36,7 @@ public sealed record Provider( Host Host = Host.NONE, HFInferenceProvider HFInferenceProvider = HFInferenceProvider.NONE, string AdditionalJsonApiParameters = "", - string TokenizerPath = "", + [property: JsonConverter(typeof(DataDirectoryRelativePathConverter))] string TokenizerPath = "", ProviderCapabilityOverrides? CapabilityOverrides = null, bool AllowUserProvidedAPIKey = false, string CustomIconDataUrl = "") : ConfigurationBaseObject, ISecretId, IUserProvidedAPIKey diff --git a/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveCollisionBehavior.cs b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveCollisionBehavior.cs new file mode 100644 index 000000000..c384473f7 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveCollisionBehavior.cs @@ -0,0 +1,17 @@ +namespace AIStudio.Tools.ChatArchive; + +/// +/// Determines what the import does when a chat from the archive already exists. +/// +public enum ChatArchiveCollisionBehavior +{ + /// + /// Keeps the existing chat and ignores the one from the archive. + /// + SKIP, + + /// + /// Imports the chat from the archive as an additional chat. + /// + IMPORT_AS_COPY, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveFormat.cs b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveFormat.cs new file mode 100644 index 000000000..daa3e0007 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveFormat.cs @@ -0,0 +1,33 @@ +using AIStudio.Tools.DataBackup; + +namespace AIStudio.Tools.ChatArchive; + +/// +/// Where a backup keeps its chats, for adding them to the chats there are. +/// +/// +/// A backup holds the chats in the layout of the data folder, cf. DataBackupFormat: every chat in a +/// directory of its own, below its workspace or among the temporary chats. +/// +public static class ChatArchiveFormat +{ + /// + /// Where a backup keeps the workspaces, each in a directory named after its ID. + /// + public static readonly string WORKSPACES_PREFIX = DataBackupFormat.DATA_ROOT + DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.WORKSPACES).RelativePath; + + /// + /// Where a backup keeps the temporary chats, each in a directory named after its ID. + /// + public static readonly string TEMPORARY_CHATS_PREFIX = DataBackupFormat.DATA_ROOT + DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.TEMPORARY_CHATS).RelativePath; + + /// + /// The file storing the workspace or chat name. + /// + public const string NAME_FILE_NAME = "name"; + + /// + /// The file storing the serialized chat thread. + /// + public const string THREAD_FILE_NAME = "thread.json"; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImportOutcome.cs b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImportOutcome.cs new file mode 100644 index 000000000..61f3dd8f7 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImportOutcome.cs @@ -0,0 +1,22 @@ +namespace AIStudio.Tools.ChatArchive; + +/// +/// The outcome of importing one single chat. +/// +public enum ChatArchiveImportOutcome +{ + /// + /// The chat was written to the chat storage. + /// + IMPORTED, + + /// + /// The chat already existed and was left alone. + /// + SKIPPED, + + /// + /// The chat could not be imported. + /// + FAILED, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImportPreview.cs b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImportPreview.cs new file mode 100644 index 000000000..317594cdd --- /dev/null +++ b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImportPreview.cs @@ -0,0 +1,9 @@ +namespace AIStudio.Tools.ChatArchive; + +/// +/// Describes the contents of an archive before anything is imported. +/// +/// Whether the archive could be read. +/// The manifest of the archive. +/// The reason why the archive could not be read, if it could not. +public sealed record ChatArchiveImportPreview(bool Success, ChatArchiveManifest Manifest, string Issue); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImportResult.cs b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImportResult.cs new file mode 100644 index 000000000..03f32b16c --- /dev/null +++ b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImportResult.cs @@ -0,0 +1,20 @@ +namespace AIStudio.Tools.ChatArchive; + +/// +/// The outcome of a chat archive import. +/// +/// Whether the import ran without an error. +/// The number of workspaces which received chats. +/// The number of imported chats. +/// The number of chats which were skipped, because they already existed. +/// The number of chats which could not be imported. +/// The reason why the import failed, if it did. +/// Whether the user stopped the import. Chats imported up to that point are kept. +public sealed record ChatArchiveImportResult( + bool Success, + int ImportedWorkspaces, + int ImportedChats, + int SkippedChats, + int FailedChats, + string Issue, + bool Cancelled); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImporter.cs b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImporter.cs new file mode 100644 index 000000000..37be8c6ba --- /dev/null +++ b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImporter.cs @@ -0,0 +1,539 @@ +using System.IO.Compression; +using System.Text.Json; + +using AIStudio.Chat; +using AIStudio.Settings; +using AIStudio.Tools.DataBackup; +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tools.ChatArchive; + +/// +/// Adds the chats of a backup to the chats there are, and leaves everything else alone. +/// +/// +/// The other way to use a backup, next to restoring it: a restore replaces all data, while this keeps +/// every chat and adds those of the backup. A chat which exists already is kept, or added once more, +/// as the user decides.

+/// +/// The backup is opened for reading only, and every entry is read as a stream, cf. DataBackupReader. +/// +public static class ChatArchiveImporter +{ + private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger(nameof(ChatArchiveImporter)); + + private const int BUFFER_SIZE = 64 * 1024; + + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ChatArchiveImporter).Namespace, nameof(ChatArchiveImporter)); + + /// + /// One entry of a backup, with its name checked and normalized. + /// + private sealed record BackupEntry(string Name, ZipArchiveEntry Entry); + + /// + /// What one import did so far. + /// + /// + /// Every counter is raised as the import goes: a cancellation leaves the chats imported so far in + /// the storage, so the numbers have to describe them even when the import never reaches its end. + /// + private sealed class ImportTally + { + public int ProcessedChats; + public int ImportedWorkspaces; + public int ImportedChats; + public int SkippedChats; + public int FailedChats; + } + + /// + /// Reads which chats a backup holds, without adding any. + /// + /// The backup to look into. + /// Cancels the reading. + /// The chats of the backup, or why the backup cannot be used. + public static async Task ReadPreviewAsync(string archivePath, CancellationToken token) + { + try + { + using var archive = ZipFile.OpenRead(archivePath); + var manifestResult = await DataBackupReader.ReadManifestAsync(archive, token); + if (manifestResult.Manifest is not { } manifest) + return new(false, new(), manifestResult.Issue); + + var chats = await ListChatsAsync(archive, GroupEntriesByChat(archive), token); + return new(true, chats with { CreatedAt = manifest.CreatedAtUtc, AppVersion = manifest.AppVersion }, string.Empty); + } + catch (OperationCanceledException) + { + throw; + } + catch (InvalidDataException) + { + return new(false, new(), TB("This file is no backup of AI Studio, or it was not written completely.")); + } + catch (Exception exception) + { + LOG.LogError(exception, "Failed to read the backup '{ArchivePath}'.", archivePath); + return new(false, new(), string.Format(TB("Unexpected error: {0}"), exception.Message)); + } + } + + /// + /// Adds all chats of a backup to the chat storage. + /// + /// The backup. + /// What to do with chats which exist already. + /// Receives the progress. + /// Cancels the import. Chats added before the cancellation are kept. + /// What the import did. + public static async Task ImportAsync(string archivePath, ChatArchiveCollisionBehavior collisionBehavior, IProgress? progress, CancellationToken token) + { + var tally = new ImportTally(); + var preview = await ReadPreviewAsync(archivePath, token); + if (!preview.Success) + return new(false, 0, 0, 0, 0, preview.Issue, false); + + try + { + var manifest = preview.Manifest; + using var archive = ZipFile.OpenRead(archivePath); + var entriesByChat = GroupEntriesByChat(archive); + + // Which workspaces exist already is asked once per workspace of the backup, so read + // the tree once instead of building a full snapshot every time: + var tree = await WorkspaceBehaviour.GetOrLoadWorkspaceTreeShellAsync(); + var knownWorkspaceIds = tree.Workspaces.Select(workspace => workspace.WorkspaceId).ToHashSet(); + + // + // A chat counts as existing wherever it is stored, not just in the workspace the backup + // names: it may have been moved since the backup was made. Adding it there again under + // the same identity would leave two chats with one ID, while the app finds a chat, and + // the answer being written for it, by that ID alone: + // + var existingChatIds = tree.TemporaryChats.Select(chat => chat.ChatId).ToHashSet(); + foreach (var workspace in tree.Workspaces) + { + token.ThrowIfCancellationRequested(); + foreach (var chat in await WorkspaceBehaviour.GetWorkspaceChatsAsync(workspace.WorkspaceId)) + existingChatIds.Add(chat.ChatId); + } + + foreach (var workspace in manifest.Workspaces) + { + token.ThrowIfCancellationRequested(); + + var targetWorkspaceId = await ResolveTargetWorkspaceAsync(workspace, knownWorkspaceIds); + if (targetWorkspaceId == Guid.Empty) + { + LOG.LogWarning("Skipping workspace '{WorkspaceName}' of the backup, because no target workspace could be prepared.", workspace.Name); + tally.FailedChats += workspace.Chats.Count; + tally.ProcessedChats += workspace.Chats.Count; + continue; + } + + var workspaceEntryPrefix = $"{ChatArchiveFormat.WORKSPACES_PREFIX}{workspace.WorkspaceId}{ZipEntryNames.SEPARATOR}"; + await ImportChatsAsync(tally, workspaceEntryPrefix, targetWorkspaceId, workspace.Chats, true, manifest.TotalChatCount, entriesByChat, collisionBehavior, existingChatIds, progress, token); + } + + await ImportChatsAsync(tally, ChatArchiveFormat.TEMPORARY_CHATS_PREFIX, Guid.Empty, manifest.TemporaryChats, false, manifest.TotalChatCount, entriesByChat, collisionBehavior, existingChatIds, progress, token); + + progress?.Report(new(tally.ProcessedChats, manifest.TotalChatCount)); + LOG.LogInformation("Imported {ImportedChats} chats from the backup '{ArchivePath}'. Skipped {SkippedChats} existing chats, {FailedChats} chats failed.", tally.ImportedChats, archivePath, tally.SkippedChats, tally.FailedChats); + + return new(true, tally.ImportedWorkspaces, tally.ImportedChats, tally.SkippedChats, tally.FailedChats, string.Empty, false); + } + catch (OperationCanceledException) + { + // + // The chats imported so far stay in the chat storage. So report what happened instead + // of throwing, otherwise the user has no idea what is now part of their chats: + // + LOG.LogInformation("The user cancelled the import of the backup '{ArchivePath}' after {ImportedChats} chats.", archivePath, tally.ImportedChats); + return new(true, tally.ImportedWorkspaces, tally.ImportedChats, tally.SkippedChats, tally.FailedChats, string.Empty, true); + } + catch (Exception exception) + { + LOG.LogError(exception, "Failed to import the backup '{ArchivePath}'.", archivePath); + return new(false, tally.ImportedWorkspaces, tally.ImportedChats, tally.SkippedChats, tally.FailedChats, string.Format(TB("Unexpected error: {0}"), exception.Message), false); + } + } + + /// + /// Adds the chats of one workspace of the backup, or its temporary chats. + /// + private static async Task ImportChatsAsync(ImportTally tally, string entryPrefix, Guid targetWorkspaceId, IReadOnlyList chats, bool countsAsWorkspace, int totalChats, Dictionary> entriesByChat, ChatArchiveCollisionBehavior collisionBehavior, HashSet existingChatIds, IProgress? progress, CancellationToken token) + { + var isFirstImportOfWorkspace = true; + foreach (var chat in chats) + { + token.ThrowIfCancellationRequested(); + progress?.Report(new(tally.ProcessedChats, totalChats)); + tally.ProcessedChats++; + + var chatEntryPrefix = $"{entryPrefix}{chat.ChatId}{ZipEntryNames.SEPARATOR}"; + var chatEntries = entriesByChat.GetValueOrDefault(chatEntryPrefix, []); + + switch (await ImportChatAsync(chatEntries, chatEntryPrefix, targetWorkspaceId, chat, collisionBehavior, existingChatIds, token)) + { + case ChatArchiveImportOutcome.IMPORTED: + tally.ImportedChats++; + if (countsAsWorkspace && isFirstImportOfWorkspace) + { + tally.ImportedWorkspaces++; + isFirstImportOfWorkspace = false; + } + + break; + + case ChatArchiveImportOutcome.SKIPPED: + tally.SkippedChats++; + break; + + case ChatArchiveImportOutcome.FAILED: + tally.FailedChats++; + break; + } + } + } + + /// + /// Sorts the entries of a backup by the chat they belong to. Without this, every chat would have + /// to scan all entries of the backup. + /// + /// + /// An entry whose name could lead outside of the chat directory is left out, cf. ZipEntryNames. + /// + private static Dictionary> GroupEntriesByChat(ZipArchive archive) + { + var entriesByChat = new Dictionary>(StringComparer.Ordinal); + foreach (var entry in archive.Entries) + { + if (!ZipEntryNames.TryNormalize(entry.FullName, out var entryName)) + { + LOG.LogWarning("Ignoring the backup entry '{EntryName}', because its name could lead outside of the chat directory.", entry.FullName); + continue; + } + + // Skip directory entries: + if (entryName.EndsWith(ZipEntryNames.SEPARATOR)) + continue; + + var chatEntryPrefix = GetChatEntryPrefix(entryName); + if (chatEntryPrefix is null) + continue; + + if (!entriesByChat.TryGetValue(chatEntryPrefix, out var chatEntries)) + { + chatEntries = []; + entriesByChat[chatEntryPrefix] = chatEntries; + } + + chatEntries.Add(new(entryName, entry)); + } + + return entriesByChat; + } + + /// + /// Determines the chat an entry belongs to: everything up to and including the ID of the chat, so + /// "data/workspaces/W/C/thread.json" becomes "data/workspaces/W/C/". + /// + /// The chat prefix, or null when the entry belongs to no chat. + private static string? GetChatEntryPrefix(string entryName) + { + string root; + int levels; + if (entryName.StartsWith(ChatArchiveFormat.WORKSPACES_PREFIX, StringComparison.Ordinal)) + { + root = ChatArchiveFormat.WORKSPACES_PREFIX; + levels = 2; + } + else if (entryName.StartsWith(ChatArchiveFormat.TEMPORARY_CHATS_PREFIX, StringComparison.Ordinal)) + { + root = ChatArchiveFormat.TEMPORARY_CHATS_PREFIX; + levels = 1; + } + else + return null; + + var index = root.Length; + for (var level = 0; level < levels; level++) + { + var next = entryName.IndexOf(ZipEntryNames.SEPARATOR, index); + if (next < 0) + return null; + + index = next + 1; + } + + return entryName[..index]; + } + + /// + /// Lists the chats of a backup, with the names of their workspaces. + /// + private static async Task ListChatsAsync(ZipArchive archive, Dictionary> entriesByChat, CancellationToken token) + { + var workspaces = new Dictionary(); + var temporaryChats = new List(); + foreach (var (chatEntryPrefix, chatEntries) in entriesByChat) + { + token.ThrowIfCancellationRequested(); + + // A chat without its thread cannot be added, so it is not listed either: + if (!chatEntries.Any(entry => entry.Name.Equals($"{chatEntryPrefix}{ChatArchiveFormat.THREAD_FILE_NAME}", StringComparison.OrdinalIgnoreCase))) + continue; + + var segments = chatEntryPrefix.TrimEnd(ZipEntryNames.SEPARATOR).Split(ZipEntryNames.SEPARATOR); + if (!Guid.TryParse(segments[^1], out var chatId)) + continue; + + var nameEntry = chatEntries.FirstOrDefault(entry => entry.Name.Equals($"{chatEntryPrefix}{ChatArchiveFormat.NAME_FILE_NAME}", StringComparison.OrdinalIgnoreCase)); + var chat = new ChatArchiveManifestChat + { + ChatId = chatId, + Name = nameEntry is null ? string.Empty : await ReadTextAsync(nameEntry.Entry, token), + }; + + if (chatEntryPrefix.StartsWith(ChatArchiveFormat.TEMPORARY_CHATS_PREFIX, StringComparison.Ordinal)) + { + temporaryChats.Add(chat); + continue; + } + + if (!Guid.TryParse(segments[^2], out var workspaceId)) + continue; + + if (!workspaces.TryGetValue(workspaceId, out var workspace)) + { + var workspaceNameEntry = archive.GetEntry($"{ChatArchiveFormat.WORKSPACES_PREFIX}{workspaceId}{ZipEntryNames.SEPARATOR}{ChatArchiveFormat.NAME_FILE_NAME}"); + workspace = new ChatArchiveManifestWorkspace + { + WorkspaceId = workspaceId, + Name = workspaceNameEntry is null ? string.Empty : await ReadTextAsync(workspaceNameEntry, token), + }; + + workspaces[workspaceId] = workspace; + } + + workspace.Chats.Add(chat); + } + + return new() + { + Workspaces = workspaces.Values.OrderBy(workspace => workspace.Name, StringComparer.CurrentCultureIgnoreCase).ToList(), + TemporaryChats = temporaryChats, + }; + } + + private static async Task ReadTextAsync(ZipArchiveEntry entry, CancellationToken token) + { + using var reader = new StreamReader(entry.Open()); + return (await reader.ReadToEndAsync(token)).Trim(); + } + + /// + /// Determines the workspace which receives the chats of a workspace of the backup. Known + /// workspaces are reused, unknown ones are created with their original identity. + /// + /// The workspace of the backup. + /// The workspaces which exist already. Newly created ones are added. + private static async Task ResolveTargetWorkspaceAsync(ChatArchiveManifestWorkspace workspace, HashSet knownWorkspaceIds) + { + if (workspace.WorkspaceId == Guid.Empty) + return Guid.Empty; + + if (knownWorkspaceIds.Contains(workspace.WorkspaceId)) + return workspace.WorkspaceId; + + var workspaceName = string.IsNullOrWhiteSpace(workspace.Name) ? TB("Imported workspace") : workspace.Name; + await WorkspaceBehaviour.EnsureWorkspace(workspace.WorkspaceId, await CreateUniqueWorkspaceNameAsync(workspaceName)); + knownWorkspaceIds.Add(workspace.WorkspaceId); + + return workspace.WorkspaceId; + } + + /// + /// Extends the workspace name by a counter until it is no longer taken. Workspace names + /// must be unique, otherwise the workspace cannot be renamed later on. + /// + private static async Task CreateUniqueWorkspaceNameAsync(string workspaceName) + { + var candidate = workspaceName; + var counter = 1; + + while (await WorkspaceBehaviour.IsWorkspaceNameExistingAsync(candidate)) + { + counter++; + candidate = $"{workspaceName} ({counter})"; + } + + return candidate; + } + + /// + /// Imports one chat: extracts its files into a fresh chat directory, points the paths of + /// those files to it, and stores the chat. + /// + /// The backup entries of the chat. + /// The path of the chat within the backup, which every entry of the chat starts with. + /// The workspace to import the chat into, or Guid.Empty for a temporary chat. + /// The chat as the preview lists it. + /// What to do when a chat with the same ID exists already. + /// The IDs of all chats in the storage. An imported chat is added. + /// Cancels the import. + private static async Task ImportChatAsync(IReadOnlyList chatEntries, string chatEntryPrefix, Guid targetWorkspaceId, ChatArchiveManifestChat chat, ChatArchiveCollisionBehavior collisionBehavior, HashSet existingChatIds, CancellationToken token) + { + var threadEntry = chatEntries.FirstOrDefault(entry => entry.Name.Equals($"{chatEntryPrefix}{ChatArchiveFormat.THREAD_FILE_NAME}", StringComparison.OrdinalIgnoreCase)); + if (threadEntry is null) + { + LOG.LogWarning("The backup contains no chat thread for the chat '{ChatId}'.", chat.ChatId); + return ChatArchiveImportOutcome.FAILED; + } + + // + // The directory is checked as well: a chat the storage could not list, for example + // because its thread is unreadable, still occupies its place on disk. + // + var targetChatId = chat.ChatId; + if (existingChatIds.Contains(targetChatId) || WorkspaceBehaviour.IsChatExisting(new(targetWorkspaceId, targetChatId))) + { + if (collisionBehavior is ChatArchiveCollisionBehavior.SKIP) + return ChatArchiveImportOutcome.SKIPPED; + + targetChatId = Guid.NewGuid(); + } + + var chatDirectory = WorkspaceBehaviour.GetChatDirectory(targetWorkspaceId, targetChatId); + try + { + Directory.CreateDirectory(chatDirectory); + + // + // Extract everything except the chat thread and the name file. Both are written + // by the chat storage itself once the paths inside the thread are corrected: + // + var extractedPaths = new HashSet(PathTools.COMPARER); + foreach (var entry in chatEntries) + { + token.ThrowIfCancellationRequested(); + + var relativePath = entry.Name[chatEntryPrefix.Length..]; + if (relativePath.Equals(ChatArchiveFormat.THREAD_FILE_NAME, StringComparison.OrdinalIgnoreCase) || + relativePath.Equals(ChatArchiveFormat.NAME_FILE_NAME, StringComparison.OrdinalIgnoreCase)) + continue; + + var destinationPath = ZipEntryNames.GetDestinationPath(chatDirectory, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!); + + await using var entryStream = entry.Entry.Open(); + await using var fileStream = new FileStream(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan); + await entryStream.CopyToAsync(fileStream, token); + + extractedPaths.Add(relativePath); + } + + ChatThread? thread; + await using (var threadStream = threadEntry.Entry.Open()) + thread = await JsonSerializer.DeserializeAsync(threadStream, WorkspaceBehaviour.JSON_OPTIONS, token); + + if (thread is null) + { + LOG.LogWarning("The chat thread of the chat '{ChatId}' could not be read.", chat.ChatId); + TryDeleteDirectory(chatDirectory); + return ChatArchiveImportOutcome.FAILED; + } + + // + // A file below the data directory is stored with a path relative to it, which reading + // the thread resolved against the data directory of this computer: where the chat lay + // when the backup was made. Only the paths of files which were actually extracted move + // to the imported chat. Everything else keeps its path: + // + var originalChatDirectory = WorkspaceBehaviour.GetChatDirectory(thread.WorkspaceId, thread.ChatId); + thread = thread with { ChatId = targetChatId }; + thread.WorkspaceId = targetWorkspaceId; + thread.RewriteFilePaths(path => + { + var chatRelativePath = Path.GetRelativePath(originalChatDirectory, path).Replace(Path.DirectorySeparatorChar, ZipEntryNames.SEPARATOR); + return extractedPaths.Contains(chatRelativePath) + ? Path.Combine(chatDirectory, chatRelativePath.Replace(ZipEntryNames.SEPARATOR, Path.DirectorySeparatorChar)) + : path; + }); + + CopyTranscriptsStoredElsewhere(thread, chatDirectory); + + // + // The storage skips a chat whose lock another operation holds for too long. The + // directory prepared above would then hold the extracted files but no chat: + // + if (!await WorkspaceBehaviour.StoreChatAsync(thread)) + { + LOG.LogWarning("The chat '{ChatId}' could not be stored, because another operation held it for too long.", chat.ChatId); + TryDeleteDirectory(chatDirectory); + return ChatArchiveImportOutcome.FAILED; + } + + existingChatIds.Add(targetChatId); + return ChatArchiveImportOutcome.IMPORTED; + } + catch (OperationCanceledException) + { + TryDeleteDirectory(chatDirectory); + throw; + } + catch (Exception exception) + { + LOG.LogError(exception, "Failed to import the chat '{ChatId}' into the workspace '{WorkspaceId}'.", chat.ChatId, targetWorkspaceId); + TryDeleteDirectory(chatDirectory); + return ChatArchiveImportOutcome.FAILED; + } + } + + /// + /// Gives the imported chat its own copy of every transcript the backup did not carry. + /// + /// + /// Such a transcript still names the place where it lay when the backup was made. On the + /// same computer that is the chat the backup came from, and since a chat deletes the + /// transcripts it names, deleting the imported chat would take the transcripts of the + /// original along. A copied chat gets its own copies for the same reason, so this uses the + /// same code. A transcript which no longer exists keeps its place in the chat and shows up + /// as missing, again as it does in a copy. + /// + /// The imported chat thread, after the paths of its extracted files were pointed to it. + /// The directory of the imported chat. + private static void CopyTranscriptsStoredElsewhere(ChatThread thread, string chatDirectory) + { + var transcriptDirectory = ManagedTranscriptAttachment.GetTranscriptDirectory(chatDirectory); + var copiedPaths = new Dictionary(PathTools.COMPARER); + + // + // Only a transcript in the data directory can belong to another chat, and only there + // does deleting a chat delete its transcripts. Any other path is left alone, so that a + // backup cannot get files from elsewhere on this computer copied into a chat: + // + thread.ReplaceAttachments(attachment => attachment is ManagedTranscriptAttachment transcript + && !PathTools.IsInsideDirectory(chatDirectory, transcript.FilePath) + && PathTools.IsInsideDirectory(SettingsManager.DataDirectory!, transcript.FilePath) + ? WorkspaceBehaviour.CopyManagedTranscriptAttachment(thread, transcript, transcriptDirectory, copiedPaths) + : attachment); + } + + private static void TryDeleteDirectory(string directory) + { + if (!Directory.Exists(directory)) + return; + + try + { + Directory.Delete(directory, true); + } + catch (Exception exception) + { + LOG.LogWarning(exception, "Failed to clean up the incomplete chat directory '{ChatDirectory}'.", directory); + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveManifest.cs b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveManifest.cs new file mode 100644 index 000000000..74417ae17 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveManifest.cs @@ -0,0 +1,32 @@ +namespace AIStudio.Tools.ChatArchive; + +/// +/// The chats a backup holds, so that the user sees them before any is added. +/// +public sealed record ChatArchiveManifest +{ + /// + /// When the backup was made. + /// + public DateTimeOffset CreatedAt { get; init; } + + /// + /// The AI Studio version which made the backup. Informational only. + /// + public string AppVersion { get; init; } = string.Empty; + + /// + /// All workspaces with chats in the backup. + /// + public List Workspaces { get; init; } = []; + + /// + /// All temporary chats in the backup. + /// + public List TemporaryChats { get; init; } = []; + + /// + /// Gets the total number of chats in the backup. + /// + public int TotalChatCount => this.Workspaces.Sum(workspace => workspace.Chats.Count) + this.TemporaryChats.Count; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveManifestChat.cs b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveManifestChat.cs new file mode 100644 index 000000000..fd2c63007 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveManifestChat.cs @@ -0,0 +1,17 @@ +namespace AIStudio.Tools.ChatArchive; + +/// +/// One chat in a backup. +/// +public sealed record ChatArchiveManifestChat +{ + /// + /// The unique identifier of the chat. + /// + public Guid ChatId { get; init; } + + /// + /// The name of the chat when the backup was made. + /// + public string Name { get; init; } = string.Empty; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveManifestWorkspace.cs b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveManifestWorkspace.cs new file mode 100644 index 000000000..0063a3741 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveManifestWorkspace.cs @@ -0,0 +1,22 @@ +namespace AIStudio.Tools.ChatArchive; + +/// +/// One workspace with chats in a backup. +/// +public sealed record ChatArchiveManifestWorkspace +{ + /// + /// The unique identifier of the workspace. + /// + public Guid WorkspaceId { get; init; } + + /// + /// The name of the workspace when the backup was made. + /// + public string Name { get; init; } = string.Empty; + + /// + /// All chats of this workspace in the backup. + /// + public List Chats { get; init; } = []; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveProgress.cs b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveProgress.cs new file mode 100644 index 000000000..0843779d2 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveProgress.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Tools.ChatArchive; + +/// +/// Reports the progress of an export or import operation. +/// +/// The number of chats processed so far. +/// The total number of chats to process. +public readonly record struct ChatArchiveProgress(int ProcessedChats, int TotalChats); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupContents.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupContents.cs new file mode 100644 index 000000000..bb687796b --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupContents.cs @@ -0,0 +1,119 @@ +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tools.DataBackup; + +/// +/// What AI Studio keeps in its data and configuration folders, and which of it belongs into a backup. +/// +/// +/// This is a positive list. A backup takes along only the entries marked as included, and restoring +/// one replaces only those. Everything else in these folders stays out of a backup and survives a +/// restore: the leftovers of a plugin installation, a file somebody else put there, or a folder a +/// future version adds before anybody listed it here.

+/// +/// The entries which stay out are listed, too. The folder panel on the information page tells users +/// who back up these folders with a tool of their own what they may leave out, and nobody has to guess +/// whether a folder was forgotten or left out on purpose. Whoever adds a file or folder to the data or +/// configuration folder adds it here, one way or the other.

+/// +/// The runtime, which applies a restore, keeps its own list of the included entries and refuses to +/// replace anything else (REPLACEABLE_ELEMENTS in runtime/src/data_restore.rs). An entry included here +/// goes there as well. +/// +public static class DataBackupContents +{ + /// + /// The separator of the relative paths, the same on every OS. + /// + public const char SEPARATOR = '/'; + + /// + /// The folder of the plugins, within the data folder. + /// + public const string PLUGINS_FOLDER = "plugins/"; + + /// + /// Every file and folder AI Studio keeps directly in its data or configuration folder, and those + /// below the folders of the plugins and the databases which need a decision of their own. + /// + public static readonly IReadOnlyList ENTRIES = + [ + new(DataBackupLocation.CONFIG_DIRECTORY, "settings.json", DataBackupEntryKind.SETTINGS), + new(DataBackupLocation.DATA_DIRECTORY, "workspaces/", DataBackupEntryKind.WORKSPACES), + new(DataBackupLocation.DATA_DIRECTORY, "tempChats/", DataBackupEntryKind.TEMPORARY_CHATS), + new(DataBackupLocation.DATA_DIRECTORY, PLUGINS_FOLDER, DataBackupEntryKind.PLUGINS), + new(DataBackupLocation.DATA_DIRECTORY, "tokenizers/", DataBackupEntryKind.TOKENIZERS), + new(DataBackupLocation.DATA_DIRECTORY, "visualBriefings/", DataBackupEntryKind.VISUAL_BRIEFINGS), + new(DataBackupLocation.DATA_DIRECTORY, "databases/sqlite/", DataBackupEntryKind.DATA_SOURCE_INDEX), + new(DataBackupLocation.DATA_DIRECTORY, "databases/vector_database/stores/", DataBackupEntryKind.VECTOR_STORES), + + new(DataBackupLocation.CONFIG_DIRECTORY, "settings.v*.json", DataBackupEntryKind.SETTINGS_OF_OTHER_VERSIONS), + new(DataBackupLocation.DATA_DIRECTORY, "plugins/.internal/", DataBackupEntryKind.BUILT_IN_PLUGINS), + new(DataBackupLocation.DATA_DIRECTORY, "plugins/.config/", DataBackupEntryKind.ORGANIZATION_PLUGINS), + new(DataBackupLocation.DATA_DIRECTORY, "plugins/.config-tests/", DataBackupEntryKind.ORGANIZATION_PLUGINS), + new(DataBackupLocation.DATA_DIRECTORY, "plugins/.lock", DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.DATA_DIRECTORY, ".plugin-staging/", DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.DATA_DIRECTORY, ".plugin-install-backups/", DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.DATA_DIRECTORY, ".plugin-delete-backups/", DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.DATA_DIRECTORY, "media-staging/", DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.DATA_DIRECTORY, "mailAttachments/", DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.DATA_DIRECTORY, "mindwork_ai_studio.pid", DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.DATA_DIRECTORY, DataBackupFormat.STAGING_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.DATA_DIRECTORY, DataRestoreFormat.PENDING_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.CONFIG_DIRECTORY, DataRestoreFormat.PENDING_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.DATA_DIRECTORY, DataRestoreFormat.PREVIOUS_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.CONFIG_DIRECTORY, DataRestoreFormat.PREVIOUS_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.DATA_DIRECTORY, DataRestoreFormat.RESULT_FILE_NAME, DataBackupEntryKind.TEMPORARY_FILES), + new(DataBackupLocation.DATA_DIRECTORY, "pandoc/", DataBackupEntryKind.DOWNLOADED_TOOLS), + new(DataBackupLocation.DATA_DIRECTORY, "data-protection-keys/", DataBackupEntryKind.INTERFACE_KEYS), + new(DataBackupLocation.DATA_DIRECTORY, "events.log", DataBackupEntryKind.LOGS), + + // The runtime hides the log of the startup on Linux and macOS: + new(DataBackupLocation.DATA_DIRECTORY, OperatingSystem.IsWindows() ? "AI Studio Events.log" : ".AI Studio Events.log", DataBackupEntryKind.LOGS), + ]; + + /// + /// Whether a path lies within an entry: below it for a folder, or the entry itself for a file. + /// + /// The path of the entry, as the list holds it. + /// The path to check, relative to the same folder and with forward slashes. + /// True when the entry covers the path. + public static bool Covers(string entryPath, string relativePath) => entryPath.EndsWith(SEPARATOR) + ? relativePath.StartsWith(entryPath, PathTools.COMPARISON) + : relativePath.Equals(entryPath, PathTools.COMPARISON); + + /// + /// Finds the plugins which the organization deployed into the folder of the user's plugins. + /// + /// + /// The config server keeps the plugins of an organization in folders of their own, which the list + /// leaves out as a whole. A device management may put them anywhere, though, so their path proves + /// nothing: such a plugin declares itself as deployed by the organization instead. A backup leaves + /// it out, because the organization brings it back, and a user with a backup tool of their own + /// needs to know which folders these are. + /// + /// The plugins AI Studio found. + /// The data folder. + /// The folders of these plugins within the data folder, in the form of the list, ordered. + public static IReadOnlyList FindOrganizationPluginsAmongUserPlugins(IEnumerable plugins, string dataDirectory) + { + var excludedPluginFolders = ENTRIES + .Where(entry => !entry.IsIncluded && entry.IsFolder && entry.Location is DataBackupLocation.DATA_DIRECTORY && entry.RelativePath.StartsWith(PLUGINS_FOLDER, StringComparison.Ordinal)) + .Select(entry => entry.RelativePath) + .ToList(); + + // + // Relative to the data folder and with forward slashes, the plugin folders take the form of + // the list. That is the form in which AI Studio stores such paths, too: + // + return plugins + .Where(plugin => plugin.IsManagedByConfigServer) + .Select(plugin => DataDirectoryRelativePathConverter.ToStored(plugin.LocalPath, dataDirectory).TrimEnd(SEPARATOR) + SEPARATOR) + .Where(folder => folder.Length > PLUGINS_FOLDER.Length + && folder.StartsWith(PLUGINS_FOLDER, PathTools.COMPARISON) + && !excludedPluginFolders.Any(excluded => folder.StartsWith(excluded, PathTools.COMPARISON))) + .Distinct(PathTools.COMPARER) + .Order(PathTools.COMPARER) + .ToList(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupEntry.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupEntry.cs new file mode 100644 index 000000000..fd19b6600 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupEntry.cs @@ -0,0 +1,27 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// A file or folder AI Studio keeps in its data or configuration folder. +/// +/// The folder in which the entry lies. +/// The path of the entry within that folder, with forward slashes. A folder ends with a slash. +/// What the entry holds, and with it, whether it belongs into a backup. +public sealed record DataBackupEntry(DataBackupLocation Location, string RelativePath, DataBackupEntryKind Kind) +{ + /// + /// Whether the entry belongs into a backup. + /// + public bool IsIncluded => this.Kind is DataBackupEntryKind.SETTINGS + or DataBackupEntryKind.WORKSPACES + or DataBackupEntryKind.TEMPORARY_CHATS + or DataBackupEntryKind.PLUGINS + or DataBackupEntryKind.TOKENIZERS + or DataBackupEntryKind.VISUAL_BRIEFINGS + or DataBackupEntryKind.DATA_SOURCE_INDEX + or DataBackupEntryKind.VECTOR_STORES; + + /// + /// Whether the entry is a folder rather than a single file. + /// + public bool IsFolder => this.RelativePath.EndsWith(DataBackupContents.SEPARATOR); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupEntryKind.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupEntryKind.cs new file mode 100644 index 000000000..97a738d7c --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupEntryKind.cs @@ -0,0 +1,83 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// What an entry of the backup contents holds, and with it, whether it belongs into a backup. +/// +public enum DataBackupEntryKind +{ + /// + /// The settings. A backup leaves out what a configuration plugin of the organization manages. + /// + SETTINGS, + + /// + /// The chats of the workspaces, with their transcripts. + /// + WORKSPACES, + + /// + /// The temporary chats, with their transcripts. + /// + TEMPORARY_CHATS, + + /// + /// The plugins of the user. A backup leaves out the plugins of the organization. + /// + PLUGINS, + + /// + /// The copies of the tokenizers the user chose for a provider. + /// + TOKENIZERS, + + /// + /// The projects of the Visual Briefing Assistant. + /// + VISUAL_BRIEFINGS, + + /// + /// The SQLite index of the data sources: which files are embedded, and their text chunks. + /// + DATA_SOURCE_INDEX, + + /// + /// The vector stores, which hold the embeddings of the data sources. + /// + VECTOR_STORES, + + /// + /// The copies of the settings which every version of AI Studio keeps for itself. + /// + SETTINGS_OF_OTHER_VERSIONS, + + /// + /// The plugins which come with AI Studio, such as the language packs. + /// + BUILT_IN_PLUGINS, + + /// + /// The plugins which the organization deploys, or tries out for one session. + /// + ORGANIZATION_PLUGINS, + + /// + /// Files AI Studio needs only for a moment, while it works on something, and removes again. + /// + TEMPORARY_FILES, + + /// + /// Tools AI Studio downloaded, which depend on the OS, such as Pandoc. + /// + DOWNLOADED_TOOLS, + + /// + /// The keys with which the user interface protects its data. Only the Flatpak keeps them in the data + /// folder, since the home folder is read-only there, and AI Studio creates new ones when they are missing. + /// + INTERFACE_KEYS, + + /// + /// The log files. + /// + LOGS, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupFormat.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupFormat.cs new file mode 100644 index 000000000..60db85142 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupFormat.cs @@ -0,0 +1,86 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace AIStudio.Tools.DataBackup; + +/// +/// The format of a backup: a ZIP file which holds the data folder below DATA_ROOT and the +/// configuration folder below CONFIG_ROOT, each in the layout AI Studio keeps on the disk, and the +/// manifest as its last entry. +/// +/// +/// What goes in follows DataBackupContents. The paths below the roots are the relative paths of +/// that list, so restoring a backup is unpacking it. The manifest comes last: an archive which lacks +/// it was not written completely. +/// +public static class DataBackupFormat +{ + /// + /// The version of the format this version of AI Studio writes. A restore refuses a backup of a + /// later version, since it cannot know what such a backup holds. + /// + public const int FORMAT_VERSION = 1; + + /// + /// The file extension of a backup, named like the one of plugin archives. + /// + public const string FILE_EXTENSION = ".mwbackup"; + + /// + /// The manifest, the last entry of every backup. + /// + public const string MANIFEST_ENTRY_NAME = "manifest.json"; + + /// + /// Where the data folder lies within a backup. + /// + public const string DATA_ROOT = "data/"; + + /// + /// Where the configuration folder lies within a backup. + /// + public const string CONFIG_ROOT = "config/"; + + /// + /// The directory in the data folder where the copies of the vector stores wait while a backup runs. + /// The runtime creates it; keep the name in sync with BACKUP_STAGING_DIRECTORY there. + /// + public const string STAGING_DIRECTORY = ".backup-staging/"; + + /// + /// How the manifest and the plan of a restore are written: readable for anybody who opens them, + /// and for the runtime, which applies the plan. + /// + public static readonly JsonSerializerOptions JSON_OPTIONS = new() + { + WriteIndented = true, + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + Converters = { new JsonStringEnumConverter() }, + }; + + /// + /// File types which are compressed already. Compressing them once more costs time and makes the + /// backup no smaller. + /// + public static readonly IReadOnlySet ALREADY_COMPRESSED_EXTENSIONS = new HashSet(StringComparer.OrdinalIgnoreCase) + { + ".jpg", ".jpeg", ".png", ".gif", ".webp", ".heic", ".tiff", + ".mp3", ".m4a", ".m4b", ".aac", ".ogg", ".opus", ".flac", ".wma", + ".mp4", ".m4v", ".mov", ".mkv", ".webm", ".avi", ".wmv", ".flv", + ".zip", ".7z", ".gz", ".bz2", ".xz", ".rar", + ".pdf", ".docx", ".xlsx", ".pptx", ".odt", ".odp", + ".mwplugin", FILE_EXTENSION, + }; + + /// + /// The name of the entry which holds a file of the data or configuration folder. + /// + /// The folder of the file. + /// The path of the file within its folder, with forward slashes. + /// The name of the entry. + public static string EntryName(DataBackupLocation location, string relativePath) => location switch + { + DataBackupLocation.CONFIG_DIRECTORY => CONFIG_ROOT + relativePath, + _ => DATA_ROOT + relativePath, + }; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupLocation.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupLocation.cs new file mode 100644 index 000000000..0e55d014a --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupLocation.cs @@ -0,0 +1,17 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// The folder in which an entry of the backup contents lies. +/// +public enum DataBackupLocation +{ + /// + /// The data folder, where AI Studio keeps the chats, the plugins, and the indexes of the data sources. + /// + DATA_DIRECTORY, + + /// + /// The configuration folder, where AI Studio keeps its settings. + /// + CONFIG_DIRECTORY, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupManifest.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupManifest.cs new file mode 100644 index 000000000..0314f1e91 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupManifest.cs @@ -0,0 +1,22 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// What a backup holds, and what wrote it. +/// +/// The version of the format, cf. DataBackupFormat.FORMAT_VERSION. +/// The version of AI Studio which wrote the backup. +/// The version of the settings in the backup. +/// When the backup was written. +/// The data folder the backup was made from, for whoever wonders where it came from. +/// The configuration folder the backup was made from. +/// Whether the backup holds the indexes of the data sources. Without them, a restore keeps the indexes there are. +/// Every file of the backup, the manifest aside. +public sealed record DataBackupManifest( + int FormatVersion, + string AppVersion, + string SettingsVersion, + DateTimeOffset CreatedAtUtc, + string SourceDataDirectory, + string SourceConfigDirectory, + bool IncludesDataSourceIndexes, + IReadOnlyList Entries); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupManifestEntry.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupManifestEntry.cs new file mode 100644 index 000000000..c595ede73 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupManifestEntry.cs @@ -0,0 +1,9 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// One file of a backup, as the manifest lists it. +/// +/// The name of the entry within the backup. +/// The size of the file in bytes. +/// The SHA-256 hash of the file, in lowercase hex digits. +public sealed record DataBackupManifestEntry(string Name, long Size, string Sha256); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupManifestReadResult.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupManifestReadResult.cs new file mode 100644 index 000000000..a84b71c6a --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupManifestReadResult.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// The manifest of a backup, or why it could not be read. +/// +/// The manifest, or null when the backup cannot be used. +/// Why the backup cannot be used, for the user. Empty when it can. +public sealed record DataBackupManifestReadResult(DataBackupManifest? Manifest, string Issue); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupPhase.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupPhase.cs new file mode 100644 index 000000000..4ec1ea7c7 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupPhase.cs @@ -0,0 +1,22 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// What a backup is busy with. +/// +public enum DataBackupPhase +{ + /// + /// It collects the settings and pauses the indexing. + /// + PREPARING, + + /// + /// It copies the indexes of the data sources, one vector store after the other. + /// + COPYING_INDEXES, + + /// + /// It writes the files into the backup. + /// + WRITING, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupProgress.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupProgress.cs new file mode 100644 index 000000000..0c6255fe5 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupProgress.cs @@ -0,0 +1,9 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// How far a backup got. +/// +/// What the backup is busy with. +/// How many steps of the phase are done: vector stores while the indexes are copied, files while they are written. +/// How many steps the phase has, or zero while that is not known yet. +public readonly record struct DataBackupProgress(DataBackupPhase Phase, int Processed, int Total); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupReader.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupReader.cs new file mode 100644 index 000000000..a6b08dd71 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupReader.cs @@ -0,0 +1,50 @@ +using System.IO.Compression; +using System.Text.Json; + +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tools.DataBackup; + +/// +/// Reads what a backup holds, cf. DataBackupFormat. +/// +/// +/// Open a backup with ZipArchiveMode.Read and read its entries as streams. ZipArchiveMode.Update would +/// buffer an entry in memory, which ends at 2 GB, and a backup may hold larger files. +/// +public static class DataBackupReader +{ + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataBackupReader).Namespace, nameof(DataBackupReader)); + + /// + /// Reads the manifest of a backup, and checks that this version of AI Studio can use the backup. + /// + /// The backup, opened for reading. + /// The cancellation token. + /// The manifest, or why the backup cannot be used. + public static async Task ReadManifestAsync(ZipArchive archive, CancellationToken token) + { + var manifestEntry = archive.GetEntry(DataBackupFormat.MANIFEST_ENTRY_NAME); + if (manifestEntry is null) + return new(null, TB("This file is no backup of AI Studio, or it was not written completely.")); + + DataBackupManifest? manifest; + try + { + await using var manifestStream = manifestEntry.Open(); + manifest = await JsonSerializer.DeserializeAsync(manifestStream, DataBackupFormat.JSON_OPTIONS, token); + } + catch (JsonException) + { + manifest = null; + } + + if (manifest is null) + return new(null, TB("The content list of this backup could not be read.")); + + if (manifest.FormatVersion > DataBackupFormat.FORMAT_VERSION) + return new(null, TB("This backup was made by a newer version of AI Studio. Please update AI Studio first.")); + + return new(manifest, string.Empty); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupResult.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupResult.cs new file mode 100644 index 000000000..54596733f --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupResult.cs @@ -0,0 +1,13 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// How a backup ended. +/// +/// Whether the backup was written completely. A cancelled backup counts as no success. +/// Whether the user stopped the backup. +/// The file of the backup. +/// How many files the backup holds. +/// How large these files are together, before they were compressed. +/// Whether the backup holds the indexes of the data sources. +/// Why the backup failed, for the user. Empty when it did not fail. +public sealed record DataBackupResult(bool Success, bool Cancelled, string ArchivePath, int FileCount, long Bytes, bool IncludesDataSourceIndexes, string Issue); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupSettings.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupSettings.cs new file mode 100644 index 000000000..4b1517c75 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupSettings.cs @@ -0,0 +1,45 @@ +using System.Text.Json; + +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tools.DataBackup; + +/// +/// The settings as a backup takes them along. +/// +public static class DataBackupSettings +{ + /// + /// Copies the settings, leaving out every object which a configuration plugin manages. + /// + /// + /// A configuration plugin brings its providers, data sources, chat templates, profiles, and + /// policies back on another computer, along with their API keys. Restored from a backup as well, + /// they would compete with what the plugin brings, and without the plugin they would stay without + /// their keys: secrets never go into a backup. That holds for the configuration plugins of an + /// organization and for those the user installed, since a backup takes the latter along.

+ /// + /// A setting which a configuration plugin locks stays in the copy, together with the plugin which + /// locks it. After a restore, the clean-up following the start of the plugins treats it like the + /// setting of a removed plugin, until the plugin locks it again. Without the plugin, the user gets + /// back the value they had chosen before.

+ /// + /// The copy is a round trip through the format in which the settings are stored, so it shares + /// nothing with the settings AI Studio runs with. Like storing the settings, it reads them while + /// the rest of the app may still change them. + /// + /// The settings AI Studio runs with. They stay as they are. + /// The copy for the backup. + public static Data CopyWithoutPluginObjects(Data settings) + { + var copy = JsonSerializer.Deserialize(JsonSerializer.Serialize(settings, SettingsManager.JSON_OPTIONS), SettingsManager.JSON_OPTIONS) + ?? throw new InvalidOperationException("The settings could not be copied."); + + foreach (var descriptor in PluginConfigurationObjectDescriptors.ALL) + descriptor.RemovePluginObjects(copy); + + return copy; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupSources.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupSources.cs new file mode 100644 index 000000000..59f85a89d --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupSources.cs @@ -0,0 +1,11 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// Where a backup takes its files from. +/// +/// The data folder. +/// The configuration folder. +/// The settings as the backup takes them along, cf. DataBackupSettings. They come from memory rather than from the disk, since the backup leaves out the objects of configuration plugins. +/// The copies of the indexes, in the layout of the data folder, or an empty string when the backup leaves the indexes out. The indexes in the data folder change while AI Studio runs, so the backup never reads them there. +/// The plugins an organization deployed among the user's own ones, cf. DataBackupContents.FindOrganizationPluginsAmongUserPlugins. +public sealed record DataBackupSources(string DataDirectory, string ConfigDirectory, string SettingsJson, string IndexCopiesDirectory, IReadOnlyList OrganizationPluginFolders); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataBackupWriter.cs b/app/MindWork AI Studio/Tools/DataBackup/DataBackupWriter.cs new file mode 100644 index 000000000..abd5bbf62 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataBackupWriter.cs @@ -0,0 +1,173 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +namespace AIStudio.Tools.DataBackup; + +/// +/// Writes a backup, cf. DataBackupFormat. +/// +/// +/// Every file is streamed into the backup, so a backup of many gigabytes needs no more memory than +/// a small one. The caller writes the backup straight into the file the user chose: in the Flatpak, +/// the portal grants access to exactly that file, and to nothing next to it. +/// +public static class DataBackupWriter +{ + private const int BUFFER_SIZE = 64 * 1024; + + /// + /// The earliest time a ZIP file can store. + /// + private static readonly DateTimeOffset EARLIEST_ZIP_TIME = new(1980, 1, 1, 0, 0, 0, TimeSpan.Zero); + + /// + /// A file to take along, and where it goes in the backup. + /// + private sealed record BackupFile(string EntryName, string SourcePath); + + /// + /// Writes a backup into the given stream. + /// + /// + /// A file which is gone by the time it is written, e.g. a temporary chat AI Studio deleted + /// meanwhile, is left out. A file which cannot be read fails the backup, though: a backup which + /// quietly lacks something is worse than none. + /// + /// Where to write the backup. It stays open. + /// Where the files come from. + /// What to note in the manifest. The entries are filled in here. + /// Receives how many files are written. + /// Cancels the backup. + /// The manifest as the backup holds it. + public static async Task WriteAsync(Stream target, DataBackupSources sources, DataBackupManifest manifest, IProgress? progress, CancellationToken token) + { + var files = CollectFiles(sources, token); + var totalFiles = files.Count + 1; + var entries = new List(totalFiles); + var buffer = new byte[BUFFER_SIZE]; + + progress?.Report(new(DataBackupPhase.WRITING, 0, totalFiles)); + using (var archive = new ZipArchive(target, ZipArchiveMode.Create, leaveOpen: true)) + { + var settingsEntry = DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.SETTINGS); + await using (var settingsStream = new MemoryStream(Encoding.UTF8.GetBytes(sources.SettingsJson), writable: false)) + entries.Add(await WriteEntryAsync(archive, DataBackupFormat.EntryName(settingsEntry.Location, settingsEntry.RelativePath), settingsStream, DateTimeOffset.Now, buffer, token)); + + progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles)); + foreach (var file in files) + { + token.ThrowIfCancellationRequested(); + + FileStream source; + try + { + source = new FileStream(file.SourcePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan); + } + catch (Exception exception) when (exception is FileNotFoundException or DirectoryNotFoundException) + { + continue; + } + + await using (source) + entries.Add(await WriteEntryAsync(archive, file.EntryName, source, File.GetLastWriteTime(file.SourcePath), buffer, token)); + + progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles)); + } + + manifest = manifest with { Entries = entries }; + await using (var manifestStream = new MemoryStream(JsonSerializer.SerializeToUtf8Bytes(manifest, DataBackupFormat.JSON_OPTIONS), writable: false)) + await WriteEntryAsync(archive, DataBackupFormat.MANIFEST_ENTRY_NAME, manifestStream, DateTimeOffset.Now, buffer, token); + } + + return manifest; + } + + /// + /// Collects the files of the data folder which belong into a backup, cf. DataBackupContents. + /// + /// + /// The settings are no part of it: they come from memory. The indexes come from their copies. + /// + private static List CollectFiles(DataBackupSources sources, CancellationToken token) + { + var excludedPaths = DataBackupContents.ENTRIES + .Where(entry => !entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY) + .Select(entry => entry.RelativePath) + .Concat(sources.OrganizationPluginFolders) + .ToList(); + + var files = new List(); + foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY)) + { + var root = entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX or DataBackupEntryKind.VECTOR_STORES + ? sources.IndexCopiesDirectory + : sources.DataDirectory; + + if (string.IsNullOrWhiteSpace(root)) + continue; + + var path = Path.Join(root, entry.RelativePath.Replace(DataBackupContents.SEPARATOR, Path.DirectorySeparatorChar)); + if (!entry.IsFolder) + { + if (File.Exists(path)) + files.Add(new(DataBackupFormat.EntryName(entry.Location, entry.RelativePath), path)); + + continue; + } + + if (!Directory.Exists(path)) + continue; + + // + // Hidden files count as well, since on Linux and macOS a dot makes a file hidden. Links + // are not followed, so nothing outside the folder gets into the backup. A folder which + // cannot be read fails the backup instead of leaving a gap: + // + var options = new EnumerationOptions { RecurseSubdirectories = true, AttributesToSkip = FileAttributes.ReparsePoint, IgnoreInaccessible = false }; + foreach (var filePath in Directory.EnumerateFiles(path, "*", options)) + { + token.ThrowIfCancellationRequested(); + + var relativePath = Path.GetRelativePath(root, filePath).Replace(Path.DirectorySeparatorChar, DataBackupContents.SEPARATOR); + if (excludedPaths.Any(excludedPath => DataBackupContents.Covers(excludedPath, relativePath))) + continue; + + files.Add(new(DataBackupFormat.EntryName(DataBackupLocation.DATA_DIRECTORY, relativePath), filePath)); + } + } + + return files; + } + + /// + /// Writes one entry, and notes its size and hash while it streams through. + /// + private static async Task WriteEntryAsync(ZipArchive archive, string entryName, Stream source, DateTimeOffset lastWriteTime, byte[] buffer, CancellationToken token) + { + var compressionLevel = DataBackupFormat.ALREADY_COMPRESSED_EXTENSIONS.Contains(Path.GetExtension(entryName)) + ? CompressionLevel.NoCompression + : CompressionLevel.Optimal; + + var entry = archive.CreateEntry(entryName, compressionLevel); + + // The time of the last change tells, e.g., the order of the chats in a workspace: + entry.LastWriteTime = lastWriteTime < EARLIEST_ZIP_TIME ? DateTimeOffset.Now : lastWriteTime; + + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + long size = 0; + await using (var entryStream = entry.Open()) + { + int read; + while ((read = await source.ReadAsync(buffer, token)) > 0) + { + hash.AppendData(buffer, 0, read); + await entryStream.WriteAsync(buffer.AsMemory(0, read), token); + size += read; + } + } + + return new(entryName, size, Convert.ToHexStringLower(hash.GetHashAndReset())); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreElement.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreElement.cs new file mode 100644 index 000000000..d9a177312 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreElement.cs @@ -0,0 +1,9 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// One element which a restore replaces, as the list of the backup contents names it. +/// +/// The folder in which the element lies. +/// The path of the element within that folder, with forward slashes. A folder ends with a slash. +/// The paths below the element which stay as they are, relative to the same folder: the plugins which come with AI Studio or from the organization, for example. The runtime moves them from the replaced element into the new one. +public sealed record DataRestoreElement(DataBackupLocation Location, string RelativePath, IReadOnlyList Keep); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreFollowUp.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreFollowUp.cs new file mode 100644 index 000000000..8423f1df1 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreFollowUp.cs @@ -0,0 +1,106 @@ +using System.Text.Json; + +namespace AIStudio.Tools.DataBackup; + +/// +/// What remains to do after the runtime applied a restore, or after a backup or the preparation of a +/// restore stopped halfway: read the report of the runtime, and remove what nobody needs any longer. +/// +/// +/// Nothing here touches a restore which the runtime has not finished, cf. IsRestoreUnfinished: its +/// journal and the data it replaced are the only way back, and the runtime continues at every start. +/// +public static class DataRestoreFollowUp +{ + /// + /// Whether a restore is prepared completely and waits for the next start. + /// + /// The data folder. + /// True when the next start applies a restore. + public static bool IsRestorePending(string dataDirectory) => IsMarkedReady(dataDirectory) && !HasJournal(dataDirectory); + + /// + /// Whether the runtime has not finished a restore: it failed to move a failed restore back, and + /// tries again at the next start. + /// + /// The data folder. + /// True while the runtime still needs the pending and the previous directories. + public static bool IsRestoreUnfinished(string dataDirectory) => IsMarkedReady(dataDirectory) && HasJournal(dataDirectory); + + /// + /// Reads how the runtime applied a restore at this start. Throws a JsonException when the report + /// cannot be read. + /// + /// The data folder. + /// Cancels reading. + /// The report, or null when the runtime applied no restore. + public static async Task ReadReportAsync(string dataDirectory, CancellationToken token) + { + var reportPath = Path.Join(dataDirectory, DataRestoreFormat.RESULT_FILE_NAME); + if (!File.Exists(reportPath)) + return null; + + await using var reportStream = File.OpenRead(reportPath); + return await JsonSerializer.DeserializeAsync(reportStream, DataBackupFormat.JSON_OPTIONS, token); + } + + /// + /// Removes the report of the runtime, once the user saw it. + /// + /// The data folder. + public static void RemoveReport(string dataDirectory) => File.Delete(Path.Join(dataDirectory, DataRestoreFormat.RESULT_FILE_NAME)); + + /// + /// Removes what a backup or the preparation of a restore left behind when it stopped halfway, e.g., + /// because AI Studio crashed. Call it only while neither runs. + /// + /// + /// A pending directory with READY_FILE_NAME stays: the restore waits for the next start, or the + /// runtime has not finished it. + /// + /// The data folder. + /// The configuration folder. + public static void RemoveLeftovers(string dataDirectory, string configDirectory) + { + DeleteDirectory(DataRestoreFormat.DirectoryIn(dataDirectory, DataBackupFormat.STAGING_DIRECTORY)); + if (IsMarkedReady(dataDirectory)) + return; + + DeleteDirectory(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PENDING_DIRECTORY)); + DeleteDirectory(DataRestoreFormat.DirectoryIn(configDirectory, DataRestoreFormat.PENDING_DIRECTORY)); + } + + /// + /// Removes the data which a restore replaced. Call it once the restored settings loaded: until + /// then, these data are the way back. + /// + /// The data folder. + /// The configuration folder. + /// Whether there were such data. + public static bool RemoveReplacedData(string dataDirectory, string configDirectory) + { + if (IsMarkedReady(dataDirectory)) + return false; + + var removedData = DeleteDirectory(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PREVIOUS_DIRECTORY)); + var removedConfiguration = DeleteDirectory(DataRestoreFormat.DirectoryIn(configDirectory, DataRestoreFormat.PREVIOUS_DIRECTORY)); + return removedData || removedConfiguration; + } + + private static bool IsMarkedReady(string dataDirectory) => File.Exists(Path.Join(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PENDING_DIRECTORY), DataRestoreFormat.READY_FILE_NAME)); + + private static bool HasJournal(string dataDirectory) => File.Exists(Path.Join(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PENDING_DIRECTORY), DataRestoreFormat.JOURNAL_FILE_NAME)); + + /// + /// Deletes a directory with everything in it. + /// + /// Whether the directory existed. + private static bool DeleteDirectory(string path) + { + if (!Directory.Exists(path)) + return false; + + Directory.Delete(path, true); + return true; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreFormat.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreFormat.cs new file mode 100644 index 000000000..b4fec28d9 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreFormat.cs @@ -0,0 +1,76 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// How a prepared restore waits for the next start, which applies it. +/// +/// +/// This is the contract between the app, which prepares a restore, and the runtime, which applies it +/// before the app starts again, when no file is in use: +/// +/// Below the data folder and below the configuration folder, PENDING_DIRECTORY holds what the +/// backup brings for that folder, in the layout of that folder. Staging lies on the same disk as its +/// target, so the runtime can move every element into place instead of copying it. +/// The plan, PLAN_FILE_NAME in the pending directory of the data folder, names the elements to +/// replace, cf. DataRestorePlan. +/// READY_FILE_NAME in the same directory comes last. Without it, the restore was not prepared +/// completely, and the runtime leaves everything as it is. +/// +/// What the runtime adds, cf. runtime/src/data_restore.rs: +/// +/// PREVIOUS_DIRECTORY, below both folders, keeps what the restore replaced. The app removes it +/// once it loaded the restored settings. +/// JOURNAL_FILE_NAME in the pending directory of the data folder lists every move. While it lies +/// next to READY_FILE_NAME, the runtime moves everything back at the next start, so the app never +/// removes a pending directory which holds READY_FILE_NAME. +/// RESULT_FILE_NAME in the data folder tells the app how the restore ended, cf. DataRestoreReport. +/// +/// Keep in sync with the runtime, which applies the restore. +/// +public static class DataRestoreFormat +{ + /// + /// The version of the plan this version of AI Studio writes. + /// + public const int PLAN_FORMAT_VERSION = 1; + + /// + /// The directory, below the data folder and below the configuration folder, which holds what a + /// prepared restore brings for that folder. + /// + public const string PENDING_DIRECTORY = ".restore-pending/"; + + /// + /// The plan of the restore, in the pending directory of the data folder. + /// + public const string PLAN_FILE_NAME = "restore.json"; + + /// + /// Marks a restore as prepared completely, in the pending directory of the data folder. + /// + public const string READY_FILE_NAME = "READY"; + + /// + /// The directory, below the data folder and below the configuration folder, which keeps what the + /// runtime replaced, in the layout of that folder. + /// + public const string PREVIOUS_DIRECTORY = ".restore-previous/"; + + /// + /// How the runtime applied a restore, in the data folder. + /// + public const string RESULT_FILE_NAME = ".restore-result.json"; + + /// + /// The journal of the runtime, in the pending directory of the data folder. Next to READY_FILE_NAME, + /// it tells that the runtime has not finished a restore: it moves it back at the next start. + /// + public const string JOURNAL_FILE_NAME = "journal.json"; + + /// + /// The path of one of the directories above, below the given folder. + /// + /// The data or the configuration folder. + /// PENDING_DIRECTORY or PREVIOUS_DIRECTORY. + /// The path of the directory. + public static string DirectoryIn(string folder, string directory) => Path.Join(folder, directory.TrimEnd(DataBackupContents.SEPARATOR)); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOpenPoint.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOpenPoint.cs new file mode 100644 index 000000000..bb3d4400f --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOpenPoint.cs @@ -0,0 +1,9 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// One thing the user still has to do after a restore. +/// +/// What to do. +/// What it concerns: the name of a provider, a data source, or a tool, or a path. +/// For a tool: the label of the setting which lacks its value; otherwise empty. +public sealed record DataRestoreOpenPoint(DataRestoreOpenPointKind Kind, string Name, string Setting = ""); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOpenPointKind.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOpenPointKind.cs new file mode 100644 index 000000000..d05e934a4 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOpenPointKind.cs @@ -0,0 +1,59 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// What the user still has to do after a restore, since a backup holds no secrets and another computer +/// may lack the files the settings name. +/// +public enum DataRestoreOpenPointKind +{ + /// + /// A provider of the user has no API key on this computer. + /// + PROVIDER_WITHOUT_API_KEY, + + /// + /// An embedding provider of the user has no API key on this computer. + /// + EMBEDDING_PROVIDER_WITHOUT_API_KEY, + + /// + /// A transcription provider of the user has no API key on this computer. + /// + TRANSCRIPTION_PROVIDER_WITHOUT_API_KEY, + + /// + /// An ERI data source of the user has no password or token on this computer. + /// + DATA_SOURCE_WITHOUT_SECRET, + + /// + /// A mailbox of the user has no password on this computer. + /// + MAILBOX_WITHOUT_PASSWORD, + + /// + /// A tool which the user set up has no value for a secret setting on this computer, e.g., for + /// the API key of a search service. + /// + TOOL_WITHOUT_SECRET, + + /// + /// A local data source names a file or folder which does not exist on this computer. + /// + DATA_SOURCE_NOT_FOUND, + + /// + /// The batch processing names a file or folder which does not exist on this computer. + /// + BATCH_PROCESSING_PATH_NOT_FOUND, + + /// + /// The file with custom root certificates does not exist on this computer. + /// + CERTIFICATE_BUNDLE_NOT_FOUND, + + /// + /// The ERI server assistant writes a server into a folder which does not exist on this computer. + /// + ERI_SERVER_DIRECTORY_NOT_FOUND, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOpenPoints.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOpenPoints.cs new file mode 100644 index 000000000..e86f29d10 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOpenPoints.cs @@ -0,0 +1,87 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.ERIClient.DataModel; +using AIStudio.Tools.ToolCallingSystem; + +namespace AIStudio.Tools.DataBackup; + +/// +/// Finds what the user still has to do after a restore. +/// +/// +/// A backup holds no secrets, so the providers, data sources, mailboxes, and tools of the user lack +/// them on another computer. Those of the organization do not: their configuration plugin brings them +/// along with their secrets. The files and folders which the settings name may be missing, too, wherever +/// they come from. +/// +public static class DataRestoreOpenPoints +{ + /// + /// Finds what the user still has to do after a restore. + /// + /// The restored settings. + /// Tells which secrets this computer holds. + /// The secret settings of the tools which the user set up. Only the tools know which of their settings are secret, cf. DataRestoreService. + /// The open points, in the order of the settings. + public static async Task> CollectAsync(Data settings, IStoredSecrets secrets, IReadOnlyList toolSecrets) + { + var openPoints = new List(); + foreach (var provider in settings.Providers.Where(provider => !provider.IsEnterpriseConfiguration && provider.UsedLLMProvider.IsAPIKeyNeeded(provider.Host))) + if (!await secrets.HasAPIKeyAsync(provider, SecretStoreType.LLM_PROVIDER)) + openPoints.Add(new(DataRestoreOpenPointKind.PROVIDER_WITHOUT_API_KEY, provider.InstanceName)); + + foreach (var provider in settings.EmbeddingProviders.Where(provider => !provider.IsEnterpriseConfiguration && provider.UsedLLMProvider.IsAPIKeyNeeded(provider.Host))) + if (!await secrets.HasAPIKeyAsync(provider, SecretStoreType.EMBEDDING_PROVIDER)) + openPoints.Add(new(DataRestoreOpenPointKind.EMBEDDING_PROVIDER_WITHOUT_API_KEY, provider.Name)); + + foreach (var provider in settings.TranscriptionProviders.Where(provider => !provider.IsEnterpriseConfiguration && provider.UsedLLMProvider.IsAPIKeyNeeded(provider.Host))) + if (!await secrets.HasAPIKeyAsync(provider, SecretStoreType.TRANSCRIPTION_PROVIDER)) + openPoints.Add(new(DataRestoreOpenPointKind.TRANSCRIPTION_PROVIDER_WITHOUT_API_KEY, provider.Name)); + + foreach (var dataSource in settings.DataSources) + { + switch (dataSource) + { + case DataSourceLocalDirectory directory when !Directory.Exists(directory.Path): + openPoints.Add(new(DataRestoreOpenPointKind.DATA_SOURCE_NOT_FOUND, directory.Name)); + break; + + case DataSourceLocalFile file when !File.Exists(file.FilePath): + openPoints.Add(new(DataRestoreOpenPointKind.DATA_SOURCE_NOT_FOUND, file.Name)); + break; + + case IERIDataSource { IsEnterpriseConfiguration: false, AuthMethod: AuthMethod.TOKEN or AuthMethod.USERNAME_PASSWORD } eriDataSource: + if (!await secrets.HasSecretAsync(eriDataSource, SecretStoreType.DATA_SOURCE)) + openPoints.Add(new(DataRestoreOpenPointKind.DATA_SOURCE_WITHOUT_SECRET, eriDataSource.Name)); + + break; + } + } + + foreach (var mailbox in settings.Mailboxes.Where(mailbox => mailbox is { IsEnterpriseConfiguration: false, AuthMethod: MailboxAuthMethod.PASSWORD })) + if (!await secrets.HasSecretAsync(mailbox, SecretStoreType.DATA_SOURCE)) + openPoints.Add(new(DataRestoreOpenPointKind.MAILBOX_WITHOUT_PASSWORD, mailbox.Name)); + + foreach (var toolSecret in toolSecrets) + if (!await secrets.HasSecretAsync(new ToolSettingsSecretId(toolSecret.ToolId, toolSecret.FieldName), SecretStoreType.TOOL_SETTINGS)) + openPoints.Add(new(DataRestoreOpenPointKind.TOOL_WITHOUT_SECRET, toolSecret.ToolName, toolSecret.SettingLabel)); + + var batchProcessing = settings.BatchProcessing; + foreach (var folder in new[] { batchProcessing.InputDirectory, batchProcessing.OutputDirectory }) + if (!string.IsNullOrWhiteSpace(folder) && !Directory.Exists(folder)) + openPoints.Add(new(DataRestoreOpenPointKind.BATCH_PROCESSING_PATH_NOT_FOUND, folder)); + + if (!string.IsNullOrWhiteSpace(batchProcessing.PromptFilePath) && !File.Exists(batchProcessing.PromptFilePath)) + openPoints.Add(new(DataRestoreOpenPointKind.BATCH_PROCESSING_PATH_NOT_FOUND, batchProcessing.PromptFilePath)); + + var certificateBundlePath = settings.App.ExternalHttpCustomRootCertificateBundlePath; + if (settings.App.ExternalHttpCustomRootCertificatesEnabled && !string.IsNullOrWhiteSpace(certificateBundlePath) && !File.Exists(certificateBundlePath)) + openPoints.Add(new(DataRestoreOpenPointKind.CERTIFICATE_BUNDLE_NOT_FOUND, certificateBundlePath)); + + foreach (var server in settings.ERI.ERIServers.Where(server => server.WriteToFilesystem && !string.IsNullOrWhiteSpace(server.BaseDirectory) && !Directory.Exists(server.BaseDirectory))) + openPoints.Add(new(DataRestoreOpenPointKind.ERI_SERVER_DIRECTORY_NOT_FOUND, server.ServerName)); + + return openPoints; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOutcome.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOutcome.cs new file mode 100644 index 000000000..408382fcb --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreOutcome.cs @@ -0,0 +1,29 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// How the runtime applied a restore, cf. DataRestoreReport. +/// +public enum DataRestoreOutcome +{ + /// + /// The restored data are in place. + /// + APPLIED, + + /// + /// The runtime did not start the restore, since the plan or the folders did not allow it. All data + /// stayed as they were. + /// + REFUSED, + + /// + /// The restore failed, and the runtime moved all data back. + /// + ROLLED_BACK, + + /// + /// The restore failed, and moving the data back failed as well. The runtime tries again at every + /// start. + /// + ROLLBACK_FAILED, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestorePlan.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestorePlan.cs new file mode 100644 index 000000000..96eb66781 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestorePlan.cs @@ -0,0 +1,11 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// What the runtime replaces at the next start, cf. DataRestoreFormat. +/// +/// The version of the plan, cf. DataRestoreFormat.PLAN_FORMAT_VERSION. +/// When the restore was prepared. +/// When the backup was made. +/// The version of AI Studio which made the backup. +/// The elements to replace. An element the backup holds nothing for stays as it is. +public sealed record DataRestorePlan(int FormatVersion, DateTimeOffset PreparedAtUtc, DateTimeOffset BackupCreatedAtUtc, string BackupAppVersion, IReadOnlyList Elements); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestorePreparer.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestorePreparer.cs new file mode 100644 index 000000000..375a7531e --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestorePreparer.cs @@ -0,0 +1,239 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text.Json; + +using AIStudio.Tools.PluginSystem; + +using SettingsVersion = AIStudio.Settings.Version; + +namespace AIStudio.Tools.DataBackup; + +/// +/// Prepares a restore: unpacks a backup next to the data and configuration folders, so that the next +/// start can put it in place, cf. DataRestoreFormat. +/// +/// +/// Nothing in use changes here. While AI Studio runs, its files are open, and on Windows, they cannot be +/// replaced at all. The runtime replaces them at the next start, before the app opens any of them, and +/// rolls back when that fails. +/// +public static class DataRestorePreparer +{ + private const int BUFFER_SIZE = 64 * 1024; + + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataRestorePreparer).Namespace, nameof(DataRestorePreparer)); + + /// + /// A file of the backup to unpack, and where it goes. + /// + private sealed record PendingFile(ZipArchiveEntry Entry, DataBackupManifestEntry Listed, string DestinationPath); + + /// + /// Unpacks a backup for the next start. + /// + /// + /// Every file has to match the size and the hash the manifest lists for it: a restore replaces all + /// data, so a damaged backup must not get that far. A file the manifest does not list, or one + /// which belongs to no element the restore replaces, stays in the backup. A restore prepared + /// before is discarded first; one which fails or is cancelled leaves nothing behind. + /// + /// The backup. + /// Where the restore goes. + /// Receives how many files are unpacked. + /// Cancels the preparation. + /// Whether the restore is prepared, or why not. + public static async Task PrepareAsync(string archivePath, DataRestoreTarget target, IProgress? progress, CancellationToken token) + { + Discard(target); + try + { + using var archive = ZipFile.OpenRead(archivePath); + var manifestResult = await DataBackupReader.ReadManifestAsync(archive, token); + if (manifestResult.Manifest is not { } manifest) + return new(false, false, manifestResult.Issue); + + var versionIssue = CheckVersions(manifest, target); + if (!string.IsNullOrWhiteSpace(versionIssue)) + return new(false, false, versionIssue); + + var elements = DetermineElements(manifest, target); + var files = DetermineFiles(archive, manifest, elements, target); + progress?.Report(new(0, files.Count)); + + var buffer = new byte[BUFFER_SIZE]; + for (var index = 0; index < files.Count; index++) + { + token.ThrowIfCancellationRequested(); + if (!await UnpackAsync(files[index], buffer, token)) + { + Discard(target); + return new(false, false, string.Format(TB("The backup is damaged: the file '{0}' does not match its content list."), files[index].Listed.Name)); + } + + progress?.Report(new(index + 1, files.Count)); + } + + var pendingDirectory = PendingDirectory(target.DataDirectory); + Directory.CreateDirectory(pendingDirectory); + + var plan = new DataRestorePlan(DataRestoreFormat.PLAN_FORMAT_VERSION, DateTimeOffset.UtcNow, manifest.CreatedAtUtc, manifest.AppVersion, elements); + await File.WriteAllBytesAsync(Path.Join(pendingDirectory, DataRestoreFormat.PLAN_FILE_NAME), JsonSerializer.SerializeToUtf8Bytes(plan, DataBackupFormat.JSON_OPTIONS), token); + + // Last of all: from now on, the next start applies the restore. + await File.WriteAllTextAsync(Path.Join(pendingDirectory, DataRestoreFormat.READY_FILE_NAME), plan.PreparedAtUtc.ToString("O"), token); + return new(true, false, string.Empty); + } + catch (InvalidDataException) + { + Discard(target); + return new(false, false, TB("This file is no backup of AI Studio, or it was not written completely.")); + } + catch + { + Discard(target); + throw; + } + } + + /// + /// Removes a prepared restore, complete or not, so that the next start leaves everything as it is. + /// + /// + /// A restore which the runtime has not finished stays: its journal is the only way back to the data + /// it replaced, cf. DataRestoreFollowUp.IsRestoreUnfinished. + /// + /// Where the restore was to go. + public static void Discard(DataRestoreTarget target) + { + if (DataRestoreFollowUp.IsRestoreUnfinished(target.DataDirectory)) + throw new InvalidOperationException("The runtime has not finished the last restore yet, so it cannot be discarded."); + + // First of all: should removing the rest fail halfway, the next start must not apply what is left. + var readyPath = Path.Join(PendingDirectory(target.DataDirectory), DataRestoreFormat.READY_FILE_NAME); + if (File.Exists(readyPath)) + File.Delete(readyPath); + + foreach (var pendingDirectory in new[] { PendingDirectory(target.DataDirectory), PendingDirectory(target.ConfigDirectory) }) + if (Directory.Exists(pendingDirectory)) + Directory.Delete(pendingDirectory, true); + } + + /// + /// Whether a restore is prepared completely and waits for the next start. + /// + /// Where the restore goes. + /// True when the next start applies a restore. + public static bool IsPrepared(DataRestoreTarget target) => DataRestoreFollowUp.IsRestorePending(target.DataDirectory); + + private static string PendingDirectory(string directory) => DataRestoreFormat.DirectoryIn(directory, DataRestoreFormat.PENDING_DIRECTORY); + + /// + /// Refuses a backup of a later version of AI Studio. Its settings or its index may have a form + /// this version cannot read, and it would refuse to write them, or break them. + /// + /// Why the backup cannot be restored, or an empty string. + private static string CheckVersions(DataBackupManifest manifest, DataRestoreTarget target) + { + var settingsAreNewer = !Enum.TryParse(manifest.SettingsVersion, out var settingsVersion) || settingsVersion > target.SettingsVersion; + var appIsNewer = Version.TryParse(manifest.AppVersion, out var backupAppVersion) && Version.TryParse(target.AppVersion, out var appVersion) && backupAppVersion > appVersion; + return settingsAreNewer || appIsNewer + ? TB("This backup was made by a newer version of AI Studio. Please update AI Studio first.") + : string.Empty; + } + + /// + /// Determines the elements the restore replaces: those of the list of the backup contents which the + /// backup holds anything for. + /// + /// + /// An element the backup holds nothing for stays as it is, since nothing tells whether it was empty + /// or whether the version of AI Studio which made the backup knew it at all. The indexes are + /// replaced only when the backup holds both of them, cf. DataBackupManifest.IncludesDataSourceIndexes. + /// + private static List DetermineElements(DataBackupManifest manifest, DataRestoreTarget target) + { + var elements = new List(); + foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded)) + { + if (entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX or DataBackupEntryKind.VECTOR_STORES && !manifest.IncludesDataSourceIndexes) + continue; + + var entryName = DataBackupFormat.EntryName(entry.Location, entry.RelativePath); + if (!manifest.Entries.Any(listed => DataBackupContents.Covers(entryName, listed.Name))) + continue; + + var keep = DataBackupContents.ENTRIES + .Where(excluded => !excluded.IsIncluded && excluded.Location == entry.Location && entry.IsFolder && DataBackupContents.Covers(entry.RelativePath, excluded.RelativePath)) + .Select(excluded => excluded.RelativePath) + .Concat(entry.Kind is DataBackupEntryKind.PLUGINS ? target.OrganizationPluginFolders : []) + .ToList(); + + elements.Add(new(entry.Location, entry.RelativePath, keep)); + } + + return elements; + } + + /// + /// Determines the files to unpack: those the manifest lists, which belong to an element the restore + /// replaces, and which the element does not keep. + /// + private static List DetermineFiles(ZipArchive archive, DataBackupManifest manifest, IReadOnlyList elements, DataRestoreTarget target) + { + var listedEntries = manifest.Entries.ToDictionary(listed => listed.Name, StringComparer.Ordinal); + var files = new List(); + foreach (var entry in archive.Entries) + { + var entryName = ZipEntryNames.Normalize(entry.FullName); + if (!listedEntries.TryGetValue(entryName, out var listed)) + continue; + + foreach (var element in elements) + { + var root = DataBackupFormat.EntryName(element.Location, string.Empty); + if (!entryName.StartsWith(root, StringComparison.Ordinal)) + continue; + + var relativePath = entryName[root.Length..]; + if (!DataBackupContents.Covers(element.RelativePath, relativePath) || element.Keep.Any(kept => DataBackupContents.Covers(kept, relativePath))) + continue; + + var directory = element.Location is DataBackupLocation.CONFIG_DIRECTORY ? target.ConfigDirectory : target.DataDirectory; + files.Add(new(entry, listed, ZipEntryNames.GetDestinationPath(PendingDirectory(directory), relativePath))); + break; + } + } + + return files; + } + + /// + /// Unpacks one file, and checks its size and hash while it streams through. + /// + /// False when the file does not match the manifest. + private static async Task UnpackAsync(PendingFile file, byte[] buffer, CancellationToken token) + { + Directory.CreateDirectory(Path.GetDirectoryName(file.DestinationPath)!); + + using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + long size = 0; + await using (var source = file.Entry.Open()) + await using (var destination = new FileStream(file.DestinationPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan)) + { + int read; + while ((read = await source.ReadAsync(buffer, token)) > 0) + { + hash.AppendData(buffer, 0, read); + await destination.WriteAsync(buffer.AsMemory(0, read), token); + size += read; + } + } + + if (size != file.Listed.Size || Convert.ToHexStringLower(hash.GetHashAndReset()) != file.Listed.Sha256) + return false; + + // The time of the last change tells, e.g., the order of the chats in a workspace: + File.SetLastWriteTime(file.DestinationPath, file.Entry.LastWriteTime.LocalDateTime); + return true; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreProgress.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreProgress.cs new file mode 100644 index 000000000..a5313084e --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreProgress.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// How far the preparation of a restore got. +/// +/// How many files of the backup are unpacked. +/// How many files there are to unpack, or zero while that is not known yet. +public readonly record struct DataRestoreProgress(int Processed, int Total); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreReport.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreReport.cs new file mode 100644 index 000000000..6f1aa7a1e --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreReport.cs @@ -0,0 +1,15 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// How the runtime applied a restore at this start, as it writes it to DataRestoreFormat.RESULT_FILE_NAME. +/// +/// +/// The points in time stay text: the runtime leaves them empty when it could not read the plan. +/// +/// How the restore ended. +/// What happened, in English, for the log and for the support. +/// When the restore was prepared, or an empty string. +/// When the backup was made, or an empty string. +/// The version of AI Studio which made the backup, or an empty string. +/// When the runtime finished. +public sealed record DataRestoreReport(DataRestoreOutcome Outcome, string Detail, string PreparedAtUtc, string BackupCreatedAtUtc, string BackupAppVersion, string FinishedAtUtc); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreResult.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreResult.cs new file mode 100644 index 000000000..95a4430d2 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreResult.cs @@ -0,0 +1,9 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// How the preparation of a restore ended. +/// +/// Whether the restore is prepared and waits for the next start. +/// Whether the user stopped the preparation. +/// Why the restore cannot be prepared, for the user. Empty when it is prepared. +public sealed record DataRestoreResult(bool Success, bool Cancelled, string Issue); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreTarget.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreTarget.cs new file mode 100644 index 000000000..2cc0245b3 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreTarget.cs @@ -0,0 +1,13 @@ +using SettingsVersion = AIStudio.Settings.Version; + +namespace AIStudio.Tools.DataBackup; + +/// +/// Where a restore goes, and what it has to fit to. +/// +/// The data folder. +/// The configuration folder. +/// The version of this AI Studio. A backup of a later version is refused. +/// The version of the settings this AI Studio writes. A backup with later settings is refused. +/// The plugins an organization deployed among the user's own ones, cf. DataBackupContents.FindOrganizationPluginsAmongUserPlugins. They stay as they are. +public sealed record DataRestoreTarget(string DataDirectory, string ConfigDirectory, string AppVersion, SettingsVersion SettingsVersion, IReadOnlyList OrganizationPluginFolders); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/DataRestoreToolSecret.cs b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreToolSecret.cs new file mode 100644 index 000000000..c647b9ff4 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/DataRestoreToolSecret.cs @@ -0,0 +1,10 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// A secret setting of a tool which the user set up, and which may lack its value after a restore. +/// +/// The tool. +/// The setting, as the schema of the tool names it. +/// The name of the tool, as the user reads it. +/// The label of the setting, as the user reads it. +public sealed record DataRestoreToolSecret(string ToolId, string FieldName, string ToolName, string SettingLabel); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/IStoredSecrets.cs b/app/MindWork AI Studio/Tools/DataBackup/IStoredSecrets.cs new file mode 100644 index 000000000..926821bce --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/IStoredSecrets.cs @@ -0,0 +1,23 @@ +namespace AIStudio.Tools.DataBackup; + +/// +/// Tells whether the OS keyring of this computer holds a secret. +/// +public interface IStoredSecrets +{ + /// + /// Whether the API key of a provider is stored. + /// + /// The provider. + /// The kind of provider. + /// True when the key is stored. + public Task HasAPIKeyAsync(ISecretId secretId, SecretStoreType storeType); + + /// + /// Whether another secret is stored, e.g., the password of a data source. + /// + /// What the secret belongs to. + /// The kind of secret. + /// True when the secret is stored. + public Task HasSecretAsync(ISecretId secretId, SecretStoreType storeType); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataBackup/RustStoredSecrets.cs b/app/MindWork AI Studio/Tools/DataBackup/RustStoredSecrets.cs new file mode 100644 index 000000000..44fbbfa2e --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataBackup/RustStoredSecrets.cs @@ -0,0 +1,16 @@ +using AIStudio.Tools.Services; + +namespace AIStudio.Tools.DataBackup; + +/// +/// Asks the runtime whether the OS keyring holds a secret. It only tries, so a missing secret logs no error. +/// +/// The runtime. +public sealed class RustStoredSecrets(RustService rustService) : IStoredSecrets +{ + /// + public async Task HasAPIKeyAsync(ISecretId secretId, SecretStoreType storeType) => (await rustService.GetAPIKey(secretId, storeType, isTrying: true)).Success; + + /// + public async Task HasSecretAsync(ISecretId secretId, SecretStoreType storeType) => (await rustService.GetSecret(secretId, storeType, isTrying: true)).Success; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataDirectoryRelativePathConverter.cs b/app/MindWork AI Studio/Tools/DataDirectoryRelativePathConverter.cs new file mode 100644 index 000000000..af04359fa --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataDirectoryRelativePathConverter.cs @@ -0,0 +1,69 @@ +using System.Text.Json; +using System.Text.Json.Serialization; + +using AIStudio.Settings; + +namespace AIStudio.Tools; + +/// +/// Stores a path below the data directory relative to it, and every other path as it is. +/// +/// +/// AI Studio keeps files of its own below the data directory: the transcripts of a chat, the copies of +/// the tokenizers, and more. A stored path to such a file outlives the computer it was written on. A +/// backup brings it to another computer, where the user name, the OS, or the data directory itself may +/// differ. Relative to the data directory, the path still names the same file there. In memory, the +/// path stays absolute, so the code which works with it never notices.

+/// +/// A relative path is written with forward slashes, which every OS reads. A path to a file of the user +/// stays absolute: it lies outside of the data directory, and a backup does not bring that file along. +/// A path written before AI Studio stored such paths relative is absolute as well, and is read as it +/// is. While the data directory is not known yet, every path stays as it is. +/// +public sealed class DataDirectoryRelativePathConverter : JsonConverter +{ + private const char STORED_SEPARATOR = '/'; + + #region Overrides of JsonConverter + + /// + public override string Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => ToAbsolute(reader.GetString() ?? string.Empty, SettingsManager.DataDirectory); + + /// + public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) => writer.WriteStringValue(ToStored(value, SettingsManager.DataDirectory)); + + #endregion + + /// + /// Turns a path into the form in which it is stored. + /// + /// The absolute path, as AI Studio works with it. + /// The data directory, or null while it is not known yet. + /// The path relative to the data directory when it lies below it, otherwise the path as it is. + internal static string ToStored(string path, string? dataDirectory) + { + if (string.IsNullOrWhiteSpace(path) || string.IsNullOrWhiteSpace(dataDirectory) || !Path.IsPathFullyQualified(path) || !PathTools.IsInsideDirectory(dataDirectory, path)) + return path; + + return Path.GetRelativePath(Path.GetFullPath(dataDirectory), Path.GetFullPath(path)).Replace(Path.DirectorySeparatorChar, STORED_SEPARATOR); + } + + /// + /// Turns a stored path back into the absolute path AI Studio works with. + /// + /// + /// A relative path which leads out of the data directory was not written by AI Studio. Rather than + /// resolving it against whatever directory the process runs in, it is read as no path at all. + /// + /// The path as it was stored. + /// The data directory, or null while it is not known yet. + /// The absolute path. + internal static string ToAbsolute(string storedPath, string? dataDirectory) + { + if (string.IsNullOrWhiteSpace(storedPath) || string.IsNullOrWhiteSpace(dataDirectory) || Path.IsPathRooted(storedPath)) + return storedPath; + + var path = Path.Join(dataDirectory, storedPath.Replace(STORED_SEPARATOR, Path.DirectorySeparatorChar)); + return PathTools.IsInsideDirectory(dataDirectory, path) ? path : string.Empty; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/DataSourceRepair.cs b/app/MindWork AI Studio/Tools/DataSourceRepair.cs index 14fb3611f..b2b07fb7c 100644 --- a/app/MindWork AI Studio/Tools/DataSourceRepair.cs +++ b/app/MindWork AI Studio/Tools/DataSourceRepair.cs @@ -26,6 +26,13 @@ public static class DataSourceRepair /// True when the rebuild was started. public static async Task ConfirmAndRepairAsync(IDialogService dialogService, DataSourceEmbeddingService embeddingService, string dataSourceId, string dataSourceName) { + // A repair discards the index, which waits for the end of a pause: + if (embeddingService.IsIndexingPaused) + { + await dialogService.ShowMessageBox(TB("Repair Data Source"), TB("AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done."), TB("Close")); + return false; + } + var dialogParameters = new DialogParameters { { x => x.Message, string.Format(TB("The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?"), dataSourceName) }, diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs index 87020a6fd..a3cede7e2 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs @@ -53,4 +53,16 @@ public abstract partial class IndexStoreClient(string name, string path) : Datab /// The cancellation token. /// The number of chunks, or null when the index cannot tell. Null and zero mean different things here. public abstract Task GetTotalChunkCountAsync(CancellationToken token); + + /// + /// Writes a consistent copy of the index into the given directory, e.g., for a backup. + /// + /// + /// Pause the indexing first, cf. DataSourceEmbeddingService.PauseIndexingAsync. The copy is + /// consistent on its own, but the vector stores copied for the same backup have to fit to it. + /// + /// Where to write the copy. A copy written there before is replaced. + /// The cancellation token. + /// The path of the copy. + public abstract Task BackupToDirectoryAsync(string targetDirectory, CancellationToken token); } diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs index b44628d8b..63daab4e2 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs @@ -67,6 +67,15 @@ public sealed partial class NoIndexStoreClient(string name, string? unavailableR public override Task GetTotalChunkCountAsync(CancellationToken token) => Task.FromResult(null); + /// + /// + /// Unlike the other methods, this one does not pretend success. A backup without the index would + /// still take the vector stores along, and after a restore, they would belong to nothing the index + /// knows. Whoever makes the backup decides what to do instead. + /// + public override Task BackupToDirectoryAsync(string targetDirectory, CancellationToken token) => + Task.FromException(new InvalidOperationException(unavailableReason ?? "The index of the data sources is not available.")); + public override void Dispose() { } diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs index 1ec2aab73..1f9e6348a 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs @@ -7,6 +7,7 @@ using AIStudio.Tools.Mail; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Services; +using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; namespace AIStudio.Tools.Databases.IndexStore; @@ -580,6 +581,68 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st return versions.FirstOrDefault() ?? string.Empty; } + /// + /// + /// VACUUM INTO writes a consistent snapshot of the database, including what the write-ahead log + /// still holds, and in WAL mode, reading it holds back no writer.

+ /// + /// It runs on connections of their own, although the databases guide has raw SQL go through + /// GetDbConnection. The connections of the context share one cache, and a shared cache locks + /// whole tables: reading the whole database for the copy would hold back every writer until the + /// copy is done. Without pooling, nothing keeps the copy open afterward, so the backup can + /// delete it on every OS. + /// + public override async Task BackupToDirectoryAsync(string targetDirectory, CancellationToken token) + { + Directory.CreateDirectory(targetDirectory); + var targetPath = Path.Join(targetDirectory, DATABASE_FILENAME); + + // VACUUM INTO refuses to write into a file which holds anything: + if (File.Exists(targetPath)) + File.Delete(targetPath); + + try + { + await using (var connection = new SqliteConnection(BuildBackupConnectionString(this.databasePath, SqliteOpenMode.ReadWrite))) + { + await connection.OpenAsync(token); + await using var command = connection.CreateCommand(); + command.CommandText = "VACUUM INTO $targetPath;"; + command.Parameters.AddWithValue("$targetPath", targetPath); + await command.ExecuteNonQueryAsync(token); + } + + await using (var connection = new SqliteConnection(BuildBackupConnectionString(targetPath, SqliteOpenMode.ReadOnly))) + { + await connection.OpenAsync(token); + await using var command = connection.CreateCommand(); + command.CommandText = "PRAGMA quick_check;"; + + var checkResult = (await command.ExecuteScalarAsync(token))?.ToString(); + if (checkResult is not "ok") + throw new InvalidOperationException($"The copy of the index at '{targetPath}' failed its check: '{checkResult}'."); + } + + return targetPath; + } + catch + { + if (File.Exists(targetPath)) + File.Delete(targetPath); + + throw; + } + } + + private static string BuildBackupConnectionString(string path, SqliteOpenMode mode) => new SqliteConnectionStringBuilder + { + DataSource = path, + Mode = mode, + Cache = SqliteCacheMode.Private, + Pooling = false, + DefaultTimeout = 30, + }.ToString(); + private IndexStoreDbContext CreateContext() => new(this.dbContextOptions); private static void ApplyDataSource( diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs index c2ad86847..b057ddc75 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/NoVectorStoreClient.cs @@ -25,6 +25,9 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason, public override Task EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) => Task.FromException(this.CreateUnavailableException()); + public override Task VectorStoreExists(string storeName, CancellationToken token) => + Task.FromException(this.CreateUnavailableException()); + public override Task InsertEmbedding(string storeName, IReadOnlyList points, CancellationToken token) => Task.FromException(this.CreateUnavailableException()); @@ -43,6 +46,12 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason, public override Task DeleteVectorStore(string storeName, CancellationToken token) => Task.FromException(this.CreateUnavailableException()); + public override Task PrepareBackupAsync(CancellationToken token) => + Task.FromException(this.CreateUnavailableException()); + + public override Task CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) => + Task.FromException(this.CreateUnavailableException()); + private InvalidOperationException CreateUnavailableException() => new(unavailableReason ?? "The vector store is not available."); diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs index ea3fb07fb..24663977c 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/QdrantEdgeClientImplementation.cs @@ -23,12 +23,15 @@ public sealed class QdrantEdgeClientImplementation( private const string DATABASE_NAME = "Qdrant Edge"; private const string INFO_PATH = "/system/qdrant-edge/info"; private const string ENSURE_PATH = "/system/qdrant-edge/ensure"; + private const string EXISTS_PATH = "/system/qdrant-edge/exists"; private const string INSERT_PATH = "/system/qdrant-edge/insert"; private const string SEARCH_PATH = "/system/qdrant-edge/search"; private const string DELETE_FILE_PATH = "/system/qdrant-edge/delete-file"; private const string OPTIMIZE_PATH = "/system/qdrant-edge/optimize"; private const string DELETE_STORE_PATH = "/system/qdrant-edge/delete-store"; - + private const string PREPARE_BACKUP_PATH = "/system/qdrant-edge/backup/prepare"; + private const string COPY_STORE_FOR_BACKUP_PATH = "/system/qdrant-edge/backup/copy"; + private readonly string path = path; private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(QdrantEdgeClientImplementation).Namespace, nameof(QdrantEdgeClientImplementation)); @@ -122,6 +125,10 @@ public sealed class QdrantEdgeClientImplementation( await rustService.ExecuteDatabaseQuery(DATABASE_NAME, ENSURE_PATH, new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token) ?? throw new InvalidOperationException("The vector store ensure response was empty."); + public override async Task VectorStoreExists(string storeName, CancellationToken token) => + await rustService.ExecuteDatabaseQuery(DATABASE_NAME, EXISTS_PATH, + new CheckVectorStoreRequest(storeName), token) ?? throw new InvalidOperationException("The vector store exists response was empty."); + public override Task InsertEmbedding(string storeName, IReadOnlyList points, CancellationToken token) => rustService.ExecuteDatabaseOperation(DATABASE_NAME, INSERT_PATH, new InsertEmbeddingRequest(storeName, points), token); @@ -158,6 +165,14 @@ public sealed class QdrantEdgeClientImplementation( public override Task DeleteVectorStore(string storeName, CancellationToken token) => rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_STORE_PATH, new DeleteVectorStoreRequest(storeName), token); + public override async Task PrepareBackupAsync(CancellationToken token) => + await rustService.ExecuteDatabaseQuery(DATABASE_NAME, PREPARE_BACKUP_PATH, + new PrepareBackupRequest(), token) ?? throw new InvalidOperationException("The response preparing the backup of the vector stores was empty."); + + public override async Task CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) => + await rustService.ExecuteDatabaseQuery(DATABASE_NAME, COPY_STORE_FOR_BACKUP_PATH, + new CopyStoreForBackupRequest(backupId, storeName), token) ?? throw new InvalidOperationException("The response copying a vector store for the backup was empty."); + public override void Dispose() { } @@ -172,6 +187,8 @@ public sealed class QdrantEdgeClientImplementation( // ReSharper disable NotAccessedPositionalProperty.Local private sealed record EnsureVectorStoreRequest(string StoreName, string DataSourceName, int VectorSize); + private sealed record CheckVectorStoreRequest(string StoreName); + private sealed record InsertEmbeddingRequest(string StoreName, IReadOnlyList Points); /// The only points the search may return, or null for the whole store. @@ -182,5 +199,12 @@ public sealed class QdrantEdgeClientImplementation( private sealed record OptimizeVectorStoreRequest(string StoreName); private sealed record DeleteVectorStoreRequest(string StoreName); + + /// + /// Preparing a backup needs nothing but the request itself. + /// + private sealed record PrepareBackupRequest; + + private sealed record CopyStoreForBackupRequest(string BackupId, string StoreName); // ReSharper restore NotAccessedPositionalProperty.Local } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreBackupPreparation.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreBackupPreparation.cs new file mode 100644 index 000000000..ce6273659 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreBackupPreparation.cs @@ -0,0 +1,9 @@ +namespace AIStudio.Tools.Databases.VectorStore; + +/// +/// What a backup needs to copy the stores, one at a time. +/// +/// Names the backup when its stores are copied. +/// The directory of the backup's copies. Below it, every copy has the place it has below the data directory. The backup removes it once it is written. +/// Every store which was created completely. +public sealed record VectorStoreBackupPreparation(string BackupId, string StagingPath, IReadOnlyList Stores); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreBackupStore.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreBackupStore.cs new file mode 100644 index 000000000..c018e1617 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreBackupStore.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Tools.Databases.VectorStore; + +/// +/// A store which a backup takes along. +/// +/// The name of the store. +/// How large the store is on the disk. +public sealed record VectorStoreBackupStore(string StoreName, long Bytes); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs index 595281532..9e50b91a3 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs @@ -4,6 +4,14 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien { public abstract Task EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token); + /// + /// Whether the store was created completely, without creating or opening it. + /// + /// The name of the store. + /// The cancellation token. + /// True when the store exists. + public abstract Task VectorStoreExists(string storeName, CancellationToken token); + public abstract Task InsertEmbedding(string storeName, IReadOnlyList points, CancellationToken token); public abstract Task> SearchEmbeddingAsync(string storeName, IReadOnlyList vector, int maxMatches, CancellationToken token); @@ -24,4 +32,29 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien public abstract Task OptimizeVectorStore(string storeName, CancellationToken token); public abstract Task DeleteVectorStore(string storeName, CancellationToken token); + + /// + /// Prepares a backup of the stores: lists them, checks that their copies fit on the disk, and + /// creates the directory for the copies below the data directory. + /// + /// + /// Pause the indexing first, cf. DataSourceEmbeddingService.PauseIndexingAsync, so that no store + /// changes until all of them are copied. + /// + /// The cancellation token. + /// The backup, with every store to copy. + public abstract Task PrepareBackupAsync(CancellationToken token); + + /// + /// Copies one store into the directory of a prepared backup. + /// + /// + /// The store is unloaded for the copy and loads again with the next request. Requests for any + /// store wait while one is copied. + /// + /// The backup, as PrepareBackupAsync named it. + /// The name of the store. + /// The cancellation token. + /// How many bytes were copied. Zero when the store was not created completely. + public abstract Task CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token); } diff --git a/app/MindWork AI Studio/Tools/PathTools.cs b/app/MindWork AI Studio/Tools/PathTools.cs new file mode 100644 index 000000000..bff05ce75 --- /dev/null +++ b/app/MindWork AI Studio/Tools/PathTools.cs @@ -0,0 +1,45 @@ +namespace AIStudio.Tools; + +/// +/// Shared rules for comparing file system paths. +/// +public static class PathTools +{ + /// + /// How to compare file system paths: Windows treats them case-insensitively, Linux and + /// macOS do not. There, two files may differ in casing alone. + /// + public static readonly StringComparison COMPARISON = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + + /// + /// The comparer matching COMPARISON, for path-keyed lookups. + /// + public static readonly StringComparer COMPARER = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; + + /// + /// Determines whether a path lies below the given directory. + /// + /// + /// The directory itself does not count as lying below itself. A path which cannot be resolved + /// counts as lying elsewhere. + /// + /// The directory. + /// The path to check. + /// Whether the path lies below the directory. + public static bool IsInsideDirectory(string directory, string path) + { + try + { + var fullDirectory = Path.GetFullPath(directory); + var directoryPrefix = fullDirectory.EndsWith(Path.DirectorySeparatorChar) + ? fullDirectory + : fullDirectory + Path.DirectorySeparatorChar; + + return Path.GetFullPath(path).StartsWith(directoryPrefix, COMPARISON); + } + catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException) + { + return false; + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/AGENTS.md b/app/MindWork AI Studio/Tools/PluginSystem/AGENTS.md index efcf99445..5afa3f3d1 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/AGENTS.md +++ b/app/MindWork AI Studio/Tools/PluginSystem/AGENTS.md @@ -33,11 +33,11 @@ Plugins can configure: Configuration plugins provide three kinds of values: - **Managed settings:** simple values such as booleans, numbers, strings, enums, lists, or sets handled through `ManagedConfiguration`. These values may be locked or used as organization defaults. Which configuration plugin owns a locked setting is persisted in `Data.ManagedLockedConfigurations`, and organization defaults are tracked in `Data.ManagedEditableDefaults`. Both are cleaned up generically by `ManagedConfiguration.CleanupLeftOverManagedConfigurations(...)` when the owning plugin is gone. The value a setting had before a configuration plugin took it over is kept in `Data.ManagedUserValueSnapshots` and restored by that same clean-up, so removing a plugin hands the user's own value back instead of the app default. -- **Managed configuration objects:** complex Lua tables that are persisted into `SettingsManager.ConfigurationData`, implement `IConfigurationObject`, and are cleaned up through `PluginConfigurationObject.CleanLeftOverConfigurationObjects(...)`. Examples include providers, profiles, chat templates, data sources, and document analysis policies. +- **Managed configuration objects:** complex Lua tables that are persisted into `SettingsManager.ConfigurationData`, implement `IConfigurationObject`, and are described by a `PluginConfigurationObjectDescriptor` in `PluginConfigurationObjectDescriptors`. Reading them from a plugin and cleaning up the left-over ones go through these descriptors. Examples include providers, profiles, chat templates, data sources, and document analysis policies. - **Live plugin content:** complex Lua tables that implement `ILivePluginContent` and are read live from running plugins instead of being persisted to `ConfigurationData`. Examples include `MANDATORY_INFOS` and `INTRODUCTIONS`. If live plugin content creates persistent side data, add a dedicated cleanup path for that side data, like mandatory-info acceptances. When adding configuration plugin capabilities: - For managed settings, update the corresponding data class in `app/MindWork AI Studio/Settings/DataModel/` to call `ManagedConfiguration.Register(...)` and process the setting in `PluginConfiguration.TryProcessConfiguration`. Cleaning up the setting when its configuration plugin was removed needs no extra step: `ManagedConfiguration.CleanupLeftOverManagedConfigurations(...)` iterates all registered settings. Do not add per-setting cleanup calls to `PluginFactory.Loading.LoadAll`. -- For managed configuration objects, update `PluginConfigurationObject.cs` and `PluginConfigurationObjectType.cs`, persist them in the appropriate `ConfigurationData` collection, and add cleanup via `PluginConfigurationObject.CleanLeftOverConfigurationObjects(...)`. +- For managed configuration objects, add the type to `PluginConfigurationObjectType.cs`, persist the objects in the appropriate `ConfigurationData` collection, and add a descriptor to `PluginConfigurationObjectDescriptors` and its list `ALL`: its Lua table, its parser, the collection, the next number, and where secrets are kept. Parsing and cleanup need no extra call; `PluginConfigurationObjectDescriptorsTests` fails for a type without a descriptor. - For live plugin content, add a data type implementing `ILivePluginContent`, parse it in `PluginConfiguration`, expose it through `PluginFactory`, and add any required cleanup only for persistent side data. - Always document the new capability in `app/MindWork AI Studio/Plugins/configuration/plugin.lua`. diff --git a/app/MindWork AI Studio/Tools/PluginSystem/ConfigurationBaseObjectDescriptor.cs b/app/MindWork AI Studio/Tools/PluginSystem/ConfigurationBaseObjectDescriptor.cs new file mode 100644 index 000000000..9bed49c6b --- /dev/null +++ b/app/MindWork AI Studio/Tools/PluginSystem/ConfigurationBaseObjectDescriptor.cs @@ -0,0 +1,52 @@ +using System.Linq.Expressions; + +using AIStudio.Settings.DataModel; + +using Lua; + +namespace AIStudio.Tools.PluginSystem; + +/// +/// Describes a type of configuration object which derives from ConfigurationBaseObject. +/// +/// The type of the configuration objects. +/// The Lua table in which a configuration plugin defines these objects. +/// Where the settings keep these objects. +/// The number which the next of these objects gets. +/// Reads one of these objects from its Lua table. +/// Where the API keys of these objects are kept, or null for objects without one. +/// The class of the configuration objects. +public sealed class ConfigurationBaseObjectDescriptor(PluginConfigurationObjectType type, string luaTableName, Expression>> selection, Expression> nextNumSelection, ConfigurationObjectParser parser, SecretStoreType? secretStoreType = null) : PluginConfigurationObjectDescriptor where TClass : ConfigurationBaseObject +{ + /// + public override PluginConfigurationObjectType Type => type; + + /// + /// The Lua table in which a configuration plugin defines these objects. + /// + public string LuaTableName => luaTableName; + + /// + /// Where the settings keep these objects. + /// + public Expression>> Selection => selection; + + /// + /// The number which the next of these objects gets. + /// + public Expression> NextNumSelection => nextNumSelection; + + /// + /// Reads one of these objects from its Lua table. + /// + public ConfigurationObjectParser Parser => parser; + + /// + public override bool TryParse(LuaTable mainTable, Guid configPluginId, List configObjects, bool dryRun, string pluginPath) => PluginConfigurationObject.TryParse(this, mainTable, configPluginId, configObjects, dryRun, pluginPath); + + /// + public override Task CleanLeftOverAsync(IList availablePlugins, IReadOnlySet deployedEnterpriseConfigPluginIds, IReadOnlySet notStartedConfigPluginIds, IList configObjectList) => PluginConfigurationObject.CleanLeftOverConfigurationObjects(type, selection, availablePlugins, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, secretStoreType); + + /// + public override int RemovePluginObjects(Data settings) => selection.Compile()(settings).RemoveAll(configObject => configObject.IsEnterpriseConfiguration); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/ConfigurationObjectParser.cs b/app/MindWork AI Studio/Tools/PluginSystem/ConfigurationObjectParser.cs new file mode 100644 index 000000000..c52530f61 --- /dev/null +++ b/app/MindWork AI Studio/Tools/PluginSystem/ConfigurationObjectParser.cs @@ -0,0 +1,15 @@ +using Lua; + +namespace AIStudio.Tools.PluginSystem; + +/// +/// Reads one configuration object from the Lua table in which a configuration plugin defines it. +/// +/// The position of the entry within its Lua table, which the log names. +/// The entry to read. +/// The configuration plugin which defines the object. +/// The directory of the plugin, which paths in the entry are relative to. +/// Whether the plugin is only checked, e.g., before an import. Nothing must be stored then. +/// The configuration object read from the entry. +/// True when the entry defines a valid configuration object. +public delegate bool ConfigurationObjectParser(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/DataSourceConfigurationObjectDescriptor.cs b/app/MindWork AI Studio/Tools/PluginSystem/DataSourceConfigurationObjectDescriptor.cs new file mode 100644 index 000000000..126b46b06 --- /dev/null +++ b/app/MindWork AI Studio/Tools/PluginSystem/DataSourceConfigurationObjectDescriptor.cs @@ -0,0 +1,32 @@ +using AIStudio.Settings.DataModel; + +using Lua; + +namespace AIStudio.Tools.PluginSystem; + +/// +/// Describes the data sources which configuration plugins provide. +/// +/// +/// The settings keep every data source in a list of IDataSource, and a data source of a configuration +/// plugin is a record struct rather than a ConfigurationBaseObject. That is why data sources are read +/// in a way of their own. +/// +public sealed class DataSourceConfigurationObjectDescriptor : PluginConfigurationObjectDescriptor +{ + /// + public override PluginConfigurationObjectType Type => PluginConfigurationObjectType.DATA_SOURCE; + + /// + public override bool TryParse(LuaTable mainTable, Guid configPluginId, List configObjects, bool dryRun, string pluginPath) => PluginConfigurationObject.TryParseDataSources(mainTable, configPluginId, configObjects, dryRun); + + /// + /// + /// The secret of a data source is not necessarily an API key, e.g., it is a password, so it is + /// deleted as a secret. + /// + public override Task CleanLeftOverAsync(IList availablePlugins, IReadOnlySet deployedEnterpriseConfigPluginIds, IReadOnlySet notStartedConfigPluginIds, IList configObjectList) => PluginConfigurationObject.CleanLeftOverConfigurationObjects(this.Type, x => x.DataSources, availablePlugins, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.DATA_SOURCE, deleteSecret: true); + + /// + public override int RemovePluginObjects(Data settings) => settings.DataSources.RemoveAll(dataSource => dataSource.IsEnterpriseConfiguration); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/NoConfigurationObject.cs b/app/MindWork AI Studio/Tools/PluginSystem/NoConfigurationObject.cs deleted file mode 100644 index d42cb6f70..000000000 --- a/app/MindWork AI Studio/Tools/PluginSystem/NoConfigurationObject.cs +++ /dev/null @@ -1,26 +0,0 @@ -namespace AIStudio.Tools.PluginSystem; - -public sealed record NoConfigurationObject : ConfigurationBaseObject -{ - public static readonly NoConfigurationObject INSTANCE = new(); - - private NoConfigurationObject() - { - this.Id = Guid.Empty.ToString(); - this.Name = "No Configuration"; - } - - #region Overrides of ConfigurationBaseObject - - public override string Id { get; init; } - - public override uint Num { get; init; } - - public override string Name { get; init; } - - public override bool IsEnterpriseConfiguration { get; init; } - - public override Guid EnterpriseConfigurationPluginId { get; init; } - - #endregion -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginArchive.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginArchive.cs index 4db08c729..30424c844 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginArchive.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginArchive.cs @@ -22,16 +22,12 @@ public static class PluginArchive using var archive = ZipFile.OpenRead(sourceArchiveFileName); Directory.CreateDirectory(destinationDirectory); - var destinationDirectoryFullPath = Path.GetFullPath(destinationDirectory); - if (!destinationDirectoryFullPath.EndsWith(Path.DirectorySeparatorChar)) - destinationDirectoryFullPath += Path.DirectorySeparatorChar; - foreach (var entry in archive.Entries) { - var normalizedEntryName = NormalizeEntryName(entry.FullName); - var destinationPath = GetEntryDestinationPath(destinationDirectoryFullPath, normalizedEntryName); + var normalizedEntryName = ZipEntryNames.Normalize(entry.FullName); + var destinationPath = ZipEntryNames.GetDestinationPath(destinationDirectory, normalizedEntryName); - if (normalizedEntryName.EndsWith('/')) + if (normalizedEntryName.EndsWith(ZipEntryNames.SEPARATOR)) { if (entry.Length != 0) throw new InvalidDataException($"The plugin archive contains a directory entry with data: '{entry.FullName}'."); @@ -44,37 +40,4 @@ public static class PluginArchive entry.ExtractToFile(destinationPath); } } - - private static string NormalizeEntryName(string entryName) - { - var normalizedEntryName = entryName.Replace('\\', '/'); - if (string.IsNullOrWhiteSpace(normalizedEntryName)) - throw new InvalidDataException("The plugin archive contains an empty entry name."); - - if (normalizedEntryName.Contains('\0')) - throw new InvalidDataException($"The plugin archive contains an invalid entry name: '{entryName}'."); - - if (normalizedEntryName.StartsWith('/')) - throw new InvalidDataException($"The plugin archive contains a rooted entry name: '{entryName}'."); - - if (normalizedEntryName is [_, ':', ..]) - throw new InvalidDataException($"The plugin archive contains a drive-qualified entry name: '{entryName}'."); - - var pathSegments = normalizedEntryName.Split('/', StringSplitOptions.RemoveEmptyEntries); - if (pathSegments.Length == 0 || pathSegments.Any(segment => segment is "." or "..")) - throw new InvalidDataException($"The plugin archive contains an unsafe entry name: '{entryName}'."); - - return normalizedEntryName; - } - - private static string GetEntryDestinationPath(string destinationDirectoryFullPath, string normalizedEntryName) - { - var pathSegments = normalizedEntryName.Split('/', StringSplitOptions.RemoveEmptyEntries); - var relativePath = Path.Combine(pathSegments); - var destinationPath = Path.GetFullPath(Path.Combine(destinationDirectoryFullPath, relativePath)); - if (!destinationPath.StartsWith(destinationDirectoryFullPath, StringComparison.Ordinal)) - throw new InvalidDataException($"The plugin archive contains an entry outside the destination directory: '{normalizedEntryName}'."); - - return destinationPath; - } -} +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs index d3ea8d6c6..3bd75a98a 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs @@ -272,6 +272,12 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT // Config: allow the user to share or export plugins? ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToSharePlugins, this.Id, settingsTable, dryRun); + // Config: allow the user to back up the data of AI Studio? + ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToBackUpData, this.Id, settingsTable, dryRun); + + // Config: allow the user to restore the data of AI Studio from a backup? + ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToRestoreData, this.Id, settingsTable, dryRun); + // Config: show administration settings? ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.ShowAdminSettings, this.Id, settingsTable, dryRun); @@ -349,26 +355,9 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT // Config: enterprise-managed approvals for assistant plugins this.TryProcessEnterpriseApprovedAssistantPlugins(settingsTable, dryRun); - // Handle configured LLM providers: - PluginConfigurationObject.TryParse(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, x => x.NextProviderNum, mainTable, this.Id, ref this.configObjects, dryRun, this.PluginPath); - - // Handle configured transcription providers: - PluginConfigurationObject.TryParse(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER, x => x.TranscriptionProviders, x => x.NextTranscriptionNum, mainTable, this.Id, ref this.configObjects, dryRun, this.PluginPath); - - // Handle configured embedding providers: - PluginConfigurationObject.TryParse(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, x => x.NextEmbeddingNum, mainTable, this.Id, ref this.configObjects, dryRun, this.PluginPath); - - // Handle configured chat templates: - PluginConfigurationObject.TryParse(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, x => x.NextChatTemplateNum, mainTable, this.Id, ref this.configObjects, dryRun, this.PluginPath); - - // Handle configured data sources: - PluginConfigurationObject.TryParseDataSources(mainTable, this.Id, ref this.configObjects, dryRun); - - // Handle configured profiles: - PluginConfigurationObject.TryParse(PluginConfigurationObjectType.PROFILE, x => x.Profiles, x => x.NextProfileNum, mainTable, this.Id, ref this.configObjects, dryRun); - - // Handle configured document analysis policies: - PluginConfigurationObject.TryParse(PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY, x => x.DocumentAnalysis.Policies, x => x.NextDocumentAnalysisPolicyNum, mainTable, this.Id, ref this.configObjects, dryRun); + // Handle configured objects such as LLM providers, chat templates, or data sources: + foreach (var descriptor in PluginConfigurationObjectDescriptors.ALL) + descriptor.TryParse(mainTable, this.Id, this.configObjects, dryRun, this.PluginPath); // Handle configured mandatory infos: this.TryReadMandatoryInfos(mainTable); diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs index 4e2704171..e04a48fdc 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs @@ -125,53 +125,22 @@ public sealed record PluginConfigurationObject /// provided list with results. /// /// The type of configuration object to parse, which must - /// inherit from . - /// The type of configuration object to process, as specified - /// in . - /// An expression to retrieve existing configuration objects from - /// the main configuration data. - /// An expression to retrieve the next available configuration - /// object number from the main configuration data. + /// inherit from ConfigurationBaseObject. + /// Describes the type of configuration object: its Lua table, how to + /// read one entry, and where the settings keep these objects. /// The Lua table containing entries to parse into configuration objects. /// The unique identifier of the plugin associated with the configuration /// objects being parsed. - /// The list to populate with the parsed configuration objects. - /// This parameter is passed by reference. + /// The list to populate with the parsed configuration objects. /// Specifies whether to perform the operation as a dry run, where changes /// are not persisted. - /// An optional parameter specifying the file path of the plugin, used for relative paths in the Lua table. + /// The directory of the plugin, used for relative paths in the Lua table. /// Returns true if parsing succeeds and configuration objects are added /// to the list; otherwise, false. - public static bool TryParse( - PluginConfigurationObjectType configObjectType, - Expression>> configObjectSelection, - Expression> nextConfigObjectNumSelection, - LuaTable mainTable, - Guid configPluginId, - ref List configObjects, - bool dryRun, - string pluginPath = "" - ) where TClass : ConfigurationBaseObject + public static bool TryParse(ConfigurationBaseObjectDescriptor descriptor, LuaTable mainTable, Guid configPluginId, List configObjects, bool dryRun, string pluginPath) where TClass : ConfigurationBaseObject { - var luaTableName = configObjectType switch - { - PluginConfigurationObjectType.LLM_PROVIDER => "LLM_PROVIDERS", - PluginConfigurationObjectType.CHAT_TEMPLATE => "CHAT_TEMPLATES", - PluginConfigurationObjectType.DATA_SOURCE => "DATA_SOURCES", - PluginConfigurationObjectType.EMBEDDING_PROVIDER => "EMBEDDING_PROVIDERS", - PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER => "TRANSCRIPTION_PROVIDERS", - PluginConfigurationObjectType.PROFILE => "PROFILES", - PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY => "DOCUMENT_ANALYSIS_POLICIES", - - _ => null, - }; - - if (luaTableName is null) - { - LOG.LogError("The configuration object type '{ConfigObjectType}' is not supported yet (config plugin id: {ConfigPluginId}).", configObjectType, configPluginId); - return false; - } - + var configObjectType = descriptor.Type; + var luaTableName = descriptor.LuaTableName; if (!mainTable.TryGetValue(luaTableName, out var luaValue) || !luaValue.TryRead(out var luaTable)) { LOG.LogWarning("The table '{LuaTableName}' does not exist or is not a valid table (config plugin id: {ConfigPluginId}).", luaTableName, configPluginId); @@ -179,7 +148,7 @@ public sealed record PluginConfigurationObject } var localSettingsManager = SettingsManagerAccess; - var storedObjects = configObjectSelection.Compile()(localSettingsManager.ConfigurationData); + var storedObjects = descriptor.Selection.Compile()(localSettingsManager.ConfigurationData); var numberObjects = luaTable.ArrayLength; ThreadSafeRandom? random = null; for (var i = 1; i <= numberObjects; i++) @@ -191,19 +160,7 @@ public sealed record PluginConfigurationObject continue; } - var (wasParsingSuccessful, configObject) = configObjectType switch - { - PluginConfigurationObjectType.LLM_PROVIDER => (Settings.Provider.TryParseProviderTable(i, luaObjectTable, configPluginId, pluginPath, dryRun, out var configurationObject) && configurationObject != Settings.Provider.NONE, configurationObject), - PluginConfigurationObjectType.CHAT_TEMPLATE => (ChatTemplate.TryParseChatTemplateTable(i, luaObjectTable, configPluginId, pluginPath, out var configurationObject) && configurationObject != ChatTemplate.NO_CHAT_TEMPLATE, configurationObject), - PluginConfigurationObjectType.PROFILE => (Profile.TryParseProfileTable(i, luaObjectTable, configPluginId, out var configurationObject) && configurationObject != Profile.NO_PROFILE, configurationObject), - PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER => (TranscriptionProvider.TryParseTranscriptionProviderTable(i, luaObjectTable, configPluginId, pluginPath, dryRun, out var configurationObject) && configurationObject != TranscriptionProvider.NONE, configurationObject), - PluginConfigurationObjectType.EMBEDDING_PROVIDER => (EmbeddingProvider.TryParseEmbeddingProviderTable(i, luaObjectTable, configPluginId, pluginPath, dryRun, out var configurationObject) && configurationObject != EmbeddingProvider.NONE, configurationObject), - PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY => (DataDocumentAnalysisPolicy.TryProcessConfiguration(i, luaObjectTable, configPluginId, out var configurationObject) && configurationObject is DataDocumentAnalysisPolicy, configurationObject), - - _ => (false, NoConfigurationObject.INSTANCE) - }; - - if (wasParsingSuccessful) + if (descriptor.Parser(i, luaObjectTable, configPluginId, pluginPath, dryRun, out var configObject)) { // Store it in the config object list: configObjects.Add(new() @@ -236,7 +193,7 @@ public sealed record PluginConfigurationObject else { configObject = KeepSynchronizedTokenizer(configObject, null); - if (nextConfigObjectNumSelection.TryIncrement(localSettingsManager.ConfigurationData, IncrementType.POST) is { Success: true, UpdatedValue: var nextNum }) + if (descriptor.NextNumSelection.TryIncrement(localSettingsManager.ConfigurationData, IncrementType.POST) is { Success: true, UpdatedValue: var nextNum }) { // Case: Increment the next number was successful configObject = configObject with { Num = nextNum }; @@ -330,11 +287,7 @@ public sealed record PluginConfigurationObject /// The list to populate with the parsed configuration objects. /// Specifies whether to perform the operation as a dry run. /// True if the table was present and processed; otherwise false. - public static bool TryParseDataSources( - LuaTable mainTable, - Guid configPluginId, - ref List configObjects, - bool dryRun) + public static bool TryParseDataSources(LuaTable mainTable, Guid configPluginId, List configObjects, bool dryRun) { const string LUA_TABLE_NAME = "DATA_SOURCES"; if (!mainTable.TryGetValue(LUA_TABLE_NAME, out var luaValue) || !luaValue.TryRead(out var luaTable)) @@ -698,16 +651,6 @@ public sealed record PluginConfigurationObject var fullPluginPath = Path.GetFullPath(pluginPath); var candidatePath = Path.GetFullPath(Path.Combine(fullPluginPath, configuredTokenizerPath)); - - if (candidatePath.Equals(fullPluginPath, StringComparison.OrdinalIgnoreCase)) - return null; - - var pluginPrefix = fullPluginPath.EndsWith(Path.DirectorySeparatorChar) - ? fullPluginPath - : fullPluginPath + Path.DirectorySeparatorChar; - - return candidatePath.StartsWith(pluginPrefix, StringComparison.OrdinalIgnoreCase) - ? candidatePath - : null; + return PathTools.IsInsideDirectory(fullPluginPath, candidatePath) ? candidatePath : null; } } diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObjectDescriptor.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObjectDescriptor.cs new file mode 100644 index 000000000..263e24cec --- /dev/null +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObjectDescriptor.cs @@ -0,0 +1,54 @@ +using AIStudio.Settings.DataModel; + +using Lua; + +namespace AIStudio.Tools.PluginSystem; + +/// +/// Describes one type of configuration object which configuration plugins provide, such as LLM +/// providers or chat templates. +/// +/// +/// Whatever treats every type of configuration object alike goes through these descriptors, so that +/// none of these places can forget a type: reading the objects from a configuration plugin, removing +/// those a plugin no longer defines, and leaving all of them out of a backup. A new type of +/// configuration object needs a descriptor in PluginConfigurationObjectDescriptors, and every such place +/// handles it. +/// +public abstract class PluginConfigurationObjectDescriptor +{ + /// + /// The type of the configuration objects. + /// + public abstract PluginConfigurationObjectType Type { get; } + + /// + /// Reads the configuration objects of this type which a configuration plugin defines, and stores + /// them in the settings unless the plugin is only checked. + /// + /// The main table of the configuration plugin. + /// The configuration plugin. + /// The list to which every object read is added. + /// Whether the plugin is only checked, e.g., before an import. Nothing is stored then. + /// The directory of the plugin, which paths in the objects are relative to. + /// True when the plugin defines a table for this type. + public abstract bool TryParse(LuaTable mainTable, Guid configPluginId, List configObjects, bool dryRun, string pluginPath); + + /// + /// Removes the configuration objects of this type which belong to a configuration plugin that is + /// gone or no longer defines them, along with their secrets and tokenizers. + /// + /// The plugins AI Studio found. + /// The configuration plugins an organization deployed, including those which could not be loaded. + /// The configuration plugins which were loaded, but did not start. + /// The configuration objects all running configuration plugins define. + /// True when the settings changed. + public abstract Task CleanLeftOverAsync(IList availablePlugins, IReadOnlySet deployedEnterpriseConfigPluginIds, IReadOnlySet notStartedConfigPluginIds, IList configObjectList); + + /// + /// Removes the configuration objects of this type which a configuration plugin manages. + /// + /// The settings to change, e.g., a copy for a backup. Never the settings AI Studio runs with. + /// How many objects were removed. + public abstract int RemovePluginObjects(Data settings); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObjectDescriptors.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObjectDescriptors.cs new file mode 100644 index 000000000..02de9755c --- /dev/null +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObjectDescriptors.cs @@ -0,0 +1,63 @@ +using AIStudio.Settings; +using AIStudio.Settings.DataModel; + +using Lua; + +namespace AIStudio.Tools.PluginSystem; + +/// +/// The descriptors of every type of configuration object which configuration plugins provide. +/// +/// +/// A new type of configuration object gets its descriptor here and joins ALL. Reading the objects from +/// a configuration plugin and removing the left-over ones then handle it, and a test checks that no +/// type of PluginConfigurationObjectType is missing. +/// +public static class PluginConfigurationObjectDescriptors +{ + public static readonly ConfigurationBaseObjectDescriptor LLM_PROVIDER = new(PluginConfigurationObjectType.LLM_PROVIDER, "LLM_PROVIDERS", x => x.Providers, x => x.NextProviderNum, ParseLLMProvider, SecretStoreType.LLM_PROVIDER); + + public static readonly ConfigurationBaseObjectDescriptor TRANSCRIPTION_PROVIDER = new(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER, "TRANSCRIPTION_PROVIDERS", x => x.TranscriptionProviders, x => x.NextTranscriptionNum, ParseTranscriptionProvider, SecretStoreType.TRANSCRIPTION_PROVIDER); + + public static readonly ConfigurationBaseObjectDescriptor EMBEDDING_PROVIDER = new(PluginConfigurationObjectType.EMBEDDING_PROVIDER, "EMBEDDING_PROVIDERS", x => x.EmbeddingProviders, x => x.NextEmbeddingNum, ParseEmbeddingProvider, SecretStoreType.EMBEDDING_PROVIDER); + + public static readonly ConfigurationBaseObjectDescriptor CHAT_TEMPLATE = new(PluginConfigurationObjectType.CHAT_TEMPLATE, "CHAT_TEMPLATES", x => x.ChatTemplates, x => x.NextChatTemplateNum, ParseChatTemplate); + + public static readonly DataSourceConfigurationObjectDescriptor DATA_SOURCE = new(); + + public static readonly ConfigurationBaseObjectDescriptor PROFILE = new(PluginConfigurationObjectType.PROFILE, "PROFILES", x => x.Profiles, x => x.NextProfileNum, ParseProfile); + + public static readonly ConfigurationBaseObjectDescriptor DOCUMENT_ANALYSIS_POLICY = new(PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY, "DOCUMENT_ANALYSIS_POLICIES", x => x.DocumentAnalysis.Policies, x => x.NextDocumentAnalysisPolicyNum, ParseDocumentAnalysisPolicy); + + /// + /// Every descriptor, in the order in which configuration plugins are read. + /// + public static readonly IReadOnlyList ALL = + [ + LLM_PROVIDER, + TRANSCRIPTION_PROVIDER, + EMBEDDING_PROVIDER, + CHAT_TEMPLATE, + DATA_SOURCE, + PROFILE, + DOCUMENT_ANALYSIS_POLICY, + ]; + + private static bool ParseLLMProvider(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) => + Settings.Provider.TryParseProviderTable(index, table, configPluginId, pluginPath, dryRun, out configObject) && configObject != Settings.Provider.NONE; + + private static bool ParseTranscriptionProvider(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) => + TranscriptionProvider.TryParseTranscriptionProviderTable(index, table, configPluginId, pluginPath, dryRun, out configObject) && configObject != TranscriptionProvider.NONE; + + private static bool ParseEmbeddingProvider(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) => + EmbeddingProvider.TryParseEmbeddingProviderTable(index, table, configPluginId, pluginPath, dryRun, out configObject) && configObject != EmbeddingProvider.NONE; + + private static bool ParseChatTemplate(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) => + ChatTemplate.TryParseChatTemplateTable(index, table, configPluginId, pluginPath, out configObject) && configObject != ChatTemplate.NO_CHAT_TEMPLATE; + + private static bool ParseProfile(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) => + Profile.TryParseProfileTable(index, table, configPluginId, out configObject) && configObject != Profile.NO_PROFILE; + + private static bool ParseDocumentAnalysisPolicy(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) => + DataDocumentAnalysisPolicy.TryProcessConfiguration(index, table, configPluginId, out configObject) && configObject is DataDocumentAnalysisPolicy; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs index 3b5253bac..b96782cc3 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Loading.cs @@ -14,7 +14,19 @@ public static partial class PluginFactory { private static readonly List AVAILABLE_PLUGINS = []; private static readonly SemaphoreSlim PLUGIN_LOAD_SEMAPHORE = new(1, 1); - + private static readonly TaskCompletionSource FIRST_LOAD_COMPLETED = new(TaskCreationOptions.RunContinuationsAsynchronously); + + /// + /// Completes once LoadAll got through for the first time, the clean-up of the configuration included. + /// + /// + /// Until then, the settings may lack whatever the configuration of an organization provides, such + /// as its embedding providers: on a new computer, or after restoring a backup, which leaves them + /// out. Whoever acts on such objects while AI Studio starts waits for this, or takes them for + /// objects which are gone. It never completes when loading failed, so wait with a limit. + /// + public static Task FirstLoadCompleted => FIRST_LOAD_COMPLETED.Task; + /// /// How long a single plugin may take to load, and again to start. /// @@ -303,33 +315,10 @@ public static partial class PluginFactory var wasConfigurationChanged = hasStartedConfigurationPlugins; - // Check LLM providers: - if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.LLM_PROVIDER)) - wasConfigurationChanged = true; - - // Check transcription providers: - if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER, x => x.TranscriptionProviders, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.TRANSCRIPTION_PROVIDER)) - wasConfigurationChanged = true; - - // Check embedding providers: - if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.EMBEDDING_PROVIDER)) - wasConfigurationChanged = true; - - // Check data sources: - if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.DATA_SOURCE, x => x.DataSources, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.DATA_SOURCE, deleteSecret: true)) - wasConfigurationChanged = true; - - // Check chat templates: - if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList)) - wasConfigurationChanged = true; - - // Check profiles: - if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.PROFILE, x => x.Profiles, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList)) - wasConfigurationChanged = true; - - // Check document analysis policies: - if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY, x => x.DocumentAnalysis.Policies, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList)) - wasConfigurationChanged = true; + // Check configured objects such as LLM providers, chat templates, or data sources: + foreach (var descriptor in PluginConfigurationObjectDescriptors.ALL) + if(await descriptor.CleanLeftOverAsync(AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList)) + wasConfigurationChanged = true; // // Check left-over mandatory info acceptances. The mandatory infos come from the running @@ -372,6 +361,8 @@ public static partial class PluginFactory await SettingsManagerAccess.StoreSettings(); await MessageBus.INSTANCE.SendMessage(null, Event.CONFIGURATION_CHANGED); } + + FIRST_LOAD_COMPLETED.TrySetResult(); } /// diff --git a/app/MindWork AI Studio/Tools/Rust/AppRestartResponse.cs b/app/MindWork AI Studio/Tools/Rust/AppRestartResponse.cs new file mode 100644 index 000000000..38aeea2a8 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/AppRestartResponse.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Rust; + +public sealed record AppRestartResponse(bool Success, string ErrorMessage); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/FileTypes.cs b/app/MindWork AI Studio/Tools/Rust/FileTypes.cs index d57cb88d3..f2d32e11e 100644 --- a/app/MindWork AI Studio/Tools/Rust/FileTypes.cs +++ b/app/MindWork AI Studio/Tools/Rust/FileTypes.cs @@ -1,3 +1,4 @@ +using AIStudio.Tools.DataBackup; using AIStudio.Tools.PluginSystem; // ReSharper disable MemberCanBePrivate.Global @@ -104,6 +105,7 @@ public static class FileTypes public static readonly FileTypeFilter EXECUTABLES = FileTypeFilter.Leaf(TB("Executable"), "exe", "app", "bin", "appimage"); public static readonly FileTypeFilter SHORTCUT = FileTypeFilter.Leaf(TB("Shortcut"), "lnk"); public static readonly FileTypeFilter PLUGIN_ARCHIVE = FileTypeFilter.Leaf(TB("Plugin archive"), PluginArchive.PLUGIN_FILE_EXTENSION.TrimStart('.'), "zip"); + public static readonly FileTypeFilter BACKUP_ARCHIVE = FileTypeFilter.Leaf(TB("AI Studio backup"), DataBackupFormat.FILE_EXTENSION.TrimStart('.'), "zip"); /// /// The file types AI Studio converts using Pandoc. diff --git a/app/MindWork AI Studio/Tools/Services/DataBackupService.cs b/app/MindWork AI Studio/Tools/Services/DataBackupService.cs new file mode 100644 index 000000000..5ecb88bdf --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataBackupService.cs @@ -0,0 +1,280 @@ +using System.Reflection; +using System.Text.Json; + +using AIStudio.Settings; +using AIStudio.Tools.AIJobs; +using AIStudio.Tools.Databases; +using AIStudio.Tools.Databases.IndexStore; +using AIStudio.Tools.Databases.VectorStore; +using AIStudio.Tools.DataBackup; +using AIStudio.Tools.Media; +using AIStudio.Tools.Metadata; +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tools.Services; + +/// +/// Backs up all data of AI Studio into one file, one backup at a time. +/// +/// +/// The backup runs here rather than in the component which started it, so leaving the page does not +/// stop it, and a schedule can start it later on as well. The components only watch: they read the +/// state and hear about every change through StateChanged.

+/// +/// The service enforces what the organization allows on its own, since a component may show an +/// outdated state, and a schedule has no component at all. +/// +public sealed class DataBackupService(ILogger logger, SettingsManager settingsManager, DataSourceEmbeddingService embeddingService, DatabaseClientProvider databaseClientProvider, AIJobService aiJobService, MediaTranscriptionService mediaTranscriptionService) +{ + private const int BUFFER_SIZE = 64 * 1024; + + /// + /// ERROR_FILE_TOO_LARGE of Windows, as an HRESULT. + /// + private const int WINDOWS_FILE_TOO_LARGE = unchecked((int)0x800700DF); + + /// + /// ERROR_FILE_SYSTEM_LIMITATION of Windows, as an HRESULT. + /// + private const int WINDOWS_FILE_SYSTEM_LIMITATION = unchecked((int)0x80070299); + + /// + /// EFBIG of Linux and macOS, which .NET passes on as the HResult. + /// + private const int UNIX_FILE_TOO_LARGE = 27; + + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataBackupService).Namespace, nameof(DataBackupService)); + + private readonly Lock stateLock = new(); + + private CancellationTokenSource? running; + + /// + /// Fires whenever the state changes: a backup starts, makes progress, or ends. + /// + /// + /// It fires on whatever thread the backup runs. A component hands it over to its own context. + /// + public event Action? StateChanged; + + /// + /// Whether a backup runs. + /// + public bool IsRunning + { + get + { + lock (this.stateLock) + return this.running is not null; + } + } + + /// + /// How far the running backup got. + /// + public DataBackupProgress Progress { get; private set; } + + /// + /// How the last backup ended, or null when none ended yet. + /// + public DataBackupResult? LastResult { get; private set; } + + /// + /// Backs up all data into the given file. + /// + /// + /// The file is written in place; a file which was there before is replaced. A backup which fails + /// or is cancelled removes what it wrote. + /// + /// The file of the backup, as the user chose it. + /// How the backup ended. + public async Task CreateBackupAsync(string archivePath) + { + if (!settingsManager.ConfigurationData.App.AllowUserToBackUpData) + return this.Refuse(archivePath, TB("Your organization does not allow backups.")); + + // + // A chat which is being written while it is copied may end up broken in the backup. An + // answer or a transcript stores its chat once it is done, so the backup waits for them: + // + if (aiJobService.HasActiveJobs || mediaTranscriptionService.GetSnapshots().Any(snapshot => snapshot.Status is MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING)) + return this.Refuse(archivePath, TB("AI Studio is still writing an answer or a transcript. Please start the backup once it is done.")); + + CancellationTokenSource cancellation; + lock (this.stateLock) + { + if (this.running is not null) + return new(false, false, archivePath, 0, 0, false, TB("A backup is running already.")); + + cancellation = this.running = new(); + } + + this.ReportProgress(new(DataBackupPhase.PREPARING, 0, 0)); + var indexCopiesDirectory = string.Empty; + var token = cancellation.Token; + try + { + var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet."); + var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet."); + var settings = DataBackupSettings.CopyWithoutPluginObjects(settingsManager.ConfigurationData); + var settingsJson = JsonSerializer.Serialize(settings, SettingsManager.JSON_OPTIONS); + var organizationPluginFolders = DataBackupContents.FindOrganizationPluginsAmongUserPlugins(PluginFactory.AvailablePlugins.OfType(), dataDirectory); + + indexCopiesDirectory = await this.CopyIndexesAsync(token); + + var sources = new DataBackupSources(dataDirectory, configDirectory, settingsJson, indexCopiesDirectory, organizationPluginFolders); + var manifest = new DataBackupManifest( + DataBackupFormat.FORMAT_VERSION, + Assembly.GetExecutingAssembly().GetCustomAttribute()?.Version ?? "unknown", + settings.Version.ToString(), + DateTimeOffset.UtcNow, + dataDirectory, + configDirectory, + !string.IsNullOrWhiteSpace(indexCopiesDirectory), + []); + + var progress = new Progress(this.ReportProgress); + await using (var archiveStream = new FileStream(archivePath, FileMode.Create, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan)) + manifest = await Task.Run(() => DataBackupWriter.WriteAsync(archiveStream, sources, manifest, progress, token), token); + + logger.LogInformation("Backed up {FileCount} files with {Megabytes:F1} MB into '{ArchivePath}'. The indexes of the data sources are included: {IncludesIndexes}.", manifest.Entries.Count, manifest.Entries.Sum(entry => entry.Size) / (1024d * 1024d), archivePath, manifest.IncludesDataSourceIndexes); + return this.End(new(true, false, archivePath, manifest.Entries.Count, manifest.Entries.Sum(entry => entry.Size), manifest.IncludesDataSourceIndexes, string.Empty)); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + this.TryDeleteArchive(archivePath); + logger.LogInformation("The user cancelled the backup into '{ArchivePath}'. The incomplete backup was removed.", archivePath); + return this.End(new(false, true, archivePath, 0, 0, false, string.Empty)); + } + catch (IOException exception) when (IsFileTooLarge(exception)) + { + this.TryDeleteArchive(archivePath); + logger.LogError(exception, "The backup into '{ArchivePath}' failed, since the file system takes no file this large. The incomplete backup was removed.", archivePath); + return this.End(new(false, false, archivePath, 0, 0, false, TB("The backup is too large for the chosen drive, which takes no files beyond 4 GB. USB sticks formatted with FAT32 are such drives. Please choose another drive."))); + } + catch (Exception exception) + { + this.TryDeleteArchive(archivePath); + logger.LogError(exception, "The backup into '{ArchivePath}' failed. The incomplete backup was removed.", archivePath); + return this.End(new(false, false, archivePath, 0, 0, false, string.Format(TB("The backup failed: {0}"), exception.Message))); + } + finally + { + this.TryDeleteIndexCopies(indexCopiesDirectory); + lock (this.stateLock) + this.running = null; + + cancellation.Dispose(); + this.StateChanged?.Invoke(); + } + } + + /// + /// Whether writing failed since the file system takes no file this large, as FAT32 does beyond 4 GiB. + /// + private static bool IsFileTooLarge(IOException exception) => exception.HResult is WINDOWS_FILE_TOO_LARGE or WINDOWS_FILE_SYSTEM_LIMITATION or UNIX_FILE_TOO_LARGE; + + /// + /// Stops the running backup. It removes what it wrote so far. + /// + public void Cancel() + { + lock (this.stateLock) + this.running?.Cancel(); + } + + /// + /// Copies both indexes of the data sources, while the indexing pauses. + /// + /// + /// Both copies are made during one pause: the vector stores have to fit to the index, or a + /// restore would leave embeddings which belong to nothing, or files which count as embedded + /// without their embeddings. Writing the backup afterward takes far longer, so the indexing + /// goes on meanwhile.

+ /// + /// Without either database, the backup leaves both indexes out, and its manifest says so. + /// + /// The directory of the copies, in the layout of the data folder, or an empty string when the backup leaves the indexes out. + private async Task CopyIndexesAsync(CancellationToken token) + { + var indexStore = await databaseClientProvider.GetIndexStoreAsync(token); + var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token); + if (indexStore is NoIndexStoreClient || vectorStore is NoVectorStoreClient) + { + logger.LogWarning("The backup leaves the indexes of the data sources out, because the index store or the vector store is not available."); + return string.Empty; + } + + await using var pause = await embeddingService.PauseIndexingAsync(token); + var preparation = await vectorStore.PrepareBackupAsync(token); + try + { + var indexEntry = DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX); + await indexStore.BackupToDirectoryAsync(Path.Join(preparation.StagingPath, indexEntry.RelativePath.Replace(DataBackupContents.SEPARATOR, Path.DirectorySeparatorChar)), token); + + for (var index = 0; index < preparation.Stores.Count; index++) + { + this.ReportProgress(new(DataBackupPhase.COPYING_INDEXES, index, preparation.Stores.Count)); + await vectorStore.CopyStoreForBackupAsync(preparation.BackupId, preparation.Stores[index].StoreName, token); + } + + return preparation.StagingPath; + } + catch + { + this.TryDeleteIndexCopies(preparation.StagingPath); + throw; + } + } + + private DataBackupResult Refuse(string archivePath, string issue) + { + logger.LogWarning("Refused to start a backup into '{ArchivePath}': {Issue}", archivePath, issue); + return this.End(new(false, false, archivePath, 0, 0, false, issue)); + } + + private DataBackupResult End(DataBackupResult result) + { + this.LastResult = result; + this.StateChanged?.Invoke(); + return result; + } + + private void ReportProgress(DataBackupProgress progress) + { + this.Progress = progress; + this.StateChanged?.Invoke(); + } + + private void TryDeleteArchive(string archivePath) + { + try + { + if (File.Exists(archivePath)) + File.Delete(archivePath); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not remove the incomplete backup '{ArchivePath}'.", archivePath); + } + } + + /// + /// Removes the copies of the indexes. Should that fail, the next start removes them. + /// + private void TryDeleteIndexCopies(string indexCopiesDirectory) + { + if (string.IsNullOrWhiteSpace(indexCopiesDirectory)) + return; + + try + { + if (Directory.Exists(indexCopiesDirectory)) + Directory.Delete(indexCopiesDirectory, true); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not remove the copies of the indexes at '{IndexCopiesDirectory}'.", indexCopiesDirectory); + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataRestoreFollowUpService.cs b/app/MindWork AI Studio/Tools/Services/DataRestoreFollowUpService.cs new file mode 100644 index 000000000..d50543c40 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataRestoreFollowUpService.cs @@ -0,0 +1,45 @@ +using AIStudio.Settings; + +namespace AIStudio.Tools.Services; + +/// +/// One-shot startup service which picks up what a restore or a backup left behind: it learns how the +/// runtime applied a restore, removes what a backup or a preparation left halfway, and removes the data +/// a restore replaced once the restored settings loaded. +/// +/// +/// The report of the runtime waits in DataRestoreService until the main layout shows it, once the +/// plugins started: only then do the providers of the organization exist, and the open points name +/// none of them. +/// +public sealed class DataRestoreFollowUpService(ILogger logger, SettingsManager settingsManager, DataRestoreService dataRestoreService) : BackgroundService +{ + #region Overrides of BackgroundService + + /// Waits for the folders and the settings, picks up what the last start left, and then exits. + /// Host shutdown token. + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while ((string.IsNullOrWhiteSpace(SettingsManager.DataDirectory) || string.IsNullOrWhiteSpace(SettingsManager.ConfigDirectory)) && !stoppingToken.IsCancellationRequested) + await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); + + await dataRestoreService.LoadStartupStateAsync(stoppingToken); + + while (!settingsManager.HasCompletedInitialSettingsLoad && !stoppingToken.IsCancellationRequested) + await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); + + // + // Settings which could not be loaded leave the replaced data in place: they hold the settings + // which worked before the restore. + // + if (settingsManager.SettingsWriteBlocked) + { + logger.LogWarning("The settings could not be loaded safely. The data which a restore replaced stay where they are."); + return; + } + + dataRestoreService.RemoveReplacedData(); + } + + #endregion +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataRestoreService.cs b/app/MindWork AI Studio/Tools/Services/DataRestoreService.cs new file mode 100644 index 000000000..c2b3927b5 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataRestoreService.cs @@ -0,0 +1,385 @@ +using System.Reflection; +using System.Text.Json; + +using AIStudio.Settings; +using AIStudio.Tools.AIJobs; +using AIStudio.Tools.DataBackup; +using AIStudio.Tools.Media; +using AIStudio.Tools.Metadata; +using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.ToolCallingSystem; + +namespace AIStudio.Tools.Services; + +/// +/// Prepares the restore of a backup, which the next start of AI Studio applies. +/// +/// +/// A restore replaces all data, so it runs only while nothing else changes it: no answer, no +/// transcript, and no backup may be running, and the indexing pauses from the start of the preparation +/// until the restart. Whatever changes after the preparation is lost with the restart, which is why +/// the app asks for the restart right away, and offers to discard the restore instead.

+/// +/// The service enforces what the organization allows on its own, since a component may show an +/// outdated state.

+/// +/// At the start which applied a restore, the service holds the report of the runtime until the user +/// saw it, cf. DataRestoreFollowUpService. +/// +public sealed class DataRestoreService(ILogger logger, SettingsManager settingsManager, RustService rustService, ToolRegistry toolRegistry, ToolSettingsService toolSettingsService, DataSourceEmbeddingService embeddingService, DataBackupService dataBackupService, AIJobService aiJobService, MediaTranscriptionService mediaTranscriptionService) +{ + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataRestoreService).Namespace, nameof(DataRestoreService)); + + private readonly Lock stateLock = new(); + + private readonly TaskCompletionSource startupStateLoaded = new(TaskCreationOptions.RunContinuationsAsynchronously); + + private CancellationTokenSource? preparing; + + /// + /// Holds the indexing back while a restore is prepared or waits for the restart. + /// + private IAsyncDisposable? indexingPause; + + /// + /// Fires whenever the state changes: a preparation starts, makes progress, or ends, or a prepared restore is discarded. + /// + /// + /// It fires on whatever thread the preparation runs. A component hands it over to its own context. + /// + public event Action? StateChanged; + + /// + /// Whether a restore is being prepared. + /// + public bool IsPreparing + { + get + { + lock (this.stateLock) + return this.preparing is not null; + } + } + + /// + /// Whether a restore is prepared and waits for the restart. + /// + public bool IsRestorePending { get; private set; } + + /// + /// How far the running preparation got. + /// + public DataRestoreProgress Progress { get; private set; } + + /// + /// How the last preparation ended, or null when none ended yet. + /// + public DataRestoreResult? LastResult { get; private set; } + + /// + /// How the runtime applied a restore at this start, until the user saw it; otherwise null. + /// + public DataRestoreReport? Report { get; private set; } + + /// + /// Whether the runtime has not finished a restore and continues at the next start, cf. + /// DataRestoreFollowUp.IsRestoreUnfinished. Until then, AI Studio neither prepares nor discards a + /// restore. + /// + public bool IsRestoreUnfinished { get; private set; } + + /// + /// Completes once the service knows what the last start left behind, cf. LoadStartupStateAsync. + /// + public Task StartupStateLoaded => this.startupStateLoaded.Task; + + /// + /// Learns at the start what the runtime and the last run left behind: the report of a restore, a + /// restore which still waits for a restart, and what a backup or a preparation left halfway. + /// + /// Cancels reading the report. + public async Task LoadStartupStateAsync(CancellationToken token) + { + try + { + var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet."); + var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet."); + try + { + this.Report = await DataRestoreFollowUp.ReadReportAsync(dataDirectory, token); + if (this.Report is not null) + logger.LogInformation("The runtime applied a restore at this start: {Outcome}. {Detail}", this.Report.Outcome, this.Report.Detail); + } + catch (Exception exception) when (exception is JsonException or IOException) + { + logger.LogError(exception, "Could not read how the runtime applied a restore."); + DataRestoreFollowUp.RemoveReport(dataDirectory); + } + + this.IsRestoreUnfinished = DataRestoreFollowUp.IsRestoreUnfinished(dataDirectory); + if (this.IsRestoreUnfinished) + logger.LogError("The runtime has not finished a restore. It tries to move the replaced data back at the next start."); + + if (!dataBackupService.IsRunning && !this.IsPreparing) + DataRestoreFollowUp.RemoveLeftovers(dataDirectory, configDirectory); + + // + // A restore which still waits for its start, e.g., in the development environment, where the + // .NET server may restart without the runtime: whatever changes until then is lost. + // + if (DataRestoreFollowUp.IsRestorePending(dataDirectory)) + { + logger.LogWarning("A prepared restore still waits for the restart which applies it."); + this.IsRestorePending = true; + this.indexingPause ??= await embeddingService.PauseIndexingAsync(token); + } + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception exception) + { + logger.LogError(exception, "Could not learn what the last start left behind of a restore or a backup."); + } + finally + { + this.startupStateLoaded.TrySetResult(); + this.StateChanged?.Invoke(); + } + } + + /// + /// Removes the data which a restore replaced. Call it once the restored settings loaded. + /// + public void RemoveReplacedData() + { + try + { + var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet."); + var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet."); + if (DataRestoreFollowUp.RemoveReplacedData(dataDirectory, configDirectory)) + logger.LogInformation("Removed the data which the last restore replaced."); + } + catch (Exception exception) + { + logger.LogError(exception, "Could not remove the data which the last restore replaced."); + } + } + + /// + /// Finds what the user still has to do after the restore of this start, e.g., enter API keys again. + /// + /// The open points; none when no restore was applied. + public async Task> CollectOpenPointsAsync() + { + if (this.Report?.Outcome is not DataRestoreOutcome.APPLIED) + return []; + + try + { + return await DataRestoreOpenPoints.CollectAsync(settingsManager.ConfigurationData, new RustStoredSecrets(rustService), this.FindToolSecrets()); + } + catch (Exception exception) + { + logger.LogError(exception, "Could not find what remains to do after the restore."); + return []; + } + } + + /// + /// Finds the secret settings of the tools which the user set up, i.e., saved the settings of. + /// + /// + /// A setting the organization fixed comes from its configuration plugin, and one the tool does + /// not show for the saved settings means nothing for them. Which of the remaining ones held a + /// value nobody knows: a backup holds no secrets, so the dialog asks to enter them again if they + /// were used. + /// + private List FindToolSecrets() + { + var toolSecrets = new List(); + foreach (var definition in toolRegistry.GetAllDefinitions()) + { + if (toolRegistry.GetImplementation(definition.ImplementationKey) is not { } implementation) + continue; + + if (!settingsManager.ConfigurationData.Tools.Settings.TryGetValue(definition.Id, out var storedValues)) + continue; + + foreach (var (fieldName, field) in definition.SettingsSchema.Properties) + if (field.Secret && !toolSettingsService.IsFieldLocked(definition, fieldName) && implementation.IsSettingsFieldVisible(fieldName, storedValues)) + toolSecrets.Add(new(definition.Id, fieldName, implementation.GetDisplayName(), implementation.GetSettingsFieldLabel(fieldName, field))); + } + + return toolSecrets; + } + + /// + /// Forgets the report of the runtime, once the user saw it. + /// + public void ForgetReport() + { + this.Report = null; + try + { + DataRestoreFollowUp.RemoveReport(SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet.")); + } + catch (Exception exception) + { + logger.LogError(exception, "Could not remove the report of the restore."); + } + + this.StateChanged?.Invoke(); + } + + /// + /// Prepares the restore of a backup for the next start. + /// + /// The backup. + /// Whether the restore is prepared, or why not. + public async Task PrepareRestoreAsync(string archivePath) + { + if (!settingsManager.ConfigurationData.App.AllowUserToRestoreData) + return this.Refuse(TB("Your organization does not allow restoring backups.")); + + if (dataBackupService.IsRunning) + return this.Refuse(TB("A backup is running. Please restore once it is done.")); + + if (this.IsRestoreUnfinished) + return this.Refuse(TB("AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first.")); + + if (aiJobService.HasActiveJobs || mediaTranscriptionService.GetSnapshots().Any(snapshot => snapshot.Status is MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING)) + return this.Refuse(TB("AI Studio is still writing an answer or a transcript. Please restore once it is done.")); + + CancellationTokenSource cancellation; + lock (this.stateLock) + { + if (this.preparing is not null) + return new(false, false, TB("A restore is being prepared already.")); + + cancellation = this.preparing = new(); + } + + this.ReportProgress(new(0, 0)); + var token = cancellation.Token; + try + { + var target = this.CreateTarget(); + this.indexingPause ??= await embeddingService.PauseIndexingAsync(token); + + var progress = new Progress(this.ReportProgress); + var result = await Task.Run(() => DataRestorePreparer.PrepareAsync(archivePath, target, progress, token), token); + this.IsRestorePending = result.Success; + if (result.Success) + logger.LogInformation("Prepared the restore of the backup '{ArchivePath}'. It is applied at the next start.", archivePath); + else + logger.LogWarning("Could not prepare the restore of the backup '{ArchivePath}': {Issue}", archivePath, result.Issue); + + return this.End(result); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + logger.LogInformation("The user cancelled preparing the restore of the backup '{ArchivePath}'.", archivePath); + return this.End(new(false, true, string.Empty)); + } + catch (Exception exception) + { + // The preparation removes what it unpacked when it fails: + logger.LogError(exception, "Preparing the restore of the backup '{ArchivePath}' failed.", archivePath); + return this.End(new(false, false, string.Format(TB("The restore could not be prepared: {0}"), exception.Message))); + } + finally + { + lock (this.stateLock) + this.preparing = null; + + cancellation.Dispose(); + if (!this.IsRestorePending) + await this.EndIndexingPauseAsync(); + + this.StateChanged?.Invoke(); + } + } + + /// + /// Stops the running preparation. It removes what it unpacked so far. + /// + public void Cancel() + { + lock (this.stateLock) + this.preparing?.Cancel(); + } + + /// + /// Discards a prepared restore, so that the next start leaves all data as it is. + /// + public async Task DiscardPendingRestoreAsync() + { + if (this.IsPreparing || this.IsRestoreUnfinished) + return; + + try + { + var target = this.CreateTarget(); + try + { + DataRestorePreparer.Discard(target); + logger.LogInformation("Discarded the prepared restore."); + } + finally + { + // What remains on the disk decides, should discarding have failed halfway: + this.IsRestorePending = DataRestorePreparer.IsPrepared(target); + } + } + catch (Exception exception) + { + logger.LogError(exception, "Could not discard the prepared restore."); + } + + // A restore which the next start still applies keeps the indexing paused: + if (!this.IsRestorePending) + await this.EndIndexingPauseAsync(); + + this.StateChanged?.Invoke(); + } + + private DataRestoreTarget CreateTarget() + { + var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet."); + var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet."); + var appVersion = Assembly.GetExecutingAssembly().GetCustomAttribute()?.Version ?? string.Empty; + var organizationPluginFolders = DataBackupContents.FindOrganizationPluginsAmongUserPlugins(PluginFactory.AvailablePlugins.OfType(), dataDirectory); + + // Loading the settings brings them to the version this AI Studio writes: + return new(dataDirectory, configDirectory, appVersion, settingsManager.ConfigurationData.Version, organizationPluginFolders); + } + + private async Task EndIndexingPauseAsync() + { + var pause = this.indexingPause; + this.indexingPause = null; + if (pause is not null) + await pause.DisposeAsync(); + } + + private DataRestoreResult Refuse(string issue) + { + logger.LogWarning("Refused to prepare a restore: {Issue}", issue); + return this.End(new(false, false, issue)); + } + + private DataRestoreResult End(DataRestoreResult result) + { + this.LastResult = result; + this.StateChanged?.Invoke(); + return result; + } + + private void ReportProgress(DataRestoreProgress progress) + { + this.Progress = progress; + this.StateChanged?.Invoke(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Pause.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Pause.cs new file mode 100644 index 000000000..80550564a --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Pause.cs @@ -0,0 +1,96 @@ +namespace AIStudio.Tools.Services; + +public sealed partial class DataSourceEmbeddingService +{ + /// + /// Whether the indexing is paused right now, e.g., while a backup copies the indexes, or while a + /// restore waits for the restart. + /// + /// + /// Deleting or repairing a data source waits for the end of a pause, so the user interface asks + /// first and tells the user instead of waiting. + /// + public bool IsIndexingPaused => this.pauses.IsPaused; + + /// + /// Pauses the indexing of every data source, e.g., while a backup copies the indexes. + /// + /// + /// Every run stops, and the pause waits until they have, since a run may be about to write to + /// both stores. A document a run embedded completely stays stored, and its fingerprint keeps the + /// next run from embedding it once more. Every run asked for during the pause waits for its end, + /// the next sync of a mailbox among them. Ending the pause queues every data source whose run it + /// stopped, for the same reason that run was asked for.

+ /// + /// Pauses may overlap. The indexing goes on once the last one ended. + /// + /// Stops waiting for the runs to stop, which ends the pause as well. + /// The pause. Disposing it ends the pause. + public async Task PauseIndexingAsync(CancellationToken token) + { + // + // The pause begins before any run is stopped. A run which starts meanwhile is either among + // the active runs below, or it finds the indexing paused and waits: + // + this.pauses.Begin(); + var pause = new IndexingPause(this); + + var stoppedRuns = new List(); + foreach (var (dataSourceId, activeRun) in this.activeRuns) + { + if (!TryCancelRun(activeRun)) + continue; + + this.pauses.RecordStoppedRun(dataSourceId, activeRun.RefreshMode); + stoppedRuns.Add(activeRun.Completion.Task); + } + + try + { + await Task.WhenAll(stoppedRuns).WaitAsync(token); + } + catch + { + await pause.DisposeAsync(); + throw; + } + + logger.LogInformation("Paused the indexing of all data sources. Stopped {StoppedRuns} active runs.", stoppedRuns.Count); + return pause; + } + + /// + /// Ends a pause, and once the last one ended, queues every data source whose run a pause stopped. + /// + private async Task EndPauseAsync() + { + var stoppedRuns = this.pauses.End(); + foreach (var stoppedRun in stoppedRuns) + { + // Deleted during the pause, so there is nothing to carry on with: + if (!this.TryGetConfiguredIndexedSource(stoppedRun.DataSourceId, out var dataSource)) + continue; + + await this.QueueDataSourceAsync(dataSource, true, stoppedRun.RefreshMode); + } + } + + /// + /// One pause, which ends once it is disposed. + /// + private sealed class IndexingPause(DataSourceEmbeddingService service) : IAsyncDisposable + { + private int ended; + + /// + /// Ends the pause. Ending it again does nothing. + /// + public async ValueTask DisposeAsync() + { + if (Interlocked.Exchange(ref this.ended, 1) == 1) + return; + + await service.EndPauseAsync(); + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs index 15d9e5d3f..ff9ff07a0 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs @@ -36,6 +36,9 @@ public sealed partial class DataSourceEmbeddingService if (activeRun is not null) await activeRun.Completion.Task; + // A backup copies the stores during a pause, and the copies would no longer fit together: + await this.pauses.WaitWhilePausedAsync(CancellationToken.None); + await this.ResetPersistedStateAsync(dataSourceId, null, null, CancellationToken.None); this.statuses.TryRemove(dataSourceId, out _); PublishStatusChanged(); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 534d4f86f..e1c5b8fa8 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -27,6 +27,16 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM /// private static readonly TimeSpan REINDEX_CHECK_TIMEOUT = TimeSpan.FromSeconds(2); + /// + /// How long the first run waits for the plugins to be loaded, cf. WaitForFirstPluginLoadAsync. + /// + /// + /// Every plugin may take ten seconds to load and ten more to start, and the configuration of an + /// organization may have to be downloaded first. Should loading fail, the data sources are + /// indexed all the same, with whatever the settings hold by then. + /// + private static readonly TimeSpan FIRST_PLUGIN_LOAD_TIMEOUT = TimeSpan.FromMinutes(2); + /// /// One indexer per kind of data source this service indexes. /// @@ -57,7 +67,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService)); - private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource Completion) + /// + /// The pauses of the indexing, cf. PauseIndexingAsync. + /// + private readonly IndexingPauses pauses = new(); + + /// + /// An active run. + /// + /// Stops the run. + /// Completes once the run has ended. + /// Why the run was asked for. + private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource Completion, DataSourceEmbeddingRefreshMode RefreshMode) { /// /// The embedding settings the run works for, as GetRunSignature describes them. @@ -386,11 +407,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } /// - /// Whether a data source cannot be searched because its vector store cannot be read anymore. + /// Whether a data source cannot be searched because its vector store cannot be read anymore, or is missing. /// /// /// Unlike the re-index check above, this reads no database at all: the state comes from the run - /// or the search which ran into the unreadable store, and is kept in memory only. That it does + /// or the search which ran into the unreadable or missing store, and is kept in memory only. That it does /// not survive a restart is deliberate. The very same store may well open on the next start, /// and a mark written to disk would then be wrong with nobody noticing. Until something touches /// the store again, the data source counts as usable, and a failing search says so on its own. @@ -512,6 +533,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM await activeRun.Completion.Task; } + // A backup copies the stores during a pause, and the copies would no longer fit together: + await this.pauses.WaitWhilePausedAsync(CancellationToken.None); + this.statuses.TryRemove(dataSource.Id, out _); await this.ResetPersistedStateAsync(dataSource.Id, null, null, CancellationToken.None); this.statuses.TryRemove(dataSource.Id, out _); @@ -633,7 +657,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM var runTokenSource = CancellationTokenSource.CreateLinkedTokenSource(parentToken); var runControl = new DataSourceRunControl( runTokenSource, - new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously)); + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously), + refreshMode); if (!this.activeRuns.TryAdd(dataSource.Id, runControl)) { @@ -652,8 +677,10 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM // One run at a time, whichever way it started: the startup check goes on next to the // queue once the automatic refresh is switched on while the app runs. The wait counts as // part of the run, so deleting or repairing the data source ends the wait as well, - // instead of waiting for the run before it to finish first. + // instead of waiting for the run before it to finish first. A paused indexing holds + // every run back in front of the gate, cf. PauseIndexingAsync: // + await this.pauses.WaitWhilePausedAsync(runTokenSource.Token); await this.runGate.WaitAsync(runTokenSource.Token); try { @@ -746,16 +773,27 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM var persistedState = await indexStore.GetDataSourceStateAsync(dataSource.Id, token); if (persistedState is { VectorSize: > 0 }) { - var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedState.VectorSize, token); - if (ensureResult.Created) + // + // The index database knows what was indexed, but the vectors are gone, e.g. because the + // folder of the vector database was deleted, or restored from another backup than the + // index database. Rebuilding sends every document to the embedding provider once more, + // which costs money and hours, so it is for the user to decide, cf. RepairDataSourceAsync. + // Until then, the data source stays out of every search. + // + if (!await vectorStore.VectorStoreExists(collectionName, token)) { logger.LogWarning( - "Vector store '{CollectionName}' for data source '{DataSourceName}' ({DataSourceId}) was missing although persisted embedding state exists. Resetting the stale state so all vectors are rebuilt.", + "Vector store '{CollectionName}' for data source '{DataSourceName}' ({DataSourceId}) is missing although persisted embedding state exists. The data source is waiting for a repair.", collectionName, dataSource.Name, dataSource.Id); - await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, indexStore, token); + token.ThrowIfCancellationRequested(); + this.UpsertStatus(this.GetMissingVectorStoreStatus(dataSource)); + return null; } + + // Opens the store, so one which cannot be read shows up before the run starts: + await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedState.VectorSize, token); } if (!this.TryResolveEmbeddingProvider(dataSource, out var embeddingProvider)) @@ -925,6 +963,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM token.ThrowIfCancellationRequested(); + // + // The embedding providers and data sources of an organization come with its configuration + // plugin. On a new computer, or after restoring a backup, the settings do not hold them yet, + // and the first run would find such a data source without its embedding provider: + // + if (!await WaitForFirstPluginLoadAsync(PluginFactory.FirstLoadCompleted, FIRST_PLUGIN_LOAD_TIMEOUT, token)) + logger.LogWarning("The plugins were not loaded within {TimeoutSeconds:F0} seconds. Indexing starts with the settings as they are, without what the configuration of an organization may add later.", FIRST_PLUGIN_LOAD_TIMEOUT.TotalSeconds); + // Before the first run: from then on, a sync may be writing an attachment there. MailAttachmentFiles.DeleteLeftovers(logger); await this.DeleteOrphanedMailboxIndexesAsync(token); @@ -933,6 +979,26 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM await this.RunInitialDataSourceHashCheckAsync(token); } + /// + /// Waits until the plugins were loaded for the first time, but not longer than the given time. + /// + /// Completes once the plugins were loaded, cf. PluginFactory.FirstLoadCompleted. + /// How long to wait at most. + /// The cancellation token. + /// True when the plugins were loaded in time, false when the time ran out. + internal static async Task WaitForFirstPluginLoadAsync(Task firstLoad, TimeSpan timeout, CancellationToken token) + { + try + { + await firstLoad.WaitAsync(timeout, token); + return true; + } + catch (TimeoutException) + { + return false; + } + } + private async Task RunInitialDataSourceHashCheckAsync(CancellationToken token) { if (!settingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh) @@ -1230,9 +1296,23 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM /// path, is written in English for the log file, and says nothing about what happens next. What /// the user needs to read is what this means for their chats and where the way out is. /// - private DataSourceEmbeddingStatus GetUnreadableVectorStoreStatus(IDataSourceBase dataSource) + private DataSourceEmbeddingStatus GetUnreadableVectorStoreStatus(IDataSourceBase dataSource) => + this.GetRepairStatus(dataSource, string.Format(TB("The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name)); + + /// + /// The same way out as for a store which cannot be read, cf. GetUnreadableVectorStoreStatus. + /// + private DataSourceEmbeddingStatus GetMissingVectorStoreStatus(IDataSourceBase dataSource) => + this.GetRepairStatus(dataSource, string.Format(TB("The index of the data source '{0}' is missing. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name)); + + /// + /// Shows a data source as waiting for the user to have its index built anew, cf. NeedsIndexRepair. + /// + /// The data source. + /// What is wrong with the index, and where the way out is. + /// The failed status, which offers the repair. + private DataSourceEmbeddingStatus GetRepairStatus(IDataSourceBase dataSource, string errorMessage) { - var errorMessage = string.Format(TB("The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name); return this.CreateStatus( dataSource, DataSourceEmbeddingState.FAILED, diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs index 34aabb1d4..cd3c15257 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs @@ -12,7 +12,8 @@ namespace AIStudio.Tools.Services; /// document has pages to count. Block numbers start at one, the way the page states them. /// /// VectorStoreUnreadable says why a data source failed, not only that it did. The UI needs that -/// difference to offer the repair for this one case, and it is carried as its own flag so nothing +/// difference to offer the repair for this one case: the vector store cannot be read, or it is +/// missing although the index database knows what it held. It is carried as its own flag so nothing /// has to read it back out of the message in LastError. Attention does the same for what the user /// has to decide, and PendingRemovalCount is the number a held back removal asks about: agreeing to /// it hands that very number back, cf. ApprovePendingMailRemovalAsync. diff --git a/app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs b/app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs index bce3fc84e..e7eb1618a 100644 --- a/app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs +++ b/app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs @@ -10,10 +10,26 @@ public sealed class FileEnumerationResult public string LastError { get; set; } = string.Empty; + /// + /// The folders whose content could not be listed. The files below them are out of reach, not gone. + /// + public List UnreadableDirectories { get; } = []; + + /// + /// Why the file or the folder of the data source itself could not be reached, or null when it could. + /// + public string? UnreachableRootReason { get; set; } + public void AddFailure(string filePath, string reason) { this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason, DateTimeOffset.UtcNow)); this.FailedFiles = this.Failures.Count; this.LastError = reason; } + + public void AddUnreadableDirectory(string directoryPath, string reason) + { + this.UnreadableDirectories.Add(directoryPath); + this.AddFailure(directoryPath, reason); + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md b/app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md index d168feccc..7f808c291 100644 --- a/app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md +++ b/app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md @@ -67,6 +67,12 @@ Rules which are easy to break: data source selection that a run got through the whole data source (`IsIndexAwaitingRebuild`). Files pass `null` to `PauseRunAsync`, so a folder stays out of the selection until its first indexing or its rebuild is complete. The mailbox passes its hash, since every mail it indexed can be searched right away. +- **Every run passes the pause.** A backup pauses the indexing with `PauseIndexingAsync` while it copies the + stores. Runs wait for the end of a pause in `ProcessDataSourceRunAsync`, in front of the run gate, so start + every run there. A run which starts any other way writes to the stores while they are copied, and the backup + holds two stores which do not fit together. Discarding the stored index of a data source, when it is deleted + or repaired, waits for the end of a pause as well (`WaitWhilePausedAsync`); the user interface asks + `IsIndexingPaused` first and tells the user instead of waiting. - **A pass lives only while the last step left more to do.** The service drops the pass of a data source after every other outcome and every exception, and when the stored index is reset. A retry the user asks for, and a step with other embedding settings, begin a new pass (`IndexingPass.ContinueOrBegin`). A pass kept beyond that diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs b/app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs index 0ad62ebb3..f45f6b387 100644 --- a/app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs +++ b/app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs @@ -49,6 +49,29 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DocumentRunProgress).Namespace, nameof(DocumentRunProgress)); + /// + /// Makes the progress of a run which reads nothing, showing the data source as the index holds it. + /// + /// + /// For a run which stops before it looks at a single document, e.g. because the server of a + /// mailbox or the folder of a data source cannot be reached. The documents skipped for good keep + /// their place in the list, so the user still sees why. + /// + /// The run. + /// How many documents the index holds. + /// The logger of the embedding service. + /// The progress, ready to publish the stored state or why the run stopped. + public static DocumentRunProgress FromStoredState(IndexedRunContext context, int indexedDocuments, ILogger logger) + { + var permanentFailures = context.Manifest.PermanentFailures; + var progress = new DocumentRunProgress(context, indexedDocuments + permanentFailures.Count, 0, string.Empty, [], logger); + progress.RecordUnchanged(indexedDocuments); + foreach (var (key, failure) in permanentFailures) + progress.RecordStillUnreadable(key, failure); + + return progress; + } + public int TotalDocuments => totalDocuments; /// diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs b/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs index da8411a1c..25e36cb3c 100644 --- a/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs +++ b/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs @@ -68,6 +68,9 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, var enumerationStartedUtc = DateTimeOffset.UtcNow; var inputFiles = this.GetInputFiles(dataSource); var enumerationDuration = DateTimeOffset.UtcNow - enumerationStartedUtc; + if (inputFiles.UnreachableRootReason is { } unreachableRootReason) + return this.StopWithoutRoot(context, dataSource, unreachableRootReason); + var indexedFiles = inputFiles.Files; var totalFiles = indexedFiles.Count + inputFiles.FailedFiles; @@ -95,7 +98,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, context.CollectionName); var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles); - var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(context, indexedFiles, token); + var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(context, inputFiles, token); token.ThrowIfCancellationRequested(); logger.LogInformation( @@ -323,6 +326,32 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, return IndexedRunOutcome.MORE_TO_DO; } + /// + /// Ends a run which cannot reach the file or the folder of its data source, and keeps the index as it is. + /// + /// + /// A folder out of reach is never the same as an empty one. Read that way, a drive which is not + /// connected, or a network share while the VPN is down, would remove every embedding of the + /// data source, and embedding it all again once the folder is back costs as long as it took the + /// first time. Nothing is written: the stored hash still tells which state of the data source + /// the index holds, so the next run picks up from there. + /// + /// The run. + /// The data source. + /// Why the file or the folder cannot be reached, and what the user can do about it. + /// DONE, since nothing can be done until the file or the folder is back. + private IndexedRunOutcome StopWithoutRoot(IndexedRunContext context, IDataSource dataSource, string reason) + { + logger.LogWarning( + "Cannot reach the file or folder of data source '{DataSourceName}' ({DataSourceId}). The run stops, and nothing is removed from the index. StoredFiles={StoredFiles}.", + dataSource.Name, + dataSource.Id, + context.Manifest.Files.Count); + + DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger).PublishRunFailure(reason); + return IndexedRunOutcome.DONE; + } + /// /// Whether a file name marks a file which is never indexed, whatever its type. /// @@ -344,15 +373,11 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT); } - private async Task RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, IReadOnlyCollection indexedFiles, CancellationToken token) + private async Task RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, FileEnumerationResult inputFiles, CancellationToken token) { var manifest = context.Manifest; - var existingPaths = indexedFiles - .Select(file => file.FullName) - .ToHashSet(StringComparer.OrdinalIgnoreCase); - var removedFiles = 0; - foreach (var removedFilePath in manifest.Files.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList()) + foreach (var removedFilePath in GetGoneFiles(manifest.Files.Keys, inputFiles.Files, inputFiles.UnreadableDirectories)) { await context.DeleteDocumentPointsAsync(removedFilePath, token); await context.IndexStore.DeleteFileAsync(context.DataSource.Id, removedFilePath, token); @@ -369,12 +394,44 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, // A file which is gone needs no mark keeping it out of the index. Without this, the table // would grow with every document the user ever deleted: // - foreach (var removedFilePath in manifest.PermanentFailures.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList()) + foreach (var removedFilePath in GetGoneFiles(manifest.PermanentFailures.Keys, inputFiles.Files, inputFiles.UnreadableDirectories)) await context.ForgetPermanentFailureAsync(removedFilePath, token); return removedFiles; } + /// + /// Picks the files which are gone from the data source, out of those the index knows about. + /// + /// + /// A file below a folder which could not be listed is not gone, only out of reach, e.g. while the + /// user may not read that folder. Removing it would throw away its index, and embedding it again + /// once the folder can be read costs as long as it took the first time. + ///

+ /// Paths are compared ignoring their case, as everywhere in the manifest. On Linux and macOS, that + /// keeps a file whose path differs from an unreadable folder in its case alone, which costs no more + /// than a stale entry until the folder can be read again. + /// + /// The paths of the files the index knows about. + /// The files this run found. + /// The folders this run could not list. + /// The paths of the files which are gone. + internal static List GetGoneFiles(IEnumerable storedPaths, IEnumerable files, IEnumerable unreadableDirectories) + { + var existingPaths = files + .Select(file => file.FullName) + .ToHashSet(StringComparer.OrdinalIgnoreCase); + + var unreadablePrefixes = unreadableDirectories + .Select(directory => Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)) + Path.DirectorySeparatorChar) + .ToList(); + + return storedPaths + .Where(path => !existingPaths.Contains(path)) + .Where(path => !unreadablePrefixes.Any(prefix => path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase))) + .ToList(); + } + /// /// A file counts as settled when it was indexed or when it was skipped for good, both with a /// matching fingerprint. Counting only the indexed ones would let a single unreadable document @@ -468,17 +525,23 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, case DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path): this.EnumerateAccessibleFiles(localDirectory.Path, result); + if (result.UnreadableDirectories.Contains(localDirectory.Path)) + result.UnreachableRootReason = string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."), localDirectory.Path); + return result; } + // + // Missing is not the same as empty, cf. StopWithoutRoot: + // switch (dataSource) { case DataSourceLocalFile localFile: - result.AddFailure(localFile.FilePath, string.Format(TB("The file '{0}' does not exist."), localFile.FilePath)); + result.UnreachableRootReason = string.Format(TB("The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source."), localFile.FilePath); break; case DataSourceLocalDirectory localDirectory: - result.AddFailure(localDirectory.Path, string.Format(TB("The folder '{0}' does not exist."), localDirectory.Path)); + result.UnreachableRootReason = string.Format(TB("The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source."), localDirectory.Path); break; } @@ -511,7 +574,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, catch (Exception exception) { logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", currentPath); - result.AddFailure(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath)); + result.AddUnreadableDirectory(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath)); continue; } diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/MailboxIndexer.cs b/app/MindWork AI Studio/Tools/Services/Indexing/MailboxIndexer.cs index cb15e2668..2a38c879e 100644 --- a/app/MindWork AI Studio/Tools/Services/Indexing/MailboxIndexer.cs +++ b/app/MindWork AI Studio/Tools/Services/Indexing/MailboxIndexer.cs @@ -569,16 +569,8 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru foreach (var folder in await context.IndexStore.GetMailFoldersAsync(dataSourceId, token)) locationCount += (await context.IndexStore.GetMailLocationsAsync(dataSourceId, folder.Path, token)).Count; - var permanentFailures = context.Manifest.PermanentFailures; - var progress = new DocumentRunProgress(context, locationCount + permanentFailures.Count, 0, string.Empty, [], logger) - { - LastSyncUtc = (await context.IndexStore.GetMailboxSyncStateAsync(dataSourceId, token)).LastSyncCompletedUtc, - }; - - progress.RecordUnchanged(locationCount); - foreach (var (key, failure) in permanentFailures) - progress.RecordStillUnreadable(key, failure); - + var progress = DocumentRunProgress.FromStoredState(context, locationCount, logger); + progress.LastSyncUtc = (await context.IndexStore.GetMailboxSyncStateAsync(dataSourceId, token)).LastSyncCompletedUtc; return progress; } diff --git a/app/MindWork AI Studio/Tools/Services/IndexingPauses.cs b/app/MindWork AI Studio/Tools/Services/IndexingPauses.cs new file mode 100644 index 000000000..96d82ce3c --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/IndexingPauses.cs @@ -0,0 +1,92 @@ +namespace AIStudio.Tools.Services; + +/// +/// The pauses of the indexing, e.g., while a backup copies the indexes. +/// +/// +/// Pauses may overlap, and the indexing goes on once the last one ended. Every run which a pause +/// stopped is noted together with why it was asked for, so that it can be queued again for the same +/// reason once the indexing goes on: a retry the user asked for still signs in to a mailbox again +/// after its server refused a sign-in, cf. DataSourceRunQueue. +/// +internal sealed class IndexingPauses +{ + private readonly Lock stateLock = new(); + + /// + /// The runs which a pause stopped, each with why it was asked for. + /// + private readonly Dictionary stoppedRuns = new(StringComparer.OrdinalIgnoreCase); + + private int count; + + /// + /// Completes once the last pause ended. Null while the indexing is not paused. + /// + private TaskCompletionSource? lastPauseEnded; + + /// + /// Whether the indexing is paused right now. + /// + public bool IsPaused + { + get + { + lock (this.stateLock) + return this.count > 0; + } + } + + /// + /// Begins a pause. From now on, every run which is about to start waits. + /// + public void Begin() + { + lock (this.stateLock) + { + if (this.count++ == 0) + this.lastPauseEnded = new(TaskCreationOptions.RunContinuationsAsynchronously); + } + } + + /// + /// Notes a run which a pause stopped. + /// + /// The id of the data source. + /// Why the run was asked for. + public void RecordStoppedRun(string dataSourceId, DataSourceEmbeddingRefreshMode refreshMode) + { + lock (this.stateLock) + this.stoppedRuns.TryAdd(dataSourceId, refreshMode); + } + + /// + /// Ends a pause. + /// + /// The runs which the pauses stopped, once the last pause ended, so the caller queues them again. Otherwise, none. + public IReadOnlyList End() + { + lock (this.stateLock) + { + if (this.count == 0 || --this.count > 0) + return []; + + this.lastPauseEnded?.TrySetResult(); + this.lastPauseEnded = null; + + var runsToQueueAgain = this.stoppedRuns.Select(stoppedRun => new DataSourceRunRequest(stoppedRun.Key, stoppedRun.Value)).ToList(); + this.stoppedRuns.Clear(); + return runsToQueueAgain; + } + } + + /// + /// Waits while the indexing is paused. + /// + /// Stops the waiting, e.g., because the run waiting here was stopped. + public Task WaitWhilePausedAsync(CancellationToken token) + { + lock (this.stateLock) + return this.lastPauseEnded?.Task.WaitAsync(token) ?? Task.CompletedTask; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/RelativeDataPathsMigrationService.cs b/app/MindWork AI Studio/Tools/Services/RelativeDataPathsMigrationService.cs new file mode 100644 index 000000000..c20168501 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/RelativeDataPathsMigrationService.cs @@ -0,0 +1,73 @@ +using AIStudio.Settings; +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tools.Services; + +/// +/// One-shot startup service which stores the chats and the settings once more, so that every path +/// below the data directory in them becomes relative. +/// +/// +/// Compatibility shim, see documentation/compatibility-shims/2026-10-relative-data-paths.md. Remove +/// after 2027-04-14.

+/// +/// AI Studio stores such a path relative whenever it stores a chat or the settings. A chat which is +/// never stored again would keep its absolute paths, though, and a backup of the data directory +/// brings them to another computer, where they lead nowhere. +/// +public sealed class RelativeDataPathsMigrationService(ILogger logger, SettingsManager settingsManager) : BackgroundService +{ + /// + /// How long to wait for the plugins to load for the first time, before storing the settings anyway. + /// + private static readonly TimeSpan FIRST_PLUGIN_LOAD_TIMEOUT = TimeSpan.FromMinutes(2); + + #region Overrides of BackgroundService + + /// Waits for the settings, stores every chat and the settings once more, and then exits. + /// Host shutdown token. + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (!settingsManager.HasCompletedInitialSettingsLoad && !stoppingToken.IsCancellationRequested) + await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); + + if (settingsManager.ConfigurationData.StoredDataPathsAreRelative) + return; + + try + { + if (!await WorkspaceBehaviour.StoreChatsWithRelativePathsAsync(stoppingToken)) + { + logger.LogWarning("Not every chat could be stored with relative paths yet. The next start tries again."); + return; + } + + // + // While the plugins load for the first time, they bring their providers and tokenizers into + // the settings. Storing the settings meanwhile could meet a list in the middle of a change: + // + try + { + await PluginFactory.FirstLoadCompleted.WaitAsync(FIRST_PLUGIN_LOAD_TIMEOUT, stoppingToken); + } + catch (TimeoutException) + { + logger.LogWarning("The plugins did not finish loading within {Timeout} minutes. The settings are stored with relative paths anyway.", FIRST_PLUGIN_LOAD_TIMEOUT.TotalMinutes); + } + + settingsManager.ConfigurationData.StoredDataPathsAreRelative = true; + await settingsManager.StoreSettings(); + logger.LogInformation("Every chat and the settings are stored with paths relative to the data directory now."); + } + catch (OperationCanceledException) + { + throw; + } + catch (Exception exception) + { + logger.LogError(exception, "Storing the chats and the settings with relative paths failed. The next start tries again."); + } + } + + #endregion +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/RustService.App.cs b/app/MindWork AI Studio/Tools/Services/RustService.App.cs index 974d9c19e..21d59b62a 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.App.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.App.cs @@ -156,4 +156,41 @@ public sealed partial class RustService return false; } } + + /// + /// Requests the Rust runtime to restart the entire desktop application, e.g., to apply a prepared + /// restore of a backup. + /// + /// + /// In the development environment, the runtime cannot restart the .NET server, which it did not + /// start, and refuses. The developer restarts both instead. + /// + /// True when the restart is on its way. + public async Task RestartApplication() + { + try + { + var response = await this.http.PostAsync("/app/restart", null); + if (!response.IsSuccessStatusCode) + { + this.logger?.LogError("Failed to restart the app due to network error: {StatusCode}.", response.StatusCode); + return false; + } + + var result = await response.Content.ReadFromJsonAsync(this.jsonRustSerializerOptions); + if (result is null || !result.Success) + { + this.logger?.LogError("Failed to restart the app: {Error}", result?.ErrorMessage ?? "Unknown error"); + return false; + } + + this.logger?.LogInformation("Restart request sent to Rust runtime."); + return true; + } + catch (Exception ex) + { + this.logger?.LogError(ex, "Exception while requesting application restart."); + return false; + } + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.RelativeDataPaths.cs b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.RelativeDataPaths.cs new file mode 100644 index 000000000..e4d175782 --- /dev/null +++ b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.RelativeDataPaths.cs @@ -0,0 +1,115 @@ +using System.Text; +using System.Text.Json; + +using AIStudio.Chat; + +namespace AIStudio.Tools; + +// +// Compatibility shim, see documentation/compatibility-shims/2026-10-relative-data-paths.md. +// Remove after 2027-04-14. +// +public static partial class WorkspaceBehaviour +{ + /// + /// Writes every stored chat once more, so that the paths of its files below the data directory become relative. + /// + /// + /// A chat stored before carries absolute paths. They still work on this computer, but a backup of + /// the data directory brings them to another one, where they lead nowhere. Each chat is read and + /// written within one hold of its lock, so that nothing AI Studio stores meanwhile gets lost. The + /// time of the last change stays as it was: the workspaces sort their chats by it, and the + /// maintenance of the temporary chats deletes them by it. + /// + /// Cancels the pass, e.g., when AI Studio shuts down. + /// Whether every chat was taken care of; false when one has to wait for the next start. + internal static async Task StoreChatsWithRelativePathsAsync(CancellationToken token) + { + var everyChatStored = true; + foreach (var chat in EnumerateStoredChats()) + { + token.ThrowIfCancellationRequested(); + if (!await StoreChatWithRelativePathsAsync(chat, token)) + everyChatStored = false; + } + + return everyChatStored; + } + + private static IEnumerable EnumerateStoredChats() + { + if (Directory.Exists(TemporaryChatsRootDirectory)) + foreach (var chatDirectory in Directory.EnumerateDirectories(TemporaryChatsRootDirectory)) + if (Guid.TryParse(Path.GetFileName(chatDirectory), out var chatId)) + yield return new(Guid.Empty, chatId); + + if (!Directory.Exists(WorkspaceRootDirectory)) + yield break; + + foreach (var workspaceDirectory in Directory.EnumerateDirectories(WorkspaceRootDirectory)) + { + if (!Guid.TryParse(Path.GetFileName(workspaceDirectory), out var workspaceId)) + continue; + + foreach (var chatDirectory in Directory.EnumerateDirectories(workspaceDirectory)) + if (Guid.TryParse(Path.GetFileName(chatDirectory), out var chatId)) + yield return new(workspaceId, chatId); + } + } + + /// Whether the chat was taken care of: written anew, or with nothing to change. + private static async Task StoreChatWithRelativePathsAsync(LoadChat chat, CancellationToken token) + { + var (acquired, semaphore) = await TryAcquireChatSemaphoreAsync(chat.WorkspaceId, chat.ChatId, nameof(StoreChatWithRelativePathsAsync)); + if (!acquired) + return false; + + try + { + var chatDirectory = GetChatDirectory(chat.WorkspaceId, chat.ChatId); + var chatPath = Path.Join(chatDirectory, "thread.json"); + if (!File.Exists(chatPath)) + return true; + + var storedJson = await File.ReadAllTextAsync(chatPath, Encoding.UTF8, token); + var thread = JsonSerializer.Deserialize(storedJson, JSON_OPTIONS); + if (thread is null) + return true; + + var json = JsonSerializer.Serialize(thread, JSON_OPTIONS); + if (json == storedJson) + return true; + + var lastEditTime = File.GetLastWriteTimeUtc(chatPath); + var temporaryPath = Path.Join(chatDirectory, $".thread-{Guid.NewGuid():N}.tmp"); + try + { + await File.WriteAllTextAsync(temporaryPath, json, Encoding.UTF8, token); + File.Move(temporaryPath, chatPath, overwrite: true); + File.SetLastWriteTimeUtc(chatPath, lastEditTime); + } + finally + { + if (File.Exists(temporaryPath)) + File.Delete(temporaryPath); + } + + return true; + } + catch (Exception exception) when (exception is JsonException or NotSupportedException) + { + // Opening the chat fails the same way, so there is nothing to store, now or at the next start: + LOG.LogWarning(exception, "The chat '{ChatId}' of workspace '{WorkspaceId}' could not be read, so its paths stay as they are.", chat.ChatId, chat.WorkspaceId); + return true; + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + LOG.LogWarning(exception, "The chat '{ChatId}' of workspace '{WorkspaceId}' could not be stored with relative paths. The next start tries again.", chat.ChatId, chat.WorkspaceId); + return false; + } + finally + { + semaphore.Release(); + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs index 03969a524..02920f974 100644 --- a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs +++ b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs @@ -10,7 +10,7 @@ using AIStudio.Tools.PluginSystem; namespace AIStudio.Tools; -public static class WorkspaceBehaviour +public static partial class WorkspaceBehaviour { public readonly record struct TryCreateWorkspaceResult(bool Success, WorkspaceTreeWorkspace Workspace); @@ -872,7 +872,7 @@ public static class WorkspaceBehaviour Directory.CreateDirectory(chatDirectory); - await FinalizeStagedTranscriptsAsync(chat, chatDirectory); + FinalizeStagedTranscripts(chat, chatDirectory); var chatNamePath = Path.Join(chatDirectory, "name"); await File.WriteAllTextAsync(chatNamePath, chat.Name); @@ -970,25 +970,15 @@ public static class WorkspaceBehaviour private static void CopyManagedTranscriptAttachments(ChatThread chat, string targetChatDirectory) { - var targetTranscriptDirectory = Path.Combine(targetChatDirectory, "attachments", "transcripts"); - var pathComparer = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal; - var copiedPaths = new Dictionary(pathComparer); + var targetTranscriptDirectory = ManagedTranscriptAttachment.GetTranscriptDirectory(targetChatDirectory); + var copiedPaths = new Dictionary(PathTools.COMPARER); - foreach (var content in chat.Blocks.Select(block => block.Content).OfType()) - { - for (var index = 0; index < content.FileAttachments.Count; index++) - { - if (content.FileAttachments[index] is ManagedTranscriptAttachment transcript) - content.FileAttachments[index] = CopyManagedTranscriptAttachment(chat, transcript, targetTranscriptDirectory, copiedPaths); - } - } - - chat.Draft.ReplaceAttachments(attachment => attachment is ManagedTranscriptAttachment transcript + chat.ReplaceAttachments(attachment => attachment is ManagedTranscriptAttachment transcript ? CopyManagedTranscriptAttachment(chat, transcript, targetTranscriptDirectory, copiedPaths) : attachment); } - private static ManagedTranscriptAttachment CopyManagedTranscriptAttachment( + internal static ManagedTranscriptAttachment CopyManagedTranscriptAttachment( ChatThread chat, ManagedTranscriptAttachment source, string targetTranscriptDirectory, @@ -1045,7 +1035,7 @@ public static class WorkspaceBehaviour if (!Directory.Exists(chatDirectory)) throw new DirectoryNotFoundException($"The owning chat directory does not exist: '{chatDirectory}'."); - var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts"); + var transcriptDirectory = ManagedTranscriptAttachment.GetTranscriptDirectory(chatDirectory); Directory.CreateDirectory(transcriptDirectory); ReconcileTranscriptCounter(chat, transcriptDirectory); @@ -1096,7 +1086,7 @@ public static class WorkspaceBehaviour UpdateAttachmentPathsAfterMove(chat, sourceDirectory, targetDirectory); chat.WorkspaceId = targetWorkspaceId; - await FinalizeStagedTranscriptsAsync(chat, targetDirectory); + FinalizeStagedTranscripts(chat, targetDirectory); await StoreMovedChatFilesAsync(chat, targetDirectory); var lastEditTime = File.GetLastWriteTimeUtc(Path.Join(targetDirectory, "thread.json")); @@ -1153,22 +1143,14 @@ public static class WorkspaceBehaviour } } - /// Rewrites absolute attachment paths after moving the complete chat directory. - private static void UpdateAttachmentPathsAfterMove(ChatThread chat, string sourceDirectory, string targetDirectory) - { - foreach (var content in chat.Blocks.Select(block => block.Content).OfType()) - { - for (var index = 0; index < content.FileAttachments.Count; index++) - content.FileAttachments[index] = RelocateAttachment(content.FileAttachments[index], sourceDirectory, targetDirectory); - } - - // - // The transcripts of the message not sent yet live in the chat directory as well. Left with - // their old paths, the draft points at a directory which is gone after the move, and the - // transcripts would never reach the message. - // - chat.Draft.ReplaceAttachments(attachment => RelocateAttachment(attachment, sourceDirectory, targetDirectory)); - } + /// Rewrites absolute file paths after moving the complete chat directory. + /// + /// The transcripts of the message not sent yet live in the chat directory as well. Left with + /// their old paths, the draft points at a directory which is gone after the move, and the + /// transcripts would never reach the message. ChatThread.RewriteFilePaths takes the draft along. + /// + private static void UpdateAttachmentPathsAfterMove(ChatThread chat, string sourceDirectory, string targetDirectory) => + chat.RewriteFilePaths(path => RelocatePath(path, sourceDirectory, targetDirectory)); /// /// Gives an attachment the path it has after its chat moved into another workspace. @@ -1191,55 +1173,54 @@ public static class WorkspaceBehaviour /// Gives an attachment inside a moved directory its path in the target directory. private static FileAttachment RelocateAttachment(FileAttachment attachment, string sourceDirectory, string targetDirectory) { - var sourcePrefix = sourceDirectory.EndsWith(Path.DirectorySeparatorChar) - ? sourceDirectory - : sourceDirectory + Path.DirectorySeparatorChar; - - var pathComparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - if (!Path.GetFullPath(attachment.FilePath).StartsWith(sourcePrefix, pathComparison)) - return attachment; - - var relativePath = Path.GetRelativePath(sourceDirectory, attachment.FilePath); - var movedPath = Path.Combine(targetDirectory, relativePath); - - return attachment switch - { - ManagedTranscriptAttachment managed => managed with { FilePath = movedPath }, - FileAttachmentImage image => image with { FilePath = movedPath }, - _ => attachment with { FilePath = movedPath }, - }; + var movedPath = RelocatePath(attachment.FilePath, sourceDirectory, targetDirectory); + return movedPath == attachment.FilePath ? attachment : attachment.WithFilePath(movedPath); } - private static async Task FinalizeStagedTranscriptsAsync(ChatThread chat, string chatDirectory) + /// Gives a path inside a moved directory its place in the target directory, and leaves every other path as it is. + private static string RelocatePath(string path, string sourceDirectory, string targetDirectory) { - var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts"); + if (!PathTools.IsInsideDirectory(sourceDirectory, path)) + return path; + + return Path.Combine(targetDirectory, Path.GetRelativePath(sourceDirectory, path)); + } + + /// + /// Moves the transcripts of a chat which still wait in the staging area into the chat directory. + /// + /// + /// A transcript waits there when its chat was not stored yet when the transcript was made. The draft + /// counts as well: leaving a new chat stores it along with the message the user started in it, and + /// the next start of AI Studio empties the staging area. A transcript of the draft left behind there + /// would be gone, while the stored draft still pointed at it. + /// + private static void FinalizeStagedTranscripts(ChatThread chat, string chatDirectory) + { + var transcriptDirectory = ManagedTranscriptAttachment.GetTranscriptDirectory(chatDirectory); ReconcileTranscriptCounter(chat, transcriptDirectory); - foreach (var content in chat.Blocks.Select(block => block.Content).OfType()) - { - for (var index = 0; index < content.FileAttachments.Count; index++) - { - if (content.FileAttachments[index] is not ManagedTranscriptAttachment { IsStaged: true } staged - || !File.Exists(staged.FilePath)) - continue; + chat.ReplaceAttachments(attachment => attachment is ManagedTranscriptAttachment { IsStaged: true } staged && File.Exists(staged.FilePath) + ? FinalizeStagedTranscript(chat, staged, transcriptDirectory) + : attachment); + } - Directory.CreateDirectory(transcriptDirectory); - var targetPath = NextTranscriptPath(chat, transcriptDirectory, staged.OriginalFileName); + /// Moves one staged transcript into the transcript directory of its chat. + private static ManagedTranscriptAttachment FinalizeStagedTranscript(ChatThread chat, ManagedTranscriptAttachment staged, string transcriptDirectory) + { + Directory.CreateDirectory(transcriptDirectory); + var targetPath = NextTranscriptPath(chat, transcriptDirectory, staged.OriginalFileName); - File.Move(staged.FilePath, targetPath); - var sourceDirectory = Path.GetDirectoryName(staged.FilePath); - if (sourceDirectory is not null && Directory.Exists(sourceDirectory) && !Directory.EnumerateFileSystemEntries(sourceDirectory).Any()) - Directory.Delete(sourceDirectory); + File.Move(staged.FilePath, targetPath); + var sourceDirectory = Path.GetDirectoryName(staged.FilePath); + if (sourceDirectory is not null && Directory.Exists(sourceDirectory) && !Directory.EnumerateFileSystemEntries(sourceDirectory).Any()) + Directory.Delete(sourceDirectory); - content.FileAttachments[index] = new ManagedTranscriptAttachment( - Path.GetFileName(targetPath), - targetPath, - new FileInfo(targetPath).Length, - staged.OriginalFileName, - false); - } - } - - await Task.CompletedTask; + return new ManagedTranscriptAttachment( + Path.GetFileName(targetPath), + targetPath, + new FileInfo(targetPath).Length, + staged.OriginalFileName, + false); } /// Raises the persisted counter to the highest transcript suffix found chat-wide. @@ -1276,7 +1257,7 @@ public static class WorkspaceBehaviour } /// Returns the canonical storage directory for a chat identity. - private static string GetChatDirectory(Guid workspaceId, Guid chatId) => workspaceId == Guid.Empty + public static string GetChatDirectory(Guid workspaceId, Guid chatId) => workspaceId == Guid.Empty ? Path.Join(SettingsManager.DataDirectory, "tempChats", chatId.ToString()) : Path.Join(SettingsManager.DataDirectory, "workspaces", workspaceId.ToString(), chatId.ToString()); @@ -1458,7 +1439,11 @@ public static class WorkspaceBehaviour return true; } - private static async Task EnsureWorkspace(Guid workspaceId, string workspaceName) + /// + /// Ensures that a workspace with the given identity exists on disk and in the cache. + /// An existing workspace keeps its current name. + /// + public static async Task EnsureWorkspace(Guid workspaceId, string workspaceName) { var workspacePath = Path.Join(WorkspaceRootDirectory, workspaceId.ToString()); var workspaceNamePath = Path.Join(workspacePath, "name"); diff --git a/app/MindWork AI Studio/Tools/ZipEntryNames.cs b/app/MindWork AI Studio/Tools/ZipEntryNames.cs new file mode 100644 index 000000000..0b4f55be6 --- /dev/null +++ b/app/MindWork AI Studio/Tools/ZipEntryNames.cs @@ -0,0 +1,92 @@ +namespace AIStudio.Tools; + +/// +/// Checks the names of entries in ZIP files before anything is written to the disk. +/// +/// +/// A ZIP file comes from somewhere else: a plugin archive, the configuration of an organization, or a +/// backup. An entry name such as "../settings.json" or an absolute path would make extracting it +/// write outside of the target directory (zip slip). Every name is therefore checked here before it +/// becomes a path.

+/// +/// Backslashes count as separators as well. Compatibility shim for ZIP files created on Windows +/// with backslashes in their entry names (dotnet/runtime#27620); remove after dotnet/runtime#27620 +/// and #41914 are fixed. See documentation/compatibility-shims/2026-07-plugin-archive-zip-backslashes.md. +/// +public static class ZipEntryNames +{ + /// + /// The separator of entry names, which the ZIP format prescribes. + /// + public const char SEPARATOR = '/'; + + /// + /// Normalizes the name of an entry, and rejects a name which could lead outside of the target + /// directory with an InvalidDataException: an empty, rooted, or drive-qualified name, or one which + /// contains "." or "..". + /// + /// The name as the ZIP file holds it. + /// The name with forward slashes. + public static string Normalize(string entryName) + { + var normalizedEntryName = entryName.Replace('\\', SEPARATOR); + if (string.IsNullOrWhiteSpace(normalizedEntryName)) + throw new InvalidDataException("The archive contains an empty entry name."); + + if (normalizedEntryName.Contains('\0')) + throw new InvalidDataException($"The archive contains an invalid entry name: '{entryName}'."); + + if (normalizedEntryName.StartsWith(SEPARATOR)) + throw new InvalidDataException($"The archive contains a rooted entry name: '{entryName}'."); + + if (normalizedEntryName is [_, ':', ..]) + throw new InvalidDataException($"The archive contains a drive-qualified entry name: '{entryName}'."); + + var pathSegments = normalizedEntryName.Split(SEPARATOR, StringSplitOptions.RemoveEmptyEntries); + if (pathSegments.Length == 0 || pathSegments.Any(segment => segment is "." or "..")) + throw new InvalidDataException($"The archive contains an unsafe entry name: '{entryName}'."); + + return normalizedEntryName; + } + + /// + /// Normalizes the name of an entry, cf. Normalize, without throwing. + /// + /// The name as the ZIP file holds it. + /// The name with forward slashes, or an empty string when it is unsafe. + /// True when the name is safe. + public static bool TryNormalize(string entryName, out string normalizedEntryName) + { + try + { + normalizedEntryName = Normalize(entryName); + return true; + } + catch (InvalidDataException) + { + normalizedEntryName = string.Empty; + return false; + } + } + + /// + /// The path at which an entry is written below the target directory. A path which would lie + /// outside of it is rejected with an InvalidDataException. + /// + /// The target directory. + /// The entry name, as Normalize returned it. + /// The full path of the entry. + public static string GetDestinationPath(string destinationDirectory, string normalizedEntryName) + { + var fullDestinationDirectory = Path.GetFullPath(destinationDirectory); + if (!fullDestinationDirectory.EndsWith(Path.DirectorySeparatorChar)) + fullDestinationDirectory += Path.DirectorySeparatorChar; + + var pathSegments = normalizedEntryName.Split(SEPARATOR, StringSplitOptions.RemoveEmptyEntries); + var destinationPath = Path.GetFullPath(Path.Combine(fullDestinationDirectory, Path.Combine(pathSegments))); + if (!destinationPath.StartsWith(fullDestinationDirectory, StringComparison.Ordinal)) + throw new InvalidDataException($"The archive contains an entry outside the destination directory: '{normalizedEntryName}'."); + + return destinationPath; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md index 32dc49564..d4467f9a3 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md @@ -10,6 +10,9 @@ - Added Requesty as a new LLM provider for chats, embeddings, and transcriptions. Requesty is an AI gateway from the United Kingdom that gives you models from many providers with a single API key. Because where your data is processed depends on the model you choose, you set its trust level yourself. Thanks, Thibault Jaigu (`Thibaultjaigu`), for this first contribution. - Added [llmman](https://github.com/llmmanorg/llmman) as a new option for the models you host yourself. llmman runs open models on your own computer or server, and AI Studio can use them for chats, embeddings, and transcriptions. Thanks, Eric Curtin (`ericcurtin`), for this first contribution. - Added support for Mistral Large 4. +- Added backups of all your data. Under Data Backup & Restore, on the information page and in the settings, you save your chats, settings, plugins, and the index of your data sources into a single file, for example, to take everything to a new computer. Restoring such a backup replaces your data when AI Studio restarts, or you add only its chats to the ones you have. API keys and passwords never go into a backup, and after a restore, AI Studio tells you where to enter them again. Thanks, Jens Erler (`j-erler`), for this contribution. +- Added organization-wide management for backups. IT departments can prevent users from backing up their data or from restoring a backup. A backup leaves out what the configurations of your organization bring along, such as their providers and plugins, so nothing appears twice after a restore. +- Added an overview of the data folders of AI Studio to the information page, for when you back up your computer with a tool of your own. It shows where the folders are and which of their contents belong into a backup. AI Studio now stores the locations of its own files, such as transcripts, relative to its data folder, so such a copy also works on a computer with another user name. - Improved the Read Web Page tool. When you have not allowed the AI to choose web addresses freely, the tool now refuses every address that does not appear in your chat or in what the tools returned, instead of only asking the AI not to read it. - Improved the installation on macOS: AI Studio is now signed and checked by Apple, so it opens like any other app. You no longer need the Terminal. - Improved the installation on Windows: AI Studio is now signed, so Windows shows who published it. For a while, Windows may still ask you to confirm the installation until it knows the app better. @@ -25,6 +28,7 @@ - Fixed a warning about the confidence level that stayed in the dialog for adding a local data source, even after you chose an embedding provider that meets it. The dialog now checks again whenever you change either of them, so a warning also appears when you switch to a provider that falls short. - Fixed two confidence levels sharing the same name in the German user interface. Moderate is called "Mäßig" again, and Medium stays "Mittel". - Fixed renaming an external data source (ERI server) leaving its secret behind under the old name in the keychain of your operating system. +- Fixed the ERI server assistant deleting files in a folder you no longer used. When it updates a server, it deletes the files it created before, and after you chose another folder for the server, it also deleted them in the old one. It now only deletes files within the folder you chose. - Fixed the Search Confluence tool missing pages that the search of your Confluence wiki finds. The tool now searches the same way as the search box of your wiki. - Fixed AI Studio starting only partly when the configuration of your organization took a while to apply. The plugins late in line never started, so the app stayed in English instead of your language, and the configuration of your organization was missing, together with its models and its welcome text. Now every plugin gets its own time to start, and a slow one no longer holds up the others. - Fixed AI Studio deleting the providers, chat templates, and profiles of your organization, including their API keys, when the configuration of your organization did not start in time. They now stay in place until the configuration starts again. @@ -33,9 +37,11 @@ - Fixed answers breaking off with an error message from self-hosted AI servers such as vLLM when the AI made up the name of a tool or sent broken details for one. AI Studio now tells the AI that its tool call was invalid, and the AI can finish its answer. - Fixed OpenAI models using about 4,300 more tokens than necessary with every request. AI Studio offered them OpenAI's own web search in the background, even when you had not selected a web search. To let a model search the web, select the Web Search tool below the message field. - Fixed AI Studio sometimes rebuilding the index of a data source from scratch when the configuration of your organization was applied while the data source was being updated. -- Fixed AI Studio is continuing to update the index of a data source with the old settings after you, or the configuration of your organization, changed its embedding provider or the expert settings of the data source, which could take hours for nothing. AI Studio now stops right away and starts over with the new settings. -- Fixed AI Studio is continuing to send the content of a data source to an embedding provider after you deleted that provider, or after a confidence level changed so that the provider may no longer read the data source. AI Studio now stops right away. -- Fixed data sources are staying unavailable after the configuration of your organization changed their embedding provider, until one of their documents changed, or you restarted AI Studio. When AI Studio updates your data sources automatically, it now rebuilds their index right away. +- Fixed AI Studio continuing to update the index of a data source with the old settings after you, or the configuration of your organization, changed its embedding provider or the expert settings of the data source, which could take hours for nothing. AI Studio now stops right away and starts over with the new settings. +- Fixed AI Studio continuing to send the content of a data source to an embedding provider after you deleted that provider, or after a confidence level changed so that the provider may no longer read the data source. AI Studio now stops right away. +- Fixed data sources staying unavailable after the configuration of your organization changed their embedding provider, until one of their documents changed, or you restarted AI Studio. When AI Studio updates your data sources automatically, it now rebuilds their index right away. +- Fixed AI Studio deleting the index of a local data source when it could not reach its folder or file, for example, because the drive was not connected. Once the folder was back, AI Studio indexed everything again, which could take hours. AI Studio now keeps the index and tells you that it cannot reach the folder. The same applies to subfolders you are not allowed to open. +- Fixed AI Studio rebuilding the index of a data source from scratch without asking you when part of the index was missing on your computer. Since this sends every document to your embedding provider once more, AI Studio now tells you about the missing index and offers to repair the data source. - Fixed justified texts being hyphenated by the rules of English even when AI Studio shows another language, such as German. Screen readers now also know which language AI Studio uses. - Fixed the log file of AI Studio showing the wrong time for some entries when AI Studio was busy. Each entry now shows when it actually happened. - Fixed AI Studio pausing for a moment while it opened the local index of a data source, such as a mailbox. Other work, for example applying the configuration of your organization, had to wait until then. diff --git a/app/Tests/Chat/ChatThreadFilePathTests.cs b/app/Tests/Chat/ChatThreadFilePathTests.cs new file mode 100644 index 000000000..f9e4e2202 --- /dev/null +++ b/app/Tests/Chat/ChatThreadFilePathTests.cs @@ -0,0 +1,106 @@ +using AIStudio.Chat; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks that rewriting the file paths of a chat reaches every one of them, and nothing else. +/// +/// +/// Moving a chat, copying it, and writing it into a backup or reading it back all change the paths +/// a chat stores. A path left out points at a place where the file is no longer, and the file drops +/// out of the chat without a word: a transcript of the message which is still being written, or an +/// image the model no longer gets to see. +/// +[TestFixture] +public sealed class ChatThreadFilePathTests +{ + private static readonly DateTimeOffset START = new(2026, 10, 14, 9, 0, 0, TimeSpan.Zero); + + private static readonly string OLD_DIRECTORY = Path.Combine(Path.GetTempPath(), "old-chat"); + + private static readonly string NEW_DIRECTORY = Path.Combine(Path.GetTempPath(), "new-chat"); + + [Test] + public void EveryPathOfTheChatIsRewritten() + { + var document = new FileAttachment(FileAttachmentType.DOCUMENT, "report.pdf", Old("report.pdf"), 1024); + var transcript = new ManagedTranscriptAttachment("meeting.md", Old("meeting.md"), 512, "meeting.mp3", false); + var picture = new FileAttachmentImage("chart.png", Old("chart.png"), 2048); + var photo = new ContentImage { SourceType = ContentImageSource.LOCAL_PATH, Source = Old("photo.jpg") }; + var chat = new ChatThread + { + ChatId = Guid.NewGuid(), + Name = "Quarterly report", + Blocks = [Block(new ContentText { Text = "Summarize the report", FileAttachments = [document, transcript, picture] }), Block(photo)], + }; + chat.Draft.AddAttachments([new ManagedTranscriptAttachment("call.md", Old("call.md"), 256, "call.m4a", false)]); + + chat.RewriteFilePaths(Move); + + var message = (ContentText)chat.Blocks[0].Content!; + Assert.Multiple(() => + { + Assert.That(message.FileAttachments.Select(attachment => attachment.FilePath), Is.EqualTo(new[] { New("report.pdf"), New("meeting.md"), New("chart.png") })); + Assert.That(photo.Source, Is.EqualTo(New("photo.jpg")), "The model gets such an image from its path."); + Assert.That(chat.Draft.FileAttachments.Select(attachment => attachment.FilePath), Is.EqualTo(new[] { New("call.md") }), "The draft is the one place every path rewrite once forgot."); + }); + } + + [Test] + public void WhatTheRewriteLeavesAloneStaysAsItIs() + { + var elsewhere = new FileAttachment(FileAttachmentType.DOCUMENT, "notes.txt", Path.Combine(Path.GetTempPath(), "elsewhere", "notes.txt"), 64); + var linked = new ContentImage { SourceType = ContentImageSource.URL, Source = "https://example.org/chart.png" }; + var chat = new ChatThread + { + ChatId = Guid.NewGuid(), + Name = "Notes", + Blocks = [Block(new ContentText { Text = "Read my notes", FileAttachments = [elsewhere] }), Block(linked)], + }; + + chat.RewriteFilePaths(Move); + + Assert.Multiple(() => + { + Assert.That(((ContentText)chat.Blocks[0].Content!).FileAttachments.Single(), Is.SameAs(elsewhere)); + Assert.That(linked.Source, Is.EqualTo("https://example.org/chart.png"), "Only an image held by a local path has a path to rewrite."); + }); + } + + [Test] + public void AnAttachmentKeepsItsKindAtItsNewPlace() + { + var transcript = new ManagedTranscriptAttachment("meeting.md", Old("meeting.md"), 512, "meeting.mp3", true); + var picture = new FileAttachmentImage("chart.png", Old("chart.png"), 2048); + var document = new FileAttachment(FileAttachmentType.DOCUMENT, "report.pdf", Old("report.pdf"), 1024); + + var movedTranscript = transcript.WithFilePath(New("meeting.md")); + var movedPicture = picture.WithFilePath(New("chart.png")); + var movedDocument = document.WithFilePath(New("report.pdf")); + + Assert.Multiple(() => + { + Assert.That(movedTranscript, Is.EqualTo(new ManagedTranscriptAttachment("meeting.md", New("meeting.md"), 512, "meeting.mp3", true)), "A transcript has to stay a transcript, or deleting the chat leaves its file behind."); + Assert.That(movedPicture, Is.TypeOf()); + Assert.That(((FileAttachmentImage)movedPicture).Source, Is.EqualTo(New("chart.png")), "The model gets the image from its source, not from its path."); + Assert.That(movedPicture.FilePathAsUrl, Is.EqualTo(new FileAttachmentImage("chart.png", New("chart.png"), 2048).FilePathAsUrl), "The preview of the image loads it from there."); + Assert.That(movedDocument, Is.EqualTo(new FileAttachment(FileAttachmentType.DOCUMENT, "report.pdf", New("report.pdf"), 1024))); + }); + } + + private static string Old(string fileName) => Path.Combine(OLD_DIRECTORY, fileName); + + private static string New(string fileName) => Path.Combine(NEW_DIRECTORY, fileName); + + private static string Move(string path) => path.StartsWith(OLD_DIRECTORY, StringComparison.Ordinal) + ? Path.Combine(NEW_DIRECTORY, Path.GetRelativePath(OLD_DIRECTORY, path)) + : path; + + private static ContentBlock Block(IContent content) => new() + { + Time = START, + ContentType = content is ContentImage ? ContentType.IMAGE : ContentType.TEXT, + Content = content, + Role = ChatRole.USER, + }; +} \ No newline at end of file diff --git a/app/Tests/Chat/ManagedTranscriptAttachmentTests.cs b/app/Tests/Chat/ManagedTranscriptAttachmentTests.cs new file mode 100644 index 000000000..83473d6dc --- /dev/null +++ b/app/Tests/Chat/ManagedTranscriptAttachmentTests.cs @@ -0,0 +1,104 @@ +using AIStudio.Chat; +using AIStudio.Settings; + +namespace AIStudio.Tests.Chat; + +/// +/// Checks which transcript files AI Studio deletes as its own. +/// +/// +/// A transcript carries a full path, which a chat file edited outside the app may point anywhere. AI +/// Studio therefore deletes a file only where it keeps transcripts itself: in the transcript folder of a +/// chat, or in the staging area. +/// +[TestFixture] +[NonParallelizable] +public sealed class ManagedTranscriptAttachmentTests +{ + private string? previousDataDirectory; + private string testDirectory = string.Empty; + + [SetUp] + public void PrepareDataDirectory() + { + // The data directory is static state of the whole application, which is why this fixture does not run alongside others: + this.previousDataDirectory = SettingsManager.DataDirectory; + this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-transcripts-{Guid.NewGuid():N}"); + Directory.CreateDirectory(this.testDirectory); + SettingsManager.DataDirectory = this.testDirectory; + } + + [TearDown] + public void RemoveDataDirectory() + { + SettingsManager.DataDirectory = this.previousDataDirectory; + + try + { + Directory.Delete(this.testDirectory, true); + } + catch (IOException) + { + // A temporary directory we could not remove says nothing about the code under test. + } + } + + [Test] + public void TheTranscriptOfAWorkspaceChatIsDeleted() + { + var chatDirectory = Path.Combine(this.testDirectory, "workspaces", Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + var transcript = WriteTranscript(ManagedTranscriptAttachment.GetTranscriptDirectory(chatDirectory)); + + Assert.Multiple(() => + { + Assert.That(ManagedTranscriptAttachment.TryDeleteOwnedFile(transcript), Is.True); + Assert.That(File.Exists(transcript.FilePath), Is.False); + }); + } + + [Test] + public void AFileInAnotherFolderOfTheChatStays() + { + var chatDirectory = Path.Combine(this.testDirectory, "workspaces", Guid.NewGuid().ToString(), Guid.NewGuid().ToString()); + var file = WriteTranscript(Path.Combine(chatDirectory, "attachments", "images")); + + Assert.Multiple(() => + { + Assert.That(ManagedTranscriptAttachment.TryDeleteOwnedFile(file), Is.False, "Only the transcript folder belongs to the transcripts."); + Assert.That(File.Exists(file.FilePath), Is.True); + }); + } + + [Test] + public void AFileOutsideTheDataDirectoryStays() + { + var outsideDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-elsewhere-{Guid.NewGuid():N}"); + try + { + var file = WriteTranscript(ManagedTranscriptAttachment.GetTranscriptDirectory(Path.Combine(outsideDirectory, "tempChats", Guid.NewGuid().ToString()))); + + Assert.Multiple(() => + { + Assert.That(ManagedTranscriptAttachment.TryDeleteOwnedFile(file), Is.False, "The same folders elsewhere belong to somebody else."); + Assert.That(File.Exists(file.FilePath), Is.True); + }); + } + finally + { + Directory.Delete(outsideDirectory, true); + } + } + + /// + /// Writes a transcript into a folder. + /// + /// Where the transcript goes. + /// The transcript, as a chat attaches it to a message. + private static ManagedTranscriptAttachment WriteTranscript(string directory) + { + Directory.CreateDirectory(directory); + var path = Path.Combine(directory, "meeting-transcript-0001.md"); + File.WriteAllText(path, "# Transcription: meeting.mp4"); + return new ManagedTranscriptAttachment(Path.GetFileName(path), path, new FileInfo(path).Length, "meeting.mp4", false); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/DataBackup/BackupChatImportTests.cs b/app/Tests/Tools/DataBackup/BackupChatImportTests.cs new file mode 100644 index 000000000..435f0ee06 --- /dev/null +++ b/app/Tests/Tools/DataBackup/BackupChatImportTests.cs @@ -0,0 +1,200 @@ +using System.IO.Compression; +using System.Text; +using System.Text.Json; + +using AIStudio.Chat; +using AIStudio.Settings; +using AIStudio.Tools; +using AIStudio.Tools.ChatArchive; +using AIStudio.Tools.DataBackup; + +namespace AIStudio.Tests.Tools.DataBackup; + +/// +/// Checks adding the chats of a backup to the chats of another computer. +/// +/// +/// A backup is written on one computer and read on another, whose data folder lies elsewhere. Its +/// chats have to arrive with their transcripts, which the chats name by a path relative to the data +/// folder, and a chat which exists already must not turn into a second chat with the same ID. +/// +[TestFixture] +[NonParallelizable] +public sealed class BackupChatImportTests +{ + private string? previousDataDirectory; + private string testDirectory = string.Empty; + + private string SourceDataDirectory => Path.Combine(this.testDirectory, "source"); + + private string TargetDataDirectory => Path.Combine(this.testDirectory, "target"); + + private string ArchivePath => Path.Combine(this.testDirectory, "backup.mwbackup"); + + [SetUp] + public void RememberDataDirectory() + { + this.previousDataDirectory = SettingsManager.DataDirectory; + this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-backup-chat-import-{Guid.NewGuid():N}"); + Directory.CreateDirectory(this.testDirectory); + } + + [TearDown] + public void RemoveDataDirectories() + { + SettingsManager.DataDirectory = this.previousDataDirectory; + WorkspaceBehaviour.TestDataDirectory = null; + + // Whoever reads the cache next must not get the workspaces of this test: + WorkspaceBehaviour.InvalidateWorkspaceTreeCache(); + + try + { + Directory.Delete(this.testDirectory, true); + } + catch (IOException) + { + // A temporary directory we could not remove says nothing about the code under test. + } + } + + [Test] + public async Task TheChatsOfABackupArriveWithTheirTranscripts() + { + var storedChat = await this.WriteBackupWithAChatAsync(); + await UseDataDirectoryAsync(this.TargetDataDirectory); + + var preview = await ChatArchiveImporter.ReadPreviewAsync(this.ArchivePath, CancellationToken.None); + var result = await ChatArchiveImporter.ImportAsync(this.ArchivePath, ChatArchiveCollisionBehavior.SKIP, null, CancellationToken.None); + var chat = await WorkspaceBehaviour.LoadChatAsync(new(storedChat.WorkspaceId, storedChat.ChatId)); + var transcript = ((ContentText)chat!.Blocks.Single().Content!).FileAttachments.Single(); + + Assert.Multiple(() => + { + Assert.That(preview.Success, Is.True, preview.Issue); + Assert.That(preview.Manifest.Workspaces.Select(workspace => workspace.Name), Is.EqualTo(new[] { "Projects" })); + Assert.That(preview.Manifest.Workspaces.Single().Chats.Select(listedChat => listedChat.Name), Is.EqualTo(new[] { "Meeting notes" })); + Assert.That(result.ImportedChats, Is.EqualTo(1)); + Assert.That(result.ImportedWorkspaces, Is.EqualTo(1)); + Assert.That(transcript.FilePath, Does.StartWith(WorkspaceBehaviour.GetChatDirectory(storedChat.WorkspaceId, storedChat.ChatId)),"The transcript belongs to the chat on this computer, not to the data folder of the other one."); + Assert.That(File.ReadAllText(transcript.FilePath), Does.Contain("Hello and welcome.")); + }); + } + + [Test] + public async Task AChatWhichExistsAlreadyIsKept() + { + await this.WriteBackupWithAChatAsync(); + await UseDataDirectoryAsync(this.TargetDataDirectory); + + await ChatArchiveImporter.ImportAsync(this.ArchivePath, ChatArchiveCollisionBehavior.SKIP, null, CancellationToken.None); + var secondResult = await ChatArchiveImporter.ImportAsync(this.ArchivePath, ChatArchiveCollisionBehavior.SKIP, null, CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(secondResult.ImportedChats, Is.Zero); + Assert.That(secondResult.SkippedChats, Is.EqualTo(1), "Two chats with one ID would leave the app unable to tell them apart."); + }); + } + + [Test] + public async Task ABackupOfANewerFormatIsRefused() + { + var manifest = new DataBackupManifest(DataBackupFormat.FORMAT_VERSION + 1, "99.0.0", "V9", DateTimeOffset.UtcNow, "/data", "/config", false, []); + await using (var archiveStream = File.Create(this.ArchivePath)) + using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create)) + { + await using var manifestStream = archive.CreateEntry(DataBackupFormat.MANIFEST_ENTRY_NAME).Open(); + await JsonSerializer.SerializeAsync(manifestStream, manifest, DataBackupFormat.JSON_OPTIONS); + } + + var preview = await ChatArchiveImporter.ReadPreviewAsync(this.ArchivePath, CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(preview.Success, Is.False, "This version cannot know what a later format holds."); + Assert.That(preview.Issue, Is.Not.Empty); + }); + } + + [Test] + public async Task AZipFileWithoutManifestIsNoBackup() + { + await using (var archiveStream = File.Create(this.ArchivePath)) + using (var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create)) + { + await using var entryStream = archive.CreateEntry("notes.txt").Open(); + await entryStream.WriteAsync(Encoding.UTF8.GetBytes("Not a backup.")); + } + + var preview = await ChatArchiveImporter.ReadPreviewAsync(this.ArchivePath, CancellationToken.None); + + Assert.That(preview.Success, Is.False); + } + + /// + /// Stores a chat with a transcript in a workspace of the source computer, and backs up its data folder. + /// + private async Task WriteBackupWithAChatAsync() + { + await UseDataDirectoryAsync(this.SourceDataDirectory); + + var workspace = await WorkspaceBehaviour.TryCreateWorkspaceAsync("Projects"); + Assert.That(workspace.Success, Is.True, "The workspace could not be created."); + + var workspaceId = workspace.Workspace.WorkspaceId; + var chatId = Guid.NewGuid(); + var transcriptDirectory = Path.Combine(WorkspaceBehaviour.GetChatDirectory(workspaceId, chatId), "attachments", "transcripts"); + Directory.CreateDirectory(transcriptDirectory); + + var transcriptPath = Path.Combine(transcriptDirectory, "meeting-transcript-0001.md"); + await File.WriteAllTextAsync(transcriptPath, "# Transcription: meeting.mp4\n\nHello and welcome."); + + var chat = new ChatThread + { + WorkspaceId = workspaceId, + ChatId = chatId, + Name = "Meeting notes", + Blocks = + [ + new ContentBlock + { + Time = DateTimeOffset.UtcNow, + ContentType = ContentType.TEXT, + Content = new ContentText { Text = "Summarize the meeting", FileAttachments = [new ManagedTranscriptAttachment("meeting-transcript-0001.md", transcriptPath, new FileInfo(transcriptPath).Length, "meeting.mp4", false)] }, + Role = ChatRole.USER, + }, + ], + }; + + Assert.That(await WorkspaceBehaviour.StoreChatAsync(chat), Is.True, "The chat could not be stored."); + + var sources = new DataBackupSources(this.SourceDataDirectory, Path.Combine(this.testDirectory, "config"), "{}", string.Empty, []); + var manifest = new DataBackupManifest(DataBackupFormat.FORMAT_VERSION, "26.10.1", "V6", DateTimeOffset.UtcNow, this.SourceDataDirectory, string.Empty, false, []); + await using (var archiveStream = File.Create(this.ArchivePath)) + await DataBackupWriter.WriteAsync(archiveStream, sources, manifest, null, CancellationToken.None); + + return new(workspaceId, chatId); + } + + /// + /// Makes the given folder the data folder, as another computer would have it. + /// + /// + /// The data directory is static state of the whole application, which is why this fixture does + /// not run alongside others. The workspace behaviour keeps the roots of the workspaces and of the + /// temporary chats fixed, so it gets told about it separately, and its cache starts over. + /// + private static async Task UseDataDirectoryAsync(string dataDirectory) + { + Directory.CreateDirectory(dataDirectory); + SettingsManager.DataDirectory = dataDirectory; + WorkspaceBehaviour.TestDataDirectory = dataDirectory; + await WorkspaceBehaviour.ForceReloadWorkspaceTreeAsync(); + } + + /// + /// A chat as the source computer stored it. + /// + private sealed record StoredChat(Guid WorkspaceId, Guid ChatId); +} \ No newline at end of file diff --git a/app/Tests/Tools/DataBackup/DataBackupContentsTests.cs b/app/Tests/Tools/DataBackup/DataBackupContentsTests.cs new file mode 100644 index 000000000..35ebcbb47 --- /dev/null +++ b/app/Tests/Tools/DataBackup/DataBackupContentsTests.cs @@ -0,0 +1,110 @@ +using AIStudio.Tools.DataBackup; +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tests.Tools.DataBackup; + +/// +/// Checks the list of what AI Studio keeps in its folders, and which of it belongs into a backup. +/// +/// +/// A backup takes along only what the list includes, and a restore replaces only that. An entry +/// listed twice, or a pattern among the included entries, would leave both open to guessing. The +/// plugins an organization deployed among the user's own ones do not show by their folder, so the +/// list has to find them by what they declare about themselves. +/// +[TestFixture] +public sealed class DataBackupContentsTests +{ + private static readonly string DATA_DIRECTORY = Path.Combine(Path.GetTempPath(), "ai-studio-backup-contents"); + + [Test] + public void EveryEntryNamesItsPlaceOnce() + { + var places = DataBackupContents.ENTRIES.Select(entry => $"{entry.Location}:{entry.RelativePath}").ToList(); + + Assert.That(places, Is.Unique); + } + + [Test] + public void AnIncludedEntryNamesOnePlaceExactly() + { + var paths = DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded).Select(entry => entry.RelativePath).ToList(); + + Assert.Multiple(() => + { + Assert.That(paths.Where(path => path.Contains('*')), Is.Empty, "A restore replaces exactly what the backup holds, so it has to know the names."); + Assert.That(paths.Where(path => path.Contains('\\')), Is.Empty, "Paths use forward slashes on every OS."); + Assert.That(paths.Where(path => path.StartsWith(DataBackupContents.SEPARATOR)), Is.Empty, "Paths are relative to their folder."); + }); + } + + [Test] + public void EveryKindAppearsInTheList() + { + var listedKinds = DataBackupContents.ENTRIES.Select(entry => entry.Kind).ToHashSet(); + + Assert.That(Enum.GetValues(), Is.SubsetOf(listedKinds), "A kind without an entry means an entry was dropped from the list."); + } + + [Test] + public void APluginTheOrganizationDeployedAmongTheUsersPluginsIsFound() + { + var plugins = new[] + { + new AvailablePlugin(PluginFolder("company-assistant"), isManagedByConfigServer: true), + new AvailablePlugin(PluginFolder("my-assistant"), isManagedByConfigServer: false), + new AvailablePlugin(PluginFolder(".config", Guid.NewGuid().ToString(), "assistant"), isManagedByConfigServer: true), + new AvailablePlugin(Path.Combine(Path.GetTempPath(), "ai-studio-plugins-elsewhere", "assistant"), isManagedByConfigServer: true), + }; + + var found = DataBackupContents.FindOrganizationPluginsAmongUserPlugins(plugins, DATA_DIRECTORY); + + Assert.That(found, Is.EqualTo(new[] { "plugins/company-assistant/" }), "The folder of the configuration server is left out as a whole already, and a plugin outside of the data folder is no part of a backup anyway."); + } + + private static string PluginFolder(params string[] parts) => Path.Combine(DATA_DIRECTORY, "plugins", Path.Combine(parts)); + + /// + /// A plugin as the plugin factory lists it after loading it. + /// + private sealed class AvailablePlugin(string localPath, bool isManagedByConfigServer) : IAvailablePlugin + { + public string IconDataUrl => string.Empty; + + public PluginType Type => PluginType.ASSISTANT; + + public Guid Id { get; } = Guid.NewGuid(); + + public string Name => "Assistant"; + + public string Description => string.Empty; + + public PluginVersion Version => new(1, 0, 0); + + public DateTimeOffset? LastChanged => null; + + public string[] Authors => []; + + public string SupportContact => string.Empty; + + public string SourceURL => string.Empty; + + public PluginCategory[] Categories => []; + + public PluginTargetGroup[] TargetGroups => []; + + public bool IsMaintained => true; + + public string DeprecationMessage => string.Empty; + + public bool IsInternal => false; + + public string LocalPath => localPath; + + public bool IsManagedByConfigServer => isManagedByConfigServer; + + public Guid? ManagedConfigurationId => null; + + public int ConfigurationPriority => 0; + } +} \ No newline at end of file diff --git a/app/Tests/Tools/DataBackup/DataBackupSettingsTests.cs b/app/Tests/Tools/DataBackup/DataBackupSettingsTests.cs new file mode 100644 index 000000000..afbc6313a --- /dev/null +++ b/app/Tests/Tools/DataBackup/DataBackupSettingsTests.cs @@ -0,0 +1,100 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.DataBackup; + +namespace AIStudio.Tests.Tools.DataBackup; + +/// +/// Checks the settings as a backup takes them along. +/// +/// +/// A configuration plugin brings its objects back on another computer, along with their API keys. +/// Restored from a backup as well, they would compete with what the plugin brings, and without the +/// plugin they would stay without their keys, since secrets never go into a backup. What the user +/// configured themselves has to come along, though, and the settings AI Studio runs with must not +/// lose anything while a backup is made. +/// +[TestFixture] +public sealed class DataBackupSettingsTests +{ + private static readonly Guid PLUGIN_ID = Guid.Parse("7b2d4f6a-8c1e-4a3b-9d5f-2e4a6c8b0d1f"); + + [Test] + public void EveryObjectOfAConfigurationPluginIsLeftOut() + { + var settings = CreateSettingsWithObjectsOfTheUserAndOfAPlugin(); + + var copy = DataBackupSettings.CopyWithoutPluginObjects(settings); + + Assert.Multiple(() => + { + Assert.That(copy.Providers.Select(provider => provider.Id), Is.EqualTo(new[] { "user-provider" })); + Assert.That(copy.TranscriptionProviders.Select(provider => provider.Id), Is.EqualTo(new[] { "user-transcription" })); + Assert.That(copy.EmbeddingProviders.Select(provider => provider.Id), Is.EqualTo(new[] { "user-embedding" })); + Assert.That(copy.ChatTemplates.Select(template => template.Id), Is.EqualTo(new[] { "user-template" })); + Assert.That(copy.DataSources.Select(dataSource => dataSource.Id), Is.EqualTo(new[] { "user-data-source" })); + Assert.That(copy.Profiles.Select(profile => profile.Id), Is.EqualTo(new[] { "user-profile" })); + Assert.That(copy.DocumentAnalysis.Policies.Select(policy => policy.Id), Is.EqualTo(new[] { "user-policy" })); + }); + } + + [Test] + public void TheSettingsAIStudioRunsWithStayAsTheyAre() + { + var settings = CreateSettingsWithObjectsOfTheUserAndOfAPlugin(); + + DataBackupSettings.CopyWithoutPluginObjects(settings); + + Assert.Multiple(() => + { + Assert.That(settings.Providers, Has.Count.EqualTo(2)); + Assert.That(settings.DataSources, Has.Count.EqualTo(2)); + Assert.That(settings.DocumentAnalysis.Policies, Has.Count.EqualTo(2)); + }); + } + + [Test] + public void ALockedSettingStaysWithItsPlugin() + { + var settings = CreateSettingsWithObjectsOfTheUserAndOfAPlugin(); + settings.ManagedLockedConfigurations["App.UpdateInterval"] = PLUGIN_ID; + settings.ManagedUserValueSnapshots["App.UpdateInterval"] = "\"CHECK_EVERY_DAY\""; + + var copy = DataBackupSettings.CopyWithoutPluginObjects(settings); + + Assert.Multiple(() => + { + Assert.That(copy.ManagedLockedConfigurations["App.UpdateInterval"], Is.EqualTo(PLUGIN_ID), "After a restore, the clean-up hands the setting back to the user unless the plugin locks it again."); + Assert.That(copy.ManagedUserValueSnapshots["App.UpdateInterval"], Is.EqualTo("\"CHECK_EVERY_DAY\""), "That is the value the user gets back."); + }); + } + + private static Data CreateSettingsWithObjectsOfTheUserAndOfAPlugin() + { + var settings = new Data(); + + settings.Providers.Add(new AIStudio.Settings.Provider(1, "user-provider", "My provider", LLMProviders.OPEN_AI, new("gpt-6", "gpt-6"))); + settings.Providers.Add(new AIStudio.Settings.Provider(2, "plugin-provider", "Provider of the organization", LLMProviders.SELF_HOSTED, new("llama", "llama"), IsSelfHosted: true, IsEnterpriseConfiguration: true, EnterpriseConfigurationPluginId: PLUGIN_ID)); + + settings.TranscriptionProviders.Add(new TranscriptionProvider(1, "user-transcription", "My transcription", LLMProviders.OPEN_AI, new("whisper-1", "whisper-1"))); + settings.TranscriptionProviders.Add(new TranscriptionProvider(2, "plugin-transcription", "Transcription of the organization", LLMProviders.OPEN_AI, new("whisper-1", "whisper-1"), IsEnterpriseConfiguration: true, EnterpriseConfigurationPluginId: PLUGIN_ID)); + + settings.EmbeddingProviders.Add(new EmbeddingProvider(1, "user-embedding", "My embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"))); + settings.EmbeddingProviders.Add(new EmbeddingProvider(2, "plugin-embedding", "Embeddings of the organization", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"), IsEnterpriseConfiguration: true, EnterpriseConfigurationPluginId: PLUGIN_ID)); + + settings.ChatTemplates.Add(new ChatTemplate { Id = "user-template", Name = "My template" }); + settings.ChatTemplates.Add(new ChatTemplate { Id = "plugin-template", Name = "Template of the organization", IsEnterpriseConfiguration = true, EnterpriseConfigurationPluginId = PLUGIN_ID }); + + settings.DataSources.Add(new DataSourceERI_V1 { Id = "user-data-source", Name = "My server" }); + settings.DataSources.Add(new DataSourceERI_V1 { Id = "plugin-data-source", Name = "Server of the organization", IsEnterpriseConfiguration = true, EnterpriseConfigurationPluginId = PLUGIN_ID }); + + settings.Profiles.Add(new Profile { Id = "user-profile", Name = "My profile" }); + settings.Profiles.Add(new Profile { Id = "plugin-profile", Name = "Profile of the organization", IsEnterpriseConfiguration = true, EnterpriseConfigurationPluginId = PLUGIN_ID }); + + settings.DocumentAnalysis.Policies.Add(new DataDocumentAnalysisPolicy { Id = "user-policy", Name = "My policy" }); + settings.DocumentAnalysis.Policies.Add(new DataDocumentAnalysisPolicy { Id = "plugin-policy", Name = "Policy of the organization", IsEnterpriseConfiguration = true, EnterpriseConfigurationPluginId = PLUGIN_ID }); + + return settings; + } +} \ No newline at end of file diff --git a/app/Tests/Tools/DataBackup/DataBackupWriterTests.cs b/app/Tests/Tools/DataBackup/DataBackupWriterTests.cs new file mode 100644 index 000000000..2a7c157f5 --- /dev/null +++ b/app/Tests/Tools/DataBackup/DataBackupWriterTests.cs @@ -0,0 +1,212 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using System.Text.Json; + +using AIStudio.Tools.DataBackup; + +namespace AIStudio.Tests.Tools.DataBackup; + +/// +/// Checks what a backup holds. +/// +/// +/// A restore unpacks the backup and replaces what the list of the backup contents names. A file +/// missing from it is lost on the next computer, and a file of the organization or of AI Studio +/// itself which got in would compete with the one the organization or AI Studio brings. The indexes +/// come from their copies only: read while AI Studio writes to them, they would not fit together. +/// +[TestFixture] +public sealed class DataBackupWriterTests +{ + private const string SETTINGS_JSON = """{ "Version": "V6" }"""; + + private static readonly Guid WORKSPACE_ID = Guid.Parse("3f2b7c1e-9a4d-4e6b-8c2f-1d5e7a9b0c3d"); + + private static readonly Guid CHAT_ID = Guid.Parse("8a1c4e7f-2b5d-4f9a-b3c6-0e2d4f6a8b1c"); + + private string testDirectory = string.Empty; + + private string DataDirectory => Path.Combine(this.testDirectory, "data"); + + private string ConfigDirectory => Path.Combine(this.testDirectory, "config"); + + private string IndexCopiesDirectory => Path.Combine(this.testDirectory, "index-copies"); + + [SetUp] + public void CreateFolders() + { + this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-backup-writer-{Guid.NewGuid():N}"); + + this.WriteDataFile($"workspaces/{WORKSPACE_ID}/name", "Projects"); + this.WriteDataFile($"workspaces/{WORKSPACE_ID}/{CHAT_ID}/thread.json", "{}"); + this.WriteDataFile($"workspaces/{WORKSPACE_ID}/{CHAT_ID}/attachments/transcripts/meeting-transcript-0001.md", "Hello and welcome."); + this.WriteDataFile($"tempChats/{Guid.Empty}/thread.json", "{}"); + this.WriteDataFile("plugins/my-assistant/plugin.lua", "-- mine"); + this.WriteDataFile("plugins/my-assistant/.hidden.lua", "-- hidden, but mine as well"); + this.WriteDataFile("plugins/company-assistant/plugin.lua", "-- deployed by the organization"); + this.WriteDataFile("plugins/.internal/language/plugin.lua", "-- comes with AI Studio"); + this.WriteDataFile("plugins/.config/0b5f1e8a-3c2d-4e6f-9a1b-7c8d9e0f1a2b/plugin.lua", "-- configuration of the organization"); + this.WriteDataFile("plugins/.lock", string.Empty); + this.WriteDataFile("tokenizers/openai-gpt-6/tokenizer.json", "{}"); + this.WriteDataFile("visualBriefings/briefing/manifest.json", "{}"); + this.WriteDataFile("media-staging/operation/transcript.md", "temporary"); + this.WriteDataFile("databases/sqlite/rag-index.sqlite3", "the index while AI Studio writes to it"); + this.WriteDataFile("databases/vector_database/stores/store_1/segments/segment", "the store while AI Studio writes to it"); + + WriteFile(Path.Combine(this.ConfigDirectory, "settings.json"), "the settings on the disk, with the objects of the organization"); + WriteFile(Path.Combine(this.IndexCopiesDirectory, "databases", "sqlite", "rag-index.sqlite3"), "the copy of the index"); + WriteFile(Path.Combine(this.IndexCopiesDirectory, "databases", "vector_database", "stores", "store_1", "segments", "segment"), "the copy of the store"); + } + + [TearDown] + public void RemoveFolders() + { + if (Directory.Exists(this.testDirectory)) + Directory.Delete(this.testDirectory, true); + } + + [Test] + public async Task ABackupHoldsWhatBelongsIntoItAndNothingElse() + { + var archive = await this.WriteBackupAsync(this.IndexCopiesDirectory); + + Assert.That(archive.Keys, Is.EquivalentTo(new[] + { + "config/settings.json", + $"data/workspaces/{WORKSPACE_ID}/name", + $"data/workspaces/{WORKSPACE_ID}/{CHAT_ID}/thread.json", + $"data/workspaces/{WORKSPACE_ID}/{CHAT_ID}/attachments/transcripts/meeting-transcript-0001.md", + $"data/tempChats/{Guid.Empty}/thread.json", + "data/plugins/my-assistant/plugin.lua", + "data/plugins/my-assistant/.hidden.lua", + "data/tokenizers/openai-gpt-6/tokenizer.json", + "data/visualBriefings/briefing/manifest.json", + "data/databases/sqlite/rag-index.sqlite3", + "data/databases/vector_database/stores/store_1/segments/segment", + DataBackupFormat.MANIFEST_ENTRY_NAME, + })); + + Assert.Multiple(() => + { + Assert.That(archive["config/settings.json"], Is.EqualTo(SETTINGS_JSON), "The settings come from memory, without the objects of configuration plugins."); + Assert.That(archive["data/databases/sqlite/rag-index.sqlite3"], Is.EqualTo("the copy of the index")); + Assert.That(archive["data/databases/vector_database/stores/store_1/segments/segment"], Is.EqualTo("the copy of the store")); + }); + } + + [Test] + public async Task WithoutCopiesTheBackupLeavesTheIndexesOut() + { + var archive = await this.WriteBackupAsync(string.Empty); + + Assert.That(archive.Keys.Where(name => name.StartsWith("data/databases/", StringComparison.Ordinal)), Is.Empty, "The indexes in the data folder change while AI Studio runs."); + } + + [Test] + public async Task TheManifestComesLastAndListsEveryOtherFile() + { + await using var target = new MemoryStream(); + await DataBackupWriter.WriteAsync(target, this.CreateSources(this.IndexCopiesDirectory), CreateManifest(), null, CancellationToken.None); + target.Position = 0; + + using var zip = new ZipArchive(target, ZipArchiveMode.Read); + var manifestEntry = zip.Entries[^1]; + var manifest = await ReadManifestAsync(manifestEntry); + var otherEntries = zip.Entries.Take(zip.Entries.Count - 1).ToList(); + + Assert.That(manifestEntry.FullName, Is.EqualTo(DataBackupFormat.MANIFEST_ENTRY_NAME), "Without the manifest, a backup was not written completely."); + Assert.That(manifest.Entries.Select(entry => entry.Name), Is.EqualTo(otherEntries.Select(entry => entry.FullName))); + + foreach (var entry in otherEntries) + { + await using var stream = entry.Open(); + var hash = Convert.ToHexStringLower(await SHA256.HashDataAsync(stream)); + var listed = manifest.Entries.Single(listedEntry => listedEntry.Name == entry.FullName); + + Assert.Multiple(() => + { + Assert.That(listed.Sha256, Is.EqualTo(hash), $"The hash of '{entry.FullName}' lets a restore notice a damaged backup."); + Assert.That(listed.Size, Is.EqualTo(entry.Length)); + }); + } + } + + /// + /// The classic ZIP format stores sizes and positions in 32 bits, which ends at 4 GiB. The index + /// of large data sources gets there, so the backup has to switch to ZIP64 on its own: for the + /// size of the entry, and for the position of every entry behind it, the manifest among them. The file is sparse, so it takes no space, but the backup takes about + /// 4.5 GiB until the test ends. + /// + [Test] + [Explicit("Writes a backup of about 4.5 GiB, which takes about a minute and as much free space.")] + public async Task ABackupBeyondFourGibibytesStaysReadable() + { + const long LARGE_FILE_SIZE = 4_608L * 1024 * 1024; + const long FOUR_GIBIBYTES = 4L * 1024 * 1024 * 1024; + + // Stored without compression, like every video, so the backup grows beyond 4 GiB as well: + var largeFilePath = Path.Combine(this.DataDirectory, "visualBriefings", "large", "recording.mp4"); + Directory.CreateDirectory(Path.GetDirectoryName(largeFilePath)!); + await using (var largeFile = new FileStream(largeFilePath, FileMode.Create, FileAccess.Write)) + largeFile.SetLength(LARGE_FILE_SIZE); + + var archivePath = Path.Combine(this.testDirectory, "backup.mwbackup"); + await using (var target = new FileStream(archivePath, FileMode.Create, FileAccess.Write, FileShare.None, 64 * 1024, FileOptions.Asynchronous | FileOptions.SequentialScan)) + await DataBackupWriter.WriteAsync(target, this.CreateSources(string.Empty), CreateManifest(), null, CancellationToken.None); + + await using var archiveStream = File.OpenRead(archivePath); + using var zip = new ZipArchive(archiveStream, ZipArchiveMode.Read); + var largeEntry = zip.GetEntry("data/visualBriefings/large/recording.mp4")!; + var manifest = await ReadManifestAsync(zip.Entries[^1]); + var listed = manifest.Entries.Single(entry => entry.Name == largeEntry.FullName); + + string hash; + await using (var entryStream = largeEntry.Open()) + hash = Convert.ToHexStringLower(await SHA256.HashDataAsync(entryStream)); + + Assert.Multiple(() => + { + Assert.That(new FileInfo(archivePath).Length, Is.GreaterThan(FOUR_GIBIBYTES), "Only then do positions need ZIP64 as well."); + Assert.That(largeEntry.Length, Is.EqualTo(LARGE_FILE_SIZE)); + Assert.That(listed.Size, Is.EqualTo(LARGE_FILE_SIZE)); + Assert.That(listed.Sha256, Is.EqualTo(hash), "The large file comes back as it was."); + Assert.That(manifest.Entries, Has.Count.EqualTo(zip.Entries.Count - 1), "The manifest lies behind the large file, beyond 4 GiB, and is still found."); + }); + } + + private async Task> WriteBackupAsync(string indexCopiesDirectory) + { + await using var target = new MemoryStream(); + await DataBackupWriter.WriteAsync(target, this.CreateSources(indexCopiesDirectory), CreateManifest(), null, CancellationToken.None); + target.Position = 0; + + using var zip = new ZipArchive(target, ZipArchiveMode.Read); + var contents = new Dictionary(StringComparer.Ordinal); + foreach (var entry in zip.Entries) + { + using var reader = new StreamReader(entry.Open(), Encoding.UTF8); + contents[entry.FullName] = await reader.ReadToEndAsync(); + } + + return contents; + } + + private DataBackupSources CreateSources(string indexCopiesDirectory) => new(this.DataDirectory, this.ConfigDirectory, SETTINGS_JSON, indexCopiesDirectory, ["plugins/company-assistant/"]); + + private static DataBackupManifest CreateManifest() => new(DataBackupFormat.FORMAT_VERSION, "26.10.1", "V6", DateTimeOffset.UtcNow, "/data", "/config", true, []); + + private static async Task ReadManifestAsync(ZipArchiveEntry manifestEntry) + { + await using var stream = manifestEntry.Open(); + return (await JsonSerializer.DeserializeAsync(stream, DataBackupFormat.JSON_OPTIONS))!; + } + + private void WriteDataFile(string relativePath, string content) => WriteFile(Path.Combine(this.DataDirectory, relativePath.Replace('/', Path.DirectorySeparatorChar)), content); + + private static void WriteFile(string path, string content) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, content); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/DataBackup/DataRestoreFollowUpTests.cs b/app/Tests/Tools/DataBackup/DataRestoreFollowUpTests.cs new file mode 100644 index 000000000..221522a89 --- /dev/null +++ b/app/Tests/Tools/DataBackup/DataRestoreFollowUpTests.cs @@ -0,0 +1,155 @@ +using System.Text.Json; + +using AIStudio.Tools.DataBackup; + +using SettingsVersion = AIStudio.Settings.Version; + +namespace AIStudio.Tests.Tools.DataBackup; + +/// +/// Checks what the first start after a restore, a crashed backup, or a crashed preparation removes, +/// and what it must keep. +/// +/// +/// While the runtime has not finished a restore, its journal and the data it replaced are the only way +/// back. Removing them would lose the data of the user for good. +/// +[TestFixture] +public sealed class DataRestoreFollowUpTests +{ + private string testDirectory = string.Empty; + + private string DataDirectory => Path.Combine(this.testDirectory, "data"); + + private string ConfigDirectory => Path.Combine(this.testDirectory, "config"); + + [SetUp] + public void CreateFolders() => this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-restore-follow-up-{Guid.NewGuid():N}"); + + [TearDown] + public void RemoveFolders() + { + if (Directory.Exists(this.testDirectory)) + Directory.Delete(this.testDirectory, true); + } + + [Test] + public void WhatABackupOrAPreparationLeftHalfwayIsRemoved() + { + WriteFile(Path.Combine(this.DataDirectory, ".backup-staging", "0123456789abcdef", "store"), "copy"); + WriteFile(Path.Combine(this.DataDirectory, ".restore-pending", "workspaces", "chat", "thread.json"), "{}"); + WriteFile(Path.Combine(this.ConfigDirectory, ".restore-pending", "settings.json"), "{}"); + + DataRestoreFollowUp.RemoveLeftovers(this.DataDirectory, this.ConfigDirectory); + + Assert.Multiple(() => + { + Assert.That(Directory.Exists(Path.Combine(this.DataDirectory, ".backup-staging")), Is.False); + Assert.That(Directory.Exists(Path.Combine(this.DataDirectory, ".restore-pending")), Is.False); + Assert.That(Directory.Exists(Path.Combine(this.ConfigDirectory, ".restore-pending")), Is.False); + }); + } + + [Test] + public void ARestoreWhichWaitsForTheRestartIsKept() + { + WriteFile(Path.Combine(this.DataDirectory, ".restore-pending", "workspaces", "chat", "thread.json"), "{}"); + WriteFile(Path.Combine(this.DataDirectory, ".restore-pending", DataRestoreFormat.READY_FILE_NAME), "now"); + + DataRestoreFollowUp.RemoveLeftovers(this.DataDirectory, this.ConfigDirectory); + + Assert.Multiple(() => + { + Assert.That(DataRestoreFollowUp.IsRestorePending(this.DataDirectory), Is.True); + Assert.That(DataRestoreFollowUp.IsRestoreUnfinished(this.DataDirectory), Is.False); + Assert.That(File.Exists(Path.Combine(this.DataDirectory, ".restore-pending", "workspaces", "chat", "thread.json")), Is.True); + }); + } + + [Test] + public void ARestoreWhichTheRuntimeHasNotFinishedKeepsEverything() + { + WriteFile(Path.Combine(this.DataDirectory, ".restore-pending", DataRestoreFormat.READY_FILE_NAME), "now"); + WriteFile(Path.Combine(this.DataDirectory, ".restore-pending", DataRestoreFormat.JOURNAL_FILE_NAME), """{ "moves": [], "started": 0 }"""); + WriteFile(Path.Combine(this.DataDirectory, ".restore-previous", "workspaces", "chat", "thread.json"), "{}"); + var target = new DataRestoreTarget(this.DataDirectory, this.ConfigDirectory, "26.10.1", SettingsVersion.V6, []); + + DataRestoreFollowUp.RemoveLeftovers(this.DataDirectory, this.ConfigDirectory); + var removedReplacedData = DataRestoreFollowUp.RemoveReplacedData(this.DataDirectory, this.ConfigDirectory); + + Assert.Multiple(() => + { + Assert.That(DataRestoreFollowUp.IsRestoreUnfinished(this.DataDirectory), Is.True); + Assert.That(DataRestoreFollowUp.IsRestorePending(this.DataDirectory), Is.False, "The runtime moves it back rather than applying it."); + Assert.That(removedReplacedData, Is.False); + Assert.That(File.Exists(Path.Combine(this.DataDirectory, ".restore-previous", "workspaces", "chat", "thread.json")), Is.True, "These are the data of the user."); + Assert.That(() => DataRestorePreparer.Discard(target), Throws.InvalidOperationException); + Assert.That(File.Exists(Path.Combine(this.DataDirectory, ".restore-pending", DataRestoreFormat.JOURNAL_FILE_NAME)), Is.True); + }); + } + + [Test] + public void TheReplacedDataAreRemovedOnceNoRestoreWaits() + { + WriteFile(Path.Combine(this.DataDirectory, ".restore-previous", "workspaces", "chat", "thread.json"), "{}"); + WriteFile(Path.Combine(this.ConfigDirectory, ".restore-previous", "settings.json"), "{}"); + + var removedReplacedData = DataRestoreFollowUp.RemoveReplacedData(this.DataDirectory, this.ConfigDirectory); + + Assert.Multiple(() => + { + Assert.That(removedReplacedData, Is.True); + Assert.That(Directory.Exists(Path.Combine(this.DataDirectory, ".restore-previous")), Is.False); + Assert.That(Directory.Exists(Path.Combine(this.ConfigDirectory, ".restore-previous")), Is.False); + }); + } + + [Test] + public async Task TheReportOfTheRuntimeIsRead() + { + WriteFile(Path.Combine(this.DataDirectory, DataRestoreFormat.RESULT_FILE_NAME), """ + { + "outcome": "ROLLED_BACK", + "detail": "Could not move '.restore-pending/plugins' to 'plugins': access denied", + "prepared_at_utc": "2026-10-15T08:00:00.0000000+00:00", + "backup_created_at_utc": "2026-10-01T17:30:00.0000000+00:00", + "backup_app_version": "26.10.1", + "finished_at_utc": "2026-10-15T08:01:00.123456+00:00" + } + """); + + var report = await DataRestoreFollowUp.ReadReportAsync(this.DataDirectory, CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(report, Is.Not.Null); + Assert.That(report!.Outcome, Is.EqualTo(DataRestoreOutcome.ROLLED_BACK)); + Assert.That(report.BackupAppVersion, Is.EqualTo("26.10.1")); + Assert.That(report.Detail, Does.Contain("access denied")); + }); + } + + [Test] + public async Task WithoutARestoreThereIsNoReport() + { + Directory.CreateDirectory(this.DataDirectory); + + var report = await DataRestoreFollowUp.ReadReportAsync(this.DataDirectory, CancellationToken.None); + + Assert.That(report, Is.Null); + } + + [Test] + public void AReportWithAnUnknownOutcomeCannotBeRead() + { + WriteFile(Path.Combine(this.DataDirectory, DataRestoreFormat.RESULT_FILE_NAME), """{ "outcome": "SOMETHING_NEW", "detail": "", "prepared_at_utc": "", "backup_created_at_utc": "", "backup_app_version": "", "finished_at_utc": "" }"""); + + Assert.That(async () => await DataRestoreFollowUp.ReadReportAsync(this.DataDirectory, CancellationToken.None), Throws.InstanceOf()); + } + + private static void WriteFile(string path, string content) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, content); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/DataBackup/DataRestoreOpenPointsTests.cs b/app/Tests/Tools/DataBackup/DataRestoreOpenPointsTests.cs new file mode 100644 index 000000000..9a301aeed --- /dev/null +++ b/app/Tests/Tools/DataBackup/DataRestoreOpenPointsTests.cs @@ -0,0 +1,132 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools; +using AIStudio.Tools.DataBackup; +using AIStudio.Tools.ERIClient.DataModel; + +using ProviderSettings = AIStudio.Settings.Provider; + +namespace AIStudio.Tests.Tools.DataBackup; + +/// +/// Checks what the user hears to do after a restore. +/// +/// +/// A backup holds no secrets, so whatever of the user needs one lacks it on the new computer. The +/// objects of the organization come with their secrets from the configuration plugin, and naming them +/// would send the user hunting for keys they never had. +/// +[TestFixture] +public sealed class DataRestoreOpenPointsTests +{ + private static readonly Guid PLUGIN_ID = Guid.Parse("4f1c2b7a-8d3e-4a6b-9c5d-2e7f1a3b5c8d"); + + private string testDirectory = string.Empty; + + [SetUp] + public void CreateFolder() + { + this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-restore-open-points-{Guid.NewGuid():N}"); + Directory.CreateDirectory(this.testDirectory); + } + + [TearDown] + public void RemoveFolder() + { + if (Directory.Exists(this.testDirectory)) + Directory.Delete(this.testDirectory, true); + } + + [Test] + public async Task TheMissingSecretsOfTheUserAreOpenPoints() + { + var settings = new Data(); + settings.Providers.Add(new ProviderSettings(1, "user-provider", "My OpenAI", LLMProviders.OPEN_AI, new("gpt-5", "gpt-5"))); + settings.Providers.Add(new ProviderSettings(2, "stored-provider", "My Anthropic", LLMProviders.ANTHROPIC, new("claude", "claude"))); + settings.Providers.Add(new ProviderSettings(3, "plugin-provider", "OpenAI of the organization", LLMProviders.OPEN_AI, new("gpt-5", "gpt-5"), IsEnterpriseConfiguration: true, EnterpriseConfigurationPluginId: PLUGIN_ID)); + settings.EmbeddingProviders.Add(new EmbeddingProvider(1, "user-embedding", "My embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"))); + settings.TranscriptionProviders.Add(new TranscriptionProvider(1, "user-transcription", "My transcription", LLMProviders.OPEN_AI, new("whisper-1", "whisper-1"))); + settings.DataSources.Add(new DataSourceERI_V1 { Id = "token-server", Name = "My ERI server", AuthMethod = AuthMethod.TOKEN }); + settings.DataSources.Add(new DataSourceERI_V1 { Id = "open-server", Name = "Open ERI server", AuthMethod = AuthMethod.NONE }); + settings.DataSources.Add(new DataSourceERI_V1 { Id = "plugin-server", Name = "ERI server of the organization", AuthMethod = AuthMethod.TOKEN, IsEnterpriseConfiguration = true, EnterpriseConfigurationPluginId = PLUGIN_ID }); + settings.Mailboxes.Add(new DataSourceMailbox { Id = "user-mailbox", Name = "My mailbox", AuthMethod = MailboxAuthMethod.PASSWORD }); + + var openPoints = await DataRestoreOpenPoints.CollectAsync(settings, new StoredSecrets("My Anthropic"), []); + + Assert.That(openPoints, Is.EqualTo(new DataRestoreOpenPoint[] + { + new(DataRestoreOpenPointKind.PROVIDER_WITHOUT_API_KEY, "My OpenAI"), + new(DataRestoreOpenPointKind.EMBEDDING_PROVIDER_WITHOUT_API_KEY, "My embeddings"), + new(DataRestoreOpenPointKind.TRANSCRIPTION_PROVIDER_WITHOUT_API_KEY, "My transcription"), + new(DataRestoreOpenPointKind.DATA_SOURCE_WITHOUT_SECRET, "My ERI server"), + new(DataRestoreOpenPointKind.MAILBOX_WITHOUT_PASSWORD, "My mailbox"), + })); + } + + [Test] + public async Task TheFilesAndFoldersMissingOnThisComputerAreOpenPoints() + { + var missingFolder = Path.Combine(this.testDirectory, "missing"); + var missingFile = Path.Combine(this.testDirectory, "missing.txt"); + var settings = new Data(); + settings.DataSources.Add(new DataSourceLocalDirectory { Id = "missing-directory", Name = "Projects", Path = missingFolder }); + settings.DataSources.Add(new DataSourceLocalDirectory { Id = "existing-directory", Name = "Notes", Path = this.testDirectory }); + settings.DataSources.Add(new DataSourceLocalFile { Id = "missing-file", Name = "Handbook", FilePath = missingFile }); + settings.BatchProcessing.InputDirectory = missingFolder; + settings.BatchProcessing.OutputDirectory = this.testDirectory; + settings.BatchProcessing.PromptFilePath = missingFile; + settings.App.ExternalHttpCustomRootCertificatesEnabled = true; + settings.App.ExternalHttpCustomRootCertificateBundlePath = missingFile; + settings.ERI.ERIServers.Add(new DataERIServer { ServerName = "Wiki server", WriteToFilesystem = true, BaseDirectory = missingFolder }); + settings.ERI.ERIServers.Add(new DataERIServer { ServerName = "Draft server", WriteToFilesystem = false, BaseDirectory = missingFolder }); + + var openPoints = await DataRestoreOpenPoints.CollectAsync(settings, new StoredSecrets(), []); + + Assert.That(openPoints, Is.EqualTo(new DataRestoreOpenPoint[] + { + new(DataRestoreOpenPointKind.DATA_SOURCE_NOT_FOUND, "Projects"), + new(DataRestoreOpenPointKind.DATA_SOURCE_NOT_FOUND, "Handbook"), + new(DataRestoreOpenPointKind.BATCH_PROCESSING_PATH_NOT_FOUND, missingFolder), + new(DataRestoreOpenPointKind.BATCH_PROCESSING_PATH_NOT_FOUND, missingFile), + new(DataRestoreOpenPointKind.CERTIFICATE_BUNDLE_NOT_FOUND, missingFile), + new(DataRestoreOpenPointKind.ERI_SERVER_DIRECTORY_NOT_FOUND, "Wiki server"), + })); + } + + [Test] + public async Task AnUnusedCertificateBundleIsNoOpenPoint() + { + var settings = new Data(); + settings.App.ExternalHttpCustomRootCertificatesEnabled = false; + settings.App.ExternalHttpCustomRootCertificateBundlePath = Path.Combine(this.testDirectory, "missing.pem"); + + var openPoints = await DataRestoreOpenPoints.CollectAsync(settings, new StoredSecrets(), []); + + Assert.That(openPoints, Is.Empty); + } + + [Test] + public async Task TheMissingSecretsOfTheToolsAreOpenPoints() + { + DataRestoreToolSecret[] toolSecrets = + [ + new("web_search", "tavily.apiKey", "Web search", "Tavily API Key"), + new("web_search", "staan.apiKey", "Web search", "Staan API Key"), + ]; + + var openPoints = await DataRestoreOpenPoints.CollectAsync(new Data(), new StoredSecrets("staan.apiKey"), toolSecrets); + + Assert.That(openPoints, Is.EqualTo(new DataRestoreOpenPoint[] { new(DataRestoreOpenPointKind.TOOL_WITHOUT_SECRET, "Web search", "Tavily API Key") })); + } + + /// + /// A keyring which holds the secrets of the given names. + /// + private sealed class StoredSecrets(params string[] storedSecretNames) : IStoredSecrets + { + public Task HasAPIKeyAsync(ISecretId secretId, SecretStoreType storeType) => Task.FromResult(storedSecretNames.Contains(secretId.SecretName)); + + public Task HasSecretAsync(ISecretId secretId, SecretStoreType storeType) => Task.FromResult(storedSecretNames.Contains(secretId.SecretName)); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/DataBackup/DataRestorePreparerTests.cs b/app/Tests/Tools/DataBackup/DataRestorePreparerTests.cs new file mode 100644 index 000000000..166fc468d --- /dev/null +++ b/app/Tests/Tools/DataBackup/DataRestorePreparerTests.cs @@ -0,0 +1,171 @@ +using System.IO.Compression; +using System.Text.Json; + +using AIStudio.Tools.DataBackup; + +using SettingsVersion = AIStudio.Settings.Version; + +namespace AIStudio.Tests.Tools.DataBackup; + +/// +/// Checks how a restore is prepared for the next start. +/// +/// +/// The runtime replaces the data at the next start with whatever the preparation left behind, as soon +/// as it finds it marked as ready. A damaged backup or one this version cannot read must therefore +/// never get that far, and the plan has to name exactly what to replace and what to keep. +/// +[TestFixture] +public sealed class DataRestorePreparerTests +{ + private static readonly Guid WORKSPACE_ID = Guid.Parse("3f2b7c1e-9a4d-4e6b-8c2f-1d5e7a9b0c3d"); + + private static readonly Guid CHAT_ID = Guid.Parse("8a1c4e7f-2b5d-4f9a-b3c6-0e2d4f6a8b1c"); + + private static readonly DateTime CHAT_LAST_CHANGED = new(2026, 9, 1, 8, 30, 0, DateTimeKind.Local); + + private string testDirectory = string.Empty; + + private string ArchivePath => Path.Combine(this.testDirectory, "backup.mwbackup"); + + private DataRestoreTarget Target => new(Path.Combine(this.testDirectory, "target-data"), Path.Combine(this.testDirectory, "target-config"), "26.10.1", SettingsVersion.V6, ["plugins/company-assistant/"]); + + [SetUp] + public void CreateFolders() => this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-restore-preparer-{Guid.NewGuid():N}"); + + [TearDown] + public void RemoveFolders() + { + if (Directory.Exists(this.testDirectory)) + Directory.Delete(this.testDirectory, true); + } + + [Test] + public async Task ARestoreIsUnpackedNextToTheFoldersAndMarkedReadyLast() + { + await this.WriteBackupAsync("26.10.1", "V6"); + + var result = await DataRestorePreparer.PrepareAsync(this.ArchivePath, this.Target, null, CancellationToken.None); + var dataPending = Path.Combine(this.Target.DataDirectory, ".restore-pending"); + var threadPath = Path.Combine(dataPending, "workspaces", WORKSPACE_ID.ToString(), CHAT_ID.ToString(), "thread.json"); + var plan = JsonSerializer.Deserialize(await File.ReadAllTextAsync(Path.Combine(dataPending, DataRestoreFormat.PLAN_FILE_NAME)), DataBackupFormat.JSON_OPTIONS)!; + + Assert.Multiple(() => + { + Assert.That(result.Success, Is.True, result.Issue); + Assert.That(DataRestorePreparer.IsPrepared(this.Target), Is.True); + Assert.That(File.ReadAllText(threadPath), Is.EqualTo("{}")); + Assert.That(File.GetLastWriteTime(threadPath), Is.EqualTo(CHAT_LAST_CHANGED).Within(TimeSpan.FromSeconds(2)), "The workspaces sort their chats by it."); + Assert.That(File.ReadAllText(Path.Combine(dataPending, "plugins", "my-assistant", "plugin.lua")), Is.EqualTo("-- mine")); + Assert.That(File.ReadAllText(Path.Combine(this.Target.ConfigDirectory, ".restore-pending", "settings.json")), Does.Contain("V6"), "The settings go next to the configuration folder."); + + Assert.That(plan.Elements.Select(element => $"{element.Location}:{element.RelativePath}"), Is.EquivalentTo((string[])["CONFIG_DIRECTORY:settings.json", "DATA_DIRECTORY:workspaces/", "DATA_DIRECTORY:plugins/"]), "An element the backup holds nothing for, such as the tokenizers here, stays as it is."); + Assert.That(plan.Elements.Single(element => element.RelativePath == "plugins/").Keep, Is.SupersetOf((string[])["plugins/.internal/", "plugins/.config/", "plugins/.config-tests/", "plugins/company-assistant/"]), "AI Studio and the organization bring their plugins themselves."); + }); + } + + [Test] + public async Task ADamagedBackupLeavesNothingBehind() + { + await this.WriteBackupAsync("26.10.1", "V6", damageThread: true); + + var result = await DataRestorePreparer.PrepareAsync(this.ArchivePath, this.Target, null, CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(result.Success, Is.False); + Assert.That(result.Issue, Does.Contain("thread.json")); + Assert.That(Directory.Exists(Path.Combine(this.Target.DataDirectory, ".restore-pending")), Is.False, "The next start must find nothing to apply."); + Assert.That(Directory.Exists(Path.Combine(this.Target.ConfigDirectory, ".restore-pending")), Is.False); + }); + } + + [TestCase("27.1.0", "V6")] + [TestCase("26.10.1", "V7")] + public async Task ABackupOfANewerVersionIsRefused(string appVersion, string settingsVersion) + { + await this.WriteBackupAsync(appVersion, settingsVersion); + + var result = await DataRestorePreparer.PrepareAsync(this.ArchivePath, this.Target, null, CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(result.Success, Is.False, "Its settings or its index may have a form this version cannot read."); + Assert.That(DataRestorePreparer.IsPrepared(this.Target), Is.False); + }); + } + + [Test] + public async Task ADiscardedRestoreLeavesNothingBehind() + { + await this.WriteBackupAsync("26.10.1", "V6"); + await DataRestorePreparer.PrepareAsync(this.ArchivePath, this.Target, null, CancellationToken.None); + + DataRestorePreparer.Discard(this.Target); + + Assert.Multiple(() => + { + Assert.That(DataRestorePreparer.IsPrepared(this.Target), Is.False); + Assert.That(Directory.Exists(Path.Combine(this.Target.ConfigDirectory, ".restore-pending")), Is.False); + }); + } + + [Test] + public void DiscardingWithoutAPreparedRestoreChangesNothing() + { + Assert.That(() => DataRestorePreparer.Discard(this.Target), Throws.Nothing, "A preparation discards whatever an earlier one left, and mostly, there is nothing."); + } + + /// + /// Writes a backup of a data folder with a chat and a plugin of the user. + /// + /// The version of AI Studio the manifest names. + /// The version of the settings the manifest names. + /// Whether to change the chat thread after the manifest noted its hash. + private async Task WriteBackupAsync(string appVersion, string settingsVersion, bool damageThread = false) + { + var sourceDataDirectory = Path.Combine(this.testDirectory, "source-data"); + var threadPath = Path.Combine(sourceDataDirectory, "workspaces", WORKSPACE_ID.ToString(), CHAT_ID.ToString(), "thread.json"); + WriteFile(Path.Combine(sourceDataDirectory, "workspaces", WORKSPACE_ID.ToString(), "name"), "Projects"); + WriteFile(threadPath, "{}"); + File.SetLastWriteTime(threadPath, CHAT_LAST_CHANGED); + WriteFile(Path.Combine(sourceDataDirectory, "plugins", "my-assistant", "plugin.lua"), "-- mine"); + + var sources = new DataBackupSources(sourceDataDirectory, Path.Combine(this.testDirectory, "source-config"), """{ "Version": "V6" }""", string.Empty, []); + var manifest = new DataBackupManifest(DataBackupFormat.FORMAT_VERSION, appVersion, settingsVersion, DateTimeOffset.UtcNow, sourceDataDirectory, string.Empty, false, []); + await using (var archiveStream = File.Create(this.ArchivePath)) + await DataBackupWriter.WriteAsync(archiveStream, sources, manifest, null, CancellationToken.None); + + if (!damageThread) + return; + + // Rewrite the backup with another chat thread, but the manifest of the original one: + var damagedArchivePath = $"{this.ArchivePath}.damaged"; + await using (var originalStream = File.OpenRead(this.ArchivePath)) + using (var original = new ZipArchive(originalStream, ZipArchiveMode.Read)) + await using (var damagedStream = File.Create(damagedArchivePath)) + using (var damaged = new ZipArchive(damagedStream, ZipArchiveMode.Create)) + { + foreach (var entry in original.Entries) + { + await using var target = damaged.CreateEntry(entry.FullName).Open(); + if (entry.FullName.EndsWith("thread.json", StringComparison.Ordinal)) + { + await target.WriteAsync("{ }"u8.ToArray()); + continue; + } + + await using var source = entry.Open(); + await source.CopyToAsync(target); + } + } + + File.Move(damagedArchivePath, this.ArchivePath, overwrite: true); + } + + private static void WriteFile(string path, string content) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + File.WriteAllText(path, content); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/DataDirectoryRelativePathTests.cs b/app/Tests/Tools/DataDirectoryRelativePathTests.cs new file mode 100644 index 000000000..439bdad19 --- /dev/null +++ b/app/Tests/Tools/DataDirectoryRelativePathTests.cs @@ -0,0 +1,138 @@ +using System.Text.Json; + +using AIStudio.Chat; +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Tools; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks that the paths of files below the data directory are stored relative to it, and every other path as it is. +/// +/// +/// A backup of the data directory brings the stored paths to another computer, where the user name, +/// the OS, or the data directory itself may differ. An absolute path to a transcript or a tokenizer +/// copy leads nowhere there, so the transcript drops out of its chat, and the token count turns into +/// an estimate. Relative to the data directory, the path still names the same file. +/// +[TestFixture] +[NonParallelizable] +public sealed class DataDirectoryRelativePathTests +{ + private static readonly DateTimeOffset START = new(2026, 10, 14, 9, 0, 0, TimeSpan.Zero); + + private static readonly Guid WORKSPACE_ID = Guid.Parse("3f2b7c1e-9a4d-4e6b-8c2f-1d5e7a9b0c3d"); + + private static readonly Guid CHAT_ID = Guid.Parse("8a1c4e7f-2b5d-4f9a-b3c6-0e2d4f6a8b1c"); + + private static readonly string ELSEWHERE = Path.Combine(Path.GetTempPath(), "ai-studio-documents-elsewhere"); + + private string? previousDataDirectory; + private string dataDirectory = string.Empty; + + [SetUp] + public void PrepareDataDirectory() + { + // + // The data directory is static state of the whole application, which is why this fixture + // does not run alongside others. Nothing is written to it: storing a path only asks where + // the path lies. + // + this.previousDataDirectory = SettingsManager.DataDirectory; + this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-relative-paths-{Guid.NewGuid():N}"); + SettingsManager.DataDirectory = this.dataDirectory; + } + + [TearDown] + public void RestoreDataDirectory() => SettingsManager.DataDirectory = this.previousDataDirectory; + + [Test] + public void AFileBelowTheDataDirectoryIsStoredRelative() + { + var transcriptPath = Path.Combine(this.dataDirectory, "workspaces", WORKSPACE_ID.ToString(), CHAT_ID.ToString(), "attachments", "transcripts", "meeting-transcript-0001.md"); + var picturePath = Path.Combine(this.dataDirectory, "plugins", "templates", "chart.png"); + var documentPath = Path.Combine(ELSEWHERE, "report.pdf"); + var message = new ContentText + { + Text = "Summarize the meeting", + FileAttachments = + [ + new ManagedTranscriptAttachment("meeting-transcript-0001.md", transcriptPath, 512, "meeting.mp3", false), + new FileAttachmentImage("chart.png", picturePath, 2048), + new FileAttachment(FileAttachmentType.DOCUMENT, "report.pdf", documentPath, 1024), + ], + }; + var chat = new ChatThread + { + WorkspaceId = WORKSPACE_ID, + ChatId = CHAT_ID, + Name = "Meeting notes", + Blocks = [new ContentBlock { Time = START, ContentType = ContentType.TEXT, Content = message, Role = ChatRole.USER }], + }; + + var json = JsonSerializer.Serialize(chat, WorkspaceBehaviour.JSON_OPTIONS); + var attachments = ((ContentText)JsonSerializer.Deserialize(json, WorkspaceBehaviour.JSON_OPTIONS)!.Blocks.Single().Content!).FileAttachments; + + Assert.Multiple(() => + { + Assert.That(json, Does.Contain($"\"workspaces/{WORKSPACE_ID}/{CHAT_ID}/attachments/transcripts/meeting-transcript-0001.md\""), "Forward slashes, so that every OS reads the path."); + Assert.That(json, Does.Not.Contain(JsonEncodedText.Encode(this.dataDirectory).ToString()), "Not a single path may name the data directory of this computer."); + Assert.That(json, Does.Contain(JsonEncodedText.Encode(documentPath).ToString()), "A file of the user stays where it is, so its path stays absolute."); + Assert.That(attachments.Select(attachment => attachment.FilePath), Is.EqualTo(new[] { transcriptPath, picturePath, documentPath }), "In memory, every path is absolute again."); + Assert.That(attachments[0], Is.TypeOf(), "A transcript has to stay a transcript, or deleting the chat leaves its file behind."); + Assert.That(((FileAttachmentImage)attachments[1]).Source, Is.EqualTo(picturePath), "The model gets the image from its source, which has to follow the path."); + }); + } + + [Test] + public void ATokenizerCopyIsStoredRelative() + { + var tokenizerPath = Path.Combine(this.dataDirectory, "tokenizers", "openai-gpt-6", "tokenizer.json"); + var provider = new AIStudio.Settings.Provider(1, "provider", "Chat provider", LLMProviders.OPEN_AI, new("gpt-6", "gpt-6"), TokenizerPath: tokenizerPath); + var embeddingProvider = new EmbeddingProvider(1, "embedding", "Embedding provider", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"), TokenizerPath: tokenizerPath); + + var providerJson = JsonSerializer.Serialize(provider, SettingsManager.JSON_OPTIONS); + var embeddingProviderJson = JsonSerializer.Serialize(embeddingProvider, SettingsManager.JSON_OPTIONS); + + Assert.Multiple(() => + { + Assert.That(providerJson, Does.Contain("\"tokenizers/openai-gpt-6/tokenizer.json\"")); + Assert.That(embeddingProviderJson, Does.Contain("\"tokenizers/openai-gpt-6/tokenizer.json\"")); + Assert.That(JsonSerializer.Deserialize(providerJson, SettingsManager.JSON_OPTIONS)!.TokenizerPath, Is.EqualTo(tokenizerPath)); + Assert.That(JsonSerializer.Deserialize(embeddingProviderJson, SettingsManager.JSON_OPTIONS)!.TokenizerPath, Is.EqualTo(tokenizerPath)); + }); + } + + [Test] + public void AnAbsolutePathIsReadAsItIs() + { + // Stored before AI Studio stored such paths relative: + var transcriptPath = Path.Combine(this.dataDirectory, "tempChats", CHAT_ID.ToString(), "attachments", "transcripts", "call-transcript-0001.md"); + var documentPath = Path.Combine(ELSEWHERE, "notes.txt"); + + Assert.Multiple(() => + { + Assert.That(DataDirectoryRelativePathConverter.ToAbsolute(transcriptPath, this.dataDirectory), Is.EqualTo(transcriptPath), "On this computer, the path still names the transcript."); + Assert.That(DataDirectoryRelativePathConverter.ToAbsolute(documentPath, this.dataDirectory), Is.EqualTo(documentPath)); + }); + } + + [Test] + public void ARelativePathLeadingOutOfTheDataDirectoryIsNoPath() + { + Assert.That(DataDirectoryRelativePathConverter.ToAbsolute("workspaces/../../notes.txt", this.dataDirectory), Is.Empty, "AI Studio never writes such a path, so it must not name a file anywhere else."); + } + + [Test] + public void WithoutADataDirectoryEveryPathStaysAsItIs() + { + var transcriptPath = Path.Combine(this.dataDirectory, "tempChats", CHAT_ID.ToString(), "attachments", "transcripts", "call-transcript-0001.md"); + + Assert.Multiple(() => + { + Assert.That(DataDirectoryRelativePathConverter.ToStored(transcriptPath, null), Is.EqualTo(transcriptPath)); + Assert.That(DataDirectoryRelativePathConverter.ToAbsolute("tokenizers/openai-gpt-6/tokenizer.json", null), Is.EqualTo("tokenizers/openai-gpt-6/tokenizer.json")); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/Databases/IndexStoreBackupTests.cs b/app/Tests/Tools/Databases/IndexStoreBackupTests.cs new file mode 100644 index 000000000..6d469e6d0 --- /dev/null +++ b/app/Tests/Tools/Databases/IndexStoreBackupTests.cs @@ -0,0 +1,101 @@ +using AIStudio.Tools.Databases.IndexStore; +using AIStudio.Tools.Services.Indexing; + +using Microsoft.Data.Sqlite; + +namespace AIStudio.Tests.Tools.Databases; + +/// +/// Checks the copy of the index which a backup takes along. +/// +/// +/// After a restore, the copy is the index: what it lacks, AI Studio has to embed anew, at the cost of +/// the user's time and, with a cloud provider, money. The full-text search has to work from it right +/// away as well, since it lives in a table of its own. +/// +[TestFixture] +public sealed class IndexStoreBackupTests +{ + private const string DATA_SOURCE_ID = "6f1d6a4e-6a5e-4c62-9a4f-0f2d2c8b7a11"; + + private const string FILE_PATH = "/tmp/test-data/report.md"; + + private static readonly CancellationToken TOKEN = CancellationToken.None; + + [Test] + public async Task TheCopyHoldsTheIndexAndItsFullTextSearch() + { + await using var store = await TemporaryIndexStore.CreateAsync(); + await StoreReportAsync(store); + var targetDirectory = CreateTargetDirectory(); + + try + { + var copyPath = await store.Client.BackupToDirectoryAsync(targetDirectory, TOKEN); + var copy = new SqliteIndexStoreClientImplementation("SQLite", copyPath, targetDirectory, string.Empty); + + var manifest = await copy.GetManifestAsync(DATA_SOURCE_ID, TOKEN); + var found = await copy.SearchChunksAsync(DATA_SOURCE_ID, "quarterly", 10, TOKEN); + + Assert.Multiple(() => + { + Assert.That(Path.GetFileName(copyPath), Is.EqualTo("rag-index.sqlite3"), "A restore puts the copy where the index belongs, under the same name."); + Assert.That(manifest.Files.Keys, Is.EqualTo(new[] { FILE_PATH }), "Without the fingerprint, the file would be embedded anew."); + Assert.That(found.Select(result => result.ChunkText), Is.EqualTo(new[] { "The quarterly report is due." })); + }); + } + finally + { + RemoveTargetDirectory(targetDirectory); + } + } + + [Test] + public async Task ACopyWrittenBeforeIsReplaced() + { + await using var store = await TemporaryIndexStore.CreateAsync(); + var targetDirectory = CreateTargetDirectory(); + + try + { + await store.Client.BackupToDirectoryAsync(targetDirectory, TOKEN); + await StoreReportAsync(store); + var copyPath = await store.Client.BackupToDirectoryAsync(targetDirectory, TOKEN); + + var copy = new SqliteIndexStoreClientImplementation("SQLite", copyPath, targetDirectory, string.Empty); + Assert.That(await copy.GetTotalChunkCountAsync(TOKEN), Is.EqualTo(1)); + } + finally + { + RemoveTargetDirectory(targetDirectory); + } + } + + [Test] + public void WithoutAnIndexThereIsNothingToCopy() + { + var client = new NoIndexStoreClient("SQLite", "The native library could not be loaded."); + + Assert.That(async () => await client.BackupToDirectoryAsync(CreateTargetDirectory(), TOKEN), Throws.InvalidOperationException, "A backup without the index would take along vector stores which belong to nothing."); + } + + private static async Task StoreReportAsync(TemporaryIndexStore store) + { + var parentFileId = IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, FILE_PATH); + var now = DateTimeOffset.UtcNow; + + await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "LOCAL_DIRECTORY", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", string.Empty, 3, TOKEN); + await store.Client.UpsertFileAsync(DATA_SOURCE_ID, new EmbeddingStateFile(parentFileId, FILE_PATH, "report.md", "report.md", "md", FILE_PATH, 64, now, now, now, 1), TOKEN); + await store.Client.UpsertChunksAsync(DATA_SOURCE_ID, [new EmbeddingStateChunk(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, FILE_PATH, 0), parentFileId, null, 0, "The quarterly report is due.", now)], TOKEN); + } + + private static string CreateTargetDirectory() => Path.Combine(Path.GetTempPath(), $"ai-studio-index-backup-{Guid.NewGuid():N}"); + + private static void RemoveTargetDirectory(string targetDirectory) + { + // The client which opened the copy pools its connections, which would keep the file open on Windows: + SqliteConnection.ClearAllPools(); + if (Directory.Exists(targetDirectory)) + Directory.Delete(targetDirectory, true); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/FirstPluginLoadTests.cs b/app/Tests/Tools/FirstPluginLoadTests.cs new file mode 100644 index 000000000..1df96d690 --- /dev/null +++ b/app/Tests/Tools/FirstPluginLoadTests.cs @@ -0,0 +1,50 @@ +using AIStudio.Tools.Services; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks that indexing waits for the plugins before its first run, but never for good. +/// +/// +/// The embedding providers of an organization come with its configuration plugin. Right after a +/// backup was restored, the settings do not hold them yet, and a first run which starts before the +/// plugins were loaded finds the data sources of the user without their embedding provider. Should +/// loading fail, though, waiting for it must not keep every data source from being indexed. +/// +[TestFixture] +public sealed class FirstPluginLoadTests +{ + [Test] + public async Task TheFirstRunWaitsUntilThePluginsWereLoaded() + { + var firstLoad = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var wait = DataSourceEmbeddingService.WaitForFirstPluginLoadAsync(firstLoad.Task, TimeSpan.FromMinutes(1), CancellationToken.None); + + await Task.Delay(50); + Assert.That(wait.IsCompleted, Is.False, "The plugins are still loading."); + + firstLoad.SetResult(); + Assert.That(await wait, Is.True); + } + + [Test] + public async Task TheFirstRunStartsAnywayWhenLoadingThePluginsTakesTooLong() + { + var neverLoaded = new TaskCompletionSource(); + + var loadedInTime = await DataSourceEmbeddingService.WaitForFirstPluginLoadAsync(neverLoaded.Task, TimeSpan.FromMilliseconds(50), CancellationToken.None); + + Assert.That(loadedInTime, Is.False, "Loading may fail for good, and the data sources must be indexed all the same."); + } + + [Test] + public void StoppingAIStudioEndsTheWait() + { + var neverLoaded = new TaskCompletionSource(); + using var stopping = new CancellationTokenSource(); + stopping.Cancel(); + + var waiting = DataSourceEmbeddingService.WaitForFirstPluginLoadAsync(neverLoaded.Task, TimeSpan.FromMinutes(1), stopping.Token); + Assert.That(async () => await waiting, Throws.InstanceOf()); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/IndexingPausesTests.cs b/app/Tests/Tools/IndexingPausesTests.cs new file mode 100644 index 000000000..ec4166a6a --- /dev/null +++ b/app/Tests/Tools/IndexingPausesTests.cs @@ -0,0 +1,108 @@ +using AIStudio.Tools.Services; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks the pauses of the indexing, which a backup takes while it copies the indexes. +/// +/// +/// A run which wrote to the stores while they are copied would leave a backup whose two stores do +/// not fit together. Once the backup is done, though, every data source whose run the pause stopped +/// has to carry on, for the same reason it ran: a retry the user asked for is the only run which +/// signs in to a mailbox again after its server refused a sign-in. +/// +[TestFixture] +public sealed class IndexingPausesTests +{ + [Test] + public async Task ARunWaitsUntilThePauseEnded() + { + var pauses = new IndexingPauses(); + pauses.Begin(); + + var waiting = pauses.WaitWhilePausedAsync(CancellationToken.None); + await Task.Delay(50); + Assert.That(waiting.IsCompleted, Is.False, "The backup still copies the indexes."); + + pauses.End(); + await waiting; + } + + [Test] + public void WithoutAPauseARunStartsRightAway() + { + var pauses = new IndexingPauses(); + + Assert.Multiple(() => + { + Assert.That(pauses.IsPaused, Is.False); + Assert.That(pauses.WaitWhilePausedAsync(CancellationToken.None).IsCompleted, Is.True); + }); + } + + [Test] + public void TheIndexingCountsAsPausedUntilTheLastPauseEnded() + { + var pauses = new IndexingPauses(); + pauses.Begin(); + pauses.Begin(); + pauses.End(); + + Assert.That(pauses.IsPaused, Is.True, "Deleting a data source would discard its index while a backup copies it."); + + pauses.End(); + Assert.That(pauses.IsPaused, Is.False); + } + + [Test] + public async Task TheIndexingGoesOnOnceTheLastPauseEnded() + { + var pauses = new IndexingPauses(); + pauses.Begin(); + pauses.Begin(); + pauses.RecordStoppedRun("mailbox", DataSourceEmbeddingRefreshMode.MANUAL_RETRY); + + var stoppedRunsAfterTheFirstEnd = pauses.End(); + var waiting = pauses.WaitWhilePausedAsync(CancellationToken.None); + await Task.Delay(50); + + Assert.Multiple(() => + { + Assert.That(stoppedRunsAfterTheFirstEnd, Is.Empty, "The other pause still holds the indexing back, so nothing may start yet."); + Assert.That(waiting.IsCompleted, Is.False); + }); + + var stoppedRuns = pauses.End(); + await waiting; + + Assert.That(stoppedRuns, Is.EqualTo(new[] { new DataSourceRunRequest("mailbox", DataSourceEmbeddingRefreshMode.MANUAL_RETRY) }), "The retry the user asked for stays a retry."); + } + + [Test] + public void AStoppedRunIsQueuedAgainOnlyOnce() + { + var pauses = new IndexingPauses(); + pauses.Begin(); + pauses.RecordStoppedRun("folder", DataSourceEmbeddingRefreshMode.WATCHER_HASH_CHECK); + pauses.RecordStoppedRun("folder", DataSourceEmbeddingRefreshMode.HASH_CHECK); + + Assert.Multiple(() => + { + Assert.That(pauses.End(), Has.Count.EqualTo(1)); + Assert.That(pauses.End(), Is.Empty, "Ending a pause which does not exist changes nothing."); + }); + } + + [Test] + public void AStoppedRunStopsWaiting() + { + var pauses = new IndexingPauses(); + pauses.Begin(); + using var stopRun = new CancellationTokenSource(); + + var waiting = pauses.WaitWhilePausedAsync(stopRun.Token); + stopRun.Cancel(); + + Assert.That(async () => await waiting, Throws.InstanceOf(), "Deleting a data source ends the wait of its run, as it does in front of the gate."); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/PluginSystem/LeftOverConfigurationObjectsTests.cs b/app/Tests/Tools/PluginSystem/LeftOverConfigurationObjectsTests.cs index 41626a6cd..0a15fc0cf 100644 --- a/app/Tests/Tools/PluginSystem/LeftOverConfigurationObjectsTests.cs +++ b/app/Tests/Tools/PluginSystem/LeftOverConfigurationObjectsTests.cs @@ -60,7 +60,7 @@ public sealed class LeftOverConfigurationObjectsTests [Test] public async Task TheObjectsOfAPluginWhichDidNotStartStay() { - var wasChanged = await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, [new AvailablePlugin()], new HashSet(), new HashSet { PLUGIN_ID }, []); + var wasChanged = await PluginConfigurationObjectDescriptors.CHAT_TEMPLATE.CleanLeftOverAsync([new AvailablePlugin()], new HashSet(), new HashSet { PLUGIN_ID }, []); Assert.Multiple(() => { @@ -72,7 +72,7 @@ public sealed class LeftOverConfigurationObjectsTests [Test] public async Task TheObjectsOfAPluginWhichStartedWithoutThemAreRemoved() { - var wasChanged = await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, [new AvailablePlugin()], new HashSet(), new HashSet(), []); + var wasChanged = await PluginConfigurationObjectDescriptors.CHAT_TEMPLATE.CleanLeftOverAsync([new AvailablePlugin()], new HashSet(), new HashSet(), []); Assert.Multiple(() => { diff --git a/app/Tests/Tools/PluginSystem/ManagedTokenizerStateTests.cs b/app/Tests/Tools/PluginSystem/ManagedTokenizerStateTests.cs index 45e5a965d..1a7bfc770 100644 --- a/app/Tests/Tools/PluginSystem/ManagedTokenizerStateTests.cs +++ b/app/Tests/Tools/PluginSystem/ManagedTokenizerStateTests.cs @@ -162,8 +162,8 @@ public sealed class ManagedTokenizerStateTests throw new InvalidOperationException("The configuration of this test is not a Lua table."); var configObjects = new List(); - PluginConfigurationObject.TryParse(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, x => x.NextProviderNum, mainTable, PLUGIN_ID, ref configObjects, dryRun: false, PLUGIN_PATH); - PluginConfigurationObject.TryParse(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, x => x.NextEmbeddingNum, mainTable, PLUGIN_ID, ref configObjects, dryRun: false, PLUGIN_PATH); + PluginConfigurationObjectDescriptors.LLM_PROVIDER.TryParse(mainTable, PLUGIN_ID, configObjects, dryRun: false, PLUGIN_PATH); + PluginConfigurationObjectDescriptors.EMBEDDING_PROVIDER.TryParse(mainTable, PLUGIN_ID, configObjects, dryRun: false, PLUGIN_PATH); return configObjects; } diff --git a/app/Tests/Tools/PluginSystem/PluginConfigurationObjectDescriptorsTests.cs b/app/Tests/Tools/PluginSystem/PluginConfigurationObjectDescriptorsTests.cs new file mode 100644 index 000000000..ef9dcbb4b --- /dev/null +++ b/app/Tests/Tools/PluginSystem/PluginConfigurationObjectDescriptorsTests.cs @@ -0,0 +1,28 @@ +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tests.Tools.PluginSystem; + +/// +/// Checks that every type of configuration object has its descriptor. +/// +/// +/// Reading the objects from a configuration plugin and removing the left-over ones go through the +/// descriptors. A type without one would be read by neither: a configuration plugin could define such +/// objects without effect, and the ones a removed plugin left behind would stay in the settings. +/// +[TestFixture] +public sealed class PluginConfigurationObjectDescriptorsTests +{ + [Test] + public void EveryTypeHasExactlyOneDescriptor() + { + var describedTypes = PluginConfigurationObjectDescriptors.ALL.Select(descriptor => descriptor.Type).ToList(); + var configurableTypes = Enum.GetValues().Where(type => type is not PluginConfigurationObjectType.NONE and not PluginConfigurationObjectType.UNKNOWN); + + Assert.Multiple(() => + { + Assert.That(describedTypes, Is.EquivalentTo(configurableTypes)); + Assert.That(describedTypes, Is.Unique, "A type described twice would be read and cleaned up twice."); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/RelativeDataPathsMigrationTests.cs b/app/Tests/Tools/RelativeDataPathsMigrationTests.cs new file mode 100644 index 000000000..0c3dbb11b --- /dev/null +++ b/app/Tests/Tools/RelativeDataPathsMigrationTests.cs @@ -0,0 +1,125 @@ +using System.Text.Json; + +using AIStudio.Chat; +using AIStudio.Settings; +using AIStudio.Tools; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks that a chat stored before AI Studio stored paths relative gets them once, and nothing else changes. +/// +/// +/// A chat which is never stored again would keep its absolute paths, and a backup of the data +/// directory would bring them to a computer where they lead nowhere. Writing the chat once more +/// must not look like a change, though: the workspaces sort their chats by the time of the last +/// change, and the maintenance of the temporary chats deletes them by it. +/// +[TestFixture] +[NonParallelizable] +public sealed class RelativeDataPathsMigrationTests +{ + private static readonly DateTime LAST_EDIT_TIME = new(2026, 9, 1, 8, 30, 0, DateTimeKind.Utc); + + private string? previousDataDirectory; + private string testDirectory = string.Empty; + + [SetUp] + public void PrepareDataDirectory() + { + // + // The data directory is static state of the whole application, which is why this fixture + // does not run alongside others. The workspace behaviour keeps the roots of the workspaces + // and of the temporary chats fixed, as the app never changes its data directory, so it gets + // told about the one of this test separately. + // + this.previousDataDirectory = SettingsManager.DataDirectory; + this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-relative-paths-migration-{Guid.NewGuid():N}"); + Directory.CreateDirectory(this.testDirectory); + SettingsManager.DataDirectory = this.testDirectory; + WorkspaceBehaviour.TestDataDirectory = this.testDirectory; + } + + [TearDown] + public void RemoveDataDirectory() + { + SettingsManager.DataDirectory = this.previousDataDirectory; + WorkspaceBehaviour.TestDataDirectory = null; + + try + { + Directory.Delete(this.testDirectory, true); + } + catch (IOException) + { + // A temporary directory we could not remove says nothing about the code under test. + } + } + + [Test] + public async Task EveryStoredChatGetsRelativePathsAndKeepsItsTime() + { + var workspaceChat = this.StoreChatAsBefore(Guid.NewGuid(), Guid.NewGuid()); + var temporaryChat = this.StoreChatAsBefore(Guid.Empty, Guid.NewGuid()); + + Assert.That(await WorkspaceBehaviour.StoreChatsWithRelativePathsAsync(CancellationToken.None), Is.True, "Nothing kept a chat busy, so every chat has to be done."); + + foreach (var chat in new[] { workspaceChat, temporaryChat }) + { + var json = await File.ReadAllTextAsync(chat.ThreadPath); + var stored = JsonSerializer.Deserialize(json, WorkspaceBehaviour.JSON_OPTIONS)!; + + Assert.Multiple(() => + { + Assert.That(json, Does.Not.Contain(JsonEncodedText.Encode(this.testDirectory).ToString()), "Not a single path may name the data directory of this computer anymore."); + Assert.That(stored.Draft.MediaTranscripts.Single().FilePath, Is.EqualTo(chat.TranscriptPath), "The draft still finds its transcript."); + Assert.That(((ContentText)stored.Blocks.Single().Content!).FileAttachments.Single().FilePath, Is.EqualTo(chat.TranscriptPath), "The sent message still finds its transcript."); + Assert.That(File.GetLastWriteTimeUtc(chat.ThreadPath), Is.EqualTo(LAST_EDIT_TIME), "Writing the chat once more is no change of the chat."); + }); + } + } + + /// + /// Stores a chat with a transcript in a sent message and in its draft, with absolute paths as AI Studio stored them before. + /// + private StoredChat StoreChatAsBefore(Guid workspaceId, Guid chatId) + { + var chatDirectory = WorkspaceBehaviour.GetChatDirectory(workspaceId, chatId); + var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts"); + Directory.CreateDirectory(transcriptDirectory); + + var transcriptPath = Path.Combine(transcriptDirectory, "meeting-transcript-0001.md"); + File.WriteAllText(transcriptPath, $"# Transcription: meeting.mp4{Environment.NewLine}{Environment.NewLine}Hello and welcome."); + var transcript = new ManagedTranscriptAttachment("meeting-transcript-0001.md", transcriptPath, new FileInfo(transcriptPath).Length, "meeting.mp4", false); + + var chat = new ChatThread + { + WorkspaceId = workspaceId, + ChatId = chatId, + Name = "Meeting notes", + Blocks = + [ + new ContentBlock + { + Time = LAST_EDIT_TIME, + ContentType = ContentType.TEXT, + Content = new ContentText { Text = "Summarize the meeting", FileAttachments = [transcript] }, + Role = ChatRole.USER, + }, + ], + }; + chat.Draft.AddAttachments([transcript]); + + // Without a known data directory, every path is stored as it is, i.e., absolute: + SettingsManager.DataDirectory = null; + var json = JsonSerializer.Serialize(chat, WorkspaceBehaviour.JSON_OPTIONS); + SettingsManager.DataDirectory = this.testDirectory; + + var threadPath = Path.Combine(chatDirectory, "thread.json"); + File.WriteAllText(threadPath, json); + File.SetLastWriteTimeUtc(threadPath, LAST_EDIT_TIME); + return new(threadPath, transcriptPath); + } + + private sealed record StoredChat(string ThreadPath, string TranscriptPath); +} \ No newline at end of file diff --git a/app/Tests/Tools/UnreachableDataSourceTests.cs b/app/Tests/Tools/UnreachableDataSourceTests.cs new file mode 100644 index 000000000..0521c2205 --- /dev/null +++ b/app/Tests/Tools/UnreachableDataSourceTests.cs @@ -0,0 +1,209 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools; +using AIStudio.Tools.Databases.IndexStore; +using AIStudio.Tools.Databases.VectorStore; +using AIStudio.Tools.Security; +using AIStudio.Tools.Services; +using AIStudio.Tools.Services.Indexing; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks that a local data source keeps its index while its file or one of its folders cannot be reached. +/// +/// +/// A drive which is not connected, or a network share while the VPN is down, looks like an empty +/// folder to anyone who only lists files. Read that way, every embedding of the data source was +/// removed, and once the folder was back, all of it was embedded again, which takes hours for a +/// large folder. The vector store of these tests refuses every removal, so a run which removes +/// anything fails right away. +/// +[TestFixture] +public sealed class UnreachableDataSourceTests +{ + private const string DATA_SOURCE_ID = "3c9e2f1a-7b4d-4e8a-9f6c-1d2e3f4a5b60"; + + private const string EMBEDDING_ID = "8a7b6c5d-4e3f-4a1b-9c8d-7e6f5a4b3c21"; + + private const string STORED_SOURCE_HASH = "STORED-SOURCE-HASH"; + + private string testRoot = string.Empty; + + private readonly List lockedDirectories = []; + + [SetUp] + public void CreateTestRoot() + { + this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-unreachable-{Guid.NewGuid():N}"); + this.lockedDirectories.Clear(); + } + + [TearDown] + public void DeleteTestRoot() + { + if (!OperatingSystem.IsWindows()) + foreach (var directory in this.lockedDirectories) + File.SetUnixFileMode(directory, UnixFileMode.UserRead | UnixFileMode.UserWrite | UnixFileMode.UserExecute); + + if (Directory.Exists(this.testRoot)) + Directory.Delete(this.testRoot, true); + } + + [Test] + public async Task AMissingFolderKeepsItsIndex() + { + var folder = Path.Combine(this.testRoot, "documents"); + var (outcome, statuses, manifest) = await RunAsync(Folder(folder), Path.Combine(folder, "report.pdf"), Path.Combine(folder, "notes.docx")); + + AssertTheIndexWasKept(outcome, statuses, manifest, 2, folder); + } + + [Test] + public async Task AMissingFileKeepsItsIndex() + { + var file = Path.Combine(this.testRoot, "report.pdf"); + var dataSource = new DataSourceLocalFile + { + Num = 1, + Id = DATA_SOURCE_ID, + Name = "Test file", + Type = DataSourceType.LOCAL_FILE, + EmbeddingId = EMBEDDING_ID, + ConfidenceLevel = ConfidenceLevel.LOW, + FilePath = file, + }; + + var (outcome, statuses, manifest) = await RunAsync(dataSource, file); + + AssertTheIndexWasKept(outcome, statuses, manifest, 1, file); + } + + [Test] + public async Task AFolderWhichCannotBeOpenedKeepsItsIndex() + { + var folder = Path.Combine(this.testRoot, "documents"); + Directory.CreateDirectory(folder); + this.TakeAwayTheRightToRead(folder); + + var (outcome, statuses, manifest) = await RunAsync(Folder(folder), Path.Combine(folder, "report.pdf")); + + AssertTheIndexWasKept(outcome, statuses, manifest, 1, folder); + } + + [Test] + public async Task TheFilesOfASubfolderWhichCannotBeOpenedKeepTheirIndex() + { + var folder = Path.Combine(this.testRoot, "documents"); + var subfolder = Path.Combine(folder, "contracts"); + var fileInSubfolder = Path.Combine(subfolder, "contract.pdf"); + Directory.CreateDirectory(subfolder); + this.TakeAwayTheRightToRead(subfolder); + + var (outcome, statuses, manifest) = await RunAsync(Folder(folder), fileInSubfolder); + + Assert.Multiple(() => + { + Assert.That(outcome, Is.EqualTo(IndexedRunOutcome.DONE)); + Assert.That(manifest.Files.Keys, Is.EquivalentTo([fileInSubfolder]), "The file is out of reach, not gone. Once the subfolder can be read again, it is still indexed."); + Assert.That(statuses[^1].Failures.Select(failure => failure.DocumentKey), Has.Member(subfolder), "The user has to learn which subfolder could not be read."); + }); + } + + [Test] + public void OnlyFilesWhichAreGoneAreRemoved() + { + var folder = Path.Combine(this.testRoot, "documents"); + var found = Path.Combine(folder, "found.pdf"); + var deleted = Path.Combine(folder, "deleted.pdf"); + var subfolder = Path.Combine(folder, "contracts"); + var inSubfolder = Path.Combine(subfolder, "2026", "contract.pdf"); + var besideSubfolder = Path.Combine(folder, "contracts-old", "contract.pdf"); + + var goneFiles = FileSourceIndexer.GetGoneFiles([found, deleted, inSubfolder, besideSubfolder], [new FileInfo(found)], [subfolder]); + + Assert.That(goneFiles, Is.EquivalentTo([deleted, besideSubfolder]), "A folder whose name only begins like the one which could not be read is no part of it."); + } + + private static DataSourceLocalDirectory Folder(string path) => new() + { + Num = 1, + Id = DATA_SOURCE_ID, + Name = "Test folder", + Type = DataSourceType.LOCAL_DIRECTORY, + EmbeddingId = EMBEDDING_ID, + ConfidenceLevel = ConfidenceLevel.LOW, + Path = path, + }; + + private static void AssertTheIndexWasKept(IndexedRunOutcome outcome, List statuses, DataSourceEmbeddingManifest manifest, int indexedFiles, string unreachablePath) + { + Assert.Multiple(() => + { + Assert.That(outcome, Is.EqualTo(IndexedRunOutcome.DONE), "Nothing can be done until the file or the folder is back."); + Assert.That(manifest.Files, Has.Count.EqualTo(indexedFiles), "Embedding everything again once the file or the folder is back costs hours."); + Assert.That(manifest.PermanentFailures, Has.Count.EqualTo(1), "A file skipped for good would otherwise be read again, only to fail the same way."); + Assert.That(manifest.SourceHash, Is.EqualTo(STORED_SOURCE_HASH), "The stored hash tells which state of the data source the index holds."); + Assert.That(statuses[^1].State, Is.EqualTo(DataSourceEmbeddingState.FAILED)); + Assert.That(statuses[^1].IndexedDocuments, Is.EqualTo(indexedFiles), "The index still holds these files, and the page has to say so."); + Assert.That(statuses[^1].LastError, Does.Contain(unreachablePath)); + }); + } + + private void TakeAwayTheRightToRead(string directory) + { + if (OperatingSystem.IsWindows()) + Assert.Ignore("Taking away the right to read a folder needs an access control list on Windows."); + else + File.SetUnixFileMode(directory, UnixFileMode.None); + + this.lockedDirectories.Add(directory); + try + { + _ = Directory.EnumerateFiles(directory).Any(); + Assert.Ignore("This account may read every folder, e.g. as root, so no folder can be locked."); + } + catch (UnauthorizedAccessException) + { + // The folder is locked, as intended. + } + } + + private static async Task RunAsync(TDataSource dataSource, params string[] indexedFiles) where TDataSource : IDataSource, IIndexedDataSource + { + var statuses = new List(); + var manifest = new DataSourceEmbeddingManifest { SourceHash = STORED_SOURCE_HASH }; + foreach (var file in indexedFiles) + manifest.Files[file] = new EmbeddedFileRecord("FINGERPRINT", 1024, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 4); + + var scannedFile = Path.Combine(Path.GetDirectoryName(indexedFiles[0])!, "scan.pdf"); + manifest.PermanentFailures[scannedFile] = new PermanentIndexingFailureRecord("FINGERPRINT", FileExtractionErrorCode.NO_TEXT_EXTRACTED, "No text could be read.", DateTimeOffset.UnixEpoch); + + var embeddingProvider = new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small")); + var settingsManager = new SettingsManager(NullLogger.Instance, null!); + var context = new IndexedRunContext( + dataSource, + embeddingProvider, + embeddingProvider.CreateProvider(), + new NoVectorStoreClient("Test vector store", "Not available in tests."), + new NoIndexStoreClient("Test index store", "Not available in tests."), + manifest, + new RunShare(RunShare.DEFAULT_DURATION), + new IndexingPass(new DataSourceRunSignature("TEST-EMBEDDING-SIGNATURE")), + new PromptInjectionGuardService(null!, settingsManager, NullLogger.Instance, NullLoggerFactory.Instance).BeginAction(), + settingsManager, + statuses.Add, + NullLogger.Instance); + + // A run which reaches no file reads none, so it needs neither the runtime nor the chunker: + var indexer = new FileSourceIndexer(settingsManager, null!, null!, NullLogger.Instance); + var outcome = await indexer.ProcessAsync(context, DataSourceEmbeddingRefreshMode.HASH_CHECK, CancellationToken.None); + + return new(outcome, statuses, manifest); + } + + private sealed record RunResult(IndexedRunOutcome Outcome, List Statuses, DataSourceEmbeddingManifest Manifest); +} \ No newline at end of file diff --git a/app/Tests/Tools/WorkspaceChatMoveTests.cs b/app/Tests/Tools/WorkspaceChatMoveTests.cs index 5a960484d..843437605 100644 --- a/app/Tests/Tools/WorkspaceChatMoveTests.cs +++ b/app/Tests/Tools/WorkspaceChatMoveTests.cs @@ -94,20 +94,18 @@ public sealed class WorkspaceChatMoveTests } [Test] - public async Task ATranscriptOutsideTheChatDirectoryKeepsItsPath() + public async Task AFileOutsideTheChatDirectoryKeepsItsPath() { var chatId = Guid.NewGuid(); Directory.CreateDirectory(Path.Combine(this.testDirectory, "tempChats", chatId.ToString())); - // A transcript which is still waiting in the staging area, because its chat was not stored yet when it was made: - var stagingDirectory = Path.Combine(this.testDirectory, "media-staging", Guid.NewGuid().ToString("N")); - var staged = WriteTranscript(stagingDirectory, $"{Guid.NewGuid():N}.md", "meeting", isStaged: true); + var document = FileAttachment.FromPath(Path.Combine(this.testDirectory, "documents", "agenda.pdf")); var chat = new ChatThread { ChatId = chatId, Name = "Meeting notes" }; - chat.Draft.AddAttachments([staged]); + chat.Draft.AddAttachments([document]); await WorkspaceBehaviour.MoveChatAsync(chat, Guid.NewGuid()); - Assert.That(chat.Draft.MediaTranscripts.Single().FilePath, Is.EqualTo(staged.FilePath), "The move takes only the chat directory along, so a path elsewhere is still right."); + Assert.That(chat.Draft.FileAttachments.Single(), Is.SameAs(document), "The move takes only the chat directory along, so a path elsewhere is still right."); } [Test] @@ -145,14 +143,11 @@ public sealed class WorkspaceChatMoveTests private ManagedTranscriptAttachment CreateTranscript(Guid chatId, string stem, int number) { var transcriptDirectory = Path.Combine(this.testDirectory, "tempChats", chatId.ToString(), "attachments", "transcripts"); - return WriteTranscript(transcriptDirectory, $"{stem}-transcript-{number:D4}.md", stem, isStaged: false); - } + Directory.CreateDirectory(transcriptDirectory); - private static ManagedTranscriptAttachment WriteTranscript(string directory, string fileName, string stem, bool isStaged) - { - Directory.CreateDirectory(directory); - var path = Path.Combine(directory, fileName); + var fileName = $"{stem}-transcript-{number:D4}.md"; + var path = Path.Combine(transcriptDirectory, fileName); File.WriteAllText(path, $"# Transcription: {stem}.mp4{Environment.NewLine}{Environment.NewLine}Hello and welcome."); - return new ManagedTranscriptAttachment(fileName, path, new FileInfo(path).Length, $"{stem}.mp4", isStaged); + return new ManagedTranscriptAttachment(fileName, path, new FileInfo(path).Length, $"{stem}.mp4", false); } } \ No newline at end of file diff --git a/app/Tests/Tools/WorkspaceChatStoreTests.cs b/app/Tests/Tools/WorkspaceChatStoreTests.cs new file mode 100644 index 000000000..65f3f6e36 --- /dev/null +++ b/app/Tests/Tools/WorkspaceChatStoreTests.cs @@ -0,0 +1,90 @@ +using System.Text.Json; + +using AIStudio.Chat; +using AIStudio.Settings; +using AIStudio.Tools; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks where the transcripts of a chat end up when the chat is stored. +/// +/// +/// A transcript made before its chat was stored waits in the staging area, which every start of AI +/// Studio empties. Storing the chat therefore takes each such transcript into the chat directory: the +/// ones of the sent messages, and the ones of the message the user is still writing. Leaving a new +/// chat stores it together with that message, so a transcript left behind would be gone after the next +/// start, while the draft still pointed at it. +/// +[TestFixture] +[NonParallelizable] +public sealed class WorkspaceChatStoreTests +{ + private string? previousDataDirectory; + private string testDirectory = string.Empty; + + [SetUp] + public async Task PrepareDataDirectory() + { + // + // The data directory is static state of the whole application, which is why this fixture + // does not run alongside others. The workspace behaviour keeps the roots of the workspaces + // and of the temporary chats fixed, as the app never changes its data directory, so it gets + // told about the one of this test separately. The cache is static as well, so it starts + // over from what the new data directory holds: nothing. + // + this.previousDataDirectory = SettingsManager.DataDirectory; + this.testDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-chat-store-{Guid.NewGuid():N}"); + Directory.CreateDirectory(this.testDirectory); + SettingsManager.DataDirectory = this.testDirectory; + WorkspaceBehaviour.TestDataDirectory = this.testDirectory; + + await WorkspaceBehaviour.ForceReloadWorkspaceTreeAsync(); + } + + [TearDown] + public void RemoveDataDirectory() + { + SettingsManager.DataDirectory = this.previousDataDirectory; + WorkspaceBehaviour.TestDataDirectory = null; + + // Whoever reads the cache next must not get the workspaces of this test: + WorkspaceBehaviour.InvalidateWorkspaceTreeCache(); + + try + { + Directory.Delete(this.testDirectory, true); + } + catch (IOException) + { + // A temporary directory we could not remove says nothing about the code under test. + } + } + + [Test] + public async Task TheStagedTranscriptOfTheDraftMovesIntoTheChat() + { + var workspace = await WorkspaceBehaviour.TryCreateWorkspaceAsync("Projects"); + Assert.That(workspace.Success, Is.True, "The workspace could not be created."); + + // A new chat, which was not stored yet when the transcript of its draft was made: + var chat = new ChatThread { WorkspaceId = workspace.Workspace.WorkspaceId, ChatId = Guid.NewGuid(), Name = "Meeting notes" }; + var staged = await ManagedTranscriptAttachment.CreateStagedAsync(Path.Combine(this.testDirectory, "meeting.mp4"), "Hello and welcome."); + chat.Draft.AddAttachments([staged]); + + Assert.That(await WorkspaceBehaviour.StoreChatAsync(chat), Is.True, "The chat could not be stored."); + + var chatDirectory = WorkspaceBehaviour.GetChatDirectory(chat.WorkspaceId, chat.ChatId); + var transcript = chat.Draft.MediaTranscripts.Single(); + var stored = JsonSerializer.Deserialize(await File.ReadAllTextAsync(Path.Combine(chatDirectory, "thread.json")), WorkspaceBehaviour.JSON_OPTIONS)!; + + Assert.Multiple(() => + { + Assert.That(transcript.FilePath, Does.StartWith(chatDirectory), "The next start empties the staging area, but leaves the chat directory alone."); + Assert.That(File.Exists(transcript.FilePath), Is.True, "Otherwise the transcript drops out of the message which is still being written."); + Assert.That(transcript.IsStaged, Is.False, "The chat owns the transcript now, so deleting the chat deletes its file as well."); + Assert.That(File.Exists(staged.FilePath), Is.False, "The transcript moves, so there is only one file to own."); + Assert.That(stored.Draft.MediaTranscripts.Single().FilePath, Is.EqualTo(transcript.FilePath), "Opening the chat again brings the draft back from the stored chat, so that has to know the new path as well."); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/ZipEntryNamesTests.cs b/app/Tests/Tools/ZipEntryNamesTests.cs new file mode 100644 index 000000000..d7128081c --- /dev/null +++ b/app/Tests/Tools/ZipEntryNamesTests.cs @@ -0,0 +1,52 @@ +using AIStudio.Tools; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks the names of ZIP entries before anything is extracted. +/// +/// +/// A plugin archive, the configuration of an organization, or a backup comes from somewhere else. A +/// name which leads outside of the target directory would let it overwrite any file the user may +/// write, the settings among them. +/// +[TestFixture] +public sealed class ZipEntryNamesTests +{ + [TestCase("plugin/plugin.lua", "plugin/plugin.lua")] + [TestCase("plugin\\plugin.lua", "plugin/plugin.lua")] + [TestCase("data/workspaces/", "data/workspaces/")] + public void ASafeNameIsKeptWithForwardSlashes(string entryName, string expected) + { + Assert.That(ZipEntryNames.Normalize(entryName), Is.EqualTo(expected)); + } + + [TestCase("")] + [TestCase("/etc/passwd")] + [TestCase("\\Windows\\win.ini")] + [TestCase("C:/Users/settings.json")] + [TestCase("../settings.json")] + [TestCase("data/../../settings.json")] + [TestCase("data\\..\\..\\settings.json")] + [TestCase("./plugin.lua")] + [TestCase("plugin\0.lua")] + public void ANameWhichCouldLeadElsewhereIsRejected(string entryName) + { + Assert.Multiple(() => + { + Assert.That(() => ZipEntryNames.Normalize(entryName), Throws.InstanceOf()); + Assert.That(ZipEntryNames.TryNormalize(entryName, out var normalizedEntryName), Is.False); + Assert.That(normalizedEntryName, Is.Empty); + }); + } + + [Test] + public void ASafeNameLiesBelowTheTargetDirectory() + { + var targetDirectory = Path.Combine(Path.GetTempPath(), "ai-studio-zip-target"); + + var destinationPath = ZipEntryNames.GetDestinationPath(targetDirectory, ZipEntryNames.Normalize("attachments/transcripts/meeting.md")); + + Assert.That(destinationPath, Is.EqualTo(Path.Combine(Path.GetFullPath(targetDirectory), "attachments", "transcripts", "meeting.md"))); + } +} \ No newline at end of file diff --git a/documentation/Enterprise IT.md b/documentation/Enterprise IT.md index 69158cf4b..564539dc6 100644 --- a/documentation/Enterprise IT.md +++ b/documentation/Enterprise IT.md @@ -535,16 +535,18 @@ AI Studio has a dedicated directory for this: `.config-tests`. Anything stored t Do not use the `.config` directory for this. It belongs to your configuration web server, and AI Studio removes everything there that your organization does not reference anymore. -### The data directory +### The data and configuration directories -Plugins live in the data directory of AI Studio: +Plugins live in the data directory of AI Studio. The settings live in its configuration directory: -| Platform | Data directory | -| --- | --- | -| Windows | `%LOCALAPPDATA%\com.github.mindwork-ai.ai-studio\data` | -| macOS | `~/Library/Application Support/com.github.mindwork-ai.ai-studio/data` | -| Linux | `$XDG_DATA_HOME/com.github.mindwork-ai.ai-studio/data`, usually `~/.local/share/com.github.mindwork-ai.ai-studio/data` | -| Linux (Flatpak) | `~/.var/app/org.mindworkai.AIStudio/data/com.github.mindwork-ai.ai-studio/data` | +| Platform | Data directory | Configuration directory | +| --- | --- | --- | +| Windows | `%LOCALAPPDATA%\com.github.mindwork-ai.ai-studio\data` | `%APPDATA%\com.github.mindwork-ai.ai-studio` | +| macOS | `~/Library/Application Support/com.github.mindwork-ai.ai-studio/data` | `~/Library/Application Support/com.github.mindwork-ai.ai-studio` | +| Linux | `$XDG_DATA_HOME/org.mindworkai.AIStudio/data`, usually `~/.local/share/org.mindworkai.AIStudio/data` | `$XDG_CONFIG_HOME/org.mindworkai.AIStudio`, usually `~/.config/org.mindworkai.AIStudio` | +| Linux (Flatpak) | `~/.var/app/org.mindworkai.AIStudio/data/org.mindworkai.AIStudio/data` | `~/.var/app/org.mindworkai.AIStudio/config/org.mindworkai.AIStudio` | + +The panel **Your Data Folders** on the Information page shows both directories as AI Studio resolved them on the device at hand. ### Staging a deployment @@ -773,6 +775,65 @@ With `DataMailboxes.MinimumOutboundDataRestriction`, you rule out the less stric With `DataMailboxes.AllowOnlyOrganizationMailServers`, AI Studio connects to no other IMAP server. Before every connection, it compares the host of the mailbox with the hosts of your mail servers; port and encryption make no difference. A mailbox somebody added on another server before stops synchronizing at once, the embeddings page says why, and the AI no longer reads it. Its local index stays, so the mailbox comes back as it was, should you allow its server again. Switch it on together with your first mail servers, and nobody gets to add a mailbox elsewhere at all. +## Backing up and restoring data + +Users can back up the data of AI Studio into a single `.mwbackup` file, for example to take it to a new computer, and restore it there. They find both under **Data Backup & Restore**, on the Information page and in the settings. Instead of replacing all their data, users may also add only the chats of a backup to the chats they have. Secrets such as API keys never go into a backup. These settings decide whether they may: + +| Setting | Effect | +|---|---| +| `DataApp.AllowUserToBackUpData` | `false` keeps users from writing a backup. The button stays visible but is disabled. | +| `DataApp.AllowUserToRestoreData` | `false` keeps users from restoring a backup and from adding the chats of one. The buttons stay visible but are disabled. | + +Both settings start as `true`. A backup holds the chats, the text of indexed documents, and indexed mails in a single file. The file is not encrypted, and it can leave the computer on any USB stick or cloud drive. Where that must not happen, set `DataApp.AllowUserToBackUpData` to `false`. + +### What a backup holds + +A backup takes along what AI Studio needs to carry on with the same data on another computer: + +- the settings, without what your configurations manage, see [Settings and objects of your configurations](#settings-and-objects-of-your-configurations), +- the chats in workspaces and the temporary chats, along with their transcripts, +- the plugins of the user, such as the assistants they built, +- the tokenizers chosen for providers, +- the projects of the Visual Briefing Assistant, +- the local index of data sources and mailboxes, together with its embeddings, so that nothing has to be embedded again. While a backup copies the index, the indexing pauses, and it carries on afterward. + +What stays out: + +- **Secrets.** API keys, passwords, and tokens stay in the OS keyring. After a restore on another computer, AI Studio lists the providers, data sources, mailboxes, and tools of the user which need theirs again. The secrets of your configurations come back with your configurations. +- **Plugins of your organization.** Everything below `.config` and `.config-tests` stays out, and so does every plugin elsewhere which is marked with `DEPLOYED_USING_CONFIG_SERVER = true`, see [Important: Mark enterprise-managed plugins explicitly](#important-mark-enterprise-managed-plugins-explicitly). Your deployment brings them back. A plugin which your MDM solution placed without that marker counts as one of the user's own: it goes into backups, and a restore can replace or remove it. +- **Files outside the folders of AI Studio**, such as documents attached to a chat, the folders of local data sources, the folders of the batch processing, or a bundle of root certificates. A backup keeps only their locations. After a restore, AI Studio lists the settings whose files or folders it cannot find on the new computer. +- Plugins which come with AI Studio, tools AI Studio downloaded such as Pandoc, log files, and temporary files. + +### Settings and objects of your configurations + +Before the settings go into a backup, AI Studio leaves out every object which a configuration plugin manages: LLM, embedding, and transcription providers, profiles, chat templates, data sources, and document analysis policies. On a computer which receives your configurations, these objects come back from there, along with their API keys, and nothing appears twice. The same holds for configuration plugins the user installed themselves: they go into the backup as plugins and bring their objects back after a restore. + +A setting which your configuration locks stays in the backup, together with the configuration which locked it. After a restore, AI Studio treats it like the setting of a withdrawn configuration until your configuration locks it again, see [Withdrawing a configuration](#withdrawing-a-configuration). On a device which receives your configurations, the setting therefore ends up as you configured it. On one without them, the user gets back the value they had chosen before your configuration took the setting over. + +### How a restore works + +A restore replaces the data of the user with the backup, and it needs a restart: while AI Studio runs, its files are open, and on Windows, they cannot be replaced at all. + +1. AI Studio checks the backup. It refuses a backup made by a newer version of AI Studio, and one whose files do not match its list of contents. +2. AI Studio unpacks the backup into a folder `.restore-pending`, one within the data directory and one within the configuration directory. This needs free space for the unpacked backup on top of the current data. Until the restart, the indexing pauses, and AI Studio asks the user to restart now or to discard the restore. +3. At the next start, before AI Studio opens any of its files, it moves the current data into a folder `.restore-previous` and the restored data into its place, piece by piece. A piece for which the backup holds nothing stays as it was. +4. Should a step fail, AI Studio moves everything back, and the user keeps their data as it was. Should even that fail, AI Studio tries again at every start. +5. Once the restored settings have loaded, AI Studio deletes the replaced data from `.restore-previous`, reports the outcome, and lists what remains to do, such as entering API keys again. + +A restore leaves the plugins of your organization, the plugins which come with AI Studio, and the log files as they are. When a restore fails, the log file `events.log` in the data directory tells why, in the entries from the source `Restore`. + +### Backing up with a tool of your own + +If you back up the user profiles of your devices anyway, for example with Restic, Borg, or the backup solution of your organization, the Information page helps: its panel **Your Data Folders** shows where the data and configuration directories lie, and which of their contents belong into a backup. [The data and configuration directories](#the-data-and-configuration-directories) lists their usual places. The panel derives this from the same list the backups of AI Studio follow, and it stays complete even when you set `DataApp.AllowUserToBackUpData` to `false`. + +Keep in mind: + +- Copy the local index below `databases/` only while AI Studio is closed. While it runs, these files change, and on Windows, they are locked. A copy taken then may not be readable afterward. +- Secrets do not lie in these folders. They stay in the OS keyring. +- On macOS, the data directory lies within the configuration directory. +- No pattern of paths tells the plugins which your MDM solution placed outside `.config` apart from the user's own plugins. The panel lists the folders of those marked with `DEPLOYED_USING_CONFIG_SERVER = true`, so you can leave them out. +- AI Studio stores the paths to its own files, such as transcripts and tokenizers, relative to the data directory. A copy therefore also works on a computer whose folders lie elsewhere, for example under another user name, as long as AI Studio v26.10.1 or later ran before the copy was taken. Paths to the user's own files, such as attachments or the folders of data sources, stay as they are. + ## Letting users provide their own API key Sometimes you want to hand out a preconfigured provider -- a fixed host, model, and instance name diff --git a/documentation/compatibility-shims/2026-07-plugin-archive-zip-backslashes.md b/documentation/compatibility-shims/2026-07-plugin-archive-zip-backslashes.md index f486fa7ec..aa984339e 100644 --- a/documentation/compatibility-shims/2026-07-plugin-archive-zip-backslashes.md +++ b/documentation/compatibility-shims/2026-07-plugin-archive-zip-backslashes.md @@ -4,9 +4,12 @@ - Introduced: 2026-07-09 - Remove after: when Microsoft fixes dotnet/runtime#27620 and dotnet/runtime#41914 - Code references: + - `app/MindWork AI Studio/Tools/ZipEntryNames.cs` (`Normalize`, which treats backslashes as separators) - `app/MindWork AI Studio/Tools/PluginSystem/PluginArchive.cs` - `app/MindWork AI Studio/Tools/PluginSystem/PluginFactory.Download.cs` - - `app/MindWork AI Studio/Tools/Services/AssistantPluginInstallService.cs` + - `app/MindWork AI Studio/Tools/Services/PluginInstallService.Import.cs` + - `app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImporter.cs` + - `app/Tests/Tools/ZipEntryNamesTests.cs` ## User Impact @@ -18,7 +21,7 @@ The same applies to plugin archives that users import from their disk. Archives ## Compatibility Behavior -AI Studio manually extracts enterprise configuration plugin archives and plugin archives imported by users. During extraction, entry names are normalized so both `/` and `\` are treated as archive path separators. +AI Studio manually extracts enterprise configuration plugin archives and plugin archives imported by users. During extraction, entry names are normalized so both `/` and `\` are treated as archive path separators. The same normalization in `ZipEntryNames` serves adding the chats of a backup, which AI Studio always writes with forward slashes. The extraction still preserves the archive structure and validates each entry before writing it to disk. Rooted paths, drive-qualified paths, and parent-directory traversal paths are rejected. @@ -28,4 +31,5 @@ This works around the behavior described in dotnet/runtime#27620. A related upst - Confirm supported .NET runtimes and administrator packaging guidance no longer require accepting backslashes in enterprise ZIP entry names. - Replace `PluginArchive.Extract(...)` with `ZipFile.ExtractToDirectory(...)`. +- Stop treating `\` as a separator in `ZipEntryNames.Normalize`, and drop the backslash cases from `ZipEntryNamesTests`. Keep the checks against rooted, drive-qualified, and `..` names: they protect every extraction, not just this shim. - Update this document's status to `Removed`. diff --git a/documentation/compatibility-shims/2026-10-relative-data-paths.md b/documentation/compatibility-shims/2026-10-relative-data-paths.md new file mode 100644 index 000000000..0bc3f4058 --- /dev/null +++ b/documentation/compatibility-shims/2026-10-relative-data-paths.md @@ -0,0 +1,33 @@ +# Relative Data Paths + +- Status: Active +- Introduced: 2026-10-14 +- Remove after: 2027-04-14 +- Code references: + - `app/MindWork AI Studio/Tools/Services/RelativeDataPathsMigrationService.cs` + - `app/MindWork AI Studio/Tools/WorkspaceBehaviour.RelativeDataPaths.cs` + - `app/MindWork AI Studio/Settings/DataModel/Data.cs` (`StoredDataPathsAreRelative`) + - `app/MindWork AI Studio/Program.cs` (registration of the service) + - `app/Tests/Tools/RelativeDataPathsMigrationTests.cs` + +## User Impact + +Up to v26.9.1, AI Studio stored the paths of its own files below the data directory as absolute paths: the transcripts of a chat in its `thread.json`, and the copies of the tokenizers in the settings. From v26.10.1 on, it stores them relative to the data directory, so that a backup of the data directory works on another computer, under another user name, or on another OS. + +AI Studio reads an absolute path as it is, so a chat stored by v26.9.1 keeps working on the same computer without this shim. It stores the path relative the next time it stores the chat. A chat which is never stored again, though, would keep its absolute paths, and a backup would bring them to a computer where they lead nowhere. + +## Compatibility Behavior + +`RelativeDataPathsMigrationService` runs once after the start. It reads and writes every chat once more, within one hold of the chat's lock, so that nothing AI Studio stores meanwhile gets lost, and it keeps the time of the chat's last change. Then it stores the settings and marks the work as done in `StoredDataPathsAreRelative`. When a chat is busy for too long or cannot be written, the flag stays unset, and the next start tries again. + +A downgrade to v26.9.1 does not understand relative paths: it no longer finds the transcripts of the chats nor the tokenizers which were copied into the data directory. + +## Removal Checklist + +- Remove `RelativeDataPathsMigrationService` and its registration in `Program.cs`. +- Remove `WorkspaceBehaviour.RelativeDataPaths.cs`. When no other partial part of `WorkspaceBehaviour` is left, make the class non-partial again. +- Remove `Data.StoredDataPathsAreRelative`. Settings which still hold the property read fine without it. +- Remove `app/Tests/Tools/RelativeDataPathsMigrationTests.cs`. +- Keep `DataDirectoryRelativePathConverter`: it is no shim. Reading an absolute path stays necessary for the files of the user. +- Update this document's status to `Removed`. +- No changelog entry is needed: a chat which was not stored since then still works on the same computer. \ No newline at end of file diff --git a/runtime/src/app_window.rs b/runtime/src/app_window.rs index 9ca5d1f1f..868b300cd 100644 --- a/runtime/src/app_window.rs +++ b/runtime/src/app_window.rs @@ -21,6 +21,7 @@ use tauri_plugin_opener::OpenerExt; use tokio::sync::broadcast; use tokio::time; use crate::clipboard::shutdown_clipboard; +use crate::data_restore::apply_pending_restore; use crate::dotnet::{cleanup_dotnet_server, start_dotnet_server, stop_dotnet_server}; use crate::environment::{ installation_kind, is_prod, is_dev, is_flatpak, InstallationKind, CONFIG_DIRECTORY, @@ -193,23 +194,35 @@ pub fn start_tauri(tauri_context: tauri::Context) { let data_path = data_path.join("data"); // Get and store the data and config directories: + let config_path = app.path().app_config_dir().unwrap(); DATA_DIRECTORY.set(data_path.to_str().unwrap().to_string()).map_err(|_| error!("Was not able to set the data directory.")).unwrap(); - CONFIG_DIRECTORY.set(app.path().app_config_dir().unwrap().to_str().unwrap().to_string()).map_err(|_| error!("Was not able to set the config directory.")).unwrap(); + CONFIG_DIRECTORY.set(config_path.to_str().unwrap().to_string()).map_err(|_| error!("Was not able to set the config directory.")).unwrap(); + + // A restore of a backup, below, writes to the log, so the log goes into the data directory first: + info!(Source = "Bootloader Tauri"; "Reconfigure the file logger to use the app data directory {data_path:?}"); + switch_to_file_logging(data_path.clone()).map_err(|e| error!("Failed to switch logging to file: {e}")).unwrap(); + + // + // A restore replaces files which the .NET server and the vector database open. It + // therefore runs before both start, and after a .NET server which outlived an earlier + // run is gone: + // + if is_prod() { + cleanup_dotnet_server(); + } + + apply_pending_restore(&data_path, &config_path); if is_dev() { #[cfg(debug_assertions)] create_startup_env_file(); } else { - cleanup_dotnet_server(); start_dotnet_server(app.handle().clone()); } start_qdrant_edge_database(app.handle().clone()); set_default_tokenizer_path(app.handle().clone()); - - info!(Source = "Bootloader Tauri"; "Reconfigure the file logger to use the app data directory {data_path:?}"); - switch_to_file_logging(data_path).map_err(|e| error!("Failed to switch logging to file: {e}")).unwrap(); set_pdfium_path(app.path()); Ok(()) @@ -819,18 +832,12 @@ pub struct AppExitResponse { /// Requests a controlled shutdown of the entire desktop application. pub async fn exit_app() -> Json { - let app_handle = { - let main_window_lock = MAIN_WINDOW.lock().unwrap(); - match main_window_lock.as_ref() { - Some(window) => window.app_handle().clone(), - None => { - error!(Source = "Tauri"; "Cannot exit app: main window not available."); - return Json(AppExitResponse { - success: false, - error_message: "Main window not available".to_string(), - }); - } - } + let Some(app_handle) = main_app_handle() else { + error!(Source = "Tauri"; "Cannot exit app: main window not available."); + return Json(AppExitResponse { + success: false, + error_message: "Main window not available".to_string(), + }); }; info!(Source = "Tauri"; "Controlled app exit was requested by the UI."); @@ -845,10 +852,57 @@ pub async fn exit_app() -> Json { }) } +/// Response for application restart requests. +#[derive(Serialize)] +pub struct AppRestartResponse { + success: bool, + error_message: String, +} + +/// Requests a controlled restart of the entire desktop application, e.g., to apply a prepared restore +/// of a backup. Before the app starts again, the exit handler stops the .NET server and the vector +/// database, just as when the app exits. +/// +/// In the development environment, the runtime did not start the .NET server, so it could not start +/// it again either. It refuses, and the developer restarts both. +pub async fn restart_app() -> Json { + if is_dev() { + warn!(Source = "Tauri"; "The UI requested a restart, which the development environment does not support."); + return Json(AppRestartResponse { + success: false, + error_message: "The development environment cannot restart the app, since the runtime did not start the .NET server. Restart both yourself.".to_string(), + }); + } + + let Some(app_handle) = main_app_handle() else { + error!(Source = "Tauri"; "Cannot restart app: main window not available."); + return Json(AppRestartResponse { + success: false, + error_message: "Main window not available".to_string(), + }); + }; + + info!(Source = "Tauri"; "Controlled app restart was requested by the UI."); + tauri::async_runtime::spawn(async move { + time::sleep(Duration::from_millis(50)).await; + app_handle.request_restart(); + }); + + Json(AppRestartResponse { + success: true, + error_message: String::new(), + }) +} + +/// The handle of the app, or `None` while the main window does not exist yet. +fn main_app_handle() -> Option { + MAIN_WINDOW.lock().unwrap().as_ref().map(|window| window.app_handle().clone()) +} + /// Registers or updates a global shortcut. If the shortcut string is empty, /// the existing shortcut for that name will be unregistered. pub async fn register_shortcut(payload: Json) -> Json { - let app_handle = MAIN_WINDOW.lock().unwrap().as_ref().map(|window| window.app_handle().clone()); + let app_handle = main_app_handle(); let event_sender = EVENT_BROADCAST.lock().unwrap().clone(); Json(crate::global_shortcuts::register(app_handle, event_sender, payload.0).await) } @@ -930,7 +984,7 @@ pub async fn suspend_shortcuts() -> Json { /// Resumes shortcut processing by re-registering all shortcuts with the OS. pub async fn resume_shortcuts() -> Json { - let app_handle = MAIN_WINDOW.lock().unwrap().as_ref().map(|window| window.app_handle().clone()); + let app_handle = main_app_handle(); let event_sender = EVENT_BROADCAST.lock().unwrap().clone(); Json(crate::global_shortcuts::resume(app_handle, event_sender).await) } diff --git a/runtime/src/data_restore.rs b/runtime/src/data_restore.rs new file mode 100644 index 000000000..c84f5abfa --- /dev/null +++ b/runtime/src/data_restore.rs @@ -0,0 +1,948 @@ +//! Applies a restore of a backup which the app prepared, before anything opens the data. +//! +//! While AI Studio runs, its files are open, and on Windows, they cannot be replaced at all. The app +//! therefore unpacks a backup next to the data and marks it as ready. The next start puts it in place, +//! before the .NET server and the vector database open a single file. +//! +//! The contract with the app, cf. `DataRestoreFormat` in the app; keep both in sync: +//! +//! - `.restore-pending/`, below the data folder and below the configuration folder, holds what the +//! backup brings for that folder, in the layout of that folder. +//! - `.restore-pending/restore.json` in the data folder names the elements to replace, each with the +//! paths below it which stay as they are. +//! - `.restore-pending/READY` in the data folder comes last. Without it, the restore was not prepared +//! completely, and nothing happens. +//! +//! What the runtime adds to it: +//! +//! - `.restore-previous/`, below both folders, keeps what the restore replaced, in the layout of that +//! folder. The app removes it once it loaded the restored settings. +//! - `.restore-pending/journal.json` in the data folder lists every move before the first one happens, +//! and how many of them started. A start which finds it next to `READY` knows that an earlier one +//! stopped in the middle, and moves back what that one moved. +//! - `.restore-result.json` in the data folder tells the app how the restore ended. +//! +//! Every move renames a file or folder within the same folder, and thus on the same disk: it is atomic, +//! it takes no time for any amount of data, and it can be undone. + +use std::collections::HashSet; +use std::fs; +use std::io; +use std::io::Write; +use std::path::{Path, PathBuf}; +use std::thread; +use std::time::Duration; +use log::{error, info, warn}; +use serde::{Deserialize, Serialize}; + +const PENDING_DIRECTORY: &str = ".restore-pending"; +const PREVIOUS_DIRECTORY: &str = ".restore-previous"; +const PLAN_FILE_NAME: &str = "restore.json"; +const READY_FILE_NAME: &str = "READY"; +const JOURNAL_FILE_NAME: &str = "journal.json"; +const JOURNAL_TEMPORARY_FILE_NAME: &str = "journal.json.tmp"; +const RESULT_FILE_NAME: &str = ".restore-result.json"; + +/// The version of the plan this runtime reads. +const PLAN_FORMAT_VERSION: i32 = 1; + +const SETTINGS_FILE_NAME: &str = "settings.json"; + +/// The elements a restore may replace: those which the list of the backup contents in the app +/// (`DataBackupContents`) includes. The runtime refuses a plan which names anything else, so that +/// whatever goes wrong in the app, a restore never replaces the logs, the downloaded tools, or the +/// data folder as a whole. +const REPLACEABLE_ELEMENTS: [(Location, &str); 8] = [ + (Location::ConfigDirectory, SETTINGS_FILE_NAME), + (Location::DataDirectory, "workspaces/"), + (Location::DataDirectory, "tempChats/"), + (Location::DataDirectory, "plugins/"), + (Location::DataDirectory, "tokenizers/"), + (Location::DataDirectory, "visualBriefings/"), + (Location::DataDirectory, "databases/sqlite/"), + (Location::DataDirectory, "databases/vector_database/stores/"), +]; + +/// Paths which a restore always keeps, whatever the plan says: the plugins which come with AI Studio +/// or from the organization. The plan adds the plugins of the organization which lie among the plugins +/// of the user, since only the app knows them. +const ALWAYS_KEPT: [&str; 3] = ["plugins/.internal/", "plugins/.config/", "plugins/.config-tests/"]; + +/// The settings file which every version of AI Studio keeps for itself, `settings.v6.json` for +/// example. When `settings.json` is older than the app, the app prefers the file of its own version, +/// which would undo a restore of older settings. These files therefore go together with +/// `settings.json`. +const SETTINGS_OF_ONE_VERSION_PREFIX: &str = "settings.v"; +const SETTINGS_OF_ONE_VERSION_SUFFIX: &str = ".json"; + +/// How long to wait before each further attempt of a move. On Windows, a file stays locked for a +/// moment after the process which held it ended, and virus scanners or the search indexer lock files +/// briefly, too. +const MOVE_RETRY_DELAYS: [Duration; 6] = [ + Duration::from_millis(100), + Duration::from_millis(200), + Duration::from_millis(400), + Duration::from_millis(800), + Duration::from_millis(1_600), + Duration::from_millis(3_200), +]; + +/// The folder a path lies in. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +enum Location { + DataDirectory, + ConfigDirectory, +} + +/// What the app prepared, cf. `DataRestorePlan` in the app. +#[derive(Debug, Deserialize)] +struct RestorePlan { + format_version: i32, + prepared_at_utc: String, + backup_created_at_utc: String, + backup_app_version: String, + elements: Vec, +} + +/// One element to replace, cf. `DataRestoreElement` in the app. +#[derive(Debug, Deserialize)] +struct RestoreElement { + location: Location, + + /// The path within its folder, with forward slashes. A folder ends with a slash. + relative_path: String, + + /// The paths below the element which stay as they are, relative to the same folder. + keep: Vec, +} + +/// Renames one file or folder. Both paths are relative to the folder, with forward slashes. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +struct Move { + location: Location, + from: String, + to: String, +} + +impl Move { + fn new(location: Location, from: &str, to: &str) -> Self { + Move { location, from: from.to_string(), to: to.to_string() } + } +} + +/// Every move of a restore, written before the first one happens, and written anew before every +/// further one and after every one which was moved back. +/// +/// The file system alone cannot tell which moves happened: a move which brings back a kept plugin finds +/// its destination taken before the restore began. Only the last move the journal counts may or may +/// not have happened, and for that one, the file system tells. +#[derive(Debug, Serialize, Deserialize)] +struct Journal { + moves: Vec, + + /// How many moves started. All of them happened, except perhaps the last one. + started: usize, +} + +/// How a restore ended. +#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "SCREAMING_SNAKE_CASE")] +enum RestoreOutcome { + /// The restored data are in place. What they replaced lies in `.restore-previous`. + Applied, + + /// The restore did not start, since the plan or the folders did not allow it. Nothing changed. + Refused, + + /// The restore started, but could not finish. Everything was moved back. + RolledBack, + + /// Moving back failed as well. The journal stays, and the next start tries again. + RollbackFailed, +} + +/// What the app reads at its next start, in `.restore-result.json`. +#[derive(Debug, Serialize, Deserialize)] +struct RestoreResult { + outcome: RestoreOutcome, + + /// What happened, in English, for the log. The app tells the user in its own words. + detail: String, + + prepared_at_utc: String, + backup_created_at_utc: String, + backup_app_version: String, + finished_at_utc: String, +} + +/// How a restore ended, and why. +struct RestoreEnd { + outcome: RestoreOutcome, + detail: String, +} + +impl RestoreEnd { + fn new(outcome: RestoreOutcome, detail: impl Into) -> Self { + RestoreEnd { outcome, detail: detail.into() } + } +} + +/// The two folders a restore works in. +struct Folders<'a> { + data: &'a Path, + config: &'a Path, +} + +impl Folders<'_> { + fn root(&self, location: Location) -> &Path { + match location { + Location::DataDirectory => self.data, + Location::ConfigDirectory => self.config, + } + } + + /// The pending directory of the data folder, which holds the plan, the journal, and `READY`. + fn pending(&self) -> PathBuf { + self.data.join(PENDING_DIRECTORY) + } +} + +/// Applies a restore which the app prepared, or moves back one which an earlier start left unfinished. +/// +/// Call it before anything opens a file in the data folder. In production, the runtime starts the +/// .NET server only afterward. In the development environment, the IDE starts the .NET server on its +/// own, so start it after the runtime when you try a restore. +pub fn apply_pending_restore(data_directory: &Path, config_directory: &Path) { + let folders = Folders { data: data_directory, config: config_directory }; + apply_pending_restore_in(&folders, &MOVE_RETRY_DELAYS); +} + +/// Applies a pending restore, and returns how it ended, or `None` when there was nothing to do. +fn apply_pending_restore_in(folders: &Folders, retry_delays: &[Duration]) -> Option { + let pending = folders.pending(); + let is_ready = exists(&pending.join(READY_FILE_NAME)); + let journal_path = pending.join(JOURNAL_FILE_NAME); + + if exists(&journal_path) { + if !is_ready { + info!(Source = "Restore"; "Finishing the clean-up of the restore which an earlier start applied."); + clean_up_after_success(folders); + return None; + } + + warn!(Source = "Restore"; "An earlier start stopped while it applied a restore. Moving the replaced data back."); + let plan = read_plan(&pending).ok(); + let end = match read_journal(&journal_path) { + Ok(mut journal) => roll_back(folders, &mut journal, retry_delays, "AI Studio stopped while it applied the restore."), + Err(error) => RestoreEnd::new(RestoreOutcome::RollbackFailed, format!("The journal of the restore could not be read: {error}")), + }; + + return Some(finish(folders, plan.as_ref(), end, retry_delays)); + } + + if !is_ready { + return None; + } + + info!(Source = "Restore"; "Applying the prepared restore of a backup."); + let (plan, end) = match read_plan(&pending) { + Ok(plan) => { + let end = apply(folders, &plan, retry_delays); + (Some(plan), end) + }, + + Err(error) => (None, RestoreEnd::new(RestoreOutcome::Refused, error)), + }; + + Some(finish(folders, plan.as_ref(), end, retry_delays)) +} + +/// Moves the restored elements into place, and back again when a move fails. +fn apply(folders: &Folders, plan: &RestorePlan, retry_delays: &[Duration]) -> RestoreEnd { + let mut journal = match begin(folders, plan, retry_delays) { + Ok(journal) => journal, + Err(detail) => return RestoreEnd::new(RestoreOutcome::Refused, detail), + }; + + while journal.started < journal.moves.len() { + if let Err(detail) = execute_next(folders, &mut journal, retry_delays) { + error!(Source = "Restore"; "{detail} Moving the replaced data back."); + return roll_back(folders, &mut journal, retry_delays, &detail); + } + } + + RestoreEnd::new(RestoreOutcome::Applied, format!("Replaced {} elements with {} moves.", plan.elements.len(), journal.moves.len())) +} + +/// Determines the moves of a restore and writes them into the journal, before any of them happens. +fn begin(folders: &Folders, plan: &RestorePlan, retry_delays: &[Duration]) -> Result { + let journal = Journal { moves: plan_moves(folders, plan)?, started: 0 }; + write_journal(&folders.pending(), &journal, retry_delays)?; + Ok(journal) +} + +/// Executes the next move, once the journal counts it. +fn execute_next(folders: &Folders, journal: &mut Journal, retry_delays: &[Duration]) -> Result<(), String> { + let step = journal.moves[journal.started].clone(); + journal.started += 1; + write_journal(&folders.pending(), journal, retry_delays)?; + + execute(folders, &step, retry_delays) + .map_err(|error| format!("Could not move '{}' to '{}': {error}", step.from, step.to)) +} + +/// Determines every move of a restore, in the order they happen. For each element: move the current +/// one aside, move the restored one in, and move back what stays. +fn plan_moves(folders: &Folders, plan: &RestorePlan) -> Result, String> { + if plan.format_version != PLAN_FORMAT_VERSION { + return Err(format!("The plan has the format version {}, but this runtime reads version {PLAN_FORMAT_VERSION}.", plan.format_version)); + } + + // Empty folders are what moving back a restore may leave behind; files are the data of the user: + for root in [folders.data, folders.config] { + let previous = root.join(PREVIOUS_DIRECTORY); + if !remove_empty_directories(&previous) { + return Err(format!("'{}' still keeps the data which an earlier restore replaced.", previous.display())); + } + } + + let mut planned_elements = HashSet::new(); + let mut moves = Vec::new(); + for element in &plan.elements { + let is_replaceable = REPLACEABLE_ELEMENTS.iter().any(|(location, path)| *location == element.location && *path == element.relative_path); + if !is_replaceable { + return Err(format!("A restore may not replace '{}'.", element.relative_path)); + } + + if !planned_elements.insert((element.location, element.relative_path.as_str())) { + return Err(format!("The plan names '{}' twice.", element.relative_path)); + } + + let root = folders.root(element.location); + let target = element.relative_path.trim_end_matches('/'); + let staged = format!("{PENDING_DIRECTORY}/{target}"); + let previous = format!("{PREVIOUS_DIRECTORY}/{target}"); + + if exists(&resolve(root, target)) { + moves.push(Move::new(element.location, target, &previous)); + } + + if element.location == Location::ConfigDirectory && target == SETTINGS_FILE_NAME { + for name in settings_of_single_versions(root)? { + moves.push(Move::new(element.location, &name, &format!("{PREVIOUS_DIRECTORY}/{name}"))); + } + } + + if exists(&resolve(root, &staged)) { + moves.push(Move::new(element.location, &staged, target)); + } + + for kept in kept_paths(element)? { + if !exists(&resolve(root, &kept)) { + continue; + } + + let below = &kept[target.len()..]; + if exists(&resolve(root, &format!("{staged}{below}"))) { + return Err(format!("The backup brings '{kept}', which the restore keeps.")); + } + + moves.push(Move::new(element.location, &format!("{previous}{below}"), &kept)); + } + } + + Ok(moves) +} + +/// The paths below an element which stay as they are, without a trailing slash. A path below another +/// one moves along with that one, so only the outermost ones remain. +fn kept_paths(element: &RestoreElement) -> Result, String> { + let element_path = element.relative_path.as_str(); + let always_kept = ALWAYS_KEPT.iter().copied().filter(|path| path.starts_with(element_path)); + + let mut kept = Vec::new(); + for path in element.keep.iter().map(String::as_str).chain(always_kept) { + let lies_below = element_path.ends_with('/') && path.len() > element_path.len() && path.starts_with(element_path); + if !lies_below || !is_safe_relative_path(path) { + return Err(format!("A restore cannot keep '{path}' of '{element_path}'.")); + } + + kept.push(path.trim_end_matches('/').to_string()); + } + + kept.sort(); + kept.dedup(); + + let outermost = kept.iter() + .filter(|path| !kept.iter().any(|other| path.starts_with(&format!("{other}/")))) + .cloned() + .collect(); + + Ok(outermost) +} + +/// Whether a path from the plan stays within its folder: relative, with forward slashes, and without +/// `.` or `..`. +fn is_safe_relative_path(path: &str) -> bool { + !path.is_empty() + && !path.starts_with('/') + && !path.contains('\\') + && !path.contains(':') + && path.trim_end_matches('/').split('/').all(|segment| !segment.is_empty() && segment != "." && segment != "..") +} + +/// The settings files of single versions in the configuration folder, cf. SETTINGS_OF_ONE_VERSION_PREFIX. +fn settings_of_single_versions(config_directory: &Path) -> Result, String> { + let entries = match fs::read_dir(config_directory) { + Ok(entries) => entries, + Err(error) if error.kind() == io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => return Err(format!("Could not read '{}': {error}", config_directory.display())), + }; + + let mut names = Vec::new(); + for entry in entries { + let entry = entry.map_err(|error| format!("Could not read '{}': {error}", config_directory.display()))?; + let is_file = entry.file_type().map(|file_type| file_type.is_file()).unwrap_or(false); + let file_name = entry.file_name(); + if let Some(name) = file_name.to_str() && is_file && is_settings_of_single_version(name) { + names.push(name.to_string()); + } + } + + names.sort(); + Ok(names) +} + +fn is_settings_of_single_version(file_name: &str) -> bool { + file_name.len() > SETTINGS_OF_ONE_VERSION_PREFIX.len() + SETTINGS_OF_ONE_VERSION_SUFFIX.len() + && file_name.starts_with(SETTINGS_OF_ONE_VERSION_PREFIX) + && file_name.ends_with(SETTINGS_OF_ONE_VERSION_SUFFIX) +} + +/// Executes one move. +fn execute(folders: &Folders, step: &Move, retry_delays: &[Duration]) -> io::Result<()> { + let root = folders.root(step.location); + move_path(&resolve(root, &step.from), &resolve(root, &step.to), retry_delays) +} + +/// Undoes the last move the journal counts. That one may not have happened: then its source is still in +/// place, and its destination is free. +fn undo(folders: &Folders, step: &Move, retry_delays: &[Duration]) -> io::Result<()> { + let root = folders.root(step.location); + let from = resolve(root, &step.from); + let to = resolve(root, &step.to); + if exists(&from) || !exists(&to) { + return Ok(()); + } + + move_path(&to, &from, retry_delays) +} + +/// Undoes the moves the journal counts, in the reverse order. It stops at the first move which cannot +/// be undone: every further one could otherwise carry along data which that move left in the wrong +/// place. The journal then names that move as the last one, and the next start tries again from there. +fn roll_back(folders: &Folders, journal: &mut Journal, retry_delays: &[Duration], cause: &str) -> RestoreEnd { + while journal.started > 0 { + let step = &journal.moves[journal.started - 1]; + if let Err(error) = undo(folders, step, retry_delays) { + return RestoreEnd::new(RestoreOutcome::RollbackFailed, format!("{cause} Could not move '{}' back to '{}': {error}", step.to, step.from)); + } + + journal.started -= 1; + if let Err(error) = write_journal(&folders.pending(), journal, retry_delays) { + return RestoreEnd::new(RestoreOutcome::RollbackFailed, format!("{cause} {error}")); + } + } + + RestoreEnd::new(RestoreOutcome::RolledBack, cause) +} + +/// Writes the result and leaves the folders in the state the outcome calls for. +fn finish(folders: &Folders, plan: Option<&RestorePlan>, end: RestoreEnd, retry_delays: &[Duration]) -> RestoreOutcome { + match end.outcome { + RestoreOutcome::Applied => info!(Source = "Restore"; "The restore was applied: {}", end.detail), + RestoreOutcome::Refused => warn!(Source = "Restore"; "The restore was refused, nothing changed: {}", end.detail), + RestoreOutcome::RolledBack => warn!(Source = "Restore"; "The restore failed and was moved back: {}", end.detail), + RestoreOutcome::RollbackFailed => error!(Source = "Restore"; "The restore failed, and moving it back failed as well. The next start tries again: {}", end.detail), + } + + write_result(folders, plan, &end); + if end.outcome == RestoreOutcome::RollbackFailed { + return end.outcome; + } + + // + // Without READY, the next start does not try again. While the journal remains next to READY, it + // moves everything back instead, so READY goes first: + // + if let Err(error) = remove_file_if_present(&folders.pending().join(READY_FILE_NAME), retry_delays) { + error!(Source = "Restore"; "Could not remove the mark of the prepared restore: {error}"); + return end.outcome; + } + + match end.outcome { + RestoreOutcome::Applied => clean_up_after_success(folders), + + // + // The restored data went back to the pending directories, and the app removes them. What + // remains of the directories for the replaced data is empty: + // + RestoreOutcome::RolledBack => { + if let Err(error) = remove_file_if_present(&folders.pending().join(JOURNAL_FILE_NAME), retry_delays) { + error!(Source = "Restore"; "Could not remove the journal of the restore: {error}"); + } + + for root in [folders.data, folders.config] { + remove_empty_directories(&root.join(PREVIOUS_DIRECTORY)); + } + }, + + RestoreOutcome::Refused | RestoreOutcome::RollbackFailed => {}, + } + + end.outcome +} + +/// Removes what remains of the pending directories once the restored data are in place: the plan, the +/// journal, and empty folders. Anything else stays for the app. +fn clean_up_after_success(folders: &Folders) { + let pending = folders.pending(); + for file_name in [JOURNAL_FILE_NAME, PLAN_FILE_NAME] { + if let Err(error) = remove_file_if_present(&pending.join(file_name), &[]) { + warn!(Source = "Restore"; "Could not remove '{file_name}' of the applied restore: {error}"); + } + } + + for root in [folders.data, folders.config] { + let pending = root.join(PENDING_DIRECTORY); + if !remove_empty_directories(&pending) { + warn!(Source = "Restore"; "'{}' still holds files after the restore. The app removes them.", pending.display()); + } + } +} + +fn write_result(folders: &Folders, plan: Option<&RestorePlan>, end: &RestoreEnd) { + let result = RestoreResult { + outcome: end.outcome, + detail: end.detail.clone(), + prepared_at_utc: plan.map(|plan| plan.prepared_at_utc.clone()).unwrap_or_default(), + backup_created_at_utc: plan.map(|plan| plan.backup_created_at_utc.clone()).unwrap_or_default(), + backup_app_version: plan.map(|plan| plan.backup_app_version.clone()).unwrap_or_default(), + finished_at_utc: chrono::Utc::now().to_rfc3339(), + }; + + let path = folders.data.join(RESULT_FILE_NAME); + let written = serde_json::to_vec_pretty(&result) + .map_err(io::Error::other) + .and_then(|json| fs::write(&path, json)); + + if let Err(error) = written { + error!(Source = "Restore"; "Could not write the result of the restore to '{}': {error}", path.display()); + } +} + +fn read_plan(pending: &Path) -> Result { + let path = pending.join(PLAN_FILE_NAME); + let json = fs::read(&path).map_err(|error| format!("Could not read the plan '{}': {error}", path.display()))?; + serde_json::from_slice(&json).map_err(|error| format!("Could not read the plan '{}': {error}", path.display())) +} + +fn read_journal(path: &Path) -> Result { + let json = fs::read(path).map_err(|error| error.to_string())?; + serde_json::from_slice(&json).map_err(|error| error.to_string()) +} + +/// Writes the journal under another name first, so that it is either complete or the one before. +fn write_journal(pending: &Path, journal: &Journal, retry_delays: &[Duration]) -> Result<(), String> { + let temporary_path = pending.join(JOURNAL_TEMPORARY_FILE_NAME); + let written = serde_json::to_vec_pretty(journal) + .map_err(io::Error::other) + .and_then(|json| { + let mut file = fs::File::create(&temporary_path)?; + file.write_all(&json)?; + file.sync_all() + }) + .and_then(|_| with_retries(retry_delays, || fs::rename(&temporary_path, pending.join(JOURNAL_FILE_NAME)))); + + written.map_err(|error| format!("Could not write the journal of the restore: {error}")) +} + +/// Renames a file or folder, and never replaces anything at the destination. +fn move_path(from: &Path, to: &Path, retry_delays: &[Duration]) -> io::Result<()> { + if exists(to) { + return Err(io::Error::new(io::ErrorKind::AlreadyExists, format!("'{}' exists already", to.display()))); + } + + if let Some(parent) = to.parent() { + fs::create_dir_all(parent)?; + } + + with_retries(retry_delays, || fs::rename(from, to)) +} + +fn remove_file_if_present(path: &Path, retry_delays: &[Duration]) -> io::Result<()> { + match with_retries(retry_delays, || fs::remove_file(path)) { + Err(error) if error.kind() != io::ErrorKind::NotFound => Err(error), + _ => Ok(()), + } +} + +/// Runs an operation again after each of the given delays, as long as it fails for another reason +/// than a missing file. +fn with_retries(retry_delays: &[Duration], mut operation: impl FnMut() -> io::Result<()>) -> io::Result<()> { + let mut delays = retry_delays.iter(); + loop { + let error = match operation() { + Ok(()) => return Ok(()), + Err(error) => error, + }; + + match delays.next() { + Some(delay) if error.kind() != io::ErrorKind::NotFound => { + warn!(Source = "Restore"; "{error}. Trying again in {delay:?}."); + thread::sleep(*delay); + }, + + _ => return Err(error), + } + } +} + +/// Removes a folder with all folders below it, as long as none of them holds a file. Returns whether +/// the folder is gone. +fn remove_empty_directories(path: &Path) -> bool { + let Ok(metadata) = fs::symlink_metadata(path) else { + return true; + }; + + if !metadata.is_dir() { + return false; + } + + let Ok(entries) = fs::read_dir(path) else { + return false; + }; + + let mut is_empty = true; + for entry in entries.flatten() { + if !remove_empty_directories(&entry.path()) { + is_empty = false; + } + } + + is_empty && fs::remove_dir(path).is_ok() +} + +/// Whether a file or folder exists. A link counts as itself, wherever it points to. +fn exists(path: &Path) -> bool { + fs::symlink_metadata(path).is_ok() +} + +/// The path of a relative path with forward slashes, below the given folder. +fn resolve(root: &Path, relative_path: &str) -> PathBuf { + relative_path + .split('/') + .filter(|segment| !segment.is_empty()) + .fold(root.to_path_buf(), |path, segment| path.join(segment)) +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + /// The plugins of the organization which lie among the plugins of the user, as the app names them. + const ORGANIZATION_PLUGIN: &str = "plugins/company-assistant/"; + + /// A data folder and a configuration folder in a temporary directory of their own. + struct TestFolders { + _directory: TempDir, + data: PathBuf, + config: PathBuf, + } + + impl TestFolders { + fn new() -> Self { + let directory = tempfile::tempdir().unwrap(); + let data = directory.path().join("data"); + let config = directory.path().join("config"); + fs::create_dir_all(&data).unwrap(); + fs::create_dir_all(&config).unwrap(); + TestFolders { _directory: directory, data, config } + } + + fn folders(&self) -> Folders<'_> { + Folders { data: &self.data, config: &self.config } + } + + fn write(&self, location: Location, relative_path: &str, content: &str) { + let path = resolve(self.folders().root(location), relative_path); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + fs::write(path, content).unwrap(); + } + + fn read(&self, location: Location, relative_path: &str) -> Option { + fs::read_to_string(resolve(self.folders().root(location), relative_path)).ok() + } + + fn exists(&self, location: Location, relative_path: &str) -> bool { + exists(&resolve(self.folders().root(location), relative_path)) + } + + fn result(&self) -> RestoreResult { + serde_json::from_slice(&fs::read(self.data.join(RESULT_FILE_NAME)).unwrap()).unwrap() + } + + /// The data of a user who has used AI Studio for a while. + fn write_current_data(&self) { + self.write(Location::ConfigDirectory, "settings.json", "current settings"); + self.write(Location::ConfigDirectory, "settings.v6.json", "current settings of v6"); + self.write(Location::DataDirectory, "workspaces/current/chat/thread.json", "current chat"); + self.write(Location::DataDirectory, "plugins/my-old-assistant/plugin.lua", "current plugin"); + self.write(Location::DataDirectory, "plugins/.internal/languages/plugin.lua", "built-in plugin"); + self.write(Location::DataDirectory, "plugins/.config/organization/plugin.lua", "organization plugin"); + self.write(Location::DataDirectory, "plugins/company-assistant/plugin.lua", "managed plugin"); + self.write(Location::DataDirectory, "events.log", "log"); + } + + /// A restore of a backup with other settings, another chat, and another plugin, as the app + /// prepares it. + fn prepare_restore(&self, elements: &str) { + self.write(Location::ConfigDirectory, ".restore-pending/settings.json", "restored settings"); + self.write(Location::DataDirectory, ".restore-pending/workspaces/restored/chat/thread.json", "restored chat"); + self.write(Location::DataDirectory, ".restore-pending/plugins/my-assistant/plugin.lua", "restored plugin"); + self.write(Location::DataDirectory, ".restore-pending/restore.json", &plan_json(elements)); + self.write(Location::DataDirectory, ".restore-pending/READY", "2026-10-15T08:00:00.0000000+00:00"); + } + } + + /// A plan as the app writes it. + fn plan_json(elements: &str) -> String { + format!(r#"{{ + "format_version": 1, + "prepared_at_utc": "2026-10-15T08:00:00.0000000+00:00", + "backup_created_at_utc": "2026-10-01T17:30:00.0000000+00:00", + "backup_app_version": "26.10.1", + "elements": [{elements}] +}}"#) + } + + /// The elements of a full restore, as the app names them. + fn all_elements() -> String { + format!(r#" + {{ "location": "CONFIG_DIRECTORY", "relative_path": "settings.json", "keep": [] }}, + {{ "location": "DATA_DIRECTORY", "relative_path": "workspaces/", "keep": [] }}, + {{ "location": "DATA_DIRECTORY", "relative_path": "plugins/", "keep": ["plugins/.internal/", "plugins/.config/", "plugins/.config-tests/", "plugins/.lock", "{ORGANIZATION_PLUGIN}"] }}"#) + } + + /// Checks that the data are exactly as before the restore. + fn assert_current_data_untouched(test: &TestFolders) { + assert_eq!(test.read(Location::ConfigDirectory, "settings.json").as_deref(), Some("current settings")); + assert_eq!(test.read(Location::ConfigDirectory, "settings.v6.json").as_deref(), Some("current settings of v6")); + assert_eq!(test.read(Location::DataDirectory, "workspaces/current/chat/thread.json").as_deref(), Some("current chat")); + assert_eq!(test.read(Location::DataDirectory, "plugins/my-old-assistant/plugin.lua").as_deref(), Some("current plugin")); + assert_eq!(test.read(Location::DataDirectory, "plugins/.internal/languages/plugin.lua").as_deref(), Some("built-in plugin")); + assert_eq!(test.read(Location::DataDirectory, "plugins/company-assistant/plugin.lua").as_deref(), Some("managed plugin")); + assert!(!test.exists(Location::DataDirectory, "workspaces/restored")); + assert!(!test.exists(Location::DataDirectory, "plugins/my-assistant")); + } + + #[test] + fn a_restore_replaces_the_elements_and_keeps_what_belongs_to_this_computer() { + let test = TestFolders::new(); + test.write_current_data(); + test.prepare_restore(&all_elements()); + + let outcome = apply_pending_restore_in(&test.folders(), &[]); + + assert_eq!(outcome, Some(RestoreOutcome::Applied)); + assert_eq!(test.read(Location::ConfigDirectory, "settings.json").as_deref(), Some("restored settings")); + assert!(!test.exists(Location::ConfigDirectory, "settings.v6.json"), "The app would prefer it over older restored settings."); + assert_eq!(test.read(Location::DataDirectory, "workspaces/restored/chat/thread.json").as_deref(), Some("restored chat")); + assert!(!test.exists(Location::DataDirectory, "workspaces/current")); + assert_eq!(test.read(Location::DataDirectory, "plugins/my-assistant/plugin.lua").as_deref(), Some("restored plugin")); + assert!(!test.exists(Location::DataDirectory, "plugins/my-old-assistant")); + + assert_eq!(test.read(Location::DataDirectory, "plugins/.internal/languages/plugin.lua").as_deref(), Some("built-in plugin")); + assert_eq!(test.read(Location::DataDirectory, "plugins/.config/organization/plugin.lua").as_deref(), Some("organization plugin")); + assert_eq!(test.read(Location::DataDirectory, "plugins/company-assistant/plugin.lua").as_deref(), Some("managed plugin")); + assert_eq!(test.read(Location::DataDirectory, "events.log").as_deref(), Some("log"), "The logs are not part of a restore."); + + assert_eq!(test.read(Location::DataDirectory, ".restore-previous/workspaces/current/chat/thread.json").as_deref(), Some("current chat")); + assert_eq!(test.read(Location::ConfigDirectory, ".restore-previous/settings.json").as_deref(), Some("current settings")); + assert_eq!(test.read(Location::ConfigDirectory, ".restore-previous/settings.v6.json").as_deref(), Some("current settings of v6")); + + assert!(!test.exists(Location::DataDirectory, ".restore-pending")); + assert!(!test.exists(Location::ConfigDirectory, ".restore-pending")); + + let result = test.result(); + assert_eq!(result.outcome, RestoreOutcome::Applied); + assert_eq!(result.backup_app_version, "26.10.1"); + } + + #[test] + fn a_restore_which_is_not_marked_ready_changes_nothing() { + let test = TestFolders::new(); + test.write_current_data(); + test.prepare_restore(&all_elements()); + fs::remove_file(test.data.join(".restore-pending").join(READY_FILE_NAME)).unwrap(); + + let outcome = apply_pending_restore_in(&test.folders(), &[]); + + assert_eq!(outcome, None); + assert_current_data_untouched(&test); + } + + #[test] + fn a_plan_which_names_another_element_is_refused() { + for relative_path in ["", "events.log", "databases/", "../data/", "pandoc/"] { + let test = TestFolders::new(); + test.write_current_data(); + test.prepare_restore(&format!(r#"{{ "location": "DATA_DIRECTORY", "relative_path": "{relative_path}", "keep": [] }}"#)); + + let outcome = apply_pending_restore_in(&test.folders(), &[]); + + assert_eq!(outcome, Some(RestoreOutcome::Refused), "'{relative_path}' must not be replaced."); + assert_current_data_untouched(&test); + assert!(!test.exists(Location::DataDirectory, ".restore-pending/READY"), "A refused restore must not be tried again at every start."); + } + } + + #[test] + fn a_kept_path_outside_its_element_is_refused() { + for kept in ["../events.log", "workspaces/", "plugins/../events.log", "plugins/", "/plugins/x/", "plugins\\\\x"] { + let test = TestFolders::new(); + test.write_current_data(); + test.prepare_restore(&format!(r#"{{ "location": "DATA_DIRECTORY", "relative_path": "plugins/", "keep": ["{kept}"] }}"#)); + + let outcome = apply_pending_restore_in(&test.folders(), &[]); + + assert_eq!(outcome, Some(RestoreOutcome::Refused), "'{kept}' must not be kept."); + assert_current_data_untouched(&test); + } + } + + #[test] + fn a_backup_which_brings_a_kept_path_is_refused() { + let test = TestFolders::new(); + test.write_current_data(); + test.prepare_restore(&all_elements()); + test.write(Location::DataDirectory, ".restore-pending/plugins/.internal/languages/plugin.lua", "foreign built-in plugin"); + + let outcome = apply_pending_restore_in(&test.folders(), &[]); + + assert_eq!(outcome, Some(RestoreOutcome::Refused)); + assert_current_data_untouched(&test); + } + + #[test] + fn a_restore_is_refused_while_the_data_of_an_earlier_one_are_kept() { + let test = TestFolders::new(); + test.write_current_data(); + test.prepare_restore(&all_elements()); + test.write(Location::DataDirectory, ".restore-previous/workspaces/older/chat/thread.json", "older chat"); + + let outcome = apply_pending_restore_in(&test.folders(), &[]); + + assert_eq!(outcome, Some(RestoreOutcome::Refused)); + assert_current_data_untouched(&test); + assert_eq!(test.read(Location::DataDirectory, ".restore-previous/workspaces/older/chat/thread.json").as_deref(), Some("older chat")); + } + + #[test] + fn empty_folders_which_an_earlier_restore_left_do_not_block_a_restore() { + let test = TestFolders::new(); + test.write_current_data(); + test.prepare_restore(&all_elements()); + fs::create_dir_all(test.data.join(".restore-previous").join("plugins")).unwrap(); + + let outcome = apply_pending_restore_in(&test.folders(), &[]); + + assert_eq!(outcome, Some(RestoreOutcome::Applied)); + } + + #[test] + fn a_move_which_fails_moves_everything_back() { + let test = TestFolders::new(); + test.write_current_data(); + + // A file in the way of the folder of the vector stores lets the last move fail: + test.write(Location::DataDirectory, "databases", "not a folder"); + test.write(Location::DataDirectory, ".restore-pending/databases/vector_database/stores/store_1/segment", "restored store"); + test.prepare_restore(&format!(r#"{}, {{ "location": "DATA_DIRECTORY", "relative_path": "databases/vector_database/stores/", "keep": [] }}"#, all_elements())); + + let outcome = apply_pending_restore_in(&test.folders(), &[]); + + assert_eq!(outcome, Some(RestoreOutcome::RolledBack)); + assert_eq!(test.result().outcome, RestoreOutcome::RolledBack); + assert_current_data_untouched(&test); + assert!(!test.exists(Location::DataDirectory, ".restore-previous")); + assert!(!test.exists(Location::ConfigDirectory, ".restore-previous")); + assert!(!test.exists(Location::DataDirectory, ".restore-pending/READY")); + assert!(!test.exists(Location::DataDirectory, ".restore-pending/journal.json")); + assert_eq!(test.read(Location::DataDirectory, ".restore-pending/workspaces/restored/chat/thread.json").as_deref(), Some("restored chat"), "The app removes the restored data."); + } + + #[test] + fn a_restore_which_stopped_in_the_middle_is_moved_back_at_the_next_start() { + let move_count = { + let test = TestFolders::new(); + test.write_current_data(); + test.prepare_restore(&all_elements()); + plan_moves(&test.folders(), &read_plan(&test.folders().pending()).unwrap()).unwrap().len() + }; + + for completed_moves in 0..=move_count { + let test = TestFolders::new(); + test.write_current_data(); + test.prepare_restore(&all_elements()); + + let mut journal = begin(&test.folders(), &read_plan(&test.folders().pending()).unwrap(), &[]).unwrap(); + for _ in 0..completed_moves { + execute_next(&test.folders(), &mut journal, &[]).unwrap(); + } + + let outcome = apply_pending_restore_in(&test.folders(), &[]); + + assert_eq!(outcome, Some(RestoreOutcome::RolledBack), "after {completed_moves} of {move_count} moves"); + assert_current_data_untouched(&test); + assert!(!test.exists(Location::DataDirectory, ".restore-previous"), "after {completed_moves} of {move_count} moves"); + assert!(!test.exists(Location::DataDirectory, ".restore-pending/READY")); + assert_eq!(apply_pending_restore_in(&test.folders(), &[]), None, "The next start finds nothing to do."); + } + } + + #[test] + fn a_clean_up_which_stopped_is_finished_at_the_next_start() { + let test = TestFolders::new(); + test.write(Location::DataDirectory, ".restore-pending/journal.json", r#"{ "moves": [], "started": 0 }"#); + test.write(Location::DataDirectory, ".restore-pending/restore.json", &plan_json("")); + fs::create_dir_all(test.data.join(".restore-pending").join("databases").join("sqlite")).unwrap(); + + let outcome = apply_pending_restore_in(&test.folders(), &[]); + + assert_eq!(outcome, None); + assert!(!test.exists(Location::DataDirectory, ".restore-pending")); + } + + #[test] + fn a_kept_path_below_another_one_moves_along_with_it() { + let element = RestoreElement { + location: Location::DataDirectory, + relative_path: "plugins/".to_string(), + keep: vec!["plugins/.config/".to_string(), "plugins/.config/organization/".to_string(), "plugins/.lock".to_string()], + }; + + assert_eq!(kept_paths(&element).unwrap(), vec!["plugins/.config", "plugins/.config-tests", "plugins/.internal", "plugins/.lock"]); + } + + #[test] + fn the_settings_of_single_versions_are_recognized_by_their_name() { + assert!(is_settings_of_single_version("settings.v6.json")); + assert!(is_settings_of_single_version("settings.v12.json")); + assert!(!is_settings_of_single_version("settings.json")); + assert!(!is_settings_of_single_version("settings.v.json")); + assert!(!is_settings_of_single_version("settings.v6.json.tmp")); + } +} \ No newline at end of file diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index 5fcb30a4f..dec7fd142 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -17,6 +17,7 @@ pub mod image; pub mod pdfium; pub mod pandoc; pub mod qdrant_edge_database; +pub mod data_restore; pub mod certificate_factory; pub mod runtime_api_token; pub mod stale_process_cleanup; diff --git a/runtime/src/qdrant_edge_database.rs b/runtime/src/qdrant_edge_database.rs index 7352e7d0c..a82a36abb 100644 --- a/runtime/src/qdrant_edge_database.rs +++ b/runtime/src/qdrant_edge_database.rs @@ -34,6 +34,11 @@ const STORE_DISPLAY_NAME_MARKER_TEMP: &str = "data_source_name.tmp"; const WAL_DIRECTORY: &str = "wal"; const CLOSED_WAL_SEGMENT_PREFIX: &str = "closed-"; +/// The directory below the data directory in which a backup collects its copies, one directory per +/// backup. The app removes a backup's directory once the backup is written. +const BACKUP_STAGING_DIRECTORY: &str = ".backup-staging"; +const MAX_BACKUP_ID_LENGTH: usize = 64; + /// Marks a response whose store exists on disk but cannot be opened. The .NET side keys its repair /// offer off this value instead of parsing `issue`, so rewording the message stays harmless. const ISSUE_CODE_STORE_UNREADABLE: &str = "store-unreadable"; @@ -100,6 +105,11 @@ pub struct EnsureQdrantEdgeStoreRequest { pub vector_size: usize, } +#[derive(Deserialize)] +pub struct CheckQdrantEdgeStoreRequest { + pub store_name: String, +} + #[derive(Deserialize)] pub struct InsertQdrantEdgeEmbeddingRequest { pub store_name: String, @@ -133,6 +143,39 @@ pub struct DeleteQdrantEdgeStoreRequest { pub store_name: String, } +#[derive(Deserialize)] +pub struct CopyQdrantEdgeStoreForBackupRequest { + pub backup_id: String, + pub store_name: String, +} + +/// A store which a backup takes along. +#[derive(Serialize)] +pub struct QdrantEdgeBackupStore { + pub store_name: String, + pub bytes: u64, +} + +/// What a backup needs to copy the stores, one request per store. +#[derive(Serialize)] +pub struct QdrantEdgeBackupPreparation { + /// Names the backup in the requests which copy its stores. + pub backup_id: String, + + /// The directory of the backup's copies. Below it, every copy has the place it has below the + /// data directory, so the backup can take the copies along under the same paths. + pub staging_path: String, + + /// Every store which was created completely, in the order of their names. + pub stores: Vec, +} + +/// How much space is free on one disk. +struct DiskSpace { + mount_point: PathBuf, + available_bytes: u64, +} + #[derive(Serialize)] pub struct QdrantEdgeResponse { pub success: bool, @@ -344,6 +387,17 @@ impl QdrantEdgeDatabase { }) } + /// Whether a store was created completely, without creating or loading it. + /// + /// The app asks this when its index database knows about a store. Should the store be gone, + /// the user decides whether to rebuild it, since that sends every document to the embedding + /// provider once more. Creating an empty store on the way, as `ensure_store_exists` does, + /// would hide that the vectors are missing. + fn store_exists(&self, store_name: &str) -> QdrantEdgeResult { + let path = self.store_path(store_name)?; + store_is_initialized(&path, store_name) + } + fn insert_embedding(&mut self, store_name: &str, points: Vec) -> QdrantEdgeResult<()> { let Some(first_point) = points.first() else { return Ok(()); @@ -440,6 +494,92 @@ impl QdrantEdgeDatabase { Ok(()) } + /// Prepares a backup of the stores: lists them, checks that their copies fit on the disk, and + /// creates the directory of the backup's copies below the data directory. + /// + /// Only stores which were created completely count. One whose creation broke off holds nothing + /// the app could use, and the next request against it removes it anyway. + fn prepare_backup(&self, data_directory: &Path) -> QdrantEdgeResult { + let stores = self.backup_stores()?; + let required_bytes = stores.iter().map(|store| store.bytes).sum(); + ensure_enough_space(required_bytes, available_space(data_directory))?; + + let backup_id = format!("{:016x}", rand::random::()); + let staging_path = backup_staging_path(data_directory, &backup_id)?; + fs::create_dir_all(&staging_path).map_err(|error| { + format!("Failed to create the directory '{}' for the copies of the vector stores: {error}", staging_path.display()) + })?; + + info!(Source = "Qdrant Edge"; "Prepared the backup '{backup_id}' of {} vector store(s) with {:.1} MiB.", stores.len(), required_bytes as f64 / (1024.0 * 1024.0)); + Ok(QdrantEdgeBackupPreparation { + backup_id, + staging_path: staging_path.to_string_lossy().to_string(), + stores, + }) + } + + fn backup_stores(&self) -> QdrantEdgeResult> { + let stores_path = self.base_path.join("stores"); + if !stores_path.is_dir() { + return Ok(Vec::new()); + } + + let mut stores = Vec::new(); + for entry in fs::read_dir(&stores_path)? { + let path = entry?.path(); + let Ok(store_name) = fs::read_to_string(path.join(STORE_INITIALIZATION_MARKER)) else { + continue; + }; + + let belongs_to_its_directory = self.store_path(&store_name).is_ok_and(|store_path| store_path == path); + if !belongs_to_its_directory || !store_is_initialized(&path, &store_name).unwrap_or(false) { + continue; + } + + stores.push(QdrantEdgeBackupStore { + bytes: directory_size(&path)?, + store_name, + }); + } + + stores.sort_by(|left, right| left.store_name.cmp(&right.store_name)); + Ok(stores) + } + + /// Copies one store into the directory of a backup's copies, and returns how many bytes it copied. + /// + /// A loaded store is flushed and unloaded first: everything it holds is on the disk then, and + /// nothing changes its files while they are copied, since the lock of the database is held all + /// the while. The next request loads the store again. A store which was not created completely + /// is not copied. + fn copy_store_for_backup(&mut self, store_name: &str, data_directory: &Path, backup_id: &str) -> QdrantEdgeResult { + let staging_path = backup_staging_path(data_directory, backup_id)?; + if !staging_path.is_dir() { + return Err(format!("The backup '{backup_id}' was not prepared.").into()); + } + + let path = self.store_path(store_name)?; + if !store_is_initialized(&path, store_name)? { + return Ok(0); + } + + if let Some(shard) = self.shards.get(store_name) { + shard.flush()?; + } + + drop(self.shards.remove(store_name)); + let relative_path = path.strip_prefix(data_directory).map_err(|_| { + format!("The vector store '{store_name}' at '{}' does not lie below the data directory '{}'.", path.display(), data_directory.display()) + })?; + + let target_path = staging_path.join(relative_path); + let copied_bytes = copy_directory(&path, &target_path).map_err(|error| { + format!("Failed to copy the vector store '{store_name}' to '{}': {error}", target_path.display()) + })?; + + Ok(copied_bytes) + } + fn delete_store(&mut self, store_name: &str) -> QdrantEdgeResult<()> { self.shards.remove(store_name); @@ -461,16 +601,114 @@ fn store_directory_name(store_name: &str) -> String { format!("store_{stable_id}") } -fn qdrant_edge_base_path() -> QdrantEdgeResult { +fn data_directory() -> QdrantEdgeResult { let data_directory = DATA_DIRECTORY .get() .ok_or("The data directory has not been initialized.")?; - Ok(Path::new(data_directory) + Ok(PathBuf::from(data_directory)) +} + +fn qdrant_edge_base_path() -> QdrantEdgeResult { + Ok(data_directory()? .join("databases") .join("vector_database")) } +/// The directory of one backup's copies. The backup ID must not be able to name any other path. +fn backup_staging_path(data_directory: &Path, backup_id: &str) -> QdrantEdgeResult { + if backup_id.is_empty() || backup_id.len() > MAX_BACKUP_ID_LENGTH || !backup_id.chars().all(|c| c.is_ascii_hexdigit()) { + return Err(format!("'{backup_id}' is not a valid backup ID.").into()); + } + + Ok(data_directory.join(BACKUP_STAGING_DIRECTORY).join(backup_id)) +} + +/// Refuses a backup whose copies would not fit on the disk. When the free space cannot be told, the +/// backup goes ahead: running out of space then fails the copy, which removes nothing. +fn ensure_enough_space(required_bytes: u64, available_bytes: Option) -> QdrantEdgeResult<()> { + match available_bytes { + Some(available_bytes) if available_bytes < required_bytes => Err(format!( + "The copies of the vector stores need {:.1} MiB, but only {:.1} MiB are free on the disk.", + required_bytes as f64 / (1024.0 * 1024.0), + available_bytes as f64 / (1024.0 * 1024.0) + ).into()), + + Some(_) => Ok(()), + + None => { + warn!(Source = "Qdrant Edge"; "Could not tell the free space on the disk of the data directory. Copying the vector stores anyway."); + Ok(()) + }, + } +} + +/// The free space on the disk which holds the given path, or None when it cannot be told. +/// +/// The canonical path comes first, since the path the app knows may lead through a symbolic link +/// onto another disk. On Windows, the canonical path carries a prefix no mount point has, so the +/// path as it is serves as the fallback. +fn available_space(path: &Path) -> Option { + let disks = sysinfo::Disks::new_with_refreshed_list() + .list() + .iter() + .map(|disk| DiskSpace { + mount_point: disk.mount_point().to_path_buf(), + available_bytes: disk.available_space(), + }) + .collect::>(); + + fs::canonicalize(path) + .ok() + .and_then(|canonical_path| available_space_on(&canonical_path, &disks)) + .or_else(|| available_space_on(path, &disks)) +} + +/// The free space of the disk with the most specific mount point which holds the given path. +fn available_space_on(path: &Path, disks: &[DiskSpace]) -> Option { + disks + .iter() + .filter(|disk| path.starts_with(&disk.mount_point)) + .max_by_key(|disk| disk.mount_point.components().count()) + .map(|disk| disk.available_bytes) +} + +/// The size of all files below a directory. Symbolic links are not followed. +fn directory_size(path: &Path) -> std::io::Result { + let mut size = 0; + for entry in fs::read_dir(path)? { + let entry = entry?; + let file_type = entry.file_type()?; + if file_type.is_dir() { + size += directory_size(&entry.path())?; + } else if file_type.is_file() { + size += entry.metadata()?.len(); + } + } + + Ok(size) +} + +/// Copies a directory with all its files, and returns how many bytes it copied. Symbolic links are +/// not copied. +fn copy_directory(source: &Path, target: &Path) -> std::io::Result { + fs::create_dir_all(target)?; + + let mut copied_bytes = 0; + for entry in fs::read_dir(source)? { + let entry = entry?; + let file_type = entry.file_type()?; + let target_path = target.join(entry.file_name()); + if file_type.is_dir() { + copied_bytes += copy_directory(&entry.path(), &target_path)?; + } else if file_type.is_file() { + copied_bytes += fs::copy(entry.path(), &target_path)?; + } + } + + Ok(copied_bytes) +} + pub async fn qdrant_edge_info() -> Json { let (current_status, mut unavailable_reason) = { let status = QDRANT_EDGE_STATUS.lock().unwrap(); @@ -518,6 +756,12 @@ pub async fn ensure_qdrant_edge_store(Json(request): Json) -> Json> { + run_qdrant_edge_request(move |database| { + database.store_exists(&request.store_name) + }).await +} + pub async fn insert_qdrant_edge_embedding(Json(request): Json) -> Json> { run_qdrant_edge_request(move |database| { database.insert_embedding(&request.store_name, request.points) @@ -548,6 +792,18 @@ pub async fn delete_qdrant_edge_store(Json(request): Json Json> { + run_qdrant_edge_request(move |database| { + database.prepare_backup(&data_directory()?) + }).await +} + +pub async fn copy_qdrant_edge_store_for_backup(Json(request): Json) -> Json> { + run_qdrant_edge_request(move |database| { + database.copy_store_for_backup(&request.store_name, &data_directory()?, &request.backup_id) + }).await +} + pub fn start_qdrant_edge_database(app_handle: tauri::AppHandle) { set_qdrant_edge_starting(); remove_obsolete_qdrant_sidecar_files(&app_handle); @@ -1131,6 +1387,26 @@ mod tests { fs::remove_dir_all(test_directory).unwrap(); } + #[test] + fn asking_whether_a_store_exists_never_creates_it() { + let test_directory = test_directory("exists"); + let store_name = "rag_6cc665a82b1e4d42bc748015b7b391ec"; + let mut database = QdrantEdgeDatabase::new(test_directory.clone()); + + assert!(!database.store_exists(store_name).unwrap()); + assert!(!database.store_exists(store_name).unwrap(), "Asking once must not create the store for the next question."); + assert!(!database.store_path(store_name).unwrap().exists()); + + database.ensure_store_exists(store_name, "Some source", 3).unwrap(); + assert!(database.store_exists(store_name).unwrap()); + + database.delete_store(store_name).unwrap(); + assert!(!database.store_exists(store_name).unwrap()); + + drop(database); + fs::remove_dir_all(test_directory).unwrap(); + } + #[test] fn an_unreadable_store_is_reported_but_never_deleted() { let test_directory = test_directory("unreadable"); @@ -1318,6 +1594,96 @@ mod tests { assert!(!info.is_available); } + #[tokio::test(flavor = "multi_thread", worker_threads = 1)] + async fn copying_a_store_for_a_backup_leaves_the_runtime_free() { + let request = copy_qdrant_edge_store_for_backup(Json(CopyQdrantEdgeStoreForBackupRequest { + backup_id: "0123abcd".to_string(), + store_name: SEARCH_STORE.to_string(), + })); + + let response = assert_runtime_stays_free(&QDRANT_EDGE_DATABASE, request).await; + + // No database runs in the tests, so the request ends as soon as it gets the lock. + assert!(!response.success); + } + + #[test] + fn a_backup_copies_every_complete_store_and_the_copy_loads_on_its_own() { + let data_directory = test_directory("backup"); + let base_path = data_directory.join("databases").join("vector_database"); + let mut database = store_with_one_point_per_axis(&base_path); + + // A store whose creation broke off: its directory exists, but it holds no store. + fs::create_dir_all(base_path.join("stores").join("store_broken")).unwrap(); + + let preparation = database.prepare_backup(&data_directory).unwrap(); + assert_eq!(preparation.stores.iter().map(|store| store.store_name.as_str()).collect::>(), vec![SEARCH_STORE]); + assert!(preparation.stores[0].bytes > 0); + + let copied_bytes = database.copy_store_for_backup(SEARCH_STORE, &data_directory, &preparation.backup_id).unwrap(); + assert!(copied_bytes > 0); + + // The copy lies where the store lies below the data directory, and it is a complete store: + let staging_path = PathBuf::from(&preparation.staging_path); + let mut copy = QdrantEdgeDatabase::new(staging_path.join("databases").join("vector_database")); + let found = copy.search_embedding(SEARCH_STORE, search_vector(), 3, None).unwrap(); + assert_eq!(found_point_ids(&found), vec![POINT_X, POINT_Y, POINT_Z]); + + // The copy unloaded the store, and the next request loads it again: + let found = database.search_embedding(SEARCH_STORE, search_vector(), 3, None).unwrap(); + assert_eq!(found_point_ids(&found), vec![POINT_X, POINT_Y, POINT_Z]); + + drop(copy); + drop(database); + fs::remove_dir_all(data_directory).unwrap(); + } + + #[test] + fn a_store_is_copied_only_into_a_prepared_backup() { + let data_directory = test_directory("backup-unprepared"); + let base_path = data_directory.join("databases").join("vector_database"); + let mut database = store_with_one_point_per_axis(&base_path); + + let error = database.copy_store_for_backup(SEARCH_STORE, &data_directory, "0123456789abcdef").unwrap_err(); + assert!(error.to_string().contains("was not prepared")); + + drop(database); + fs::remove_dir_all(data_directory).unwrap(); + } + + #[test] + fn a_backup_id_cannot_name_another_directory() { + let data_directory = Path::new("/data"); + + assert!(backup_staging_path(data_directory, "").is_err()); + assert!(backup_staging_path(data_directory, "../settings").is_err()); + assert!(backup_staging_path(data_directory, "/etc").is_err()); + assert!(backup_staging_path(data_directory, &"a".repeat(MAX_BACKUP_ID_LENGTH + 1)).is_err()); + assert_eq!( + backup_staging_path(data_directory, "0123abcd").unwrap(), + data_directory.join(BACKUP_STAGING_DIRECTORY).join("0123abcd") + ); + } + + #[test] + fn a_backup_which_does_not_fit_on_the_disk_is_refused() { + assert!(ensure_enough_space(2_000, Some(1_000)).is_err()); + assert!(ensure_enough_space(1_000, Some(1_000)).is_ok()); + assert!(ensure_enough_space(2_000, None).is_ok(), "Without knowing the free space, the copy itself fails when the disk is full."); + } + + #[test] + fn the_free_space_comes_from_the_disk_with_the_most_specific_mount_point() { + let disks = vec![ + DiskSpace { mount_point: PathBuf::from("/"), available_bytes: 10 }, + DiskSpace { mount_point: PathBuf::from("/home"), available_bytes: 20 }, + ]; + + assert_eq!(available_space_on(Path::new("/home/user/data"), &disks), Some(20)); + assert_eq!(available_space_on(Path::new("/var/data"), &disks), Some(10)); + assert_eq!(available_space_on(Path::new("relative/data"), &disks), None); + } + const SEARCH_STORE: &str = "rag_6cc665a82b1e4d42bc748015b7b391ec"; const POINT_X: &str = "0b5f1e8a-3c2d-4e6f-9a1b-7c8d9e0f1a2b"; const POINT_Y: &str = "1c6a2f9b-4d3e-4f70-8b2c-8d9e0f1a2b3c"; diff --git a/runtime/src/runtime_api.rs b/runtime/src/runtime_api.rs index b5dc11f4b..64c0d65fd 100644 --- a/runtime/src/runtime_api.rs +++ b/runtime/src/runtime_api.rs @@ -37,17 +37,21 @@ fn create_router() -> Router { .route("/system/dotnet/ready", get(crate::dotnet::dotnet_ready)) .route("/system/qdrant-edge/info", get(crate::qdrant_edge_database::qdrant_edge_info)) .route("/system/qdrant-edge/ensure", post(crate::qdrant_edge_database::ensure_qdrant_edge_store)) + .route("/system/qdrant-edge/exists", post(crate::qdrant_edge_database::check_qdrant_edge_store)) .route("/system/qdrant-edge/insert", post(crate::qdrant_edge_database::insert_qdrant_edge_embedding)) .route("/system/qdrant-edge/search", post(crate::qdrant_edge_database::search_qdrant_edge_embeddings)) .route("/system/qdrant-edge/delete-file", post(crate::qdrant_edge_database::delete_qdrant_edge_embedding_by_file)) .route("/system/qdrant-edge/optimize", post(crate::qdrant_edge_database::optimize_qdrant_edge_store)) .route("/system/qdrant-edge/delete-store", post(crate::qdrant_edge_database::delete_qdrant_edge_store)) + .route("/system/qdrant-edge/backup/prepare", post(crate::qdrant_edge_database::prepare_qdrant_edge_backup)) + .route("/system/qdrant-edge/backup/copy", post(crate::qdrant_edge_database::copy_qdrant_edge_store_for_backup)) .route("/clipboard/set", post(crate::clipboard::set_clipboard)) .route("/share/file", post(crate::share_sheet::share_file)) .route("/events", get(crate::app_window::get_event_stream)) .route("/updates/check", get(crate::app_window::check_for_update)) .route("/updates/install", get(crate::app_window::install_update)) .route("/app/exit", post(crate::app_window::exit_app)) + .route("/app/restart", post(crate::app_window::restart_app)) .route("/select/directory", post(crate::file_actions::select_directory)) .route("/select/file", post(crate::file_actions::select_file)) .route("/select/files", post(crate::file_actions::select_files))