mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-17 17:38:00 +00:00
Added backups of all data, restorable on another computer (#1008)
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
j-erlerandThorsten Sommer authored and 
GitHub committed
1 parent
ab45580473
commit
936043319c
144 files changed
+10417
-424
No files matched your search
@@ -0,0 +1,119 @@
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What AI Studio keeps in its data and configuration folders, and which of it belongs into a backup.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is a positive list. A backup takes along only the entries marked as included, and restoring
|
||||
/// one replaces only those. Everything else in these folders stays out of a backup and survives a
|
||||
/// restore: the leftovers of a plugin installation, a file somebody else put there, or a folder a
|
||||
/// future version adds before anybody listed it here.<br/><br/>
|
||||
///
|
||||
/// The entries which stay out are listed, too. The folder panel on the information page tells users
|
||||
/// who back up these folders with a tool of their own what they may leave out, and nobody has to guess
|
||||
/// whether a folder was forgotten or left out on purpose. Whoever adds a file or folder to the data or
|
||||
/// configuration folder adds it here, one way or the other.<br/><br/>
|
||||
///
|
||||
/// The runtime, which applies a restore, keeps its own list of the included entries and refuses to
|
||||
/// replace anything else (REPLACEABLE_ELEMENTS in runtime/src/data_restore.rs). An entry included here
|
||||
/// goes there as well.
|
||||
/// </remarks>
|
||||
public static class DataBackupContents
|
||||
{
|
||||
/// <summary>
|
||||
/// The separator of the relative paths, the same on every OS.
|
||||
/// </summary>
|
||||
public const char SEPARATOR = '/';
|
||||
|
||||
/// <summary>
|
||||
/// The folder of the plugins, within the data folder.
|
||||
/// </summary>
|
||||
public const string PLUGINS_FOLDER = "plugins/";
|
||||
|
||||
/// <summary>
|
||||
/// Every file and folder AI Studio keeps directly in its data or configuration folder, and those
|
||||
/// below the folders of the plugins and the databases which need a decision of their own.
|
||||
/// </summary>
|
||||
public static readonly IReadOnlyList<DataBackupEntry> ENTRIES =
|
||||
[
|
||||
new(DataBackupLocation.CONFIG_DIRECTORY, "settings.json", DataBackupEntryKind.SETTINGS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "workspaces/", DataBackupEntryKind.WORKSPACES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "tempChats/", DataBackupEntryKind.TEMPORARY_CHATS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, PLUGINS_FOLDER, DataBackupEntryKind.PLUGINS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "tokenizers/", DataBackupEntryKind.TOKENIZERS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "visualBriefings/", DataBackupEntryKind.VISUAL_BRIEFINGS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "databases/sqlite/", DataBackupEntryKind.DATA_SOURCE_INDEX),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "databases/vector_database/stores/", DataBackupEntryKind.VECTOR_STORES),
|
||||
|
||||
new(DataBackupLocation.CONFIG_DIRECTORY, "settings.v*.json", DataBackupEntryKind.SETTINGS_OF_OTHER_VERSIONS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "plugins/.internal/", DataBackupEntryKind.BUILT_IN_PLUGINS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "plugins/.config/", DataBackupEntryKind.ORGANIZATION_PLUGINS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "plugins/.config-tests/", DataBackupEntryKind.ORGANIZATION_PLUGINS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "plugins/.lock", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, ".plugin-staging/", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, ".plugin-install-backups/", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, ".plugin-delete-backups/", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "media-staging/", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "mailAttachments/", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "mindwork_ai_studio.pid", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, DataBackupFormat.STAGING_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, DataRestoreFormat.PENDING_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.CONFIG_DIRECTORY, DataRestoreFormat.PENDING_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, DataRestoreFormat.PREVIOUS_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.CONFIG_DIRECTORY, DataRestoreFormat.PREVIOUS_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, DataRestoreFormat.RESULT_FILE_NAME, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "pandoc/", DataBackupEntryKind.DOWNLOADED_TOOLS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "data-protection-keys/", DataBackupEntryKind.INTERFACE_KEYS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "events.log", DataBackupEntryKind.LOGS),
|
||||
|
||||
// The runtime hides the log of the startup on Linux and macOS:
|
||||
new(DataBackupLocation.DATA_DIRECTORY, OperatingSystem.IsWindows() ? "AI Studio Events.log" : ".AI Studio Events.log", DataBackupEntryKind.LOGS),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Whether a path lies within an entry: below it for a folder, or the entry itself for a file.
|
||||
/// </summary>
|
||||
/// <param name="entryPath">The path of the entry, as the list holds it.</param>
|
||||
/// <param name="relativePath">The path to check, relative to the same folder and with forward slashes.</param>
|
||||
/// <returns>True when the entry covers the path.</returns>
|
||||
public static bool Covers(string entryPath, string relativePath) => entryPath.EndsWith(SEPARATOR)
|
||||
? relativePath.StartsWith(entryPath, PathTools.COMPARISON)
|
||||
: relativePath.Equals(entryPath, PathTools.COMPARISON);
|
||||
|
||||
/// <summary>
|
||||
/// Finds the plugins which the organization deployed into the folder of the user's plugins.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The config server keeps the plugins of an organization in folders of their own, which the list
|
||||
/// leaves out as a whole. A device management may put them anywhere, though, so their path proves
|
||||
/// nothing: such a plugin declares itself as deployed by the organization instead. A backup leaves
|
||||
/// it out, because the organization brings it back, and a user with a backup tool of their own
|
||||
/// needs to know which folders these are.
|
||||
/// </remarks>
|
||||
/// <param name="plugins">The plugins AI Studio found.</param>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <returns>The folders of these plugins within the data folder, in the form of the list, ordered.</returns>
|
||||
public static IReadOnlyList<string> FindOrganizationPluginsAmongUserPlugins(IEnumerable<IAvailablePlugin> plugins, string dataDirectory)
|
||||
{
|
||||
var excludedPluginFolders = ENTRIES
|
||||
.Where(entry => !entry.IsIncluded && entry.IsFolder && entry.Location is DataBackupLocation.DATA_DIRECTORY && entry.RelativePath.StartsWith(PLUGINS_FOLDER, StringComparison.Ordinal))
|
||||
.Select(entry => entry.RelativePath)
|
||||
.ToList();
|
||||
|
||||
//
|
||||
// Relative to the data folder and with forward slashes, the plugin folders take the form of
|
||||
// the list. That is the form in which AI Studio stores such paths, too:
|
||||
//
|
||||
return plugins
|
||||
.Where(plugin => plugin.IsManagedByConfigServer)
|
||||
.Select(plugin => DataDirectoryRelativePathConverter.ToStored(plugin.LocalPath, dataDirectory).TrimEnd(SEPARATOR) + SEPARATOR)
|
||||
.Where(folder => folder.Length > PLUGINS_FOLDER.Length
|
||||
&& folder.StartsWith(PLUGINS_FOLDER, PathTools.COMPARISON)
|
||||
&& !excludedPluginFolders.Any(excluded => folder.StartsWith(excluded, PathTools.COMPARISON)))
|
||||
.Distinct(PathTools.COMPARER)
|
||||
.Order(PathTools.COMPARER)
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// A file or folder AI Studio keeps in its data or configuration folder.
|
||||
/// </summary>
|
||||
/// <param name="Location">The folder in which the entry lies.</param>
|
||||
/// <param name="RelativePath">The path of the entry within that folder, with forward slashes. A folder ends with a slash.</param>
|
||||
/// <param name="Kind">What the entry holds, and with it, whether it belongs into a backup.</param>
|
||||
public sealed record DataBackupEntry(DataBackupLocation Location, string RelativePath, DataBackupEntryKind Kind)
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the entry belongs into a backup.
|
||||
/// </summary>
|
||||
public bool IsIncluded => this.Kind is DataBackupEntryKind.SETTINGS
|
||||
or DataBackupEntryKind.WORKSPACES
|
||||
or DataBackupEntryKind.TEMPORARY_CHATS
|
||||
or DataBackupEntryKind.PLUGINS
|
||||
or DataBackupEntryKind.TOKENIZERS
|
||||
or DataBackupEntryKind.VISUAL_BRIEFINGS
|
||||
or DataBackupEntryKind.DATA_SOURCE_INDEX
|
||||
or DataBackupEntryKind.VECTOR_STORES;
|
||||
|
||||
/// <summary>
|
||||
/// Whether the entry is a folder rather than a single file.
|
||||
/// </summary>
|
||||
public bool IsFolder => this.RelativePath.EndsWith(DataBackupContents.SEPARATOR);
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What an entry of the backup contents holds, and with it, whether it belongs into a backup.
|
||||
/// </summary>
|
||||
public enum DataBackupEntryKind
|
||||
{
|
||||
/// <summary>
|
||||
/// The settings. A backup leaves out what a configuration plugin of the organization manages.
|
||||
/// </summary>
|
||||
SETTINGS,
|
||||
|
||||
/// <summary>
|
||||
/// The chats of the workspaces, with their transcripts.
|
||||
/// </summary>
|
||||
WORKSPACES,
|
||||
|
||||
/// <summary>
|
||||
/// The temporary chats, with their transcripts.
|
||||
/// </summary>
|
||||
TEMPORARY_CHATS,
|
||||
|
||||
/// <summary>
|
||||
/// The plugins of the user. A backup leaves out the plugins of the organization.
|
||||
/// </summary>
|
||||
PLUGINS,
|
||||
|
||||
/// <summary>
|
||||
/// The copies of the tokenizers the user chose for a provider.
|
||||
/// </summary>
|
||||
TOKENIZERS,
|
||||
|
||||
/// <summary>
|
||||
/// The projects of the Visual Briefing Assistant.
|
||||
/// </summary>
|
||||
VISUAL_BRIEFINGS,
|
||||
|
||||
/// <summary>
|
||||
/// The SQLite index of the data sources: which files are embedded, and their text chunks.
|
||||
/// </summary>
|
||||
DATA_SOURCE_INDEX,
|
||||
|
||||
/// <summary>
|
||||
/// The vector stores, which hold the embeddings of the data sources.
|
||||
/// </summary>
|
||||
VECTOR_STORES,
|
||||
|
||||
/// <summary>
|
||||
/// The copies of the settings which every version of AI Studio keeps for itself.
|
||||
/// </summary>
|
||||
SETTINGS_OF_OTHER_VERSIONS,
|
||||
|
||||
/// <summary>
|
||||
/// The plugins which come with AI Studio, such as the language packs.
|
||||
/// </summary>
|
||||
BUILT_IN_PLUGINS,
|
||||
|
||||
/// <summary>
|
||||
/// The plugins which the organization deploys, or tries out for one session.
|
||||
/// </summary>
|
||||
ORGANIZATION_PLUGINS,
|
||||
|
||||
/// <summary>
|
||||
/// Files AI Studio needs only for a moment, while it works on something, and removes again.
|
||||
/// </summary>
|
||||
TEMPORARY_FILES,
|
||||
|
||||
/// <summary>
|
||||
/// Tools AI Studio downloaded, which depend on the OS, such as Pandoc.
|
||||
/// </summary>
|
||||
DOWNLOADED_TOOLS,
|
||||
|
||||
/// <summary>
|
||||
/// The keys with which the user interface protects its data. Only the Flatpak keeps them in the data
|
||||
/// folder, since the home folder is read-only there, and AI Studio creates new ones when they are missing.
|
||||
/// </summary>
|
||||
INTERFACE_KEYS,
|
||||
|
||||
/// <summary>
|
||||
/// The log files.
|
||||
/// </summary>
|
||||
LOGS,
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// The format of a backup: a ZIP file which holds the data folder below DATA_ROOT and the
|
||||
/// configuration folder below CONFIG_ROOT, each in the layout AI Studio keeps on the disk, and the
|
||||
/// manifest as its last entry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// What goes in follows DataBackupContents. The paths below the roots are the relative paths of
|
||||
/// that list, so restoring a backup is unpacking it. The manifest comes last: an archive which lacks
|
||||
/// it was not written completely.
|
||||
/// </remarks>
|
||||
public static class DataBackupFormat
|
||||
{
|
||||
/// <summary>
|
||||
/// The version of the format this version of AI Studio writes. A restore refuses a backup of a
|
||||
/// later version, since it cannot know what such a backup holds.
|
||||
/// </summary>
|
||||
public const int FORMAT_VERSION = 1;
|
||||
|
||||
/// <summary>
|
||||
/// The file extension of a backup, named like the one of plugin archives.
|
||||
/// </summary>
|
||||
public const string FILE_EXTENSION = ".mwbackup";
|
||||
|
||||
/// <summary>
|
||||
/// The manifest, the last entry of every backup.
|
||||
/// </summary>
|
||||
public const string MANIFEST_ENTRY_NAME = "manifest.json";
|
||||
|
||||
/// <summary>
|
||||
/// Where the data folder lies within a backup.
|
||||
/// </summary>
|
||||
public const string DATA_ROOT = "data/";
|
||||
|
||||
/// <summary>
|
||||
/// Where the configuration folder lies within a backup.
|
||||
/// </summary>
|
||||
public const string CONFIG_ROOT = "config/";
|
||||
|
||||
/// <summary>
|
||||
/// The directory in the data folder where the copies of the vector stores wait while a backup runs.
|
||||
/// The runtime creates it; keep the name in sync with BACKUP_STAGING_DIRECTORY there.
|
||||
/// </summary>
|
||||
public const string STAGING_DIRECTORY = ".backup-staging/";
|
||||
|
||||
/// <summary>
|
||||
/// How the manifest and the plan of a restore are written: readable for anybody who opens them,
|
||||
/// and for the runtime, which applies the plan.
|
||||
/// </summary>
|
||||
public static readonly JsonSerializerOptions JSON_OPTIONS = new()
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
Converters = { new JsonStringEnumConverter() },
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// File types which are compressed already. Compressing them once more costs time and makes the
|
||||
/// backup no smaller.
|
||||
/// </summary>
|
||||
public static readonly IReadOnlySet<string> ALREADY_COMPRESSED_EXTENSIONS = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
".jpg", ".jpeg", ".png", ".gif", ".webp", ".heic", ".tiff",
|
||||
".mp3", ".m4a", ".m4b", ".aac", ".ogg", ".opus", ".flac", ".wma",
|
||||
".mp4", ".m4v", ".mov", ".mkv", ".webm", ".avi", ".wmv", ".flv",
|
||||
".zip", ".7z", ".gz", ".bz2", ".xz", ".rar",
|
||||
".pdf", ".docx", ".xlsx", ".pptx", ".odt", ".odp",
|
||||
".mwplugin", FILE_EXTENSION,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The name of the entry which holds a file of the data or configuration folder.
|
||||
/// </summary>
|
||||
/// <param name="location">The folder of the file.</param>
|
||||
/// <param name="relativePath">The path of the file within its folder, with forward slashes.</param>
|
||||
/// <returns>The name of the entry.</returns>
|
||||
public static string EntryName(DataBackupLocation location, string relativePath) => location switch
|
||||
{
|
||||
DataBackupLocation.CONFIG_DIRECTORY => CONFIG_ROOT + relativePath,
|
||||
_ => DATA_ROOT + relativePath,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// The folder in which an entry of the backup contents lies.
|
||||
/// </summary>
|
||||
public enum DataBackupLocation
|
||||
{
|
||||
/// <summary>
|
||||
/// The data folder, where AI Studio keeps the chats, the plugins, and the indexes of the data sources.
|
||||
/// </summary>
|
||||
DATA_DIRECTORY,
|
||||
|
||||
/// <summary>
|
||||
/// The configuration folder, where AI Studio keeps its settings.
|
||||
/// </summary>
|
||||
CONFIG_DIRECTORY,
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What a backup holds, and what wrote it.
|
||||
/// </summary>
|
||||
/// <param name="FormatVersion">The version of the format, cf. DataBackupFormat.FORMAT_VERSION.</param>
|
||||
/// <param name="AppVersion">The version of AI Studio which wrote the backup.</param>
|
||||
/// <param name="SettingsVersion">The version of the settings in the backup.</param>
|
||||
/// <param name="CreatedAtUtc">When the backup was written.</param>
|
||||
/// <param name="SourceDataDirectory">The data folder the backup was made from, for whoever wonders where it came from.</param>
|
||||
/// <param name="SourceConfigDirectory">The configuration folder the backup was made from.</param>
|
||||
/// <param name="IncludesDataSourceIndexes">Whether the backup holds the indexes of the data sources. Without them, a restore keeps the indexes there are.</param>
|
||||
/// <param name="Entries">Every file of the backup, the manifest aside.</param>
|
||||
public sealed record DataBackupManifest(
|
||||
int FormatVersion,
|
||||
string AppVersion,
|
||||
string SettingsVersion,
|
||||
DateTimeOffset CreatedAtUtc,
|
||||
string SourceDataDirectory,
|
||||
string SourceConfigDirectory,
|
||||
bool IncludesDataSourceIndexes,
|
||||
IReadOnlyList<DataBackupManifestEntry> Entries);
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// One file of a backup, as the manifest lists it.
|
||||
/// </summary>
|
||||
/// <param name="Name">The name of the entry within the backup.</param>
|
||||
/// <param name="Size">The size of the file in bytes.</param>
|
||||
/// <param name="Sha256">The SHA-256 hash of the file, in lowercase hex digits.</param>
|
||||
public sealed record DataBackupManifestEntry(string Name, long Size, string Sha256);
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// The manifest of a backup, or why it could not be read.
|
||||
/// </summary>
|
||||
/// <param name="Manifest">The manifest, or null when the backup cannot be used.</param>
|
||||
/// <param name="Issue">Why the backup cannot be used, for the user. Empty when it can.</param>
|
||||
public sealed record DataBackupManifestReadResult(DataBackupManifest? Manifest, string Issue);
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What a backup is busy with.
|
||||
/// </summary>
|
||||
public enum DataBackupPhase
|
||||
{
|
||||
/// <summary>
|
||||
/// It collects the settings and pauses the indexing.
|
||||
/// </summary>
|
||||
PREPARING,
|
||||
|
||||
/// <summary>
|
||||
/// It copies the indexes of the data sources, one vector store after the other.
|
||||
/// </summary>
|
||||
COPYING_INDEXES,
|
||||
|
||||
/// <summary>
|
||||
/// It writes the files into the backup.
|
||||
/// </summary>
|
||||
WRITING,
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How far a backup got.
|
||||
/// </summary>
|
||||
/// <param name="Phase">What the backup is busy with.</param>
|
||||
/// <param name="Processed">How many steps of the phase are done: vector stores while the indexes are copied, files while they are written.</param>
|
||||
/// <param name="Total">How many steps the phase has, or zero while that is not known yet.</param>
|
||||
public readonly record struct DataBackupProgress(DataBackupPhase Phase, int Processed, int Total);
|
||||
@@ -0,0 +1,50 @@
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Reads what a backup holds, cf. DataBackupFormat.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Open a backup with ZipArchiveMode.Read and read its entries as streams. ZipArchiveMode.Update would
|
||||
/// buffer an entry in memory, which ends at 2 GB, and a backup may hold larger files.
|
||||
/// </remarks>
|
||||
public static class DataBackupReader
|
||||
{
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataBackupReader).Namespace, nameof(DataBackupReader));
|
||||
|
||||
/// <summary>
|
||||
/// Reads the manifest of a backup, and checks that this version of AI Studio can use the backup.
|
||||
/// </summary>
|
||||
/// <param name="archive">The backup, opened for reading.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The manifest, or why the backup cannot be used.</returns>
|
||||
public static async Task<DataBackupManifestReadResult> ReadManifestAsync(ZipArchive archive, CancellationToken token)
|
||||
{
|
||||
var manifestEntry = archive.GetEntry(DataBackupFormat.MANIFEST_ENTRY_NAME);
|
||||
if (manifestEntry is null)
|
||||
return new(null, TB("This file is no backup of AI Studio, or it was not written completely."));
|
||||
|
||||
DataBackupManifest? manifest;
|
||||
try
|
||||
{
|
||||
await using var manifestStream = manifestEntry.Open();
|
||||
manifest = await JsonSerializer.DeserializeAsync<DataBackupManifest>(manifestStream, DataBackupFormat.JSON_OPTIONS, token);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
manifest = null;
|
||||
}
|
||||
|
||||
if (manifest is null)
|
||||
return new(null, TB("The content list of this backup could not be read."));
|
||||
|
||||
if (manifest.FormatVersion > DataBackupFormat.FORMAT_VERSION)
|
||||
return new(null, TB("This backup was made by a newer version of AI Studio. Please update AI Studio first."));
|
||||
|
||||
return new(manifest, string.Empty);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How a backup ended.
|
||||
/// </summary>
|
||||
/// <param name="Success">Whether the backup was written completely. A cancelled backup counts as no success.</param>
|
||||
/// <param name="Cancelled">Whether the user stopped the backup.</param>
|
||||
/// <param name="ArchivePath">The file of the backup.</param>
|
||||
/// <param name="FileCount">How many files the backup holds.</param>
|
||||
/// <param name="Bytes">How large these files are together, before they were compressed.</param>
|
||||
/// <param name="IncludesDataSourceIndexes">Whether the backup holds the indexes of the data sources.</param>
|
||||
/// <param name="Issue">Why the backup failed, for the user. Empty when it did not fail.</param>
|
||||
public sealed record DataBackupResult(bool Success, bool Cancelled, string ArchivePath, int FileCount, long Bytes, bool IncludesDataSourceIndexes, string Issue);
|
||||
@@ -0,0 +1,45 @@
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// The settings as a backup takes them along.
|
||||
/// </summary>
|
||||
public static class DataBackupSettings
|
||||
{
|
||||
/// <summary>
|
||||
/// Copies the settings, leaving out every object which a configuration plugin manages.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A configuration plugin brings its providers, data sources, chat templates, profiles, and
|
||||
/// policies back on another computer, along with their API keys. Restored from a backup as well,
|
||||
/// they would compete with what the plugin brings, and without the plugin they would stay without
|
||||
/// their keys: secrets never go into a backup. That holds for the configuration plugins of an
|
||||
/// organization and for those the user installed, since a backup takes the latter along.<br/><br/>
|
||||
///
|
||||
/// A setting which a configuration plugin locks stays in the copy, together with the plugin which
|
||||
/// locks it. After a restore, the clean-up following the start of the plugins treats it like the
|
||||
/// setting of a removed plugin, until the plugin locks it again. Without the plugin, the user gets
|
||||
/// back the value they had chosen before.<br/><br/>
|
||||
///
|
||||
/// The copy is a round trip through the format in which the settings are stored, so it shares
|
||||
/// nothing with the settings AI Studio runs with. Like storing the settings, it reads them while
|
||||
/// the rest of the app may still change them.
|
||||
/// </remarks>
|
||||
/// <param name="settings">The settings AI Studio runs with. They stay as they are.</param>
|
||||
/// <returns>The copy for the backup.</returns>
|
||||
public static Data CopyWithoutPluginObjects(Data settings)
|
||||
{
|
||||
var copy = JsonSerializer.Deserialize<Data>(JsonSerializer.Serialize(settings, SettingsManager.JSON_OPTIONS), SettingsManager.JSON_OPTIONS)
|
||||
?? throw new InvalidOperationException("The settings could not be copied.");
|
||||
|
||||
foreach (var descriptor in PluginConfigurationObjectDescriptors.ALL)
|
||||
descriptor.RemovePluginObjects(copy);
|
||||
|
||||
return copy;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Where a backup takes its files from.
|
||||
/// </summary>
|
||||
/// <param name="DataDirectory">The data folder.</param>
|
||||
/// <param name="ConfigDirectory">The configuration folder.</param>
|
||||
/// <param name="SettingsJson">The settings as the backup takes them along, cf. DataBackupSettings. They come from memory rather than from the disk, since the backup leaves out the objects of configuration plugins.</param>
|
||||
/// <param name="IndexCopiesDirectory">The copies of the indexes, in the layout of the data folder, or an empty string when the backup leaves the indexes out. The indexes in the data folder change while AI Studio runs, so the backup never reads them there.</param>
|
||||
/// <param name="OrganizationPluginFolders">The plugins an organization deployed among the user's own ones, cf. DataBackupContents.FindOrganizationPluginsAmongUserPlugins.</param>
|
||||
public sealed record DataBackupSources(string DataDirectory, string ConfigDirectory, string SettingsJson, string IndexCopiesDirectory, IReadOnlyList<string> OrganizationPluginFolders);
|
||||
@@ -0,0 +1,173 @@
|
||||
using System.IO.Compression;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Writes a backup, cf. DataBackupFormat.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every file is streamed into the backup, so a backup of many gigabytes needs no more memory than
|
||||
/// a small one. The caller writes the backup straight into the file the user chose: in the Flatpak,
|
||||
/// the portal grants access to exactly that file, and to nothing next to it.
|
||||
/// </remarks>
|
||||
public static class DataBackupWriter
|
||||
{
|
||||
private const int BUFFER_SIZE = 64 * 1024;
|
||||
|
||||
/// <summary>
|
||||
/// The earliest time a ZIP file can store.
|
||||
/// </summary>
|
||||
private static readonly DateTimeOffset EARLIEST_ZIP_TIME = new(1980, 1, 1, 0, 0, 0, TimeSpan.Zero);
|
||||
|
||||
/// <summary>
|
||||
/// A file to take along, and where it goes in the backup.
|
||||
/// </summary>
|
||||
private sealed record BackupFile(string EntryName, string SourcePath);
|
||||
|
||||
/// <summary>
|
||||
/// Writes a backup into the given stream.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A file which is gone by the time it is written, e.g. a temporary chat AI Studio deleted
|
||||
/// meanwhile, is left out. A file which cannot be read fails the backup, though: a backup which
|
||||
/// quietly lacks something is worse than none.
|
||||
/// </remarks>
|
||||
/// <param name="target">Where to write the backup. It stays open.</param>
|
||||
/// <param name="sources">Where the files come from.</param>
|
||||
/// <param name="manifest">What to note in the manifest. The entries are filled in here.</param>
|
||||
/// <param name="progress">Receives how many files are written.</param>
|
||||
/// <param name="token">Cancels the backup.</param>
|
||||
/// <returns>The manifest as the backup holds it.</returns>
|
||||
public static async Task<DataBackupManifest> WriteAsync(Stream target, DataBackupSources sources, DataBackupManifest manifest, IProgress<DataBackupProgress>? progress, CancellationToken token)
|
||||
{
|
||||
var files = CollectFiles(sources, token);
|
||||
var totalFiles = files.Count + 1;
|
||||
var entries = new List<DataBackupManifestEntry>(totalFiles);
|
||||
var buffer = new byte[BUFFER_SIZE];
|
||||
|
||||
progress?.Report(new(DataBackupPhase.WRITING, 0, totalFiles));
|
||||
using (var archive = new ZipArchive(target, ZipArchiveMode.Create, leaveOpen: true))
|
||||
{
|
||||
var settingsEntry = DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.SETTINGS);
|
||||
await using (var settingsStream = new MemoryStream(Encoding.UTF8.GetBytes(sources.SettingsJson), writable: false))
|
||||
entries.Add(await WriteEntryAsync(archive, DataBackupFormat.EntryName(settingsEntry.Location, settingsEntry.RelativePath), settingsStream, DateTimeOffset.Now, buffer, token));
|
||||
|
||||
progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles));
|
||||
foreach (var file in files)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
FileStream source;
|
||||
try
|
||||
{
|
||||
source = new FileStream(file.SourcePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
}
|
||||
catch (Exception exception) when (exception is FileNotFoundException or DirectoryNotFoundException)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
await using (source)
|
||||
entries.Add(await WriteEntryAsync(archive, file.EntryName, source, File.GetLastWriteTime(file.SourcePath), buffer, token));
|
||||
|
||||
progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles));
|
||||
}
|
||||
|
||||
manifest = manifest with { Entries = entries };
|
||||
await using (var manifestStream = new MemoryStream(JsonSerializer.SerializeToUtf8Bytes(manifest, DataBackupFormat.JSON_OPTIONS), writable: false))
|
||||
await WriteEntryAsync(archive, DataBackupFormat.MANIFEST_ENTRY_NAME, manifestStream, DateTimeOffset.Now, buffer, token);
|
||||
}
|
||||
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Collects the files of the data folder which belong into a backup, cf. DataBackupContents.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The settings are no part of it: they come from memory. The indexes come from their copies.
|
||||
/// </remarks>
|
||||
private static List<BackupFile> CollectFiles(DataBackupSources sources, CancellationToken token)
|
||||
{
|
||||
var excludedPaths = DataBackupContents.ENTRIES
|
||||
.Where(entry => !entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY)
|
||||
.Select(entry => entry.RelativePath)
|
||||
.Concat(sources.OrganizationPluginFolders)
|
||||
.ToList();
|
||||
|
||||
var files = new List<BackupFile>();
|
||||
foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY))
|
||||
{
|
||||
var root = entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX or DataBackupEntryKind.VECTOR_STORES
|
||||
? sources.IndexCopiesDirectory
|
||||
: sources.DataDirectory;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(root))
|
||||
continue;
|
||||
|
||||
var path = Path.Join(root, entry.RelativePath.Replace(DataBackupContents.SEPARATOR, Path.DirectorySeparatorChar));
|
||||
if (!entry.IsFolder)
|
||||
{
|
||||
if (File.Exists(path))
|
||||
files.Add(new(DataBackupFormat.EntryName(entry.Location, entry.RelativePath), path));
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Directory.Exists(path))
|
||||
continue;
|
||||
|
||||
//
|
||||
// Hidden files count as well, since on Linux and macOS a dot makes a file hidden. Links
|
||||
// are not followed, so nothing outside the folder gets into the backup. A folder which
|
||||
// cannot be read fails the backup instead of leaving a gap:
|
||||
//
|
||||
var options = new EnumerationOptions { RecurseSubdirectories = true, AttributesToSkip = FileAttributes.ReparsePoint, IgnoreInaccessible = false };
|
||||
foreach (var filePath in Directory.EnumerateFiles(path, "*", options))
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var relativePath = Path.GetRelativePath(root, filePath).Replace(Path.DirectorySeparatorChar, DataBackupContents.SEPARATOR);
|
||||
if (excludedPaths.Any(excludedPath => DataBackupContents.Covers(excludedPath, relativePath)))
|
||||
continue;
|
||||
|
||||
files.Add(new(DataBackupFormat.EntryName(DataBackupLocation.DATA_DIRECTORY, relativePath), filePath));
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes one entry, and notes its size and hash while it streams through.
|
||||
/// </summary>
|
||||
private static async Task<DataBackupManifestEntry> WriteEntryAsync(ZipArchive archive, string entryName, Stream source, DateTimeOffset lastWriteTime, byte[] buffer, CancellationToken token)
|
||||
{
|
||||
var compressionLevel = DataBackupFormat.ALREADY_COMPRESSED_EXTENSIONS.Contains(Path.GetExtension(entryName))
|
||||
? CompressionLevel.NoCompression
|
||||
: CompressionLevel.Optimal;
|
||||
|
||||
var entry = archive.CreateEntry(entryName, compressionLevel);
|
||||
|
||||
// The time of the last change tells, e.g., the order of the chats in a workspace:
|
||||
entry.LastWriteTime = lastWriteTime < EARLIEST_ZIP_TIME ? DateTimeOffset.Now : lastWriteTime;
|
||||
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
long size = 0;
|
||||
await using (var entryStream = entry.Open())
|
||||
{
|
||||
int read;
|
||||
while ((read = await source.ReadAsync(buffer, token)) > 0)
|
||||
{
|
||||
hash.AppendData(buffer, 0, read);
|
||||
await entryStream.WriteAsync(buffer.AsMemory(0, read), token);
|
||||
size += read;
|
||||
}
|
||||
}
|
||||
|
||||
return new(entryName, size, Convert.ToHexStringLower(hash.GetHashAndReset()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// One element which a restore replaces, as the list of the backup contents names it.
|
||||
/// </summary>
|
||||
/// <param name="Location">The folder in which the element lies.</param>
|
||||
/// <param name="RelativePath">The path of the element within that folder, with forward slashes. A folder ends with a slash.</param>
|
||||
/// <param name="Keep">The paths below the element which stay as they are, relative to the same folder: the plugins which come with AI Studio or from the organization, for example. The runtime moves them from the replaced element into the new one.</param>
|
||||
public sealed record DataRestoreElement(DataBackupLocation Location, string RelativePath, IReadOnlyList<string> Keep);
|
||||
@@ -0,0 +1,106 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What remains to do after the runtime applied a restore, or after a backup or the preparation of a
|
||||
/// restore stopped halfway: read the report of the runtime, and remove what nobody needs any longer.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Nothing here touches a restore which the runtime has not finished, cf. IsRestoreUnfinished: its
|
||||
/// journal and the data it replaced are the only way back, and the runtime continues at every start.
|
||||
/// </remarks>
|
||||
public static class DataRestoreFollowUp
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether a restore is prepared completely and waits for the next start.
|
||||
/// </summary>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <returns>True when the next start applies a restore.</returns>
|
||||
public static bool IsRestorePending(string dataDirectory) => IsMarkedReady(dataDirectory) && !HasJournal(dataDirectory);
|
||||
|
||||
/// <summary>
|
||||
/// Whether the runtime has not finished a restore: it failed to move a failed restore back, and
|
||||
/// tries again at the next start.
|
||||
/// </summary>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <returns>True while the runtime still needs the pending and the previous directories.</returns>
|
||||
public static bool IsRestoreUnfinished(string dataDirectory) => IsMarkedReady(dataDirectory) && HasJournal(dataDirectory);
|
||||
|
||||
/// <summary>
|
||||
/// Reads how the runtime applied a restore at this start. Throws a JsonException when the report
|
||||
/// cannot be read.
|
||||
/// </summary>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <param name="token">Cancels reading.</param>
|
||||
/// <returns>The report, or null when the runtime applied no restore.</returns>
|
||||
public static async Task<DataRestoreReport?> ReadReportAsync(string dataDirectory, CancellationToken token)
|
||||
{
|
||||
var reportPath = Path.Join(dataDirectory, DataRestoreFormat.RESULT_FILE_NAME);
|
||||
if (!File.Exists(reportPath))
|
||||
return null;
|
||||
|
||||
await using var reportStream = File.OpenRead(reportPath);
|
||||
return await JsonSerializer.DeserializeAsync<DataRestoreReport>(reportStream, DataBackupFormat.JSON_OPTIONS, token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the report of the runtime, once the user saw it.
|
||||
/// </summary>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
public static void RemoveReport(string dataDirectory) => File.Delete(Path.Join(dataDirectory, DataRestoreFormat.RESULT_FILE_NAME));
|
||||
|
||||
/// <summary>
|
||||
/// Removes what a backup or the preparation of a restore left behind when it stopped halfway, e.g.,
|
||||
/// because AI Studio crashed. Call it only while neither runs.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A pending directory with READY_FILE_NAME stays: the restore waits for the next start, or the
|
||||
/// runtime has not finished it.
|
||||
/// </remarks>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <param name="configDirectory">The configuration folder.</param>
|
||||
public static void RemoveLeftovers(string dataDirectory, string configDirectory)
|
||||
{
|
||||
DeleteDirectory(DataRestoreFormat.DirectoryIn(dataDirectory, DataBackupFormat.STAGING_DIRECTORY));
|
||||
if (IsMarkedReady(dataDirectory))
|
||||
return;
|
||||
|
||||
DeleteDirectory(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PENDING_DIRECTORY));
|
||||
DeleteDirectory(DataRestoreFormat.DirectoryIn(configDirectory, DataRestoreFormat.PENDING_DIRECTORY));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the data which a restore replaced. Call it once the restored settings loaded: until
|
||||
/// then, these data are the way back.
|
||||
/// </summary>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <param name="configDirectory">The configuration folder.</param>
|
||||
/// <returns>Whether there were such data.</returns>
|
||||
public static bool RemoveReplacedData(string dataDirectory, string configDirectory)
|
||||
{
|
||||
if (IsMarkedReady(dataDirectory))
|
||||
return false;
|
||||
|
||||
var removedData = DeleteDirectory(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PREVIOUS_DIRECTORY));
|
||||
var removedConfiguration = DeleteDirectory(DataRestoreFormat.DirectoryIn(configDirectory, DataRestoreFormat.PREVIOUS_DIRECTORY));
|
||||
return removedData || removedConfiguration;
|
||||
}
|
||||
|
||||
private static bool IsMarkedReady(string dataDirectory) => File.Exists(Path.Join(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PENDING_DIRECTORY), DataRestoreFormat.READY_FILE_NAME));
|
||||
|
||||
private static bool HasJournal(string dataDirectory) => File.Exists(Path.Join(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PENDING_DIRECTORY), DataRestoreFormat.JOURNAL_FILE_NAME));
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a directory with everything in it.
|
||||
/// </summary>
|
||||
/// <returns>Whether the directory existed.</returns>
|
||||
private static bool DeleteDirectory(string path)
|
||||
{
|
||||
if (!Directory.Exists(path))
|
||||
return false;
|
||||
|
||||
Directory.Delete(path, true);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How a prepared restore waits for the next start, which applies it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is the contract between the app, which prepares a restore, and the runtime, which applies it
|
||||
/// before the app starts again, when no file is in use:
|
||||
/// <list type="bullet">
|
||||
/// <item>Below the data folder and below the configuration folder, PENDING_DIRECTORY holds what the
|
||||
/// backup brings for that folder, in the layout of that folder. Staging lies on the same disk as its
|
||||
/// target, so the runtime can move every element into place instead of copying it.</item>
|
||||
/// <item>The plan, PLAN_FILE_NAME in the pending directory of the data folder, names the elements to
|
||||
/// replace, cf. DataRestorePlan.</item>
|
||||
/// <item>READY_FILE_NAME in the same directory comes last. Without it, the restore was not prepared
|
||||
/// completely, and the runtime leaves everything as it is.</item>
|
||||
/// </list>
|
||||
/// What the runtime adds, cf. runtime/src/data_restore.rs:
|
||||
/// <list type="bullet">
|
||||
/// <item>PREVIOUS_DIRECTORY, below both folders, keeps what the restore replaced. The app removes it
|
||||
/// once it loaded the restored settings.</item>
|
||||
/// <item>JOURNAL_FILE_NAME in the pending directory of the data folder lists every move. While it lies
|
||||
/// next to READY_FILE_NAME, the runtime moves everything back at the next start, so the app never
|
||||
/// removes a pending directory which holds READY_FILE_NAME.</item>
|
||||
/// <item>RESULT_FILE_NAME in the data folder tells the app how the restore ended, cf. DataRestoreReport.</item>
|
||||
/// </list>
|
||||
/// Keep in sync with the runtime, which applies the restore.
|
||||
/// </remarks>
|
||||
public static class DataRestoreFormat
|
||||
{
|
||||
/// <summary>
|
||||
/// The version of the plan this version of AI Studio writes.
|
||||
/// </summary>
|
||||
public const int PLAN_FORMAT_VERSION = 1;
|
||||
|
||||
/// <summary>
|
||||
/// The directory, below the data folder and below the configuration folder, which holds what a
|
||||
/// prepared restore brings for that folder.
|
||||
/// </summary>
|
||||
public const string PENDING_DIRECTORY = ".restore-pending/";
|
||||
|
||||
/// <summary>
|
||||
/// The plan of the restore, in the pending directory of the data folder.
|
||||
/// </summary>
|
||||
public const string PLAN_FILE_NAME = "restore.json";
|
||||
|
||||
/// <summary>
|
||||
/// Marks a restore as prepared completely, in the pending directory of the data folder.
|
||||
/// </summary>
|
||||
public const string READY_FILE_NAME = "READY";
|
||||
|
||||
/// <summary>
|
||||
/// The directory, below the data folder and below the configuration folder, which keeps what the
|
||||
/// runtime replaced, in the layout of that folder.
|
||||
/// </summary>
|
||||
public const string PREVIOUS_DIRECTORY = ".restore-previous/";
|
||||
|
||||
/// <summary>
|
||||
/// How the runtime applied a restore, in the data folder.
|
||||
/// </summary>
|
||||
public const string RESULT_FILE_NAME = ".restore-result.json";
|
||||
|
||||
/// <summary>
|
||||
/// The journal of the runtime, in the pending directory of the data folder. Next to READY_FILE_NAME,
|
||||
/// it tells that the runtime has not finished a restore: it moves it back at the next start.
|
||||
/// </summary>
|
||||
public const string JOURNAL_FILE_NAME = "journal.json";
|
||||
|
||||
/// <summary>
|
||||
/// The path of one of the directories above, below the given folder.
|
||||
/// </summary>
|
||||
/// <param name="folder">The data or the configuration folder.</param>
|
||||
/// <param name="directory">PENDING_DIRECTORY or PREVIOUS_DIRECTORY.</param>
|
||||
/// <returns>The path of the directory.</returns>
|
||||
public static string DirectoryIn(string folder, string directory) => Path.Join(folder, directory.TrimEnd(DataBackupContents.SEPARATOR));
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// One thing the user still has to do after a restore.
|
||||
/// </summary>
|
||||
/// <param name="Kind">What to do.</param>
|
||||
/// <param name="Name">What it concerns: the name of a provider, a data source, or a tool, or a path.</param>
|
||||
/// <param name="Setting">For a tool: the label of the setting which lacks its value; otherwise empty.</param>
|
||||
public sealed record DataRestoreOpenPoint(DataRestoreOpenPointKind Kind, string Name, string Setting = "");
|
||||
@@ -0,0 +1,59 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What the user still has to do after a restore, since a backup holds no secrets and another computer
|
||||
/// may lack the files the settings name.
|
||||
/// </summary>
|
||||
public enum DataRestoreOpenPointKind
|
||||
{
|
||||
/// <summary>
|
||||
/// A provider of the user has no API key on this computer.
|
||||
/// </summary>
|
||||
PROVIDER_WITHOUT_API_KEY,
|
||||
|
||||
/// <summary>
|
||||
/// An embedding provider of the user has no API key on this computer.
|
||||
/// </summary>
|
||||
EMBEDDING_PROVIDER_WITHOUT_API_KEY,
|
||||
|
||||
/// <summary>
|
||||
/// A transcription provider of the user has no API key on this computer.
|
||||
/// </summary>
|
||||
TRANSCRIPTION_PROVIDER_WITHOUT_API_KEY,
|
||||
|
||||
/// <summary>
|
||||
/// An ERI data source of the user has no password or token on this computer.
|
||||
/// </summary>
|
||||
DATA_SOURCE_WITHOUT_SECRET,
|
||||
|
||||
/// <summary>
|
||||
/// A mailbox of the user has no password on this computer.
|
||||
/// </summary>
|
||||
MAILBOX_WITHOUT_PASSWORD,
|
||||
|
||||
/// <summary>
|
||||
/// A tool which the user set up has no value for a secret setting on this computer, e.g., for
|
||||
/// the API key of a search service.
|
||||
/// </summary>
|
||||
TOOL_WITHOUT_SECRET,
|
||||
|
||||
/// <summary>
|
||||
/// A local data source names a file or folder which does not exist on this computer.
|
||||
/// </summary>
|
||||
DATA_SOURCE_NOT_FOUND,
|
||||
|
||||
/// <summary>
|
||||
/// The batch processing names a file or folder which does not exist on this computer.
|
||||
/// </summary>
|
||||
BATCH_PROCESSING_PATH_NOT_FOUND,
|
||||
|
||||
/// <summary>
|
||||
/// The file with custom root certificates does not exist on this computer.
|
||||
/// </summary>
|
||||
CERTIFICATE_BUNDLE_NOT_FOUND,
|
||||
|
||||
/// <summary>
|
||||
/// The ERI server assistant writes a server into a folder which does not exist on this computer.
|
||||
/// </summary>
|
||||
ERI_SERVER_DIRECTORY_NOT_FOUND,
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.ERIClient.DataModel;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Finds what the user still has to do after a restore.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A backup holds no secrets, so the providers, data sources, mailboxes, and tools of the user lack
|
||||
/// them on another computer. Those of the organization do not: their configuration plugin brings them
|
||||
/// along with their secrets. The files and folders which the settings name may be missing, too, wherever
|
||||
/// they come from.
|
||||
/// </remarks>
|
||||
public static class DataRestoreOpenPoints
|
||||
{
|
||||
/// <summary>
|
||||
/// Finds what the user still has to do after a restore.
|
||||
/// </summary>
|
||||
/// <param name="settings">The restored settings.</param>
|
||||
/// <param name="secrets">Tells which secrets this computer holds.</param>
|
||||
/// <param name="toolSecrets">The secret settings of the tools which the user set up. Only the tools know which of their settings are secret, cf. DataRestoreService.</param>
|
||||
/// <returns>The open points, in the order of the settings.</returns>
|
||||
public static async Task<IReadOnlyList<DataRestoreOpenPoint>> CollectAsync(Data settings, IStoredSecrets secrets, IReadOnlyList<DataRestoreToolSecret> toolSecrets)
|
||||
{
|
||||
var openPoints = new List<DataRestoreOpenPoint>();
|
||||
foreach (var provider in settings.Providers.Where(provider => !provider.IsEnterpriseConfiguration && provider.UsedLLMProvider.IsAPIKeyNeeded(provider.Host)))
|
||||
if (!await secrets.HasAPIKeyAsync(provider, SecretStoreType.LLM_PROVIDER))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.PROVIDER_WITHOUT_API_KEY, provider.InstanceName));
|
||||
|
||||
foreach (var provider in settings.EmbeddingProviders.Where(provider => !provider.IsEnterpriseConfiguration && provider.UsedLLMProvider.IsAPIKeyNeeded(provider.Host)))
|
||||
if (!await secrets.HasAPIKeyAsync(provider, SecretStoreType.EMBEDDING_PROVIDER))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.EMBEDDING_PROVIDER_WITHOUT_API_KEY, provider.Name));
|
||||
|
||||
foreach (var provider in settings.TranscriptionProviders.Where(provider => !provider.IsEnterpriseConfiguration && provider.UsedLLMProvider.IsAPIKeyNeeded(provider.Host)))
|
||||
if (!await secrets.HasAPIKeyAsync(provider, SecretStoreType.TRANSCRIPTION_PROVIDER))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.TRANSCRIPTION_PROVIDER_WITHOUT_API_KEY, provider.Name));
|
||||
|
||||
foreach (var dataSource in settings.DataSources)
|
||||
{
|
||||
switch (dataSource)
|
||||
{
|
||||
case DataSourceLocalDirectory directory when !Directory.Exists(directory.Path):
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.DATA_SOURCE_NOT_FOUND, directory.Name));
|
||||
break;
|
||||
|
||||
case DataSourceLocalFile file when !File.Exists(file.FilePath):
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.DATA_SOURCE_NOT_FOUND, file.Name));
|
||||
break;
|
||||
|
||||
case IERIDataSource { IsEnterpriseConfiguration: false, AuthMethod: AuthMethod.TOKEN or AuthMethod.USERNAME_PASSWORD } eriDataSource:
|
||||
if (!await secrets.HasSecretAsync(eriDataSource, SecretStoreType.DATA_SOURCE))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.DATA_SOURCE_WITHOUT_SECRET, eriDataSource.Name));
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var mailbox in settings.Mailboxes.Where(mailbox => mailbox is { IsEnterpriseConfiguration: false, AuthMethod: MailboxAuthMethod.PASSWORD }))
|
||||
if (!await secrets.HasSecretAsync(mailbox, SecretStoreType.DATA_SOURCE))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.MAILBOX_WITHOUT_PASSWORD, mailbox.Name));
|
||||
|
||||
foreach (var toolSecret in toolSecrets)
|
||||
if (!await secrets.HasSecretAsync(new ToolSettingsSecretId(toolSecret.ToolId, toolSecret.FieldName), SecretStoreType.TOOL_SETTINGS))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.TOOL_WITHOUT_SECRET, toolSecret.ToolName, toolSecret.SettingLabel));
|
||||
|
||||
var batchProcessing = settings.BatchProcessing;
|
||||
foreach (var folder in new[] { batchProcessing.InputDirectory, batchProcessing.OutputDirectory })
|
||||
if (!string.IsNullOrWhiteSpace(folder) && !Directory.Exists(folder))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.BATCH_PROCESSING_PATH_NOT_FOUND, folder));
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(batchProcessing.PromptFilePath) && !File.Exists(batchProcessing.PromptFilePath))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.BATCH_PROCESSING_PATH_NOT_FOUND, batchProcessing.PromptFilePath));
|
||||
|
||||
var certificateBundlePath = settings.App.ExternalHttpCustomRootCertificateBundlePath;
|
||||
if (settings.App.ExternalHttpCustomRootCertificatesEnabled && !string.IsNullOrWhiteSpace(certificateBundlePath) && !File.Exists(certificateBundlePath))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.CERTIFICATE_BUNDLE_NOT_FOUND, certificateBundlePath));
|
||||
|
||||
foreach (var server in settings.ERI.ERIServers.Where(server => server.WriteToFilesystem && !string.IsNullOrWhiteSpace(server.BaseDirectory) && !Directory.Exists(server.BaseDirectory)))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.ERI_SERVER_DIRECTORY_NOT_FOUND, server.ServerName));
|
||||
|
||||
return openPoints;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How the runtime applied a restore, cf. DataRestoreReport.
|
||||
/// </summary>
|
||||
public enum DataRestoreOutcome
|
||||
{
|
||||
/// <summary>
|
||||
/// The restored data are in place.
|
||||
/// </summary>
|
||||
APPLIED,
|
||||
|
||||
/// <summary>
|
||||
/// The runtime did not start the restore, since the plan or the folders did not allow it. All data
|
||||
/// stayed as they were.
|
||||
/// </summary>
|
||||
REFUSED,
|
||||
|
||||
/// <summary>
|
||||
/// The restore failed, and the runtime moved all data back.
|
||||
/// </summary>
|
||||
ROLLED_BACK,
|
||||
|
||||
/// <summary>
|
||||
/// The restore failed, and moving the data back failed as well. The runtime tries again at every
|
||||
/// start.
|
||||
/// </summary>
|
||||
ROLLBACK_FAILED,
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What the runtime replaces at the next start, cf. DataRestoreFormat.
|
||||
/// </summary>
|
||||
/// <param name="FormatVersion">The version of the plan, cf. DataRestoreFormat.PLAN_FORMAT_VERSION.</param>
|
||||
/// <param name="PreparedAtUtc">When the restore was prepared.</param>
|
||||
/// <param name="BackupCreatedAtUtc">When the backup was made.</param>
|
||||
/// <param name="BackupAppVersion">The version of AI Studio which made the backup.</param>
|
||||
/// <param name="Elements">The elements to replace. An element the backup holds nothing for stays as it is.</param>
|
||||
public sealed record DataRestorePlan(int FormatVersion, DateTimeOffset PreparedAtUtc, DateTimeOffset BackupCreatedAtUtc, string BackupAppVersion, IReadOnlyList<DataRestoreElement> Elements);
|
||||
@@ -0,0 +1,239 @@
|
||||
using System.IO.Compression;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
using SettingsVersion = AIStudio.Settings.Version;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Prepares a restore: unpacks a backup next to the data and configuration folders, so that the next
|
||||
/// start can put it in place, cf. DataRestoreFormat.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Nothing in use changes here. While AI Studio runs, its files are open, and on Windows, they cannot be
|
||||
/// replaced at all. The runtime replaces them at the next start, before the app opens any of them, and
|
||||
/// rolls back when that fails.
|
||||
/// </remarks>
|
||||
public static class DataRestorePreparer
|
||||
{
|
||||
private const int BUFFER_SIZE = 64 * 1024;
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataRestorePreparer).Namespace, nameof(DataRestorePreparer));
|
||||
|
||||
/// <summary>
|
||||
/// A file of the backup to unpack, and where it goes.
|
||||
/// </summary>
|
||||
private sealed record PendingFile(ZipArchiveEntry Entry, DataBackupManifestEntry Listed, string DestinationPath);
|
||||
|
||||
/// <summary>
|
||||
/// Unpacks a backup for the next start.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every file has to match the size and the hash the manifest lists for it: a restore replaces all
|
||||
/// data, so a damaged backup must not get that far. A file the manifest does not list, or one
|
||||
/// which belongs to no element the restore replaces, stays in the backup. A restore prepared
|
||||
/// before is discarded first; one which fails or is cancelled leaves nothing behind.
|
||||
/// </remarks>
|
||||
/// <param name="archivePath">The backup.</param>
|
||||
/// <param name="target">Where the restore goes.</param>
|
||||
/// <param name="progress">Receives how many files are unpacked.</param>
|
||||
/// <param name="token">Cancels the preparation.</param>
|
||||
/// <returns>Whether the restore is prepared, or why not.</returns>
|
||||
public static async Task<DataRestoreResult> PrepareAsync(string archivePath, DataRestoreTarget target, IProgress<DataRestoreProgress>? progress, CancellationToken token)
|
||||
{
|
||||
Discard(target);
|
||||
try
|
||||
{
|
||||
using var archive = ZipFile.OpenRead(archivePath);
|
||||
var manifestResult = await DataBackupReader.ReadManifestAsync(archive, token);
|
||||
if (manifestResult.Manifest is not { } manifest)
|
||||
return new(false, false, manifestResult.Issue);
|
||||
|
||||
var versionIssue = CheckVersions(manifest, target);
|
||||
if (!string.IsNullOrWhiteSpace(versionIssue))
|
||||
return new(false, false, versionIssue);
|
||||
|
||||
var elements = DetermineElements(manifest, target);
|
||||
var files = DetermineFiles(archive, manifest, elements, target);
|
||||
progress?.Report(new(0, files.Count));
|
||||
|
||||
var buffer = new byte[BUFFER_SIZE];
|
||||
for (var index = 0; index < files.Count; index++)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
if (!await UnpackAsync(files[index], buffer, token))
|
||||
{
|
||||
Discard(target);
|
||||
return new(false, false, string.Format(TB("The backup is damaged: the file '{0}' does not match its content list."), files[index].Listed.Name));
|
||||
}
|
||||
|
||||
progress?.Report(new(index + 1, files.Count));
|
||||
}
|
||||
|
||||
var pendingDirectory = PendingDirectory(target.DataDirectory);
|
||||
Directory.CreateDirectory(pendingDirectory);
|
||||
|
||||
var plan = new DataRestorePlan(DataRestoreFormat.PLAN_FORMAT_VERSION, DateTimeOffset.UtcNow, manifest.CreatedAtUtc, manifest.AppVersion, elements);
|
||||
await File.WriteAllBytesAsync(Path.Join(pendingDirectory, DataRestoreFormat.PLAN_FILE_NAME), JsonSerializer.SerializeToUtf8Bytes(plan, DataBackupFormat.JSON_OPTIONS), token);
|
||||
|
||||
// Last of all: from now on, the next start applies the restore.
|
||||
await File.WriteAllTextAsync(Path.Join(pendingDirectory, DataRestoreFormat.READY_FILE_NAME), plan.PreparedAtUtc.ToString("O"), token);
|
||||
return new(true, false, string.Empty);
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
Discard(target);
|
||||
return new(false, false, TB("This file is no backup of AI Studio, or it was not written completely."));
|
||||
}
|
||||
catch
|
||||
{
|
||||
Discard(target);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes a prepared restore, complete or not, so that the next start leaves everything as it is.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A restore which the runtime has not finished stays: its journal is the only way back to the data
|
||||
/// it replaced, cf. DataRestoreFollowUp.IsRestoreUnfinished.
|
||||
/// </remarks>
|
||||
/// <param name="target">Where the restore was to go.</param>
|
||||
public static void Discard(DataRestoreTarget target)
|
||||
{
|
||||
if (DataRestoreFollowUp.IsRestoreUnfinished(target.DataDirectory))
|
||||
throw new InvalidOperationException("The runtime has not finished the last restore yet, so it cannot be discarded.");
|
||||
|
||||
// First of all: should removing the rest fail halfway, the next start must not apply what is left.
|
||||
var readyPath = Path.Join(PendingDirectory(target.DataDirectory), DataRestoreFormat.READY_FILE_NAME);
|
||||
if (File.Exists(readyPath))
|
||||
File.Delete(readyPath);
|
||||
|
||||
foreach (var pendingDirectory in new[] { PendingDirectory(target.DataDirectory), PendingDirectory(target.ConfigDirectory) })
|
||||
if (Directory.Exists(pendingDirectory))
|
||||
Directory.Delete(pendingDirectory, true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a restore is prepared completely and waits for the next start.
|
||||
/// </summary>
|
||||
/// <param name="target">Where the restore goes.</param>
|
||||
/// <returns>True when the next start applies a restore.</returns>
|
||||
public static bool IsPrepared(DataRestoreTarget target) => DataRestoreFollowUp.IsRestorePending(target.DataDirectory);
|
||||
|
||||
private static string PendingDirectory(string directory) => DataRestoreFormat.DirectoryIn(directory, DataRestoreFormat.PENDING_DIRECTORY);
|
||||
|
||||
/// <summary>
|
||||
/// Refuses a backup of a later version of AI Studio. Its settings or its index may have a form
|
||||
/// this version cannot read, and it would refuse to write them, or break them.
|
||||
/// </summary>
|
||||
/// <returns>Why the backup cannot be restored, or an empty string.</returns>
|
||||
private static string CheckVersions(DataBackupManifest manifest, DataRestoreTarget target)
|
||||
{
|
||||
var settingsAreNewer = !Enum.TryParse<SettingsVersion>(manifest.SettingsVersion, out var settingsVersion) || settingsVersion > target.SettingsVersion;
|
||||
var appIsNewer = Version.TryParse(manifest.AppVersion, out var backupAppVersion) && Version.TryParse(target.AppVersion, out var appVersion) && backupAppVersion > appVersion;
|
||||
return settingsAreNewer || appIsNewer
|
||||
? TB("This backup was made by a newer version of AI Studio. Please update AI Studio first.")
|
||||
: string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the elements the restore replaces: those of the list of the backup contents which the
|
||||
/// backup holds anything for.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// An element the backup holds nothing for stays as it is, since nothing tells whether it was empty
|
||||
/// or whether the version of AI Studio which made the backup knew it at all. The indexes are
|
||||
/// replaced only when the backup holds both of them, cf. DataBackupManifest.IncludesDataSourceIndexes.
|
||||
/// </remarks>
|
||||
private static List<DataRestoreElement> DetermineElements(DataBackupManifest manifest, DataRestoreTarget target)
|
||||
{
|
||||
var elements = new List<DataRestoreElement>();
|
||||
foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded))
|
||||
{
|
||||
if (entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX or DataBackupEntryKind.VECTOR_STORES && !manifest.IncludesDataSourceIndexes)
|
||||
continue;
|
||||
|
||||
var entryName = DataBackupFormat.EntryName(entry.Location, entry.RelativePath);
|
||||
if (!manifest.Entries.Any(listed => DataBackupContents.Covers(entryName, listed.Name)))
|
||||
continue;
|
||||
|
||||
var keep = DataBackupContents.ENTRIES
|
||||
.Where(excluded => !excluded.IsIncluded && excluded.Location == entry.Location && entry.IsFolder && DataBackupContents.Covers(entry.RelativePath, excluded.RelativePath))
|
||||
.Select(excluded => excluded.RelativePath)
|
||||
.Concat(entry.Kind is DataBackupEntryKind.PLUGINS ? target.OrganizationPluginFolders : [])
|
||||
.ToList();
|
||||
|
||||
elements.Add(new(entry.Location, entry.RelativePath, keep));
|
||||
}
|
||||
|
||||
return elements;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the files to unpack: those the manifest lists, which belong to an element the restore
|
||||
/// replaces, and which the element does not keep.
|
||||
/// </summary>
|
||||
private static List<PendingFile> DetermineFiles(ZipArchive archive, DataBackupManifest manifest, IReadOnlyList<DataRestoreElement> elements, DataRestoreTarget target)
|
||||
{
|
||||
var listedEntries = manifest.Entries.ToDictionary(listed => listed.Name, StringComparer.Ordinal);
|
||||
var files = new List<PendingFile>();
|
||||
foreach (var entry in archive.Entries)
|
||||
{
|
||||
var entryName = ZipEntryNames.Normalize(entry.FullName);
|
||||
if (!listedEntries.TryGetValue(entryName, out var listed))
|
||||
continue;
|
||||
|
||||
foreach (var element in elements)
|
||||
{
|
||||
var root = DataBackupFormat.EntryName(element.Location, string.Empty);
|
||||
if (!entryName.StartsWith(root, StringComparison.Ordinal))
|
||||
continue;
|
||||
|
||||
var relativePath = entryName[root.Length..];
|
||||
if (!DataBackupContents.Covers(element.RelativePath, relativePath) || element.Keep.Any(kept => DataBackupContents.Covers(kept, relativePath)))
|
||||
continue;
|
||||
|
||||
var directory = element.Location is DataBackupLocation.CONFIG_DIRECTORY ? target.ConfigDirectory : target.DataDirectory;
|
||||
files.Add(new(entry, listed, ZipEntryNames.GetDestinationPath(PendingDirectory(directory), relativePath)));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unpacks one file, and checks its size and hash while it streams through.
|
||||
/// </summary>
|
||||
/// <returns>False when the file does not match the manifest.</returns>
|
||||
private static async Task<bool> UnpackAsync(PendingFile file, byte[] buffer, CancellationToken token)
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(file.DestinationPath)!);
|
||||
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
long size = 0;
|
||||
await using (var source = file.Entry.Open())
|
||||
await using (var destination = new FileStream(file.DestinationPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
{
|
||||
int read;
|
||||
while ((read = await source.ReadAsync(buffer, token)) > 0)
|
||||
{
|
||||
hash.AppendData(buffer, 0, read);
|
||||
await destination.WriteAsync(buffer.AsMemory(0, read), token);
|
||||
size += read;
|
||||
}
|
||||
}
|
||||
|
||||
if (size != file.Listed.Size || Convert.ToHexStringLower(hash.GetHashAndReset()) != file.Listed.Sha256)
|
||||
return false;
|
||||
|
||||
// The time of the last change tells, e.g., the order of the chats in a workspace:
|
||||
File.SetLastWriteTime(file.DestinationPath, file.Entry.LastWriteTime.LocalDateTime);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How far the preparation of a restore got.
|
||||
/// </summary>
|
||||
/// <param name="Processed">How many files of the backup are unpacked.</param>
|
||||
/// <param name="Total">How many files there are to unpack, or zero while that is not known yet.</param>
|
||||
public readonly record struct DataRestoreProgress(int Processed, int Total);
|
||||
@@ -0,0 +1,15 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How the runtime applied a restore at this start, as it writes it to DataRestoreFormat.RESULT_FILE_NAME.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The points in time stay text: the runtime leaves them empty when it could not read the plan.
|
||||
/// </remarks>
|
||||
/// <param name="Outcome">How the restore ended.</param>
|
||||
/// <param name="Detail">What happened, in English, for the log and for the support.</param>
|
||||
/// <param name="PreparedAtUtc">When the restore was prepared, or an empty string.</param>
|
||||
/// <param name="BackupCreatedAtUtc">When the backup was made, or an empty string.</param>
|
||||
/// <param name="BackupAppVersion">The version of AI Studio which made the backup, or an empty string.</param>
|
||||
/// <param name="FinishedAtUtc">When the runtime finished.</param>
|
||||
public sealed record DataRestoreReport(DataRestoreOutcome Outcome, string Detail, string PreparedAtUtc, string BackupCreatedAtUtc, string BackupAppVersion, string FinishedAtUtc);
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How the preparation of a restore ended.
|
||||
/// </summary>
|
||||
/// <param name="Success">Whether the restore is prepared and waits for the next start.</param>
|
||||
/// <param name="Cancelled">Whether the user stopped the preparation.</param>
|
||||
/// <param name="Issue">Why the restore cannot be prepared, for the user. Empty when it is prepared.</param>
|
||||
public sealed record DataRestoreResult(bool Success, bool Cancelled, string Issue);
|
||||
@@ -0,0 +1,13 @@
|
||||
using SettingsVersion = AIStudio.Settings.Version;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Where a restore goes, and what it has to fit to.
|
||||
/// </summary>
|
||||
/// <param name="DataDirectory">The data folder.</param>
|
||||
/// <param name="ConfigDirectory">The configuration folder.</param>
|
||||
/// <param name="AppVersion">The version of this AI Studio. A backup of a later version is refused.</param>
|
||||
/// <param name="SettingsVersion">The version of the settings this AI Studio writes. A backup with later settings is refused.</param>
|
||||
/// <param name="OrganizationPluginFolders">The plugins an organization deployed among the user's own ones, cf. DataBackupContents.FindOrganizationPluginsAmongUserPlugins. They stay as they are.</param>
|
||||
public sealed record DataRestoreTarget(string DataDirectory, string ConfigDirectory, string AppVersion, SettingsVersion SettingsVersion, IReadOnlyList<string> OrganizationPluginFolders);
|
||||
@@ -0,0 +1,10 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// A secret setting of a tool which the user set up, and which may lack its value after a restore.
|
||||
/// </summary>
|
||||
/// <param name="ToolId">The tool.</param>
|
||||
/// <param name="FieldName">The setting, as the schema of the tool names it.</param>
|
||||
/// <param name="ToolName">The name of the tool, as the user reads it.</param>
|
||||
/// <param name="SettingLabel">The label of the setting, as the user reads it.</param>
|
||||
public sealed record DataRestoreToolSecret(string ToolId, string FieldName, string ToolName, string SettingLabel);
|
||||
@@ -0,0 +1,23 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Tells whether the OS keyring of this computer holds a secret.
|
||||
/// </summary>
|
||||
public interface IStoredSecrets
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the API key of a provider is stored.
|
||||
/// </summary>
|
||||
/// <param name="secretId">The provider.</param>
|
||||
/// <param name="storeType">The kind of provider.</param>
|
||||
/// <returns>True when the key is stored.</returns>
|
||||
public Task<bool> HasAPIKeyAsync(ISecretId secretId, SecretStoreType storeType);
|
||||
|
||||
/// <summary>
|
||||
/// Whether another secret is stored, e.g., the password of a data source.
|
||||
/// </summary>
|
||||
/// <param name="secretId">What the secret belongs to.</param>
|
||||
/// <param name="storeType">The kind of secret.</param>
|
||||
/// <returns>True when the secret is stored.</returns>
|
||||
public Task<bool> HasSecretAsync(ISecretId secretId, SecretStoreType storeType);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Asks the runtime whether the OS keyring holds a secret. It only tries, so a missing secret logs no error.
|
||||
/// </summary>
|
||||
/// <param name="rustService">The runtime.</param>
|
||||
public sealed class RustStoredSecrets(RustService rustService) : IStoredSecrets
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> HasAPIKeyAsync(ISecretId secretId, SecretStoreType storeType) => (await rustService.GetAPIKey(secretId, storeType, isTrying: true)).Success;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> HasSecretAsync(ISecretId secretId, SecretStoreType storeType) => (await rustService.GetSecret(secretId, storeType, isTrying: true)).Success;
|
||||
}
|
||||
Reference in new issue
Block a user