mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-17 22:57:59 +00:00
Added backups of all data, restorable on another computer (#1008)
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
1 parent
ab45580473
commit
936043319c
144 files changed
+10417
-424
No files matched your search
@@ -0,0 +1,17 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Determines what the import does when a chat from the archive already exists.
|
||||
/// </summary>
|
||||
public enum ChatArchiveCollisionBehavior
|
||||
{
|
||||
/// <summary>
|
||||
/// Keeps the existing chat and ignores the one from the archive.
|
||||
/// </summary>
|
||||
SKIP,
|
||||
|
||||
/// <summary>
|
||||
/// Imports the chat from the archive as an additional chat.
|
||||
/// </summary>
|
||||
IMPORT_AS_COPY,
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
using AIStudio.Tools.DataBackup;
|
||||
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Where a backup keeps its chats, for adding them to the chats there are.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A backup holds the chats in the layout of the data folder, cf. DataBackupFormat: every chat in a
|
||||
/// directory of its own, below its workspace or among the temporary chats.
|
||||
/// </remarks>
|
||||
public static class ChatArchiveFormat
|
||||
{
|
||||
/// <summary>
|
||||
/// Where a backup keeps the workspaces, each in a directory named after its ID.
|
||||
/// </summary>
|
||||
public static readonly string WORKSPACES_PREFIX = DataBackupFormat.DATA_ROOT + DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.WORKSPACES).RelativePath;
|
||||
|
||||
/// <summary>
|
||||
/// Where a backup keeps the temporary chats, each in a directory named after its ID.
|
||||
/// </summary>
|
||||
public static readonly string TEMPORARY_CHATS_PREFIX = DataBackupFormat.DATA_ROOT + DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.TEMPORARY_CHATS).RelativePath;
|
||||
|
||||
/// <summary>
|
||||
/// The file storing the workspace or chat name.
|
||||
/// </summary>
|
||||
public const string NAME_FILE_NAME = "name";
|
||||
|
||||
/// <summary>
|
||||
/// The file storing the serialized chat thread.
|
||||
/// </summary>
|
||||
public const string THREAD_FILE_NAME = "thread.json";
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// The outcome of importing one single chat.
|
||||
/// </summary>
|
||||
public enum ChatArchiveImportOutcome
|
||||
{
|
||||
/// <summary>
|
||||
/// The chat was written to the chat storage.
|
||||
/// </summary>
|
||||
IMPORTED,
|
||||
|
||||
/// <summary>
|
||||
/// The chat already existed and was left alone.
|
||||
/// </summary>
|
||||
SKIPPED,
|
||||
|
||||
/// <summary>
|
||||
/// The chat could not be imported.
|
||||
/// </summary>
|
||||
FAILED,
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Describes the contents of an archive before anything is imported.
|
||||
/// </summary>
|
||||
/// <param name="Success">Whether the archive could be read.</param>
|
||||
/// <param name="Manifest">The manifest of the archive.</param>
|
||||
/// <param name="Issue">The reason why the archive could not be read, if it could not.</param>
|
||||
public sealed record ChatArchiveImportPreview(bool Success, ChatArchiveManifest Manifest, string Issue);
|
||||
@@ -0,0 +1,20 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// The outcome of a chat archive import.
|
||||
/// </summary>
|
||||
/// <param name="Success">Whether the import ran without an error.</param>
|
||||
/// <param name="ImportedWorkspaces">The number of workspaces which received chats.</param>
|
||||
/// <param name="ImportedChats">The number of imported chats.</param>
|
||||
/// <param name="SkippedChats">The number of chats which were skipped, because they already existed.</param>
|
||||
/// <param name="FailedChats">The number of chats which could not be imported.</param>
|
||||
/// <param name="Issue">The reason why the import failed, if it did.</param>
|
||||
/// <param name="Cancelled">Whether the user stopped the import. Chats imported up to that point are kept.</param>
|
||||
public sealed record ChatArchiveImportResult(
|
||||
bool Success,
|
||||
int ImportedWorkspaces,
|
||||
int ImportedChats,
|
||||
int SkippedChats,
|
||||
int FailedChats,
|
||||
string Issue,
|
||||
bool Cancelled);
|
||||
@@ -0,0 +1,539 @@
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools.DataBackup;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Adds the chats of a backup to the chats there are, and leaves everything else alone.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The other way to use a backup, next to restoring it: a restore replaces all data, while this keeps
|
||||
/// every chat and adds those of the backup. A chat which exists already is kept, or added once more,
|
||||
/// as the user decides.<br/><br/>
|
||||
///
|
||||
/// The backup is opened for reading only, and every entry is read as a stream, cf. DataBackupReader.
|
||||
/// </remarks>
|
||||
public static class ChatArchiveImporter
|
||||
{
|
||||
private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger(nameof(ChatArchiveImporter));
|
||||
|
||||
private const int BUFFER_SIZE = 64 * 1024;
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ChatArchiveImporter).Namespace, nameof(ChatArchiveImporter));
|
||||
|
||||
/// <summary>
|
||||
/// One entry of a backup, with its name checked and normalized.
|
||||
/// </summary>
|
||||
private sealed record BackupEntry(string Name, ZipArchiveEntry Entry);
|
||||
|
||||
/// <summary>
|
||||
/// What one import did so far.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every counter is raised as the import goes: a cancellation leaves the chats imported so far in
|
||||
/// the storage, so the numbers have to describe them even when the import never reaches its end.
|
||||
/// </remarks>
|
||||
private sealed class ImportTally
|
||||
{
|
||||
public int ProcessedChats;
|
||||
public int ImportedWorkspaces;
|
||||
public int ImportedChats;
|
||||
public int SkippedChats;
|
||||
public int FailedChats;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads which chats a backup holds, without adding any.
|
||||
/// </summary>
|
||||
/// <param name="archivePath">The backup to look into.</param>
|
||||
/// <param name="token">Cancels the reading.</param>
|
||||
/// <returns>The chats of the backup, or why the backup cannot be used.</returns>
|
||||
public static async Task<ChatArchiveImportPreview> ReadPreviewAsync(string archivePath, CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var archive = ZipFile.OpenRead(archivePath);
|
||||
var manifestResult = await DataBackupReader.ReadManifestAsync(archive, token);
|
||||
if (manifestResult.Manifest is not { } manifest)
|
||||
return new(false, new(), manifestResult.Issue);
|
||||
|
||||
var chats = await ListChatsAsync(archive, GroupEntriesByChat(archive), token);
|
||||
return new(true, chats with { CreatedAt = manifest.CreatedAtUtc, AppVersion = manifest.AppVersion }, string.Empty);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
return new(false, new(), TB("This file is no backup of AI Studio, or it was not written completely."));
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogError(exception, "Failed to read the backup '{ArchivePath}'.", archivePath);
|
||||
return new(false, new(), string.Format(TB("Unexpected error: {0}"), exception.Message));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds all chats of a backup to the chat storage.
|
||||
/// </summary>
|
||||
/// <param name="archivePath">The backup.</param>
|
||||
/// <param name="collisionBehavior">What to do with chats which exist already.</param>
|
||||
/// <param name="progress">Receives the progress.</param>
|
||||
/// <param name="token">Cancels the import. Chats added before the cancellation are kept.</param>
|
||||
/// <returns>What the import did.</returns>
|
||||
public static async Task<ChatArchiveImportResult> ImportAsync(string archivePath, ChatArchiveCollisionBehavior collisionBehavior, IProgress<ChatArchiveProgress>? progress, CancellationToken token)
|
||||
{
|
||||
var tally = new ImportTally();
|
||||
var preview = await ReadPreviewAsync(archivePath, token);
|
||||
if (!preview.Success)
|
||||
return new(false, 0, 0, 0, 0, preview.Issue, false);
|
||||
|
||||
try
|
||||
{
|
||||
var manifest = preview.Manifest;
|
||||
using var archive = ZipFile.OpenRead(archivePath);
|
||||
var entriesByChat = GroupEntriesByChat(archive);
|
||||
|
||||
// Which workspaces exist already is asked once per workspace of the backup, so read
|
||||
// the tree once instead of building a full snapshot every time:
|
||||
var tree = await WorkspaceBehaviour.GetOrLoadWorkspaceTreeShellAsync();
|
||||
var knownWorkspaceIds = tree.Workspaces.Select(workspace => workspace.WorkspaceId).ToHashSet();
|
||||
|
||||
//
|
||||
// A chat counts as existing wherever it is stored, not just in the workspace the backup
|
||||
// names: it may have been moved since the backup was made. Adding it there again under
|
||||
// the same identity would leave two chats with one ID, while the app finds a chat, and
|
||||
// the answer being written for it, by that ID alone:
|
||||
//
|
||||
var existingChatIds = tree.TemporaryChats.Select(chat => chat.ChatId).ToHashSet();
|
||||
foreach (var workspace in tree.Workspaces)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
foreach (var chat in await WorkspaceBehaviour.GetWorkspaceChatsAsync(workspace.WorkspaceId))
|
||||
existingChatIds.Add(chat.ChatId);
|
||||
}
|
||||
|
||||
foreach (var workspace in manifest.Workspaces)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var targetWorkspaceId = await ResolveTargetWorkspaceAsync(workspace, knownWorkspaceIds);
|
||||
if (targetWorkspaceId == Guid.Empty)
|
||||
{
|
||||
LOG.LogWarning("Skipping workspace '{WorkspaceName}' of the backup, because no target workspace could be prepared.", workspace.Name);
|
||||
tally.FailedChats += workspace.Chats.Count;
|
||||
tally.ProcessedChats += workspace.Chats.Count;
|
||||
continue;
|
||||
}
|
||||
|
||||
var workspaceEntryPrefix = $"{ChatArchiveFormat.WORKSPACES_PREFIX}{workspace.WorkspaceId}{ZipEntryNames.SEPARATOR}";
|
||||
await ImportChatsAsync(tally, workspaceEntryPrefix, targetWorkspaceId, workspace.Chats, true, manifest.TotalChatCount, entriesByChat, collisionBehavior, existingChatIds, progress, token);
|
||||
}
|
||||
|
||||
await ImportChatsAsync(tally, ChatArchiveFormat.TEMPORARY_CHATS_PREFIX, Guid.Empty, manifest.TemporaryChats, false, manifest.TotalChatCount, entriesByChat, collisionBehavior, existingChatIds, progress, token);
|
||||
|
||||
progress?.Report(new(tally.ProcessedChats, manifest.TotalChatCount));
|
||||
LOG.LogInformation("Imported {ImportedChats} chats from the backup '{ArchivePath}'. Skipped {SkippedChats} existing chats, {FailedChats} chats failed.", tally.ImportedChats, archivePath, tally.SkippedChats, tally.FailedChats);
|
||||
|
||||
return new(true, tally.ImportedWorkspaces, tally.ImportedChats, tally.SkippedChats, tally.FailedChats, string.Empty, false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
//
|
||||
// The chats imported so far stay in the chat storage. So report what happened instead
|
||||
// of throwing, otherwise the user has no idea what is now part of their chats:
|
||||
//
|
||||
LOG.LogInformation("The user cancelled the import of the backup '{ArchivePath}' after {ImportedChats} chats.", archivePath, tally.ImportedChats);
|
||||
return new(true, tally.ImportedWorkspaces, tally.ImportedChats, tally.SkippedChats, tally.FailedChats, string.Empty, true);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogError(exception, "Failed to import the backup '{ArchivePath}'.", archivePath);
|
||||
return new(false, tally.ImportedWorkspaces, tally.ImportedChats, tally.SkippedChats, tally.FailedChats, string.Format(TB("Unexpected error: {0}"), exception.Message), false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Adds the chats of one workspace of the backup, or its temporary chats.
|
||||
/// </summary>
|
||||
private static async Task ImportChatsAsync(ImportTally tally, string entryPrefix, Guid targetWorkspaceId, IReadOnlyList<ChatArchiveManifestChat> chats, bool countsAsWorkspace, int totalChats, Dictionary<string, List<BackupEntry>> entriesByChat, ChatArchiveCollisionBehavior collisionBehavior, HashSet<Guid> existingChatIds, IProgress<ChatArchiveProgress>? progress, CancellationToken token)
|
||||
{
|
||||
var isFirstImportOfWorkspace = true;
|
||||
foreach (var chat in chats)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
progress?.Report(new(tally.ProcessedChats, totalChats));
|
||||
tally.ProcessedChats++;
|
||||
|
||||
var chatEntryPrefix = $"{entryPrefix}{chat.ChatId}{ZipEntryNames.SEPARATOR}";
|
||||
var chatEntries = entriesByChat.GetValueOrDefault(chatEntryPrefix, []);
|
||||
|
||||
switch (await ImportChatAsync(chatEntries, chatEntryPrefix, targetWorkspaceId, chat, collisionBehavior, existingChatIds, token))
|
||||
{
|
||||
case ChatArchiveImportOutcome.IMPORTED:
|
||||
tally.ImportedChats++;
|
||||
if (countsAsWorkspace && isFirstImportOfWorkspace)
|
||||
{
|
||||
tally.ImportedWorkspaces++;
|
||||
isFirstImportOfWorkspace = false;
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case ChatArchiveImportOutcome.SKIPPED:
|
||||
tally.SkippedChats++;
|
||||
break;
|
||||
|
||||
case ChatArchiveImportOutcome.FAILED:
|
||||
tally.FailedChats++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sorts the entries of a backup by the chat they belong to. Without this, every chat would have
|
||||
/// to scan all entries of the backup.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// An entry whose name could lead outside of the chat directory is left out, cf. ZipEntryNames.
|
||||
/// </remarks>
|
||||
private static Dictionary<string, List<BackupEntry>> GroupEntriesByChat(ZipArchive archive)
|
||||
{
|
||||
var entriesByChat = new Dictionary<string, List<BackupEntry>>(StringComparer.Ordinal);
|
||||
foreach (var entry in archive.Entries)
|
||||
{
|
||||
if (!ZipEntryNames.TryNormalize(entry.FullName, out var entryName))
|
||||
{
|
||||
LOG.LogWarning("Ignoring the backup entry '{EntryName}', because its name could lead outside of the chat directory.", entry.FullName);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip directory entries:
|
||||
if (entryName.EndsWith(ZipEntryNames.SEPARATOR))
|
||||
continue;
|
||||
|
||||
var chatEntryPrefix = GetChatEntryPrefix(entryName);
|
||||
if (chatEntryPrefix is null)
|
||||
continue;
|
||||
|
||||
if (!entriesByChat.TryGetValue(chatEntryPrefix, out var chatEntries))
|
||||
{
|
||||
chatEntries = [];
|
||||
entriesByChat[chatEntryPrefix] = chatEntries;
|
||||
}
|
||||
|
||||
chatEntries.Add(new(entryName, entry));
|
||||
}
|
||||
|
||||
return entriesByChat;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the chat an entry belongs to: everything up to and including the ID of the chat, so
|
||||
/// "data/workspaces/W/C/thread.json" becomes "data/workspaces/W/C/".
|
||||
/// </summary>
|
||||
/// <returns>The chat prefix, or null when the entry belongs to no chat.</returns>
|
||||
private static string? GetChatEntryPrefix(string entryName)
|
||||
{
|
||||
string root;
|
||||
int levels;
|
||||
if (entryName.StartsWith(ChatArchiveFormat.WORKSPACES_PREFIX, StringComparison.Ordinal))
|
||||
{
|
||||
root = ChatArchiveFormat.WORKSPACES_PREFIX;
|
||||
levels = 2;
|
||||
}
|
||||
else if (entryName.StartsWith(ChatArchiveFormat.TEMPORARY_CHATS_PREFIX, StringComparison.Ordinal))
|
||||
{
|
||||
root = ChatArchiveFormat.TEMPORARY_CHATS_PREFIX;
|
||||
levels = 1;
|
||||
}
|
||||
else
|
||||
return null;
|
||||
|
||||
var index = root.Length;
|
||||
for (var level = 0; level < levels; level++)
|
||||
{
|
||||
var next = entryName.IndexOf(ZipEntryNames.SEPARATOR, index);
|
||||
if (next < 0)
|
||||
return null;
|
||||
|
||||
index = next + 1;
|
||||
}
|
||||
|
||||
return entryName[..index];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Lists the chats of a backup, with the names of their workspaces.
|
||||
/// </summary>
|
||||
private static async Task<ChatArchiveManifest> ListChatsAsync(ZipArchive archive, Dictionary<string, List<BackupEntry>> entriesByChat, CancellationToken token)
|
||||
{
|
||||
var workspaces = new Dictionary<Guid, ChatArchiveManifestWorkspace>();
|
||||
var temporaryChats = new List<ChatArchiveManifestChat>();
|
||||
foreach (var (chatEntryPrefix, chatEntries) in entriesByChat)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
// A chat without its thread cannot be added, so it is not listed either:
|
||||
if (!chatEntries.Any(entry => entry.Name.Equals($"{chatEntryPrefix}{ChatArchiveFormat.THREAD_FILE_NAME}", StringComparison.OrdinalIgnoreCase)))
|
||||
continue;
|
||||
|
||||
var segments = chatEntryPrefix.TrimEnd(ZipEntryNames.SEPARATOR).Split(ZipEntryNames.SEPARATOR);
|
||||
if (!Guid.TryParse(segments[^1], out var chatId))
|
||||
continue;
|
||||
|
||||
var nameEntry = chatEntries.FirstOrDefault(entry => entry.Name.Equals($"{chatEntryPrefix}{ChatArchiveFormat.NAME_FILE_NAME}", StringComparison.OrdinalIgnoreCase));
|
||||
var chat = new ChatArchiveManifestChat
|
||||
{
|
||||
ChatId = chatId,
|
||||
Name = nameEntry is null ? string.Empty : await ReadTextAsync(nameEntry.Entry, token),
|
||||
};
|
||||
|
||||
if (chatEntryPrefix.StartsWith(ChatArchiveFormat.TEMPORARY_CHATS_PREFIX, StringComparison.Ordinal))
|
||||
{
|
||||
temporaryChats.Add(chat);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Guid.TryParse(segments[^2], out var workspaceId))
|
||||
continue;
|
||||
|
||||
if (!workspaces.TryGetValue(workspaceId, out var workspace))
|
||||
{
|
||||
var workspaceNameEntry = archive.GetEntry($"{ChatArchiveFormat.WORKSPACES_PREFIX}{workspaceId}{ZipEntryNames.SEPARATOR}{ChatArchiveFormat.NAME_FILE_NAME}");
|
||||
workspace = new ChatArchiveManifestWorkspace
|
||||
{
|
||||
WorkspaceId = workspaceId,
|
||||
Name = workspaceNameEntry is null ? string.Empty : await ReadTextAsync(workspaceNameEntry, token),
|
||||
};
|
||||
|
||||
workspaces[workspaceId] = workspace;
|
||||
}
|
||||
|
||||
workspace.Chats.Add(chat);
|
||||
}
|
||||
|
||||
return new()
|
||||
{
|
||||
Workspaces = workspaces.Values.OrderBy(workspace => workspace.Name, StringComparer.CurrentCultureIgnoreCase).ToList(),
|
||||
TemporaryChats = temporaryChats,
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<string> ReadTextAsync(ZipArchiveEntry entry, CancellationToken token)
|
||||
{
|
||||
using var reader = new StreamReader(entry.Open());
|
||||
return (await reader.ReadToEndAsync(token)).Trim();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the workspace which receives the chats of a workspace of the backup. Known
|
||||
/// workspaces are reused, unknown ones are created with their original identity.
|
||||
/// </summary>
|
||||
/// <param name="workspace">The workspace of the backup.</param>
|
||||
/// <param name="knownWorkspaceIds">The workspaces which exist already. Newly created ones are added.</param>
|
||||
private static async Task<Guid> ResolveTargetWorkspaceAsync(ChatArchiveManifestWorkspace workspace, HashSet<Guid> knownWorkspaceIds)
|
||||
{
|
||||
if (workspace.WorkspaceId == Guid.Empty)
|
||||
return Guid.Empty;
|
||||
|
||||
if (knownWorkspaceIds.Contains(workspace.WorkspaceId))
|
||||
return workspace.WorkspaceId;
|
||||
|
||||
var workspaceName = string.IsNullOrWhiteSpace(workspace.Name) ? TB("Imported workspace") : workspace.Name;
|
||||
await WorkspaceBehaviour.EnsureWorkspace(workspace.WorkspaceId, await CreateUniqueWorkspaceNameAsync(workspaceName));
|
||||
knownWorkspaceIds.Add(workspace.WorkspaceId);
|
||||
|
||||
return workspace.WorkspaceId;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extends the workspace name by a counter until it is no longer taken. Workspace names
|
||||
/// must be unique, otherwise the workspace cannot be renamed later on.
|
||||
/// </summary>
|
||||
private static async Task<string> CreateUniqueWorkspaceNameAsync(string workspaceName)
|
||||
{
|
||||
var candidate = workspaceName;
|
||||
var counter = 1;
|
||||
|
||||
while (await WorkspaceBehaviour.IsWorkspaceNameExistingAsync(candidate))
|
||||
{
|
||||
counter++;
|
||||
candidate = $"{workspaceName} ({counter})";
|
||||
}
|
||||
|
||||
return candidate;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Imports one chat: extracts its files into a fresh chat directory, points the paths of
|
||||
/// those files to it, and stores the chat.
|
||||
/// </summary>
|
||||
/// <param name="chatEntries">The backup entries of the chat.</param>
|
||||
/// <param name="chatEntryPrefix">The path of the chat within the backup, which every entry of the chat starts with.</param>
|
||||
/// <param name="targetWorkspaceId">The workspace to import the chat into, or Guid.Empty for a temporary chat.</param>
|
||||
/// <param name="chat">The chat as the preview lists it.</param>
|
||||
/// <param name="collisionBehavior">What to do when a chat with the same ID exists already.</param>
|
||||
/// <param name="existingChatIds">The IDs of all chats in the storage. An imported chat is added.</param>
|
||||
/// <param name="token">Cancels the import.</param>
|
||||
private static async Task<ChatArchiveImportOutcome> ImportChatAsync(IReadOnlyList<BackupEntry> chatEntries, string chatEntryPrefix, Guid targetWorkspaceId, ChatArchiveManifestChat chat, ChatArchiveCollisionBehavior collisionBehavior, HashSet<Guid> existingChatIds, CancellationToken token)
|
||||
{
|
||||
var threadEntry = chatEntries.FirstOrDefault(entry => entry.Name.Equals($"{chatEntryPrefix}{ChatArchiveFormat.THREAD_FILE_NAME}", StringComparison.OrdinalIgnoreCase));
|
||||
if (threadEntry is null)
|
||||
{
|
||||
LOG.LogWarning("The backup contains no chat thread for the chat '{ChatId}'.", chat.ChatId);
|
||||
return ChatArchiveImportOutcome.FAILED;
|
||||
}
|
||||
|
||||
//
|
||||
// The directory is checked as well: a chat the storage could not list, for example
|
||||
// because its thread is unreadable, still occupies its place on disk.
|
||||
//
|
||||
var targetChatId = chat.ChatId;
|
||||
if (existingChatIds.Contains(targetChatId) || WorkspaceBehaviour.IsChatExisting(new(targetWorkspaceId, targetChatId)))
|
||||
{
|
||||
if (collisionBehavior is ChatArchiveCollisionBehavior.SKIP)
|
||||
return ChatArchiveImportOutcome.SKIPPED;
|
||||
|
||||
targetChatId = Guid.NewGuid();
|
||||
}
|
||||
|
||||
var chatDirectory = WorkspaceBehaviour.GetChatDirectory(targetWorkspaceId, targetChatId);
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(chatDirectory);
|
||||
|
||||
//
|
||||
// Extract everything except the chat thread and the name file. Both are written
|
||||
// by the chat storage itself once the paths inside the thread are corrected:
|
||||
//
|
||||
var extractedPaths = new HashSet<string>(PathTools.COMPARER);
|
||||
foreach (var entry in chatEntries)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var relativePath = entry.Name[chatEntryPrefix.Length..];
|
||||
if (relativePath.Equals(ChatArchiveFormat.THREAD_FILE_NAME, StringComparison.OrdinalIgnoreCase) ||
|
||||
relativePath.Equals(ChatArchiveFormat.NAME_FILE_NAME, StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
var destinationPath = ZipEntryNames.GetDestinationPath(chatDirectory, relativePath);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
|
||||
|
||||
await using var entryStream = entry.Entry.Open();
|
||||
await using var fileStream = new FileStream(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await entryStream.CopyToAsync(fileStream, token);
|
||||
|
||||
extractedPaths.Add(relativePath);
|
||||
}
|
||||
|
||||
ChatThread? thread;
|
||||
await using (var threadStream = threadEntry.Entry.Open())
|
||||
thread = await JsonSerializer.DeserializeAsync<ChatThread>(threadStream, WorkspaceBehaviour.JSON_OPTIONS, token);
|
||||
|
||||
if (thread is null)
|
||||
{
|
||||
LOG.LogWarning("The chat thread of the chat '{ChatId}' could not be read.", chat.ChatId);
|
||||
TryDeleteDirectory(chatDirectory);
|
||||
return ChatArchiveImportOutcome.FAILED;
|
||||
}
|
||||
|
||||
//
|
||||
// A file below the data directory is stored with a path relative to it, which reading
|
||||
// the thread resolved against the data directory of this computer: where the chat lay
|
||||
// when the backup was made. Only the paths of files which were actually extracted move
|
||||
// to the imported chat. Everything else keeps its path:
|
||||
//
|
||||
var originalChatDirectory = WorkspaceBehaviour.GetChatDirectory(thread.WorkspaceId, thread.ChatId);
|
||||
thread = thread with { ChatId = targetChatId };
|
||||
thread.WorkspaceId = targetWorkspaceId;
|
||||
thread.RewriteFilePaths(path =>
|
||||
{
|
||||
var chatRelativePath = Path.GetRelativePath(originalChatDirectory, path).Replace(Path.DirectorySeparatorChar, ZipEntryNames.SEPARATOR);
|
||||
return extractedPaths.Contains(chatRelativePath)
|
||||
? Path.Combine(chatDirectory, chatRelativePath.Replace(ZipEntryNames.SEPARATOR, Path.DirectorySeparatorChar))
|
||||
: path;
|
||||
});
|
||||
|
||||
CopyTranscriptsStoredElsewhere(thread, chatDirectory);
|
||||
|
||||
//
|
||||
// The storage skips a chat whose lock another operation holds for too long. The
|
||||
// directory prepared above would then hold the extracted files but no chat:
|
||||
//
|
||||
if (!await WorkspaceBehaviour.StoreChatAsync(thread))
|
||||
{
|
||||
LOG.LogWarning("The chat '{ChatId}' could not be stored, because another operation held it for too long.", chat.ChatId);
|
||||
TryDeleteDirectory(chatDirectory);
|
||||
return ChatArchiveImportOutcome.FAILED;
|
||||
}
|
||||
|
||||
existingChatIds.Add(targetChatId);
|
||||
return ChatArchiveImportOutcome.IMPORTED;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
TryDeleteDirectory(chatDirectory);
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogError(exception, "Failed to import the chat '{ChatId}' into the workspace '{WorkspaceId}'.", chat.ChatId, targetWorkspaceId);
|
||||
TryDeleteDirectory(chatDirectory);
|
||||
return ChatArchiveImportOutcome.FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gives the imported chat its own copy of every transcript the backup did not carry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Such a transcript still names the place where it lay when the backup was made. On the
|
||||
/// same computer that is the chat the backup came from, and since a chat deletes the
|
||||
/// transcripts it names, deleting the imported chat would take the transcripts of the
|
||||
/// original along. A copied chat gets its own copies for the same reason, so this uses the
|
||||
/// same code. A transcript which no longer exists keeps its place in the chat and shows up
|
||||
/// as missing, again as it does in a copy.
|
||||
/// </remarks>
|
||||
/// <param name="thread">The imported chat thread, after the paths of its extracted files were pointed to it.</param>
|
||||
/// <param name="chatDirectory">The directory of the imported chat.</param>
|
||||
private static void CopyTranscriptsStoredElsewhere(ChatThread thread, string chatDirectory)
|
||||
{
|
||||
var transcriptDirectory = ManagedTranscriptAttachment.GetTranscriptDirectory(chatDirectory);
|
||||
var copiedPaths = new Dictionary<string, ManagedTranscriptAttachment>(PathTools.COMPARER);
|
||||
|
||||
//
|
||||
// Only a transcript in the data directory can belong to another chat, and only there
|
||||
// does deleting a chat delete its transcripts. Any other path is left alone, so that a
|
||||
// backup cannot get files from elsewhere on this computer copied into a chat:
|
||||
//
|
||||
thread.ReplaceAttachments(attachment => attachment is ManagedTranscriptAttachment transcript
|
||||
&& !PathTools.IsInsideDirectory(chatDirectory, transcript.FilePath)
|
||||
&& PathTools.IsInsideDirectory(SettingsManager.DataDirectory!, transcript.FilePath)
|
||||
? WorkspaceBehaviour.CopyManagedTranscriptAttachment(thread, transcript, transcriptDirectory, copiedPaths)
|
||||
: attachment);
|
||||
}
|
||||
|
||||
private static void TryDeleteDirectory(string directory)
|
||||
{
|
||||
if (!Directory.Exists(directory))
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(directory, true);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogWarning(exception, "Failed to clean up the incomplete chat directory '{ChatDirectory}'.", directory);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// The chats a backup holds, so that the user sees them before any is added.
|
||||
/// </summary>
|
||||
public sealed record ChatArchiveManifest
|
||||
{
|
||||
/// <summary>
|
||||
/// When the backup was made.
|
||||
/// </summary>
|
||||
public DateTimeOffset CreatedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The AI Studio version which made the backup. Informational only.
|
||||
/// </summary>
|
||||
public string AppVersion { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// All workspaces with chats in the backup.
|
||||
/// </summary>
|
||||
public List<ChatArchiveManifestWorkspace> Workspaces { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// All temporary chats in the backup.
|
||||
/// </summary>
|
||||
public List<ChatArchiveManifestChat> TemporaryChats { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Gets the total number of chats in the backup.
|
||||
/// </summary>
|
||||
public int TotalChatCount => this.Workspaces.Sum(workspace => workspace.Chats.Count) + this.TemporaryChats.Count;
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// One chat in a backup.
|
||||
/// </summary>
|
||||
public sealed record ChatArchiveManifestChat
|
||||
{
|
||||
/// <summary>
|
||||
/// The unique identifier of the chat.
|
||||
/// </summary>
|
||||
public Guid ChatId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The name of the chat when the backup was made.
|
||||
/// </summary>
|
||||
public string Name { get; init; } = string.Empty;
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// One workspace with chats in a backup.
|
||||
/// </summary>
|
||||
public sealed record ChatArchiveManifestWorkspace
|
||||
{
|
||||
/// <summary>
|
||||
/// The unique identifier of the workspace.
|
||||
/// </summary>
|
||||
public Guid WorkspaceId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The name of the workspace when the backup was made.
|
||||
/// </summary>
|
||||
public string Name { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// All chats of this workspace in the backup.
|
||||
/// </summary>
|
||||
public List<ChatArchiveManifestChat> Chats { get; init; } = [];
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Reports the progress of an export or import operation.
|
||||
/// </summary>
|
||||
/// <param name="ProcessedChats">The number of chats processed so far.</param>
|
||||
/// <param name="TotalChats">The total number of chats to process.</param>
|
||||
public readonly record struct ChatArchiveProgress(int ProcessedChats, int TotalChats);
|
||||
@@ -0,0 +1,119 @@
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What AI Studio keeps in its data and configuration folders, and which of it belongs into a backup.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is a positive list. A backup takes along only the entries marked as included, and restoring
|
||||
/// one replaces only those. Everything else in these folders stays out of a backup and survives a
|
||||
/// restore: the leftovers of a plugin installation, a file somebody else put there, or a folder a
|
||||
/// future version adds before anybody listed it here.<br/><br/>
|
||||
///
|
||||
/// The entries which stay out are listed, too. The folder panel on the information page tells users
|
||||
/// who back up these folders with a tool of their own what they may leave out, and nobody has to guess
|
||||
/// whether a folder was forgotten or left out on purpose. Whoever adds a file or folder to the data or
|
||||
/// configuration folder adds it here, one way or the other.<br/><br/>
|
||||
///
|
||||
/// The runtime, which applies a restore, keeps its own list of the included entries and refuses to
|
||||
/// replace anything else (REPLACEABLE_ELEMENTS in runtime/src/data_restore.rs). An entry included here
|
||||
/// goes there as well.
|
||||
/// </remarks>
|
||||
public static class DataBackupContents
|
||||
{
|
||||
/// <summary>
|
||||
/// The separator of the relative paths, the same on every OS.
|
||||
/// </summary>
|
||||
public const char SEPARATOR = '/';
|
||||
|
||||
/// <summary>
|
||||
/// The folder of the plugins, within the data folder.
|
||||
/// </summary>
|
||||
public const string PLUGINS_FOLDER = "plugins/";
|
||||
|
||||
/// <summary>
|
||||
/// Every file and folder AI Studio keeps directly in its data or configuration folder, and those
|
||||
/// below the folders of the plugins and the databases which need a decision of their own.
|
||||
/// </summary>
|
||||
public static readonly IReadOnlyList<DataBackupEntry> ENTRIES =
|
||||
[
|
||||
new(DataBackupLocation.CONFIG_DIRECTORY, "settings.json", DataBackupEntryKind.SETTINGS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "workspaces/", DataBackupEntryKind.WORKSPACES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "tempChats/", DataBackupEntryKind.TEMPORARY_CHATS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, PLUGINS_FOLDER, DataBackupEntryKind.PLUGINS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "tokenizers/", DataBackupEntryKind.TOKENIZERS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "visualBriefings/", DataBackupEntryKind.VISUAL_BRIEFINGS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "databases/sqlite/", DataBackupEntryKind.DATA_SOURCE_INDEX),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "databases/vector_database/stores/", DataBackupEntryKind.VECTOR_STORES),
|
||||
|
||||
new(DataBackupLocation.CONFIG_DIRECTORY, "settings.v*.json", DataBackupEntryKind.SETTINGS_OF_OTHER_VERSIONS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "plugins/.internal/", DataBackupEntryKind.BUILT_IN_PLUGINS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "plugins/.config/", DataBackupEntryKind.ORGANIZATION_PLUGINS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "plugins/.config-tests/", DataBackupEntryKind.ORGANIZATION_PLUGINS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "plugins/.lock", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, ".plugin-staging/", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, ".plugin-install-backups/", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, ".plugin-delete-backups/", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "media-staging/", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "mailAttachments/", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "mindwork_ai_studio.pid", DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, DataBackupFormat.STAGING_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, DataRestoreFormat.PENDING_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.CONFIG_DIRECTORY, DataRestoreFormat.PENDING_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, DataRestoreFormat.PREVIOUS_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.CONFIG_DIRECTORY, DataRestoreFormat.PREVIOUS_DIRECTORY, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, DataRestoreFormat.RESULT_FILE_NAME, DataBackupEntryKind.TEMPORARY_FILES),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "pandoc/", DataBackupEntryKind.DOWNLOADED_TOOLS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "data-protection-keys/", DataBackupEntryKind.INTERFACE_KEYS),
|
||||
new(DataBackupLocation.DATA_DIRECTORY, "events.log", DataBackupEntryKind.LOGS),
|
||||
|
||||
// The runtime hides the log of the startup on Linux and macOS:
|
||||
new(DataBackupLocation.DATA_DIRECTORY, OperatingSystem.IsWindows() ? "AI Studio Events.log" : ".AI Studio Events.log", DataBackupEntryKind.LOGS),
|
||||
];
|
||||
|
||||
/// <summary>
|
||||
/// Whether a path lies within an entry: below it for a folder, or the entry itself for a file.
|
||||
/// </summary>
|
||||
/// <param name="entryPath">The path of the entry, as the list holds it.</param>
|
||||
/// <param name="relativePath">The path to check, relative to the same folder and with forward slashes.</param>
|
||||
/// <returns>True when the entry covers the path.</returns>
|
||||
public static bool Covers(string entryPath, string relativePath) => entryPath.EndsWith(SEPARATOR)
|
||||
? relativePath.StartsWith(entryPath, PathTools.COMPARISON)
|
||||
: relativePath.Equals(entryPath, PathTools.COMPARISON);
|
||||
|
||||
/// <summary>
|
||||
/// Finds the plugins which the organization deployed into the folder of the user's plugins.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The config server keeps the plugins of an organization in folders of their own, which the list
|
||||
/// leaves out as a whole. A device management may put them anywhere, though, so their path proves
|
||||
/// nothing: such a plugin declares itself as deployed by the organization instead. A backup leaves
|
||||
/// it out, because the organization brings it back, and a user with a backup tool of their own
|
||||
/// needs to know which folders these are.
|
||||
/// </remarks>
|
||||
/// <param name="plugins">The plugins AI Studio found.</param>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <returns>The folders of these plugins within the data folder, in the form of the list, ordered.</returns>
|
||||
public static IReadOnlyList<string> FindOrganizationPluginsAmongUserPlugins(IEnumerable<IAvailablePlugin> plugins, string dataDirectory)
|
||||
{
|
||||
var excludedPluginFolders = ENTRIES
|
||||
.Where(entry => !entry.IsIncluded && entry.IsFolder && entry.Location is DataBackupLocation.DATA_DIRECTORY && entry.RelativePath.StartsWith(PLUGINS_FOLDER, StringComparison.Ordinal))
|
||||
.Select(entry => entry.RelativePath)
|
||||
.ToList();
|
||||
|
||||
//
|
||||
// Relative to the data folder and with forward slashes, the plugin folders take the form of
|
||||
// the list. That is the form in which AI Studio stores such paths, too:
|
||||
//
|
||||
return plugins
|
||||
.Where(plugin => plugin.IsManagedByConfigServer)
|
||||
.Select(plugin => DataDirectoryRelativePathConverter.ToStored(plugin.LocalPath, dataDirectory).TrimEnd(SEPARATOR) + SEPARATOR)
|
||||
.Where(folder => folder.Length > PLUGINS_FOLDER.Length
|
||||
&& folder.StartsWith(PLUGINS_FOLDER, PathTools.COMPARISON)
|
||||
&& !excludedPluginFolders.Any(excluded => folder.StartsWith(excluded, PathTools.COMPARISON)))
|
||||
.Distinct(PathTools.COMPARER)
|
||||
.Order(PathTools.COMPARER)
|
||||
.ToList();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// A file or folder AI Studio keeps in its data or configuration folder.
|
||||
/// </summary>
|
||||
/// <param name="Location">The folder in which the entry lies.</param>
|
||||
/// <param name="RelativePath">The path of the entry within that folder, with forward slashes. A folder ends with a slash.</param>
|
||||
/// <param name="Kind">What the entry holds, and with it, whether it belongs into a backup.</param>
|
||||
public sealed record DataBackupEntry(DataBackupLocation Location, string RelativePath, DataBackupEntryKind Kind)
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the entry belongs into a backup.
|
||||
/// </summary>
|
||||
public bool IsIncluded => this.Kind is DataBackupEntryKind.SETTINGS
|
||||
or DataBackupEntryKind.WORKSPACES
|
||||
or DataBackupEntryKind.TEMPORARY_CHATS
|
||||
or DataBackupEntryKind.PLUGINS
|
||||
or DataBackupEntryKind.TOKENIZERS
|
||||
or DataBackupEntryKind.VISUAL_BRIEFINGS
|
||||
or DataBackupEntryKind.DATA_SOURCE_INDEX
|
||||
or DataBackupEntryKind.VECTOR_STORES;
|
||||
|
||||
/// <summary>
|
||||
/// Whether the entry is a folder rather than a single file.
|
||||
/// </summary>
|
||||
public bool IsFolder => this.RelativePath.EndsWith(DataBackupContents.SEPARATOR);
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What an entry of the backup contents holds, and with it, whether it belongs into a backup.
|
||||
/// </summary>
|
||||
public enum DataBackupEntryKind
|
||||
{
|
||||
/// <summary>
|
||||
/// The settings. A backup leaves out what a configuration plugin of the organization manages.
|
||||
/// </summary>
|
||||
SETTINGS,
|
||||
|
||||
/// <summary>
|
||||
/// The chats of the workspaces, with their transcripts.
|
||||
/// </summary>
|
||||
WORKSPACES,
|
||||
|
||||
/// <summary>
|
||||
/// The temporary chats, with their transcripts.
|
||||
/// </summary>
|
||||
TEMPORARY_CHATS,
|
||||
|
||||
/// <summary>
|
||||
/// The plugins of the user. A backup leaves out the plugins of the organization.
|
||||
/// </summary>
|
||||
PLUGINS,
|
||||
|
||||
/// <summary>
|
||||
/// The copies of the tokenizers the user chose for a provider.
|
||||
/// </summary>
|
||||
TOKENIZERS,
|
||||
|
||||
/// <summary>
|
||||
/// The projects of the Visual Briefing Assistant.
|
||||
/// </summary>
|
||||
VISUAL_BRIEFINGS,
|
||||
|
||||
/// <summary>
|
||||
/// The SQLite index of the data sources: which files are embedded, and their text chunks.
|
||||
/// </summary>
|
||||
DATA_SOURCE_INDEX,
|
||||
|
||||
/// <summary>
|
||||
/// The vector stores, which hold the embeddings of the data sources.
|
||||
/// </summary>
|
||||
VECTOR_STORES,
|
||||
|
||||
/// <summary>
|
||||
/// The copies of the settings which every version of AI Studio keeps for itself.
|
||||
/// </summary>
|
||||
SETTINGS_OF_OTHER_VERSIONS,
|
||||
|
||||
/// <summary>
|
||||
/// The plugins which come with AI Studio, such as the language packs.
|
||||
/// </summary>
|
||||
BUILT_IN_PLUGINS,
|
||||
|
||||
/// <summary>
|
||||
/// The plugins which the organization deploys, or tries out for one session.
|
||||
/// </summary>
|
||||
ORGANIZATION_PLUGINS,
|
||||
|
||||
/// <summary>
|
||||
/// Files AI Studio needs only for a moment, while it works on something, and removes again.
|
||||
/// </summary>
|
||||
TEMPORARY_FILES,
|
||||
|
||||
/// <summary>
|
||||
/// Tools AI Studio downloaded, which depend on the OS, such as Pandoc.
|
||||
/// </summary>
|
||||
DOWNLOADED_TOOLS,
|
||||
|
||||
/// <summary>
|
||||
/// The keys with which the user interface protects its data. Only the Flatpak keeps them in the data
|
||||
/// folder, since the home folder is read-only there, and AI Studio creates new ones when they are missing.
|
||||
/// </summary>
|
||||
INTERFACE_KEYS,
|
||||
|
||||
/// <summary>
|
||||
/// The log files.
|
||||
/// </summary>
|
||||
LOGS,
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// The format of a backup: a ZIP file which holds the data folder below DATA_ROOT and the
|
||||
/// configuration folder below CONFIG_ROOT, each in the layout AI Studio keeps on the disk, and the
|
||||
/// manifest as its last entry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// What goes in follows DataBackupContents. The paths below the roots are the relative paths of
|
||||
/// that list, so restoring a backup is unpacking it. The manifest comes last: an archive which lacks
|
||||
/// it was not written completely.
|
||||
/// </remarks>
|
||||
public static class DataBackupFormat
|
||||
{
|
||||
/// <summary>
|
||||
/// The version of the format this version of AI Studio writes. A restore refuses a backup of a
|
||||
/// later version, since it cannot know what such a backup holds.
|
||||
/// </summary>
|
||||
public const int FORMAT_VERSION = 1;
|
||||
|
||||
/// <summary>
|
||||
/// The file extension of a backup, named like the one of plugin archives.
|
||||
/// </summary>
|
||||
public const string FILE_EXTENSION = ".mwbackup";
|
||||
|
||||
/// <summary>
|
||||
/// The manifest, the last entry of every backup.
|
||||
/// </summary>
|
||||
public const string MANIFEST_ENTRY_NAME = "manifest.json";
|
||||
|
||||
/// <summary>
|
||||
/// Where the data folder lies within a backup.
|
||||
/// </summary>
|
||||
public const string DATA_ROOT = "data/";
|
||||
|
||||
/// <summary>
|
||||
/// Where the configuration folder lies within a backup.
|
||||
/// </summary>
|
||||
public const string CONFIG_ROOT = "config/";
|
||||
|
||||
/// <summary>
|
||||
/// The directory in the data folder where the copies of the vector stores wait while a backup runs.
|
||||
/// The runtime creates it; keep the name in sync with BACKUP_STAGING_DIRECTORY there.
|
||||
/// </summary>
|
||||
public const string STAGING_DIRECTORY = ".backup-staging/";
|
||||
|
||||
/// <summary>
|
||||
/// How the manifest and the plan of a restore are written: readable for anybody who opens them,
|
||||
/// and for the runtime, which applies the plan.
|
||||
/// </summary>
|
||||
public static readonly JsonSerializerOptions JSON_OPTIONS = new()
|
||||
{
|
||||
WriteIndented = true,
|
||||
PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower,
|
||||
Converters = { new JsonStringEnumConverter() },
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// File types which are compressed already. Compressing them once more costs time and makes the
|
||||
/// backup no smaller.
|
||||
/// </summary>
|
||||
public static readonly IReadOnlySet<string> ALREADY_COMPRESSED_EXTENSIONS = new HashSet<string>(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
".jpg", ".jpeg", ".png", ".gif", ".webp", ".heic", ".tiff",
|
||||
".mp3", ".m4a", ".m4b", ".aac", ".ogg", ".opus", ".flac", ".wma",
|
||||
".mp4", ".m4v", ".mov", ".mkv", ".webm", ".avi", ".wmv", ".flv",
|
||||
".zip", ".7z", ".gz", ".bz2", ".xz", ".rar",
|
||||
".pdf", ".docx", ".xlsx", ".pptx", ".odt", ".odp",
|
||||
".mwplugin", FILE_EXTENSION,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The name of the entry which holds a file of the data or configuration folder.
|
||||
/// </summary>
|
||||
/// <param name="location">The folder of the file.</param>
|
||||
/// <param name="relativePath">The path of the file within its folder, with forward slashes.</param>
|
||||
/// <returns>The name of the entry.</returns>
|
||||
public static string EntryName(DataBackupLocation location, string relativePath) => location switch
|
||||
{
|
||||
DataBackupLocation.CONFIG_DIRECTORY => CONFIG_ROOT + relativePath,
|
||||
_ => DATA_ROOT + relativePath,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// The folder in which an entry of the backup contents lies.
|
||||
/// </summary>
|
||||
public enum DataBackupLocation
|
||||
{
|
||||
/// <summary>
|
||||
/// The data folder, where AI Studio keeps the chats, the plugins, and the indexes of the data sources.
|
||||
/// </summary>
|
||||
DATA_DIRECTORY,
|
||||
|
||||
/// <summary>
|
||||
/// The configuration folder, where AI Studio keeps its settings.
|
||||
/// </summary>
|
||||
CONFIG_DIRECTORY,
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What a backup holds, and what wrote it.
|
||||
/// </summary>
|
||||
/// <param name="FormatVersion">The version of the format, cf. DataBackupFormat.FORMAT_VERSION.</param>
|
||||
/// <param name="AppVersion">The version of AI Studio which wrote the backup.</param>
|
||||
/// <param name="SettingsVersion">The version of the settings in the backup.</param>
|
||||
/// <param name="CreatedAtUtc">When the backup was written.</param>
|
||||
/// <param name="SourceDataDirectory">The data folder the backup was made from, for whoever wonders where it came from.</param>
|
||||
/// <param name="SourceConfigDirectory">The configuration folder the backup was made from.</param>
|
||||
/// <param name="IncludesDataSourceIndexes">Whether the backup holds the indexes of the data sources. Without them, a restore keeps the indexes there are.</param>
|
||||
/// <param name="Entries">Every file of the backup, the manifest aside.</param>
|
||||
public sealed record DataBackupManifest(
|
||||
int FormatVersion,
|
||||
string AppVersion,
|
||||
string SettingsVersion,
|
||||
DateTimeOffset CreatedAtUtc,
|
||||
string SourceDataDirectory,
|
||||
string SourceConfigDirectory,
|
||||
bool IncludesDataSourceIndexes,
|
||||
IReadOnlyList<DataBackupManifestEntry> Entries);
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// One file of a backup, as the manifest lists it.
|
||||
/// </summary>
|
||||
/// <param name="Name">The name of the entry within the backup.</param>
|
||||
/// <param name="Size">The size of the file in bytes.</param>
|
||||
/// <param name="Sha256">The SHA-256 hash of the file, in lowercase hex digits.</param>
|
||||
public sealed record DataBackupManifestEntry(string Name, long Size, string Sha256);
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// The manifest of a backup, or why it could not be read.
|
||||
/// </summary>
|
||||
/// <param name="Manifest">The manifest, or null when the backup cannot be used.</param>
|
||||
/// <param name="Issue">Why the backup cannot be used, for the user. Empty when it can.</param>
|
||||
public sealed record DataBackupManifestReadResult(DataBackupManifest? Manifest, string Issue);
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What a backup is busy with.
|
||||
/// </summary>
|
||||
public enum DataBackupPhase
|
||||
{
|
||||
/// <summary>
|
||||
/// It collects the settings and pauses the indexing.
|
||||
/// </summary>
|
||||
PREPARING,
|
||||
|
||||
/// <summary>
|
||||
/// It copies the indexes of the data sources, one vector store after the other.
|
||||
/// </summary>
|
||||
COPYING_INDEXES,
|
||||
|
||||
/// <summary>
|
||||
/// It writes the files into the backup.
|
||||
/// </summary>
|
||||
WRITING,
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How far a backup got.
|
||||
/// </summary>
|
||||
/// <param name="Phase">What the backup is busy with.</param>
|
||||
/// <param name="Processed">How many steps of the phase are done: vector stores while the indexes are copied, files while they are written.</param>
|
||||
/// <param name="Total">How many steps the phase has, or zero while that is not known yet.</param>
|
||||
public readonly record struct DataBackupProgress(DataBackupPhase Phase, int Processed, int Total);
|
||||
@@ -0,0 +1,50 @@
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Reads what a backup holds, cf. DataBackupFormat.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Open a backup with ZipArchiveMode.Read and read its entries as streams. ZipArchiveMode.Update would
|
||||
/// buffer an entry in memory, which ends at 2 GB, and a backup may hold larger files.
|
||||
/// </remarks>
|
||||
public static class DataBackupReader
|
||||
{
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataBackupReader).Namespace, nameof(DataBackupReader));
|
||||
|
||||
/// <summary>
|
||||
/// Reads the manifest of a backup, and checks that this version of AI Studio can use the backup.
|
||||
/// </summary>
|
||||
/// <param name="archive">The backup, opened for reading.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The manifest, or why the backup cannot be used.</returns>
|
||||
public static async Task<DataBackupManifestReadResult> ReadManifestAsync(ZipArchive archive, CancellationToken token)
|
||||
{
|
||||
var manifestEntry = archive.GetEntry(DataBackupFormat.MANIFEST_ENTRY_NAME);
|
||||
if (manifestEntry is null)
|
||||
return new(null, TB("This file is no backup of AI Studio, or it was not written completely."));
|
||||
|
||||
DataBackupManifest? manifest;
|
||||
try
|
||||
{
|
||||
await using var manifestStream = manifestEntry.Open();
|
||||
manifest = await JsonSerializer.DeserializeAsync<DataBackupManifest>(manifestStream, DataBackupFormat.JSON_OPTIONS, token);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
manifest = null;
|
||||
}
|
||||
|
||||
if (manifest is null)
|
||||
return new(null, TB("The content list of this backup could not be read."));
|
||||
|
||||
if (manifest.FormatVersion > DataBackupFormat.FORMAT_VERSION)
|
||||
return new(null, TB("This backup was made by a newer version of AI Studio. Please update AI Studio first."));
|
||||
|
||||
return new(manifest, string.Empty);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How a backup ended.
|
||||
/// </summary>
|
||||
/// <param name="Success">Whether the backup was written completely. A cancelled backup counts as no success.</param>
|
||||
/// <param name="Cancelled">Whether the user stopped the backup.</param>
|
||||
/// <param name="ArchivePath">The file of the backup.</param>
|
||||
/// <param name="FileCount">How many files the backup holds.</param>
|
||||
/// <param name="Bytes">How large these files are together, before they were compressed.</param>
|
||||
/// <param name="IncludesDataSourceIndexes">Whether the backup holds the indexes of the data sources.</param>
|
||||
/// <param name="Issue">Why the backup failed, for the user. Empty when it did not fail.</param>
|
||||
public sealed record DataBackupResult(bool Success, bool Cancelled, string ArchivePath, int FileCount, long Bytes, bool IncludesDataSourceIndexes, string Issue);
|
||||
@@ -0,0 +1,45 @@
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// The settings as a backup takes them along.
|
||||
/// </summary>
|
||||
public static class DataBackupSettings
|
||||
{
|
||||
/// <summary>
|
||||
/// Copies the settings, leaving out every object which a configuration plugin manages.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A configuration plugin brings its providers, data sources, chat templates, profiles, and
|
||||
/// policies back on another computer, along with their API keys. Restored from a backup as well,
|
||||
/// they would compete with what the plugin brings, and without the plugin they would stay without
|
||||
/// their keys: secrets never go into a backup. That holds for the configuration plugins of an
|
||||
/// organization and for those the user installed, since a backup takes the latter along.<br/><br/>
|
||||
///
|
||||
/// A setting which a configuration plugin locks stays in the copy, together with the plugin which
|
||||
/// locks it. After a restore, the clean-up following the start of the plugins treats it like the
|
||||
/// setting of a removed plugin, until the plugin locks it again. Without the plugin, the user gets
|
||||
/// back the value they had chosen before.<br/><br/>
|
||||
///
|
||||
/// The copy is a round trip through the format in which the settings are stored, so it shares
|
||||
/// nothing with the settings AI Studio runs with. Like storing the settings, it reads them while
|
||||
/// the rest of the app may still change them.
|
||||
/// </remarks>
|
||||
/// <param name="settings">The settings AI Studio runs with. They stay as they are.</param>
|
||||
/// <returns>The copy for the backup.</returns>
|
||||
public static Data CopyWithoutPluginObjects(Data settings)
|
||||
{
|
||||
var copy = JsonSerializer.Deserialize<Data>(JsonSerializer.Serialize(settings, SettingsManager.JSON_OPTIONS), SettingsManager.JSON_OPTIONS)
|
||||
?? throw new InvalidOperationException("The settings could not be copied.");
|
||||
|
||||
foreach (var descriptor in PluginConfigurationObjectDescriptors.ALL)
|
||||
descriptor.RemovePluginObjects(copy);
|
||||
|
||||
return copy;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Where a backup takes its files from.
|
||||
/// </summary>
|
||||
/// <param name="DataDirectory">The data folder.</param>
|
||||
/// <param name="ConfigDirectory">The configuration folder.</param>
|
||||
/// <param name="SettingsJson">The settings as the backup takes them along, cf. DataBackupSettings. They come from memory rather than from the disk, since the backup leaves out the objects of configuration plugins.</param>
|
||||
/// <param name="IndexCopiesDirectory">The copies of the indexes, in the layout of the data folder, or an empty string when the backup leaves the indexes out. The indexes in the data folder change while AI Studio runs, so the backup never reads them there.</param>
|
||||
/// <param name="OrganizationPluginFolders">The plugins an organization deployed among the user's own ones, cf. DataBackupContents.FindOrganizationPluginsAmongUserPlugins.</param>
|
||||
public sealed record DataBackupSources(string DataDirectory, string ConfigDirectory, string SettingsJson, string IndexCopiesDirectory, IReadOnlyList<string> OrganizationPluginFolders);
|
||||
@@ -0,0 +1,173 @@
|
||||
using System.IO.Compression;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Writes a backup, cf. DataBackupFormat.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every file is streamed into the backup, so a backup of many gigabytes needs no more memory than
|
||||
/// a small one. The caller writes the backup straight into the file the user chose: in the Flatpak,
|
||||
/// the portal grants access to exactly that file, and to nothing next to it.
|
||||
/// </remarks>
|
||||
public static class DataBackupWriter
|
||||
{
|
||||
private const int BUFFER_SIZE = 64 * 1024;
|
||||
|
||||
/// <summary>
|
||||
/// The earliest time a ZIP file can store.
|
||||
/// </summary>
|
||||
private static readonly DateTimeOffset EARLIEST_ZIP_TIME = new(1980, 1, 1, 0, 0, 0, TimeSpan.Zero);
|
||||
|
||||
/// <summary>
|
||||
/// A file to take along, and where it goes in the backup.
|
||||
/// </summary>
|
||||
private sealed record BackupFile(string EntryName, string SourcePath);
|
||||
|
||||
/// <summary>
|
||||
/// Writes a backup into the given stream.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A file which is gone by the time it is written, e.g. a temporary chat AI Studio deleted
|
||||
/// meanwhile, is left out. A file which cannot be read fails the backup, though: a backup which
|
||||
/// quietly lacks something is worse than none.
|
||||
/// </remarks>
|
||||
/// <param name="target">Where to write the backup. It stays open.</param>
|
||||
/// <param name="sources">Where the files come from.</param>
|
||||
/// <param name="manifest">What to note in the manifest. The entries are filled in here.</param>
|
||||
/// <param name="progress">Receives how many files are written.</param>
|
||||
/// <param name="token">Cancels the backup.</param>
|
||||
/// <returns>The manifest as the backup holds it.</returns>
|
||||
public static async Task<DataBackupManifest> WriteAsync(Stream target, DataBackupSources sources, DataBackupManifest manifest, IProgress<DataBackupProgress>? progress, CancellationToken token)
|
||||
{
|
||||
var files = CollectFiles(sources, token);
|
||||
var totalFiles = files.Count + 1;
|
||||
var entries = new List<DataBackupManifestEntry>(totalFiles);
|
||||
var buffer = new byte[BUFFER_SIZE];
|
||||
|
||||
progress?.Report(new(DataBackupPhase.WRITING, 0, totalFiles));
|
||||
using (var archive = new ZipArchive(target, ZipArchiveMode.Create, leaveOpen: true))
|
||||
{
|
||||
var settingsEntry = DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.SETTINGS);
|
||||
await using (var settingsStream = new MemoryStream(Encoding.UTF8.GetBytes(sources.SettingsJson), writable: false))
|
||||
entries.Add(await WriteEntryAsync(archive, DataBackupFormat.EntryName(settingsEntry.Location, settingsEntry.RelativePath), settingsStream, DateTimeOffset.Now, buffer, token));
|
||||
|
||||
progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles));
|
||||
foreach (var file in files)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
FileStream source;
|
||||
try
|
||||
{
|
||||
source = new FileStream(file.SourcePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
}
|
||||
catch (Exception exception) when (exception is FileNotFoundException or DirectoryNotFoundException)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
await using (source)
|
||||
entries.Add(await WriteEntryAsync(archive, file.EntryName, source, File.GetLastWriteTime(file.SourcePath), buffer, token));
|
||||
|
||||
progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles));
|
||||
}
|
||||
|
||||
manifest = manifest with { Entries = entries };
|
||||
await using (var manifestStream = new MemoryStream(JsonSerializer.SerializeToUtf8Bytes(manifest, DataBackupFormat.JSON_OPTIONS), writable: false))
|
||||
await WriteEntryAsync(archive, DataBackupFormat.MANIFEST_ENTRY_NAME, manifestStream, DateTimeOffset.Now, buffer, token);
|
||||
}
|
||||
|
||||
return manifest;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Collects the files of the data folder which belong into a backup, cf. DataBackupContents.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The settings are no part of it: they come from memory. The indexes come from their copies.
|
||||
/// </remarks>
|
||||
private static List<BackupFile> CollectFiles(DataBackupSources sources, CancellationToken token)
|
||||
{
|
||||
var excludedPaths = DataBackupContents.ENTRIES
|
||||
.Where(entry => !entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY)
|
||||
.Select(entry => entry.RelativePath)
|
||||
.Concat(sources.OrganizationPluginFolders)
|
||||
.ToList();
|
||||
|
||||
var files = new List<BackupFile>();
|
||||
foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY))
|
||||
{
|
||||
var root = entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX or DataBackupEntryKind.VECTOR_STORES
|
||||
? sources.IndexCopiesDirectory
|
||||
: sources.DataDirectory;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(root))
|
||||
continue;
|
||||
|
||||
var path = Path.Join(root, entry.RelativePath.Replace(DataBackupContents.SEPARATOR, Path.DirectorySeparatorChar));
|
||||
if (!entry.IsFolder)
|
||||
{
|
||||
if (File.Exists(path))
|
||||
files.Add(new(DataBackupFormat.EntryName(entry.Location, entry.RelativePath), path));
|
||||
|
||||
continue;
|
||||
}
|
||||
|
||||
if (!Directory.Exists(path))
|
||||
continue;
|
||||
|
||||
//
|
||||
// Hidden files count as well, since on Linux and macOS a dot makes a file hidden. Links
|
||||
// are not followed, so nothing outside the folder gets into the backup. A folder which
|
||||
// cannot be read fails the backup instead of leaving a gap:
|
||||
//
|
||||
var options = new EnumerationOptions { RecurseSubdirectories = true, AttributesToSkip = FileAttributes.ReparsePoint, IgnoreInaccessible = false };
|
||||
foreach (var filePath in Directory.EnumerateFiles(path, "*", options))
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var relativePath = Path.GetRelativePath(root, filePath).Replace(Path.DirectorySeparatorChar, DataBackupContents.SEPARATOR);
|
||||
if (excludedPaths.Any(excludedPath => DataBackupContents.Covers(excludedPath, relativePath)))
|
||||
continue;
|
||||
|
||||
files.Add(new(DataBackupFormat.EntryName(DataBackupLocation.DATA_DIRECTORY, relativePath), filePath));
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes one entry, and notes its size and hash while it streams through.
|
||||
/// </summary>
|
||||
private static async Task<DataBackupManifestEntry> WriteEntryAsync(ZipArchive archive, string entryName, Stream source, DateTimeOffset lastWriteTime, byte[] buffer, CancellationToken token)
|
||||
{
|
||||
var compressionLevel = DataBackupFormat.ALREADY_COMPRESSED_EXTENSIONS.Contains(Path.GetExtension(entryName))
|
||||
? CompressionLevel.NoCompression
|
||||
: CompressionLevel.Optimal;
|
||||
|
||||
var entry = archive.CreateEntry(entryName, compressionLevel);
|
||||
|
||||
// The time of the last change tells, e.g., the order of the chats in a workspace:
|
||||
entry.LastWriteTime = lastWriteTime < EARLIEST_ZIP_TIME ? DateTimeOffset.Now : lastWriteTime;
|
||||
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
long size = 0;
|
||||
await using (var entryStream = entry.Open())
|
||||
{
|
||||
int read;
|
||||
while ((read = await source.ReadAsync(buffer, token)) > 0)
|
||||
{
|
||||
hash.AppendData(buffer, 0, read);
|
||||
await entryStream.WriteAsync(buffer.AsMemory(0, read), token);
|
||||
size += read;
|
||||
}
|
||||
}
|
||||
|
||||
return new(entryName, size, Convert.ToHexStringLower(hash.GetHashAndReset()));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// One element which a restore replaces, as the list of the backup contents names it.
|
||||
/// </summary>
|
||||
/// <param name="Location">The folder in which the element lies.</param>
|
||||
/// <param name="RelativePath">The path of the element within that folder, with forward slashes. A folder ends with a slash.</param>
|
||||
/// <param name="Keep">The paths below the element which stay as they are, relative to the same folder: the plugins which come with AI Studio or from the organization, for example. The runtime moves them from the replaced element into the new one.</param>
|
||||
public sealed record DataRestoreElement(DataBackupLocation Location, string RelativePath, IReadOnlyList<string> Keep);
|
||||
@@ -0,0 +1,106 @@
|
||||
using System.Text.Json;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What remains to do after the runtime applied a restore, or after a backup or the preparation of a
|
||||
/// restore stopped halfway: read the report of the runtime, and remove what nobody needs any longer.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Nothing here touches a restore which the runtime has not finished, cf. IsRestoreUnfinished: its
|
||||
/// journal and the data it replaced are the only way back, and the runtime continues at every start.
|
||||
/// </remarks>
|
||||
public static class DataRestoreFollowUp
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether a restore is prepared completely and waits for the next start.
|
||||
/// </summary>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <returns>True when the next start applies a restore.</returns>
|
||||
public static bool IsRestorePending(string dataDirectory) => IsMarkedReady(dataDirectory) && !HasJournal(dataDirectory);
|
||||
|
||||
/// <summary>
|
||||
/// Whether the runtime has not finished a restore: it failed to move a failed restore back, and
|
||||
/// tries again at the next start.
|
||||
/// </summary>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <returns>True while the runtime still needs the pending and the previous directories.</returns>
|
||||
public static bool IsRestoreUnfinished(string dataDirectory) => IsMarkedReady(dataDirectory) && HasJournal(dataDirectory);
|
||||
|
||||
/// <summary>
|
||||
/// Reads how the runtime applied a restore at this start. Throws a JsonException when the report
|
||||
/// cannot be read.
|
||||
/// </summary>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <param name="token">Cancels reading.</param>
|
||||
/// <returns>The report, or null when the runtime applied no restore.</returns>
|
||||
public static async Task<DataRestoreReport?> ReadReportAsync(string dataDirectory, CancellationToken token)
|
||||
{
|
||||
var reportPath = Path.Join(dataDirectory, DataRestoreFormat.RESULT_FILE_NAME);
|
||||
if (!File.Exists(reportPath))
|
||||
return null;
|
||||
|
||||
await using var reportStream = File.OpenRead(reportPath);
|
||||
return await JsonSerializer.DeserializeAsync<DataRestoreReport>(reportStream, DataBackupFormat.JSON_OPTIONS, token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the report of the runtime, once the user saw it.
|
||||
/// </summary>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
public static void RemoveReport(string dataDirectory) => File.Delete(Path.Join(dataDirectory, DataRestoreFormat.RESULT_FILE_NAME));
|
||||
|
||||
/// <summary>
|
||||
/// Removes what a backup or the preparation of a restore left behind when it stopped halfway, e.g.,
|
||||
/// because AI Studio crashed. Call it only while neither runs.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A pending directory with READY_FILE_NAME stays: the restore waits for the next start, or the
|
||||
/// runtime has not finished it.
|
||||
/// </remarks>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <param name="configDirectory">The configuration folder.</param>
|
||||
public static void RemoveLeftovers(string dataDirectory, string configDirectory)
|
||||
{
|
||||
DeleteDirectory(DataRestoreFormat.DirectoryIn(dataDirectory, DataBackupFormat.STAGING_DIRECTORY));
|
||||
if (IsMarkedReady(dataDirectory))
|
||||
return;
|
||||
|
||||
DeleteDirectory(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PENDING_DIRECTORY));
|
||||
DeleteDirectory(DataRestoreFormat.DirectoryIn(configDirectory, DataRestoreFormat.PENDING_DIRECTORY));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the data which a restore replaced. Call it once the restored settings loaded: until
|
||||
/// then, these data are the way back.
|
||||
/// </summary>
|
||||
/// <param name="dataDirectory">The data folder.</param>
|
||||
/// <param name="configDirectory">The configuration folder.</param>
|
||||
/// <returns>Whether there were such data.</returns>
|
||||
public static bool RemoveReplacedData(string dataDirectory, string configDirectory)
|
||||
{
|
||||
if (IsMarkedReady(dataDirectory))
|
||||
return false;
|
||||
|
||||
var removedData = DeleteDirectory(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PREVIOUS_DIRECTORY));
|
||||
var removedConfiguration = DeleteDirectory(DataRestoreFormat.DirectoryIn(configDirectory, DataRestoreFormat.PREVIOUS_DIRECTORY));
|
||||
return removedData || removedConfiguration;
|
||||
}
|
||||
|
||||
private static bool IsMarkedReady(string dataDirectory) => File.Exists(Path.Join(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PENDING_DIRECTORY), DataRestoreFormat.READY_FILE_NAME));
|
||||
|
||||
private static bool HasJournal(string dataDirectory) => File.Exists(Path.Join(DataRestoreFormat.DirectoryIn(dataDirectory, DataRestoreFormat.PENDING_DIRECTORY), DataRestoreFormat.JOURNAL_FILE_NAME));
|
||||
|
||||
/// <summary>
|
||||
/// Deletes a directory with everything in it.
|
||||
/// </summary>
|
||||
/// <returns>Whether the directory existed.</returns>
|
||||
private static bool DeleteDirectory(string path)
|
||||
{
|
||||
if (!Directory.Exists(path))
|
||||
return false;
|
||||
|
||||
Directory.Delete(path, true);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How a prepared restore waits for the next start, which applies it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This is the contract between the app, which prepares a restore, and the runtime, which applies it
|
||||
/// before the app starts again, when no file is in use:
|
||||
/// <list type="bullet">
|
||||
/// <item>Below the data folder and below the configuration folder, PENDING_DIRECTORY holds what the
|
||||
/// backup brings for that folder, in the layout of that folder. Staging lies on the same disk as its
|
||||
/// target, so the runtime can move every element into place instead of copying it.</item>
|
||||
/// <item>The plan, PLAN_FILE_NAME in the pending directory of the data folder, names the elements to
|
||||
/// replace, cf. DataRestorePlan.</item>
|
||||
/// <item>READY_FILE_NAME in the same directory comes last. Without it, the restore was not prepared
|
||||
/// completely, and the runtime leaves everything as it is.</item>
|
||||
/// </list>
|
||||
/// What the runtime adds, cf. runtime/src/data_restore.rs:
|
||||
/// <list type="bullet">
|
||||
/// <item>PREVIOUS_DIRECTORY, below both folders, keeps what the restore replaced. The app removes it
|
||||
/// once it loaded the restored settings.</item>
|
||||
/// <item>JOURNAL_FILE_NAME in the pending directory of the data folder lists every move. While it lies
|
||||
/// next to READY_FILE_NAME, the runtime moves everything back at the next start, so the app never
|
||||
/// removes a pending directory which holds READY_FILE_NAME.</item>
|
||||
/// <item>RESULT_FILE_NAME in the data folder tells the app how the restore ended, cf. DataRestoreReport.</item>
|
||||
/// </list>
|
||||
/// Keep in sync with the runtime, which applies the restore.
|
||||
/// </remarks>
|
||||
public static class DataRestoreFormat
|
||||
{
|
||||
/// <summary>
|
||||
/// The version of the plan this version of AI Studio writes.
|
||||
/// </summary>
|
||||
public const int PLAN_FORMAT_VERSION = 1;
|
||||
|
||||
/// <summary>
|
||||
/// The directory, below the data folder and below the configuration folder, which holds what a
|
||||
/// prepared restore brings for that folder.
|
||||
/// </summary>
|
||||
public const string PENDING_DIRECTORY = ".restore-pending/";
|
||||
|
||||
/// <summary>
|
||||
/// The plan of the restore, in the pending directory of the data folder.
|
||||
/// </summary>
|
||||
public const string PLAN_FILE_NAME = "restore.json";
|
||||
|
||||
/// <summary>
|
||||
/// Marks a restore as prepared completely, in the pending directory of the data folder.
|
||||
/// </summary>
|
||||
public const string READY_FILE_NAME = "READY";
|
||||
|
||||
/// <summary>
|
||||
/// The directory, below the data folder and below the configuration folder, which keeps what the
|
||||
/// runtime replaced, in the layout of that folder.
|
||||
/// </summary>
|
||||
public const string PREVIOUS_DIRECTORY = ".restore-previous/";
|
||||
|
||||
/// <summary>
|
||||
/// How the runtime applied a restore, in the data folder.
|
||||
/// </summary>
|
||||
public const string RESULT_FILE_NAME = ".restore-result.json";
|
||||
|
||||
/// <summary>
|
||||
/// The journal of the runtime, in the pending directory of the data folder. Next to READY_FILE_NAME,
|
||||
/// it tells that the runtime has not finished a restore: it moves it back at the next start.
|
||||
/// </summary>
|
||||
public const string JOURNAL_FILE_NAME = "journal.json";
|
||||
|
||||
/// <summary>
|
||||
/// The path of one of the directories above, below the given folder.
|
||||
/// </summary>
|
||||
/// <param name="folder">The data or the configuration folder.</param>
|
||||
/// <param name="directory">PENDING_DIRECTORY or PREVIOUS_DIRECTORY.</param>
|
||||
/// <returns>The path of the directory.</returns>
|
||||
public static string DirectoryIn(string folder, string directory) => Path.Join(folder, directory.TrimEnd(DataBackupContents.SEPARATOR));
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// One thing the user still has to do after a restore.
|
||||
/// </summary>
|
||||
/// <param name="Kind">What to do.</param>
|
||||
/// <param name="Name">What it concerns: the name of a provider, a data source, or a tool, or a path.</param>
|
||||
/// <param name="Setting">For a tool: the label of the setting which lacks its value; otherwise empty.</param>
|
||||
public sealed record DataRestoreOpenPoint(DataRestoreOpenPointKind Kind, string Name, string Setting = "");
|
||||
@@ -0,0 +1,59 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What the user still has to do after a restore, since a backup holds no secrets and another computer
|
||||
/// may lack the files the settings name.
|
||||
/// </summary>
|
||||
public enum DataRestoreOpenPointKind
|
||||
{
|
||||
/// <summary>
|
||||
/// A provider of the user has no API key on this computer.
|
||||
/// </summary>
|
||||
PROVIDER_WITHOUT_API_KEY,
|
||||
|
||||
/// <summary>
|
||||
/// An embedding provider of the user has no API key on this computer.
|
||||
/// </summary>
|
||||
EMBEDDING_PROVIDER_WITHOUT_API_KEY,
|
||||
|
||||
/// <summary>
|
||||
/// A transcription provider of the user has no API key on this computer.
|
||||
/// </summary>
|
||||
TRANSCRIPTION_PROVIDER_WITHOUT_API_KEY,
|
||||
|
||||
/// <summary>
|
||||
/// An ERI data source of the user has no password or token on this computer.
|
||||
/// </summary>
|
||||
DATA_SOURCE_WITHOUT_SECRET,
|
||||
|
||||
/// <summary>
|
||||
/// A mailbox of the user has no password on this computer.
|
||||
/// </summary>
|
||||
MAILBOX_WITHOUT_PASSWORD,
|
||||
|
||||
/// <summary>
|
||||
/// A tool which the user set up has no value for a secret setting on this computer, e.g., for
|
||||
/// the API key of a search service.
|
||||
/// </summary>
|
||||
TOOL_WITHOUT_SECRET,
|
||||
|
||||
/// <summary>
|
||||
/// A local data source names a file or folder which does not exist on this computer.
|
||||
/// </summary>
|
||||
DATA_SOURCE_NOT_FOUND,
|
||||
|
||||
/// <summary>
|
||||
/// The batch processing names a file or folder which does not exist on this computer.
|
||||
/// </summary>
|
||||
BATCH_PROCESSING_PATH_NOT_FOUND,
|
||||
|
||||
/// <summary>
|
||||
/// The file with custom root certificates does not exist on this computer.
|
||||
/// </summary>
|
||||
CERTIFICATE_BUNDLE_NOT_FOUND,
|
||||
|
||||
/// <summary>
|
||||
/// The ERI server assistant writes a server into a folder which does not exist on this computer.
|
||||
/// </summary>
|
||||
ERI_SERVER_DIRECTORY_NOT_FOUND,
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.ERIClient.DataModel;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Finds what the user still has to do after a restore.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A backup holds no secrets, so the providers, data sources, mailboxes, and tools of the user lack
|
||||
/// them on another computer. Those of the organization do not: their configuration plugin brings them
|
||||
/// along with their secrets. The files and folders which the settings name may be missing, too, wherever
|
||||
/// they come from.
|
||||
/// </remarks>
|
||||
public static class DataRestoreOpenPoints
|
||||
{
|
||||
/// <summary>
|
||||
/// Finds what the user still has to do after a restore.
|
||||
/// </summary>
|
||||
/// <param name="settings">The restored settings.</param>
|
||||
/// <param name="secrets">Tells which secrets this computer holds.</param>
|
||||
/// <param name="toolSecrets">The secret settings of the tools which the user set up. Only the tools know which of their settings are secret, cf. DataRestoreService.</param>
|
||||
/// <returns>The open points, in the order of the settings.</returns>
|
||||
public static async Task<IReadOnlyList<DataRestoreOpenPoint>> CollectAsync(Data settings, IStoredSecrets secrets, IReadOnlyList<DataRestoreToolSecret> toolSecrets)
|
||||
{
|
||||
var openPoints = new List<DataRestoreOpenPoint>();
|
||||
foreach (var provider in settings.Providers.Where(provider => !provider.IsEnterpriseConfiguration && provider.UsedLLMProvider.IsAPIKeyNeeded(provider.Host)))
|
||||
if (!await secrets.HasAPIKeyAsync(provider, SecretStoreType.LLM_PROVIDER))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.PROVIDER_WITHOUT_API_KEY, provider.InstanceName));
|
||||
|
||||
foreach (var provider in settings.EmbeddingProviders.Where(provider => !provider.IsEnterpriseConfiguration && provider.UsedLLMProvider.IsAPIKeyNeeded(provider.Host)))
|
||||
if (!await secrets.HasAPIKeyAsync(provider, SecretStoreType.EMBEDDING_PROVIDER))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.EMBEDDING_PROVIDER_WITHOUT_API_KEY, provider.Name));
|
||||
|
||||
foreach (var provider in settings.TranscriptionProviders.Where(provider => !provider.IsEnterpriseConfiguration && provider.UsedLLMProvider.IsAPIKeyNeeded(provider.Host)))
|
||||
if (!await secrets.HasAPIKeyAsync(provider, SecretStoreType.TRANSCRIPTION_PROVIDER))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.TRANSCRIPTION_PROVIDER_WITHOUT_API_KEY, provider.Name));
|
||||
|
||||
foreach (var dataSource in settings.DataSources)
|
||||
{
|
||||
switch (dataSource)
|
||||
{
|
||||
case DataSourceLocalDirectory directory when !Directory.Exists(directory.Path):
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.DATA_SOURCE_NOT_FOUND, directory.Name));
|
||||
break;
|
||||
|
||||
case DataSourceLocalFile file when !File.Exists(file.FilePath):
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.DATA_SOURCE_NOT_FOUND, file.Name));
|
||||
break;
|
||||
|
||||
case IERIDataSource { IsEnterpriseConfiguration: false, AuthMethod: AuthMethod.TOKEN or AuthMethod.USERNAME_PASSWORD } eriDataSource:
|
||||
if (!await secrets.HasSecretAsync(eriDataSource, SecretStoreType.DATA_SOURCE))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.DATA_SOURCE_WITHOUT_SECRET, eriDataSource.Name));
|
||||
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var mailbox in settings.Mailboxes.Where(mailbox => mailbox is { IsEnterpriseConfiguration: false, AuthMethod: MailboxAuthMethod.PASSWORD }))
|
||||
if (!await secrets.HasSecretAsync(mailbox, SecretStoreType.DATA_SOURCE))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.MAILBOX_WITHOUT_PASSWORD, mailbox.Name));
|
||||
|
||||
foreach (var toolSecret in toolSecrets)
|
||||
if (!await secrets.HasSecretAsync(new ToolSettingsSecretId(toolSecret.ToolId, toolSecret.FieldName), SecretStoreType.TOOL_SETTINGS))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.TOOL_WITHOUT_SECRET, toolSecret.ToolName, toolSecret.SettingLabel));
|
||||
|
||||
var batchProcessing = settings.BatchProcessing;
|
||||
foreach (var folder in new[] { batchProcessing.InputDirectory, batchProcessing.OutputDirectory })
|
||||
if (!string.IsNullOrWhiteSpace(folder) && !Directory.Exists(folder))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.BATCH_PROCESSING_PATH_NOT_FOUND, folder));
|
||||
|
||||
if (!string.IsNullOrWhiteSpace(batchProcessing.PromptFilePath) && !File.Exists(batchProcessing.PromptFilePath))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.BATCH_PROCESSING_PATH_NOT_FOUND, batchProcessing.PromptFilePath));
|
||||
|
||||
var certificateBundlePath = settings.App.ExternalHttpCustomRootCertificateBundlePath;
|
||||
if (settings.App.ExternalHttpCustomRootCertificatesEnabled && !string.IsNullOrWhiteSpace(certificateBundlePath) && !File.Exists(certificateBundlePath))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.CERTIFICATE_BUNDLE_NOT_FOUND, certificateBundlePath));
|
||||
|
||||
foreach (var server in settings.ERI.ERIServers.Where(server => server.WriteToFilesystem && !string.IsNullOrWhiteSpace(server.BaseDirectory) && !Directory.Exists(server.BaseDirectory)))
|
||||
openPoints.Add(new(DataRestoreOpenPointKind.ERI_SERVER_DIRECTORY_NOT_FOUND, server.ServerName));
|
||||
|
||||
return openPoints;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How the runtime applied a restore, cf. DataRestoreReport.
|
||||
/// </summary>
|
||||
public enum DataRestoreOutcome
|
||||
{
|
||||
/// <summary>
|
||||
/// The restored data are in place.
|
||||
/// </summary>
|
||||
APPLIED,
|
||||
|
||||
/// <summary>
|
||||
/// The runtime did not start the restore, since the plan or the folders did not allow it. All data
|
||||
/// stayed as they were.
|
||||
/// </summary>
|
||||
REFUSED,
|
||||
|
||||
/// <summary>
|
||||
/// The restore failed, and the runtime moved all data back.
|
||||
/// </summary>
|
||||
ROLLED_BACK,
|
||||
|
||||
/// <summary>
|
||||
/// The restore failed, and moving the data back failed as well. The runtime tries again at every
|
||||
/// start.
|
||||
/// </summary>
|
||||
ROLLBACK_FAILED,
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// What the runtime replaces at the next start, cf. DataRestoreFormat.
|
||||
/// </summary>
|
||||
/// <param name="FormatVersion">The version of the plan, cf. DataRestoreFormat.PLAN_FORMAT_VERSION.</param>
|
||||
/// <param name="PreparedAtUtc">When the restore was prepared.</param>
|
||||
/// <param name="BackupCreatedAtUtc">When the backup was made.</param>
|
||||
/// <param name="BackupAppVersion">The version of AI Studio which made the backup.</param>
|
||||
/// <param name="Elements">The elements to replace. An element the backup holds nothing for stays as it is.</param>
|
||||
public sealed record DataRestorePlan(int FormatVersion, DateTimeOffset PreparedAtUtc, DateTimeOffset BackupCreatedAtUtc, string BackupAppVersion, IReadOnlyList<DataRestoreElement> Elements);
|
||||
@@ -0,0 +1,239 @@
|
||||
using System.IO.Compression;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
using SettingsVersion = AIStudio.Settings.Version;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Prepares a restore: unpacks a backup next to the data and configuration folders, so that the next
|
||||
/// start can put it in place, cf. DataRestoreFormat.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Nothing in use changes here. While AI Studio runs, its files are open, and on Windows, they cannot be
|
||||
/// replaced at all. The runtime replaces them at the next start, before the app opens any of them, and
|
||||
/// rolls back when that fails.
|
||||
/// </remarks>
|
||||
public static class DataRestorePreparer
|
||||
{
|
||||
private const int BUFFER_SIZE = 64 * 1024;
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataRestorePreparer).Namespace, nameof(DataRestorePreparer));
|
||||
|
||||
/// <summary>
|
||||
/// A file of the backup to unpack, and where it goes.
|
||||
/// </summary>
|
||||
private sealed record PendingFile(ZipArchiveEntry Entry, DataBackupManifestEntry Listed, string DestinationPath);
|
||||
|
||||
/// <summary>
|
||||
/// Unpacks a backup for the next start.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every file has to match the size and the hash the manifest lists for it: a restore replaces all
|
||||
/// data, so a damaged backup must not get that far. A file the manifest does not list, or one
|
||||
/// which belongs to no element the restore replaces, stays in the backup. A restore prepared
|
||||
/// before is discarded first; one which fails or is cancelled leaves nothing behind.
|
||||
/// </remarks>
|
||||
/// <param name="archivePath">The backup.</param>
|
||||
/// <param name="target">Where the restore goes.</param>
|
||||
/// <param name="progress">Receives how many files are unpacked.</param>
|
||||
/// <param name="token">Cancels the preparation.</param>
|
||||
/// <returns>Whether the restore is prepared, or why not.</returns>
|
||||
public static async Task<DataRestoreResult> PrepareAsync(string archivePath, DataRestoreTarget target, IProgress<DataRestoreProgress>? progress, CancellationToken token)
|
||||
{
|
||||
Discard(target);
|
||||
try
|
||||
{
|
||||
using var archive = ZipFile.OpenRead(archivePath);
|
||||
var manifestResult = await DataBackupReader.ReadManifestAsync(archive, token);
|
||||
if (manifestResult.Manifest is not { } manifest)
|
||||
return new(false, false, manifestResult.Issue);
|
||||
|
||||
var versionIssue = CheckVersions(manifest, target);
|
||||
if (!string.IsNullOrWhiteSpace(versionIssue))
|
||||
return new(false, false, versionIssue);
|
||||
|
||||
var elements = DetermineElements(manifest, target);
|
||||
var files = DetermineFiles(archive, manifest, elements, target);
|
||||
progress?.Report(new(0, files.Count));
|
||||
|
||||
var buffer = new byte[BUFFER_SIZE];
|
||||
for (var index = 0; index < files.Count; index++)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
if (!await UnpackAsync(files[index], buffer, token))
|
||||
{
|
||||
Discard(target);
|
||||
return new(false, false, string.Format(TB("The backup is damaged: the file '{0}' does not match its content list."), files[index].Listed.Name));
|
||||
}
|
||||
|
||||
progress?.Report(new(index + 1, files.Count));
|
||||
}
|
||||
|
||||
var pendingDirectory = PendingDirectory(target.DataDirectory);
|
||||
Directory.CreateDirectory(pendingDirectory);
|
||||
|
||||
var plan = new DataRestorePlan(DataRestoreFormat.PLAN_FORMAT_VERSION, DateTimeOffset.UtcNow, manifest.CreatedAtUtc, manifest.AppVersion, elements);
|
||||
await File.WriteAllBytesAsync(Path.Join(pendingDirectory, DataRestoreFormat.PLAN_FILE_NAME), JsonSerializer.SerializeToUtf8Bytes(plan, DataBackupFormat.JSON_OPTIONS), token);
|
||||
|
||||
// Last of all: from now on, the next start applies the restore.
|
||||
await File.WriteAllTextAsync(Path.Join(pendingDirectory, DataRestoreFormat.READY_FILE_NAME), plan.PreparedAtUtc.ToString("O"), token);
|
||||
return new(true, false, string.Empty);
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
Discard(target);
|
||||
return new(false, false, TB("This file is no backup of AI Studio, or it was not written completely."));
|
||||
}
|
||||
catch
|
||||
{
|
||||
Discard(target);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes a prepared restore, complete or not, so that the next start leaves everything as it is.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A restore which the runtime has not finished stays: its journal is the only way back to the data
|
||||
/// it replaced, cf. DataRestoreFollowUp.IsRestoreUnfinished.
|
||||
/// </remarks>
|
||||
/// <param name="target">Where the restore was to go.</param>
|
||||
public static void Discard(DataRestoreTarget target)
|
||||
{
|
||||
if (DataRestoreFollowUp.IsRestoreUnfinished(target.DataDirectory))
|
||||
throw new InvalidOperationException("The runtime has not finished the last restore yet, so it cannot be discarded.");
|
||||
|
||||
// First of all: should removing the rest fail halfway, the next start must not apply what is left.
|
||||
var readyPath = Path.Join(PendingDirectory(target.DataDirectory), DataRestoreFormat.READY_FILE_NAME);
|
||||
if (File.Exists(readyPath))
|
||||
File.Delete(readyPath);
|
||||
|
||||
foreach (var pendingDirectory in new[] { PendingDirectory(target.DataDirectory), PendingDirectory(target.ConfigDirectory) })
|
||||
if (Directory.Exists(pendingDirectory))
|
||||
Directory.Delete(pendingDirectory, true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a restore is prepared completely and waits for the next start.
|
||||
/// </summary>
|
||||
/// <param name="target">Where the restore goes.</param>
|
||||
/// <returns>True when the next start applies a restore.</returns>
|
||||
public static bool IsPrepared(DataRestoreTarget target) => DataRestoreFollowUp.IsRestorePending(target.DataDirectory);
|
||||
|
||||
private static string PendingDirectory(string directory) => DataRestoreFormat.DirectoryIn(directory, DataRestoreFormat.PENDING_DIRECTORY);
|
||||
|
||||
/// <summary>
|
||||
/// Refuses a backup of a later version of AI Studio. Its settings or its index may have a form
|
||||
/// this version cannot read, and it would refuse to write them, or break them.
|
||||
/// </summary>
|
||||
/// <returns>Why the backup cannot be restored, or an empty string.</returns>
|
||||
private static string CheckVersions(DataBackupManifest manifest, DataRestoreTarget target)
|
||||
{
|
||||
var settingsAreNewer = !Enum.TryParse<SettingsVersion>(manifest.SettingsVersion, out var settingsVersion) || settingsVersion > target.SettingsVersion;
|
||||
var appIsNewer = Version.TryParse(manifest.AppVersion, out var backupAppVersion) && Version.TryParse(target.AppVersion, out var appVersion) && backupAppVersion > appVersion;
|
||||
return settingsAreNewer || appIsNewer
|
||||
? TB("This backup was made by a newer version of AI Studio. Please update AI Studio first.")
|
||||
: string.Empty;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the elements the restore replaces: those of the list of the backup contents which the
|
||||
/// backup holds anything for.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// An element the backup holds nothing for stays as it is, since nothing tells whether it was empty
|
||||
/// or whether the version of AI Studio which made the backup knew it at all. The indexes are
|
||||
/// replaced only when the backup holds both of them, cf. DataBackupManifest.IncludesDataSourceIndexes.
|
||||
/// </remarks>
|
||||
private static List<DataRestoreElement> DetermineElements(DataBackupManifest manifest, DataRestoreTarget target)
|
||||
{
|
||||
var elements = new List<DataRestoreElement>();
|
||||
foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded))
|
||||
{
|
||||
if (entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX or DataBackupEntryKind.VECTOR_STORES && !manifest.IncludesDataSourceIndexes)
|
||||
continue;
|
||||
|
||||
var entryName = DataBackupFormat.EntryName(entry.Location, entry.RelativePath);
|
||||
if (!manifest.Entries.Any(listed => DataBackupContents.Covers(entryName, listed.Name)))
|
||||
continue;
|
||||
|
||||
var keep = DataBackupContents.ENTRIES
|
||||
.Where(excluded => !excluded.IsIncluded && excluded.Location == entry.Location && entry.IsFolder && DataBackupContents.Covers(entry.RelativePath, excluded.RelativePath))
|
||||
.Select(excluded => excluded.RelativePath)
|
||||
.Concat(entry.Kind is DataBackupEntryKind.PLUGINS ? target.OrganizationPluginFolders : [])
|
||||
.ToList();
|
||||
|
||||
elements.Add(new(entry.Location, entry.RelativePath, keep));
|
||||
}
|
||||
|
||||
return elements;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the files to unpack: those the manifest lists, which belong to an element the restore
|
||||
/// replaces, and which the element does not keep.
|
||||
/// </summary>
|
||||
private static List<PendingFile> DetermineFiles(ZipArchive archive, DataBackupManifest manifest, IReadOnlyList<DataRestoreElement> elements, DataRestoreTarget target)
|
||||
{
|
||||
var listedEntries = manifest.Entries.ToDictionary(listed => listed.Name, StringComparer.Ordinal);
|
||||
var files = new List<PendingFile>();
|
||||
foreach (var entry in archive.Entries)
|
||||
{
|
||||
var entryName = ZipEntryNames.Normalize(entry.FullName);
|
||||
if (!listedEntries.TryGetValue(entryName, out var listed))
|
||||
continue;
|
||||
|
||||
foreach (var element in elements)
|
||||
{
|
||||
var root = DataBackupFormat.EntryName(element.Location, string.Empty);
|
||||
if (!entryName.StartsWith(root, StringComparison.Ordinal))
|
||||
continue;
|
||||
|
||||
var relativePath = entryName[root.Length..];
|
||||
if (!DataBackupContents.Covers(element.RelativePath, relativePath) || element.Keep.Any(kept => DataBackupContents.Covers(kept, relativePath)))
|
||||
continue;
|
||||
|
||||
var directory = element.Location is DataBackupLocation.CONFIG_DIRECTORY ? target.ConfigDirectory : target.DataDirectory;
|
||||
files.Add(new(entry, listed, ZipEntryNames.GetDestinationPath(PendingDirectory(directory), relativePath)));
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Unpacks one file, and checks its size and hash while it streams through.
|
||||
/// </summary>
|
||||
/// <returns>False when the file does not match the manifest.</returns>
|
||||
private static async Task<bool> UnpackAsync(PendingFile file, byte[] buffer, CancellationToken token)
|
||||
{
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(file.DestinationPath)!);
|
||||
|
||||
using var hash = IncrementalHash.CreateHash(HashAlgorithmName.SHA256);
|
||||
long size = 0;
|
||||
await using (var source = file.Entry.Open())
|
||||
await using (var destination = new FileStream(file.DestinationPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
{
|
||||
int read;
|
||||
while ((read = await source.ReadAsync(buffer, token)) > 0)
|
||||
{
|
||||
hash.AppendData(buffer, 0, read);
|
||||
await destination.WriteAsync(buffer.AsMemory(0, read), token);
|
||||
size += read;
|
||||
}
|
||||
}
|
||||
|
||||
if (size != file.Listed.Size || Convert.ToHexStringLower(hash.GetHashAndReset()) != file.Listed.Sha256)
|
||||
return false;
|
||||
|
||||
// The time of the last change tells, e.g., the order of the chats in a workspace:
|
||||
File.SetLastWriteTime(file.DestinationPath, file.Entry.LastWriteTime.LocalDateTime);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How far the preparation of a restore got.
|
||||
/// </summary>
|
||||
/// <param name="Processed">How many files of the backup are unpacked.</param>
|
||||
/// <param name="Total">How many files there are to unpack, or zero while that is not known yet.</param>
|
||||
public readonly record struct DataRestoreProgress(int Processed, int Total);
|
||||
@@ -0,0 +1,15 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How the runtime applied a restore at this start, as it writes it to DataRestoreFormat.RESULT_FILE_NAME.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The points in time stay text: the runtime leaves them empty when it could not read the plan.
|
||||
/// </remarks>
|
||||
/// <param name="Outcome">How the restore ended.</param>
|
||||
/// <param name="Detail">What happened, in English, for the log and for the support.</param>
|
||||
/// <param name="PreparedAtUtc">When the restore was prepared, or an empty string.</param>
|
||||
/// <param name="BackupCreatedAtUtc">When the backup was made, or an empty string.</param>
|
||||
/// <param name="BackupAppVersion">The version of AI Studio which made the backup, or an empty string.</param>
|
||||
/// <param name="FinishedAtUtc">When the runtime finished.</param>
|
||||
public sealed record DataRestoreReport(DataRestoreOutcome Outcome, string Detail, string PreparedAtUtc, string BackupCreatedAtUtc, string BackupAppVersion, string FinishedAtUtc);
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// How the preparation of a restore ended.
|
||||
/// </summary>
|
||||
/// <param name="Success">Whether the restore is prepared and waits for the next start.</param>
|
||||
/// <param name="Cancelled">Whether the user stopped the preparation.</param>
|
||||
/// <param name="Issue">Why the restore cannot be prepared, for the user. Empty when it is prepared.</param>
|
||||
public sealed record DataRestoreResult(bool Success, bool Cancelled, string Issue);
|
||||
@@ -0,0 +1,13 @@
|
||||
using SettingsVersion = AIStudio.Settings.Version;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Where a restore goes, and what it has to fit to.
|
||||
/// </summary>
|
||||
/// <param name="DataDirectory">The data folder.</param>
|
||||
/// <param name="ConfigDirectory">The configuration folder.</param>
|
||||
/// <param name="AppVersion">The version of this AI Studio. A backup of a later version is refused.</param>
|
||||
/// <param name="SettingsVersion">The version of the settings this AI Studio writes. A backup with later settings is refused.</param>
|
||||
/// <param name="OrganizationPluginFolders">The plugins an organization deployed among the user's own ones, cf. DataBackupContents.FindOrganizationPluginsAmongUserPlugins. They stay as they are.</param>
|
||||
public sealed record DataRestoreTarget(string DataDirectory, string ConfigDirectory, string AppVersion, SettingsVersion SettingsVersion, IReadOnlyList<string> OrganizationPluginFolders);
|
||||
@@ -0,0 +1,10 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// A secret setting of a tool which the user set up, and which may lack its value after a restore.
|
||||
/// </summary>
|
||||
/// <param name="ToolId">The tool.</param>
|
||||
/// <param name="FieldName">The setting, as the schema of the tool names it.</param>
|
||||
/// <param name="ToolName">The name of the tool, as the user reads it.</param>
|
||||
/// <param name="SettingLabel">The label of the setting, as the user reads it.</param>
|
||||
public sealed record DataRestoreToolSecret(string ToolId, string FieldName, string ToolName, string SettingLabel);
|
||||
@@ -0,0 +1,23 @@
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Tells whether the OS keyring of this computer holds a secret.
|
||||
/// </summary>
|
||||
public interface IStoredSecrets
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the API key of a provider is stored.
|
||||
/// </summary>
|
||||
/// <param name="secretId">The provider.</param>
|
||||
/// <param name="storeType">The kind of provider.</param>
|
||||
/// <returns>True when the key is stored.</returns>
|
||||
public Task<bool> HasAPIKeyAsync(ISecretId secretId, SecretStoreType storeType);
|
||||
|
||||
/// <summary>
|
||||
/// Whether another secret is stored, e.g., the password of a data source.
|
||||
/// </summary>
|
||||
/// <param name="secretId">What the secret belongs to.</param>
|
||||
/// <param name="storeType">The kind of secret.</param>
|
||||
/// <returns>True when the secret is stored.</returns>
|
||||
public Task<bool> HasSecretAsync(ISecretId secretId, SecretStoreType storeType);
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
namespace AIStudio.Tools.DataBackup;
|
||||
|
||||
/// <summary>
|
||||
/// Asks the runtime whether the OS keyring holds a secret. It only tries, so a missing secret logs no error.
|
||||
/// </summary>
|
||||
/// <param name="rustService">The runtime.</param>
|
||||
public sealed class RustStoredSecrets(RustService rustService) : IStoredSecrets
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> HasAPIKeyAsync(ISecretId secretId, SecretStoreType storeType) => (await rustService.GetAPIKey(secretId, storeType, isTrying: true)).Success;
|
||||
|
||||
/// <inheritdoc />
|
||||
public async Task<bool> HasSecretAsync(ISecretId secretId, SecretStoreType storeType) => (await rustService.GetSecret(secretId, storeType, isTrying: true)).Success;
|
||||
}
|
||||
@@ -0,0 +1,69 @@
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
using AIStudio.Settings;
|
||||
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Stores a path below the data directory relative to it, and every other path as it is.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// AI Studio keeps files of its own below the data directory: the transcripts of a chat, the copies of
|
||||
/// the tokenizers, and more. A stored path to such a file outlives the computer it was written on. A
|
||||
/// backup brings it to another computer, where the user name, the OS, or the data directory itself may
|
||||
/// differ. Relative to the data directory, the path still names the same file there. In memory, the
|
||||
/// path stays absolute, so the code which works with it never notices.<br/><br/>
|
||||
///
|
||||
/// A relative path is written with forward slashes, which every OS reads. A path to a file of the user
|
||||
/// stays absolute: it lies outside of the data directory, and a backup does not bring that file along.
|
||||
/// A path written before AI Studio stored such paths relative is absolute as well, and is read as it
|
||||
/// is. While the data directory is not known yet, every path stays as it is.
|
||||
/// </remarks>
|
||||
public sealed class DataDirectoryRelativePathConverter : JsonConverter<string>
|
||||
{
|
||||
private const char STORED_SEPARATOR = '/';
|
||||
|
||||
#region Overrides of JsonConverter<string>
|
||||
|
||||
/// <inheritdoc />
|
||||
public override string Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => ToAbsolute(reader.GetString() ?? string.Empty, SettingsManager.DataDirectory);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override void Write(Utf8JsonWriter writer, string value, JsonSerializerOptions options) => writer.WriteStringValue(ToStored(value, SettingsManager.DataDirectory));
|
||||
|
||||
#endregion
|
||||
|
||||
/// <summary>
|
||||
/// Turns a path into the form in which it is stored.
|
||||
/// </summary>
|
||||
/// <param name="path">The absolute path, as AI Studio works with it.</param>
|
||||
/// <param name="dataDirectory">The data directory, or null while it is not known yet.</param>
|
||||
/// <returns>The path relative to the data directory when it lies below it, otherwise the path as it is.</returns>
|
||||
internal static string ToStored(string path, string? dataDirectory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path) || string.IsNullOrWhiteSpace(dataDirectory) || !Path.IsPathFullyQualified(path) || !PathTools.IsInsideDirectory(dataDirectory, path))
|
||||
return path;
|
||||
|
||||
return Path.GetRelativePath(Path.GetFullPath(dataDirectory), Path.GetFullPath(path)).Replace(Path.DirectorySeparatorChar, STORED_SEPARATOR);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Turns a stored path back into the absolute path AI Studio works with.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A relative path which leads out of the data directory was not written by AI Studio. Rather than
|
||||
/// resolving it against whatever directory the process runs in, it is read as no path at all.
|
||||
/// </remarks>
|
||||
/// <param name="storedPath">The path as it was stored.</param>
|
||||
/// <param name="dataDirectory">The data directory, or null while it is not known yet.</param>
|
||||
/// <returns>The absolute path.</returns>
|
||||
internal static string ToAbsolute(string storedPath, string? dataDirectory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(storedPath) || string.IsNullOrWhiteSpace(dataDirectory) || Path.IsPathRooted(storedPath))
|
||||
return storedPath;
|
||||
|
||||
var path = Path.Join(dataDirectory, storedPath.Replace(STORED_SEPARATOR, Path.DirectorySeparatorChar));
|
||||
return PathTools.IsInsideDirectory(dataDirectory, path) ? path : string.Empty;
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,13 @@ public static class DataSourceRepair
|
||||
/// <returns>True when the rebuild was started.</returns>
|
||||
public static async Task<bool> ConfirmAndRepairAsync(IDialogService dialogService, DataSourceEmbeddingService embeddingService, string dataSourceId, string dataSourceName)
|
||||
{
|
||||
// A repair discards the index, which waits for the end of a pause:
|
||||
if (embeddingService.IsIndexingPaused)
|
||||
{
|
||||
await dialogService.ShowMessageBox(TB("Repair Data Source"), TB("AI Studio is backing up or restoring the indexes of your data sources right now. Please try again once that is done."), TB("Close"));
|
||||
return false;
|
||||
}
|
||||
|
||||
var dialogParameters = new DialogParameters<ConfirmDialog>
|
||||
{
|
||||
{ x => x.Message, string.Format(TB("The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?"), dataSourceName) },
|
||||
|
||||
@@ -53,4 +53,16 @@ public abstract partial class IndexStoreClient(string name, string path) : Datab
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The number of chunks, or null when the index cannot tell. Null and zero mean different things here.</returns>
|
||||
public abstract Task<long?> GetTotalChunkCountAsync(CancellationToken token);
|
||||
|
||||
/// <summary>
|
||||
/// Writes a consistent copy of the index into the given directory, e.g., for a backup.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Pause the indexing first, cf. DataSourceEmbeddingService.PauseIndexingAsync. The copy is
|
||||
/// consistent on its own, but the vector stores copied for the same backup have to fit to it.
|
||||
/// </remarks>
|
||||
/// <param name="targetDirectory">Where to write the copy. A copy written there before is replaced.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The path of the copy.</returns>
|
||||
public abstract Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token);
|
||||
}
|
||||
@@ -67,6 +67,15 @@ public sealed partial class NoIndexStoreClient(string name, string? unavailableR
|
||||
|
||||
public override Task<long?> GetTotalChunkCountAsync(CancellationToken token) => Task.FromResult<long?>(null);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// Unlike the other methods, this one does not pretend success. A backup without the index would
|
||||
/// still take the vector stores along, and after a restore, they would belong to nothing the index
|
||||
/// knows. Whoever makes the backup decides what to do instead.
|
||||
/// </remarks>
|
||||
public override Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token) =>
|
||||
Task.FromException<string>(new InvalidOperationException(unavailableReason ?? "The index of the data sources is not available."));
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
+63
@@ -7,6 +7,7 @@ using AIStudio.Tools.Mail;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
@@ -580,6 +581,68 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
|
||||
return versions.FirstOrDefault() ?? string.Empty;
|
||||
}
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// VACUUM INTO writes a consistent snapshot of the database, including what the write-ahead log
|
||||
/// still holds, and in WAL mode, reading it holds back no writer.<br/><br/>
|
||||
///
|
||||
/// It runs on connections of their own, although the databases guide has raw SQL go through
|
||||
/// GetDbConnection. The connections of the context share one cache, and a shared cache locks
|
||||
/// whole tables: reading the whole database for the copy would hold back every writer until the
|
||||
/// copy is done. Without pooling, nothing keeps the copy open afterward, so the backup can
|
||||
/// delete it on every OS.
|
||||
/// </remarks>
|
||||
public override async Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token)
|
||||
{
|
||||
Directory.CreateDirectory(targetDirectory);
|
||||
var targetPath = Path.Join(targetDirectory, DATABASE_FILENAME);
|
||||
|
||||
// VACUUM INTO refuses to write into a file which holds anything:
|
||||
if (File.Exists(targetPath))
|
||||
File.Delete(targetPath);
|
||||
|
||||
try
|
||||
{
|
||||
await using (var connection = new SqliteConnection(BuildBackupConnectionString(this.databasePath, SqliteOpenMode.ReadWrite)))
|
||||
{
|
||||
await connection.OpenAsync(token);
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = "VACUUM INTO $targetPath;";
|
||||
command.Parameters.AddWithValue("$targetPath", targetPath);
|
||||
await command.ExecuteNonQueryAsync(token);
|
||||
}
|
||||
|
||||
await using (var connection = new SqliteConnection(BuildBackupConnectionString(targetPath, SqliteOpenMode.ReadOnly)))
|
||||
{
|
||||
await connection.OpenAsync(token);
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = "PRAGMA quick_check;";
|
||||
|
||||
var checkResult = (await command.ExecuteScalarAsync(token))?.ToString();
|
||||
if (checkResult is not "ok")
|
||||
throw new InvalidOperationException($"The copy of the index at '{targetPath}' failed its check: '{checkResult}'.");
|
||||
}
|
||||
|
||||
return targetPath;
|
||||
}
|
||||
catch
|
||||
{
|
||||
if (File.Exists(targetPath))
|
||||
File.Delete(targetPath);
|
||||
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private static string BuildBackupConnectionString(string path, SqliteOpenMode mode) => new SqliteConnectionStringBuilder
|
||||
{
|
||||
DataSource = path,
|
||||
Mode = mode,
|
||||
Cache = SqliteCacheMode.Private,
|
||||
Pooling = false,
|
||||
DefaultTimeout = 30,
|
||||
}.ToString();
|
||||
|
||||
private IndexStoreDbContext CreateContext() => new(this.dbContextOptions);
|
||||
|
||||
private static void ApplyDataSource(
|
||||
|
||||
@@ -25,6 +25,9 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
|
||||
public override Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
|
||||
Task.FromException<VectorStoreEnsureResult>(this.CreateUnavailableException());
|
||||
|
||||
public override Task<bool> VectorStoreExists(string storeName, CancellationToken token) =>
|
||||
Task.FromException<bool>(this.CreateUnavailableException());
|
||||
|
||||
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
|
||||
Task.FromException(this.CreateUnavailableException());
|
||||
|
||||
@@ -43,6 +46,12 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
|
||||
public override Task DeleteVectorStore(string storeName, CancellationToken token) =>
|
||||
Task.FromException(this.CreateUnavailableException());
|
||||
|
||||
public override Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token) =>
|
||||
Task.FromException<VectorStoreBackupPreparation>(this.CreateUnavailableException());
|
||||
|
||||
public override Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) =>
|
||||
Task.FromException<long>(this.CreateUnavailableException());
|
||||
|
||||
private InvalidOperationException CreateUnavailableException() =>
|
||||
new(unavailableReason ?? "The vector store is not available.");
|
||||
|
||||
|
||||
+25
-1
@@ -23,12 +23,15 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
private const string DATABASE_NAME = "Qdrant Edge";
|
||||
private const string INFO_PATH = "/system/qdrant-edge/info";
|
||||
private const string ENSURE_PATH = "/system/qdrant-edge/ensure";
|
||||
private const string EXISTS_PATH = "/system/qdrant-edge/exists";
|
||||
private const string INSERT_PATH = "/system/qdrant-edge/insert";
|
||||
private const string SEARCH_PATH = "/system/qdrant-edge/search";
|
||||
private const string DELETE_FILE_PATH = "/system/qdrant-edge/delete-file";
|
||||
private const string OPTIMIZE_PATH = "/system/qdrant-edge/optimize";
|
||||
private const string DELETE_STORE_PATH = "/system/qdrant-edge/delete-store";
|
||||
|
||||
private const string PREPARE_BACKUP_PATH = "/system/qdrant-edge/backup/prepare";
|
||||
private const string COPY_STORE_FOR_BACKUP_PATH = "/system/qdrant-edge/backup/copy";
|
||||
|
||||
private readonly string path = path;
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(QdrantEdgeClientImplementation).Namespace, nameof(QdrantEdgeClientImplementation));
|
||||
@@ -122,6 +125,10 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>(DATABASE_NAME, ENSURE_PATH,
|
||||
new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token) ?? throw new InvalidOperationException("The vector store ensure response was empty.");
|
||||
|
||||
public override async Task<bool> VectorStoreExists(string storeName, CancellationToken token) =>
|
||||
await rustService.ExecuteDatabaseQuery<CheckVectorStoreRequest, bool?>(DATABASE_NAME, EXISTS_PATH,
|
||||
new CheckVectorStoreRequest(storeName), token) ?? throw new InvalidOperationException("The vector store exists response was empty.");
|
||||
|
||||
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
|
||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, INSERT_PATH, new InsertEmbeddingRequest(storeName, points), token);
|
||||
|
||||
@@ -158,6 +165,14 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
public override Task DeleteVectorStore(string storeName, CancellationToken token) =>
|
||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_STORE_PATH, new DeleteVectorStoreRequest(storeName), token);
|
||||
|
||||
public override async Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token) =>
|
||||
await rustService.ExecuteDatabaseQuery<PrepareBackupRequest, VectorStoreBackupPreparation>(DATABASE_NAME, PREPARE_BACKUP_PATH,
|
||||
new PrepareBackupRequest(), token) ?? throw new InvalidOperationException("The response preparing the backup of the vector stores was empty.");
|
||||
|
||||
public override async Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) =>
|
||||
await rustService.ExecuteDatabaseQuery<CopyStoreForBackupRequest, long?>(DATABASE_NAME, COPY_STORE_FOR_BACKUP_PATH,
|
||||
new CopyStoreForBackupRequest(backupId, storeName), token) ?? throw new InvalidOperationException("The response copying a vector store for the backup was empty.");
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
@@ -172,6 +187,8 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
// ReSharper disable NotAccessedPositionalProperty.Local
|
||||
private sealed record EnsureVectorStoreRequest(string StoreName, string DataSourceName, int VectorSize);
|
||||
|
||||
private sealed record CheckVectorStoreRequest(string StoreName);
|
||||
|
||||
private sealed record InsertEmbeddingRequest(string StoreName, IReadOnlyList<VectorStoragePoint> Points);
|
||||
|
||||
/// <param name="PointIds">The only points the search may return, or null for the whole store.</param>
|
||||
@@ -182,5 +199,12 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
private sealed record OptimizeVectorStoreRequest(string StoreName);
|
||||
|
||||
private sealed record DeleteVectorStoreRequest(string StoreName);
|
||||
|
||||
/// <summary>
|
||||
/// Preparing a backup needs nothing but the request itself.
|
||||
/// </summary>
|
||||
private sealed record PrepareBackupRequest;
|
||||
|
||||
private sealed record CopyStoreForBackupRequest(string BackupId, string StoreName);
|
||||
// ReSharper restore NotAccessedPositionalProperty.Local
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.Databases.VectorStore;
|
||||
|
||||
/// <summary>
|
||||
/// What a backup needs to copy the stores, one at a time.
|
||||
/// </summary>
|
||||
/// <param name="BackupId">Names the backup when its stores are copied.</param>
|
||||
/// <param name="StagingPath">The directory of the backup's copies. Below it, every copy has the place it has below the data directory. The backup removes it once it is written.</param>
|
||||
/// <param name="Stores">Every store which was created completely.</param>
|
||||
public sealed record VectorStoreBackupPreparation(string BackupId, string StagingPath, IReadOnlyList<VectorStoreBackupStore> Stores);
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace AIStudio.Tools.Databases.VectorStore;
|
||||
|
||||
/// <summary>
|
||||
/// A store which a backup takes along.
|
||||
/// </summary>
|
||||
/// <param name="StoreName">The name of the store.</param>
|
||||
/// <param name="Bytes">How large the store is on the disk.</param>
|
||||
public sealed record VectorStoreBackupStore(string StoreName, long Bytes);
|
||||
@@ -4,6 +4,14 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
|
||||
{
|
||||
public abstract Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token);
|
||||
|
||||
/// <summary>
|
||||
/// Whether the store was created completely, without creating or opening it.
|
||||
/// </summary>
|
||||
/// <param name="storeName">The name of the store.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>True when the store exists.</returns>
|
||||
public abstract Task<bool> VectorStoreExists(string storeName, CancellationToken token);
|
||||
|
||||
public abstract Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token);
|
||||
|
||||
public abstract Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, CancellationToken token);
|
||||
@@ -24,4 +32,29 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
|
||||
public abstract Task OptimizeVectorStore(string storeName, CancellationToken token);
|
||||
|
||||
public abstract Task DeleteVectorStore(string storeName, CancellationToken token);
|
||||
|
||||
/// <summary>
|
||||
/// Prepares a backup of the stores: lists them, checks that their copies fit on the disk, and
|
||||
/// creates the directory for the copies below the data directory.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Pause the indexing first, cf. DataSourceEmbeddingService.PauseIndexingAsync, so that no store
|
||||
/// changes until all of them are copied.
|
||||
/// </remarks>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The backup, with every store to copy.</returns>
|
||||
public abstract Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token);
|
||||
|
||||
/// <summary>
|
||||
/// Copies one store into the directory of a prepared backup.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The store is unloaded for the copy and loads again with the next request. Requests for any
|
||||
/// store wait while one is copied.
|
||||
/// </remarks>
|
||||
/// <param name="backupId">The backup, as PrepareBackupAsync named it.</param>
|
||||
/// <param name="storeName">The name of the store.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>How many bytes were copied. Zero when the store was not created completely.</returns>
|
||||
public abstract Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token);
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Shared rules for comparing file system paths.
|
||||
/// </summary>
|
||||
public static class PathTools
|
||||
{
|
||||
/// <summary>
|
||||
/// How to compare file system paths: Windows treats them case-insensitively, Linux and
|
||||
/// macOS do not. There, two files may differ in casing alone.
|
||||
/// </summary>
|
||||
public static readonly StringComparison COMPARISON = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||
|
||||
/// <summary>
|
||||
/// The comparer matching COMPARISON, for path-keyed lookups.
|
||||
/// </summary>
|
||||
public static readonly StringComparer COMPARER = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal;
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a path lies below the given directory.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The directory itself does not count as lying below itself. A path which cannot be resolved
|
||||
/// counts as lying elsewhere.
|
||||
/// </remarks>
|
||||
/// <param name="directory">The directory.</param>
|
||||
/// <param name="path">The path to check.</param>
|
||||
/// <returns>Whether the path lies below the directory.</returns>
|
||||
public static bool IsInsideDirectory(string directory, string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
var fullDirectory = Path.GetFullPath(directory);
|
||||
var directoryPrefix = fullDirectory.EndsWith(Path.DirectorySeparatorChar)
|
||||
? fullDirectory
|
||||
: fullDirectory + Path.DirectorySeparatorChar;
|
||||
|
||||
return Path.GetFullPath(path).StartsWith(directoryPrefix, COMPARISON);
|
||||
}
|
||||
catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -33,11 +33,11 @@ Plugins can configure:
|
||||
|
||||
Configuration plugins provide three kinds of values:
|
||||
- **Managed settings:** simple values such as booleans, numbers, strings, enums, lists, or sets handled through `ManagedConfiguration`. These values may be locked or used as organization defaults. Which configuration plugin owns a locked setting is persisted in `Data.ManagedLockedConfigurations`, and organization defaults are tracked in `Data.ManagedEditableDefaults`. Both are cleaned up generically by `ManagedConfiguration.CleanupLeftOverManagedConfigurations(...)` when the owning plugin is gone. The value a setting had before a configuration plugin took it over is kept in `Data.ManagedUserValueSnapshots` and restored by that same clean-up, so removing a plugin hands the user's own value back instead of the app default.
|
||||
- **Managed configuration objects:** complex Lua tables that are persisted into `SettingsManager.ConfigurationData`, implement `IConfigurationObject`, and are cleaned up through `PluginConfigurationObject.CleanLeftOverConfigurationObjects(...)`. Examples include providers, profiles, chat templates, data sources, and document analysis policies.
|
||||
- **Managed configuration objects:** complex Lua tables that are persisted into `SettingsManager.ConfigurationData`, implement `IConfigurationObject`, and are described by a `PluginConfigurationObjectDescriptor` in `PluginConfigurationObjectDescriptors`. Reading them from a plugin and cleaning up the left-over ones go through these descriptors. Examples include providers, profiles, chat templates, data sources, and document analysis policies.
|
||||
- **Live plugin content:** complex Lua tables that implement `ILivePluginContent` and are read live from running plugins instead of being persisted to `ConfigurationData`. Examples include `MANDATORY_INFOS` and `INTRODUCTIONS`. If live plugin content creates persistent side data, add a dedicated cleanup path for that side data, like mandatory-info acceptances.
|
||||
|
||||
When adding configuration plugin capabilities:
|
||||
- For managed settings, update the corresponding data class in `app/MindWork AI Studio/Settings/DataModel/` to call `ManagedConfiguration.Register(...)` and process the setting in `PluginConfiguration.TryProcessConfiguration`. Cleaning up the setting when its configuration plugin was removed needs no extra step: `ManagedConfiguration.CleanupLeftOverManagedConfigurations(...)` iterates all registered settings. Do not add per-setting cleanup calls to `PluginFactory.Loading.LoadAll`.
|
||||
- For managed configuration objects, update `PluginConfigurationObject.cs` and `PluginConfigurationObjectType.cs`, persist them in the appropriate `ConfigurationData` collection, and add cleanup via `PluginConfigurationObject.CleanLeftOverConfigurationObjects(...)`.
|
||||
- For managed configuration objects, add the type to `PluginConfigurationObjectType.cs`, persist the objects in the appropriate `ConfigurationData` collection, and add a descriptor to `PluginConfigurationObjectDescriptors` and its list `ALL`: its Lua table, its parser, the collection, the next number, and where secrets are kept. Parsing and cleanup need no extra call; `PluginConfigurationObjectDescriptorsTests` fails for a type without a descriptor.
|
||||
- For live plugin content, add a data type implementing `ILivePluginContent`, parse it in `PluginConfiguration`, expose it through `PluginFactory`, and add any required cleanup only for persistent side data.
|
||||
- Always document the new capability in `app/MindWork AI Studio/Plugins/configuration/plugin.lua`.
|
||||
@@ -0,0 +1,52 @@
|
||||
using System.Linq.Expressions;
|
||||
|
||||
using AIStudio.Settings.DataModel;
|
||||
|
||||
using Lua;
|
||||
|
||||
namespace AIStudio.Tools.PluginSystem;
|
||||
|
||||
/// <summary>
|
||||
/// Describes a type of configuration object which derives from ConfigurationBaseObject.
|
||||
/// </summary>
|
||||
/// <param name="type">The type of the configuration objects.</param>
|
||||
/// <param name="luaTableName">The Lua table in which a configuration plugin defines these objects.</param>
|
||||
/// <param name="selection">Where the settings keep these objects.</param>
|
||||
/// <param name="nextNumSelection">The number which the next of these objects gets.</param>
|
||||
/// <param name="parser">Reads one of these objects from its Lua table.</param>
|
||||
/// <param name="secretStoreType">Where the API keys of these objects are kept, or null for objects without one.</param>
|
||||
/// <typeparam name="TClass">The class of the configuration objects.</typeparam>
|
||||
public sealed class ConfigurationBaseObjectDescriptor<TClass>(PluginConfigurationObjectType type, string luaTableName, Expression<Func<Data, List<TClass>>> selection, Expression<Func<Data, uint>> nextNumSelection, ConfigurationObjectParser parser, SecretStoreType? secretStoreType = null) : PluginConfigurationObjectDescriptor where TClass : ConfigurationBaseObject
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override PluginConfigurationObjectType Type => type;
|
||||
|
||||
/// <summary>
|
||||
/// The Lua table in which a configuration plugin defines these objects.
|
||||
/// </summary>
|
||||
public string LuaTableName => luaTableName;
|
||||
|
||||
/// <summary>
|
||||
/// Where the settings keep these objects.
|
||||
/// </summary>
|
||||
public Expression<Func<Data, List<TClass>>> Selection => selection;
|
||||
|
||||
/// <summary>
|
||||
/// The number which the next of these objects gets.
|
||||
/// </summary>
|
||||
public Expression<Func<Data, uint>> NextNumSelection => nextNumSelection;
|
||||
|
||||
/// <summary>
|
||||
/// Reads one of these objects from its Lua table.
|
||||
/// </summary>
|
||||
public ConfigurationObjectParser Parser => parser;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool TryParse(LuaTable mainTable, Guid configPluginId, List<PluginConfigurationObject> configObjects, bool dryRun, string pluginPath) => PluginConfigurationObject.TryParse(this, mainTable, configPluginId, configObjects, dryRun, pluginPath);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override Task<bool> CleanLeftOverAsync(IList<IAvailablePlugin> availablePlugins, IReadOnlySet<Guid> deployedEnterpriseConfigPluginIds, IReadOnlySet<Guid> notStartedConfigPluginIds, IList<PluginConfigurationObject> configObjectList) => PluginConfigurationObject.CleanLeftOverConfigurationObjects(type, selection, availablePlugins, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, secretStoreType);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int RemovePluginObjects(Data settings) => selection.Compile()(settings).RemoveAll(configObject => configObject.IsEnterpriseConfiguration);
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
using Lua;
|
||||
|
||||
namespace AIStudio.Tools.PluginSystem;
|
||||
|
||||
/// <summary>
|
||||
/// Reads one configuration object from the Lua table in which a configuration plugin defines it.
|
||||
/// </summary>
|
||||
/// <param name="index">The position of the entry within its Lua table, which the log names.</param>
|
||||
/// <param name="table">The entry to read.</param>
|
||||
/// <param name="configPluginId">The configuration plugin which defines the object.</param>
|
||||
/// <param name="pluginPath">The directory of the plugin, which paths in the entry are relative to.</param>
|
||||
/// <param name="dryRun">Whether the plugin is only checked, e.g., before an import. Nothing must be stored then.</param>
|
||||
/// <param name="configObject">The configuration object read from the entry.</param>
|
||||
/// <returns>True when the entry defines a valid configuration object.</returns>
|
||||
public delegate bool ConfigurationObjectParser(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject);
|
||||
@@ -0,0 +1,32 @@
|
||||
using AIStudio.Settings.DataModel;
|
||||
|
||||
using Lua;
|
||||
|
||||
namespace AIStudio.Tools.PluginSystem;
|
||||
|
||||
/// <summary>
|
||||
/// Describes the data sources which configuration plugins provide.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The settings keep every data source in a list of IDataSource, and a data source of a configuration
|
||||
/// plugin is a record struct rather than a ConfigurationBaseObject. That is why data sources are read
|
||||
/// in a way of their own.
|
||||
/// </remarks>
|
||||
public sealed class DataSourceConfigurationObjectDescriptor : PluginConfigurationObjectDescriptor
|
||||
{
|
||||
/// <inheritdoc />
|
||||
public override PluginConfigurationObjectType Type => PluginConfigurationObjectType.DATA_SOURCE;
|
||||
|
||||
/// <inheritdoc />
|
||||
public override bool TryParse(LuaTable mainTable, Guid configPluginId, List<PluginConfigurationObject> configObjects, bool dryRun, string pluginPath) => PluginConfigurationObject.TryParseDataSources(mainTable, configPluginId, configObjects, dryRun);
|
||||
|
||||
/// <inheritdoc />
|
||||
/// <remarks>
|
||||
/// The secret of a data source is not necessarily an API key, e.g., it is a password, so it is
|
||||
/// deleted as a secret.
|
||||
/// </remarks>
|
||||
public override Task<bool> CleanLeftOverAsync(IList<IAvailablePlugin> availablePlugins, IReadOnlySet<Guid> deployedEnterpriseConfigPluginIds, IReadOnlySet<Guid> notStartedConfigPluginIds, IList<PluginConfigurationObject> configObjectList) => PluginConfigurationObject.CleanLeftOverConfigurationObjects(this.Type, x => x.DataSources, availablePlugins, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.DATA_SOURCE, deleteSecret: true);
|
||||
|
||||
/// <inheritdoc />
|
||||
public override int RemovePluginObjects(Data settings) => settings.DataSources.RemoveAll(dataSource => dataSource.IsEnterpriseConfiguration);
|
||||
}
|
||||
@@ -1,26 +0,0 @@
|
||||
namespace AIStudio.Tools.PluginSystem;
|
||||
|
||||
public sealed record NoConfigurationObject : ConfigurationBaseObject
|
||||
{
|
||||
public static readonly NoConfigurationObject INSTANCE = new();
|
||||
|
||||
private NoConfigurationObject()
|
||||
{
|
||||
this.Id = Guid.Empty.ToString();
|
||||
this.Name = "No Configuration";
|
||||
}
|
||||
|
||||
#region Overrides of ConfigurationBaseObject
|
||||
|
||||
public override string Id { get; init; }
|
||||
|
||||
public override uint Num { get; init; }
|
||||
|
||||
public override string Name { get; init; }
|
||||
|
||||
public override bool IsEnterpriseConfiguration { get; init; }
|
||||
|
||||
public override Guid EnterpriseConfigurationPluginId { get; init; }
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -22,16 +22,12 @@ public static class PluginArchive
|
||||
using var archive = ZipFile.OpenRead(sourceArchiveFileName);
|
||||
Directory.CreateDirectory(destinationDirectory);
|
||||
|
||||
var destinationDirectoryFullPath = Path.GetFullPath(destinationDirectory);
|
||||
if (!destinationDirectoryFullPath.EndsWith(Path.DirectorySeparatorChar))
|
||||
destinationDirectoryFullPath += Path.DirectorySeparatorChar;
|
||||
|
||||
foreach (var entry in archive.Entries)
|
||||
{
|
||||
var normalizedEntryName = NormalizeEntryName(entry.FullName);
|
||||
var destinationPath = GetEntryDestinationPath(destinationDirectoryFullPath, normalizedEntryName);
|
||||
var normalizedEntryName = ZipEntryNames.Normalize(entry.FullName);
|
||||
var destinationPath = ZipEntryNames.GetDestinationPath(destinationDirectory, normalizedEntryName);
|
||||
|
||||
if (normalizedEntryName.EndsWith('/'))
|
||||
if (normalizedEntryName.EndsWith(ZipEntryNames.SEPARATOR))
|
||||
{
|
||||
if (entry.Length != 0)
|
||||
throw new InvalidDataException($"The plugin archive contains a directory entry with data: '{entry.FullName}'.");
|
||||
@@ -44,37 +40,4 @@ public static class PluginArchive
|
||||
entry.ExtractToFile(destinationPath);
|
||||
}
|
||||
}
|
||||
|
||||
private static string NormalizeEntryName(string entryName)
|
||||
{
|
||||
var normalizedEntryName = entryName.Replace('\\', '/');
|
||||
if (string.IsNullOrWhiteSpace(normalizedEntryName))
|
||||
throw new InvalidDataException("The plugin archive contains an empty entry name.");
|
||||
|
||||
if (normalizedEntryName.Contains('\0'))
|
||||
throw new InvalidDataException($"The plugin archive contains an invalid entry name: '{entryName}'.");
|
||||
|
||||
if (normalizedEntryName.StartsWith('/'))
|
||||
throw new InvalidDataException($"The plugin archive contains a rooted entry name: '{entryName}'.");
|
||||
|
||||
if (normalizedEntryName is [_, ':', ..])
|
||||
throw new InvalidDataException($"The plugin archive contains a drive-qualified entry name: '{entryName}'.");
|
||||
|
||||
var pathSegments = normalizedEntryName.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
if (pathSegments.Length == 0 || pathSegments.Any(segment => segment is "." or ".."))
|
||||
throw new InvalidDataException($"The plugin archive contains an unsafe entry name: '{entryName}'.");
|
||||
|
||||
return normalizedEntryName;
|
||||
}
|
||||
|
||||
private static string GetEntryDestinationPath(string destinationDirectoryFullPath, string normalizedEntryName)
|
||||
{
|
||||
var pathSegments = normalizedEntryName.Split('/', StringSplitOptions.RemoveEmptyEntries);
|
||||
var relativePath = Path.Combine(pathSegments);
|
||||
var destinationPath = Path.GetFullPath(Path.Combine(destinationDirectoryFullPath, relativePath));
|
||||
if (!destinationPath.StartsWith(destinationDirectoryFullPath, StringComparison.Ordinal))
|
||||
throw new InvalidDataException($"The plugin archive contains an entry outside the destination directory: '{normalizedEntryName}'.");
|
||||
|
||||
return destinationPath;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -272,6 +272,12 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
|
||||
// Config: allow the user to share or export plugins?
|
||||
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToSharePlugins, this.Id, settingsTable, dryRun);
|
||||
|
||||
// Config: allow the user to back up the data of AI Studio?
|
||||
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToBackUpData, this.Id, settingsTable, dryRun);
|
||||
|
||||
// Config: allow the user to restore the data of AI Studio from a backup?
|
||||
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.AllowUserToRestoreData, this.Id, settingsTable, dryRun);
|
||||
|
||||
// Config: show administration settings?
|
||||
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.ShowAdminSettings, this.Id, settingsTable, dryRun);
|
||||
|
||||
@@ -349,26 +355,9 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
|
||||
// Config: enterprise-managed approvals for assistant plugins
|
||||
this.TryProcessEnterpriseApprovedAssistantPlugins(settingsTable, dryRun);
|
||||
|
||||
// Handle configured LLM providers:
|
||||
PluginConfigurationObject.TryParse(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, x => x.NextProviderNum, mainTable, this.Id, ref this.configObjects, dryRun, this.PluginPath);
|
||||
|
||||
// Handle configured transcription providers:
|
||||
PluginConfigurationObject.TryParse(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER, x => x.TranscriptionProviders, x => x.NextTranscriptionNum, mainTable, this.Id, ref this.configObjects, dryRun, this.PluginPath);
|
||||
|
||||
// Handle configured embedding providers:
|
||||
PluginConfigurationObject.TryParse(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, x => x.NextEmbeddingNum, mainTable, this.Id, ref this.configObjects, dryRun, this.PluginPath);
|
||||
|
||||
// Handle configured chat templates:
|
||||
PluginConfigurationObject.TryParse(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, x => x.NextChatTemplateNum, mainTable, this.Id, ref this.configObjects, dryRun, this.PluginPath);
|
||||
|
||||
// Handle configured data sources:
|
||||
PluginConfigurationObject.TryParseDataSources(mainTable, this.Id, ref this.configObjects, dryRun);
|
||||
|
||||
// Handle configured profiles:
|
||||
PluginConfigurationObject.TryParse(PluginConfigurationObjectType.PROFILE, x => x.Profiles, x => x.NextProfileNum, mainTable, this.Id, ref this.configObjects, dryRun);
|
||||
|
||||
// Handle configured document analysis policies:
|
||||
PluginConfigurationObject.TryParse(PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY, x => x.DocumentAnalysis.Policies, x => x.NextDocumentAnalysisPolicyNum, mainTable, this.Id, ref this.configObjects, dryRun);
|
||||
// Handle configured objects such as LLM providers, chat templates, or data sources:
|
||||
foreach (var descriptor in PluginConfigurationObjectDescriptors.ALL)
|
||||
descriptor.TryParse(mainTable, this.Id, this.configObjects, dryRun, this.PluginPath);
|
||||
|
||||
// Handle configured mandatory infos:
|
||||
this.TryReadMandatoryInfos(mainTable);
|
||||
|
||||
@@ -125,53 +125,22 @@ public sealed record PluginConfigurationObject
|
||||
/// provided list with results.
|
||||
/// </summary>
|
||||
/// <typeparam name="TClass">The type of configuration object to parse, which must
|
||||
/// inherit from <see cref="ConfigurationBaseObject"/>.</typeparam>
|
||||
/// <param name="configObjectType">The type of configuration object to process, as specified
|
||||
/// in <see cref="PluginConfigurationObjectType"/>.</param>
|
||||
/// <param name="configObjectSelection">An expression to retrieve existing configuration objects from
|
||||
/// the main configuration data.</param>
|
||||
/// <param name="nextConfigObjectNumSelection">An expression to retrieve the next available configuration
|
||||
/// object number from the main configuration data.</param>
|
||||
/// inherit from ConfigurationBaseObject.</typeparam>
|
||||
/// <param name="descriptor">Describes the type of configuration object: its Lua table, how to
|
||||
/// read one entry, and where the settings keep these objects.</param>
|
||||
/// <param name="mainTable">The Lua table containing entries to parse into configuration objects.</param>
|
||||
/// <param name="configPluginId">The unique identifier of the plugin associated with the configuration
|
||||
/// objects being parsed.</param>
|
||||
/// <param name="configObjects">The list to populate with the parsed configuration objects.
|
||||
/// This parameter is passed by reference.</param>
|
||||
/// <param name="configObjects">The list to populate with the parsed configuration objects.</param>
|
||||
/// <param name="dryRun">Specifies whether to perform the operation as a dry run, where changes
|
||||
/// are not persisted.</param>
|
||||
/// <param name="pluginPath">An optional parameter specifying the file path of the plugin, used for relative paths in the Lua table.</param>
|
||||
/// <param name="pluginPath">The directory of the plugin, used for relative paths in the Lua table.</param>
|
||||
/// <returns>Returns true if parsing succeeds and configuration objects are added
|
||||
/// to the list; otherwise, false.</returns>
|
||||
public static bool TryParse<TClass>(
|
||||
PluginConfigurationObjectType configObjectType,
|
||||
Expression<Func<Data, List<TClass>>> configObjectSelection,
|
||||
Expression<Func<Data, uint>> nextConfigObjectNumSelection,
|
||||
LuaTable mainTable,
|
||||
Guid configPluginId,
|
||||
ref List<PluginConfigurationObject> configObjects,
|
||||
bool dryRun,
|
||||
string pluginPath = ""
|
||||
) where TClass : ConfigurationBaseObject
|
||||
public static bool TryParse<TClass>(ConfigurationBaseObjectDescriptor<TClass> descriptor, LuaTable mainTable, Guid configPluginId, List<PluginConfigurationObject> configObjects, bool dryRun, string pluginPath) where TClass : ConfigurationBaseObject
|
||||
{
|
||||
var luaTableName = configObjectType switch
|
||||
{
|
||||
PluginConfigurationObjectType.LLM_PROVIDER => "LLM_PROVIDERS",
|
||||
PluginConfigurationObjectType.CHAT_TEMPLATE => "CHAT_TEMPLATES",
|
||||
PluginConfigurationObjectType.DATA_SOURCE => "DATA_SOURCES",
|
||||
PluginConfigurationObjectType.EMBEDDING_PROVIDER => "EMBEDDING_PROVIDERS",
|
||||
PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER => "TRANSCRIPTION_PROVIDERS",
|
||||
PluginConfigurationObjectType.PROFILE => "PROFILES",
|
||||
PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY => "DOCUMENT_ANALYSIS_POLICIES",
|
||||
|
||||
_ => null,
|
||||
};
|
||||
|
||||
if (luaTableName is null)
|
||||
{
|
||||
LOG.LogError("The configuration object type '{ConfigObjectType}' is not supported yet (config plugin id: {ConfigPluginId}).", configObjectType, configPluginId);
|
||||
return false;
|
||||
}
|
||||
|
||||
var configObjectType = descriptor.Type;
|
||||
var luaTableName = descriptor.LuaTableName;
|
||||
if (!mainTable.TryGetValue(luaTableName, out var luaValue) || !luaValue.TryRead<LuaTable>(out var luaTable))
|
||||
{
|
||||
LOG.LogWarning("The table '{LuaTableName}' does not exist or is not a valid table (config plugin id: {ConfigPluginId}).", luaTableName, configPluginId);
|
||||
@@ -179,7 +148,7 @@ public sealed record PluginConfigurationObject
|
||||
}
|
||||
|
||||
var localSettingsManager = SettingsManagerAccess;
|
||||
var storedObjects = configObjectSelection.Compile()(localSettingsManager.ConfigurationData);
|
||||
var storedObjects = descriptor.Selection.Compile()(localSettingsManager.ConfigurationData);
|
||||
var numberObjects = luaTable.ArrayLength;
|
||||
ThreadSafeRandom? random = null;
|
||||
for (var i = 1; i <= numberObjects; i++)
|
||||
@@ -191,19 +160,7 @@ public sealed record PluginConfigurationObject
|
||||
continue;
|
||||
}
|
||||
|
||||
var (wasParsingSuccessful, configObject) = configObjectType switch
|
||||
{
|
||||
PluginConfigurationObjectType.LLM_PROVIDER => (Settings.Provider.TryParseProviderTable(i, luaObjectTable, configPluginId, pluginPath, dryRun, out var configurationObject) && configurationObject != Settings.Provider.NONE, configurationObject),
|
||||
PluginConfigurationObjectType.CHAT_TEMPLATE => (ChatTemplate.TryParseChatTemplateTable(i, luaObjectTable, configPluginId, pluginPath, out var configurationObject) && configurationObject != ChatTemplate.NO_CHAT_TEMPLATE, configurationObject),
|
||||
PluginConfigurationObjectType.PROFILE => (Profile.TryParseProfileTable(i, luaObjectTable, configPluginId, out var configurationObject) && configurationObject != Profile.NO_PROFILE, configurationObject),
|
||||
PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER => (TranscriptionProvider.TryParseTranscriptionProviderTable(i, luaObjectTable, configPluginId, pluginPath, dryRun, out var configurationObject) && configurationObject != TranscriptionProvider.NONE, configurationObject),
|
||||
PluginConfigurationObjectType.EMBEDDING_PROVIDER => (EmbeddingProvider.TryParseEmbeddingProviderTable(i, luaObjectTable, configPluginId, pluginPath, dryRun, out var configurationObject) && configurationObject != EmbeddingProvider.NONE, configurationObject),
|
||||
PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY => (DataDocumentAnalysisPolicy.TryProcessConfiguration(i, luaObjectTable, configPluginId, out var configurationObject) && configurationObject is DataDocumentAnalysisPolicy, configurationObject),
|
||||
|
||||
_ => (false, NoConfigurationObject.INSTANCE)
|
||||
};
|
||||
|
||||
if (wasParsingSuccessful)
|
||||
if (descriptor.Parser(i, luaObjectTable, configPluginId, pluginPath, dryRun, out var configObject))
|
||||
{
|
||||
// Store it in the config object list:
|
||||
configObjects.Add(new()
|
||||
@@ -236,7 +193,7 @@ public sealed record PluginConfigurationObject
|
||||
else
|
||||
{
|
||||
configObject = KeepSynchronizedTokenizer(configObject, null);
|
||||
if (nextConfigObjectNumSelection.TryIncrement(localSettingsManager.ConfigurationData, IncrementType.POST) is { Success: true, UpdatedValue: var nextNum })
|
||||
if (descriptor.NextNumSelection.TryIncrement(localSettingsManager.ConfigurationData, IncrementType.POST) is { Success: true, UpdatedValue: var nextNum })
|
||||
{
|
||||
// Case: Increment the next number was successful
|
||||
configObject = configObject with { Num = nextNum };
|
||||
@@ -330,11 +287,7 @@ public sealed record PluginConfigurationObject
|
||||
/// <param name="configObjects">The list to populate with the parsed configuration objects.</param>
|
||||
/// <param name="dryRun">Specifies whether to perform the operation as a dry run.</param>
|
||||
/// <returns>True if the table was present and processed; otherwise false.</returns>
|
||||
public static bool TryParseDataSources(
|
||||
LuaTable mainTable,
|
||||
Guid configPluginId,
|
||||
ref List<PluginConfigurationObject> configObjects,
|
||||
bool dryRun)
|
||||
public static bool TryParseDataSources(LuaTable mainTable, Guid configPluginId, List<PluginConfigurationObject> configObjects, bool dryRun)
|
||||
{
|
||||
const string LUA_TABLE_NAME = "DATA_SOURCES";
|
||||
if (!mainTable.TryGetValue(LUA_TABLE_NAME, out var luaValue) || !luaValue.TryRead<LuaTable>(out var luaTable))
|
||||
@@ -698,16 +651,6 @@ public sealed record PluginConfigurationObject
|
||||
|
||||
var fullPluginPath = Path.GetFullPath(pluginPath);
|
||||
var candidatePath = Path.GetFullPath(Path.Combine(fullPluginPath, configuredTokenizerPath));
|
||||
|
||||
if (candidatePath.Equals(fullPluginPath, StringComparison.OrdinalIgnoreCase))
|
||||
return null;
|
||||
|
||||
var pluginPrefix = fullPluginPath.EndsWith(Path.DirectorySeparatorChar)
|
||||
? fullPluginPath
|
||||
: fullPluginPath + Path.DirectorySeparatorChar;
|
||||
|
||||
return candidatePath.StartsWith(pluginPrefix, StringComparison.OrdinalIgnoreCase)
|
||||
? candidatePath
|
||||
: null;
|
||||
return PathTools.IsInsideDirectory(fullPluginPath, candidatePath) ? candidatePath : null;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,54 @@
|
||||
using AIStudio.Settings.DataModel;
|
||||
|
||||
using Lua;
|
||||
|
||||
namespace AIStudio.Tools.PluginSystem;
|
||||
|
||||
/// <summary>
|
||||
/// Describes one type of configuration object which configuration plugins provide, such as LLM
|
||||
/// providers or chat templates.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Whatever treats every type of configuration object alike goes through these descriptors, so that
|
||||
/// none of these places can forget a type: reading the objects from a configuration plugin, removing
|
||||
/// those a plugin no longer defines, and leaving all of them out of a backup. A new type of
|
||||
/// configuration object needs a descriptor in PluginConfigurationObjectDescriptors, and every such place
|
||||
/// handles it.
|
||||
/// </remarks>
|
||||
public abstract class PluginConfigurationObjectDescriptor
|
||||
{
|
||||
/// <summary>
|
||||
/// The type of the configuration objects.
|
||||
/// </summary>
|
||||
public abstract PluginConfigurationObjectType Type { get; }
|
||||
|
||||
/// <summary>
|
||||
/// Reads the configuration objects of this type which a configuration plugin defines, and stores
|
||||
/// them in the settings unless the plugin is only checked.
|
||||
/// </summary>
|
||||
/// <param name="mainTable">The main table of the configuration plugin.</param>
|
||||
/// <param name="configPluginId">The configuration plugin.</param>
|
||||
/// <param name="configObjects">The list to which every object read is added.</param>
|
||||
/// <param name="dryRun">Whether the plugin is only checked, e.g., before an import. Nothing is stored then.</param>
|
||||
/// <param name="pluginPath">The directory of the plugin, which paths in the objects are relative to.</param>
|
||||
/// <returns>True when the plugin defines a table for this type.</returns>
|
||||
public abstract bool TryParse(LuaTable mainTable, Guid configPluginId, List<PluginConfigurationObject> configObjects, bool dryRun, string pluginPath);
|
||||
|
||||
/// <summary>
|
||||
/// Removes the configuration objects of this type which belong to a configuration plugin that is
|
||||
/// gone or no longer defines them, along with their secrets and tokenizers.
|
||||
/// </summary>
|
||||
/// <param name="availablePlugins">The plugins AI Studio found.</param>
|
||||
/// <param name="deployedEnterpriseConfigPluginIds">The configuration plugins an organization deployed, including those which could not be loaded.</param>
|
||||
/// <param name="notStartedConfigPluginIds">The configuration plugins which were loaded, but did not start.</param>
|
||||
/// <param name="configObjectList">The configuration objects all running configuration plugins define.</param>
|
||||
/// <returns>True when the settings changed.</returns>
|
||||
public abstract Task<bool> CleanLeftOverAsync(IList<IAvailablePlugin> availablePlugins, IReadOnlySet<Guid> deployedEnterpriseConfigPluginIds, IReadOnlySet<Guid> notStartedConfigPluginIds, IList<PluginConfigurationObject> configObjectList);
|
||||
|
||||
/// <summary>
|
||||
/// Removes the configuration objects of this type which a configuration plugin manages.
|
||||
/// </summary>
|
||||
/// <param name="settings">The settings to change, e.g., a copy for a backup. Never the settings AI Studio runs with.</param>
|
||||
/// <returns>How many objects were removed.</returns>
|
||||
public abstract int RemovePluginObjects(Data settings);
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
|
||||
using Lua;
|
||||
|
||||
namespace AIStudio.Tools.PluginSystem;
|
||||
|
||||
/// <summary>
|
||||
/// The descriptors of every type of configuration object which configuration plugins provide.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A new type of configuration object gets its descriptor here and joins ALL. Reading the objects from
|
||||
/// a configuration plugin and removing the left-over ones then handle it, and a test checks that no
|
||||
/// type of PluginConfigurationObjectType is missing.
|
||||
/// </remarks>
|
||||
public static class PluginConfigurationObjectDescriptors
|
||||
{
|
||||
public static readonly ConfigurationBaseObjectDescriptor<Settings.Provider> LLM_PROVIDER = new(PluginConfigurationObjectType.LLM_PROVIDER, "LLM_PROVIDERS", x => x.Providers, x => x.NextProviderNum, ParseLLMProvider, SecretStoreType.LLM_PROVIDER);
|
||||
|
||||
public static readonly ConfigurationBaseObjectDescriptor<TranscriptionProvider> TRANSCRIPTION_PROVIDER = new(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER, "TRANSCRIPTION_PROVIDERS", x => x.TranscriptionProviders, x => x.NextTranscriptionNum, ParseTranscriptionProvider, SecretStoreType.TRANSCRIPTION_PROVIDER);
|
||||
|
||||
public static readonly ConfigurationBaseObjectDescriptor<EmbeddingProvider> EMBEDDING_PROVIDER = new(PluginConfigurationObjectType.EMBEDDING_PROVIDER, "EMBEDDING_PROVIDERS", x => x.EmbeddingProviders, x => x.NextEmbeddingNum, ParseEmbeddingProvider, SecretStoreType.EMBEDDING_PROVIDER);
|
||||
|
||||
public static readonly ConfigurationBaseObjectDescriptor<ChatTemplate> CHAT_TEMPLATE = new(PluginConfigurationObjectType.CHAT_TEMPLATE, "CHAT_TEMPLATES", x => x.ChatTemplates, x => x.NextChatTemplateNum, ParseChatTemplate);
|
||||
|
||||
public static readonly DataSourceConfigurationObjectDescriptor DATA_SOURCE = new();
|
||||
|
||||
public static readonly ConfigurationBaseObjectDescriptor<Profile> PROFILE = new(PluginConfigurationObjectType.PROFILE, "PROFILES", x => x.Profiles, x => x.NextProfileNum, ParseProfile);
|
||||
|
||||
public static readonly ConfigurationBaseObjectDescriptor<DataDocumentAnalysisPolicy> DOCUMENT_ANALYSIS_POLICY = new(PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY, "DOCUMENT_ANALYSIS_POLICIES", x => x.DocumentAnalysis.Policies, x => x.NextDocumentAnalysisPolicyNum, ParseDocumentAnalysisPolicy);
|
||||
|
||||
/// <summary>
|
||||
/// Every descriptor, in the order in which configuration plugins are read.
|
||||
/// </summary>
|
||||
public static readonly IReadOnlyList<PluginConfigurationObjectDescriptor> ALL =
|
||||
[
|
||||
LLM_PROVIDER,
|
||||
TRANSCRIPTION_PROVIDER,
|
||||
EMBEDDING_PROVIDER,
|
||||
CHAT_TEMPLATE,
|
||||
DATA_SOURCE,
|
||||
PROFILE,
|
||||
DOCUMENT_ANALYSIS_POLICY,
|
||||
];
|
||||
|
||||
private static bool ParseLLMProvider(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) =>
|
||||
Settings.Provider.TryParseProviderTable(index, table, configPluginId, pluginPath, dryRun, out configObject) && configObject != Settings.Provider.NONE;
|
||||
|
||||
private static bool ParseTranscriptionProvider(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) =>
|
||||
TranscriptionProvider.TryParseTranscriptionProviderTable(index, table, configPluginId, pluginPath, dryRun, out configObject) && configObject != TranscriptionProvider.NONE;
|
||||
|
||||
private static bool ParseEmbeddingProvider(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) =>
|
||||
EmbeddingProvider.TryParseEmbeddingProviderTable(index, table, configPluginId, pluginPath, dryRun, out configObject) && configObject != EmbeddingProvider.NONE;
|
||||
|
||||
private static bool ParseChatTemplate(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) =>
|
||||
ChatTemplate.TryParseChatTemplateTable(index, table, configPluginId, pluginPath, out configObject) && configObject != ChatTemplate.NO_CHAT_TEMPLATE;
|
||||
|
||||
private static bool ParseProfile(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) =>
|
||||
Profile.TryParseProfileTable(index, table, configPluginId, out configObject) && configObject != Profile.NO_PROFILE;
|
||||
|
||||
private static bool ParseDocumentAnalysisPolicy(int index, LuaTable table, Guid configPluginId, string pluginPath, bool dryRun, out ConfigurationBaseObject configObject) =>
|
||||
DataDocumentAnalysisPolicy.TryProcessConfiguration(index, table, configPluginId, out configObject) && configObject is DataDocumentAnalysisPolicy;
|
||||
}
|
||||
@@ -14,7 +14,19 @@ public static partial class PluginFactory
|
||||
{
|
||||
private static readonly List<IAvailablePlugin> AVAILABLE_PLUGINS = [];
|
||||
private static readonly SemaphoreSlim PLUGIN_LOAD_SEMAPHORE = new(1, 1);
|
||||
|
||||
private static readonly TaskCompletionSource FIRST_LOAD_COMPLETED = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
/// <summary>
|
||||
/// Completes once LoadAll got through for the first time, the clean-up of the configuration included.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Until then, the settings may lack whatever the configuration of an organization provides, such
|
||||
/// as its embedding providers: on a new computer, or after restoring a backup, which leaves them
|
||||
/// out. Whoever acts on such objects while AI Studio starts waits for this, or takes them for
|
||||
/// objects which are gone. It never completes when loading failed, so wait with a limit.
|
||||
/// </remarks>
|
||||
public static Task FirstLoadCompleted => FIRST_LOAD_COMPLETED.Task;
|
||||
|
||||
/// <summary>
|
||||
/// How long a single plugin may take to load, and again to start.
|
||||
/// </summary>
|
||||
@@ -303,33 +315,10 @@ public static partial class PluginFactory
|
||||
|
||||
var wasConfigurationChanged = hasStartedConfigurationPlugins;
|
||||
|
||||
// Check LLM providers:
|
||||
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.LLM_PROVIDER, x => x.Providers, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.LLM_PROVIDER))
|
||||
wasConfigurationChanged = true;
|
||||
|
||||
// Check transcription providers:
|
||||
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.TRANSCRIPTION_PROVIDER, x => x.TranscriptionProviders, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.TRANSCRIPTION_PROVIDER))
|
||||
wasConfigurationChanged = true;
|
||||
|
||||
// Check embedding providers:
|
||||
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.EMBEDDING_PROVIDER, x => x.EmbeddingProviders, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.EMBEDDING_PROVIDER))
|
||||
wasConfigurationChanged = true;
|
||||
|
||||
// Check data sources:
|
||||
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.DATA_SOURCE, x => x.DataSources, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList, SecretStoreType.DATA_SOURCE, deleteSecret: true))
|
||||
wasConfigurationChanged = true;
|
||||
|
||||
// Check chat templates:
|
||||
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.CHAT_TEMPLATE, x => x.ChatTemplates, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList))
|
||||
wasConfigurationChanged = true;
|
||||
|
||||
// Check profiles:
|
||||
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.PROFILE, x => x.Profiles, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList))
|
||||
wasConfigurationChanged = true;
|
||||
|
||||
// Check document analysis policies:
|
||||
if(await PluginConfigurationObject.CleanLeftOverConfigurationObjects(PluginConfigurationObjectType.DOCUMENT_ANALYSIS_POLICY, x => x.DocumentAnalysis.Policies, AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList))
|
||||
wasConfigurationChanged = true;
|
||||
// Check configured objects such as LLM providers, chat templates, or data sources:
|
||||
foreach (var descriptor in PluginConfigurationObjectDescriptors.ALL)
|
||||
if(await descriptor.CleanLeftOverAsync(AVAILABLE_PLUGINS, deployedEnterpriseConfigPluginIds, notStartedConfigPluginIds, configObjectList))
|
||||
wasConfigurationChanged = true;
|
||||
|
||||
//
|
||||
// Check left-over mandatory info acceptances. The mandatory infos come from the running
|
||||
@@ -372,6 +361,8 @@ public static partial class PluginFactory
|
||||
await SettingsManagerAccess.StoreSettings();
|
||||
await MessageBus.INSTANCE.SendMessage<bool>(null, Event.CONFIGURATION_CHANGED);
|
||||
}
|
||||
|
||||
FIRST_LOAD_COMPLETED.TrySetResult();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -0,0 +1,3 @@
|
||||
namespace AIStudio.Tools.Rust;
|
||||
|
||||
public sealed record AppRestartResponse(bool Success, string ErrorMessage);
|
||||
@@ -1,3 +1,4 @@
|
||||
using AIStudio.Tools.DataBackup;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
// ReSharper disable MemberCanBePrivate.Global
|
||||
@@ -104,6 +105,7 @@ public static class FileTypes
|
||||
public static readonly FileTypeFilter EXECUTABLES = FileTypeFilter.Leaf(TB("Executable"), "exe", "app", "bin", "appimage");
|
||||
public static readonly FileTypeFilter SHORTCUT = FileTypeFilter.Leaf(TB("Shortcut"), "lnk");
|
||||
public static readonly FileTypeFilter PLUGIN_ARCHIVE = FileTypeFilter.Leaf(TB("Plugin archive"), PluginArchive.PLUGIN_FILE_EXTENSION.TrimStart('.'), "zip");
|
||||
public static readonly FileTypeFilter BACKUP_ARCHIVE = FileTypeFilter.Leaf(TB("AI Studio backup"), DataBackupFormat.FILE_EXTENSION.TrimStart('.'), "zip");
|
||||
|
||||
/// <summary>
|
||||
/// The file types AI Studio converts using Pandoc.
|
||||
|
||||
@@ -0,0 +1,280 @@
|
||||
using System.Reflection;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools.AIJobs;
|
||||
using AIStudio.Tools.Databases;
|
||||
using AIStudio.Tools.Databases.IndexStore;
|
||||
using AIStudio.Tools.Databases.VectorStore;
|
||||
using AIStudio.Tools.DataBackup;
|
||||
using AIStudio.Tools.Media;
|
||||
using AIStudio.Tools.Metadata;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Backs up all data of AI Studio into one file, one backup at a time.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The backup runs here rather than in the component which started it, so leaving the page does not
|
||||
/// stop it, and a schedule can start it later on as well. The components only watch: they read the
|
||||
/// state and hear about every change through StateChanged.<br/><br/>
|
||||
///
|
||||
/// The service enforces what the organization allows on its own, since a component may show an
|
||||
/// outdated state, and a schedule has no component at all.
|
||||
/// </remarks>
|
||||
public sealed class DataBackupService(ILogger<DataBackupService> logger, SettingsManager settingsManager, DataSourceEmbeddingService embeddingService, DatabaseClientProvider databaseClientProvider, AIJobService aiJobService, MediaTranscriptionService mediaTranscriptionService)
|
||||
{
|
||||
private const int BUFFER_SIZE = 64 * 1024;
|
||||
|
||||
/// <summary>
|
||||
/// ERROR_FILE_TOO_LARGE of Windows, as an HRESULT.
|
||||
/// </summary>
|
||||
private const int WINDOWS_FILE_TOO_LARGE = unchecked((int)0x800700DF);
|
||||
|
||||
/// <summary>
|
||||
/// ERROR_FILE_SYSTEM_LIMITATION of Windows, as an HRESULT.
|
||||
/// </summary>
|
||||
private const int WINDOWS_FILE_SYSTEM_LIMITATION = unchecked((int)0x80070299);
|
||||
|
||||
/// <summary>
|
||||
/// EFBIG of Linux and macOS, which .NET passes on as the HResult.
|
||||
/// </summary>
|
||||
private const int UNIX_FILE_TOO_LARGE = 27;
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataBackupService).Namespace, nameof(DataBackupService));
|
||||
|
||||
private readonly Lock stateLock = new();
|
||||
|
||||
private CancellationTokenSource? running;
|
||||
|
||||
/// <summary>
|
||||
/// Fires whenever the state changes: a backup starts, makes progress, or ends.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// It fires on whatever thread the backup runs. A component hands it over to its own context.
|
||||
/// </remarks>
|
||||
public event Action? StateChanged;
|
||||
|
||||
/// <summary>
|
||||
/// Whether a backup runs.
|
||||
/// </summary>
|
||||
public bool IsRunning
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (this.stateLock)
|
||||
return this.running is not null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// How far the running backup got.
|
||||
/// </summary>
|
||||
public DataBackupProgress Progress { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// How the last backup ended, or null when none ended yet.
|
||||
/// </summary>
|
||||
public DataBackupResult? LastResult { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Backs up all data into the given file.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The file is written in place; a file which was there before is replaced. A backup which fails
|
||||
/// or is cancelled removes what it wrote.
|
||||
/// </remarks>
|
||||
/// <param name="archivePath">The file of the backup, as the user chose it.</param>
|
||||
/// <returns>How the backup ended.</returns>
|
||||
public async Task<DataBackupResult> CreateBackupAsync(string archivePath)
|
||||
{
|
||||
if (!settingsManager.ConfigurationData.App.AllowUserToBackUpData)
|
||||
return this.Refuse(archivePath, TB("Your organization does not allow backups."));
|
||||
|
||||
//
|
||||
// A chat which is being written while it is copied may end up broken in the backup. An
|
||||
// answer or a transcript stores its chat once it is done, so the backup waits for them:
|
||||
//
|
||||
if (aiJobService.HasActiveJobs || mediaTranscriptionService.GetSnapshots().Any(snapshot => snapshot.Status is MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING))
|
||||
return this.Refuse(archivePath, TB("AI Studio is still writing an answer or a transcript. Please start the backup once it is done."));
|
||||
|
||||
CancellationTokenSource cancellation;
|
||||
lock (this.stateLock)
|
||||
{
|
||||
if (this.running is not null)
|
||||
return new(false, false, archivePath, 0, 0, false, TB("A backup is running already."));
|
||||
|
||||
cancellation = this.running = new();
|
||||
}
|
||||
|
||||
this.ReportProgress(new(DataBackupPhase.PREPARING, 0, 0));
|
||||
var indexCopiesDirectory = string.Empty;
|
||||
var token = cancellation.Token;
|
||||
try
|
||||
{
|
||||
var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet.");
|
||||
var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet.");
|
||||
var settings = DataBackupSettings.CopyWithoutPluginObjects(settingsManager.ConfigurationData);
|
||||
var settingsJson = JsonSerializer.Serialize(settings, SettingsManager.JSON_OPTIONS);
|
||||
var organizationPluginFolders = DataBackupContents.FindOrganizationPluginsAmongUserPlugins(PluginFactory.AvailablePlugins.OfType<IAvailablePlugin>(), dataDirectory);
|
||||
|
||||
indexCopiesDirectory = await this.CopyIndexesAsync(token);
|
||||
|
||||
var sources = new DataBackupSources(dataDirectory, configDirectory, settingsJson, indexCopiesDirectory, organizationPluginFolders);
|
||||
var manifest = new DataBackupManifest(
|
||||
DataBackupFormat.FORMAT_VERSION,
|
||||
Assembly.GetExecutingAssembly().GetCustomAttribute<MetaDataAttribute>()?.Version ?? "unknown",
|
||||
settings.Version.ToString(),
|
||||
DateTimeOffset.UtcNow,
|
||||
dataDirectory,
|
||||
configDirectory,
|
||||
!string.IsNullOrWhiteSpace(indexCopiesDirectory),
|
||||
[]);
|
||||
|
||||
var progress = new Progress<DataBackupProgress>(this.ReportProgress);
|
||||
await using (var archiveStream = new FileStream(archivePath, FileMode.Create, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
manifest = await Task.Run(() => DataBackupWriter.WriteAsync(archiveStream, sources, manifest, progress, token), token);
|
||||
|
||||
logger.LogInformation("Backed up {FileCount} files with {Megabytes:F1} MB into '{ArchivePath}'. The indexes of the data sources are included: {IncludesIndexes}.", manifest.Entries.Count, manifest.Entries.Sum(entry => entry.Size) / (1024d * 1024d), archivePath, manifest.IncludesDataSourceIndexes);
|
||||
return this.End(new(true, false, archivePath, manifest.Entries.Count, manifest.Entries.Sum(entry => entry.Size), manifest.IncludesDataSourceIndexes, string.Empty));
|
||||
}
|
||||
catch (OperationCanceledException) when (token.IsCancellationRequested)
|
||||
{
|
||||
this.TryDeleteArchive(archivePath);
|
||||
logger.LogInformation("The user cancelled the backup into '{ArchivePath}'. The incomplete backup was removed.", archivePath);
|
||||
return this.End(new(false, true, archivePath, 0, 0, false, string.Empty));
|
||||
}
|
||||
catch (IOException exception) when (IsFileTooLarge(exception))
|
||||
{
|
||||
this.TryDeleteArchive(archivePath);
|
||||
logger.LogError(exception, "The backup into '{ArchivePath}' failed, since the file system takes no file this large. The incomplete backup was removed.", archivePath);
|
||||
return this.End(new(false, false, archivePath, 0, 0, false, TB("The backup is too large for the chosen drive, which takes no files beyond 4 GB. USB sticks formatted with FAT32 are such drives. Please choose another drive.")));
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
this.TryDeleteArchive(archivePath);
|
||||
logger.LogError(exception, "The backup into '{ArchivePath}' failed. The incomplete backup was removed.", archivePath);
|
||||
return this.End(new(false, false, archivePath, 0, 0, false, string.Format(TB("The backup failed: {0}"), exception.Message)));
|
||||
}
|
||||
finally
|
||||
{
|
||||
this.TryDeleteIndexCopies(indexCopiesDirectory);
|
||||
lock (this.stateLock)
|
||||
this.running = null;
|
||||
|
||||
cancellation.Dispose();
|
||||
this.StateChanged?.Invoke();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether writing failed since the file system takes no file this large, as FAT32 does beyond 4 GiB.
|
||||
/// </summary>
|
||||
private static bool IsFileTooLarge(IOException exception) => exception.HResult is WINDOWS_FILE_TOO_LARGE or WINDOWS_FILE_SYSTEM_LIMITATION or UNIX_FILE_TOO_LARGE;
|
||||
|
||||
/// <summary>
|
||||
/// Stops the running backup. It removes what it wrote so far.
|
||||
/// </summary>
|
||||
public void Cancel()
|
||||
{
|
||||
lock (this.stateLock)
|
||||
this.running?.Cancel();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Copies both indexes of the data sources, while the indexing pauses.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Both copies are made during one pause: the vector stores have to fit to the index, or a
|
||||
/// restore would leave embeddings which belong to nothing, or files which count as embedded
|
||||
/// without their embeddings. Writing the backup afterward takes far longer, so the indexing
|
||||
/// goes on meanwhile.<br/><br/>
|
||||
///
|
||||
/// Without either database, the backup leaves both indexes out, and its manifest says so.
|
||||
/// </remarks>
|
||||
/// <returns>The directory of the copies, in the layout of the data folder, or an empty string when the backup leaves the indexes out.</returns>
|
||||
private async Task<string> CopyIndexesAsync(CancellationToken token)
|
||||
{
|
||||
var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
|
||||
var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token);
|
||||
if (indexStore is NoIndexStoreClient || vectorStore is NoVectorStoreClient)
|
||||
{
|
||||
logger.LogWarning("The backup leaves the indexes of the data sources out, because the index store or the vector store is not available.");
|
||||
return string.Empty;
|
||||
}
|
||||
|
||||
await using var pause = await embeddingService.PauseIndexingAsync(token);
|
||||
var preparation = await vectorStore.PrepareBackupAsync(token);
|
||||
try
|
||||
{
|
||||
var indexEntry = DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX);
|
||||
await indexStore.BackupToDirectoryAsync(Path.Join(preparation.StagingPath, indexEntry.RelativePath.Replace(DataBackupContents.SEPARATOR, Path.DirectorySeparatorChar)), token);
|
||||
|
||||
for (var index = 0; index < preparation.Stores.Count; index++)
|
||||
{
|
||||
this.ReportProgress(new(DataBackupPhase.COPYING_INDEXES, index, preparation.Stores.Count));
|
||||
await vectorStore.CopyStoreForBackupAsync(preparation.BackupId, preparation.Stores[index].StoreName, token);
|
||||
}
|
||||
|
||||
return preparation.StagingPath;
|
||||
}
|
||||
catch
|
||||
{
|
||||
this.TryDeleteIndexCopies(preparation.StagingPath);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
private DataBackupResult Refuse(string archivePath, string issue)
|
||||
{
|
||||
logger.LogWarning("Refused to start a backup into '{ArchivePath}': {Issue}", archivePath, issue);
|
||||
return this.End(new(false, false, archivePath, 0, 0, false, issue));
|
||||
}
|
||||
|
||||
private DataBackupResult End(DataBackupResult result)
|
||||
{
|
||||
this.LastResult = result;
|
||||
this.StateChanged?.Invoke();
|
||||
return result;
|
||||
}
|
||||
|
||||
private void ReportProgress(DataBackupProgress progress)
|
||||
{
|
||||
this.Progress = progress;
|
||||
this.StateChanged?.Invoke();
|
||||
}
|
||||
|
||||
private void TryDeleteArchive(string archivePath)
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(archivePath))
|
||||
File.Delete(archivePath);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not remove the incomplete backup '{ArchivePath}'.", archivePath);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the copies of the indexes. Should that fail, the next start removes them.
|
||||
/// </summary>
|
||||
private void TryDeleteIndexCopies(string indexCopiesDirectory)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(indexCopiesDirectory))
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
if (Directory.Exists(indexCopiesDirectory))
|
||||
Directory.Delete(indexCopiesDirectory, true);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Could not remove the copies of the indexes at '{IndexCopiesDirectory}'.", indexCopiesDirectory);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using AIStudio.Settings;
|
||||
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
/// <summary>
|
||||
/// One-shot startup service which picks up what a restore or a backup left behind: it learns how the
|
||||
/// runtime applied a restore, removes what a backup or a preparation left halfway, and removes the data
|
||||
/// a restore replaced once the restored settings loaded.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The report of the runtime waits in DataRestoreService until the main layout shows it, once the
|
||||
/// plugins started: only then do the providers of the organization exist, and the open points name
|
||||
/// none of them.
|
||||
/// </remarks>
|
||||
public sealed class DataRestoreFollowUpService(ILogger<DataRestoreFollowUpService> logger, SettingsManager settingsManager, DataRestoreService dataRestoreService) : BackgroundService
|
||||
{
|
||||
#region Overrides of BackgroundService
|
||||
|
||||
/// <summary>Waits for the folders and the settings, picks up what the last start left, and then exits.</summary>
|
||||
/// <param name="stoppingToken">Host shutdown token.</param>
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while ((string.IsNullOrWhiteSpace(SettingsManager.DataDirectory) || string.IsNullOrWhiteSpace(SettingsManager.ConfigDirectory)) && !stoppingToken.IsCancellationRequested)
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken);
|
||||
|
||||
await dataRestoreService.LoadStartupStateAsync(stoppingToken);
|
||||
|
||||
while (!settingsManager.HasCompletedInitialSettingsLoad && !stoppingToken.IsCancellationRequested)
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken);
|
||||
|
||||
//
|
||||
// Settings which could not be loaded leave the replaced data in place: they hold the settings
|
||||
// which worked before the restore.
|
||||
//
|
||||
if (settingsManager.SettingsWriteBlocked)
|
||||
{
|
||||
logger.LogWarning("The settings could not be loaded safely. The data which a restore replaced stay where they are.");
|
||||
return;
|
||||
}
|
||||
|
||||
dataRestoreService.RemoveReplacedData();
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -0,0 +1,385 @@
|
||||
using System.Reflection;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools.AIJobs;
|
||||
using AIStudio.Tools.DataBackup;
|
||||
using AIStudio.Tools.Media;
|
||||
using AIStudio.Tools.Metadata;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.ToolCallingSystem;
|
||||
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
/// <summary>
|
||||
/// Prepares the restore of a backup, which the next start of AI Studio applies.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A restore replaces all data, so it runs only while nothing else changes it: no answer, no
|
||||
/// transcript, and no backup may be running, and the indexing pauses from the start of the preparation
|
||||
/// until the restart. Whatever changes after the preparation is lost with the restart, which is why
|
||||
/// the app asks for the restart right away, and offers to discard the restore instead.<br/><br/>
|
||||
///
|
||||
/// The service enforces what the organization allows on its own, since a component may show an
|
||||
/// outdated state.<br/><br/>
|
||||
///
|
||||
/// At the start which applied a restore, the service holds the report of the runtime until the user
|
||||
/// saw it, cf. DataRestoreFollowUpService.
|
||||
/// </remarks>
|
||||
public sealed class DataRestoreService(ILogger<DataRestoreService> logger, SettingsManager settingsManager, RustService rustService, ToolRegistry toolRegistry, ToolSettingsService toolSettingsService, DataSourceEmbeddingService embeddingService, DataBackupService dataBackupService, AIJobService aiJobService, MediaTranscriptionService mediaTranscriptionService)
|
||||
{
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataRestoreService).Namespace, nameof(DataRestoreService));
|
||||
|
||||
private readonly Lock stateLock = new();
|
||||
|
||||
private readonly TaskCompletionSource startupStateLoaded = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
private CancellationTokenSource? preparing;
|
||||
|
||||
/// <summary>
|
||||
/// Holds the indexing back while a restore is prepared or waits for the restart.
|
||||
/// </summary>
|
||||
private IAsyncDisposable? indexingPause;
|
||||
|
||||
/// <summary>
|
||||
/// Fires whenever the state changes: a preparation starts, makes progress, or ends, or a prepared restore is discarded.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// It fires on whatever thread the preparation runs. A component hands it over to its own context.
|
||||
/// </remarks>
|
||||
public event Action? StateChanged;
|
||||
|
||||
/// <summary>
|
||||
/// Whether a restore is being prepared.
|
||||
/// </summary>
|
||||
public bool IsPreparing
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (this.stateLock)
|
||||
return this.preparing is not null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a restore is prepared and waits for the restart.
|
||||
/// </summary>
|
||||
public bool IsRestorePending { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// How far the running preparation got.
|
||||
/// </summary>
|
||||
public DataRestoreProgress Progress { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// How the last preparation ended, or null when none ended yet.
|
||||
/// </summary>
|
||||
public DataRestoreResult? LastResult { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// How the runtime applied a restore at this start, until the user saw it; otherwise null.
|
||||
/// </summary>
|
||||
public DataRestoreReport? Report { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the runtime has not finished a restore and continues at the next start, cf.
|
||||
/// DataRestoreFollowUp.IsRestoreUnfinished. Until then, AI Studio neither prepares nor discards a
|
||||
/// restore.
|
||||
/// </summary>
|
||||
public bool IsRestoreUnfinished { get; private set; }
|
||||
|
||||
/// <summary>
|
||||
/// Completes once the service knows what the last start left behind, cf. LoadStartupStateAsync.
|
||||
/// </summary>
|
||||
public Task StartupStateLoaded => this.startupStateLoaded.Task;
|
||||
|
||||
/// <summary>
|
||||
/// Learns at the start what the runtime and the last run left behind: the report of a restore, a
|
||||
/// restore which still waits for a restart, and what a backup or a preparation left halfway.
|
||||
/// </summary>
|
||||
/// <param name="token">Cancels reading the report.</param>
|
||||
public async Task LoadStartupStateAsync(CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet.");
|
||||
var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet.");
|
||||
try
|
||||
{
|
||||
this.Report = await DataRestoreFollowUp.ReadReportAsync(dataDirectory, token);
|
||||
if (this.Report is not null)
|
||||
logger.LogInformation("The runtime applied a restore at this start: {Outcome}. {Detail}", this.Report.Outcome, this.Report.Detail);
|
||||
}
|
||||
catch (Exception exception) when (exception is JsonException or IOException)
|
||||
{
|
||||
logger.LogError(exception, "Could not read how the runtime applied a restore.");
|
||||
DataRestoreFollowUp.RemoveReport(dataDirectory);
|
||||
}
|
||||
|
||||
this.IsRestoreUnfinished = DataRestoreFollowUp.IsRestoreUnfinished(dataDirectory);
|
||||
if (this.IsRestoreUnfinished)
|
||||
logger.LogError("The runtime has not finished a restore. It tries to move the replaced data back at the next start.");
|
||||
|
||||
if (!dataBackupService.IsRunning && !this.IsPreparing)
|
||||
DataRestoreFollowUp.RemoveLeftovers(dataDirectory, configDirectory);
|
||||
|
||||
//
|
||||
// A restore which still waits for its start, e.g., in the development environment, where the
|
||||
// .NET server may restart without the runtime: whatever changes until then is lost.
|
||||
//
|
||||
if (DataRestoreFollowUp.IsRestorePending(dataDirectory))
|
||||
{
|
||||
logger.LogWarning("A prepared restore still waits for the restart which applies it.");
|
||||
this.IsRestorePending = true;
|
||||
this.indexingPause ??= await embeddingService.PauseIndexingAsync(token);
|
||||
}
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Could not learn what the last start left behind of a restore or a backup.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
this.startupStateLoaded.TrySetResult();
|
||||
this.StateChanged?.Invoke();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Removes the data which a restore replaced. Call it once the restored settings loaded.
|
||||
/// </summary>
|
||||
public void RemoveReplacedData()
|
||||
{
|
||||
try
|
||||
{
|
||||
var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet.");
|
||||
var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet.");
|
||||
if (DataRestoreFollowUp.RemoveReplacedData(dataDirectory, configDirectory))
|
||||
logger.LogInformation("Removed the data which the last restore replaced.");
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Could not remove the data which the last restore replaced.");
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds what the user still has to do after the restore of this start, e.g., enter API keys again.
|
||||
/// </summary>
|
||||
/// <returns>The open points; none when no restore was applied.</returns>
|
||||
public async Task<IReadOnlyList<DataRestoreOpenPoint>> CollectOpenPointsAsync()
|
||||
{
|
||||
if (this.Report?.Outcome is not DataRestoreOutcome.APPLIED)
|
||||
return [];
|
||||
|
||||
try
|
||||
{
|
||||
return await DataRestoreOpenPoints.CollectAsync(settingsManager.ConfigurationData, new RustStoredSecrets(rustService), this.FindToolSecrets());
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Could not find what remains to do after the restore.");
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the secret settings of the tools which the user set up, i.e., saved the settings of.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A setting the organization fixed comes from its configuration plugin, and one the tool does
|
||||
/// not show for the saved settings means nothing for them. Which of the remaining ones held a
|
||||
/// value nobody knows: a backup holds no secrets, so the dialog asks to enter them again if they
|
||||
/// were used.
|
||||
/// </remarks>
|
||||
private List<DataRestoreToolSecret> FindToolSecrets()
|
||||
{
|
||||
var toolSecrets = new List<DataRestoreToolSecret>();
|
||||
foreach (var definition in toolRegistry.GetAllDefinitions())
|
||||
{
|
||||
if (toolRegistry.GetImplementation(definition.ImplementationKey) is not { } implementation)
|
||||
continue;
|
||||
|
||||
if (!settingsManager.ConfigurationData.Tools.Settings.TryGetValue(definition.Id, out var storedValues))
|
||||
continue;
|
||||
|
||||
foreach (var (fieldName, field) in definition.SettingsSchema.Properties)
|
||||
if (field.Secret && !toolSettingsService.IsFieldLocked(definition, fieldName) && implementation.IsSettingsFieldVisible(fieldName, storedValues))
|
||||
toolSecrets.Add(new(definition.Id, fieldName, implementation.GetDisplayName(), implementation.GetSettingsFieldLabel(fieldName, field)));
|
||||
}
|
||||
|
||||
return toolSecrets;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Forgets the report of the runtime, once the user saw it.
|
||||
/// </summary>
|
||||
public void ForgetReport()
|
||||
{
|
||||
this.Report = null;
|
||||
try
|
||||
{
|
||||
DataRestoreFollowUp.RemoveReport(SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet."));
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Could not remove the report of the restore.");
|
||||
}
|
||||
|
||||
this.StateChanged?.Invoke();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Prepares the restore of a backup for the next start.
|
||||
/// </summary>
|
||||
/// <param name="archivePath">The backup.</param>
|
||||
/// <returns>Whether the restore is prepared, or why not.</returns>
|
||||
public async Task<DataRestoreResult> PrepareRestoreAsync(string archivePath)
|
||||
{
|
||||
if (!settingsManager.ConfigurationData.App.AllowUserToRestoreData)
|
||||
return this.Refuse(TB("Your organization does not allow restoring backups."));
|
||||
|
||||
if (dataBackupService.IsRunning)
|
||||
return this.Refuse(TB("A backup is running. Please restore once it is done."));
|
||||
|
||||
if (this.IsRestoreUnfinished)
|
||||
return this.Refuse(TB("AI Studio has not finished restoring an earlier backup yet. Please restart AI Studio first."));
|
||||
|
||||
if (aiJobService.HasActiveJobs || mediaTranscriptionService.GetSnapshots().Any(snapshot => snapshot.Status is MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING))
|
||||
return this.Refuse(TB("AI Studio is still writing an answer or a transcript. Please restore once it is done."));
|
||||
|
||||
CancellationTokenSource cancellation;
|
||||
lock (this.stateLock)
|
||||
{
|
||||
if (this.preparing is not null)
|
||||
return new(false, false, TB("A restore is being prepared already."));
|
||||
|
||||
cancellation = this.preparing = new();
|
||||
}
|
||||
|
||||
this.ReportProgress(new(0, 0));
|
||||
var token = cancellation.Token;
|
||||
try
|
||||
{
|
||||
var target = this.CreateTarget();
|
||||
this.indexingPause ??= await embeddingService.PauseIndexingAsync(token);
|
||||
|
||||
var progress = new Progress<DataRestoreProgress>(this.ReportProgress);
|
||||
var result = await Task.Run(() => DataRestorePreparer.PrepareAsync(archivePath, target, progress, token), token);
|
||||
this.IsRestorePending = result.Success;
|
||||
if (result.Success)
|
||||
logger.LogInformation("Prepared the restore of the backup '{ArchivePath}'. It is applied at the next start.", archivePath);
|
||||
else
|
||||
logger.LogWarning("Could not prepare the restore of the backup '{ArchivePath}': {Issue}", archivePath, result.Issue);
|
||||
|
||||
return this.End(result);
|
||||
}
|
||||
catch (OperationCanceledException) when (token.IsCancellationRequested)
|
||||
{
|
||||
logger.LogInformation("The user cancelled preparing the restore of the backup '{ArchivePath}'.", archivePath);
|
||||
return this.End(new(false, true, string.Empty));
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
// The preparation removes what it unpacked when it fails:
|
||||
logger.LogError(exception, "Preparing the restore of the backup '{ArchivePath}' failed.", archivePath);
|
||||
return this.End(new(false, false, string.Format(TB("The restore could not be prepared: {0}"), exception.Message)));
|
||||
}
|
||||
finally
|
||||
{
|
||||
lock (this.stateLock)
|
||||
this.preparing = null;
|
||||
|
||||
cancellation.Dispose();
|
||||
if (!this.IsRestorePending)
|
||||
await this.EndIndexingPauseAsync();
|
||||
|
||||
this.StateChanged?.Invoke();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Stops the running preparation. It removes what it unpacked so far.
|
||||
/// </summary>
|
||||
public void Cancel()
|
||||
{
|
||||
lock (this.stateLock)
|
||||
this.preparing?.Cancel();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Discards a prepared restore, so that the next start leaves all data as it is.
|
||||
/// </summary>
|
||||
public async Task DiscardPendingRestoreAsync()
|
||||
{
|
||||
if (this.IsPreparing || this.IsRestoreUnfinished)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
var target = this.CreateTarget();
|
||||
try
|
||||
{
|
||||
DataRestorePreparer.Discard(target);
|
||||
logger.LogInformation("Discarded the prepared restore.");
|
||||
}
|
||||
finally
|
||||
{
|
||||
// What remains on the disk decides, should discarding have failed halfway:
|
||||
this.IsRestorePending = DataRestorePreparer.IsPrepared(target);
|
||||
}
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Could not discard the prepared restore.");
|
||||
}
|
||||
|
||||
// A restore which the next start still applies keeps the indexing paused:
|
||||
if (!this.IsRestorePending)
|
||||
await this.EndIndexingPauseAsync();
|
||||
|
||||
this.StateChanged?.Invoke();
|
||||
}
|
||||
|
||||
private DataRestoreTarget CreateTarget()
|
||||
{
|
||||
var dataDirectory = SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet.");
|
||||
var configDirectory = SettingsManager.ConfigDirectory ?? throw new InvalidOperationException("The configuration directory is not known yet.");
|
||||
var appVersion = Assembly.GetExecutingAssembly().GetCustomAttribute<MetaDataAttribute>()?.Version ?? string.Empty;
|
||||
var organizationPluginFolders = DataBackupContents.FindOrganizationPluginsAmongUserPlugins(PluginFactory.AvailablePlugins.OfType<IAvailablePlugin>(), dataDirectory);
|
||||
|
||||
// Loading the settings brings them to the version this AI Studio writes:
|
||||
return new(dataDirectory, configDirectory, appVersion, settingsManager.ConfigurationData.Version, organizationPluginFolders);
|
||||
}
|
||||
|
||||
private async Task EndIndexingPauseAsync()
|
||||
{
|
||||
var pause = this.indexingPause;
|
||||
this.indexingPause = null;
|
||||
if (pause is not null)
|
||||
await pause.DisposeAsync();
|
||||
}
|
||||
|
||||
private DataRestoreResult Refuse(string issue)
|
||||
{
|
||||
logger.LogWarning("Refused to prepare a restore: {Issue}", issue);
|
||||
return this.End(new(false, false, issue));
|
||||
}
|
||||
|
||||
private DataRestoreResult End(DataRestoreResult result)
|
||||
{
|
||||
this.LastResult = result;
|
||||
this.StateChanged?.Invoke();
|
||||
return result;
|
||||
}
|
||||
|
||||
private void ReportProgress(DataRestoreProgress progress)
|
||||
{
|
||||
this.Progress = progress;
|
||||
this.StateChanged?.Invoke();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed partial class DataSourceEmbeddingService
|
||||
{
|
||||
/// <summary>
|
||||
/// Whether the indexing is paused right now, e.g., while a backup copies the indexes, or while a
|
||||
/// restore waits for the restart.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Deleting or repairing a data source waits for the end of a pause, so the user interface asks
|
||||
/// first and tells the user instead of waiting.
|
||||
/// </remarks>
|
||||
public bool IsIndexingPaused => this.pauses.IsPaused;
|
||||
|
||||
/// <summary>
|
||||
/// Pauses the indexing of every data source, e.g., while a backup copies the indexes.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every run stops, and the pause waits until they have, since a run may be about to write to
|
||||
/// both stores. A document a run embedded completely stays stored, and its fingerprint keeps the
|
||||
/// next run from embedding it once more. Every run asked for during the pause waits for its end,
|
||||
/// the next sync of a mailbox among them. Ending the pause queues every data source whose run it
|
||||
/// stopped, for the same reason that run was asked for.<br/><br/>
|
||||
///
|
||||
/// Pauses may overlap. The indexing goes on once the last one ended.
|
||||
/// </remarks>
|
||||
/// <param name="token">Stops waiting for the runs to stop, which ends the pause as well.</param>
|
||||
/// <returns>The pause. Disposing it ends the pause.</returns>
|
||||
public async Task<IAsyncDisposable> PauseIndexingAsync(CancellationToken token)
|
||||
{
|
||||
//
|
||||
// The pause begins before any run is stopped. A run which starts meanwhile is either among
|
||||
// the active runs below, or it finds the indexing paused and waits:
|
||||
//
|
||||
this.pauses.Begin();
|
||||
var pause = new IndexingPause(this);
|
||||
|
||||
var stoppedRuns = new List<Task>();
|
||||
foreach (var (dataSourceId, activeRun) in this.activeRuns)
|
||||
{
|
||||
if (!TryCancelRun(activeRun))
|
||||
continue;
|
||||
|
||||
this.pauses.RecordStoppedRun(dataSourceId, activeRun.RefreshMode);
|
||||
stoppedRuns.Add(activeRun.Completion.Task);
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(stoppedRuns).WaitAsync(token);
|
||||
}
|
||||
catch
|
||||
{
|
||||
await pause.DisposeAsync();
|
||||
throw;
|
||||
}
|
||||
|
||||
logger.LogInformation("Paused the indexing of all data sources. Stopped {StoppedRuns} active runs.", stoppedRuns.Count);
|
||||
return pause;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ends a pause, and once the last one ended, queues every data source whose run a pause stopped.
|
||||
/// </summary>
|
||||
private async Task EndPauseAsync()
|
||||
{
|
||||
var stoppedRuns = this.pauses.End();
|
||||
foreach (var stoppedRun in stoppedRuns)
|
||||
{
|
||||
// Deleted during the pause, so there is nothing to carry on with:
|
||||
if (!this.TryGetConfiguredIndexedSource(stoppedRun.DataSourceId, out var dataSource))
|
||||
continue;
|
||||
|
||||
await this.QueueDataSourceAsync(dataSource, true, stoppedRun.RefreshMode);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// One pause, which ends once it is disposed.
|
||||
/// </summary>
|
||||
private sealed class IndexingPause(DataSourceEmbeddingService service) : IAsyncDisposable
|
||||
{
|
||||
private int ended;
|
||||
|
||||
/// <summary>
|
||||
/// Ends the pause. Ending it again does nothing.
|
||||
/// </summary>
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (Interlocked.Exchange(ref this.ended, 1) == 1)
|
||||
return;
|
||||
|
||||
await service.EndPauseAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -36,6 +36,9 @@ public sealed partial class DataSourceEmbeddingService
|
||||
if (activeRun is not null)
|
||||
await activeRun.Completion.Task;
|
||||
|
||||
// A backup copies the stores during a pause, and the copies would no longer fit together:
|
||||
await this.pauses.WaitWhilePausedAsync(CancellationToken.None);
|
||||
|
||||
await this.ResetPersistedStateAsync(dataSourceId, null, null, CancellationToken.None);
|
||||
this.statuses.TryRemove(dataSourceId, out _);
|
||||
PublishStatusChanged();
|
||||
|
||||
@@ -27,6 +27,16 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
/// </remarks>
|
||||
private static readonly TimeSpan REINDEX_CHECK_TIMEOUT = TimeSpan.FromSeconds(2);
|
||||
|
||||
/// <summary>
|
||||
/// How long the first run waits for the plugins to be loaded, cf. WaitForFirstPluginLoadAsync.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every plugin may take ten seconds to load and ten more to start, and the configuration of an
|
||||
/// organization may have to be downloaded first. Should loading fail, the data sources are
|
||||
/// indexed all the same, with whatever the settings hold by then.
|
||||
/// </remarks>
|
||||
private static readonly TimeSpan FIRST_PLUGIN_LOAD_TIMEOUT = TimeSpan.FromMinutes(2);
|
||||
|
||||
/// <summary>
|
||||
/// One indexer per kind of data source this service indexes.
|
||||
/// </summary>
|
||||
@@ -57,7 +67,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService));
|
||||
|
||||
private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource<object?> Completion)
|
||||
/// <summary>
|
||||
/// The pauses of the indexing, cf. PauseIndexingAsync.
|
||||
/// </summary>
|
||||
private readonly IndexingPauses pauses = new();
|
||||
|
||||
/// <summary>
|
||||
/// An active run.
|
||||
/// </summary>
|
||||
/// <param name="TokenSource">Stops the run.</param>
|
||||
/// <param name="Completion">Completes once the run has ended.</param>
|
||||
/// <param name="RefreshMode">Why the run was asked for.</param>
|
||||
private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource<object?> Completion, DataSourceEmbeddingRefreshMode RefreshMode)
|
||||
{
|
||||
/// <summary>
|
||||
/// The embedding settings the run works for, as GetRunSignature describes them.
|
||||
@@ -386,11 +407,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a data source cannot be searched because its vector store cannot be read anymore.
|
||||
/// Whether a data source cannot be searched because its vector store cannot be read anymore, or is missing.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Unlike the re-index check above, this reads no database at all: the state comes from the run
|
||||
/// or the search which ran into the unreadable store, and is kept in memory only. That it does
|
||||
/// or the search which ran into the unreadable or missing store, and is kept in memory only. That it does
|
||||
/// not survive a restart is deliberate. The very same store may well open on the next start,
|
||||
/// and a mark written to disk would then be wrong with nobody noticing. Until something touches
|
||||
/// the store again, the data source counts as usable, and a failing search says so on its own.
|
||||
@@ -512,6 +533,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
await activeRun.Completion.Task;
|
||||
}
|
||||
|
||||
// A backup copies the stores during a pause, and the copies would no longer fit together:
|
||||
await this.pauses.WaitWhilePausedAsync(CancellationToken.None);
|
||||
|
||||
this.statuses.TryRemove(dataSource.Id, out _);
|
||||
await this.ResetPersistedStateAsync(dataSource.Id, null, null, CancellationToken.None);
|
||||
this.statuses.TryRemove(dataSource.Id, out _);
|
||||
@@ -633,7 +657,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
var runTokenSource = CancellationTokenSource.CreateLinkedTokenSource(parentToken);
|
||||
var runControl = new DataSourceRunControl(
|
||||
runTokenSource,
|
||||
new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously));
|
||||
new TaskCompletionSource<object?>(TaskCreationOptions.RunContinuationsAsynchronously),
|
||||
refreshMode);
|
||||
|
||||
if (!this.activeRuns.TryAdd(dataSource.Id, runControl))
|
||||
{
|
||||
@@ -652,8 +677,10 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
// One run at a time, whichever way it started: the startup check goes on next to the
|
||||
// queue once the automatic refresh is switched on while the app runs. The wait counts as
|
||||
// part of the run, so deleting or repairing the data source ends the wait as well,
|
||||
// instead of waiting for the run before it to finish first.
|
||||
// instead of waiting for the run before it to finish first. A paused indexing holds
|
||||
// every run back in front of the gate, cf. PauseIndexingAsync:
|
||||
//
|
||||
await this.pauses.WaitWhilePausedAsync(runTokenSource.Token);
|
||||
await this.runGate.WaitAsync(runTokenSource.Token);
|
||||
try
|
||||
{
|
||||
@@ -746,16 +773,27 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
var persistedState = await indexStore.GetDataSourceStateAsync(dataSource.Id, token);
|
||||
if (persistedState is { VectorSize: > 0 })
|
||||
{
|
||||
var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedState.VectorSize, token);
|
||||
if (ensureResult.Created)
|
||||
//
|
||||
// The index database knows what was indexed, but the vectors are gone, e.g. because the
|
||||
// folder of the vector database was deleted, or restored from another backup than the
|
||||
// index database. Rebuilding sends every document to the embedding provider once more,
|
||||
// which costs money and hours, so it is for the user to decide, cf. RepairDataSourceAsync.
|
||||
// Until then, the data source stays out of every search.
|
||||
//
|
||||
if (!await vectorStore.VectorStoreExists(collectionName, token))
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Vector store '{CollectionName}' for data source '{DataSourceName}' ({DataSourceId}) was missing although persisted embedding state exists. Resetting the stale state so all vectors are rebuilt.",
|
||||
"Vector store '{CollectionName}' for data source '{DataSourceName}' ({DataSourceId}) is missing although persisted embedding state exists. The data source is waiting for a repair.",
|
||||
collectionName,
|
||||
dataSource.Name,
|
||||
dataSource.Id);
|
||||
await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, indexStore, token);
|
||||
token.ThrowIfCancellationRequested();
|
||||
this.UpsertStatus(this.GetMissingVectorStoreStatus(dataSource));
|
||||
return null;
|
||||
}
|
||||
|
||||
// Opens the store, so one which cannot be read shows up before the run starts:
|
||||
await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedState.VectorSize, token);
|
||||
}
|
||||
|
||||
if (!this.TryResolveEmbeddingProvider(dataSource, out var embeddingProvider))
|
||||
@@ -925,6 +963,14 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
//
|
||||
// The embedding providers and data sources of an organization come with its configuration
|
||||
// plugin. On a new computer, or after restoring a backup, the settings do not hold them yet,
|
||||
// and the first run would find such a data source without its embedding provider:
|
||||
//
|
||||
if (!await WaitForFirstPluginLoadAsync(PluginFactory.FirstLoadCompleted, FIRST_PLUGIN_LOAD_TIMEOUT, token))
|
||||
logger.LogWarning("The plugins were not loaded within {TimeoutSeconds:F0} seconds. Indexing starts with the settings as they are, without what the configuration of an organization may add later.", FIRST_PLUGIN_LOAD_TIMEOUT.TotalSeconds);
|
||||
|
||||
// Before the first run: from then on, a sync may be writing an attachment there.
|
||||
MailAttachmentFiles.DeleteLeftovers(logger);
|
||||
await this.DeleteOrphanedMailboxIndexesAsync(token);
|
||||
@@ -933,6 +979,26 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
await this.RunInitialDataSourceHashCheckAsync(token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Waits until the plugins were loaded for the first time, but not longer than the given time.
|
||||
/// </summary>
|
||||
/// <param name="firstLoad">Completes once the plugins were loaded, cf. PluginFactory.FirstLoadCompleted.</param>
|
||||
/// <param name="timeout">How long to wait at most.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>True when the plugins were loaded in time, false when the time ran out.</returns>
|
||||
internal static async Task<bool> WaitForFirstPluginLoadAsync(Task firstLoad, TimeSpan timeout, CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
await firstLoad.WaitAsync(timeout, token);
|
||||
return true;
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task RunInitialDataSourceHashCheckAsync(CancellationToken token)
|
||||
{
|
||||
if (!settingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh)
|
||||
@@ -1230,9 +1296,23 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
/// path, is written in English for the log file, and says nothing about what happens next. What
|
||||
/// the user needs to read is what this means for their chats and where the way out is.
|
||||
/// </remarks>
|
||||
private DataSourceEmbeddingStatus GetUnreadableVectorStoreStatus(IDataSourceBase dataSource)
|
||||
private DataSourceEmbeddingStatus GetUnreadableVectorStoreStatus(IDataSourceBase dataSource) =>
|
||||
this.GetRepairStatus(dataSource, string.Format(TB("The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name));
|
||||
|
||||
/// <remarks>
|
||||
/// The same way out as for a store which cannot be read, cf. GetUnreadableVectorStoreStatus.
|
||||
/// </remarks>
|
||||
private DataSourceEmbeddingStatus GetMissingVectorStoreStatus(IDataSourceBase dataSource) =>
|
||||
this.GetRepairStatus(dataSource, string.Format(TB("The index of the data source '{0}' is missing. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name));
|
||||
|
||||
/// <summary>
|
||||
/// Shows a data source as waiting for the user to have its index built anew, cf. NeedsIndexRepair.
|
||||
/// </summary>
|
||||
/// <param name="dataSource">The data source.</param>
|
||||
/// <param name="errorMessage">What is wrong with the index, and where the way out is.</param>
|
||||
/// <returns>The failed status, which offers the repair.</returns>
|
||||
private DataSourceEmbeddingStatus GetRepairStatus(IDataSourceBase dataSource, string errorMessage)
|
||||
{
|
||||
var errorMessage = string.Format(TB("The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name);
|
||||
return this.CreateStatus(
|
||||
dataSource,
|
||||
DataSourceEmbeddingState.FAILED,
|
||||
|
||||
@@ -12,7 +12,8 @@ namespace AIStudio.Tools.Services;
|
||||
/// document has pages to count. Block numbers start at one, the way the page states them.
|
||||
///
|
||||
/// VectorStoreUnreadable says why a data source failed, not only that it did. The UI needs that
|
||||
/// difference to offer the repair for this one case, and it is carried as its own flag so nothing
|
||||
/// difference to offer the repair for this one case: the vector store cannot be read, or it is
|
||||
/// missing although the index database knows what it held. It is carried as its own flag so nothing
|
||||
/// has to read it back out of the message in LastError. Attention does the same for what the user
|
||||
/// has to decide, and PendingRemovalCount is the number a held back removal asks about: agreeing to
|
||||
/// it hands that very number back, cf. ApprovePendingMailRemovalAsync.
|
||||
|
||||
@@ -10,10 +10,26 @@ public sealed class FileEnumerationResult
|
||||
|
||||
public string LastError { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The folders whose content could not be listed. The files below them are out of reach, not gone.
|
||||
/// </summary>
|
||||
public List<string> UnreadableDirectories { get; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Why the file or the folder of the data source itself could not be reached, or null when it could.
|
||||
/// </summary>
|
||||
public string? UnreachableRootReason { get; set; }
|
||||
|
||||
public void AddFailure(string filePath, string reason)
|
||||
{
|
||||
this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason, DateTimeOffset.UtcNow));
|
||||
this.FailedFiles = this.Failures.Count;
|
||||
this.LastError = reason;
|
||||
}
|
||||
|
||||
public void AddUnreadableDirectory(string directoryPath, string reason)
|
||||
{
|
||||
this.UnreadableDirectories.Add(directoryPath);
|
||||
this.AddFailure(directoryPath, reason);
|
||||
}
|
||||
}
|
||||
@@ -67,6 +67,12 @@ Rules which are easy to break:
|
||||
data source selection that a run got through the whole data source (`IsIndexAwaitingRebuild`). Files pass
|
||||
`null` to `PauseRunAsync`, so a folder stays out of the selection until its first indexing or its rebuild is
|
||||
complete. The mailbox passes its hash, since every mail it indexed can be searched right away.
|
||||
- **Every run passes the pause.** A backup pauses the indexing with `PauseIndexingAsync` while it copies the
|
||||
stores. Runs wait for the end of a pause in `ProcessDataSourceRunAsync`, in front of the run gate, so start
|
||||
every run there. A run which starts any other way writes to the stores while they are copied, and the backup
|
||||
holds two stores which do not fit together. Discarding the stored index of a data source, when it is deleted
|
||||
or repaired, waits for the end of a pause as well (`WaitWhilePausedAsync`); the user interface asks
|
||||
`IsIndexingPaused` first and tells the user instead of waiting.
|
||||
- **A pass lives only while the last step left more to do.** The service drops the pass of a data source after
|
||||
every other outcome and every exception, and when the stored index is reset. A retry the user asks for, and a
|
||||
step with other embedding settings, begin a new pass (`IndexingPass.ContinueOrBegin`). A pass kept beyond that
|
||||
|
||||
@@ -49,6 +49,29 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DocumentRunProgress).Namespace, nameof(DocumentRunProgress));
|
||||
|
||||
/// <summary>
|
||||
/// Makes the progress of a run which reads nothing, showing the data source as the index holds it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// For a run which stops before it looks at a single document, e.g. because the server of a
|
||||
/// mailbox or the folder of a data source cannot be reached. The documents skipped for good keep
|
||||
/// their place in the list, so the user still sees why.
|
||||
/// </remarks>
|
||||
/// <param name="context">The run.</param>
|
||||
/// <param name="indexedDocuments">How many documents the index holds.</param>
|
||||
/// <param name="logger">The logger of the embedding service.</param>
|
||||
/// <returns>The progress, ready to publish the stored state or why the run stopped.</returns>
|
||||
public static DocumentRunProgress FromStoredState(IndexedRunContext context, int indexedDocuments, ILogger logger)
|
||||
{
|
||||
var permanentFailures = context.Manifest.PermanentFailures;
|
||||
var progress = new DocumentRunProgress(context, indexedDocuments + permanentFailures.Count, 0, string.Empty, [], logger);
|
||||
progress.RecordUnchanged(indexedDocuments);
|
||||
foreach (var (key, failure) in permanentFailures)
|
||||
progress.RecordStillUnreadable(key, failure);
|
||||
|
||||
return progress;
|
||||
}
|
||||
|
||||
public int TotalDocuments => totalDocuments;
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -68,6 +68,9 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
var enumerationStartedUtc = DateTimeOffset.UtcNow;
|
||||
var inputFiles = this.GetInputFiles(dataSource);
|
||||
var enumerationDuration = DateTimeOffset.UtcNow - enumerationStartedUtc;
|
||||
if (inputFiles.UnreachableRootReason is { } unreachableRootReason)
|
||||
return this.StopWithoutRoot(context, dataSource, unreachableRootReason);
|
||||
|
||||
var indexedFiles = inputFiles.Files;
|
||||
var totalFiles = indexedFiles.Count + inputFiles.FailedFiles;
|
||||
|
||||
@@ -95,7 +98,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
context.CollectionName);
|
||||
|
||||
var metadataSnapshot = this.BuildDataSourceMetadataSnapshot(dataSource, indexedFiles);
|
||||
var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(context, indexedFiles, token);
|
||||
var removedMissingFiles = await this.RemoveMissingFileEmbeddingsAsync(context, inputFiles, token);
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
logger.LogInformation(
|
||||
@@ -323,6 +326,32 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
return IndexedRunOutcome.MORE_TO_DO;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ends a run which cannot reach the file or the folder of its data source, and keeps the index as it is.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A folder out of reach is never the same as an empty one. Read that way, a drive which is not
|
||||
/// connected, or a network share while the VPN is down, would remove every embedding of the
|
||||
/// data source, and embedding it all again once the folder is back costs as long as it took the
|
||||
/// first time. Nothing is written: the stored hash still tells which state of the data source
|
||||
/// the index holds, so the next run picks up from there.
|
||||
/// </remarks>
|
||||
/// <param name="context">The run.</param>
|
||||
/// <param name="dataSource">The data source.</param>
|
||||
/// <param name="reason">Why the file or the folder cannot be reached, and what the user can do about it.</param>
|
||||
/// <returns>DONE, since nothing can be done until the file or the folder is back.</returns>
|
||||
private IndexedRunOutcome StopWithoutRoot(IndexedRunContext context, IDataSource dataSource, string reason)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Cannot reach the file or folder of data source '{DataSourceName}' ({DataSourceId}). The run stops, and nothing is removed from the index. StoredFiles={StoredFiles}.",
|
||||
dataSource.Name,
|
||||
dataSource.Id,
|
||||
context.Manifest.Files.Count);
|
||||
|
||||
DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger).PublishRunFailure(reason);
|
||||
return IndexedRunOutcome.DONE;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a file name marks a file which is never indexed, whatever its type.
|
||||
/// </summary>
|
||||
@@ -344,15 +373,11 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT);
|
||||
}
|
||||
|
||||
private async Task<int> RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, IReadOnlyCollection<FileInfo> indexedFiles, CancellationToken token)
|
||||
private async Task<int> RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, FileEnumerationResult inputFiles, CancellationToken token)
|
||||
{
|
||||
var manifest = context.Manifest;
|
||||
var existingPaths = indexedFiles
|
||||
.Select(file => file.FullName)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var removedFiles = 0;
|
||||
foreach (var removedFilePath in manifest.Files.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList())
|
||||
foreach (var removedFilePath in GetGoneFiles(manifest.Files.Keys, inputFiles.Files, inputFiles.UnreadableDirectories))
|
||||
{
|
||||
await context.DeleteDocumentPointsAsync(removedFilePath, token);
|
||||
await context.IndexStore.DeleteFileAsync(context.DataSource.Id, removedFilePath, token);
|
||||
@@ -369,12 +394,44 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
// A file which is gone needs no mark keeping it out of the index. Without this, the table
|
||||
// would grow with every document the user ever deleted:
|
||||
//
|
||||
foreach (var removedFilePath in manifest.PermanentFailures.Keys.Except(existingPaths, StringComparer.OrdinalIgnoreCase).ToList())
|
||||
foreach (var removedFilePath in GetGoneFiles(manifest.PermanentFailures.Keys, inputFiles.Files, inputFiles.UnreadableDirectories))
|
||||
await context.ForgetPermanentFailureAsync(removedFilePath, token);
|
||||
|
||||
return removedFiles;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Picks the files which are gone from the data source, out of those the index knows about.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A file below a folder which could not be listed is not gone, only out of reach, e.g. while the
|
||||
/// user may not read that folder. Removing it would throw away its index, and embedding it again
|
||||
/// once the folder can be read costs as long as it took the first time.
|
||||
/// <br /><br />
|
||||
/// Paths are compared ignoring their case, as everywhere in the manifest. On Linux and macOS, that
|
||||
/// keeps a file whose path differs from an unreadable folder in its case alone, which costs no more
|
||||
/// than a stale entry until the folder can be read again.
|
||||
/// </remarks>
|
||||
/// <param name="storedPaths">The paths of the files the index knows about.</param>
|
||||
/// <param name="files">The files this run found.</param>
|
||||
/// <param name="unreadableDirectories">The folders this run could not list.</param>
|
||||
/// <returns>The paths of the files which are gone.</returns>
|
||||
internal static List<string> GetGoneFiles(IEnumerable<string> storedPaths, IEnumerable<FileInfo> files, IEnumerable<string> unreadableDirectories)
|
||||
{
|
||||
var existingPaths = files
|
||||
.Select(file => file.FullName)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
var unreadablePrefixes = unreadableDirectories
|
||||
.Select(directory => Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)) + Path.DirectorySeparatorChar)
|
||||
.ToList();
|
||||
|
||||
return storedPaths
|
||||
.Where(path => !existingPaths.Contains(path))
|
||||
.Where(path => !unreadablePrefixes.Any(prefix => path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
/// <remarks>
|
||||
/// A file counts as settled when it was indexed or when it was skipped for good, both with a
|
||||
/// matching fingerprint. Counting only the indexed ones would let a single unreadable document
|
||||
@@ -468,17 +525,23 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
|
||||
case DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path):
|
||||
this.EnumerateAccessibleFiles(localDirectory.Path, result);
|
||||
if (result.UnreadableDirectories.Contains(localDirectory.Path))
|
||||
result.UnreachableRootReason = string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."), localDirectory.Path);
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
//
|
||||
// Missing is not the same as empty, cf. StopWithoutRoot:
|
||||
//
|
||||
switch (dataSource)
|
||||
{
|
||||
case DataSourceLocalFile localFile:
|
||||
result.AddFailure(localFile.FilePath, string.Format(TB("The file '{0}' does not exist."), localFile.FilePath));
|
||||
result.UnreachableRootReason = string.Format(TB("The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source."), localFile.FilePath);
|
||||
break;
|
||||
|
||||
case DataSourceLocalDirectory localDirectory:
|
||||
result.AddFailure(localDirectory.Path, string.Format(TB("The folder '{0}' does not exist."), localDirectory.Path));
|
||||
result.UnreachableRootReason = string.Format(TB("The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source."), localDirectory.Path);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -511,7 +574,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogWarning(exception, "Cannot access directory '{DirectoryPath}' while indexing.", currentPath);
|
||||
result.AddFailure(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath));
|
||||
result.AddUnreadableDirectory(currentPath, string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it."), currentPath));
|
||||
continue;
|
||||
}
|
||||
|
||||
|
||||
@@ -569,16 +569,8 @@ internal sealed partial class MailboxIndexer(SettingsManager settingsManager, Ru
|
||||
foreach (var folder in await context.IndexStore.GetMailFoldersAsync(dataSourceId, token))
|
||||
locationCount += (await context.IndexStore.GetMailLocationsAsync(dataSourceId, folder.Path, token)).Count;
|
||||
|
||||
var permanentFailures = context.Manifest.PermanentFailures;
|
||||
var progress = new DocumentRunProgress(context, locationCount + permanentFailures.Count, 0, string.Empty, [], logger)
|
||||
{
|
||||
LastSyncUtc = (await context.IndexStore.GetMailboxSyncStateAsync(dataSourceId, token)).LastSyncCompletedUtc,
|
||||
};
|
||||
|
||||
progress.RecordUnchanged(locationCount);
|
||||
foreach (var (key, failure) in permanentFailures)
|
||||
progress.RecordStillUnreadable(key, failure);
|
||||
|
||||
var progress = DocumentRunProgress.FromStoredState(context, locationCount, logger);
|
||||
progress.LastSyncUtc = (await context.IndexStore.GetMailboxSyncStateAsync(dataSourceId, token)).LastSyncCompletedUtc;
|
||||
return progress;
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
/// <summary>
|
||||
/// The pauses of the indexing, e.g., while a backup copies the indexes.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Pauses may overlap, and the indexing goes on once the last one ended. Every run which a pause
|
||||
/// stopped is noted together with why it was asked for, so that it can be queued again for the same
|
||||
/// reason once the indexing goes on: a retry the user asked for still signs in to a mailbox again
|
||||
/// after its server refused a sign-in, cf. DataSourceRunQueue.
|
||||
/// </remarks>
|
||||
internal sealed class IndexingPauses
|
||||
{
|
||||
private readonly Lock stateLock = new();
|
||||
|
||||
/// <summary>
|
||||
/// The runs which a pause stopped, each with why it was asked for.
|
||||
/// </summary>
|
||||
private readonly Dictionary<string, DataSourceEmbeddingRefreshMode> stoppedRuns = new(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
private int count;
|
||||
|
||||
/// <summary>
|
||||
/// Completes once the last pause ended. Null while the indexing is not paused.
|
||||
/// </summary>
|
||||
private TaskCompletionSource? lastPauseEnded;
|
||||
|
||||
/// <summary>
|
||||
/// Whether the indexing is paused right now.
|
||||
/// </summary>
|
||||
public bool IsPaused
|
||||
{
|
||||
get
|
||||
{
|
||||
lock (this.stateLock)
|
||||
return this.count > 0;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Begins a pause. From now on, every run which is about to start waits.
|
||||
/// </summary>
|
||||
public void Begin()
|
||||
{
|
||||
lock (this.stateLock)
|
||||
{
|
||||
if (this.count++ == 0)
|
||||
this.lastPauseEnded = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Notes a run which a pause stopped.
|
||||
/// </summary>
|
||||
/// <param name="dataSourceId">The id of the data source.</param>
|
||||
/// <param name="refreshMode">Why the run was asked for.</param>
|
||||
public void RecordStoppedRun(string dataSourceId, DataSourceEmbeddingRefreshMode refreshMode)
|
||||
{
|
||||
lock (this.stateLock)
|
||||
this.stoppedRuns.TryAdd(dataSourceId, refreshMode);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Ends a pause.
|
||||
/// </summary>
|
||||
/// <returns>The runs which the pauses stopped, once the last pause ended, so the caller queues them again. Otherwise, none.</returns>
|
||||
public IReadOnlyList<DataSourceRunRequest> End()
|
||||
{
|
||||
lock (this.stateLock)
|
||||
{
|
||||
if (this.count == 0 || --this.count > 0)
|
||||
return [];
|
||||
|
||||
this.lastPauseEnded?.TrySetResult();
|
||||
this.lastPauseEnded = null;
|
||||
|
||||
var runsToQueueAgain = this.stoppedRuns.Select(stoppedRun => new DataSourceRunRequest(stoppedRun.Key, stoppedRun.Value)).ToList();
|
||||
this.stoppedRuns.Clear();
|
||||
return runsToQueueAgain;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Waits while the indexing is paused.
|
||||
/// </summary>
|
||||
/// <param name="token">Stops the waiting, e.g., because the run waiting here was stopped.</param>
|
||||
public Task WaitWhilePausedAsync(CancellationToken token)
|
||||
{
|
||||
lock (this.stateLock)
|
||||
return this.lastPauseEnded?.Task.WaitAsync(token) ?? Task.CompletedTask;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
/// <summary>
|
||||
/// One-shot startup service which stores the chats and the settings once more, so that every path
|
||||
/// below the data directory in them becomes relative.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Compatibility shim, see documentation/compatibility-shims/2026-10-relative-data-paths.md. Remove
|
||||
/// after 2027-04-14.<br/><br/>
|
||||
///
|
||||
/// AI Studio stores such a path relative whenever it stores a chat or the settings. A chat which is
|
||||
/// never stored again would keep its absolute paths, though, and a backup of the data directory
|
||||
/// brings them to another computer, where they lead nowhere.
|
||||
/// </remarks>
|
||||
public sealed class RelativeDataPathsMigrationService(ILogger<RelativeDataPathsMigrationService> logger, SettingsManager settingsManager) : BackgroundService
|
||||
{
|
||||
/// <summary>
|
||||
/// How long to wait for the plugins to load for the first time, before storing the settings anyway.
|
||||
/// </summary>
|
||||
private static readonly TimeSpan FIRST_PLUGIN_LOAD_TIMEOUT = TimeSpan.FromMinutes(2);
|
||||
|
||||
#region Overrides of BackgroundService
|
||||
|
||||
/// <summary>Waits for the settings, stores every chat and the settings once more, and then exits.</summary>
|
||||
/// <param name="stoppingToken">Host shutdown token.</param>
|
||||
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
|
||||
{
|
||||
while (!settingsManager.HasCompletedInitialSettingsLoad && !stoppingToken.IsCancellationRequested)
|
||||
await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken);
|
||||
|
||||
if (settingsManager.ConfigurationData.StoredDataPathsAreRelative)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
if (!await WorkspaceBehaviour.StoreChatsWithRelativePathsAsync(stoppingToken))
|
||||
{
|
||||
logger.LogWarning("Not every chat could be stored with relative paths yet. The next start tries again.");
|
||||
return;
|
||||
}
|
||||
|
||||
//
|
||||
// While the plugins load for the first time, they bring their providers and tokenizers into
|
||||
// the settings. Storing the settings meanwhile could meet a list in the middle of a change:
|
||||
//
|
||||
try
|
||||
{
|
||||
await PluginFactory.FirstLoadCompleted.WaitAsync(FIRST_PLUGIN_LOAD_TIMEOUT, stoppingToken);
|
||||
}
|
||||
catch (TimeoutException)
|
||||
{
|
||||
logger.LogWarning("The plugins did not finish loading within {Timeout} minutes. The settings are stored with relative paths anyway.", FIRST_PLUGIN_LOAD_TIMEOUT.TotalMinutes);
|
||||
}
|
||||
|
||||
settingsManager.ConfigurationData.StoredDataPathsAreRelative = true;
|
||||
await settingsManager.StoreSettings();
|
||||
logger.LogInformation("Every chat and the settings are stored with paths relative to the data directory now.");
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
logger.LogError(exception, "Storing the chats and the settings with relative paths failed. The next start tries again.");
|
||||
}
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
@@ -156,4 +156,41 @@ public sealed partial class RustService
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Requests the Rust runtime to restart the entire desktop application, e.g., to apply a prepared
|
||||
/// restore of a backup.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// In the development environment, the runtime cannot restart the .NET server, which it did not
|
||||
/// start, and refuses. The developer restarts both instead.
|
||||
/// </remarks>
|
||||
/// <returns>True when the restart is on its way.</returns>
|
||||
public async Task<bool> RestartApplication()
|
||||
{
|
||||
try
|
||||
{
|
||||
var response = await this.http.PostAsync("/app/restart", null);
|
||||
if (!response.IsSuccessStatusCode)
|
||||
{
|
||||
this.logger?.LogError("Failed to restart the app due to network error: {StatusCode}.", response.StatusCode);
|
||||
return false;
|
||||
}
|
||||
|
||||
var result = await response.Content.ReadFromJsonAsync<AppRestartResponse>(this.jsonRustSerializerOptions);
|
||||
if (result is null || !result.Success)
|
||||
{
|
||||
this.logger?.LogError("Failed to restart the app: {Error}", result?.ErrorMessage ?? "Unknown error");
|
||||
return false;
|
||||
}
|
||||
|
||||
this.logger?.LogInformation("Restart request sent to Rust runtime.");
|
||||
return true;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
this.logger?.LogError(ex, "Exception while requesting application restart.");
|
||||
return false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,115 @@
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Chat;
|
||||
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
//
|
||||
// Compatibility shim, see documentation/compatibility-shims/2026-10-relative-data-paths.md.
|
||||
// Remove after 2027-04-14.
|
||||
//
|
||||
public static partial class WorkspaceBehaviour
|
||||
{
|
||||
/// <summary>
|
||||
/// Writes every stored chat once more, so that the paths of its files below the data directory become relative.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A chat stored before carries absolute paths. They still work on this computer, but a backup of
|
||||
/// the data directory brings them to another one, where they lead nowhere. Each chat is read and
|
||||
/// written within one hold of its lock, so that nothing AI Studio stores meanwhile gets lost. The
|
||||
/// time of the last change stays as it was: the workspaces sort their chats by it, and the
|
||||
/// maintenance of the temporary chats deletes them by it.
|
||||
/// </remarks>
|
||||
/// <param name="token">Cancels the pass, e.g., when AI Studio shuts down.</param>
|
||||
/// <returns>Whether every chat was taken care of; false when one has to wait for the next start.</returns>
|
||||
internal static async Task<bool> StoreChatsWithRelativePathsAsync(CancellationToken token)
|
||||
{
|
||||
var everyChatStored = true;
|
||||
foreach (var chat in EnumerateStoredChats())
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
if (!await StoreChatWithRelativePathsAsync(chat, token))
|
||||
everyChatStored = false;
|
||||
}
|
||||
|
||||
return everyChatStored;
|
||||
}
|
||||
|
||||
private static IEnumerable<LoadChat> EnumerateStoredChats()
|
||||
{
|
||||
if (Directory.Exists(TemporaryChatsRootDirectory))
|
||||
foreach (var chatDirectory in Directory.EnumerateDirectories(TemporaryChatsRootDirectory))
|
||||
if (Guid.TryParse(Path.GetFileName(chatDirectory), out var chatId))
|
||||
yield return new(Guid.Empty, chatId);
|
||||
|
||||
if (!Directory.Exists(WorkspaceRootDirectory))
|
||||
yield break;
|
||||
|
||||
foreach (var workspaceDirectory in Directory.EnumerateDirectories(WorkspaceRootDirectory))
|
||||
{
|
||||
if (!Guid.TryParse(Path.GetFileName(workspaceDirectory), out var workspaceId))
|
||||
continue;
|
||||
|
||||
foreach (var chatDirectory in Directory.EnumerateDirectories(workspaceDirectory))
|
||||
if (Guid.TryParse(Path.GetFileName(chatDirectory), out var chatId))
|
||||
yield return new(workspaceId, chatId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <returns>Whether the chat was taken care of: written anew, or with nothing to change.</returns>
|
||||
private static async Task<bool> StoreChatWithRelativePathsAsync(LoadChat chat, CancellationToken token)
|
||||
{
|
||||
var (acquired, semaphore) = await TryAcquireChatSemaphoreAsync(chat.WorkspaceId, chat.ChatId, nameof(StoreChatWithRelativePathsAsync));
|
||||
if (!acquired)
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
var chatDirectory = GetChatDirectory(chat.WorkspaceId, chat.ChatId);
|
||||
var chatPath = Path.Join(chatDirectory, "thread.json");
|
||||
if (!File.Exists(chatPath))
|
||||
return true;
|
||||
|
||||
var storedJson = await File.ReadAllTextAsync(chatPath, Encoding.UTF8, token);
|
||||
var thread = JsonSerializer.Deserialize<ChatThread>(storedJson, JSON_OPTIONS);
|
||||
if (thread is null)
|
||||
return true;
|
||||
|
||||
var json = JsonSerializer.Serialize(thread, JSON_OPTIONS);
|
||||
if (json == storedJson)
|
||||
return true;
|
||||
|
||||
var lastEditTime = File.GetLastWriteTimeUtc(chatPath);
|
||||
var temporaryPath = Path.Join(chatDirectory, $".thread-{Guid.NewGuid():N}.tmp");
|
||||
try
|
||||
{
|
||||
await File.WriteAllTextAsync(temporaryPath, json, Encoding.UTF8, token);
|
||||
File.Move(temporaryPath, chatPath, overwrite: true);
|
||||
File.SetLastWriteTimeUtc(chatPath, lastEditTime);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (File.Exists(temporaryPath))
|
||||
File.Delete(temporaryPath);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
catch (Exception exception) when (exception is JsonException or NotSupportedException)
|
||||
{
|
||||
// Opening the chat fails the same way, so there is nothing to store, now or at the next start:
|
||||
LOG.LogWarning(exception, "The chat '{ChatId}' of workspace '{WorkspaceId}' could not be read, so its paths stay as they are.", chat.ChatId, chat.WorkspaceId);
|
||||
return true;
|
||||
}
|
||||
catch (Exception exception) when (exception is IOException or UnauthorizedAccessException)
|
||||
{
|
||||
LOG.LogWarning(exception, "The chat '{ChatId}' of workspace '{WorkspaceId}' could not be stored with relative paths. The next start tries again.", chat.ChatId, chat.WorkspaceId);
|
||||
return false;
|
||||
}
|
||||
finally
|
||||
{
|
||||
semaphore.Release();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -10,7 +10,7 @@ using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
public static class WorkspaceBehaviour
|
||||
public static partial class WorkspaceBehaviour
|
||||
{
|
||||
public readonly record struct TryCreateWorkspaceResult(bool Success, WorkspaceTreeWorkspace Workspace);
|
||||
|
||||
@@ -872,7 +872,7 @@ public static class WorkspaceBehaviour
|
||||
|
||||
Directory.CreateDirectory(chatDirectory);
|
||||
|
||||
await FinalizeStagedTranscriptsAsync(chat, chatDirectory);
|
||||
FinalizeStagedTranscripts(chat, chatDirectory);
|
||||
var chatNamePath = Path.Join(chatDirectory, "name");
|
||||
await File.WriteAllTextAsync(chatNamePath, chat.Name);
|
||||
|
||||
@@ -970,25 +970,15 @@ public static class WorkspaceBehaviour
|
||||
|
||||
private static void CopyManagedTranscriptAttachments(ChatThread chat, string targetChatDirectory)
|
||||
{
|
||||
var targetTranscriptDirectory = Path.Combine(targetChatDirectory, "attachments", "transcripts");
|
||||
var pathComparer = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal;
|
||||
var copiedPaths = new Dictionary<string, ManagedTranscriptAttachment>(pathComparer);
|
||||
var targetTranscriptDirectory = ManagedTranscriptAttachment.GetTranscriptDirectory(targetChatDirectory);
|
||||
var copiedPaths = new Dictionary<string, ManagedTranscriptAttachment>(PathTools.COMPARER);
|
||||
|
||||
foreach (var content in chat.Blocks.Select(block => block.Content).OfType<ContentText>())
|
||||
{
|
||||
for (var index = 0; index < content.FileAttachments.Count; index++)
|
||||
{
|
||||
if (content.FileAttachments[index] is ManagedTranscriptAttachment transcript)
|
||||
content.FileAttachments[index] = CopyManagedTranscriptAttachment(chat, transcript, targetTranscriptDirectory, copiedPaths);
|
||||
}
|
||||
}
|
||||
|
||||
chat.Draft.ReplaceAttachments(attachment => attachment is ManagedTranscriptAttachment transcript
|
||||
chat.ReplaceAttachments(attachment => attachment is ManagedTranscriptAttachment transcript
|
||||
? CopyManagedTranscriptAttachment(chat, transcript, targetTranscriptDirectory, copiedPaths)
|
||||
: attachment);
|
||||
}
|
||||
|
||||
private static ManagedTranscriptAttachment CopyManagedTranscriptAttachment(
|
||||
internal static ManagedTranscriptAttachment CopyManagedTranscriptAttachment(
|
||||
ChatThread chat,
|
||||
ManagedTranscriptAttachment source,
|
||||
string targetTranscriptDirectory,
|
||||
@@ -1045,7 +1035,7 @@ public static class WorkspaceBehaviour
|
||||
if (!Directory.Exists(chatDirectory))
|
||||
throw new DirectoryNotFoundException($"The owning chat directory does not exist: '{chatDirectory}'.");
|
||||
|
||||
var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts");
|
||||
var transcriptDirectory = ManagedTranscriptAttachment.GetTranscriptDirectory(chatDirectory);
|
||||
Directory.CreateDirectory(transcriptDirectory);
|
||||
ReconcileTranscriptCounter(chat, transcriptDirectory);
|
||||
|
||||
@@ -1096,7 +1086,7 @@ public static class WorkspaceBehaviour
|
||||
UpdateAttachmentPathsAfterMove(chat, sourceDirectory, targetDirectory);
|
||||
chat.WorkspaceId = targetWorkspaceId;
|
||||
|
||||
await FinalizeStagedTranscriptsAsync(chat, targetDirectory);
|
||||
FinalizeStagedTranscripts(chat, targetDirectory);
|
||||
await StoreMovedChatFilesAsync(chat, targetDirectory);
|
||||
|
||||
var lastEditTime = File.GetLastWriteTimeUtc(Path.Join(targetDirectory, "thread.json"));
|
||||
@@ -1153,22 +1143,14 @@ public static class WorkspaceBehaviour
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Rewrites absolute attachment paths after moving the complete chat directory.</summary>
|
||||
private static void UpdateAttachmentPathsAfterMove(ChatThread chat, string sourceDirectory, string targetDirectory)
|
||||
{
|
||||
foreach (var content in chat.Blocks.Select(block => block.Content).OfType<ContentText>())
|
||||
{
|
||||
for (var index = 0; index < content.FileAttachments.Count; index++)
|
||||
content.FileAttachments[index] = RelocateAttachment(content.FileAttachments[index], sourceDirectory, targetDirectory);
|
||||
}
|
||||
|
||||
//
|
||||
// The transcripts of the message not sent yet live in the chat directory as well. Left with
|
||||
// their old paths, the draft points at a directory which is gone after the move, and the
|
||||
// transcripts would never reach the message.
|
||||
//
|
||||
chat.Draft.ReplaceAttachments(attachment => RelocateAttachment(attachment, sourceDirectory, targetDirectory));
|
||||
}
|
||||
/// <summary>Rewrites absolute file paths after moving the complete chat directory.</summary>
|
||||
/// <remarks>
|
||||
/// The transcripts of the message not sent yet live in the chat directory as well. Left with
|
||||
/// their old paths, the draft points at a directory which is gone after the move, and the
|
||||
/// transcripts would never reach the message. ChatThread.RewriteFilePaths takes the draft along.
|
||||
/// </remarks>
|
||||
private static void UpdateAttachmentPathsAfterMove(ChatThread chat, string sourceDirectory, string targetDirectory) =>
|
||||
chat.RewriteFilePaths(path => RelocatePath(path, sourceDirectory, targetDirectory));
|
||||
|
||||
/// <summary>
|
||||
/// Gives an attachment the path it has after its chat moved into another workspace.
|
||||
@@ -1191,55 +1173,54 @@ public static class WorkspaceBehaviour
|
||||
/// <summary>Gives an attachment inside a moved directory its path in the target directory.</summary>
|
||||
private static FileAttachment RelocateAttachment(FileAttachment attachment, string sourceDirectory, string targetDirectory)
|
||||
{
|
||||
var sourcePrefix = sourceDirectory.EndsWith(Path.DirectorySeparatorChar)
|
||||
? sourceDirectory
|
||||
: sourceDirectory + Path.DirectorySeparatorChar;
|
||||
|
||||
var pathComparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||
if (!Path.GetFullPath(attachment.FilePath).StartsWith(sourcePrefix, pathComparison))
|
||||
return attachment;
|
||||
|
||||
var relativePath = Path.GetRelativePath(sourceDirectory, attachment.FilePath);
|
||||
var movedPath = Path.Combine(targetDirectory, relativePath);
|
||||
|
||||
return attachment switch
|
||||
{
|
||||
ManagedTranscriptAttachment managed => managed with { FilePath = movedPath },
|
||||
FileAttachmentImage image => image with { FilePath = movedPath },
|
||||
_ => attachment with { FilePath = movedPath },
|
||||
};
|
||||
var movedPath = RelocatePath(attachment.FilePath, sourceDirectory, targetDirectory);
|
||||
return movedPath == attachment.FilePath ? attachment : attachment.WithFilePath(movedPath);
|
||||
}
|
||||
|
||||
private static async Task FinalizeStagedTranscriptsAsync(ChatThread chat, string chatDirectory)
|
||||
/// <summary>Gives a path inside a moved directory its place in the target directory, and leaves every other path as it is.</summary>
|
||||
private static string RelocatePath(string path, string sourceDirectory, string targetDirectory)
|
||||
{
|
||||
var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts");
|
||||
if (!PathTools.IsInsideDirectory(sourceDirectory, path))
|
||||
return path;
|
||||
|
||||
return Path.Combine(targetDirectory, Path.GetRelativePath(sourceDirectory, path));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Moves the transcripts of a chat which still wait in the staging area into the chat directory.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A transcript waits there when its chat was not stored yet when the transcript was made. The draft
|
||||
/// counts as well: leaving a new chat stores it along with the message the user started in it, and
|
||||
/// the next start of AI Studio empties the staging area. A transcript of the draft left behind there
|
||||
/// would be gone, while the stored draft still pointed at it.
|
||||
/// </remarks>
|
||||
private static void FinalizeStagedTranscripts(ChatThread chat, string chatDirectory)
|
||||
{
|
||||
var transcriptDirectory = ManagedTranscriptAttachment.GetTranscriptDirectory(chatDirectory);
|
||||
ReconcileTranscriptCounter(chat, transcriptDirectory);
|
||||
foreach (var content in chat.Blocks.Select(block => block.Content).OfType<ContentText>())
|
||||
{
|
||||
for (var index = 0; index < content.FileAttachments.Count; index++)
|
||||
{
|
||||
if (content.FileAttachments[index] is not ManagedTranscriptAttachment { IsStaged: true } staged
|
||||
|| !File.Exists(staged.FilePath))
|
||||
continue;
|
||||
chat.ReplaceAttachments(attachment => attachment is ManagedTranscriptAttachment { IsStaged: true } staged && File.Exists(staged.FilePath)
|
||||
? FinalizeStagedTranscript(chat, staged, transcriptDirectory)
|
||||
: attachment);
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(transcriptDirectory);
|
||||
var targetPath = NextTranscriptPath(chat, transcriptDirectory, staged.OriginalFileName);
|
||||
/// <summary>Moves one staged transcript into the transcript directory of its chat.</summary>
|
||||
private static ManagedTranscriptAttachment FinalizeStagedTranscript(ChatThread chat, ManagedTranscriptAttachment staged, string transcriptDirectory)
|
||||
{
|
||||
Directory.CreateDirectory(transcriptDirectory);
|
||||
var targetPath = NextTranscriptPath(chat, transcriptDirectory, staged.OriginalFileName);
|
||||
|
||||
File.Move(staged.FilePath, targetPath);
|
||||
var sourceDirectory = Path.GetDirectoryName(staged.FilePath);
|
||||
if (sourceDirectory is not null && Directory.Exists(sourceDirectory) && !Directory.EnumerateFileSystemEntries(sourceDirectory).Any())
|
||||
Directory.Delete(sourceDirectory);
|
||||
File.Move(staged.FilePath, targetPath);
|
||||
var sourceDirectory = Path.GetDirectoryName(staged.FilePath);
|
||||
if (sourceDirectory is not null && Directory.Exists(sourceDirectory) && !Directory.EnumerateFileSystemEntries(sourceDirectory).Any())
|
||||
Directory.Delete(sourceDirectory);
|
||||
|
||||
content.FileAttachments[index] = new ManagedTranscriptAttachment(
|
||||
Path.GetFileName(targetPath),
|
||||
targetPath,
|
||||
new FileInfo(targetPath).Length,
|
||||
staged.OriginalFileName,
|
||||
false);
|
||||
}
|
||||
}
|
||||
|
||||
await Task.CompletedTask;
|
||||
return new ManagedTranscriptAttachment(
|
||||
Path.GetFileName(targetPath),
|
||||
targetPath,
|
||||
new FileInfo(targetPath).Length,
|
||||
staged.OriginalFileName,
|
||||
false);
|
||||
}
|
||||
|
||||
/// <summary>Raises the persisted counter to the highest transcript suffix found chat-wide.</summary>
|
||||
@@ -1276,7 +1257,7 @@ public static class WorkspaceBehaviour
|
||||
}
|
||||
|
||||
/// <summary>Returns the canonical storage directory for a chat identity.</summary>
|
||||
private static string GetChatDirectory(Guid workspaceId, Guid chatId) => workspaceId == Guid.Empty
|
||||
public static string GetChatDirectory(Guid workspaceId, Guid chatId) => workspaceId == Guid.Empty
|
||||
? Path.Join(SettingsManager.DataDirectory, "tempChats", chatId.ToString())
|
||||
: Path.Join(SettingsManager.DataDirectory, "workspaces", workspaceId.ToString(), chatId.ToString());
|
||||
|
||||
@@ -1458,7 +1439,11 @@ public static class WorkspaceBehaviour
|
||||
return true;
|
||||
}
|
||||
|
||||
private static async Task EnsureWorkspace(Guid workspaceId, string workspaceName)
|
||||
/// <summary>
|
||||
/// Ensures that a workspace with the given identity exists on disk and in the cache.
|
||||
/// An existing workspace keeps its current name.
|
||||
/// </summary>
|
||||
public static async Task EnsureWorkspace(Guid workspaceId, string workspaceName)
|
||||
{
|
||||
var workspacePath = Path.Join(WorkspaceRootDirectory, workspaceId.ToString());
|
||||
var workspaceNamePath = Path.Join(workspacePath, "name");
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Checks the names of entries in ZIP files before anything is written to the disk.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A ZIP file comes from somewhere else: a plugin archive, the configuration of an organization, or a
|
||||
/// backup. An entry name such as "../settings.json" or an absolute path would make extracting it
|
||||
/// write outside of the target directory (zip slip). Every name is therefore checked here before it
|
||||
/// becomes a path.<br/><br/>
|
||||
///
|
||||
/// Backslashes count as separators as well. Compatibility shim for ZIP files created on Windows
|
||||
/// with backslashes in their entry names (dotnet/runtime#27620); remove after dotnet/runtime#27620
|
||||
/// and #41914 are fixed. See documentation/compatibility-shims/2026-07-plugin-archive-zip-backslashes.md.
|
||||
/// </remarks>
|
||||
public static class ZipEntryNames
|
||||
{
|
||||
/// <summary>
|
||||
/// The separator of entry names, which the ZIP format prescribes.
|
||||
/// </summary>
|
||||
public const char SEPARATOR = '/';
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes the name of an entry, and rejects a name which could lead outside of the target
|
||||
/// directory with an InvalidDataException: an empty, rooted, or drive-qualified name, or one which
|
||||
/// contains "." or "..".
|
||||
/// </summary>
|
||||
/// <param name="entryName">The name as the ZIP file holds it.</param>
|
||||
/// <returns>The name with forward slashes.</returns>
|
||||
public static string Normalize(string entryName)
|
||||
{
|
||||
var normalizedEntryName = entryName.Replace('\\', SEPARATOR);
|
||||
if (string.IsNullOrWhiteSpace(normalizedEntryName))
|
||||
throw new InvalidDataException("The archive contains an empty entry name.");
|
||||
|
||||
if (normalizedEntryName.Contains('\0'))
|
||||
throw new InvalidDataException($"The archive contains an invalid entry name: '{entryName}'.");
|
||||
|
||||
if (normalizedEntryName.StartsWith(SEPARATOR))
|
||||
throw new InvalidDataException($"The archive contains a rooted entry name: '{entryName}'.");
|
||||
|
||||
if (normalizedEntryName is [_, ':', ..])
|
||||
throw new InvalidDataException($"The archive contains a drive-qualified entry name: '{entryName}'.");
|
||||
|
||||
var pathSegments = normalizedEntryName.Split(SEPARATOR, StringSplitOptions.RemoveEmptyEntries);
|
||||
if (pathSegments.Length == 0 || pathSegments.Any(segment => segment is "." or ".."))
|
||||
throw new InvalidDataException($"The archive contains an unsafe entry name: '{entryName}'.");
|
||||
|
||||
return normalizedEntryName;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Normalizes the name of an entry, cf. Normalize, without throwing.
|
||||
/// </summary>
|
||||
/// <param name="entryName">The name as the ZIP file holds it.</param>
|
||||
/// <param name="normalizedEntryName">The name with forward slashes, or an empty string when it is unsafe.</param>
|
||||
/// <returns>True when the name is safe.</returns>
|
||||
public static bool TryNormalize(string entryName, out string normalizedEntryName)
|
||||
{
|
||||
try
|
||||
{
|
||||
normalizedEntryName = Normalize(entryName);
|
||||
return true;
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
normalizedEntryName = string.Empty;
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The path at which an entry is written below the target directory. A path which would lie
|
||||
/// outside of it is rejected with an InvalidDataException.
|
||||
/// </summary>
|
||||
/// <param name="destinationDirectory">The target directory.</param>
|
||||
/// <param name="normalizedEntryName">The entry name, as Normalize returned it.</param>
|
||||
/// <returns>The full path of the entry.</returns>
|
||||
public static string GetDestinationPath(string destinationDirectory, string normalizedEntryName)
|
||||
{
|
||||
var fullDestinationDirectory = Path.GetFullPath(destinationDirectory);
|
||||
if (!fullDestinationDirectory.EndsWith(Path.DirectorySeparatorChar))
|
||||
fullDestinationDirectory += Path.DirectorySeparatorChar;
|
||||
|
||||
var pathSegments = normalizedEntryName.Split(SEPARATOR, StringSplitOptions.RemoveEmptyEntries);
|
||||
var destinationPath = Path.GetFullPath(Path.Combine(fullDestinationDirectory, Path.Combine(pathSegments)));
|
||||
if (!destinationPath.StartsWith(fullDestinationDirectory, StringComparison.Ordinal))
|
||||
throw new InvalidDataException($"The archive contains an entry outside the destination directory: '{normalizedEntryName}'.");
|
||||
|
||||
return destinationPath;
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user