mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-14 03:53:48 +00:00
Added data backup and restore for chats
The settings gain a data backup panel. It writes the chats of the chosen workspaces, and optionally the temporary chats, into a single archive file with the extension .aistudio-chats, and it imports such archives again. The archive mirrors the chat storage, so restoring needs no conversion. Documents the user attached are never copied into the archive. The chats keep the absolute path of each document, which keeps them usable wherever the documents live and avoids a second copy. Files AI Studio created itself, such as transcripts, exist nowhere else; when the selection holds any, the user decides whether they travel with the archive. On import, chats which already exist, in any workspace, are either kept or imported as additional chats with a new ID. Workspaces are merged by their identity. An imported chat gets its own copy of every transcript the archive did not carry, as a copied chat does, so deleting it never removes the transcripts of the chat it came from. An existing archive is only replaced once the new one is complete. Chats which could not be read are reported instead of missing silently. Both directions can be stopped at any time and report what was written. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
j-erlerandClaude Opus 5.5 committed
1 parent
4baf21656a
commit
a8e8edc028
27 files changed
+2342
-3
No files matched your search
@@ -0,0 +1,63 @@
|
||||
using AIStudio.Chat;
|
||||
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Visits every file path stored inside a chat thread. Export and import use this to
|
||||
/// translate between absolute paths on this machine and archive-relative paths.
|
||||
/// </summary>
|
||||
public static class ChatArchiveAttachmentPaths
|
||||
{
|
||||
/// <summary>
|
||||
/// Replaces every file path of the given chat thread by the result of the rewrite function.
|
||||
/// Paths which the rewrite function returns unchanged stay untouched.
|
||||
/// </summary>
|
||||
/// <param name="chat">The chat thread to process. It is modified in place.</param>
|
||||
/// <param name="rewrite">Maps an existing path to its replacement.</param>
|
||||
public static void Rewrite(ChatThread chat, Func<string, string> rewrite)
|
||||
{
|
||||
foreach (var block in chat.Blocks)
|
||||
{
|
||||
if (block.Content is not { } content)
|
||||
continue;
|
||||
|
||||
RewriteAttachments(content.FileAttachments, rewrite);
|
||||
|
||||
// Images might reference a file on this machine as well:
|
||||
if (content is ContentImage { SourceType: ContentImageSource.LOCAL_PATH } image && !string.IsNullOrWhiteSpace(image.Source))
|
||||
image.Source = rewrite(image.Source);
|
||||
}
|
||||
|
||||
// Transcripts which were prepared for the composer but not sent yet:
|
||||
for (var index = 0; index < chat.PendingMediaTranscripts.Count; index++)
|
||||
{
|
||||
var transcript = chat.PendingMediaTranscripts[index];
|
||||
if (string.IsNullOrWhiteSpace(transcript.FilePath))
|
||||
continue;
|
||||
|
||||
chat.PendingMediaTranscripts[index] = transcript with { FilePath = rewrite(transcript.FilePath) };
|
||||
}
|
||||
}
|
||||
|
||||
private static void RewriteAttachments(List<FileAttachment> attachments, Func<string, string> rewrite)
|
||||
{
|
||||
for (var index = 0; index < attachments.Count; index++)
|
||||
{
|
||||
var attachment = attachments[index];
|
||||
if (string.IsNullOrWhiteSpace(attachment.FilePath))
|
||||
continue;
|
||||
|
||||
var rewrittenPath = rewrite(attachment.FilePath);
|
||||
if (rewrittenPath == attachment.FilePath)
|
||||
continue;
|
||||
|
||||
attachments[index] = attachment switch
|
||||
{
|
||||
ManagedTranscriptAttachment managed => managed with { FilePath = rewrittenPath },
|
||||
FileAttachmentImage image => image with { FilePath = rewrittenPath },
|
||||
|
||||
_ => attachment with { FilePath = rewrittenPath },
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,67 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Finds the files AI Studio keeps inside a chat directory itself, such as transcripts.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Documents the user attaches are never copied into a chat directory: they stay where the
|
||||
/// user put them and the chat only stores their path. Everything below a chat directory was
|
||||
/// therefore written by AI Studio, which makes the directory contents a reliable definition
|
||||
/// of "the app's own files" — minus the two files the chat storage owns itself.
|
||||
/// </remarks>
|
||||
public static class ChatArchiveChatFiles
|
||||
{
|
||||
/// <summary>
|
||||
/// Collects the app-owned files of one chat.
|
||||
/// </summary>
|
||||
/// <param name="chatDirectory">The directory of the chat.</param>
|
||||
/// <returns>The absolute paths of the files, keyed by their chat-relative path.</returns>
|
||||
public static Dictionary<string, string> Collect(string chatDirectory)
|
||||
{
|
||||
var files = new Dictionary<string, string>(PathTools.COMPARER);
|
||||
if (!Directory.Exists(chatDirectory))
|
||||
return files;
|
||||
|
||||
foreach (var filePath in Directory.EnumerateFiles(chatDirectory, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
// Skip the temporary files the chat storage writes while saving a chat:
|
||||
var fileName = Path.GetFileName(filePath);
|
||||
if (fileName.StartsWith('.') && fileName.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
var chatRelativePath = Path.GetRelativePath(chatDirectory, filePath).Replace(Path.DirectorySeparatorChar, ChatArchiveFormat.ENTRY_SEPARATOR);
|
||||
|
||||
// The chat itself is always exported and is not one of its own attachments:
|
||||
if (chatRelativePath.Equals(ChatArchiveFormat.THREAD_FILE_NAME, StringComparison.OrdinalIgnoreCase) ||
|
||||
chatRelativePath.Equals(ChatArchiveFormat.NAME_FILE_NAME, StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
files[chatRelativePath] = filePath;
|
||||
}
|
||||
|
||||
return files;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether at least one of the given chats owns a file.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// This stops at the first file found, so that asking the question stays cheap even for
|
||||
/// a large chat storage.
|
||||
/// </remarks>
|
||||
/// <param name="chatDirectories">The directories of the chats to check.</param>
|
||||
/// <param name="token">Cancels the search.</param>
|
||||
/// <returns>Whether any app-owned file exists.</returns>
|
||||
public static bool HasAny(IEnumerable<string> chatDirectories, CancellationToken token)
|
||||
{
|
||||
foreach (var chatDirectory in chatDirectories)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
if (Collect(chatDirectory).Count > 0)
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Determines what the import does when a chat from the archive already exists.
|
||||
/// </summary>
|
||||
public enum ChatArchiveCollisionBehavior
|
||||
{
|
||||
/// <summary>
|
||||
/// Keeps the existing chat and ignores the one from the archive.
|
||||
/// </summary>
|
||||
SKIP,
|
||||
|
||||
/// <summary>
|
||||
/// Imports the chat from the archive as an additional chat.
|
||||
/// </summary>
|
||||
IMPORT_AS_COPY,
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// The outcome of a chat archive export.
|
||||
/// </summary>
|
||||
/// <param name="Success">Whether the archive was written completely.</param>
|
||||
/// <param name="ArchivePath">The path of the written archive.</param>
|
||||
/// <param name="ExportedWorkspaces">The number of exported workspaces.</param>
|
||||
/// <param name="ExportedChats">The number of exported chats.</param>
|
||||
/// <param name="IncludedFiles">The number of app-owned files, such as transcripts, which were included in the archive.</param>
|
||||
/// <param name="UnreadableChats">The number of selected chats which could not be read and are missing from the archive.</param>
|
||||
/// <param name="Issue">The reason why the export failed, if it did.</param>
|
||||
/// <param name="Cancelled">Whether the user stopped the export. No archive is left behind then.</param>
|
||||
public sealed record ChatArchiveExportResult(
|
||||
bool Success,
|
||||
string ArchivePath,
|
||||
int ExportedWorkspaces,
|
||||
int ExportedChats,
|
||||
int IncludedFiles,
|
||||
int UnreadableChats,
|
||||
string Issue,
|
||||
bool Cancelled);
|
||||
@@ -0,0 +1,329 @@
|
||||
using System.IO.Compression;
|
||||
using System.Reflection;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Tools.Metadata;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Writes selected workspaces and temporary chats into a chat archive.
|
||||
/// </summary>
|
||||
public static class ChatArchiveExporter
|
||||
{
|
||||
private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger(nameof(ChatArchiveExporter));
|
||||
|
||||
/// <summary>
|
||||
/// File types which are compressed already. Deflating them again costs time without
|
||||
/// making the archive smaller.
|
||||
/// </summary>
|
||||
private static readonly HashSet<string> ALREADY_COMPRESSED_EXTENSIONS = new(StringComparer.OrdinalIgnoreCase)
|
||||
{
|
||||
".jpg", ".jpeg", ".png", ".gif", ".webp", ".heic", ".tiff",
|
||||
".mp3", ".m4a", ".m4b", ".aac", ".ogg", ".opus", ".flac", ".wma",
|
||||
".mp4", ".m4v", ".mov", ".mkv", ".webm", ".avi", ".wmv", ".flv",
|
||||
".zip", ".7z", ".gz", ".bz2", ".xz", ".rar",
|
||||
".pdf", ".docx", ".xlsx", ".pptx", ".odt", ".odp",
|
||||
};
|
||||
|
||||
private static readonly int BUFFER_SIZE = 64 * 1024;
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ChatArchiveExporter).Namespace, nameof(ChatArchiveExporter));
|
||||
|
||||
/// <summary>
|
||||
/// Exports the given workspaces and, when requested, all temporary chats into one archive.
|
||||
/// </summary>
|
||||
/// <param name="workspaceIds">The workspaces to export.</param>
|
||||
/// <param name="includeTemporaryChats">Whether to include all temporary chats.</param>
|
||||
/// <param name="includeChatFiles">Whether to include the app-owned files of the chats, such as transcripts.</param>
|
||||
/// <param name="archivePath">The target path of the archive. An existing file is replaced once the new archive is complete.</param>
|
||||
/// <param name="progress">Receives the export progress.</param>
|
||||
/// <param name="token">Cancels the export. A cancelled export leaves no archive behind and an existing one untouched.</param>
|
||||
/// <returns>The export result.</returns>
|
||||
public static async Task<ChatArchiveExportResult> ExportAsync(IReadOnlyList<Guid> workspaceIds, bool includeTemporaryChats, bool includeChatFiles, string archivePath, IProgress<ChatArchiveProgress>? progress, CancellationToken token)
|
||||
{
|
||||
//
|
||||
// The archive is written next to its target and only takes its place once complete.
|
||||
// Writing over the target directly would destroy the backup it replaces the moment the
|
||||
// export starts, and a failed or cancelled export would then leave none at all:
|
||||
//
|
||||
var temporaryArchivePath = $"{archivePath}.{Guid.NewGuid():N}.tmp";
|
||||
try
|
||||
{
|
||||
//
|
||||
// Collect everything to export first, so that we know the total amount of
|
||||
// work before we start writing the archive:
|
||||
//
|
||||
var tree = await WorkspaceBehaviour.GetOrLoadWorkspaceTreeShellAsync();
|
||||
var selectedWorkspaces = new List<(Guid WorkspaceId, string Name, IReadOnlyList<WorkspaceTreeChat> Chats)>();
|
||||
foreach (var workspaceId in workspaceIds)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var workspace = tree.Workspaces.FirstOrDefault(candidate => candidate.WorkspaceId == workspaceId);
|
||||
if (workspace.WorkspaceId == Guid.Empty)
|
||||
{
|
||||
LOG.LogWarning("Skipping workspace '{WorkspaceId}' for the export, because it is unknown.", workspaceId);
|
||||
continue;
|
||||
}
|
||||
|
||||
var chats = await WorkspaceBehaviour.GetWorkspaceChatsAsync(workspaceId);
|
||||
selectedWorkspaces.Add((workspaceId, workspace.Name, chats));
|
||||
}
|
||||
|
||||
IReadOnlyList<WorkspaceTreeChat> temporaryChats = includeTemporaryChats ? tree.TemporaryChats : [];
|
||||
var totalChats = selectedWorkspaces.Sum(workspace => workspace.Chats.Count) + temporaryChats.Count;
|
||||
var processedChats = 0;
|
||||
var unreadableChats = 0;
|
||||
|
||||
var manifestWorkspaces = new List<ChatArchiveManifestWorkspace>();
|
||||
List<ChatArchiveManifestChat> manifestTemporaryChats;
|
||||
|
||||
await using (var archiveStream = new FileStream(temporaryArchivePath, FileMode.CreateNew, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan))
|
||||
{
|
||||
using var archive = new ZipArchive(archiveStream, ZipArchiveMode.Create);
|
||||
|
||||
// Workspace chats and temporary chats differ only in where they are stored:
|
||||
async Task<List<ChatArchiveManifestChat>> ExportChatsAsync(string entryPrefix, Guid workspaceId, IReadOnlyList<WorkspaceTreeChat> chats)
|
||||
{
|
||||
var manifestChats = new List<ChatArchiveManifestChat>();
|
||||
foreach (var chat in chats)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
progress?.Report(new(processedChats, totalChats));
|
||||
processedChats++;
|
||||
|
||||
var manifestChat = await ExportChatAsync(archive, entryPrefix, workspaceId, chat, includeChatFiles, token);
|
||||
if (manifestChat is not null)
|
||||
manifestChats.Add(manifestChat);
|
||||
else
|
||||
unreadableChats++;
|
||||
}
|
||||
|
||||
return manifestChats;
|
||||
}
|
||||
|
||||
foreach (var workspace in selectedWorkspaces)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var workspaceEntryPrefix = $"{ChatArchiveFormat.WORKSPACES_DIRECTORY}{ChatArchiveFormat.ENTRY_SEPARATOR}{workspace.WorkspaceId}{ChatArchiveFormat.ENTRY_SEPARATOR}";
|
||||
await WriteTextEntryAsync(archive, $"{workspaceEntryPrefix}{ChatArchiveFormat.NAME_FILE_NAME}", workspace.Name, token);
|
||||
|
||||
manifestWorkspaces.Add(new()
|
||||
{
|
||||
WorkspaceId = workspace.WorkspaceId,
|
||||
Name = workspace.Name,
|
||||
Chats = await ExportChatsAsync(workspaceEntryPrefix, workspace.WorkspaceId, workspace.Chats),
|
||||
});
|
||||
}
|
||||
|
||||
var temporaryChatsEntryPrefix = $"{ChatArchiveFormat.TEMPORARY_CHATS_DIRECTORY}{ChatArchiveFormat.ENTRY_SEPARATOR}";
|
||||
manifestTemporaryChats = await ExportChatsAsync(temporaryChatsEntryPrefix, Guid.Empty, temporaryChats);
|
||||
|
||||
var manifest = new ChatArchiveManifest
|
||||
{
|
||||
FormatVersion = ChatArchiveFormat.FORMAT_VERSION,
|
||||
ExportedAt = DateTimeOffset.UtcNow,
|
||||
AppVersion = Assembly.GetExecutingAssembly().GetCustomAttribute<MetaDataAttribute>()?.Version ?? "unknown",
|
||||
Workspaces = manifestWorkspaces,
|
||||
TemporaryChats = manifestTemporaryChats,
|
||||
};
|
||||
|
||||
var manifestJson = JsonSerializer.Serialize(manifest, WorkspaceBehaviour.JSON_OPTIONS);
|
||||
await WriteTextEntryAsync(archive, ChatArchiveFormat.MANIFEST_FILE_NAME, manifestJson, token);
|
||||
}
|
||||
|
||||
token.ThrowIfCancellationRequested();
|
||||
File.Move(temporaryArchivePath, archivePath, overwrite: true);
|
||||
|
||||
progress?.Report(new(processedChats, totalChats));
|
||||
|
||||
var exportedChats = manifestWorkspaces.SelectMany(workspace => workspace.Chats).Concat(manifestTemporaryChats).ToList();
|
||||
LOG.LogInformation("Exported {ChatCount} chats from {WorkspaceCount} workspaces to the archive '{ArchivePath}'.", exportedChats.Count, manifestWorkspaces.Count, archivePath);
|
||||
|
||||
return new(
|
||||
true,
|
||||
archivePath,
|
||||
manifestWorkspaces.Count,
|
||||
exportedChats.Count,
|
||||
exportedChats.Sum(chat => chat.IncludedFiles),
|
||||
unreadableChats,
|
||||
string.Empty,
|
||||
false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
TryDeleteArchive(temporaryArchivePath);
|
||||
LOG.LogInformation("The user cancelled the export to '{ArchivePath}'. The incomplete archive was removed.", archivePath);
|
||||
return new(true, archivePath, 0, 0, 0, 0, string.Empty, true);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
TryDeleteArchive(temporaryArchivePath);
|
||||
LOG.LogError(exception, "Failed to export chats to the archive '{ArchivePath}'.", archivePath);
|
||||
return new(false, archivePath, 0, 0, 0, 0, string.Format(TB("Unexpected error: {0}"), exception.Message), false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether the given selection contains any app-owned file which the export
|
||||
/// could include, so that the user is only asked about them when there are any.
|
||||
/// </summary>
|
||||
/// <param name="workspaceIds">The workspaces of the selection.</param>
|
||||
/// <param name="includeTemporaryChats">Whether the selection includes all temporary chats.</param>
|
||||
/// <param name="token">Cancels the search.</param>
|
||||
/// <returns>Whether at least one chat of the selection owns a file.</returns>
|
||||
public static async Task<bool> HasChatFilesAsync(IReadOnlyList<Guid> workspaceIds, bool includeTemporaryChats, CancellationToken token)
|
||||
{
|
||||
var tree = await WorkspaceBehaviour.GetOrLoadWorkspaceTreeShellAsync();
|
||||
foreach (var workspaceId in workspaceIds)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var chats = await WorkspaceBehaviour.GetWorkspaceChatsAsync(workspaceId);
|
||||
if (ChatArchiveChatFiles.HasAny(chats.Select(chat => chat.ChatPath), token))
|
||||
return true;
|
||||
}
|
||||
|
||||
return includeTemporaryChats && ChatArchiveChatFiles.HasAny(tree.TemporaryChats.Select(chat => chat.ChatPath), token);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes one chat into the archive: its name, optionally the files AI Studio owns for it,
|
||||
/// and its chat thread.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Documents the user attached are never written into the archive. Their paths stay
|
||||
/// absolute in the chat thread, which keeps them working wherever the documents live,
|
||||
/// for example on a document server, and keeps the archive free of copies of them.
|
||||
/// </remarks>
|
||||
/// <returns>The manifest entry for the chat, or null when the chat could not be read.</returns>
|
||||
private static async Task<ChatArchiveManifestChat?> ExportChatAsync(ZipArchive archive, string entryPrefix, Guid workspaceId, WorkspaceTreeChat chat, bool includeChatFiles, CancellationToken token)
|
||||
{
|
||||
var thread = await WorkspaceBehaviour.LoadChatAsync(new(workspaceId, chat.ChatId));
|
||||
if (thread is null)
|
||||
{
|
||||
LOG.LogWarning("Skipping chat '{ChatId}' of workspace '{WorkspaceId}' for the export, because it could not be read.", chat.ChatId, workspaceId);
|
||||
return null;
|
||||
}
|
||||
|
||||
var chatDirectory = chat.ChatPath;
|
||||
var chatEntryPrefix = $"{entryPrefix}{chat.ChatId}{ChatArchiveFormat.ENTRY_SEPARATOR}";
|
||||
await WriteTextEntryAsync(archive, $"{chatEntryPrefix}{ChatArchiveFormat.NAME_FILE_NAME}", chat.Name, token);
|
||||
|
||||
//
|
||||
// Write the files before rewriting any path: only a file which is really in the
|
||||
// archive may become an archive-relative path. Otherwise a file which could not be
|
||||
// read would leave a relative path behind, which the import cannot resolve.
|
||||
//
|
||||
var includedFiles = new Dictionary<string, string>(PathTools.COMPARER);
|
||||
if (includeChatFiles)
|
||||
{
|
||||
foreach (var (chatRelativePath, filePath) in ChatArchiveChatFiles.Collect(chatDirectory))
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
try
|
||||
{
|
||||
await WriteFileEntryAsync(archive, $"{chatEntryPrefix}{chatRelativePath}", filePath, token);
|
||||
includedFiles[chatRelativePath] = filePath;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogWarning(exception, "Failed to add the file '{FilePath}' of chat '{ChatId}' to the archive.", filePath, chat.ChatId);
|
||||
}
|
||||
}
|
||||
|
||||
ChatArchiveAttachmentPaths.Rewrite(thread, originalPath =>
|
||||
TryGetChatRelativePath(chatDirectory, originalPath, out var relativePath) && includedFiles.ContainsKey(relativePath)
|
||||
? relativePath
|
||||
: originalPath);
|
||||
}
|
||||
|
||||
var threadJson = JsonSerializer.Serialize(thread, WorkspaceBehaviour.JSON_OPTIONS);
|
||||
await WriteTextEntryAsync(archive, $"{chatEntryPrefix}{ChatArchiveFormat.THREAD_FILE_NAME}", threadJson, token);
|
||||
|
||||
return new()
|
||||
{
|
||||
ChatId = chat.ChatId,
|
||||
Name = chat.Name,
|
||||
LastEditTime = chat.LastEditTime,
|
||||
IncludedFiles = includedFiles.Count,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether the given path points into the chat directory and returns the
|
||||
/// archive-relative path for it.
|
||||
/// </summary>
|
||||
private static bool TryGetChatRelativePath(string chatDirectory, string path, out string relativePath)
|
||||
{
|
||||
relativePath = string.Empty;
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
var relative = Path.GetRelativePath(Path.GetFullPath(chatDirectory), Path.GetFullPath(path));
|
||||
if (Path.IsPathRooted(relative) || relative == ".." || relative.StartsWith($"..{Path.DirectorySeparatorChar}", PathTools.COMPARISON))
|
||||
return false;
|
||||
|
||||
relativePath = relative.Replace(Path.DirectorySeparatorChar, ChatArchiveFormat.ENTRY_SEPARATOR);
|
||||
return true;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogWarning(exception, "Could not determine whether the path '{Path}' belongs to the chat directory '{ChatDirectory}'.", path, chatDirectory);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task WriteTextEntryAsync(ZipArchive archive, string entryName, string content, CancellationToken token)
|
||||
{
|
||||
var entry = archive.CreateEntry(entryName, CompressionLevel.Optimal);
|
||||
await using var entryStream = entry.Open();
|
||||
await using var writer = new StreamWriter(entryStream, new UTF8Encoding(false));
|
||||
await writer.WriteAsync(content.AsMemory(), token);
|
||||
}
|
||||
|
||||
private static async Task WriteFileEntryAsync(ZipArchive archive, string entryName, string filePath, CancellationToken token)
|
||||
{
|
||||
// Compressing an already compressed attachment costs time for nothing:
|
||||
var compressionLevel = ALREADY_COMPRESSED_EXTENSIONS.Contains(Path.GetExtension(filePath))
|
||||
? CompressionLevel.Fastest
|
||||
: CompressionLevel.Optimal;
|
||||
|
||||
var entry = archive.CreateEntry(entryName, compressionLevel);
|
||||
|
||||
// The ZIP format cannot store timestamps before 1980:
|
||||
var lastWriteTime = File.GetLastWriteTime(filePath);
|
||||
if (lastWriteTime.Year >= 1980)
|
||||
entry.LastWriteTime = lastWriteTime;
|
||||
|
||||
await using var entryStream = entry.Open();
|
||||
await using var fileStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await fileStream.CopyToAsync(entryStream, token);
|
||||
}
|
||||
|
||||
private static void TryDeleteArchive(string archivePath)
|
||||
{
|
||||
if (!File.Exists(archivePath))
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
File.Delete(archivePath);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogWarning(exception, "Failed to delete the incomplete archive '{ArchivePath}'.", archivePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Central definition of the chat archive format. A chat archive is a ZIP file which mirrors
|
||||
/// the chat storage layout, so that exported chats can be restored without any conversion.
|
||||
/// </summary>
|
||||
public static class ChatArchiveFormat
|
||||
{
|
||||
/// <summary>
|
||||
/// The archive format version written by this app version. The importer refuses archives
|
||||
/// with a higher version, because it cannot know what those archives contain.
|
||||
/// </summary>
|
||||
public const int FORMAT_VERSION = 1;
|
||||
|
||||
/// <summary>
|
||||
/// The file extension used for chat archives.
|
||||
/// </summary>
|
||||
public const string FILE_EXTENSION = ".aistudio-chats";
|
||||
|
||||
/// <summary>
|
||||
/// The manifest file at the archive root. It describes what the archive contains.
|
||||
/// </summary>
|
||||
public const string MANIFEST_FILE_NAME = "manifest.json";
|
||||
|
||||
/// <summary>
|
||||
/// The archive directory containing all exported workspaces.
|
||||
/// </summary>
|
||||
public const string WORKSPACES_DIRECTORY = "workspaces";
|
||||
|
||||
/// <summary>
|
||||
/// The archive directory containing all exported temporary chats.
|
||||
/// </summary>
|
||||
public const string TEMPORARY_CHATS_DIRECTORY = "tempChats";
|
||||
|
||||
/// <summary>
|
||||
/// The file storing the workspace or chat name.
|
||||
/// </summary>
|
||||
public const string NAME_FILE_NAME = "name";
|
||||
|
||||
/// <summary>
|
||||
/// The file storing the serialized chat thread.
|
||||
/// </summary>
|
||||
public const string THREAD_FILE_NAME = "thread.json";
|
||||
|
||||
/// <summary>
|
||||
/// The separator used inside ZIP archives. ZIP entries always use forward slashes,
|
||||
/// regardless of the operating system.
|
||||
/// </summary>
|
||||
public const char ENTRY_SEPARATOR = '/';
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// The outcome of importing one single chat.
|
||||
/// </summary>
|
||||
public enum ChatArchiveImportOutcome
|
||||
{
|
||||
/// <summary>
|
||||
/// The chat was written to the chat storage.
|
||||
/// </summary>
|
||||
IMPORTED,
|
||||
|
||||
/// <summary>
|
||||
/// The chat already existed and was left alone.
|
||||
/// </summary>
|
||||
SKIPPED,
|
||||
|
||||
/// <summary>
|
||||
/// The chat could not be imported.
|
||||
/// </summary>
|
||||
FAILED,
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Describes the contents of an archive before anything is imported.
|
||||
/// </summary>
|
||||
/// <param name="Success">Whether the archive could be read.</param>
|
||||
/// <param name="Manifest">The manifest of the archive.</param>
|
||||
/// <param name="Issue">The reason why the archive could not be read, if it could not.</param>
|
||||
public sealed record ChatArchiveImportPreview(bool Success, ChatArchiveManifest Manifest, string Issue);
|
||||
@@ -0,0 +1,20 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// The outcome of a chat archive import.
|
||||
/// </summary>
|
||||
/// <param name="Success">Whether the import ran without an error.</param>
|
||||
/// <param name="ImportedWorkspaces">The number of workspaces which received chats.</param>
|
||||
/// <param name="ImportedChats">The number of imported chats.</param>
|
||||
/// <param name="SkippedChats">The number of chats which were skipped, because they already existed.</param>
|
||||
/// <param name="FailedChats">The number of chats which could not be imported.</param>
|
||||
/// <param name="Issue">The reason why the import failed, if it did.</param>
|
||||
/// <param name="Cancelled">Whether the user stopped the import. Chats imported up to that point are kept.</param>
|
||||
public sealed record ChatArchiveImportResult(
|
||||
bool Success,
|
||||
int ImportedWorkspaces,
|
||||
int ImportedChats,
|
||||
int SkippedChats,
|
||||
int FailedChats,
|
||||
string Issue,
|
||||
bool Cancelled);
|
||||
@@ -0,0 +1,519 @@
|
||||
using System.IO.Compression;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Reads chat archives and writes their contents back into the chat storage.
|
||||
/// </summary>
|
||||
public static class ChatArchiveImporter
|
||||
{
|
||||
private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger(nameof(ChatArchiveImporter));
|
||||
|
||||
private static readonly int BUFFER_SIZE = 64 * 1024;
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ChatArchiveImporter).Namespace, nameof(ChatArchiveImporter));
|
||||
|
||||
/// <summary>
|
||||
/// Returns the entry name with forward slashes. The ZIP format prescribes them, but
|
||||
/// some tools on Windows write backslashes instead. Reading both keeps archives from
|
||||
/// such tools importable.
|
||||
/// </summary>
|
||||
private static string NormalizeEntryName(string entryName) => entryName.Replace('\\', ChatArchiveFormat.ENTRY_SEPARATOR);
|
||||
|
||||
/// <summary>
|
||||
/// Reads the manifest of an archive without importing anything.
|
||||
/// </summary>
|
||||
/// <param name="archivePath">The archive to inspect.</param>
|
||||
/// <param name="token">Cancels reading the manifest.</param>
|
||||
/// <returns>The preview of the archive contents.</returns>
|
||||
public static async Task<ChatArchiveImportPreview> ReadPreviewAsync(string archivePath, CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
using var archive = ZipFile.OpenRead(archivePath);
|
||||
var manifestEntry = archive.GetEntry(ChatArchiveFormat.MANIFEST_FILE_NAME);
|
||||
if (manifestEntry is null)
|
||||
return new(false, new(), TB("This file is not a chat archive."));
|
||||
|
||||
await using var manifestStream = manifestEntry.Open();
|
||||
var manifest = await JsonSerializer.DeserializeAsync<ChatArchiveManifest>(manifestStream, WorkspaceBehaviour.JSON_OPTIONS, token);
|
||||
if (manifest is null)
|
||||
return new(false, new(), TB("The content list of this archive could not be read."));
|
||||
|
||||
if (manifest.FormatVersion > ChatArchiveFormat.FORMAT_VERSION)
|
||||
return new(false, manifest, TB("This archive was created by a newer version of AI Studio. Please update AI Studio to import it."));
|
||||
|
||||
return new(true, manifest, string.Empty);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
throw;
|
||||
}
|
||||
catch (InvalidDataException)
|
||||
{
|
||||
return new(false, new(), TB("This file is not a chat archive."));
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogError(exception, "Failed to read the archive '{ArchivePath}'.", archivePath);
|
||||
return new(false, new(), string.Format(TB("Unexpected error: {0}"), exception.Message));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Imports all chats of an archive into the chat storage.
|
||||
/// </summary>
|
||||
/// <param name="archivePath">The archive to import.</param>
|
||||
/// <param name="collisionBehavior">What to do with chats which already exist.</param>
|
||||
/// <param name="progress">Receives the import progress.</param>
|
||||
/// <param name="token">Cancels the import. Chats imported before the cancellation are kept.</param>
|
||||
/// <returns>The import result.</returns>
|
||||
public static async Task<ChatArchiveImportResult> ImportAsync(string archivePath, ChatArchiveCollisionBehavior collisionBehavior, IProgress<ChatArchiveProgress>? progress, CancellationToken token)
|
||||
{
|
||||
var importedWorkspaces = 0;
|
||||
var importedChats = 0;
|
||||
var skippedChats = 0;
|
||||
var failedChats = 0;
|
||||
|
||||
var preview = await ReadPreviewAsync(archivePath, token);
|
||||
if (!preview.Success)
|
||||
return new(false, 0, 0, 0, 0, preview.Issue, false);
|
||||
|
||||
try
|
||||
{
|
||||
var manifest = preview.Manifest;
|
||||
var totalChats = manifest.TotalChatCount;
|
||||
var processedChats = 0;
|
||||
|
||||
using var archive = ZipFile.OpenRead(archivePath);
|
||||
var entriesByChat = GroupEntriesByChat(archive);
|
||||
|
||||
// Which workspaces exist already is asked once per archived workspace, so read
|
||||
// the tree once instead of building a full snapshot every time:
|
||||
var tree = await WorkspaceBehaviour.GetOrLoadWorkspaceTreeShellAsync();
|
||||
var knownWorkspaceIds = tree.Workspaces.Select(workspace => workspace.WorkspaceId).ToHashSet();
|
||||
|
||||
//
|
||||
// A chat counts as existing wherever it is stored, not just in the workspace the
|
||||
// archive names: it may have been moved since the export. Importing it there again
|
||||
// under the same identity would leave two chats with one ID, while the app finds a
|
||||
// chat, and the answer being written for it, by that ID alone:
|
||||
//
|
||||
var existingChatIds = tree.TemporaryChats.Select(chat => chat.ChatId).ToHashSet();
|
||||
foreach (var workspace in tree.Workspaces)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
foreach (var chat in await WorkspaceBehaviour.GetWorkspaceChatsAsync(workspace.WorkspaceId))
|
||||
existingChatIds.Add(chat.ChatId);
|
||||
}
|
||||
|
||||
//
|
||||
// Workspace chats and temporary chats differ only in where they are stored.
|
||||
// Every counter is raised inside the loop: a cancellation leaves the chats
|
||||
// imported so far in the storage, so the numbers have to describe them even
|
||||
// when the loop never reaches its end.
|
||||
//
|
||||
async Task ImportChatsAsync(string entryPrefix, Guid targetWorkspaceId, IReadOnlyList<ChatArchiveManifestChat> chats, bool countsAsWorkspace)
|
||||
{
|
||||
var isFirstImportOfWorkspace = true;
|
||||
foreach (var chat in chats)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
progress?.Report(new(processedChats, totalChats));
|
||||
processedChats++;
|
||||
|
||||
var chatEntryPrefix = $"{entryPrefix}{chat.ChatId}{ChatArchiveFormat.ENTRY_SEPARATOR}";
|
||||
var chatEntries = entriesByChat.GetValueOrDefault(chatEntryPrefix, []);
|
||||
|
||||
switch (await ImportChatAsync(chatEntries, chatEntryPrefix, targetWorkspaceId, chat, collisionBehavior, existingChatIds, token))
|
||||
{
|
||||
case ChatArchiveImportOutcome.IMPORTED:
|
||||
importedChats++;
|
||||
if (countsAsWorkspace && isFirstImportOfWorkspace)
|
||||
{
|
||||
importedWorkspaces++;
|
||||
isFirstImportOfWorkspace = false;
|
||||
}
|
||||
|
||||
break;
|
||||
|
||||
case ChatArchiveImportOutcome.SKIPPED:
|
||||
skippedChats++;
|
||||
break;
|
||||
|
||||
default:
|
||||
failedChats++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
foreach (var workspace in manifest.Workspaces)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var targetWorkspaceId = await ResolveTargetWorkspaceAsync(workspace, knownWorkspaceIds);
|
||||
if (targetWorkspaceId == Guid.Empty)
|
||||
{
|
||||
LOG.LogWarning("Skipping workspace '{WorkspaceName}' of the archive, because no target workspace could be prepared.", workspace.Name);
|
||||
failedChats += workspace.Chats.Count;
|
||||
processedChats += workspace.Chats.Count;
|
||||
continue;
|
||||
}
|
||||
|
||||
var workspaceEntryPrefix = $"{ChatArchiveFormat.WORKSPACES_DIRECTORY}{ChatArchiveFormat.ENTRY_SEPARATOR}{workspace.WorkspaceId}{ChatArchiveFormat.ENTRY_SEPARATOR}";
|
||||
await ImportChatsAsync(workspaceEntryPrefix, targetWorkspaceId, workspace.Chats, countsAsWorkspace: true);
|
||||
}
|
||||
|
||||
var temporaryChatsEntryPrefix = $"{ChatArchiveFormat.TEMPORARY_CHATS_DIRECTORY}{ChatArchiveFormat.ENTRY_SEPARATOR}";
|
||||
await ImportChatsAsync(temporaryChatsEntryPrefix, Guid.Empty, manifest.TemporaryChats, countsAsWorkspace: false);
|
||||
|
||||
progress?.Report(new(processedChats, totalChats));
|
||||
LOG.LogInformation("Imported {ImportedChats} chats from the archive '{ArchivePath}'. Skipped {SkippedChats} existing chats, {FailedChats} chats failed.", importedChats, archivePath, skippedChats, failedChats);
|
||||
|
||||
return new(true, importedWorkspaces, importedChats, skippedChats, failedChats, string.Empty, false);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
//
|
||||
// Unlike a cancelled export, which leaves no archive behind, the chats imported
|
||||
// so far stay in the chat storage. So report what happened instead of throwing,
|
||||
// otherwise the user has no idea what is now part of their chats:
|
||||
//
|
||||
LOG.LogInformation("The user cancelled the import of the archive '{ArchivePath}' after {ImportedChats} chats.", archivePath, importedChats);
|
||||
return new(true, importedWorkspaces, importedChats, skippedChats, failedChats, string.Empty, true);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogError(exception, "Failed to import the archive '{ArchivePath}'.", archivePath);
|
||||
return new(false, importedWorkspaces, importedChats, skippedChats, failedChats, string.Format(TB("Unexpected error: {0}"), exception.Message), false);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Sorts all archive entries by the chat they belong to. Without this, every chat would
|
||||
/// have to scan all entries of the archive.
|
||||
/// </summary>
|
||||
private static Dictionary<string, List<ZipArchiveEntry>> GroupEntriesByChat(ZipArchive archive)
|
||||
{
|
||||
var entriesByChat = new Dictionary<string, List<ZipArchiveEntry>>(StringComparer.Ordinal);
|
||||
foreach (var entry in archive.Entries)
|
||||
{
|
||||
var entryName = NormalizeEntryName(entry.FullName);
|
||||
|
||||
// Skip directory entries:
|
||||
if (entryName.EndsWith(ChatArchiveFormat.ENTRY_SEPARATOR))
|
||||
continue;
|
||||
|
||||
var chatEntryPrefix = GetChatEntryPrefix(entryName);
|
||||
if (chatEntryPrefix is null)
|
||||
continue;
|
||||
|
||||
if (!entriesByChat.TryGetValue(chatEntryPrefix, out var chatEntries))
|
||||
{
|
||||
chatEntries = [];
|
||||
entriesByChat[chatEntryPrefix] = chatEntries;
|
||||
}
|
||||
|
||||
chatEntries.Add(entry);
|
||||
}
|
||||
|
||||
return entriesByChat;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the chat an archive entry belongs to: everything up to and including the
|
||||
/// chat identifier, so <c>workspaces/W/C/thread.json</c> becomes <c>workspaces/W/C/</c>.
|
||||
/// </summary>
|
||||
/// <returns>The chat prefix, or null when the entry belongs to no chat.</returns>
|
||||
private static string? GetChatEntryPrefix(string entryName)
|
||||
{
|
||||
var expectedSeparators = 0;
|
||||
if (entryName.StartsWith($"{ChatArchiveFormat.WORKSPACES_DIRECTORY}{ChatArchiveFormat.ENTRY_SEPARATOR}", StringComparison.Ordinal))
|
||||
expectedSeparators = 3;
|
||||
else if (entryName.StartsWith($"{ChatArchiveFormat.TEMPORARY_CHATS_DIRECTORY}{ChatArchiveFormat.ENTRY_SEPARATOR}", StringComparison.Ordinal))
|
||||
expectedSeparators = 2;
|
||||
|
||||
if (expectedSeparators is 0)
|
||||
return null;
|
||||
|
||||
var index = 0;
|
||||
for (var separator = 0; separator < expectedSeparators; separator++)
|
||||
{
|
||||
var next = entryName.IndexOf(ChatArchiveFormat.ENTRY_SEPARATOR, index);
|
||||
if (next < 0)
|
||||
return null;
|
||||
|
||||
index = next + 1;
|
||||
}
|
||||
|
||||
return entryName[..index];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines the workspace which receives the chats of an archived workspace. Known
|
||||
/// workspaces are reused, unknown ones are created with their original identity.
|
||||
/// </summary>
|
||||
/// <param name="workspace">The archived workspace.</param>
|
||||
/// <param name="knownWorkspaceIds">The workspaces which exist already. Newly created ones are added.</param>
|
||||
private static async Task<Guid> ResolveTargetWorkspaceAsync(ChatArchiveManifestWorkspace workspace, HashSet<Guid> knownWorkspaceIds)
|
||||
{
|
||||
if (workspace.WorkspaceId == Guid.Empty)
|
||||
return Guid.Empty;
|
||||
|
||||
if (knownWorkspaceIds.Contains(workspace.WorkspaceId))
|
||||
return workspace.WorkspaceId;
|
||||
|
||||
var workspaceName = string.IsNullOrWhiteSpace(workspace.Name) ? TB("Imported workspace") : workspace.Name;
|
||||
await WorkspaceBehaviour.EnsureWorkspace(workspace.WorkspaceId, await CreateUniqueWorkspaceNameAsync(workspaceName));
|
||||
knownWorkspaceIds.Add(workspace.WorkspaceId);
|
||||
|
||||
return workspace.WorkspaceId;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Extends the workspace name by a counter until it is no longer taken. Workspace names
|
||||
/// must be unique, otherwise the workspace cannot be renamed later on.
|
||||
/// </summary>
|
||||
private static async Task<string> CreateUniqueWorkspaceNameAsync(string workspaceName)
|
||||
{
|
||||
var candidate = workspaceName;
|
||||
var counter = 1;
|
||||
|
||||
while (await WorkspaceBehaviour.IsWorkspaceNameExistingAsync(candidate))
|
||||
{
|
||||
counter++;
|
||||
candidate = $"{workspaceName} ({counter})";
|
||||
}
|
||||
|
||||
return candidate;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Imports one chat: extracts its files into a fresh chat directory, turns the
|
||||
/// archive-relative attachment paths back into absolute ones, and stores the chat.
|
||||
/// </summary>
|
||||
/// <param name="existingChatIds">The IDs of all chats in the storage. An imported chat is added.</param>
|
||||
private static async Task<ChatArchiveImportOutcome> ImportChatAsync(IReadOnlyList<ZipArchiveEntry> chatEntries, string chatEntryPrefix, Guid targetWorkspaceId, ChatArchiveManifestChat chat, ChatArchiveCollisionBehavior collisionBehavior, HashSet<Guid> existingChatIds, CancellationToken token)
|
||||
{
|
||||
var threadEntry = chatEntries.FirstOrDefault(entry => NormalizeEntryName(entry.FullName).Equals($"{chatEntryPrefix}{ChatArchiveFormat.THREAD_FILE_NAME}", StringComparison.OrdinalIgnoreCase));
|
||||
if (threadEntry is null)
|
||||
{
|
||||
LOG.LogWarning("The archive contains no chat thread for the chat '{ChatId}'.", chat.ChatId);
|
||||
return ChatArchiveImportOutcome.FAILED;
|
||||
}
|
||||
|
||||
//
|
||||
// The directory is checked as well: a chat the storage could not list, for example
|
||||
// because its thread is unreadable, still occupies its place on disk.
|
||||
//
|
||||
var targetChatId = chat.ChatId;
|
||||
if (existingChatIds.Contains(targetChatId) || WorkspaceBehaviour.IsChatExisting(new(targetWorkspaceId, targetChatId)))
|
||||
{
|
||||
if (collisionBehavior is ChatArchiveCollisionBehavior.SKIP)
|
||||
return ChatArchiveImportOutcome.SKIPPED;
|
||||
|
||||
targetChatId = Guid.NewGuid();
|
||||
}
|
||||
|
||||
var chatDirectory = WorkspaceBehaviour.GetChatDirectory(targetWorkspaceId, targetChatId);
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(chatDirectory);
|
||||
|
||||
//
|
||||
// Extract everything except the chat thread and the name file. Both are written
|
||||
// by the chat storage itself once the paths inside the thread are corrected:
|
||||
//
|
||||
var extractedPaths = new HashSet<string>(PathTools.COMPARER);
|
||||
foreach (var entry in chatEntries)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
|
||||
var relativePath = NormalizeEntryName(entry.FullName)[chatEntryPrefix.Length..];
|
||||
if (relativePath.Equals(ChatArchiveFormat.THREAD_FILE_NAME, StringComparison.OrdinalIgnoreCase) ||
|
||||
relativePath.Equals(ChatArchiveFormat.NAME_FILE_NAME, StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
var destinationPath = ResolveSafeDestination(chatDirectory, relativePath);
|
||||
if (destinationPath is null)
|
||||
{
|
||||
LOG.LogWarning("Ignoring the archive entry '{EntryName}', because it points outside of the chat directory.", entry.FullName);
|
||||
continue;
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
|
||||
|
||||
await using var entryStream = entry.Open();
|
||||
await using var fileStream = new FileStream(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
await entryStream.CopyToAsync(fileStream, token);
|
||||
|
||||
extractedPaths.Add(relativePath);
|
||||
}
|
||||
|
||||
ChatThread? thread;
|
||||
await using (var threadStream = threadEntry.Open())
|
||||
thread = await JsonSerializer.DeserializeAsync<ChatThread>(threadStream, WorkspaceBehaviour.JSON_OPTIONS, token);
|
||||
|
||||
if (thread is null)
|
||||
{
|
||||
LOG.LogWarning("The chat thread of the chat '{ChatId}' could not be read.", chat.ChatId);
|
||||
TryDeleteDirectory(chatDirectory);
|
||||
return ChatArchiveImportOutcome.FAILED;
|
||||
}
|
||||
|
||||
thread = thread with { ChatId = targetChatId };
|
||||
thread.WorkspaceId = targetWorkspaceId;
|
||||
|
||||
//
|
||||
// Turn the archive-relative paths back into absolute ones. Only paths whose files
|
||||
// were actually extracted are rewritten. Everything else was already absolute when
|
||||
// the archive was created and stays as it is:
|
||||
//
|
||||
ChatArchiveAttachmentPaths.Rewrite(thread, path =>
|
||||
{
|
||||
var normalizedPath = path.Replace(Path.DirectorySeparatorChar, ChatArchiveFormat.ENTRY_SEPARATOR);
|
||||
if (!extractedPaths.Contains(normalizedPath))
|
||||
return path;
|
||||
|
||||
return Path.Combine(chatDirectory, normalizedPath.Replace(ChatArchiveFormat.ENTRY_SEPARATOR, Path.DirectorySeparatorChar));
|
||||
});
|
||||
|
||||
CopyTranscriptsStoredElsewhere(thread, chatDirectory);
|
||||
|
||||
//
|
||||
// The storage skips a chat whose lock another operation holds for too long. The
|
||||
// directory prepared above would then hold the extracted files but no chat:
|
||||
//
|
||||
if (!await WorkspaceBehaviour.StoreChatAsync(thread))
|
||||
{
|
||||
LOG.LogWarning("The chat '{ChatId}' could not be stored, because another operation held it for too long.", chat.ChatId);
|
||||
TryDeleteDirectory(chatDirectory);
|
||||
return ChatArchiveImportOutcome.FAILED;
|
||||
}
|
||||
|
||||
existingChatIds.Add(targetChatId);
|
||||
return ChatArchiveImportOutcome.IMPORTED;
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
TryDeleteDirectory(chatDirectory);
|
||||
throw;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogError(exception, "Failed to import the chat '{ChatId}' into the workspace '{WorkspaceId}'.", chat.ChatId, targetWorkspaceId);
|
||||
TryDeleteDirectory(chatDirectory);
|
||||
return ChatArchiveImportOutcome.FAILED;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Gives the imported chat its own copy of every transcript the archive did not carry.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Such a transcript still names the place where it lay when the archive was made. On the
|
||||
/// same computer that is the chat the archive came from, and since a chat deletes the
|
||||
/// transcripts it names, deleting the imported chat would take the transcripts of the
|
||||
/// original along. A copied chat gets its own copies for the same reason, so this uses the
|
||||
/// same code. A transcript which no longer exists keeps its place in the chat and shows up
|
||||
/// as missing, again as it does in a copy.
|
||||
/// </remarks>
|
||||
/// <param name="thread">The imported chat thread, after its archive-relative paths were resolved.</param>
|
||||
/// <param name="chatDirectory">The directory of the imported chat.</param>
|
||||
private static void CopyTranscriptsStoredElsewhere(ChatThread thread, string chatDirectory)
|
||||
{
|
||||
var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts");
|
||||
var copiedPaths = new Dictionary<string, ManagedTranscriptAttachment>(PathTools.COMPARER);
|
||||
|
||||
//
|
||||
// Only a transcript in the data directory can belong to another chat, and only there
|
||||
// does deleting a chat delete its transcripts. Any other path is left alone, so that an
|
||||
// archive cannot get files from elsewhere on this computer copied into a chat:
|
||||
//
|
||||
ManagedTranscriptAttachment CopyWhenElsewhere(ManagedTranscriptAttachment transcript) =>
|
||||
IsInsideDirectory(chatDirectory, transcript.FilePath) || !IsInsideDirectory(SettingsManager.DataDirectory!, transcript.FilePath)
|
||||
? transcript
|
||||
: WorkspaceBehaviour.CopyManagedTranscriptAttachment(thread, transcript, transcriptDirectory, copiedPaths);
|
||||
|
||||
foreach (var content in thread.Blocks.Select(block => block.Content).OfType<IContent>())
|
||||
{
|
||||
for (var index = 0; index < content.FileAttachments.Count; index++)
|
||||
{
|
||||
if (content.FileAttachments[index] is ManagedTranscriptAttachment transcript)
|
||||
content.FileAttachments[index] = CopyWhenElsewhere(transcript);
|
||||
}
|
||||
}
|
||||
|
||||
for (var index = 0; index < thread.PendingMediaTranscripts.Count; index++)
|
||||
thread.PendingMediaTranscripts[index] = CopyWhenElsewhere(thread.PendingMediaTranscripts[index]);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a path lies below the given directory.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A path which cannot be resolved counts as lying elsewhere. The copy which follows then
|
||||
/// decides what becomes of it, and it keeps such an attachment as it is.
|
||||
/// </remarks>
|
||||
private static bool IsInsideDirectory(string directory, string path)
|
||||
{
|
||||
try
|
||||
{
|
||||
var fullDirectory = Path.GetFullPath(directory);
|
||||
var directoryPrefix = fullDirectory.EndsWith(Path.DirectorySeparatorChar)
|
||||
? fullDirectory
|
||||
: fullDirectory + Path.DirectorySeparatorChar;
|
||||
|
||||
return Path.GetFullPath(path).StartsWith(directoryPrefix, PathTools.COMPARISON);
|
||||
}
|
||||
catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Combines the chat directory with an archive-relative path and rejects paths which
|
||||
/// would escape the chat directory.
|
||||
/// </summary>
|
||||
private static string? ResolveSafeDestination(string chatDirectory, string relativePath)
|
||||
{
|
||||
try
|
||||
{
|
||||
var fullChatDirectory = Path.GetFullPath(chatDirectory);
|
||||
var destinationPath = Path.GetFullPath(Path.Combine(fullChatDirectory, relativePath.Replace(ChatArchiveFormat.ENTRY_SEPARATOR, Path.DirectorySeparatorChar)));
|
||||
|
||||
var directoryPrefix = fullChatDirectory.EndsWith(Path.DirectorySeparatorChar)
|
||||
? fullChatDirectory
|
||||
: fullChatDirectory + Path.DirectorySeparatorChar;
|
||||
|
||||
return destinationPath.StartsWith(directoryPrefix, PathTools.COMPARISON) ? destinationPath : null;
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogWarning(exception, "Could not resolve the archive path '{RelativePath}' inside the chat directory '{ChatDirectory}'.", relativePath, chatDirectory);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static void TryDeleteDirectory(string directory)
|
||||
{
|
||||
if (!Directory.Exists(directory))
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
Directory.Delete(directory, true);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
LOG.LogWarning(exception, "Failed to clean up the incomplete chat directory '{ChatDirectory}'.", directory);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
using System.Text.Json.Serialization;
|
||||
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// The manifest at the root of a chat archive. It describes the archive contents, so that
|
||||
/// the import can show a preview without extracting anything.
|
||||
/// </summary>
|
||||
public sealed record ChatArchiveManifest
|
||||
{
|
||||
/// <summary>
|
||||
/// The format version of this archive.
|
||||
/// </summary>
|
||||
public int FormatVersion { get; init; } = ChatArchiveFormat.FORMAT_VERSION;
|
||||
|
||||
/// <summary>
|
||||
/// The time when this archive was created.
|
||||
/// </summary>
|
||||
public DateTimeOffset ExportedAt { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The AI Studio version which created this archive. Informational only.
|
||||
/// </summary>
|
||||
public string AppVersion { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// All workspaces contained in this archive.
|
||||
/// </summary>
|
||||
public List<ChatArchiveManifestWorkspace> Workspaces { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// All temporary chats contained in this archive.
|
||||
/// </summary>
|
||||
public List<ChatArchiveManifestChat> TemporaryChats { get; init; } = [];
|
||||
|
||||
/// <summary>
|
||||
/// Gets the total number of chats in this archive.
|
||||
/// </summary>
|
||||
[JsonIgnore]
|
||||
public int TotalChatCount => this.Workspaces.Sum(workspace => workspace.Chats.Count) + this.TemporaryChats.Count;
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Describes one exported chat inside a chat archive.
|
||||
/// </summary>
|
||||
public sealed record ChatArchiveManifestChat
|
||||
{
|
||||
/// <summary>
|
||||
/// The unique identifier of the chat.
|
||||
/// </summary>
|
||||
public Guid ChatId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The name of the chat at the time of the export.
|
||||
/// </summary>
|
||||
public string Name { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The time of the last edit at the time of the export.
|
||||
/// </summary>
|
||||
public DateTimeOffset LastEditTime { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The number of app-owned files of this chat which were included in the archive,
|
||||
/// such as transcripts.
|
||||
/// </summary>
|
||||
public int IncludedFiles { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,22 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Describes one exported workspace inside a chat archive.
|
||||
/// </summary>
|
||||
public sealed record ChatArchiveManifestWorkspace
|
||||
{
|
||||
/// <summary>
|
||||
/// The unique identifier of the workspace.
|
||||
/// </summary>
|
||||
public Guid WorkspaceId { get; init; }
|
||||
|
||||
/// <summary>
|
||||
/// The name of the workspace at the time of the export.
|
||||
/// </summary>
|
||||
public string Name { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// All chats of this workspace contained in the archive.
|
||||
/// </summary>
|
||||
public List<ChatArchiveManifestChat> Chats { get; init; } = [];
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace AIStudio.Tools.ChatArchive;
|
||||
|
||||
/// <summary>
|
||||
/// Reports the progress of an export or import operation.
|
||||
/// </summary>
|
||||
/// <param name="ProcessedChats">The number of chats processed so far.</param>
|
||||
/// <param name="TotalChats">The total number of chats to process.</param>
|
||||
public readonly record struct ChatArchiveProgress(int ProcessedChats, int TotalChats);
|
||||
@@ -0,0 +1,18 @@
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Shared rules for comparing file system paths.
|
||||
/// </summary>
|
||||
public static class PathTools
|
||||
{
|
||||
/// <summary>
|
||||
/// How to compare file system paths: Windows treats them case-insensitively, Linux and
|
||||
/// macOS do not. There, two files may differ in casing alone.
|
||||
/// </summary>
|
||||
public static readonly StringComparison COMPARISON = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
|
||||
|
||||
/// <summary>
|
||||
/// The comparer matching <see cref="COMPARISON"/>, for path-keyed lookups.
|
||||
/// </summary>
|
||||
public static readonly StringComparer COMPARER = OperatingSystem.IsWindows() ? StringComparer.OrdinalIgnoreCase : StringComparer.Ordinal;
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
using AIStudio.Tools.ChatArchive;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
// ReSharper disable MemberCanBePrivate.Global
|
||||
@@ -104,6 +105,7 @@ public static class FileTypes
|
||||
public static readonly FileTypeFilter EXECUTABLES = FileTypeFilter.Leaf(TB("Executable"), "exe", "app", "bin", "appimage");
|
||||
public static readonly FileTypeFilter SHORTCUT = FileTypeFilter.Leaf(TB("Shortcut"), "lnk");
|
||||
public static readonly FileTypeFilter PLUGIN_ARCHIVE = FileTypeFilter.Leaf(TB("Plugin archive"), PluginArchive.PLUGIN_FILE_EXTENSION.TrimStart('.'), "zip");
|
||||
public static readonly FileTypeFilter CHAT_ARCHIVE = FileTypeFilter.Leaf(TB("Chat archive"), ChatArchiveFormat.FILE_EXTENSION.TrimStart('.'), "zip");
|
||||
|
||||
/// <summary>
|
||||
/// The file types AI Studio converts using Pandoc.
|
||||
|
||||
@@ -875,7 +875,7 @@ public static class WorkspaceBehaviour
|
||||
chat.PendingMediaTranscripts[index] = CopyManagedTranscriptAttachment(chat, chat.PendingMediaTranscripts[index], targetTranscriptDirectory, copiedPaths);
|
||||
}
|
||||
|
||||
private static ManagedTranscriptAttachment CopyManagedTranscriptAttachment(
|
||||
internal static ManagedTranscriptAttachment CopyManagedTranscriptAttachment(
|
||||
ChatThread chat,
|
||||
ManagedTranscriptAttachment source,
|
||||
string targetTranscriptDirectory,
|
||||
@@ -1130,7 +1130,7 @@ public static class WorkspaceBehaviour
|
||||
}
|
||||
|
||||
/// <summary>Returns the canonical storage directory for a chat identity.</summary>
|
||||
private static string GetChatDirectory(Guid workspaceId, Guid chatId) => workspaceId == Guid.Empty
|
||||
public static string GetChatDirectory(Guid workspaceId, Guid chatId) => workspaceId == Guid.Empty
|
||||
? Path.Join(SettingsManager.DataDirectory, "tempChats", chatId.ToString())
|
||||
: Path.Join(SettingsManager.DataDirectory, "workspaces", workspaceId.ToString(), chatId.ToString());
|
||||
|
||||
@@ -1280,7 +1280,11 @@ public static class WorkspaceBehaviour
|
||||
return true;
|
||||
}
|
||||
|
||||
private static async Task EnsureWorkspace(Guid workspaceId, string workspaceName)
|
||||
/// <summary>
|
||||
/// Ensures that a workspace with the given identity exists on disk and in the cache.
|
||||
/// An existing workspace keeps its current name.
|
||||
/// </summary>
|
||||
public static async Task EnsureWorkspace(Guid workspaceId, string workspaceName)
|
||||
{
|
||||
var workspacePath = Path.Join(WORKSPACE_ROOT_DIRECTORY, workspaceId.ToString());
|
||||
var workspaceNamePath = Path.Join(workspacePath, "name");
|
||||
|
||||
Reference in new issue
Block a user