From dc97db256e24651137e8a7625d0e6c4ba52e6da1 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Sat, 14 Nov 2026 21:16:40 +0100 Subject: [PATCH] Improved how AI Studio saves its files (#1201) --- ...istantBatchProcessing.razor.Persistence.cs | 26 +- .../BeyondFactsLab/BeyondFactsLabStore.cs | 57 +--- .../BeyondFactsLab/BflPromptOverrides.cs | 9 +- .../Assistants/I18N/allTexts.lua | 6 +- .../ModelComparison/ModelComparisonStore.cs | 41 +-- .../VisualBriefingAssistant.razor | 24 +- .../VisualBriefingStore.Projects.cs | 3 +- .../VisualBriefing/VisualBriefingStore.cs | 24 +- .../Chat/ManagedTranscriptAttachment.cs | 34 +-- .../ProviderSelectionWithConfidence.razor | 10 +- .../ProviderSelectionWithConfidence.razor.cs | 15 + app/MindWork AI Studio/Pages/Supporters.razor | 2 +- .../plugin.lua | 6 +- .../plugin.lua | 6 +- .../Settings/SettingsManager.cs | 32 +-- app/MindWork AI Studio/Tools/AtomicFile.cs | 196 +++++++++++++ .../Tools/Media/PersistentMediaTranscript.cs | 26 +- .../WorkspaceBehaviour.RelativeDataPaths.cs | 15 +- .../Tools/WorkspaceBehaviour.cs | 38 +-- .../wwwroot/changelog/v26.11.2.md | 2 + .../BeyondFactsLabStoreTests.cs | 14 +- app/Tests/Settings/SettingsStorageTests.cs | 5 +- app/Tests/Tools/AtomicFileTests.cs | 263 ++++++++++++++++++ app/Tests/Tools/WorkspaceChatStoreTests.cs | 30 ++ 24 files changed, 626 insertions(+), 258 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/AtomicFile.cs create mode 100644 app/Tests/Tools/AtomicFileTests.cs diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Persistence.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Persistence.cs index b537248e1..bd883ee29 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Persistence.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Persistence.cs @@ -129,32 +129,20 @@ public partial class AssistantBatchProcessing private async Task WriteCsvFileAsync(string targetFilePath, string content) { - // Write to a sibling file first, then rename. This way, an aborted - // write can never destroy the results of the previous files: - var tempFilePath = targetFilePath + ".tmp"; try { - // We write the CSV file with a byte order mark, so that spreadsheet - // applications recognize the UTF-8 encoding of, e.g., umlauts: - await File.WriteAllTextAsync(tempFilePath, content, new UTF8Encoding(true), CancellationToken.None); - File.Move(tempFilePath, targetFilePath, true); + // AtomicFile writes to a sibling file first, then renames it. This way, an aborted + // write can never destroy the results of the previous files. It also removes the + // sibling file when the write fails, so that it neither looks like a result to the + // user nor piles up over several runs. We write the CSV file with a byte order + // mark, so that spreadsheet applications recognize the UTF-8 encoding of, e.g., + // umlauts: + await AtomicFile.WriteTextAsync(targetFilePath, content, new UTF8Encoding(true), CancellationToken.None, logger: this.Logger); } catch (Exception e) { this.Logger.LogError(e, "Was not able to write the batch output file '{TargetFilePath}'.", targetFilePath); - // Remove our leftover: a failing rename keeps the temporary file in - // the output folder, where it looks like a result to the user and - // piles up over several runs. - try - { - File.Delete(tempFilePath); - } - catch (Exception deleteError) - { - this.Logger.LogWarning(deleteError, "Was not able to remove the temporary file '{TempFilePath}'.", tempFilePath); - } - // A failing write repeats for every document. We report it once per // run: without any message, the UI would show a successful run // while the files on disk stay behind. diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabStore.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabStore.cs index 683655dd2..f5801cb61 100644 --- a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabStore.cs +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BeyondFactsLabStore.cs @@ -81,58 +81,32 @@ public sealed partial class BeyondFactsLabStore(ILogger log /// /// Writes a text file so that readers see either the old or the new content, never a part of it. /// + /// + /// The content reaches the disk before it replaces the file: a preparation is the work of hours, and a power + /// failure must not take it away. + /// /// The file to write. /// The new content. /// The cancellation token. - private static Task WriteTextAtomicAsync(string targetPath, string content, CancellationToken token) => WriteAtomicAsync(targetPath, temporaryPath => File.WriteAllTextAsync(temporaryPath, content, new UTF8Encoding(false), token), token); + private static Task WriteTextAtomicAsync(string targetPath, string content, CancellationToken token) + { + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + return AtomicFile.WriteTextAsync(targetPath, content, new UTF8Encoding(false), token, flushToDisk: true); + } /// /// Writes a binary file so that readers see either the old or the new content, never a part of it. /// + /// + /// Like WriteTextAtomicAsync, the content reaches the disk before it replaces the file. + /// /// The file to write. /// The new content. /// The cancellation token. - private static Task WriteBytesAtomicAsync(string targetPath, ReadOnlyMemory content, CancellationToken token) => WriteAtomicAsync(targetPath, temporaryPath => File.WriteAllBytesAsync(temporaryPath, content, token), token); - - /// - /// Writes a file beside its target first and moves it there once it is complete. - /// - /// The file to write. - /// Writes the new content into the file of the path given. - /// The cancellation token. - private static async Task WriteAtomicAsync(string targetPath, Func writeTemporaryFile, CancellationToken token) + private static Task WriteBytesAtomicAsync(string targetPath, ReadOnlyMemory content, CancellationToken token) { Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); - var temporaryPath = $"{targetPath}.tmp-{Guid.NewGuid():N}"; - try - { - await writeTemporaryFile(temporaryPath); - await using (var stream = new FileStream(temporaryPath, FileMode.Open, FileAccess.ReadWrite, FileShare.None, 4_096, true)) - await stream.FlushAsync(token); - - File.Move(temporaryPath, targetPath, overwrite: true); - } - finally - { - TryDeleteFile(temporaryPath); - } - } - - /// - /// Deletes a file when it exists, without failing. - /// - /// The file to delete. - private static void TryDeleteFile(string path) - { - try - { - if (File.Exists(path)) - File.Delete(path); - } - catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) - { - // Cleaning up a temporary file is best effort; the next start tries again. - } + return AtomicFile.WriteAsync(targetPath, (stream, writeToken) => stream.WriteAsync(content, writeToken).AsTask(), token, flushToDisk: true); } /// @@ -197,8 +171,7 @@ public sealed partial class BeyondFactsLabStore(ILogger log return; Directory.CreateDirectory(this.RootDirectory); - foreach (var temporaryPath in Directory.EnumerateFiles(this.RootDirectory, "*.tmp-*", SearchOption.AllDirectories)) - TryDeleteFile(temporaryPath); + AtomicFile.DeleteTemporaryFiles(this.RootDirectory, logger); await this.LoadSelectionAsync(token); this.initialized = true; diff --git a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflPromptOverrides.cs b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflPromptOverrides.cs index 292f9621b..2b6ce811a 100644 --- a/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflPromptOverrides.cs +++ b/app/MindWork AI Studio/Assistants/BeyondFactsLab/BflPromptOverrides.cs @@ -103,17 +103,16 @@ public static class BflPromptOverrides continue; Directory.CreateDirectory(Path.GetDirectoryName(target)!); - var temporary = $"{target}.{Guid.NewGuid():N}.tmp"; - await File.WriteAllTextAsync(temporary, await BflPromptLibrary.ReadBuiltInAsync(path, token), new UTF8Encoding(false), token); + var prompt = await BflPromptLibrary.ReadBuiltInAsync(path, token); try { - // Moving fails when the file appeared meanwhile, so the person's file is never overwritten: - File.Move(temporary, target, false); + // Writing fails when the file appeared meanwhile, so the person's file is never overwritten: + await AtomicFile.WriteTextAsync(target, prompt, new UTF8Encoding(false), token, overwrite: false, flushToDisk: true); copied++; } catch (IOException) when (File.Exists(target)) { - File.Delete(temporary); + // The person's file stays, and AtomicFile removed its temporary file already. } } diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index fbc80a555..2f3b72365 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -18769,6 +18769,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3060804484"] = "Content Contribut -- Financial Support UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3061261435"] = "Financial Support" +-- Thank you, Dominic, for supporting us not only financially but also with code: the Model Comparison assistant, a smoother chat, the audio quality setting for transcriptions, personal API keys for providers of an organization, sharing feature usage with self-hosted servers, and a better Confluence search. +UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3091322681"] = "Thank you, Dominic, for supporting us not only financially but also with code: the Model Comparison assistant, a smoother chat, the audio quality setting for transcriptions, personal API keys for providers of an organization, sharing feature usage with self-hosted servers, and a better Confluence search." + -- Thank you, Prodman, for bringing LiteLLM to AI Studio as a new provider. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3226413110"] = "Thank you, Prodman, for bringing LiteLLM to AI Studio as a new provider." @@ -18781,9 +18784,6 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3264998714"] = "German Aerospace -- The first 10 supporters who make a monthly contribution: UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3364384944"] = "The first 10 supporters who make a monthly contribution:" --- Thank you, Dominic, for supporting us not only financially but also with code: a smoother chat, the audio quality setting for transcriptions, personal API keys for providers of an organization, sharing feature usage with self-hosted servers, and a better Confluence search. -UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T348735370"] = "Thank you, Dominic, for supporting us not only financially but also with code: a smoother chat, the audio quality setting for transcriptions, personal API keys for providers of an organization, sharing feature usage with self-hosted servers, and a better Confluence search." - -- Thanks, Eric, for bringing llmman to AI Studio as a new option for self-hosted models. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3520213035"] = "Thanks, Eric, for bringing llmman to AI Studio as a new option for self-hosted models." diff --git a/app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonStore.cs b/app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonStore.cs index 22b523a54..7836713b7 100644 --- a/app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonStore.cs +++ b/app/MindWork AI Studio/Assistants/ModelComparison/ModelComparisonStore.cs @@ -1,5 +1,4 @@ using System.Collections.Concurrent; -using System.Text; using System.Text.Json; using AIStudio.Settings; @@ -79,44 +78,19 @@ public sealed partial class ModelComparisonStore(ILogger l /// /// Writes a file of the store so that readers see either the old or the new content, never a part of it. /// + /// + /// The content reaches the disk before it replaces the file: the results of a series took weeks of work, and a + /// power failure must not take them away. + /// /// The type of the file. /// The file to write. /// The new content. /// The cancellation token. /// The JSON options, those of the assistant by default; a chat takes those of the workspaces. - private static async Task WriteJsonAtomicAsync(string targetPath, T value, CancellationToken token, JsonSerializerOptions? options = null) + private static Task WriteJsonAtomicAsync(string targetPath, T value, CancellationToken token, JsonSerializerOptions? options = null) { Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); - var temporaryPath = $"{targetPath}.tmp-{Guid.NewGuid():N}"; - try - { - await File.WriteAllTextAsync(temporaryPath, JsonSerializer.Serialize(value, options ?? JSON_OPTIONS), new UTF8Encoding(false), token); - await using (var stream = new FileStream(temporaryPath, FileMode.Open, FileAccess.ReadWrite, FileShare.None, 4_096, true)) - await stream.FlushAsync(token); - - File.Move(temporaryPath, targetPath, overwrite: true); - } - finally - { - TryDeleteFile(temporaryPath); - } - } - - /// - /// Deletes a file when it exists, without failing. - /// - /// The file to delete. - private static void TryDeleteFile(string path) - { - try - { - if (File.Exists(path)) - File.Delete(path); - } - catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) - { - // Cleaning up a temporary file is best effort; the next start tries again. - } + return AtomicFile.WriteJsonAsync(targetPath, value, options ?? JSON_OPTIONS, token, flushToDisk: true); } /// @@ -150,8 +124,7 @@ public sealed partial class ModelComparisonStore(ILogger l return; Directory.CreateDirectory(this.RootDirectory); - foreach (var temporaryPath in Directory.EnumerateFiles(this.RootDirectory, "*.tmp-*", SearchOption.AllDirectories)) - TryDeleteFile(temporaryPath); + AtomicFile.DeleteTemporaryFiles(this.RootDirectory, logger); this.initialized = true; } diff --git a/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingAssistant.razor b/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingAssistant.razor index fad710c2a..b7cfbc525 100644 --- a/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingAssistant.razor +++ b/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingAssistant.razor @@ -192,27 +192,9 @@ @T("Briefing settings") - @* - The confidence belongs to the provider chosen right next to it, so both share one row. - It uses the icon trigger, like the chat does, so this row ends the same way the profile - row below it does: a field followed by one compact icon button. - Do not add a margin to that button to "correct" its height: a dense outlined select with - a label carries margin-top 8px and margin-bottom 4px of its own, so centring the boxes - already lands within a few pixels of the visible frame, and any added margin makes it - worse. Baseline alignment does not work here either, because the wrapper below takes - its baseline from its last line box, which sits under the input. - *@ - - @* ProviderSelection marks its select as flex-grow-0, and that utility is declared - !important, so StretchItems cannot widen it. The width has to come from here. *@ -
- -
- @if (this.SettingsManager.ConfigurationData.Confidence.ShowProviderConfidence) - { - - } - + @* The provider row ends in a field followed by one compact icon button, the same way the + profile row below it does. *@ + diff --git a/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingStore.Projects.cs b/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingStore.Projects.cs index 5bab50640..b737c1864 100644 --- a/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingStore.Projects.cs +++ b/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingStore.Projects.cs @@ -98,8 +98,7 @@ public sealed partial class VisualBriefingStore return; Directory.CreateDirectory(this.RootDirectory); - foreach (var temporaryPath in Directory.EnumerateFiles(this.RootDirectory, "*.tmp-*", SearchOption.AllDirectories)) - TryDeleteFile(temporaryPath); + AtomicFile.DeleteTemporaryFiles(this.RootDirectory, logger); await this.LoadSelectionAsync(token); foreach (var directory in Directory.EnumerateDirectories(this.RootDirectory)) diff --git a/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingStore.cs b/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingStore.cs index c2fec2e29..0a9965e9a 100644 --- a/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingStore.cs +++ b/app/MindWork AI Studio/Assistants/VisualBriefing/VisualBriefingStore.cs @@ -111,26 +111,22 @@ public sealed partial class VisualBriefingStore( } /// - /// Defines WriteTextAtomicAsync for the visual briefing feature. + /// Writes a text file of a briefing so that readers see either the old or the new content, never a part of it. /// - private static async Task WriteTextAtomicAsync(string targetPath, + /// + /// The content reaches the disk before it replaces the file, so that a power failure takes no briefing away. + /// + /// The file to write. + /// The new content. + /// Whether an existing file is replaced; otherwise, an existing file fails the write with an IOException. + /// The cancellation token. + private static Task WriteTextAtomicAsync(string targetPath, string content, bool overwrite, CancellationToken token) { Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); - var temporaryPath = $"{targetPath}.tmp-{Guid.NewGuid():N}"; - try - { - await File.WriteAllTextAsync(temporaryPath, content, new UTF8Encoding(false), token); - await using (var stream = new FileStream(temporaryPath, FileMode.Open, FileAccess.ReadWrite, FileShare.None, 4_096, true)) - await stream.FlushAsync(token); - File.Move(temporaryPath, targetPath, overwrite); - } - finally - { - TryDeleteFile(temporaryPath); - } + return AtomicFile.WriteTextAsync(targetPath, content, new UTF8Encoding(false), token, overwrite, flushToDisk: true); } /// diff --git a/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs b/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs index c9df93bbc..6bbfe69aa 100644 --- a/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs +++ b/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs @@ -46,7 +46,7 @@ public sealed record ManagedTranscriptAttachment(string FileName, string FilePat Directory.CreateDirectory(operationDirectory); var originalFileName = Path.GetFileName(originalPath); var stagingPath = Path.Combine(operationDirectory, $"{Guid.NewGuid():N}.md"); - await WriteMarkdownAsync(stagingPath, originalFileName, transcript); + await File.WriteAllTextAsync(stagingPath, BuildMarkdown(originalFileName, transcript), new UTF8Encoding(false)); return FromPath(stagingPath, originalFileName, isStaged: true); } @@ -58,18 +58,8 @@ public sealed record ManagedTranscriptAttachment(string FileName, string FilePat internal static async Task CreateAtomicAsync(string targetPath, string originalFileName, string transcript) { Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); - var temporaryPath = Path.Combine(Path.GetDirectoryName(targetPath)!, $".{Guid.NewGuid():N}.tmp"); - try - { - await WriteMarkdownAsync(temporaryPath, originalFileName, transcript); - File.Move(temporaryPath, targetPath); - return FromPath(targetPath, originalFileName, isStaged: false); - } - finally - { - if (File.Exists(temporaryPath)) - File.Delete(temporaryPath); - } + await AtomicFile.WriteTextAsync(targetPath, BuildMarkdown(originalFileName, transcript), new UTF8Encoding(false), CancellationToken.None, overwrite: false); + return FromPath(targetPath, originalFileName, isStaged: false); } /// Deletes a file only when its canonical path has an exact managed structure. @@ -162,15 +152,11 @@ public sealed record ManagedTranscriptAttachment(string FileName, string FilePat originalFileName, isStaged); - /// Writes localized transcript Markdown without a UTF-8 byte-order mark. - private static async Task WriteMarkdownAsync(string path, string originalFileName, string transcript) - { - var markdown = $""" - # Transcription: {originalFileName} - - {transcript.Trim()} - """; - - await File.WriteAllTextAsync(path, markdown, new UTF8Encoding(false)); - } + /// Builds the Markdown of a transcript; both callers write it without a UTF-8 byte-order mark. + private static string BuildMarkdown(string originalFileName, string transcript) => + $""" + # Transcription: {originalFileName} + + {transcript.Trim()} + """; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ProviderSelectionWithConfidence.razor b/app/MindWork AI Studio/Components/ProviderSelectionWithConfidence.razor index 66a373c3a..1959952bf 100644 --- a/app/MindWork AI Studio/Components/ProviderSelectionWithConfidence.razor +++ b/app/MindWork AI Studio/Components/ProviderSelectionWithConfidence.razor @@ -1,10 +1,18 @@ @inherits MSGComponentBase +@* + The confidence belongs to the provider chosen right next to it, so both share one row. It uses the icon trigger, + like the chat does, so the row ends in a field followed by one compact icon button. + Do not add a margin to that button to "correct" its height: a dense outlined select with a label carries + margin-top 8px and margin-bottom 4px of its own, so centring the boxes already lands within a few pixels of the + visible frame, and any added margin makes it worse. Baseline alignment does not work here either, because the + wrapper below takes its baseline from its last line box, which sits under the input. +*@ @* ProviderSelection marks its select as flex-grow-0, and that utility is declared !important, so StretchItems cannot widen it. The width has to come from here. *@
- +
@if (this.SettingsManager.ConfigurationData.Confidence.ShowProviderConfidence) { diff --git a/app/MindWork AI Studio/Components/ProviderSelectionWithConfidence.razor.cs b/app/MindWork AI Studio/Components/ProviderSelectionWithConfidence.razor.cs index d40ea3dca..5c12bdb99 100644 --- a/app/MindWork AI Studio/Components/ProviderSelectionWithConfidence.razor.cs +++ b/app/MindWork AI Studio/Components/ProviderSelectionWithConfidence.razor.cs @@ -1,3 +1,5 @@ +using AIStudio.Provider; + using Microsoft.AspNetCore.Components; namespace AIStudio.Components; @@ -29,4 +31,17 @@ public partial class ProviderSelectionWithConfidence : MSGComponentBase /// [Parameter] public Func ValidateProvider { get; set; } = _ => null; + + /// + /// Gets or sets the confidence a provider needs at least to be offered, when it is higher than the minimum of the + /// component, see ProviderSelection.ExplicitMinimumConfidence. + /// + [Parameter] + public ConfidenceLevel ExplicitMinimumConfidence { get; set; } = ConfidenceLevel.UNKNOWN; + + /// + /// Gets or sets whether the selection is disabled, see ProviderSelection.Disabled. + /// + [Parameter] + public bool Disabled { get; set; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Pages/Supporters.razor b/app/MindWork AI Studio/Pages/Supporters.razor index f74c4591b..25d5011f2 100644 --- a/app/MindWork AI Studio/Pages/Supporters.razor +++ b/app/MindWork AI Studio/Pages/Supporters.razor @@ -95,7 +95,7 @@ - + diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 23d466900..89f6347e5 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -18771,6 +18771,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3060804484"] = "Inhaltliche Beitr -- Financial Support UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3061261435"] = "Finanzielle Unterstützung" +-- Thank you, Dominic, for supporting us not only financially but also with code: the Model Comparison assistant, a smoother chat, the audio quality setting for transcriptions, personal API keys for providers of an organization, sharing feature usage with self-hosted servers, and a better Confluence search. +UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3091322681"] = "Vielen Dank, Dominic, dass du uns nicht nur finanziell, sondern auch mit Code unterstützt: mit dem Assistenten „Modellvergleich“, einem reibungsloseren Chat, der Einstellung für die Audioqualität der Transkription, persönlichen API-Schlüsseln für Anbieter einer Organisation, der Möglichkeit, die Funktionsnutzung mit selbst gehosteten Servern zu teilen, und einer besseren Confluence-Suche." + -- Thank you, Prodman, for bringing LiteLLM to AI Studio as a new provider. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3226413110"] = "Vielen Dank, Prodman, dass du LiteLLM als neuen Anbieter in AI Studio integriert hast." @@ -18783,9 +18786,6 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3264998714"] = "Deutsches Zentrum -- The first 10 supporters who make a monthly contribution: UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3364384944"] = "Die ersten 10 Spender, die einen monatlichen Beitrag leisten:" --- Thank you, Dominic, for supporting us not only financially but also with code: a smoother chat, the audio quality setting for transcriptions, personal API keys for providers of an organization, sharing feature usage with self-hosted servers, and a better Confluence search. -UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T348735370"] = "Vielen Dank, Dominic, dass du uns nicht nur finanziell, sondern auch mit Code unterstützt: mit einem reibungsloseren Chat, der Einstellung für die Audioqualität der Transkription, persönlichen API-Schlüsseln für Anbieter einer Organisation, der Möglichkeit, die Funktionsnutzung mit selbst gehosteten Servern zu teilen, und einer besseren Confluence-Suche." - -- Thanks, Eric, for bringing llmman to AI Studio as a new option for self-hosted models. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3520213035"] = "Danke, Eric, dass du llmman als neue Option für selbst gehostete Modelle in AI Studio integriert hast." diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 7d50da6ab..da20a2e2b 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -18771,6 +18771,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3060804484"] = "Content Contribut -- Financial Support UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3061261435"] = "Financial Support" +-- Thank you, Dominic, for supporting us not only financially but also with code: the Model Comparison assistant, a smoother chat, the audio quality setting for transcriptions, personal API keys for providers of an organization, sharing feature usage with self-hosted servers, and a better Confluence search. +UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3091322681"] = "Thank you, Dominic, for supporting us not only financially but also with code: the Model Comparison assistant, a smoother chat, the audio quality setting for transcriptions, personal API keys for providers of an organization, sharing feature usage with self-hosted servers, and a better Confluence search." + -- Thank you, Prodman, for bringing LiteLLM to AI Studio as a new provider. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3226413110"] = "Thank you, Prodman, for bringing LiteLLM to AI Studio as a new provider." @@ -18783,9 +18786,6 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3264998714"] = "German Aerospace -- The first 10 supporters who make a monthly contribution: UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3364384944"] = "The first 10 supporters who make a monthly contribution:" --- Thank you, Dominic, for supporting us not only financially but also with code: a smoother chat, the audio quality setting for transcriptions, personal API keys for providers of an organization, sharing feature usage with self-hosted servers, and a better Confluence search. -UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T348735370"] = "Thank you, Dominic, for supporting us not only financially but also with code: a smoother chat, the audio quality setting for transcriptions, personal API keys for providers of an organization, sharing feature usage with self-hosted servers, and a better Confluence search." - -- Thanks, Eric, for bringing llmman to AI Studio as a new option for self-hosted models. UI_TEXT_CONTENT["AISTUDIO::PAGES::SUPPORTERS::T3520213035"] = "Thanks, Eric, for bringing llmman to AI Studio as a new option for self-hosted models." diff --git a/app/MindWork AI Studio/Settings/SettingsManager.cs b/app/MindWork AI Studio/Settings/SettingsManager.cs index 6a4a82962..2424728ff 100644 --- a/app/MindWork AI Studio/Settings/SettingsManager.cs +++ b/app/MindWork AI Studio/Settings/SettingsManager.cs @@ -452,33 +452,13 @@ public sealed class SettingsManager } // - // We write the new settings next to the previous ones and replace them afterwards, so that - // no crash can leave a half-written settings file behind. The temporary file has to live in - // the configuration directory for that: replacing a file is a rename, and a rename across a - // file system boundary falls back to copying, which is exactly what we want to avoid. The - // temporary directory of the operating system is such another file system under Flatpak. + // AtomicFile writes the new settings next to the previous ones and replaces them afterwards, + // so that no crash can leave a half-written settings file behind. The settings also have to + // reach the disk first: the backup file is written at the same moment, so after a power + // failure, it would be no better off than the settings file, and falling back to it blocks + // storing the settings anyway. // - var tempFile = $"{settingsPath}.tmp-{Guid.NewGuid():N}"; - try - { - await File.WriteAllTextAsync(tempFile, settingsJson); - File.Move(tempFile, settingsPath, true); - } - catch - { - try - { - if (File.Exists(tempFile)) - File.Delete(tempFile); - } - catch (Exception cleanupException) - { - this.logger.LogWarning(cleanupException, $"Failed to delete the temporary settings file '{tempFile}'."); - } - - throw; - } - + await AtomicFile.WriteTextAsync(settingsPath, settingsJson, new UTF8Encoding(false), CancellationToken.None, flushToDisk: true, logger: this.logger); this.logger.LogInformation($"Stored the settings to '{settingsPath}'."); } diff --git a/app/MindWork AI Studio/Tools/AtomicFile.cs b/app/MindWork AI Studio/Tools/AtomicFile.cs new file mode 100644 index 000000000..e2e1843cb --- /dev/null +++ b/app/MindWork AI Studio/Tools/AtomicFile.cs @@ -0,0 +1,196 @@ +using System.Text; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace AIStudio.Tools; + +/// +/// Writes files so that readers see either the old or the new content, never a part of it. +/// +/// +/// The new content goes into a temporary file next to the target first, which then replaces the target by a rename. +/// The temporary file has to live in the directory of the target: a rename across a file system boundary falls back +/// to copying, which is exactly what this avoids, and the temporary directory of the operating system is such another +/// file system under Flatpak. The temporary file is named after its target, followed by a GUID and ".tmp", so that +/// a crash leaves a file behind which DeleteTemporaryFiles, DataSourceTranscriptStore, and MediaSidecarTranscripts +/// recognize.

+/// This works only where AI Studio may create files next to the target: in its data and configuration directories, +/// or in a folder the user chose as a whole, e.g., the output folder of the Batch Processing assistant. It does not +/// work for a single file the user chose in a save or an open dialog or dropped onto AI Studio: under Flatpak, the +/// document portal shares only that one file, so nothing can be created next to it. Prepare such a file in the +/// temporary directory and write it into the chosen file with FileMode.Create, as PresentationFile and +/// PluginShareService do. Keep in mind, too, that the Flatpak sees the home directory read-only (--filesystem=home:ro). +///

+/// The rename protects against a crash of AI Studio. Against a power failure, the content has to reach the disk +/// before the rename, which flushToDisk asks for. That costs time on every write, especially on macOS and on SD cards, +/// so only files which are hard to get back use it. +/// +public static partial class AtomicFile +{ + private const int BUFFER_SIZE = 65_536; + + /// + /// Writes a file, unless the writer decides to discard what it wrote. + /// + /// The file to write. Its directory has to exist. + /// Writes the new content into the stream and returns whether it replaces the target. + /// The cancellation token. + /// Whether an existing target is replaced; otherwise, an existing target fails the write with an IOException. + /// Whether the content has to reach the disk before it replaces the target. + /// Where to report a temporary file which cannot be removed. + /// True when the new content replaced the target, false when the writer discarded it. + public static async Task WriteOrDiscardAsync(string targetPath, Func> write, CancellationToken token, bool overwrite = true, bool flushToDisk = false, ILogger? logger = null) + { + var temporaryPath = $"{targetPath}.{Guid.NewGuid():N}.tmp"; + try + { + bool keep; + await using (var stream = new FileStream(temporaryPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous)) + { + keep = await write(stream, token); + if (keep && flushToDisk) + await FlushToDiskAsync(stream, token); + } + + if (!keep) + { + TryDelete(temporaryPath, logger); + return false; + } + + File.Move(temporaryPath, targetPath, overwrite); + return true; + } + catch + { + // Deleting never throws, so the original error is the one which reaches the caller: + TryDelete(temporaryPath, logger); + throw; + } + } + + /// + /// Writes a file. + /// + /// The file to write. Its directory has to exist. + /// Writes the new content into the stream. + /// The cancellation token. + /// Whether an existing target is replaced; otherwise, an existing target fails the write with an IOException. + /// Whether the content has to reach the disk before it replaces the target. + /// Where to report a temporary file which cannot be removed. + public static Task WriteAsync(string targetPath, Func write, CancellationToken token, bool overwrite = true, bool flushToDisk = false, ILogger? logger = null) => WriteOrDiscardAsync(targetPath, async (stream, writeToken) => + { + await write(stream, writeToken); + return true; + }, token, overwrite, flushToDisk, logger); + + /// + /// Writes a text file. + /// + /// + /// The file starts with the byte order mark of the encoding, if it has one, just like with File.WriteAllTextAsync: + /// Encoding.UTF8 writes one, new UTF8Encoding(false) does not. + /// + /// The file to write. Its directory has to exist. + /// The new content. + /// The encoding of the file. + /// The cancellation token. + /// Whether an existing target is replaced; otherwise, an existing target fails the write with an IOException. + /// Whether the content has to reach the disk before it replaces the target. + /// Where to report a temporary file which cannot be removed. + public static Task WriteTextAsync(string targetPath, string content, Encoding encoding, CancellationToken token, bool overwrite = true, bool flushToDisk = false, ILogger? logger = null) => + WriteAsync(targetPath, (stream, writeToken) => WriteTextAsync(stream, content, encoding, writeToken), token, overwrite, flushToDisk, logger); + + /// + /// Writes a text into the stream of a file, for a writer passed to WriteAsync or WriteOrDiscardAsync. + /// + /// + /// The stream gets the byte order mark of the encoding, if it has one, just like a file written by File.WriteAllTextAsync. + /// + /// The stream of the file. + /// The text. + /// The encoding of the file. + /// The cancellation token. + public static async Task WriteTextAsync(Stream stream, string content, Encoding encoding, CancellationToken token) + { + await using var writer = new StreamWriter(stream, encoding, BUFFER_SIZE, leaveOpen: true); + await writer.WriteAsync(content.AsMemory(), token); + await writer.FlushAsync(token); + } + + /// + /// Writes a JSON file, as UTF-8 without a byte order mark. + /// + /// + /// The value goes into the file piece by piece, so that the complete JSON is never in memory at once. The file + /// holds the same bytes as when the value was serialized into a string and written by File.WriteAllTextAsync. + /// + /// The type of the value. + /// The file to write. Its directory has to exist. + /// The value to write. + /// The JSON options. + /// The cancellation token. + /// Whether an existing target is replaced; otherwise, an existing target fails the write with an IOException. + /// Whether the content has to reach the disk before it replaces the target. + /// Where to report a temporary file which cannot be removed. + public static Task WriteJsonAsync(string targetPath, T value, JsonSerializerOptions options, CancellationToken token, bool overwrite = true, bool flushToDisk = false, ILogger? logger = null) => + WriteAsync(targetPath, (stream, writeToken) => JsonSerializer.SerializeAsync(stream, value, options, writeToken), token, overwrite, flushToDisk, logger); + + /// + /// Deletes the temporary files which a crash left behind anywhere below a directory. + /// + /// + /// Besides the temporary files of this class, it also deletes those which AI Studio named ".tmp-" followed by a + /// GUID before, so that the leftovers of older versions disappear as well. Call it only for a directory which + /// belongs to AI Studio alone. + /// + /// The directory to clean up. + /// Where to report a temporary file which cannot be removed. + public static void DeleteTemporaryFiles(string rootDirectory, ILogger? logger = null) + { + foreach (var path in Directory.EnumerateFiles(rootDirectory, "*.tmp*", SearchOption.AllDirectories)) + { + var fileName = Path.GetFileName(path); + if (TemporaryFileName().IsMatch(fileName) || PreviousTemporaryFileName().IsMatch(fileName)) + TryDelete(path, logger); + } + } + + /// + /// Makes sure the content of a file reached the disk. + /// + /// + /// FlushAsync hands the buffered content to the operating system only; just Flush(true) waits for the disk, and + /// that only synchronously. It runs on the thread pool, so that it does not block the caller, e.g., the renderer. + /// + /// The stream of the file. + /// The cancellation token. + private static async Task FlushToDiskAsync(FileStream stream, CancellationToken token) + { + await stream.FlushAsync(token); + await Task.Run(() => stream.Flush(flushToDisk: true), token); + } + + /// + /// Deletes a file, without failing. + /// + /// The file to delete; a file which does not exist is fine. + /// Where to report a file which cannot be removed. + private static void TryDelete(string path, ILogger? logger) + { + try + { + File.Delete(path); + } + catch (Exception exception) when (exception is IOException or UnauthorizedAccessException) + { + logger?.LogWarning(exception, "Could not remove the temporary file '{TemporaryPath}'.", path); + } + } + + [GeneratedRegex(@"\.[0-9a-f]{32}\.tmp$")] + private static partial Regex TemporaryFileName(); + + [GeneratedRegex(@"\.tmp-[0-9a-f]{32}$")] + private static partial Regex PreviousTemporaryFileName(); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscript.cs b/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscript.cs index 550e197ed..a8d29b291 100644 --- a/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscript.cs +++ b/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscript.cs @@ -119,7 +119,6 @@ internal static class PersistentMediaTranscript if (!sourceBefore.StillMatches(mediaPath)) return SourceChanged(slot); - var temporaryPath = $"{slot.Path}.{Guid.NewGuid():N}.tmp"; try { var directory = Path.GetDirectoryName(slot.Path); @@ -128,13 +127,18 @@ internal static class PersistentMediaTranscript // // The transcription is paid for at this point. Storing it is quick, so it is completed - // even when the caller cancelled meanwhile, and the next attempt can use it: + // even when the caller cancelled meanwhile, and the next attempt can use it. A transcript + // of a media file which changed meanwhile is discarded instead of replacing the old one: // - await File.WriteAllTextAsync(temporaryPath, transcription.Text, new UTF8Encoding(false), CancellationToken.None); - if (!sourceBefore.StillMatches(mediaPath)) + var stored = await AtomicFile.WriteOrDiscardAsync(slot.Path, async (stream, writeToken) => + { + await AtomicFile.WriteTextAsync(stream, transcription.Text, new UTF8Encoding(false), writeToken); + return sourceBefore.StillMatches(mediaPath); + }, CancellationToken.None, logger: logger); + + if (!stored) return SourceChanged(slot); - File.Move(temporaryPath, slot.Path, true); logger.LogInformation("Stored the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath); return PersistentMediaTranscriptResult.Ready(slot, reused: false); } @@ -143,18 +147,6 @@ internal static class PersistentMediaTranscript logger.LogWarning(exception, "Could not store the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath); return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.STORE_FAILED, slot, string.Format(TB("Was not able to store the transcript: {0}"), exception.Message)); } - finally - { - try - { - if (File.Exists(temporaryPath)) - File.Delete(temporaryPath); - } - catch (Exception exception) - { - logger.LogWarning(exception, "Could not remove the temporary transcript '{TempFilePath}'.", temporaryPath); - } - } } /// diff --git a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.RelativeDataPaths.cs b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.RelativeDataPaths.cs index e4d175782..ec9ebb9e4 100644 --- a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.RelativeDataPaths.cs +++ b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.RelativeDataPaths.cs @@ -81,19 +81,8 @@ public static partial class WorkspaceBehaviour return true; var lastEditTime = File.GetLastWriteTimeUtc(chatPath); - var temporaryPath = Path.Join(chatDirectory, $".thread-{Guid.NewGuid():N}.tmp"); - try - { - await File.WriteAllTextAsync(temporaryPath, json, Encoding.UTF8, token); - File.Move(temporaryPath, chatPath, overwrite: true); - File.SetLastWriteTimeUtc(chatPath, lastEditTime); - } - finally - { - if (File.Exists(temporaryPath)) - File.Delete(temporaryPath); - } - + await AtomicFile.WriteTextAsync(chatPath, json, Encoding.UTF8, token, logger: LOG); + File.SetLastWriteTimeUtc(chatPath, lastEditTime); return true; } catch (Exception exception) when (exception is JsonException or NotSupportedException) diff --git a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs index 7123698fb..7c6b5a87e 100644 --- a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs +++ b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs @@ -938,13 +938,9 @@ public static partial class WorkspaceBehaviour Directory.CreateDirectory(chatDirectory); FinalizeStagedTranscripts(chat, chatDirectory); - var chatNamePath = Path.Join(chatDirectory, "name"); - await File.WriteAllTextAsync(chatNamePath, chat.Name); + await StoreChatFilesAsync(chat, chatDirectory); - var chatPath = Path.Join(chatDirectory, "thread.json"); - await File.WriteAllTextAsync(chatPath, JsonSerializer.Serialize(chat, JSON_OPTIONS), Encoding.UTF8); - - var lastEditTime = File.GetLastWriteTimeUtc(chatPath); + var lastEditTime = File.GetLastWriteTimeUtc(Path.Join(chatDirectory, "thread.json")); await UpdateCacheAfterChatStored(chat.WorkspaceId, chat.ChatId, chatDirectory, chat.Name, lastEditTime); return true; } @@ -1174,7 +1170,7 @@ public static partial class WorkspaceBehaviour movedChat.WorkspaceId = targetWorkspaceId; FinalizeStagedTranscripts(movedChat, targetDirectory); - await StoreMovedChatFilesAsync(movedChat, targetDirectory); + await StoreChatFilesAsync(movedChat, targetDirectory); var lastEditTime = File.GetLastWriteTimeUtc(Path.Join(targetDirectory, "thread.json")); await UpdateCacheAfterChatMoved(sourceWorkspaceId, targetWorkspaceId, chat.ChatId, targetDirectory, movedChat.Name, lastEditTime); @@ -1202,7 +1198,7 @@ public static partial class WorkspaceBehaviour Directory.Move(targetDirectory, sourceDirectory); if (movedChat is not null && Directory.Exists(sourceDirectory)) - await StoreMovedChatFilesAsync(movedChat, sourceDirectory); + await StoreChatFilesAsync(movedChat, sourceDirectory); } catch (Exception rollbackError) { @@ -1229,23 +1225,17 @@ public static partial class WorkspaceBehaviour return JsonSerializer.Deserialize(chatData, JSON_OPTIONS) ?? throw new InvalidDataException($"The stored chat could not be read: '{chatDirectory}'."); } - /// Atomically stores the name and thread after a directory move. - private static async Task StoreMovedChatFilesAsync(ChatThread chat, string chatDirectory) + /// Stores the name and the thread of a chat into its directory. + /// + /// Both files are written atomically: a crash of AI Studio while storing leaves each of them as it was before, + /// never half written. The thread keeps its UTF-8 byte order mark, the name has none, as before. + /// + /// The chat to store. + /// The directory of the chat, which has to exist. + private static async Task StoreChatFilesAsync(ChatThread chat, string chatDirectory) { - await File.WriteAllTextAsync(Path.Join(chatDirectory, "name"), chat.Name); - var chatPath = Path.Join(chatDirectory, "thread.json"); - var temporaryPath = Path.Join(chatDirectory, $".thread-{Guid.NewGuid():N}.tmp"); - - try - { - await File.WriteAllTextAsync(temporaryPath, JsonSerializer.Serialize(chat, JSON_OPTIONS), Encoding.UTF8); - File.Move(temporaryPath, chatPath, overwrite: true); - } - finally - { - if (File.Exists(temporaryPath)) - File.Delete(temporaryPath); - } + await AtomicFile.WriteTextAsync(Path.Join(chatDirectory, "name"), chat.Name, new UTF8Encoding(false), CancellationToken.None, logger: LOG); + await AtomicFile.WriteTextAsync(Path.Join(chatDirectory, "thread.json"), JsonSerializer.Serialize(chat, JSON_OPTIONS), Encoding.UTF8, CancellationToken.None, logger: LOG); } /// Rewrites absolute file paths after moving the complete chat directory. diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.11.2.md b/app/MindWork AI Studio/wwwroot/changelog/v26.11.2.md index ceda6d299..f160da435 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.11.2.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.11.2.md @@ -8,6 +8,8 @@ - Improved follow-up questions about web pages which a tool found, such as the results of a web search: the Read Web Page tool may read these pages again later in the same chat, also after you restart AI Studio and also when it may otherwise read only addresses which stand in the chat. - Improved the texts of AI Studio: we corrected grammar mistakes, missing words, and punctuation in many English texts and improved the wording of some German ones. A few texts which every language showed in English are now translated as well. - Improved the descriptions at the top of all assistants: long words are now hyphenated, so that the justified text shows fewer large gaps. +- Improved how AI Studio saves your settings and the projects of the Visual Briefing assistant: it now waits until the data has reached the disk before it replaces the previous version, so that a power failure while saving is far less likely to damage them. +- Improved how AI Studio saves chats: if AI Studio crashes while saving a chat, the chat keeps its previous state instead of becoming unreadable. - Changed the icons of AI Studio to Lucide, an open-source icon set. All icons now share one style, and each feature shows the same icon wherever you meet it. - Fixed assistant plugins which wrote numbers and dates according to the region settings of your computer. Plugins now show them the same way on every computer: a decimal number such as `12.5` keeps its point, and the `DateTime` helper writes the names of months in English. - Fixed the translation, agenda, and e-mail assistants checking your input again and again in the background, about once a second, even while you weren't typing. diff --git a/app/Tests/BeyondFactsLab/BeyondFactsLabStoreTests.cs b/app/Tests/BeyondFactsLab/BeyondFactsLabStoreTests.cs index d12b66177..089bc5861 100644 --- a/app/Tests/BeyondFactsLab/BeyondFactsLabStoreTests.cs +++ b/app/Tests/BeyondFactsLab/BeyondFactsLabStoreTests.cs @@ -198,14 +198,20 @@ public sealed class BeyondFactsLabStoreTests await store.SetHappenedAsync(created.ProjectId, DateTimeOffset.UtcNow); await store.RememberSelectionAsync(created.ProjectId); - Assert.That(Directory.EnumerateFiles(this.RootDirectory, "*.tmp-*", SearchOption.AllDirectories), Is.Empty); + Assert.That(Directory.EnumerateFiles(this.RootDirectory, "*.tmp*", SearchOption.AllDirectories), Is.Empty); } - [Test] - public async Task ANewStoreRemovesTemporaryFilesACrashLeftBehind() + /// + /// Covers the temporary files of AtomicFile, which end in ".tmp", and those of earlier versions, which end in + /// ".tmp-" and a GUID. + /// + [TestCase(false)] + [TestCase(true)] + public async Task ANewStoreRemovesTemporaryFilesACrashLeftBehind(bool fromAnEarlierVersion) { var created = await this.CreateStore().CreateAsync("Panel"); - var leftOver = $"{this.ProjectPath(created.ProjectId)}.tmp-{Guid.NewGuid():N}"; + var projectPath = this.ProjectPath(created.ProjectId); + var leftOver = fromAnEarlierVersion ? $"{projectPath}.tmp-{Guid.NewGuid():N}" : $"{projectPath}.{Guid.NewGuid():N}.tmp"; await File.WriteAllTextAsync(leftOver, "{"); await this.CreateStore().ListProjectsAsync(); diff --git a/app/Tests/Settings/SettingsStorageTests.cs b/app/Tests/Settings/SettingsStorageTests.cs index 24ee27411..2c7ef5aae 100644 --- a/app/Tests/Settings/SettingsStorageTests.cs +++ b/app/Tests/Settings/SettingsStorageTests.cs @@ -102,9 +102,10 @@ public sealed class SettingsStorageTests // Every store writes its settings next to the previous ones and renames afterwards. The // temporary file carries a name of its own, so two stores cannot collide over it -- but a // store which gave up halfway would leave one lying around, and the next start would find - // a configuration directory filling up with them. + // a configuration directory filling up with them. The pattern covers the names of AtomicFile, + // ending in ".tmp", and the earlier ones, ending in ".tmp-" and a GUID. // - var leftovers = Directory.GetFiles(this.testDirectory, "*.tmp-*").Select(Path.GetFileName).ToList(); + var leftovers = Directory.GetFiles(this.testDirectory, "*.tmp*").Select(Path.GetFileName).ToList(); Assert.That(leftovers, Is.Empty, $"Temporary settings files were left behind: {string.Join(", ", leftovers)}."); } diff --git a/app/Tests/Tools/AtomicFileTests.cs b/app/Tests/Tools/AtomicFileTests.cs new file mode 100644 index 000000000..42bdbfc50 --- /dev/null +++ b/app/Tests/Tools/AtomicFileTests.cs @@ -0,0 +1,263 @@ +using System.Text; +using System.Text.Json; + +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks that a file written through AtomicFile holds either its old or its new content, and that no temporary file +/// stays behind. +/// +/// +/// Settings, chats, and the stores of several assistants rely on this. A file which a crash or an error left half +/// written would be read back the next time, so every way a write can end is checked here: replaced, refused, +/// failed, cancelled, and discarded. +/// +[TestFixture] +public sealed class AtomicFileTests +{ + private const string OLD_CONTENT = "old content"; + + private static readonly Encoding UTF8_WITHOUT_BOM = new UTF8Encoding(false); + + private string testRoot = string.Empty; + + private string TargetPath => Path.Combine(this.testRoot, "data.json"); + + [SetUp] + public void CreateTestRoot() + { + this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-atomic-file-{Guid.NewGuid():N}"); + Directory.CreateDirectory(this.testRoot); + } + + [TearDown] + public void DeleteTestRoot() + { + if (Directory.Exists(this.testRoot)) + Directory.Delete(this.testRoot, true); + } + + [Test] + public async Task ANewFileGetsItsContent() + { + await AtomicFile.WriteTextAsync(this.TargetPath, "new content", UTF8_WITHOUT_BOM, CancellationToken.None); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await File.ReadAllTextAsync(this.TargetPath), Is.EqualTo("new content")); + Assert.That(this.Leftovers(), Is.Empty); + } + } + + [Test] + public async Task AnExistingFileIsReplaced() + { + await File.WriteAllTextAsync(this.TargetPath, OLD_CONTENT); + + await AtomicFile.WriteTextAsync(this.TargetPath, "new content", UTF8_WITHOUT_BOM, CancellationToken.None); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await File.ReadAllTextAsync(this.TargetPath), Is.EqualTo("new content")); + Assert.That(this.Leftovers(), Is.Empty); + } + } + + [Test] + public async Task WithoutOverwritingAnExistingFileStaysAsItIs() + { + await File.WriteAllTextAsync(this.TargetPath, OLD_CONTENT); + + Assert.That(async () => await AtomicFile.WriteTextAsync(this.TargetPath, "new content", UTF8_WITHOUT_BOM, CancellationToken.None, overwrite: false), Throws.InstanceOf()); + using (Assert.EnterMultipleScope()) + { + Assert.That(await File.ReadAllTextAsync(this.TargetPath), Is.EqualTo(OLD_CONTENT)); + Assert.That(this.Leftovers(), Is.Empty); + } + } + + [Test] + public async Task AFailingWriterLeavesTheFileAsItWasAndKeepsItsError() + { + await File.WriteAllTextAsync(this.TargetPath, OLD_CONTENT); + var error = new InvalidOperationException("The writer failed."); + + var thrown = await Assert.ThrowsAsync(async () => await AtomicFile.WriteAsync(this.TargetPath, async (stream, token) => + { + await stream.WriteAsync("partial"u8.ToArray(), token); + throw error; + }, CancellationToken.None)); + + using (Assert.EnterMultipleScope()) + { + Assert.That(thrown, Is.SameAs(error), "Cleaning up must not replace the error of the writer."); + Assert.That(await File.ReadAllTextAsync(this.TargetPath), Is.EqualTo(OLD_CONTENT)); + Assert.That(this.Leftovers(), Is.Empty); + } + } + + [Test] + public async Task ACancelledWriteLeavesTheFileAsItWas() + { + await File.WriteAllTextAsync(this.TargetPath, OLD_CONTENT); + var cancelled = new CancellationToken(canceled: true); + + Assert.That(async () => await AtomicFile.WriteTextAsync(this.TargetPath, "new content", UTF8_WITHOUT_BOM, cancelled), Throws.InstanceOf()); + using (Assert.EnterMultipleScope()) + { + Assert.That(await File.ReadAllTextAsync(this.TargetPath), Is.EqualTo(OLD_CONTENT)); + Assert.That(this.Leftovers(), Is.Empty); + } + } + + [TestCase(true)] + [TestCase(false)] + public async Task ADiscardedWriteLeavesTheFileAsItWas(bool targetExists) + { + if (targetExists) + await File.WriteAllTextAsync(this.TargetPath, OLD_CONTENT); + + var replaced = await AtomicFile.WriteOrDiscardAsync(this.TargetPath, async (stream, token) => + { + await AtomicFile.WriteTextAsync(stream, "new content", UTF8_WITHOUT_BOM, token); + return false; + }, CancellationToken.None); + + using (Assert.EnterMultipleScope()) + { + Assert.That(replaced, Is.False); + Assert.That(File.Exists(this.TargetPath), Is.EqualTo(targetExists)); + if (targetExists) + Assert.That(await File.ReadAllTextAsync(this.TargetPath), Is.EqualTo(OLD_CONTENT)); + + Assert.That(this.Leftovers(), Is.Empty); + } + } + + [Test] + public async Task FlushingToTheDiskWritesTheSameContent() + { + await File.WriteAllTextAsync(this.TargetPath, OLD_CONTENT); + + await AtomicFile.WriteTextAsync(this.TargetPath, "new content", UTF8_WITHOUT_BOM, CancellationToken.None, flushToDisk: true); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await File.ReadAllTextAsync(this.TargetPath), Is.EqualTo("new content")); + Assert.That(this.Leftovers(), Is.Empty); + } + } + + /// + /// The places which switched to AtomicFile keep the bytes they wrote before, the byte order mark included: + /// chats start with one, the settings do not. + /// + [TestCase("Encoding.UTF8", "Grüße aus Köln 🙂\nZweite Zeile")] + [TestCase("Encoding.UTF8", "")] + [TestCase("UTF8Encoding(false)", "Grüße aus Köln 🙂\nZweite Zeile")] + [TestCase("UTF8Encoding(false)", "")] + [TestCase("UTF8Encoding(true)", "Grüße aus Köln 🙂\nZweite Zeile")] + public async Task TextGetsTheSameBytesAsWithWriteAllText(string encodingName, string content) + { + var encoding = encodingName switch + { + "Encoding.UTF8" => Encoding.UTF8, + "UTF8Encoding(false)" => new UTF8Encoding(false), + _ => new UTF8Encoding(true) + }; + + var expectedPath = Path.Combine(this.testRoot, "expected.txt"); + await File.WriteAllTextAsync(expectedPath, content, encoding); + + await AtomicFile.WriteTextAsync(this.TargetPath, content, encoding, CancellationToken.None); + + Assert.That(await File.ReadAllBytesAsync(this.TargetPath), Is.EqualTo(await File.ReadAllBytesAsync(expectedPath))); + } + + [Test] + public async Task JsonGetsTheSameBytesAsWithSerializeAndWriteAllText() + { + var settings = new Data(); + var expectedPath = Path.Combine(this.testRoot, "expected.json"); + await File.WriteAllTextAsync(expectedPath, JsonSerializer.Serialize(settings, SettingsManager.JSON_OPTIONS), UTF8_WITHOUT_BOM); + + await AtomicFile.WriteJsonAsync(this.TargetPath, settings, SettingsManager.JSON_OPTIONS, CancellationToken.None); + + Assert.That(await File.ReadAllBytesAsync(this.TargetPath), Is.EqualTo(await File.ReadAllBytesAsync(expectedPath))); + } + + /// + /// The transcript stores recognize an interrupted write by this name: DataSourceTranscriptStore removes it, and + /// MediaSidecarTranscripts keeps it out of the Batch Processing assistant. + /// + [Test] + public async Task TheTemporaryFileLiesNextToItsTargetUnderTheNameTheTranscriptStoresKnow() + { + var temporaryFiles = new List(); + await AtomicFile.WriteAsync(this.TargetPath, (_, _) => + { + temporaryFiles.AddRange(this.Leftovers().Select(path => Path.GetFileName(path))); + return Task.CompletedTask; + }, CancellationToken.None); + + Assert.That(temporaryFiles, Has.Count.EqualTo(1)); + Assert.That(temporaryFiles[0], Does.Match(@"^data\.json\.[0-9a-f]{32}\.tmp$")); + } + + /// + /// Two writes of the same file overlap, e.g., when two runs of the Batch Processing assistant write into the same + /// folder. Each one needs a temporary file of its own; with a shared one, the second write would fail. + /// + [Test] + public async Task OverlappingWritesOfTheSameFileDoNotCollide() + { + await AtomicFile.WriteAsync(this.TargetPath, async (stream, token) => + { + await AtomicFile.WriteTextAsync(stream, "first", UTF8_WITHOUT_BOM, token); + await AtomicFile.WriteTextAsync(this.TargetPath, "second", UTF8_WITHOUT_BOM, token); + }, CancellationToken.None); + + using (Assert.EnterMultipleScope()) + { + Assert.That(await File.ReadAllTextAsync(this.TargetPath), Is.EqualTo("first"), "The write which finished last wins."); + Assert.That(this.Leftovers(), Is.Empty); + } + } + + [Test] + public async Task DeletingTemporaryFilesRemovesTheLeftoversOfBothFormsAndNothingElse() + { + var nested = Directory.CreateDirectory(Path.Combine(this.testRoot, "nested", "deeper")).FullName; + var current = Path.Combine(nested, $"project.json.{Guid.NewGuid():N}.tmp"); + var previous = Path.Combine(this.testRoot, $"project.json.tmp-{Guid.NewGuid():N}"); + string[] unrelated = + [ + Path.Combine(this.testRoot, "project.json"), + Path.Combine(this.testRoot, "notes.tmp"), + Path.Combine(nested, "draft.tmp-copy"), + Path.Combine(nested, $"archive.{Guid.NewGuid():N}.tmp.bak"), + Path.Combine(this.testRoot, $"report.tmp-{Guid.NewGuid():N}.txt") + ]; + + foreach (var path in unrelated.Append(current).Append(previous)) + await File.WriteAllTextAsync(path, "{"); + + AtomicFile.DeleteTemporaryFiles(this.testRoot); + + using (Assert.EnterMultipleScope()) + { + Assert.That(File.Exists(current), Is.False, "A leftover of AtomicFile stayed."); + Assert.That(File.Exists(previous), Is.False, "A leftover of an older version stayed."); + Assert.That(unrelated.Where(path => !File.Exists(path)), Is.Empty, "A file which is no leftover was deleted."); + } + } + + /// + /// Gets every file below the test folder except the target. + /// + private IEnumerable Leftovers() => Directory.EnumerateFiles(this.testRoot, "*", SearchOption.AllDirectories).Where(path => path != this.TargetPath); +} \ No newline at end of file diff --git a/app/Tests/Tools/WorkspaceChatStoreTests.cs b/app/Tests/Tools/WorkspaceChatStoreTests.cs index bb393df50..06562da1b 100644 --- a/app/Tests/Tools/WorkspaceChatStoreTests.cs +++ b/app/Tests/Tools/WorkspaceChatStoreTests.cs @@ -1,3 +1,4 @@ +using System.Text; using System.Text.Json; using AIStudio.Chat; @@ -87,4 +88,33 @@ public sealed class WorkspaceChatStoreTests Assert.That(stored.Draft.MediaTranscripts.Single().FilePath, Is.EqualTo(transcript.FilePath), "Opening the chat again brings the draft back from the stored chat, so that has to know the new path as well."); } } + + /// + /// Storing writes each file of the chat next to it first and replaces it then, so that a crash while storing + /// leaves the chat as it was stored before. The files keep the encoding they always had. + /// + [Test] + public async Task StoringAChatAgainLeavesNoTemporaryFileAndKeepsTheEncodingOfItsFiles() + { + var workspace = await WorkspaceBehaviour.TryCreateWorkspaceAsync("Projects"); + Assert.That(workspace.Success, Is.True, "The workspace could not be created."); + + var chat = new ChatThread { WorkspaceId = workspace.Workspace.WorkspaceId, ChatId = Guid.NewGuid(), Name = "Meeting notes" }; + Assert.That(await WorkspaceBehaviour.StoreChatAsync(chat), Is.True, "The chat could not be stored."); + + chat.Name = "Meeting notes, revised"; + Assert.That(await WorkspaceBehaviour.StoreChatAsync(chat), Is.True, "The chat could not be stored again."); + + var chatDirectory = WorkspaceBehaviour.GetChatDirectory(chat.WorkspaceId, chat.ChatId); + var thread = await File.ReadAllBytesAsync(Path.Combine(chatDirectory, "thread.json")); + var name = await File.ReadAllBytesAsync(Path.Combine(chatDirectory, "name")); + byte[] byteOrderMark = [0xEF, 0xBB, 0xBF]; + + using (Assert.EnterMultipleScope()) + { + Assert.That(Directory.EnumerateFiles(chatDirectory, "*.tmp*"), Is.Empty, "A temporary file stayed in the chat directory."); + Assert.That(thread.Take(byteOrderMark.Length), Is.EqualTo(byteOrderMark), "The thread starts with a byte order mark, as it always did."); + Assert.That(name, Is.EqualTo(Encoding.UTF8.GetBytes("Meeting notes, revised")), "The name holds the name stored last, without a byte order mark."); + } + } } \ No newline at end of file