Fixed chats not loading in the v26.10.1 pre-release and added a trim smoke test (#1128)

app/Build/Commands/VerifyCommand.cs

@@ -1,5 +1,7 @@
 using Build.Tools;
 
+using SharedTools;
+
 // ReSharper disable ClassNeverInstantiated.Global
 // ReSharper disable UnusedType.Global
 // ReSharper disable UnusedMember.Global
@@ -20,7 +22,7 @@ public sealed class VerifyCommand
     /// </summary>
     private const string SIDECAR_PREFIX = "mindworkAIStudioServer-";
 
-    [Command("verify", Description = "Run the quality gate: .NET tests, Rust tests, Clippy, and the model sources")]
+    [Command("verify", Description = "Run the quality gate: .NET tests, the trim smoke test, Rust tests, Clippy, and the model sources")]
     public async Task<int> Verify()
     {
         if(!Environment.IsWorkingDirectoryValid())
@@ -32,6 +34,7 @@ public sealed class VerifyCommand
         var results = new List<(string What, int ExitCode)>
         {
             (".NET tests", await CommandRunner.RunAsync(Environment.GetTestsDirectory(), "dotnet", "test --nologo")),
+            ("Trim smoke test", await RunTrimSmokeTestAsync()),
         };
 
         var runtimeDirectory = Environment.GetRustRuntimeDirectory();
@@ -71,6 +74,34 @@ public sealed class VerifyCommand
         return 1;
     }
 
+    /// <summary>
+    /// Publishes the trim smoke test for this machine, trimmed like the app, and runs it.
+    /// </summary>
+    /// <remarks>
+    /// Debug builds and the tests are not trimmed, so only a published build shows that the trimmer
+    /// removed code which System.Text.Json reaches by reflection. Then AI Studio can neither load nor
+    /// save, e.g., its chats, while everything looks fine during development.
+    /// </remarks>
+    /// <returns>The exit code of the publishing, when that failed; otherwise the one of the smoke test.</returns>
+    private static async Task<int> RunTrimSmokeTestAsync()
+    {
+        var rid = Environment.GetCurrentRid();
+        if (rid is RID.NONE)
+        {
+            Console.WriteLine("- Error: The trim smoke test needs a platform AI Studio is built for.");
+            return 1;
+        }
+
+        var directory = Environment.GetTrimSmokeTestDirectory();
+        var publishDirectory = Path.Combine(directory, "bin", "verify");
+        var publishExitCode = await CommandRunner.RunAsync(directory, "dotnet", $"publish --configuration release --runtime {rid.AsMicrosoftRid()} --output \"{publishDirectory}\" --nologo");
+        if (publishExitCode is not 0)
+            return publishExitCode;
+
+        var program = Path.Combine(publishDirectory, OperatingSystem.IsWindows() ? "TrimSmokeTest.exe" : "TrimSmokeTest");
+        return await CommandRunner.RunAsync(directory, program, string.Empty);
+    }
+
     /// <summary>
     /// Whether a build has already produced the files Tauri's build script reads.
     /// </summary>

app/Build/Tools/Environment.cs

@@ -41,6 +41,13 @@ public static class Environment
         return Path.GetFullPath(directory);
     }
 
+    public static string GetTrimSmokeTestDirectory()
+    {
+        var currentDirectory = Directory.GetCurrentDirectory();
+        var directory = Path.Combine(currentDirectory, "..", "TrimSmokeTest");
+        return Path.GetFullPath(directory);
+    }
+
     /// <summary>
     /// The root of the git repository, which is what a path in a report is written relative to.
     /// </summary>

app/MindWork AI Studio.sln

@@ -12,6 +12,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SourceGeneratedMappings", "
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Tests", "Tests\Tests.csproj", "{CD46329B-D135-4594-9A70-55D3480F8FEE}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TrimSmokeTest", "TrimSmokeTest\TrimSmokeTest.csproj", "{0F4B3B3C-E8C9-4CDC-A247-5029D777DFC2}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -42,6 +44,10 @@ Global
 		{CD46329B-D135-4594-9A70-55D3480F8FEE}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{CD46329B-D135-4594-9A70-55D3480F8FEE}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{CD46329B-D135-4594-9A70-55D3480F8FEE}.Release|Any CPU.Build.0 = Release|Any CPU
+		{0F4B3B3C-E8C9-4CDC-A247-5029D777DFC2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{0F4B3B3C-E8C9-4CDC-A247-5029D777DFC2}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{0F4B3B3C-E8C9-4CDC-A247-5029D777DFC2}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{0F4B3B3C-E8C9-4CDC-A247-5029D777DFC2}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(NestedProjects) = preSolution
 	EndGlobalSection

app/MindWork AI Studio/MindWork AI Studio.csproj

@@ -27,14 +27,6 @@
             `dotnet publish -r` instead.
         -->
         <RuntimeIdentifiers>osx-arm64;osx-x64;linux-x64;linux-arm64;win-x64;win-arm64</RuntimeIdentifiers>
-        
-        <PublishTrimmed>true</PublishTrimmed> <!-- Publish with trimming, to reduce size -->
-        <TrimMode>partial</TrimMode> <!-- Trim unused assemblies, but keep all assemblies referenced by the project -->
-        <TrimmerRemoveSymbols>true</TrimmerRemoveSymbols> <!-- Remove debug symbols -->
-        <EnableUnsafeBinaryFormatterSerialization>false</EnableUnsafeBinaryFormatterSerialization> <!-- Remove unsafe binary formatter serialization -->
-        <EnableUnsafeUTF7Encoding>false</EnableUnsafeUTF7Encoding> <!-- Remove unsafe UTF7 encoding -->
-        <JsonSerializerIsReflectionEnabledByDefault>true</JsonSerializerIsReflectionEnabledByDefault> <!-- Enable reflection for JSON serialization -->
-        <SuppressTrimAnalysisWarnings>true</SuppressTrimAnalysisWarnings> <!-- Suppress trim analysis warnings -->
 
         <!--
             The Web SDK defaults to server GC. AI Studio is a single-user desktop app, though: workstation
@@ -46,15 +38,19 @@
         <ConcurrentGarbageCollection>true</ConcurrentGarbageCollection>
 
         <!--
-            IL2026: Usage of methods marked as RequiresUnreferencedCode. None issue here, since we use partial trim mode, though.
+            IL2026: Usage of methods marked as RequiresUnreferencedCode, e.g., every call of JsonSerializer. Partial trim mode
+            keeps our own types, so this is no issue for them. It is for types of .NET which System.Text.Json reaches by
+            reflection only, see the TrimmerRootAssembly in Trimming.props.
             CS8974: Converting method group to non-delegate type; Did you intend to invoke the method? We have this issue with MudBlazor validation methods.
         -->
         <NoWarn>IL2026, CS8974</NoWarn>
         <LangVersion>latest</LangVersion>
     </PropertyGroup>
 
+    <Import Project="..\Trimming.props" />
+
     <ItemGroup>
-      <!-- Remove some files from the solution view -->  
+      <!-- Remove some files from the solution view -->
       <None Remove="Properties\launchSettings.json" />
       <None Remove="build.nu" />
     </ItemGroup>
@@ -105,10 +101,12 @@
     <!--
         The test project has to reach Program.LOGGER_FACTORY. Program is internal, and it stays that
         way: it is the entry point, not API. Several types initialize a static logger field from that
-        factory, so a test process must assign it before it touches any of them.
+        factory, so a test process must assign it before it touches any of them. The trim smoke test
+        needs the same, and it uses the internal JSON options of the settings and the Visual Briefing.
     -->
     <ItemGroup>
       <InternalsVisibleTo Include="Tests" />
+      <InternalsVisibleTo Include="TrimSmokeTest" />
     </ItemGroup>
 
     <!-- Read the meta data file -->

app/TrimSmokeTest/JsonContractWalker.cs

@@ -0,0 +1,61 @@
+using System.Text.Json;
+using System.Text.Json.Serialization.Metadata;
+
+namespace AIStudio.TrimSmokeTest;
+
+/// <summary>
+/// Builds the contract System.Text.Json uses for a root and for everything it holds.
+/// </summary>
+/// <remarks>
+/// A type whose contract needs code the trimmer removed fails right here, before any value is read or
+/// written, just like it fails in AI Studio. A chat which lacks the property in question fails all the same.
+/// </remarks>
+public static class JsonContractWalker
+{
+    /// <summary>
+    /// Returns a description of every contract which cannot be built, starting at the given root.
+    /// </summary>
+    /// <param name="root">The type to start with, and the options AI Studio uses for it.</param>
+    /// <returns>One line per failure, naming the path from the root to the type which failed.</returns>
+    public static IReadOnlyList<string> FindBrokenContracts(JsonRoot root)
+    {
+        //
+        // The same System.Text.Json does with the first call of Serialize or Deserialize:
+        //
+        root.Options.MakeReadOnly(populateMissingResolver: true);
+
+        var failures = new List<string>();
+        Visit(root.Type, root.Type.Name, root.Options, [], failures);
+        return failures;
+    }
+
+    private static void Visit(Type type, string path, JsonSerializerOptions options, HashSet<Type> visited, List<string> failures)
+    {
+        if (!visited.Add(type))
+            return;
+
+        JsonTypeInfo typeInfo;
+        try
+        {
+            typeInfo = options.GetTypeInfo(type);
+        }
+        catch (Exception exception) when (exception is NotSupportedException or InvalidOperationException)
+        {
+            failures.Add($"{path}: {exception.Message}");
+            return;
+        }
+
+        foreach (var property in typeInfo.Properties)
+            Visit(property.PropertyType, $"{path} → {property.Name}", options, visited, failures);
+
+        if (typeInfo.KeyType is { } keyType)
+            Visit(keyType, $"{path} → key", options, visited, failures);
+
+        if (typeInfo.ElementType is { } elementType)
+            Visit(elementType, $"{path} → element", options, visited, failures);
+
+        if (typeInfo.PolymorphismOptions is { } polymorphism)
+            foreach (var derivedType in polymorphism.DerivedTypes)
+                Visit(derivedType.DerivedType, $"{path} → {derivedType.DerivedType.Name}", options, visited, failures);
+    }
+}

app/TrimSmokeTest/JsonRoot.cs

@@ -0,0 +1,10 @@
+using System.Text.Json;
+
+namespace AIStudio.TrimSmokeTest;
+
+/// <summary>
+/// A type which AI Studio reads or writes as JSON on its own, together with the options it uses for that.
+/// </summary>
+/// <param name="Type">The type which gets serialized. Whatever it holds gets checked along with it.</param>
+/// <param name="Options">The options AI Studio passes to System.Text.Json for this type.</param>
+public sealed record JsonRoot(Type Type, JsonSerializerOptions Options);

app/TrimSmokeTest/JsonRoots.cs

@@ -0,0 +1,65 @@
+using System.Text.Json;
+
+using AIStudio.Agents;
+using AIStudio.Agents.AssistantAudit;
+using AIStudio.Assistants.Builder;
+using AIStudio.Assistants.PromptOptimizer;
+using AIStudio.Assistants.SWOTAnalysis;
+using AIStudio.Assistants.VisualBriefing;
+using AIStudio.Chat;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Settings.DataModel.PreviousModels;
+using AIStudio.Tools;
+using AIStudio.Tools.DataBackup;
+
+namespace AIStudio.TrimSmokeTest;
+
+/// <summary>
+/// The types which AI Studio reads or writes as JSON on their own, not only as part of another type.
+/// </summary>
+/// <remarks>
+/// Whatever these types hold, i.e., the types of their properties, the elements of their collections, and
+/// their derived types, gets checked along with them, so only the outermost types belong here. Add a type
+/// whenever AI Studio starts to store a new kind of file or to read a new kind of model answer.<br/><br/>
+///
+/// The answers of models are read with options of their own, which are private and only change names and
+/// leniency. The default options build the same contracts for the types of .NET they hold. A type which is
+/// private to its parser, such as the DTO of a slide deck, is out of reach here.
+/// </remarks>
+public static class JsonRoots
+{
+    public static readonly IReadOnlyList<JsonRoot> ALL =
+    [
+        // Chats, in the workspaces, among the temporary chats, and in chat archives:
+        new(typeof(ChatThread), WorkspaceBehaviour.JSON_OPTIONS),
+
+        // The settings, and their older versions, which the migrations still read:
+        new(typeof(Data), SettingsManager.JSON_OPTIONS),
+        new(typeof(DataV1V3), SettingsManager.JSON_OPTIONS),
+        new(typeof(DataV4), SettingsManager.JSON_OPTIONS),
+        new(typeof(DataV5), SettingsManager.JSON_OPTIONS),
+
+        // The projects of the Visual Briefing assistant:
+        new(typeof(VisualBriefingManifest), VisualBriefingJson.Persistence),
+        new(typeof(VisualBriefingBuildRecord), VisualBriefingJson.Persistence),
+        new(typeof(VisualBriefingEvidenceArtifact), VisualBriefingJson.Persistence),
+        new(typeof(VisualBriefingPlanArtifact), VisualBriefingJson.Persistence),
+        new(typeof(VisualBriefingContentArtifact), VisualBriefingJson.Persistence),
+        new(typeof(VisualBriefingPresentationArtifact), VisualBriefingJson.Persistence),
+
+        // Backups and their restore:
+        new(typeof(DataBackupManifest), DataBackupFormat.JSON_OPTIONS),
+        new(typeof(DataRestorePlan), DataBackupFormat.JSON_OPTIONS),
+        new(typeof(DataRestoreReport), DataBackupFormat.JSON_OPTIONS),
+
+        // Answers of models:
+        new(typeof(LuaResponse), JsonSerializerOptions.Default),
+        new(typeof(LauncherTextsResponse), JsonSerializerOptions.Default),
+        new(typeof(PromptOptimizationResult), JsonSerializerOptions.Default),
+        new(typeof(RetrievalContextValidationResult), JsonSerializerOptions.Default),
+        new(typeof(AssistantAuditResult), JsonSerializerOptions.Default),
+        new(typeof(SwotCategoryAnalysisResult), JsonSerializerOptions.Default),
+        new(typeof(SwotModerationResult), JsonSerializerOptions.Default),
+    ];
+}

app/TrimSmokeTest/JsonRoundTrips.cs

@@ -0,0 +1,64 @@
+using System.Text.Json;
+
+using AIStudio.Assistants.VisualBriefing;
+using AIStudio.Chat;
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+
+namespace AIStudio.TrimSmokeTest;
+
+/// <summary>
+/// Writes and reads filled values of the roots which matter most, since some parts of a contract only come
+/// into play once there is a value, e.g., the factory which builds an immutable collection.
+/// </summary>
+public static class JsonRoundTrips
+{
+    private const string TEXT_WITH_WEB_ADDRESS = "The details are on https://example.org/chosen.";
+
+    /// <summary>
+    /// Returns a description of every value which does not survive being written and read again.
+    /// </summary>
+    /// <returns>One line per failure, naming the type which failed.</returns>
+    public static IReadOnlyList<string> FindLosses()
+    {
+        var restrictions = ChatRestrictions.NONE.WithWebAddressesWrittenByModelIn(TEXT_WITH_WEB_ADDRESS);
+        var chat = new ChatThread
+        {
+            Name = "Trim smoke test",
+            Blocks =
+            [
+                new ContentBlock
+                {
+                    Time = DateTimeOffset.UnixEpoch,
+                    ContentType = ContentType.TEXT,
+                    Content = new ContentText { Text = TEXT_WITH_WEB_ADDRESS },
+                    Role = ChatRole.USER,
+                },
+            ],
+        };
+
+        chat.RequireRestrictions(restrictions);
+
+        var failures = new List<string>();
+        AddLoss(chat, WorkspaceBehaviour.JSON_OPTIONS, failures);
+        AddLoss(new Data(), SettingsManager.JSON_OPTIONS, failures);
+        AddLoss(new VisualBriefingManifest { Name = "Trim smoke test", Restrictions = restrictions }, VisualBriefingJson.Persistence, failures);
+        return failures;
+    }
+
+    private static void AddLoss<T>(T value, JsonSerializerOptions options, List<string> failures)
+    {
+        try
+        {
+            var json = JsonSerializer.Serialize(value, options);
+            var jsonAgain = JsonSerializer.Serialize(JsonSerializer.Deserialize<T>(json, options), options);
+            if (jsonAgain != json)
+                failures.Add($"{typeof(T).Name}: written again after reading, it gives other JSON than before.");
+        }
+        catch (Exception exception) when (exception is NotSupportedException or InvalidOperationException or JsonException)
+        {
+            failures.Add($"{typeof(T).Name}: {exception.Message}");
+        }
+    }
+}

app/TrimSmokeTest/Program.cs

@@ -0,0 +1,34 @@
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.TrimSmokeTest;
+
+/// <summary>
+/// Checks whether a trimmed build can still read and write the JSON of AI Studio, see TrimSmokeTest.csproj.
+/// </summary>
+public static class Program
+{
+    public static int Main()
+    {
+        //
+        // Several types of the app initialize a static logger from this factory, which only the start of
+        // the app assigns. Without it, they die in their type initializer, see app/Tests/TestHost.cs:
+        //
+        AIStudio.Program.LOGGER_FACTORY = NullLoggerFactory.Instance;
+
+        var failures = new List<string>();
+        foreach (var root in JsonRoots.ALL)
+            failures.AddRange(JsonContractWalker.FindBrokenContracts(root));
+
+        failures.AddRange(JsonRoundTrips.FindLosses());
+        if (failures.Count is 0)
+        {
+            Console.WriteLine($"- All {JsonRoots.ALL.Count} JSON roots can be read and written.");
+            return 0;
+        }
+
+        foreach (var failure in failures)
+            Console.WriteLine($"- Error: {failure}");
+
+        return 1;
+    }
+}

app/TrimSmokeTest/TrimSmokeTest.csproj

@@ -0,0 +1,28 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <!--
+        Checks what only a published build of AI Studio lacks. The trimmer removes code of .NET which System.Text.Json
+        reaches by reflection only, while debug builds and the tests are not trimmed. Hence, this program has to run
+        published, trimmed by the same Trimming.props as the app; `dotnet run verify` does both.
+    -->
+    <PropertyGroup>
+        <OutputType>Exe</OutputType>
+        <TargetFramework>net9.0</TargetFramework>
+        <RootNamespace>AIStudio.TrimSmokeTest</RootNamespace>
+        <LangVersion>latest</LangVersion>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+        <IsPackable>false</IsPackable>
+        <SelfContained>true</SelfContained> <!-- Trimming works for self-contained builds only -->
+
+        <!-- IL2026: Usage of methods marked as RequiresUnreferencedCode. Finding out what this means for a trimmed build is the whole point of this program. -->
+        <NoWarn>IL2026</NoWarn>
+    </PropertyGroup>
+
+    <Import Project="..\Trimming.props" />
+
+    <ItemGroup>
+        <ProjectReference Include="..\MindWork AI Studio\MindWork AI Studio.csproj" />
+    </ItemGroup>
+
+</Project>

app/Trimming.props

@@ -0,0 +1,27 @@
+<Project>
+
+    <!--
+        How a published build of AI Studio is trimmed. Kept apart from the project of the app, so that every
+        project which has to trim exactly like the app imports the same settings, instead of a copy which drifts.
+    -->
+    <PropertyGroup>
+        <PublishTrimmed>true</PublishTrimmed> <!-- Publish with trimming, to reduce size -->
+        <TrimMode>partial</TrimMode> <!-- Trim only assemblies marked as trimmable: all of .NET, but neither our own nor most packages -->
+        <TrimmerRemoveSymbols>true</TrimmerRemoveSymbols> <!-- Remove debug symbols -->
+        <EnableUnsafeBinaryFormatterSerialization>false</EnableUnsafeBinaryFormatterSerialization> <!-- Remove unsafe binary formatter serialization -->
+        <EnableUnsafeUTF7Encoding>false</EnableUnsafeUTF7Encoding> <!-- Remove unsafe UTF7 encoding -->
+        <JsonSerializerIsReflectionEnabledByDefault>true</JsonSerializerIsReflectionEnabledByDefault> <!-- Enable reflection for JSON serialization -->
+        <SuppressTrimAnalysisWarnings>true</SuppressTrimAnalysisWarnings> <!-- Suppress trim analysis warnings -->
+    </PropertyGroup>
+
+    <!--
+        System.Text.Json builds an immutable collection, e.g., the ImmutableHashSet of a chat, through a factory method
+        like ImmutableHashSet.CreateRange, which it finds by reflection at runtime. Nothing calls that method directly,
+        so the trimmer removes it, and every type which holds such a collection fails to load and to save. Only a
+        published build shows this: debug builds and the tests are not trimmed. Hence, we keep the whole assembly.
+    -->
+    <ItemGroup>
+        <TrimmerRootAssembly Include="System.Collections.Immutable" />
+    </ItemGroup>
+
+</Project>

documentation/Build.md

@@ -30,11 +30,12 @@ One command checks that what you are about to build is sound:
 2. Navigate to the `/app/Build` directory within the repository.
 3. Run `dotnet run verify`.
 
-It runs the .NET tests, the Rust tests, Clippy (`cargo clippy --all-targets -- -D warnings`), and a report on the pages the model rules were written from. Every check runs, even after one of them has failed, so that a single run tells you everything that is wrong instead of the first thing.
+It runs the .NET tests, the trim smoke test, the Rust tests, Clippy (`cargo clippy --all-targets -- -D warnings`), and a report on the pages the model rules were written from. Every check runs, even after one of them has failed, so that a single run tells you everything that is wrong instead of the first thing.
 
 `dotnet run build` runs the gate first and stops when it does not pass. For the quick loop while you are working on something, use `dotnet run build --skip-verify`, and let the gate run before you open a pull request. The same command runs in our GitHub workflow as the `verify` job, on every pull request which changes more than the documentation — including those without the `run-pipeline` label, because a gate which is closed exactly while nobody is looking is not a gate.
 
-Two notes:
+Three notes:
+- The trim smoke test publishes `app/TrimSmokeTest` trimmed, exactly like the app, and reads and writes the JSON of chats, settings, Visual Briefing projects, backups, and model answers. Debug builds and the tests are not trimmed, so only a published build shows when the trimmer removed code which System.Text.Json needs; AI Studio can then neither load nor save that data. When AI Studio starts to store a new kind of file or to read a new kind of model answer, add its type to `JsonRoots` in `app/TrimSmokeTest`.
 - The Rust half of the gate needs the .NET sidecar (see "One-time mandatory steps" above). While that file is missing, the gate skips the Rust tests and Clippy and says so rather than failing, because the command which produces the sidecar is `dotnet run build` itself.
 - `dotnet run verify-models` reports how long ago somebody last read the pages behind the model rules, and names everything older than six months. It is a report and never a failure: a page nobody has looked at for a while is not a page which changed. Everything else about the model rules — whether two rules claim the same names, whether every family names a page and a day, whether every pattern is written the way model names arrive — is checked by the test project, and therefore by `dotnet test`.