Added the Model Comparison assistant for blind comparisons of two models (#1009)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Chat/ChatTranscriptTests.cs

@@ -0,0 +1,77 @@
+using AIStudio.Chat;
+using AIStudio.Tools.ToolCallingSystem;
+
+namespace AIStudio.Tests.Chat;
+
+/// <summary>
+/// Checks that a transcript holds what the user sees of a chat, and nothing else.
+/// </summary>
+[TestFixture]
+public sealed class ChatTranscriptTests
+{
+    [Test]
+    public void TheTranscriptHoldsTheVisibleMessagesInTheirOrder()
+    {
+        var thread = Thread(
+            Block(ChatRole.USER, new ContentText { Text = "A hidden opening." }, hidden: true),
+            Block(ChatRole.USER, new ContentText { Text = "Summarize the report." }),
+            Block(ChatRole.AI, new ContentText { Text = "<think>Let me read it.</think>The report says 42." }),
+            Block(ChatRole.USER, new ContentText { Text = "Thanks." }));
+
+        Assert.That(ChatTranscript.Of(thread), Is.EqualTo("[User]\nSummarize the report.\n\n[Assistant]\nThe report says 42.\n\n[User]\nThanks."));
+    }
+
+    [Test]
+    public void ToolCallsAreOneShortLineWithoutTheirResults()
+    {
+        var answer = new ContentText
+        {
+            Text = "Found it.",
+            ToolInvocations =
+            [
+                new ToolInvocationTrace { Order = 2, ToolName = "read_web_page", Arguments = new() { ["url"] = "https://example.org" }, Result = "The whole page." },
+                new ToolInvocationTrace { Order = 1, ToolName = "web_search", Arguments = new() { ["query"] = new string('q', 100) + "\nmore" }, Result = "Many results." },
+            ],
+        };
+
+        var transcript = ChatTranscript.Of(Thread(Block(ChatRole.AI, answer)));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(transcript, Does.StartWith($"[Assistant]\n[Tool call: web_search(query={new string('q', ChatTranscript.MAX_ARGUMENT_LENGTH)}…)]\n[Tool call: read_web_page(url=https://example.org)]\nFound it."));
+            Assert.That(transcript, Does.Not.Contain("Many results."));
+            Assert.That(transcript, Does.Not.Contain("The whole page."));
+        }
+    }
+
+    [Test]
+    public void AttachedFilesAreNamedWithoutTheirContent()
+    {
+        var message = new ContentText
+        {
+            Text = "Please check these.",
+            FileAttachments = [new FileAttachment(FileAttachmentType.DOCUMENT, "report.pdf", "/tmp/report.pdf", 100), new FileAttachment(FileAttachmentType.DOCUMENT, "notes.md", "/tmp/notes.md", 10)],
+        };
+
+        Assert.That(ChatTranscript.Of(Thread(Block(ChatRole.USER, message))), Is.EqualTo("[User]\nPlease check these.\n[Attached files: report.pdf, notes.md]"));
+    }
+
+    [Test]
+    public void AChatWithoutVisibleMessagesHasAnEmptyTranscript()
+    {
+        var thread = Thread(Block(ChatRole.AI, new ContentText { Text = "   " }), Block(ChatRole.USER, new ContentText { Text = "Hidden." }, hidden: true));
+
+        Assert.That(ChatTranscript.Of(thread), Is.Empty);
+    }
+
+    private static ChatThread Thread(params ContentBlock[] blocks) => new() { Blocks = [.. blocks] };
+
+    private static ContentBlock Block(ChatRole role, ContentText content, bool hidden = false) => new()
+    {
+        Time = DateTimeOffset.UnixEpoch,
+        ContentType = ContentType.TEXT,
+        Role = role,
+        Content = content,
+        HideFromUser = hidden,
+    };
+}

app/Tests/Chat/Embedded/EmbeddedChatOpeningTests.cs

@@ -0,0 +1,71 @@
+using AIStudio.Chat;
+using AIStudio.Chat.Embedded;
+
+namespace AIStudio.Tests.Chat.Embedded;
+
+/// <summary>
+/// Checks when an embedded chat sends the message it opens with.
+/// </summary>
+[TestFixture]
+public sealed class EmbeddedChatOpeningTests
+{
+    private static readonly ContentText OPENING_MESSAGE = new() { Text = "What is new today?" };
+
+    [Test]
+    public void AnEmptyChatSendsItsOpeningMessage()
+    {
+        Assert.That(EmbeddedChatOpening.SendsNow(OPENING_MESSAGE, Conversation(), wasSent: false), Is.True);
+    }
+
+    [Test]
+    public void AChatWithoutOpeningMessageSendsNothing()
+    {
+        Assert.That(EmbeddedChatOpening.SendsNow(null, Conversation(), wasSent: false), Is.False);
+    }
+
+    [Test]
+    public void AChatSendsItsOpeningMessageOnlyOnce()
+    {
+        // Until the request adds the message to the chat, the chat still looks empty:
+        Assert.That(EmbeddedChatOpening.SendsNow(OPENING_MESSAGE, Conversation(), wasSent: true), Is.False);
+    }
+
+    [Test]
+    public void AChatInWhichTheUserWroteSendsNothing()
+    {
+        var conversation = Conversation();
+        conversation.Thread!.Blocks.Add(new ContentBlock { Role = ChatRole.USER, ContentType = ContentType.TEXT, Content = new ContentText { Text = "My own question." } });
+
+        Assert.That(EmbeddedChatOpening.SendsNow(OPENING_MESSAGE, conversation, wasSent: false), Is.False);
+    }
+
+    [Test]
+    public void AChatWhichLoadsOrCannotBeOpenedSendsNothing()
+    {
+        var unavailable = Conversation();
+        unavailable.Thread = null;
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(EmbeddedChatOpening.SendsNow(OPENING_MESSAGE, null, wasSent: false), Is.False, "While the chat loads, the message waits.");
+            Assert.That(EmbeddedChatOpening.SendsNow(OPENING_MESSAGE, unavailable, wasSent: false), Is.False);
+        }
+    }
+
+    [Test]
+    public void AChatInWhichTheModelAnswersSendsNothing()
+    {
+        var conversation = Conversation();
+        conversation.IsRunning = true;
+
+        Assert.That(EmbeddedChatOpening.SendsNow(OPENING_MESSAGE, conversation, wasSent: false), Is.False);
+    }
+
+    private static EmbeddedConversation Conversation() => new()
+    {
+        ChatId = Guid.NewGuid(),
+        Component = AIStudio.Tools.Components.MODEL_COMPARISON_ASSISTANT,
+        Store = new InMemoryEmbeddedChatStore(),
+        Thread = new ChatThread(),
+    };
+}

app/Tests/Chat/Embedded/EmbeddedChatServiceTests.cs

@@ -0,0 +1,240 @@
+using AIStudio.Chat;
+using AIStudio.Chat.Embedded;
+using AIStudio.Provider;
+using AIStudio.Settings;
+using AIStudio.Tools;
+using AIStudio.Tools.AssistantSessions;
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Chat.Embedded;
+
+/// <summary>
+/// Checks how EmbeddedChatService runs a request, without asking any provider.
+/// </summary>
+[TestFixture]
+public sealed class EmbeddedChatServiceTests
+{
+    private const AIStudio.Tools.Components COMPONENT = AIStudio.Tools.Components.MODEL_COMPARISON_ASSISTANT;
+
+    private SettingsManager settingsManager = null!;
+    private ServiceProvider serviceProvider = null!;
+    private IServiceProvider previousServiceProvider = null!;
+    private AIStudio.Settings.Provider provider = null!;
+
+    [SetUp]
+    public void CreateSettings()
+    {
+        this.settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
+        this.previousServiceProvider = Program.SERVICE_PROVIDER;
+        this.serviceProvider = new ServiceCollection().AddSingleton(this.settingsManager).BuildServiceProvider();
+        Program.SERVICE_PROVIDER = this.serviceProvider;
+
+        this.provider = new AIStudio.Settings.Provider(1, Guid.NewGuid().ToString(), "Instance 1", LLMProviders.DEEP_SEEK, new("test-model", null));
+    }
+
+    [TearDown]
+    public void RestoreApplicationState()
+    {
+        Program.SERVICE_PROVIDER = this.previousServiceProvider;
+        this.serviceProvider.Dispose();
+    }
+
+    [Test]
+    public async Task AChatWithoutAStoredThreadStartsWithANewOne()
+    {
+        var service = this.CreateService(Answer("An answer."));
+        var store = new InMemoryEmbeddedChatStore();
+        var chatId = Guid.NewGuid();
+
+        var conversation = await service.GetAsync(store, COMPONENT, chatId, () => new ChatThread());
+        var again = await service.GetAsync(store, COMPONENT, chatId, () => new ChatThread());
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(conversation.IsAvailable, Is.True);
+            Assert.That(conversation.Thread!.ChatId, Is.EqualTo(chatId), "The new thread gets the ID of the chat.");
+            Assert.That(again, Is.SameAs(conversation), "A chat is loaded once.");
+        }
+    }
+
+    [Test]
+    public async Task AChatThisVersionCannotOpenTakesNoMessage()
+    {
+        var service = this.CreateService(Answer("An answer."));
+        var store = new InMemoryEmbeddedChatStore { CannotOpen = true };
+        var chatId = Guid.NewGuid();
+
+        var conversation = await service.GetAsync(store, COMPONENT, chatId, () => new ChatThread());
+        var sent = await service.SendAsync(chatId, this.Request("A question."));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(conversation.IsAvailable, Is.False);
+            Assert.That(sent, Is.False);
+            Assert.That(store.SaveCount, Is.Zero, "The stored chat stays as it is.");
+        }
+    }
+
+    [Test]
+    public async Task TheMessageIsStoredBeforeTheModelAnswers()
+    {
+        var store = new InMemoryEmbeddedChatStore();
+        var chatId = Guid.NewGuid();
+        var blocksStoredBeforeTheAnswer = -1;
+        var service = this.CreateService((_, answer, _, _) =>
+        {
+            blocksStoredBeforeTheAnswer = store.StoredBlocks(chatId).Count;
+            answer.Text = "An answer.";
+            return Task.CompletedTask;
+        });
+
+        var conversation = await service.GetAsync(store, COMPONENT, chatId, () => new ChatThread());
+        var sent = await service.SendAsync(chatId, this.Request("A question."));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(sent, Is.True);
+            Assert.That(blocksStoredBeforeTheAnswer, Is.EqualTo(1), "Only the message was stored when the model began.");
+            Assert.That(store.StoredBlocks(chatId).Select(block => block.Role), Is.EqualTo(new[] { ChatRole.USER, ChatRole.AI }));
+            Assert.That(conversation.UserMessageCount, Is.EqualTo(1));
+            Assert.That(conversation.HasAnswer, Is.True);
+            Assert.That(conversation.IsRunning, Is.False);
+        }
+    }
+
+    [Test]
+    public async Task AnEmptyAnswerLeavesOnlyTheMessage()
+    {
+        var store = new InMemoryEmbeddedChatStore();
+        var chatId = Guid.NewGuid();
+        var service = this.CreateService(Answer(string.Empty));
+
+        var conversation = await service.GetAsync(store, COMPONENT, chatId, () => new ChatThread());
+        var sent = await service.SendAsync(chatId, this.Request("A question."));
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(sent, Is.True, "The message went in, even without an answer.");
+            Assert.That(store.StoredBlocks(chatId).Select(block => block.Role), Is.EqualTo(new[] { ChatRole.USER }));
+            Assert.That(conversation.HasAnswer, Is.False);
+        }
+    }
+
+    [Test]
+    public async Task ARefusedPreparationSendsNothing()
+    {
+        var store = new InMemoryEmbeddedChatStore();
+        var chatId = Guid.NewGuid();
+        var service = this.CreateService(Answer("An answer."));
+
+        var conversation = await service.GetAsync(store, COMPONENT, chatId, () => new ChatThread());
+        var sent = await service.SendAsync(chatId, this.Request("A question.") with { PrepareAsync = _ => Task.FromResult(false) });
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(sent, Is.False);
+            Assert.That(conversation.Thread!.Blocks, Is.Empty);
+            Assert.That(store.SaveCount, Is.Zero);
+            Assert.That(conversation.IsRunning, Is.False, "A refused request does not block the next one.");
+        }
+    }
+
+    [Test]
+    public async Task WithoutAProviderNothingIsSent()
+    {
+        var store = new InMemoryEmbeddedChatStore();
+        var chatId = Guid.NewGuid();
+        var service = this.CreateService(Answer("An answer."));
+
+        var conversation = await service.GetAsync(store, COMPONENT, chatId, () => new ChatThread());
+        var sent = await service.SendAsync(chatId, this.Request("A question.") with { Provider = AIStudio.Settings.Provider.NONE });
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(sent, Is.False);
+            Assert.That(conversation.Thread!.Blocks, Is.Empty);
+        }
+    }
+
+    [Test]
+    public async Task ASecondMessageWaitsUntilTheFirstIsAnswered()
+    {
+        var store = new InMemoryEmbeddedChatStore();
+        var chatId = Guid.NewGuid();
+        var answering = new TaskCompletionSource();
+        var release = new TaskCompletionSource();
+        var service = this.CreateService(async (_, answer, _, _) =>
+        {
+            answering.SetResult();
+            await release.Task;
+            answer.Text = "An answer.";
+        });
+
+        var conversation = await service.GetAsync(store, COMPONENT, chatId, () => new ChatThread());
+        var first = service.SendAsync(chatId, this.Request("A question."));
+        await answering.Task;
+
+        var second = await service.SendAsync(chatId, this.Request("Another question."));
+        release.SetResult();
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(second, Is.False);
+            Assert.That(await first, Is.True);
+            Assert.That(conversation.UserMessageCount, Is.EqualTo(1));
+        }
+    }
+
+    [Test]
+    public async Task TheRequestMayCarryMoreThanTheStoredChat()
+    {
+        var store = new InMemoryEmbeddedChatStore();
+        var chatId = Guid.NewGuid();
+        var blocksSent = -1;
+        var service = this.CreateService((_, answer, thread, _) =>
+        {
+            blocksSent = thread.Blocks.Count;
+            answer.Text = "An answer.";
+            return Task.CompletedTask;
+        });
+
+        await service.GetAsync(store, COMPONENT, chatId, () => new ChatThread());
+        var extraBlock = new ContentBlock { Role = ChatRole.USER, ContentType = ContentType.TEXT, Content = new ContentText { Text = "An image description." } };
+        await service.SendAsync(chatId, this.Request("A question.") with { ThreadForRequest = thread => thread with { Blocks = [extraBlock, .. thread.Blocks] } });
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(blocksSent, Is.EqualTo(3), "The request carried the extra block, the message, and the answer.");
+            Assert.That(store.StoredBlocks(chatId), Has.Count.EqualTo(2), "The stored chat never holds the extra block.");
+        }
+    }
+
+    [Test]
+    public async Task AHiddenMessageIsNotCountedAsOneOfTheUser()
+    {
+        var store = new InMemoryEmbeddedChatStore();
+        var chatId = Guid.NewGuid();
+        var service = this.CreateService(Answer("Hello, how can I help?"));
+
+        var conversation = await service.GetAsync(store, COMPONENT, chatId, () => new ChatThread());
+        await service.SendAsync(chatId, this.Request("Open the conversation.") with { IsHidden = true });
+
+        using (Assert.EnterMultipleScope())
+        {
+            Assert.That(conversation.UserMessageCount, Is.Zero);
+            Assert.That(conversation.HasVisibleMessages, Is.True, "The answer is visible.");
+        }
+    }
+
+    private EmbeddedChatService CreateService(Func<EmbeddedChatRequest, ContentText, ChatThread, CancellationToken, Task> answerAsync) => new(this.settingsManager, new AssistantSessionService(MessageBus.INSTANCE), MessageBus.INSTANCE, NullLogger<EmbeddedChatService>.Instance, answerAsync);
+
+    private EmbeddedChatRequest Request(string text) => new() { Provider = this.provider, Message = new ContentText { Text = text } };
+
+    private static Func<EmbeddedChatRequest, ContentText, ChatThread, CancellationToken, Task> Answer(string text) => (_, answer, _, _) =>
+    {
+        answer.Text = text;
+        return Task.CompletedTask;
+    };
+}

app/Tests/Chat/Embedded/InMemoryEmbeddedChatStore.cs

@@ -0,0 +1,46 @@
+using AIStudio.Chat;
+using AIStudio.Chat.Embedded;
+
+namespace AIStudio.Tests.Chat.Embedded;
+
+/// <summary>
+/// Keeps embedded chats in memory, as they were when they were stored last.
+/// </summary>
+public sealed class InMemoryEmbeddedChatStore : IEmbeddedChatStore
+{
+    private readonly Dictionary<Guid, ChatThread> threads = [];
+
+    /// <summary>
+    /// Whether loading fails, as with a chat this version cannot open.
+    /// </summary>
+    public bool CannotOpen { get; init; }
+
+    /// <summary>
+    /// How often a chat was stored.
+    /// </summary>
+    public int SaveCount { get; private set; }
+
+    /// <summary>
+    /// Gets the blocks of a chat as they were stored last, or none.
+    /// </summary>
+    /// <param name="chatId">The ID of the chat.</param>
+    public IReadOnlyList<ContentBlock> StoredBlocks(Guid chatId) => this.threads.TryGetValue(chatId, out var thread) ? thread.Blocks : [];
+
+    /// <inheritdoc />
+    public Task<ChatThread?> LoadThreadAsync(Guid chatId, CancellationToken token = default)
+    {
+        if (this.CannotOpen)
+            throw new InvalidDataException("A newer version stored this chat.");
+
+        return Task.FromResult(this.threads.TryGetValue(chatId, out var thread) ? thread : null);
+    }
+
+    /// <inheritdoc />
+    public Task SaveThreadAsync(ChatThread thread, CancellationToken token = default)
+    {
+        // A copy of the list, so that what the chat adds later does not change what was stored:
+        this.threads[thread.ChatId] = thread with { Blocks = [.. thread.Blocks] };
+        this.SaveCount++;
+        return Task.CompletedTask;
+    }
+}