using AIStudio.Chat;
namespace AIStudio.Tests.Chat;
///
/// Checks what a conversation is counted as costing before it is sent.
///
///
/// The number under the input field used to count the sentence being typed and nothing else, which
/// answers a question nobody asks: what decides whether the next message fits is everything that
/// travels with it. So what is collected here has to be what the message builder actually sends --
/// no more, because a number which counts something that stays behind is wrong in the direction
/// that makes a person stop writing.
///
[TestFixture]
public sealed class ConversationPartsTests
{
private string directory = string.Empty;
[SetUp]
public void CreateFiles()
{
this.directory = Path.Combine(Path.GetTempPath(), $"ai-studio-tests-{Guid.NewGuid():N}");
Directory.CreateDirectory(this.directory);
}
[TearDown]
public void RemoveFiles()
{
if (Directory.Exists(this.directory))
Directory.Delete(this.directory, true);
}
[Test]
public void TheWholeConversationCountsAndNotOnlyWhatIsBeingTyped()
{
var thread = new ChatThread
{
SystemPrompt = "You are helpful.",
Blocks =
[
Block("What is the capital of France?"),
Block("Paris."),
],
};
var parts = ConversationParts.Of(thread, "You are helpful.", "And of Italy?", null, imagesAreSent: true);
Assert.Multiple(() =>
{
Assert.That(parts.Texts, Is.EqualTo(new[] { "You are helpful.", "What is the capital of France?", "Paris." }));
Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "And of Italy?" }));
});
}
[Test]
public void WhatIsStillBeingWrittenIsKeptApartFromWhatStands()
{
//
// Both cost the same and both are counted. They are kept apart because of what happens
// afterwards: a message which stands says the same thing forever and its count is worth
// remembering, while the answer being streamed is a different text three seconds later.
//
var streaming = Block("The answer so far");
((ContentText)streaming.Content!).IsStreaming = true;
var thread = new ChatThread { Blocks = [Block("A question."), streaming] };
var parts = ConversationParts.Of(thread, string.Empty, "a draft", null, imagesAreSent: true);
Assert.Multiple(() =>
{
Assert.That(parts.Texts, Is.EqualTo(new[] { "A question." }));
Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "The answer so far", "a draft" }));
});
}
[Test]
public void AnAnswerWhichIsFinishedStandsLikeAnyOtherMessage()
{
var finished = Block("The whole answer.");
((ContentText)finished.Content!).IsStreaming = false;
var parts = ConversationParts.Of(new() { Blocks = [finished] }, string.Empty, string.Empty, null, imagesAreSent: true);
Assert.Multiple(() =>
{
Assert.That(parts.Texts, Is.EqualTo(new[] { "The whole answer." }));
Assert.That(parts.GrowingTexts, Is.Empty);
});
}
[Test]
public void TheSystemPromptCountedIsTheOneWhichWouldBeSent()
{
//
// Not the one standing in the thread. A chat template may replace it, retrieved data is
// appended to it, a profile adds a paragraph and the tool policy adds another -- and
// switching a profile while writing has to move the number, which it cannot do if the
// thread's own field is what gets counted.
//
var thread = new ChatThread { SystemPrompt = "What the person typed." };
var parts = ConversationParts.Of(thread, "What the request carries.", string.Empty, null, imagesAreSent: true);
Assert.That(parts.Texts, Is.EqualTo(new[] { "What the request carries." }));
}
[Test]
public void ABlockHiddenFromTheUserStillCosts()
{
//
// Hidden on the screen, not in the request: the message builder sends it like any other
// block, so its tokens are gone whether or not anybody can see where they went.
//
var hidden = Block("An instruction the user does not see.");
var thread = new ChatThread { Blocks = [new() { ContentType = hidden.ContentType, Role = hidden.Role, Content = hidden.Content, HideFromUser = true }] };
var parts = ConversationParts.Of(thread, string.Empty, string.Empty, null, imagesAreSent: true);
Assert.That(parts.Texts, Is.EqualTo(new[] { "An instruction the user does not see." }));
}
[Test]
public void WithoutAConversationOnlyTheDraftCounts()
{
var parts = ConversationParts.Of(null, string.Empty, "Hello", null, imagesAreSent: true);
Assert.Multiple(() =>
{
Assert.That(parts.Texts, Is.Empty);
Assert.That(parts.GrowingTexts, Is.EqualTo(new[] { "Hello" }));
});
}
[TestCase("")]
[TestCase(" ")]
public void NothingWrittenIsNothingToCount(string draft)
{
var parts = ConversationParts.Of(null, string.Empty, draft, null, imagesAreSent: true);
Assert.Multiple(() =>
{
Assert.That(parts.Texts, Is.Empty);
Assert.That(parts.GrowingTexts, Is.Empty);
});
}
[Test]
public void ABlockWithoutTextIsSkippedBecauseItIsNeverSent()
{
//
// The message builder drops a block whose text is empty, whatever else hangs off it. A
// count which added that block's attachments would report tokens for a message which is
// never built.
//
var document = this.WriteFile("notes.txt", "some content");
var empty = Block(string.Empty);
((ContentText)empty.Content!).FileAttachments.Add(FileAttachment.FromPath(document));
var parts = ConversationParts.Of(new() { Blocks = [empty] }, string.Empty, string.Empty, null, imagesAreSent: true);
Assert.Multiple(() =>
{
Assert.That(parts.Texts, Is.Empty);
Assert.That(parts.Documents, Is.Empty);
});
}
[Test]
public void AttachmentsOfTheConversationAndOfTheComposerBothCount()
{
//
// A document attached three messages ago is sent again with every further message, so it
// costs its tokens again every time. That is exactly the thing a person cannot see and
// which this number is for.
//
var older = this.WriteFile("older.txt", "older content");
var draft = this.WriteFile("draft.txt", "draft content");
var block = Block("Please read this.");
((ContentText)block.Content!).FileAttachments.Add(FileAttachment.FromPath(older));
var parts = ConversationParts.Of(new() { Blocks = [block] }, string.Empty, "And this one.", [FileAttachment.FromPath(draft)], imagesAreSent: true);
Assert.That(parts.Documents.Select(document => document.FileName), Is.EqualTo(new[] { "older.txt", "draft.txt" }));
}
[Test]
public void AnAttachmentWhoseFileIsGoneCountsForNothing()
{
//
// It is not sent either: the message builder reports it as unavailable and leaves it out.
//
var attachment = FileAttachment.FromPath(Path.Combine(this.directory, "never-existed.txt"));
var parts = ConversationParts.Of(null, string.Empty, "Here", [attachment], imagesAreSent: true);
Assert.That(parts.Documents, Is.Empty);
}
[Test]
public void ImagesAreCountedSeparatelyFromDocuments()
{
var document = this.WriteFile("notes.txt", "content");
var image = this.WriteFile("photo.png", "not really a png");
var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(document), FileAttachment.FromPath(image)], imagesAreSent: true);
Assert.Multiple(() =>
{
Assert.That(parts.Documents.Select(entry => entry.FileName), Is.EqualTo(new[] { "notes.txt" }));
Assert.That(parts.Images, Is.EqualTo(1));
});
}
[Test]
public void AModelWhichTakesNoImagesIsSentNoneAndIsToldAboutNone()
{
//
// The message builder leaves the pictures out entirely for such a model, so reporting them
// as uncounted would tell a person about a cost which is not there.
//
var image = this.WriteFile("photo.png", "not really a png");
var parts = ConversationParts.Of(null, string.Empty, "Look", [FileAttachment.FromPath(image)], imagesAreSent: false);
Assert.That(parts.Images, Is.Zero);
}
private static ContentBlock Block(string text) => new()
{
ContentType = ContentType.TEXT,
Role = ChatRole.USER,
Content = new ContentText { Text = text },
};
private string WriteFile(string name, string content)
{
var path = Path.Combine(this.directory, name);
File.WriteAllText(path, content);
return path;
}
}