Merge branch 'main' into feature/token-count-from-usage

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2026-09-23 17:18:06 +02:00
commit ae10bb4809
69 files changed
+4129 -336

No files matched your search

+225
View File
@@ -0,0 +1,225 @@
using AIStudio.Agents;
using AIStudio.Chat;
using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings.DataModel;
namespace AIStudio.Tests.Chat;
/// <summary>
/// Checks what rolling a chat back to an earlier answer removes and what it keeps.
/// </summary>
/// <remarks>
/// A rollback has to remove more than the user sees. The prompts an assistant sends into a chat are
/// hidden, yet they are part of the conversation; a chat which kept them would continue with
/// messages the user can neither see nor remove. The example conversation of a chat template is
/// hidden as well, but it comes before everything else and has to keep working.
///
/// What a rollback must not remove is anything which tells the providers what this chat has seen.
/// Removing the message which brought confidential data in does not unsee it, so the chat keeps
/// demanding a self-hosted provider, and the confidence which that data asked for.
/// </remarks>
[TestFixture]
public sealed class ChatThreadRollbackTests
{
private static readonly DateTimeOffset START = new(2026, 9, 23, 10, 0, 0, TimeSpan.Zero);
[Test]
public void HiddenPromptsAfterTheAnswerGoAsWell()
{
var answer = Block(ChatRole.AI, "First answer", 2);
var thread = new ChatThread
{
Blocks =
[
Block(ChatRole.USER, "First question", 1),
answer,
Block(ChatRole.USER, "Prompt of an assistant", 3, hidden: true),
Block(ChatRole.AI, "Answer to the assistant", 4),
],
};
var rolledBack = thread.RollBackTo(answer.Content!);
Assert.Multiple(() =>
{
Assert.That(rolledBack, Is.True, "Two blocks came after the answer, so the rollback removed something.");
Assert.That(Texts(thread), Is.EqualTo(new[] { "First question", "First answer" }), "The hidden prompt goes with the answer to it, although the user never saw it.");
});
}
[Test]
public void TheExampleConversationOfAChatTemplateStays()
{
var answer = Block(ChatRole.AI, "First answer", 4);
var thread = new ChatThread
{
Blocks =
[
Block(ChatRole.USER, "Example question", 1, hidden: true),
Block(ChatRole.AI, "Example answer", 2, hidden: true),
Block(ChatRole.USER, "First question", 3),
answer,
Block(ChatRole.USER, "Second question", 5),
Block(ChatRole.AI, "Second answer", 6),
],
};
thread.RollBackTo(answer.Content!);
Assert.That(Texts(thread), Is.EqualTo(new[] { "Example question", "Example answer", "First question", "First answer" }), "Hidden blocks before the answer belong to what the user rolls back to, so the chat template keeps its example conversation.");
}
[Test]
public void TheOrderIsTheOneOfTheTimeStampsNotTheOneOfTheList()
{
var answer = Block(ChatRole.AI, "First answer", 2);
var thread = new ChatThread
{
Blocks =
[
Block(ChatRole.USER, "Second question", 3),
Block(ChatRole.AI, "Second answer", 4),
answer,
Block(ChatRole.USER, "First question", 1),
],
};
thread.RollBackTo(answer.Content!);
Assert.That(Texts(thread), Is.EqualTo(new[] { "First question", "First answer" }), "The chat shows its blocks by time, so a rollback has to count by time as well, wherever a block sits in the list.");
}
[Test]
public void TheLastRetrievalIsDroppedButTheChoiceOfDataSourcesStays()
{
var answer = Block(ChatRole.AI, "First answer", 2);
var options = new DataSourceOptions
{
DisableDataSources = false,
AutomaticDataSourceSelection = true,
PreselectedDataSourceIds = ["handbook"],
};
var thread = new ChatThread
{
DataSourceOptions = options,
AISelectedDataSources = [SelectedHandbook()],
AugmentedData = "A chunk from the handbook",
Blocks =
[
Block(ChatRole.USER, "First question", 1),
answer,
Block(ChatRole.USER, "Second question", 3),
Block(ChatRole.AI, "Second answer", 4),
],
};
thread.RollBackTo(answer.Content!);
Assert.Multiple(() =>
{
Assert.That(thread.AugmentedData, Is.Empty, "Nobody knows whether the retrieved data belongs to a kept or to a removed message, so it must not reach the next system prompt.");
Assert.That(thread.AISelectedDataSources, Is.Empty, "The data sources an agent picked belong to the same retrieval as the data.");
Assert.That(thread.DataSourceOptions, Is.SameAs(options), "The data source options are the user's choice, not the result of a message.");
Assert.That(options.DisableDataSources, Is.False);
Assert.That(options.AutomaticDataSourceSelection, Is.True);
Assert.That(options.PreselectedDataSourceIds, Is.EqualTo(new[] { "handbook" }));
});
}
[Test]
public void WhatTheChatHasSeenKeepsRestrictingTheProviders()
{
var answer = Block(ChatRole.AI, "First answer", 2);
var thread = new ChatThread
{
DataSecurity = DataSourceSecurity.SELF_HOSTED,
Blocks =
[
Block(ChatRole.USER, "First question", 1),
answer,
Block(ChatRole.USER, "Question about the confidential handbook", 3),
Block(ChatRole.AI, "Answer from the confidential handbook", 4),
],
};
thread.RequireProviderConfidence(ConfidenceLevel.HIGH);
thread.RollBackTo(answer.Content!);
Assert.Multiple(() =>
{
Assert.That(thread.DataSecurity, Is.EqualTo(DataSourceSecurity.SELF_HOSTED), "The confidential data was seen by this chat, so no cloud provider may continue it.");
Assert.That(thread.RequiredProviderConfidence, Is.EqualTo(ConfidenceLevel.HIGH), "The confidence the data demanded stays, although the message which brought it in is gone.");
});
}
[Test]
public void RollingBackToTheLastAnswerChangesNothing()
{
var answer = Block(ChatRole.AI, "First answer", 2);
var thread = ThreadWithRetrieval(Block(ChatRole.USER, "First question", 1), answer);
var rolledBack = thread.RollBackTo(answer.Content!);
AssertUnchanged(thread, rolledBack, "First question", "First answer");
}
[Test]
public void RollingBackToUnknownContentChangesNothing()
{
var thread = ThreadWithRetrieval(Block(ChatRole.USER, "First question", 1), Block(ChatRole.AI, "First answer", 2));
var rolledBack = thread.RollBackTo(new ContentText { Text = "Answer of another chat" });
AssertUnchanged(thread, rolledBack, "First question", "First answer");
}
/// <summary>
/// A chat whose last retrieval is still in place, so that a rollback which should do nothing
/// shows when it resets it anyway.
/// </summary>
/// <param name="blocks">The blocks of the chat.</param>
/// <returns>The chat.</returns>
private static ChatThread ThreadWithRetrieval(params ContentBlock[] blocks) => new()
{
AISelectedDataSources = [SelectedHandbook()],
AugmentedData = "A chunk from the handbook",
Blocks = [..blocks],
};
private static void AssertUnchanged(ChatThread thread, bool rolledBack, params string[] expectedTexts)
{
Assert.Multiple(() =>
{
Assert.That(rolledBack, Is.False, "Nothing came after the content, so there was nothing to roll back.");
Assert.That(Texts(thread), Is.EqualTo(expectedTexts), "A rollback without anything to remove leaves the blocks alone.");
Assert.That(thread.AugmentedData, Is.EqualTo("A chunk from the handbook"), "A rollback without anything to remove keeps the last retrieval, because it still belongs to the last message.");
Assert.That(thread.AISelectedDataSources, Has.Count.EqualTo(1));
});
}
private static ContentBlock Block(ChatRole role, string text, int minute, bool hidden = false) => new()
{
Time = START.AddMinutes(minute),
ContentType = ContentType.TEXT,
Content = new ContentText { Text = text },
Role = role,
HideFromUser = hidden,
};
private static DataSourceAgentSelected SelectedHandbook() => new()
{
DataSource = new DataSourceLocalDirectory { Id = "handbook", Name = "Handbook" },
AIDecision = new SelectedDataSource("handbook", "The question is about the handbook.", 0.9f),
Selected = true,
};
/// <summary>
/// The texts of the chat in the order the chat shows them.
/// </summary>
/// <param name="thread">The chat.</param>
/// <returns>The texts.</returns>
private static string[] Texts(ChatThread thread) => thread.Blocks.OrderBy(x => x.Time).Select(x => ((ContentText)x.Content!).Text).ToArray();
}
+83 -3
View File
@@ -162,16 +162,85 @@ public sealed class IContentExtensionsTests
"|---|---|",
"| Q1 | 100 |"), TOOL_SOURCE);
content.TryGetMarkdownText(out var markdown);
var tables = PlainFileExport.ExtractTables(markdown, ',');
var tables = FilesOf(content);
Assert.Multiple(() =>
{
Assert.That(tables, Has.Count.EqualTo(1), "One table in the message, one table offered for it.");
Assert.That(tables[0].Content, Does.Not.Contain("example.org"), "A data table has no column a link list would fit into.");
Assert.That(content.ToExportContent(tables[0]), Is.EqualTo(tables[0].Content), "A data table has no column a link list would fit into.");
});
}
[Test]
public void AMarkdownBlockCarriesTheSourcesVisibly()
{
var content = TextWith(Lines(
"Here are your notes:",
string.Empty,
"```markdown",
"# Notes",
string.Empty,
"The notes.",
"```"), TOOL_SOURCE);
var exported = content.ToExportContent(FilesOf(content).Single());
Assert.That(TopLevelBlocksOf(exported), Is.EqualTo(new[] { "h1", "ParagraphBlock", "h1", "h2", "ListBlock" }), "The notes without the text around them, followed by the source list just as the entire answer carries it.");
}
[Test]
public void AWebPageCarriesTheSourcesInAComment()
{
var content = TextWith(Lines(
"```html",
"<!DOCTYPE html>",
"<html><body><p>Hello</p></body></html>",
"```"), TOOL_SOURCE);
var file = FilesOf(content).Single();
var exported = content.ToExportContent(file);
var appended = exported[file.Content.Length..].Trim();
Assert.Multiple(() =>
{
Assert.That(exported, Does.StartWith(file.Content), "The page stays as the model wrote it.");
Assert.That(appended, Does.StartWith("<!--").And.EndWith("-->"), "Below the page stands one comment and nothing a browser would show.");
Assert.That(appended, Does.Contain(TOOL_SOURCE.URL));
});
}
[Test]
public void ALatexBlockCarriesTheSourcesInComments()
{
var content = TextWith(Lines(
"```latex",
@"\section{Results}",
"```"), TOOL_SOURCE);
var file = FilesOf(content).Single();
var exported = content.ToExportContent(file);
var appended = exported[file.Content.Length..].Trim();
Assert.Multiple(() =>
{
Assert.That(exported, Does.StartWith(file.Content), "The document stays as the model wrote it.");
Assert.That(appended.Split(Environment.NewLine), Has.All.StartWith("%"), "A line LaTeX would read could stop the whole run.");
Assert.That(appended, Does.Contain(TOOL_SOURCE.URL));
});
}
[TestCase("markdown")]
[TestCase("html")]
[TestCase("latex")]
public void WithoutSourcesACodeBlockStaysAsTheModelWroteIt(string language)
{
var content = TextWith(Lines($"```{language}", "The content.", "```"));
var file = FilesOf(content).Single();
Assert.That(content.ToExportContent(file), Is.EqualTo(file.Content), "No comment, no heading, no empty line.");
}
/// <summary>
/// A text message with the given sources hanging on it.
/// </summary>
@@ -184,6 +253,17 @@ public sealed class IContentExtensionsTests
Sources = [..sources],
};
/// <summary>
/// Reads the files of a message the way the export menu does.
/// </summary>
/// <param name="content">The content to read.</param>
/// <returns>The files the export menu offers for it.</returns>
private static IReadOnlyList<MessageFile> FilesOf(IContent content)
{
content.TryGetMarkdownText(out var markdown);
return PlainFileExport.ExtractFiles(markdown, ',');
}
/// <summary>
/// Names the blocks a Markdown text is made of, headings by their level.
/// </summary>
@@ -0,0 +1,299 @@
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using Lua;
using Lua.Standard;
namespace AIStudio.Tests.Settings;
/// <summary>
/// Checks the tools and data sources a chat template carries, across the two surfaces which have
/// to agree on them: the Lua a configuration plugin states, and the Lua the app exports.
/// </summary>
/// <remarks>
/// The interesting part is not that a value survives, but that the difference between "this
/// template says nothing" and "this template says none" survives. Both end up as an empty
/// selection in the chat on a fresh installation, so a mistake here stays invisible until somebody
/// sets a default tool for their chats -- and then quietly hands out a tool the template ruled out.
/// The last tests cover what the export says out loud before it runs, for the same reason: a data
/// source which cannot be rolled out goes unnoticed on the machine reading the plugin.
/// </remarks>
[TestFixture]
public sealed class ChatTemplateConfigurationTests
{
private static readonly Guid PLUGIN_ID = new("22222222-2222-2222-2222-222222222222");
[Test]
public async Task WhatTheAppExportsIsWhatAConfigurationPluginCanReadBack()
{
var written = NewTemplate() with
{
ToolIds = ["read_web_page", "web_search"],
DataSourceOptions = new()
{
DisableDataSources = false,
AutomaticDataSourceSelection = false,
AutomaticValidation = true,
PreselectedDataSourceIds = ["11111111-1111-1111-1111-111111111111"],
},
};
var read = await ExportAndReadBackAsync(written);
Assert.Multiple(() =>
{
Assert.That(read.ToolIds, Is.EquivalentTo(written.ToolIds!));
Assert.That(read.DataSourceOptions, Is.Not.Null);
Assert.That(read.DataSourceOptions!.DisableDataSources, Is.False);
Assert.That(read.DataSourceOptions.AutomaticDataSourceSelection, Is.False);
Assert.That(read.DataSourceOptions.AutomaticValidation, Is.True);
Assert.That(read.DataSourceOptions.PreselectedDataSourceIds, Is.EqualTo(written.DataSourceOptions!.PreselectedDataSourceIds));
});
}
[Test]
public async Task AnAgenticSelectionSurvivesTheExportAsWell()
{
var written = NewTemplate() with
{
DataSourceOptions = new()
{
DisableDataSources = false,
AutomaticDataSourceSelection = true,
AutomaticValidation = true,
PreselectedDataSourceIds = [],
},
};
var read = await ExportAndReadBackAsync(written);
Assert.Multiple(() =>
{
Assert.That(read.DataSourceOptions, Is.Not.Null);
Assert.That(read.DataSourceOptions!.AutomaticDataSourceSelection, Is.True, "Letting an agent pick the sources is the one thing only a chat template can state, so it must not be lost on the way through Lua.");
Assert.That(read.DataSourceOptions.PreselectedDataSourceIds, Is.Empty);
Assert.That(read.ToolIds, Is.Null);
});
}
[Test]
public async Task ATemplateWhichSaysNothingExportsNeitherTable()
{
var written = NewTemplate();
Assert.That(written.TryExportAsConfigurationSection(out var luaCode, out var issue), Is.True, issue);
Assert.Multiple(() =>
{
Assert.That(luaCode, Does.Not.Contain("ToolIds"));
Assert.That(luaCode, Does.Not.Contain("DataSourceOptions"));
});
var read = await ParseAsync(luaCode);
Assert.Multiple(() =>
{
Assert.That(read.ToolIds, Is.Null, "A template without a tool selection must stay without one, so the chat keeps using its own default.");
Assert.That(read.DataSourceOptions, Is.Null);
});
}
[Test]
public async Task ATemplateWhichRulesOutEveryToolStaysThatWay()
{
var written = NewTemplate() with { ToolIds = [] };
var read = await ExportAndReadBackAsync(written);
Assert.That(read.ToolIds, Is.Not.Null, "An empty selection is the statement that this template wants no tools. Reading it back as null would hand out the chat default instead.");
Assert.That(read.ToolIds, Is.Empty);
}
[Test]
public async Task NamingTheDataSourceOptionsAtAllSwitchesDataSourcesOn()
{
var read = await ParseAsync("""
CONFIG["CHAT_TEMPLATES"][#CONFIG["CHAT_TEMPLATES"]+1] = {
["Id"] = "33333333-3333-3333-3333-333333333333",
["Name"] = "Intranet Research",
["SystemPrompt"] = "You are a research assistant.",
["DataSourceOptions"] = {
["PreselectedDataSourceIds"] = {
"11111111-1111-1111-1111-111111111111",
},
},
}
""");
Assert.That(read.DataSourceOptions, Is.Not.Null);
Assert.Multiple(() =>
{
Assert.That(read.DataSourceOptions!.DisableDataSources, Is.False, "Writing this table is already the statement that the template wants data sources, so an omitted switch must not turn them off again.");
Assert.That(read.DataSourceOptions.AutomaticDataSourceSelection, Is.False);
Assert.That(read.DataSourceOptions.AutomaticValidation, Is.False);
Assert.That(read.DataSourceOptions.PreselectedDataSourceIds, Is.EqualTo(new[] { "11111111-1111-1111-1111-111111111111" }));
});
}
[Test]
public async Task AnUnusableEntryIsSkippedAndTheRestOfTheListSurvives()
{
var read = await ParseAsync("""
CONFIG["CHAT_TEMPLATES"][#CONFIG["CHAT_TEMPLATES"]+1] = {
["Id"] = "33333333-3333-3333-3333-333333333333",
["Name"] = "Intranet Research",
["SystemPrompt"] = "You are a research assistant.",
["ToolIds"] = {
"web_search",
"",
{},
"read_web_page",
},
["DataSourceOptions"] = {
["PreselectedDataSourceIds"] = {
"11111111-1111-1111-1111-111111111111",
" ",
},
},
}
""");
Assert.Multiple(() =>
{
Assert.That(read.ToolIds, Is.EquivalentTo(new[] { "web_search", "read_web_page" }));
Assert.That(read.DataSourceOptions!.PreselectedDataSourceIds, Is.EqualTo(new[] { "11111111-1111-1111-1111-111111111111" }));
});
}
[Test]
public void ExportedDataSourceIdsComeWithTheNoteThatTheyAreLocalOnes()
{
var withSources = NewTemplate() with
{
DataSourceOptions = new() { DisableDataSources = false, PreselectedDataSourceIds = ["11111111-1111-1111-1111-111111111111"] },
};
var agenticOnly = NewTemplate() with
{
DataSourceOptions = new() { DisableDataSources = false, AutomaticDataSourceSelection = true },
};
Assert.That(withSources.TryExportAsConfigurationSection(out var withSourcesLua, out var issue), Is.True, issue);
Assert.That(agenticOnly.TryExportAsConfigurationSection(out var agenticOnlyLua, out issue), Is.True, issue);
Assert.Multiple(() =>
{
Assert.That(withSourcesLua, Does.StartWith("--"), "Whoever pastes this into a plugin cannot see from the IDs alone that they belong to another machine.");
Assert.That(agenticOnlyLua, Does.Not.StartWith("--"), "Without IDs there is nothing to check, so the note would only be noise.");
});
}
[Test]
public void LocalDataSourcesOfATemplateAreNamedBeforeItIsExported()
{
var template = NewTemplate() with
{
DataSourceOptions = new()
{
DisableDataSources = false,
PreselectedDataSourceIds = ["11111111-1111-1111-1111-111111111111", "22222222-2222-2222-2222-222222222222"],
},
};
var localNames = ChatTemplate.GetPreselectedLocalDataSourceNames(template, ConfiguredDataSources());
Assert.That(localNames, Is.EqualTo(new[] { "Meeting notes" }), "Only the local source can be named: its ID means nothing on the machine which reads the exported plugin, while the ERI source points at something the whole organization reaches.");
}
[Test]
public void ATemplateWithoutLocalDataSourcesIsExportedWithoutAQuestion()
{
var eriOnly = NewTemplate() with
{
DataSourceOptions = new() { DisableDataSources = false, PreselectedDataSourceIds = ["22222222-2222-2222-2222-222222222222"] },
};
var agentic = NewTemplate() with
{
DataSourceOptions = new() { DisableDataSources = false, AutomaticDataSourceSelection = true },
};
Assert.Multiple(() =>
{
Assert.That(ChatTemplate.GetPreselectedLocalDataSourceNames(eriOnly, ConfiguredDataSources()), Is.Empty);
Assert.That(ChatTemplate.GetPreselectedLocalDataSourceNames(agentic, ConfiguredDataSources()), Is.Empty, "An agent picks the sources per message, so this template names none to begin with.");
Assert.That(ChatTemplate.GetPreselectedLocalDataSourceNames(NewTemplate(), ConfiguredDataSources()), Is.Empty);
});
}
[Test]
public void AnIdWhichMatchesNoDataSourceIsNotReportedAsALocalOne()
{
var template = NewTemplate() with
{
DataSourceOptions = new() { DisableDataSources = false, PreselectedDataSourceIds = ["99999999-9999-9999-9999-999999999999"] },
};
Assert.That(ChatTemplate.GetPreselectedLocalDataSourceNames(template, ConfiguredDataSources()), Is.Empty, "There is no name to warn about, and the note above the exported IDs already tells the admin to check them.");
}
/// <summary>
/// A template with the parts every export needs, and nothing said about tools or data sources.
/// </summary>
private static ChatTemplate NewTemplate() => new()
{
Num = 1,
Id = "33333333-3333-3333-3333-333333333333",
Name = "Intranet Research",
SystemPrompt = "You are a research assistant.",
PredefinedUserPrompt = string.Empty,
ExampleConversation = [],
FileAttachments = [],
AllowProfileUsage = true,
};
/// <summary>
/// One data source of each kind: a local one, which cannot be rolled out, and an ERI one, which can.
/// </summary>
private static IReadOnlyList<IDataSource> ConfiguredDataSources() =>
[
new DataSourceLocalFile { Id = "11111111-1111-1111-1111-111111111111", Name = "Meeting notes", Type = DataSourceType.LOCAL_FILE },
new DataSourceERI_V1 { Id = "22222222-2222-2222-2222-222222222222", Name = "Intranet", Type = DataSourceType.ERI_V1 },
];
private static async Task<ChatTemplate> ExportAndReadBackAsync(ChatTemplate template)
{
Assert.That(template.TryExportAsConfigurationSection(out var luaCode, out var issue), Is.True, issue);
return await ParseAsync(luaCode);
}
/// <summary>
/// Reads a chat template the way a configuration plugin states it.
/// </summary>
/// <remarks>
/// Through a real Lua state rather than a table put together in C#, so that the exported code
/// has to be valid Lua before anything else is checked.
/// </remarks>
/// <param name="luaCode">The lines a plugin would contain, including the assignment itself.</param>
/// <returns>The chat template read from it.</returns>
private static async Task<ChatTemplate> ParseAsync(string luaCode)
{
var state = LuaState.Create();
state.OpenBasicLibrary();
state.OpenTableLibrary();
await state.DoStringAsync($$"""
CONFIG = {}
CONFIG["CHAT_TEMPLATES"] = {}
{{luaCode}}
""");
if (!state.Environment["CONFIG"].TryRead<LuaTable>(out var configTable) ||
!configTable["CHAT_TEMPLATES"].TryRead<LuaTable>(out var templatesTable) ||
!templatesTable[1].TryRead<LuaTable>(out var templateTable))
throw new InvalidOperationException("The code of this test did not produce a chat template table.");
if (!ChatTemplate.TryParseChatTemplateTable(1, templateTable, PLUGIN_ID, string.Empty, out var parsed) || parsed is not ChatTemplate chatTemplate)
throw new InvalidOperationException("The chat template of this test could not be read.");
return chatTemplate;
}
}
@@ -0,0 +1,165 @@
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
namespace AIStudio.Tests.Settings;
/// <summary>
/// Checks who decides the tools and data sources when a direct chat launcher and the chat template
/// it opens its chat with both name some.
/// </summary>
/// <remarks>
/// Either side can be filled in without knowing about the other, so all four combinations happen.
/// The rule is deliberately the same for tools and for data sources: the chat template wins as a
/// whole, because it is the only one of the two which can also leave the choice of sources to an
/// agent, and a field-by-field mix of both would be something neither of them asked for.
/// </remarks>
[TestFixture]
public sealed class ChatTemplatePrecedenceTests
{
[Test]
public void WhenNeitherSideSaysAnythingTheChatDefaultsStay()
{
var toolChoice = ChatTemplate.ChooseToolIds(NewTemplate(), null);
var optionsChoice = ChatTemplate.ChooseDataSourceOptions(NewTemplate(), null);
Assert.Multiple(() =>
{
Assert.That(toolChoice.ToolIds, Is.Null, "Nobody named any tool, so the chat has to keep using the tools of its own default.");
Assert.That(toolChoice.LauncherChoiceDropped, Is.False);
Assert.That(optionsChoice.Options, Is.Null, "Nobody named any data source, so the chat has to keep using its own default options.");
Assert.That(optionsChoice.LauncherChoiceDropped, Is.False);
});
}
[Test]
public void ALauncherAloneDecidesForItself()
{
var template = NewTemplate();
var launcherOptions = NewLauncherOptions("11111111-1111-1111-1111-111111111111");
var toolChoice = ChatTemplate.ChooseToolIds(template, new[] { "web_search" });
var optionsChoice = ChatTemplate.ChooseDataSourceOptions(template, launcherOptions);
Assert.Multiple(() =>
{
Assert.That(toolChoice.ToolIds, Is.EquivalentTo(new[] { "web_search" }));
Assert.That(toolChoice.LauncherChoiceDropped, Is.False, "Nothing was dropped here, so nothing may be reported as dropped either.");
Assert.That(optionsChoice.Options, Is.SameAs(launcherOptions));
Assert.That(optionsChoice.LauncherChoiceDropped, Is.False);
});
}
[Test]
public void ATemplateAloneDecidesForItself()
{
var template = NewTemplate() with
{
ToolIds = ["read_web_page"],
DataSourceOptions = new() { DisableDataSources = false, PreselectedDataSourceIds = ["22222222-2222-2222-2222-222222222222"] },
};
var toolChoice = ChatTemplate.ChooseToolIds(template, null);
var optionsChoice = ChatTemplate.ChooseDataSourceOptions(template, null);
Assert.Multiple(() =>
{
Assert.That(toolChoice.ToolIds, Is.EquivalentTo(new[] { "read_web_page" }));
Assert.That(toolChoice.LauncherChoiceDropped, Is.False);
Assert.That(optionsChoice.Options!.PreselectedDataSourceIds, Is.EqualTo(new[] { "22222222-2222-2222-2222-222222222222" }));
Assert.That(optionsChoice.LauncherChoiceDropped, Is.False);
});
}
[Test]
public void WhenBothSidesSpeakTheTemplateWinsAndTheLossIsReported()
{
var template = NewTemplate() with
{
ToolIds = ["read_web_page"],
DataSourceOptions = new() { DisableDataSources = false, PreselectedDataSourceIds = ["22222222-2222-2222-2222-222222222222"] },
};
var toolChoice = ChatTemplate.ChooseToolIds(template, new[] { "web_search" });
var optionsChoice = ChatTemplate.ChooseDataSourceOptions(template, NewLauncherOptions("11111111-1111-1111-1111-111111111111"));
Assert.Multiple(() =>
{
Assert.That(toolChoice.ToolIds, Is.EquivalentTo(new[] { "read_web_page" }));
Assert.That(toolChoice.LauncherChoiceDropped, Is.True, "The tools of the launcher are gone, and only this flag can make the log say so.");
Assert.That(optionsChoice.Options!.PreselectedDataSourceIds, Is.EqualTo(new[] { "22222222-2222-2222-2222-222222222222" }));
Assert.That(optionsChoice.LauncherChoiceDropped, Is.True);
});
}
[Test]
public void ATemplateWhichWantsNoToolsWinsJustTheSame()
{
var template = NewTemplate() with { ToolIds = [] };
var toolChoice = ChatTemplate.ChooseToolIds(template, new[] { "web_search" });
Assert.Multiple(() =>
{
Assert.That(toolChoice.ToolIds, Is.Empty, "An empty selection is the statement that this template wants no tools, which is as much of a statement as naming one.");
Assert.That(toolChoice.LauncherChoiceDropped, Is.True);
});
}
[Test]
public void TheAgenticSelectionOfATemplateSurvivesALauncherWithItsOwnSources()
{
var template = NewTemplate() with
{
DataSourceOptions = new() { DisableDataSources = false, AutomaticDataSourceSelection = true, PreselectedDataSourceIds = [] },
};
var optionsChoice = ChatTemplate.ChooseDataSourceOptions(template, NewLauncherOptions("11111111-1111-1111-1111-111111111111"));
Assert.Multiple(() =>
{
Assert.That(optionsChoice.Options!.AutomaticDataSourceSelection, Is.True, "Letting an agent pick the sources is the one thing a launcher cannot express, so it is exactly what must not be overwritten by one.");
Assert.That(optionsChoice.Options.PreselectedDataSourceIds, Is.Empty);
Assert.That(optionsChoice.LauncherChoiceDropped, Is.True);
});
}
[Test]
public void TheChosenOptionsAreACopyRatherThanTheOnesOfTheTemplate()
{
var template = NewTemplate() with
{
DataSourceOptions = new() { DisableDataSources = false, PreselectedDataSourceIds = ["22222222-2222-2222-2222-222222222222"] },
};
var optionsChoice = ChatTemplate.ChooseDataSourceOptions(template, null);
optionsChoice.Options!.PreselectedDataSourceIds.Clear();
Assert.That(template.DataSourceOptions!.PreselectedDataSourceIds, Is.Not.Empty, "The launched chat goes on to change these options, and the template is a setting of the user which must not change with it.");
}
/// <summary>
/// A template which says nothing about tools or data sources.
/// </summary>
private static ChatTemplate NewTemplate() => new()
{
Num = 1,
Id = "33333333-3333-3333-3333-333333333333",
Name = "Intranet Research",
SystemPrompt = "You are a research assistant.",
};
/// <summary>
/// The options a launcher which names data sources ends up with.
/// </summary>
/// <remarks>
/// A launcher has no switches of its own: it names sources, and the rest is always this. Which
/// is the whole reason the chat template wins whenever both of them speak.
/// </remarks>
private static DataSourceOptions NewLauncherOptions(params string[] dataSourceIds) => new()
{
DisableDataSources = false,
AutomaticDataSourceSelection = false,
AutomaticValidation = false,
PreselectedDataSourceIds = [..dataSourceIds],
};
}
@@ -0,0 +1,71 @@
using AIStudio.Tools.Services;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Checks when the circuit state reports that the browser connection came back.
/// </summary>
/// <remarks>
/// The report is there to fetch again what the browser sent while the connection was down, because Blazor
/// drops it. The layout reads the color theme anew on it, for instance, since the machine may have switched
/// its theme during sleep. So it has to come after every loss, and only then: missing one leaves the app
/// with stale state, while a connection which was never lost has nothing to fetch again.
/// </remarks>
[TestFixture]
public sealed class CircuitStateServiceTests
{
[Test]
public void AConnectionWhichReturnsAfterALossIsReportedOnce()
{
var circuitState = new CircuitStateService();
var numReports = 0;
circuitState.ConnectionRestored += () => numReports++;
circuitState.MarkAsDisconnected();
circuitState.MarkAsConnected();
Assert.That(numReports, Is.EqualTo(1), "The connection was lost and came back, so whatever the browser sent in between is gone.");
Assert.That(circuitState.IsConnected, Is.True);
}
[Test]
public void TheFirstConnectionIsNotReported()
{
var circuitState = new CircuitStateService();
var numReports = 0;
circuitState.ConnectionRestored += () => numReports++;
circuitState.MarkAsConnected();
Assert.That(numReports, Is.Zero, "A circuit starts out connected, so its first connection has not lost anything.");
}
[Test]
public void AConnectionWhichWasNotLostIsNotReportedAgain()
{
var circuitState = new CircuitStateService();
var numReports = 0;
circuitState.ConnectionRestored += () => numReports++;
circuitState.MarkAsDisconnected();
circuitState.MarkAsConnected();
circuitState.MarkAsConnected();
Assert.That(numReports, Is.EqualTo(1), "The second call follows a connection which was up all along.");
}
[Test]
public void EveryLossIsReportedOnItsOwn()
{
var circuitState = new CircuitStateService();
var numReports = 0;
circuitState.ConnectionRestored += () => numReports++;
circuitState.MarkAsDisconnected();
circuitState.MarkAsConnected();
circuitState.MarkAsDisconnected();
circuitState.MarkAsConnected();
Assert.That(numReports, Is.EqualTo(2), "The machine went to sleep twice, and each time something may have been lost.");
}
}
+130
View File
@@ -36,4 +36,134 @@ public sealed class FileExportFormatTests
FileExportFormat.HTML,
}), "A format was added to or removed from the export menu: say in FollowsPageAnchors whether its reader follows a page in a local link, then name it here.");
}
[TestCase("Q3: Umsatz/Planung?", FileExportFormat.HTML, "Q3 Umsatz Planung.html", Description = "A chat is named after the first words of its question, and those may hold anything.")]
[TestCase("Notes for the meeting.", FileExportFormat.MARKDOWN, "Notes for the meeting.md", Description = "Windows drops a trailing dot anyway.")]
[TestCase(null, FileExportFormat.MICROSOFT_WORD, "export.docx")]
[TestCase(" ", FileExportFormat.LATEX, "export.tex", Description = "A name made of nothing is no name.")]
public void TheSaveDialogSuggestsAUsableFileName(string? name, FileExportFormat format, string expectedFileName)
{
Assert.That(format.ToSuggestedFileName(name), Is.EqualTo(expectedFileName));
}
[Test]
public void AnOverlongFileNameIsShortened()
{
var fileName = FileExportFormat.HTML.ToSuggestedFileName(new string('a', 100));
Assert.That(fileName, Is.EqualTo($"{new string('a', 60)}.html"), "The first ten words of a question easily outgrow what a dialog shows.");
}
[Test]
public void OnlyAWebPageNeedsAPageTitle()
{
Assert.Multiple(() =>
{
Assert.That(FileExportFormat.HTML.NeedsPageTitle(), Is.True, "Without one, the browser tab shows the random name of the temporary file Pandoc read.");
Assert.That(FileExportFormat.MICROSOFT_WORD.NeedsPageTitle(), Is.False, "Word would keep it as a document property nobody asked for.");
Assert.That(FileExportFormat.OPEN_DOCUMENT_TEXT.NeedsPageTitle(), Is.False, "The same goes for an OpenDocument file.");
Assert.That(FileExportFormat.LATEX.NeedsPageTitle(), Is.False);
});
}
[TestCase("html", FileExportFormat.HTML)]
[TestCase("latex", FileExportFormat.LATEX)]
[TestCase("tex", FileExportFormat.LATEX)]
[TestCase("markdown", FileExportFormat.MARKDOWN)]
[TestCase("md", FileExportFormat.MARKDOWN)]
[TestCase("csv", FileExportFormat.CSV)]
[TestCase("tsv", FileExportFormat.TSV)]
[TestCase("HTML", FileExportFormat.HTML, Description = "Models do not agree on the case.")]
[TestCase("LaTeX", FileExportFormat.LATEX)]
[TestCase("CSV", FileExportFormat.CSV)]
[TestCase(" md ", FileExportFormat.MARKDOWN, Description = "Space around the name is no part of it.")]
public void AFenceLanguageNamesItsFormat(string language, FileExportFormat expectedFormat)
{
Assert.Multiple(() =>
{
Assert.That(FileExportFormatExtensions.TryFromCodeFenceLanguage(language, out var format), Is.True);
Assert.That(format, Is.EqualTo(expectedFormat));
Assert.That(format.IsPlainText(), Is.True, "A code block holds text, so the export writes it as it is.");
});
}
[Test]
public void OnlyTheTwoOfficeFormatsAreNoPlainText()
{
Assert.Multiple(() =>
{
Assert.That(FileExportFormat.MICROSOFT_WORD.IsPlainText(), Is.False, "A Word file is an archive, and writing text into one breaks it.");
Assert.That(FileExportFormat.OPEN_DOCUMENT_TEXT.IsPlainText(), Is.False);
Assert.That(FileExportFormat.NONE.IsPlainText(), Is.False, "No format means no file.");
Assert.That(FileExportFormat.UNKNOWN.IsPlainText(), Is.False);
Assert.That(FileExportFormat.HTML.IsPlainText(), Is.True, "A page the model wrote is a finished file, even though an entire answer needs Pandoc to become one.");
Assert.That(FileExportFormat.LATEX.IsPlainText(), Is.True);
Assert.That(FileExportFormat.MARKDOWN.IsPlainText(), Is.True);
Assert.That(FileExportFormat.CSV.IsPlainText(), Is.True);
Assert.That(FileExportFormat.TSV.IsPlainText(), Is.True);
});
}
[TestCase("css", TestName = "A language AI Studio writes no file for")]
[TestCase("docx", TestName = "A format no code block can hold")]
[TestCase("", TestName = "A fence without a language")]
[TestCase(null, TestName = "A fence Markdig read no language for")]
public void AnyOtherFenceLanguageNamesNoFormat(string? language)
{
Assert.Multiple(() =>
{
Assert.That(FileExportFormatExtensions.TryFromCodeFenceLanguage(language, out var format), Is.False);
Assert.That(format, Is.EqualTo(FileExportFormat.NONE));
});
}
[TestCase(FileExportFormat.HTML)]
[TestCase(FileExportFormat.MARKDOWN)]
public void AnHtmlCommentEndsWhereItShouldAndNowhereElse(FileExportFormat format)
{
var found = format.TryToComment("A page titled --> Start, and one titled --!> Next", out var comment);
Assert.Multiple(() =>
{
Assert.That(found, Is.True);
Assert.That(comment, Does.StartWith("<!--"));
Assert.That(comment.IndexOf("-->", StringComparison.Ordinal), Is.EqualTo(comment.Length - 3), "Only the end of the comment may end it; the title would spill onto the page otherwise.");
Assert.That(comment, Does.Not.Contain("--!>"), "A browser ends a comment there as well.");
Assert.That(comment, Does.Contain("Start").And.Contain("Next"), "The title stays readable.");
});
}
[Test]
public void AnHtmlCommentKeepsTheDashesOfAnAddress()
{
FileExportFormat.HTML.TryToComment("https://xn--mnchen-3ya.de/", out var comment);
Assert.That(comment, Does.Contain("https://xn--mnchen-3ya.de/"), "A domain with an umlaut is written with two dashes, and the link has to keep working.");
}
[Test]
public void EveryLineOfALatexCommentIsOne()
{
var found = FileExportFormat.LATEX.TryToComment(Lines("# Sources", string.Empty, "- [1] A title with 100 % and a_b"), out var comment);
Assert.Multiple(() =>
{
Assert.That(found, Is.True);
Assert.That(comment.Split(Environment.NewLine), Is.EqualTo(new[] { "% # Sources", "%", "% - [1] A title with 100 % and a_b" }), "LaTeX has no end of a comment, only the end of a line.");
});
}
[TestCase(FileExportFormat.CSV)]
[TestCase(FileExportFormat.TSV)]
[TestCase(FileExportFormat.MICROSOFT_WORD)]
public void AFormatWithoutCommentsSaysSo(FileExportFormat format)
{
Assert.Multiple(() =>
{
Assert.That(format.TryToComment("A text.", out var comment), Is.False);
Assert.That(comment, Is.Empty);
});
}
private static string Lines(params string[] lines) => string.Join(Environment.NewLine, lines);
}
@@ -0,0 +1,201 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>Hello World</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Fraunces:opsz,wght@9..144,300..900&display=swap" rel="stylesheet">
<style>
:root{
--bg:#0a0f10; --bone:#f4eee3; --ember:#e2653a; --mint:#7ac9b0;
--mx:50%; --my:50%; --gx:50%; --gy:50%;
}
*{box-sizing:border-box}
html,body{height:100%}
body{margin:0;overflow:hidden;background:var(--bg);color:var(--bone);
font-family:"IBM Plex Sans",system-ui,sans-serif;-webkit-font-smoothing:antialiased}
.stage{position:fixed;inset:0;display:grid;place-items:center;isolation:isolate}
.stage > *{grid-area:1/1}
.field{
position:absolute;inset:-20%;z-index:0;filter:blur(6px);
background:
radial-gradient(46% 40% at 24% 28%, rgba(226,101,58,.20), transparent 68%),
radial-gradient(52% 46% at 78% 72%, rgba(122,201,176,.16), transparent 70%),
radial-gradient(80% 70% at 50% 50%, #121a1b, #070b0c 78%);
animation:breathe 18s ease-in-out infinite alternate;
}
@keyframes breathe{
from{transform:scale(1) translate(-1%,1%)}
to{transform:scale(1.08) translate(1.5%,-1.5%)}
}
.grid-base{
position:absolute;inset:0;z-index:1;opacity:.07;
background-image:
linear-gradient(rgba(244,238,227,.9) 1px,transparent 1px),
linear-gradient(90deg,rgba(244,238,227,.9) 1px,transparent 1px);
background-size:46px 46px;
-webkit-mask-image:radial-gradient(70% 70% at 50% 50%,#000,transparent);
mask-image:radial-gradient(70% 70% at 50% 50%,#000,transparent);
}
.grid-scan{
position:absolute;inset:0;z-index:2;
background-image:
linear-gradient(rgba(122,201,176,.55) 1px,transparent 1px),
linear-gradient(90deg,rgba(122,201,176,.55) 1px,transparent 1px);
background-size:46px 46px;
-webkit-mask-image:radial-gradient(230px 230px at var(--mx) var(--my),#000 0%,rgba(0,0,0,.35) 45%,transparent 72%);
mask-image:radial-gradient(230px 230px at var(--mx) var(--my),#000 0%,rgba(0,0,0,.35) 45%,transparent 72%);
}
.glow{position:absolute;inset:0;z-index:2;mix-blend-mode:screen;
background:radial-gradient(280px 280px at var(--gx) var(--gy),rgba(226,101,58,.22),transparent 65%)}
.sweep{
position:absolute;inset:-30% -60%;z-index:2;pointer-events:none;filter:blur(22px);
background:linear-gradient(102deg,transparent 44%,rgba(244,238,227,.085) 50%,transparent 56%);
animation:sweep 15s linear infinite;
}
@keyframes sweep{from{transform:translateX(-26%)}to{transform:translateX(26%)}}
.tilt{position:relative;z-index:5;will-change:transform}
.hello{
position:relative;display:inline-block;margin:0;text-align:center;
font-family:"Fraunces",Georgia,serif;
font-variation-settings:"opsz" 120;font-weight:500;
font-size:clamp(2.4rem,12vw,10.5rem);line-height:.92;letter-spacing:-.025em;
animation:float 11s ease-in-out infinite alternate;
}
@keyframes float{from{transform:translateY(-.02em)}to{transform:translateY(.025em)}}
.ghost{
position:absolute;inset:0;pointer-events:none;color:transparent;
-webkit-text-stroke:.012em rgba(244,238,227,.20);
transform:translate(.045em,.045em);
animation:drift 13s ease-in-out infinite alternate;
}
.ghost.deep{-webkit-text-stroke:.01em rgba(122,201,176,.16);transform:translate(-.05em,-.035em);animation-duration:16s}
@keyframes drift{to{transform:translate(.075em,.02em)}}
.w{display:inline-block;white-space:nowrap}
.m{display:inline-block;overflow:hidden;padding-bottom:.1em;margin-bottom:-.1em;vertical-align:bottom}
.l{display:inline-block;transform:translateY(115%)}
.g{display:inline-block;transition:color .35s ease,transform .4s cubic-bezier(.2,.85,.2,1),text-shadow .4s ease}
.l:hover .g{color:var(--mint);transform:translateY(-.055em);text-shadow:0 0 26px rgba(122,201,176,.45)}
.caret{
display:inline-block;width:.055em;height:.72em;margin-left:.07em;vertical-align:-.02em;
background:var(--ember);box-shadow:0 0 22px rgba(226,101,58,.6);
animation:blink 1.15s steps(1,end) infinite;
}
@keyframes blink{0%,49%{opacity:1}50%,100%{opacity:0}}
body.play .l{animation:rise .82s var(--d) cubic-bezier(.16,1,.3,1) both}
@keyframes rise{from{transform:translateY(115%)}to{transform:translateY(0)}}
body.play .caret{animation:blink 1.15s steps(1,end) .95s infinite,caretin .5s .9s both}
@keyframes caretin{from{opacity:0;transform:scaleY(.2)}to{opacity:1;transform:scaleY(1)}}
body.play .ghost{animation:ghostin 1.4s .15s ease both,drift 13s 1.6s ease-in-out infinite alternate}
@keyframes ghostin{from{opacity:0}to{opacity:1}}
.frame{position:absolute;inset:clamp(14px,3.2vw,42px);z-index:4;border:1px solid rgba(244,238,227,.09);
clip-path:inset(0 0 100% 0);animation:open 1.5s .35s cubic-bezier(.16,1,.3,1) both}
@keyframes open{to{clip-path:inset(0 0 0 0)}}
.tick{position:absolute;width:14px;height:14px;z-index:4;opacity:0;animation:tickin .6s 1.2s ease both}
@keyframes tickin{from{opacity:0;transform:scale(.4)}to{opacity:1;transform:scale(1)}}
.tl{top:clamp(14px,3.2vw,42px);left:clamp(14px,3.2vw,42px);border-top:1px solid var(--ember);border-left:1px solid var(--ember)}
.tr{top:clamp(14px,3.2vw,42px);right:clamp(14px,3.2vw,42px);border-top:1px solid var(--mint);border-right:1px solid var(--mint)}
.bl{bottom:clamp(14px,3.2vw,42px);left:clamp(14px,3.2vw,42px);border-bottom:1px solid var(--mint);border-left:1px solid var(--mint)}
.br{bottom:clamp(14px,3.2vw,42px);right:clamp(14px,3.2vw,42px);border-bottom:1px solid var(--ember);border-right:1px solid var(--ember)}
.vignette{position:absolute;inset:0;z-index:3;
background:radial-gradient(75% 65% at 50% 50%,transparent 40%,rgba(3,6,7,.72) 100%)}
.grain{
position:absolute;inset:-50%;z-index:6;pointer-events:none;opacity:.055;mix-blend-mode:overlay;
background-image:url("data:image/svg+xml,%3Csvg xmlns='http://www.w3.org/2000/svg' width='180' height='180'%3E%3Cfilter id='n'%3E%3CfeTurbulence type='fractalNoise' baseFrequency='0.85' numOctaves='3' stitchTiles='stitch'/%3E%3C/filter%3E%3Crect width='100%25' height='100%25' filter='url(%23n)'/%3E%3C/svg%3E");
animation:grain 5s steps(5) infinite;
}
@keyframes grain{
0%{transform:translate(0,0)}20%{transform:translate(-3%,2%)}40%{transform:translate(2%,-3%)}
60%{transform:translate(-2%,-2%)}80%{transform:translate(3%,1%)}100%{transform:translate(0,0)}
}
@media (prefers-reduced-motion: reduce){
*{animation-duration:.001s!important;animation-iteration-count:1!important;transition-duration:.001s!important}
.l{transform:none}.caret{opacity:1;animation:none}.sweep,.grain,.field{animation:none}
}
</style>
</head>
<body>
<main class="stage">
<div class="field" aria-hidden="true"></div>
<div class="grid-base" aria-hidden="true"></div>
<div class="grid-scan" aria-hidden="true"></div>
<div class="glow" aria-hidden="true"></div>
<div class="sweep" aria-hidden="true"></div>
<div class="tilt">
<h1 class="hello" id="hello" aria-label="Hello World"></h1>
</div>
<div class="vignette" aria-hidden="true"></div>
<div class="frame" aria-hidden="true"></div>
<i class="tick tl" aria-hidden="true"></i><i class="tick tr" aria-hidden="true"></i>
<i class="tick bl" aria-hidden="true"></i><i class="tick br" aria-hidden="true"></i>
<div class="grain" aria-hidden="true"></div>
</main>
<script>
const TEXT = "Hello World";
const hello = document.getElementById("hello");
const reduce = matchMedia("(prefers-reduced-motion: reduce)").matches;
const words = TEXT.split(" ");
hello.innerHTML =
`<span class="ghost deep" aria-hidden="true">${TEXT}</span>` +
`<span class="ghost" aria-hidden="true">${TEXT}</span>`;
let i = 0;
words.forEach((word, wi) => {
const w = document.createElement("span");
w.className = "w";
w.setAttribute("aria-hidden", "true");
[...word].forEach(ch => {
const m = document.createElement("span"); m.className = "m";
const l = document.createElement("span"); l.className = "l";
l.style.setProperty("--d", (0.28 + i * 0.052) + "s");
const g = document.createElement("span"); g.className = "g"; g.textContent = ch;
m.appendChild(l); l.appendChild(g); w.appendChild(m); i++;
});
hello.appendChild(w);
if (wi < words.length - 1) hello.appendChild(document.createTextNode(" "));
});
const caret = document.createElement("i");
caret.className = "caret"; caret.setAttribute("aria-hidden", "true");
hello.appendChild(caret);
const play = () => {
document.body.classList.remove("play");
void document.body.offsetWidth;
document.body.classList.add("play");
};
play();
addEventListener("pointerdown", play);
const root = document.documentElement, tilt = document.querySelector(".tilt");
let tx = innerWidth / 2, ty = innerHeight / 2, gx = tx, gy = ty, px = 0, py = 0;
addEventListener("pointermove", e => {
tx = e.clientX; ty = e.clientY;
root.style.setProperty("--mx", tx + "px");
root.style.setProperty("--my", ty + "px");
}, {passive:true});
addEventListener("pointerleave", () => { tx = innerWidth/2; ty = innerHeight/2; });
(function raf(){
gx += (tx - gx) * 0.07; gy += (ty - gy) * 0.07;
px += ((tx / innerWidth - .5) - px) * 0.05;
py += ((ty / innerHeight - .5) - py) * 0.05;
root.style.setProperty("--gx", gx + "px");
root.style.setProperty("--gy", gy + "px");
if (!reduce) tilt.style.transform =
`perspective(900px) rotateY(${px * 9}deg) rotateX(${-py * 7}deg) translate3d(${px * 16}px,${py * 12}px,0)`;
requestAnimationFrame(raf);
})();
</script>
</body>
</html>
+134
View File
@@ -0,0 +1,134 @@
using System.Runtime.CompilerServices;
using AIStudio.Tools;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Checks which files the export menu finds in an answer.
/// </summary>
/// <remarks>
/// Asked for a web page, a model answers with a code block marked as html, and the same goes for a
/// LaTeX document or a Markdown text. That block already is the file the user wants. Converted along
/// with the rest of the answer, Pandoc shows it as a listing of source code instead, which is what
/// PR #993 reported. The fixture is the page attached to that PR, as the model wrote it.
/// </remarks>
[TestFixture]
public sealed class PlainFileExportTests
{
private static readonly string PAGE = ReadFixture("standalone_page.html");
[Test]
public void AnAnswerMadeOfOneWebPageOffersThatPage()
{
var files = PlainFileExport.ExtractFiles(Lines("```html", PAGE, "```"), ',');
Assert.Multiple(() =>
{
Assert.That(files, Has.Count.EqualTo(1));
Assert.That(files[0].Format, Is.EqualTo(FileExportFormat.HTML));
Assert.That(files[0].Content, Is.EqualTo(PAGE), "The page leaves the answer exactly as the model wrote it, without the fence around it.");
Assert.That(files[0].Caption, Is.Empty, "Without a heading above it, a code block has nothing to be named after.");
});
}
[Test]
public void AWebPageAmidExplanationsIsOfferedAsWell()
{
var answer = Lines("Here is your page:", string.Empty, "```html", PAGE, "```", string.Empty, "Save it and open it in your browser.");
var files = PlainFileExport.ExtractFiles(answer, ',');
Assert.Multiple(() =>
{
Assert.That(files, Has.Count.EqualTo(1), "Models rarely answer with the block alone, so the text around it must not hide it.");
Assert.That(files[0].Content, Is.EqualTo(PAGE));
});
}
[TestCase("HTML", FileExportFormat.HTML)]
[TestCase("tex", FileExportFormat.LATEX)]
[TestCase("markdown", FileExportFormat.MARKDOWN)]
[TestCase("html title=\"index.html\"", FileExportFormat.HTML, Description = "Whatever follows the language is an argument, not part of it.")]
public void ACodeBlockIsOfferedInTheFormatItsLanguageNames(string infoString, FileExportFormat expectedFormat)
{
var files = PlainFileExport.ExtractFiles(Lines($"```{infoString}", "The content.", "```"), ',');
Assert.Multiple(() =>
{
Assert.That(files, Has.Count.EqualTo(1));
Assert.That(files[0].Format, Is.EqualTo(expectedFormat));
Assert.That(files[0].Content, Is.EqualTo("The content."));
});
}
[Test]
public void ATildeFenceIsOfferedAsWell()
{
var files = PlainFileExport.ExtractFiles(Lines("~~~latex", @"\section{Results}", "~~~"), ',');
Assert.That(files.Select(file => file.Format), Is.EqualTo(new[] { FileExportFormat.LATEX }));
}
[TestCase("```css", TestName = "A language AI Studio writes no file for")]
[TestCase("```", TestName = "A fence without a language")]
public void AnyOtherCodeBlockIsNotOffered(string openingFence)
{
var files = PlainFileExport.ExtractFiles(Lines(openingFence, "body { margin: 0; }", "```"), ',');
Assert.That(files, Is.Empty);
}
[TestCase("```html", "<html><body><p>The answer broke off here", TestName = "Half a web page")]
[TestCase("```csv", "Quarter,Revenue", TestName = "Half a table")]
public void ACodeBlockTheModelNeverClosedIsNotOffered(string openingFence, string content)
{
var files = PlainFileExport.ExtractFiles(Lines("The answer starts normally.", string.Empty, openingFence, content), ',');
Assert.That(files, Is.Empty, "The file would end wherever the answer broke off.");
}
[Test]
public void TablesAndCodeBlocksAreCountedApart()
{
var answer = Lines(
"# Revenue",
string.Empty,
"| Quarter | Revenue |",
"|---|---|",
"| Q1 | 100 |",
string.Empty,
"# Landing page",
string.Empty,
"```html",
"<p>First block</p>",
"```",
string.Empty,
"```latex",
@"\section{Second block}",
"```");
var files = PlainFileExport.ExtractFiles(answer, ',');
Assert.That(files.Select(file => (file.Ordinal, file.Caption, file.Format)), Is.EqualTo(new[]
{
(1, "Revenue", FileExportFormat.CSV),
(1, "Landing page", FileExportFormat.HTML),
(2, "Landing page", FileExportFormat.LATEX),
}), "The first code block is code block 1, even though a table stands before it.");
}
private static string Lines(params string[] lines) => string.Join(Environment.NewLine, lines);
/// <summary>
/// Reads a file from the fixtures next to this test.
/// </summary>
/// <remarks>
/// Read from the source tree, the way the capability snapshot is, so the fixture needs no entry in
/// the project file. A checkout on Windows may have turned its line ends into CRLF, which the
/// model never wrote.
/// </remarks>
private static string ReadFixture(string fileName, [CallerFilePath] string sourceFilePath = "") => File
.ReadAllText(Path.Combine(Path.GetDirectoryName(sourceFilePath)!, "Fixtures", fileName))
.Replace("\r\n", "\n");
}
@@ -0,0 +1,149 @@
using System.Web;
using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations;
namespace AIStudio.Tests.Tools.ToolCalling;
/// <summary>
/// Checks the parts of the Confluence search which decide where a request may go and what it asks.
/// </summary>
/// <remarks>
/// The model supplies nothing but words, and everything around them comes from here: the wiki
/// address a search may use, the CQL the words end up in, which redirects stay within the wiki,
/// and when an answer is the login page rather than a search. A mistake in any of them sends the
/// user's query or sign-in somewhere else, or lets a model change what is searched. The request
/// itself needs a real Confluence and is left to a manual test.
/// </remarks>
[TestFixture]
public sealed class ConfluenceSearchToolTests
{
private const string BASE_URL = "https://wiki.example.org/confluence/";
private static readonly Uri WIKI = new(BASE_URL);
[TestCase("https://wiki.example.org/confluence", "https://wiki.example.org/confluence/")]
[TestCase("https://wiki.example.org/confluence/", "https://wiki.example.org/confluence/")]
[TestCase(" https://wiki.example.org/confluence// ", "https://wiki.example.org/confluence/")]
[TestCase("https://wiki.example.org", "https://wiki.example.org/")]
public void ABaseUrlEndsWithExactlyOneSlash(string value, string expected)
{
Assert.That(ParseBaseUrl(value).AbsoluteUri, Is.EqualTo(expected), "Without the slash, resolving the search page against the base URL would replace its last segment, and with more than one, no page of the wiki would count as within it.");
}
[TestCase(null)]
[TestCase("")]
[TestCase(" ")]
[TestCase("wiki.example.org/confluence/")]
[TestCase("http://wiki.example.org/confluence/")]
[TestCase("https://user:secret@wiki.example.org/confluence/")]
[TestCase("https://wiki.example.org/confluence/?os_authType=basic")]
[TestCase("https://wiki.example.org/confluence/#search")]
public void ABaseUrlWhichIsNotAPlainHttpsAddressIsRefused(string? value)
{
Assert.That(ConfluenceSearchTool.TryParseBaseUrl(value, out var baseUrl), Is.False, "Plain HTTP would expose the sign-in, credentials in the address would travel with every request, and a query or fragment has no place in the root of a wiki.");
Assert.That(baseUrl, Is.Null);
}
[Test]
public void TheSearchGoesToTheSearchPageOfTheWiki()
{
var searchUrl = ConfluenceSearchTool.BuildSearchUrl(ParseBaseUrl("https://wiki.example.org/confluence"), "release plan", null);
var parameters = HttpUtility.ParseQueryString(searchUrl.Query);
Assert.That(searchUrl.GetLeftPart(UriPartial.Path), Is.EqualTo("https://wiki.example.org/confluence/dosearchsite.action"), "The search page lies below the context path of the wiki, even when the configured address lacks the final slash.");
Assert.That(parameters["cql"], Is.EqualTo("text ~ \"release plan\""));
Assert.That(parameters["queryString"], Is.EqualTo("release plan"), "Confluence shows these words in its search field, so the page reads like a search the user made.");
Assert.That(ConfluenceSearchTool.IsWithinWiki(WIKI, searchUrl), Is.True);
}
[TestCase(@"plan"" or space = ""HR", @"text ~ ""plan\"" or space = \""HR""")]
[TestCase(@"C:\temp\", @"text ~ ""C:\\temp\\""")]
[TestCase(@"plan\"" or space = \""HR", @"text ~ ""plan\\\"" or space = \\\""HR""")]
public void TheQueryCannotLeaveItsCqlString(string query, string expectedCql)
{
//
// The words land inside a quoted CQL string. Were a quote to end it, or a trailing backslash
// to turn the closing quote into a literal one, a model could append conditions of its own,
// such as one which widens the search to spaces the user never asked for:
//
Assert.That(Cql(ConfluenceSearchTool.BuildSearchUrl(WIKI, query, null)), Is.EqualTo(expectedCql), "Backslashes are escaped first and quotes second, so neither can end the string.");
}
[Test]
public void ASpaceKeyRestrictsTheSearchToThatSpace()
{
Assert.That(Cql(ConfluenceSearchTool.BuildSearchUrl(WIKI, "release plan", "DEV")), Is.EqualTo("text ~ \"release plan\" and space=\"DEV\""));
}
[Test]
public void ASpaceKeyCannotLeaveItsCqlStringEither()
{
Assert.That(Cql(ConfluenceSearchTool.BuildSearchUrl(WIKI, "release plan", @"DEV"" or space = ""HR")), Is.EqualTo(@"text ~ ""release plan"" and space=""DEV\"" or space = \""HR"""), "The space key comes from the model as well, so it gets the same escaping as the words.");
}
[TestCase(null)]
[TestCase("")]
[TestCase(" ")]
public void WithoutASpaceKeyTheWholeWikiIsSearched(string? spaceKey)
{
Assert.That(Cql(ConfluenceSearchTool.BuildSearchUrl(WIKI, "release plan", spaceKey)), Is.EqualTo("text ~ \"release plan\""), "An empty space key would otherwise ask for a space which does not exist and find nothing.");
}
[Test]
public void TheQueryStaysInsideItsParameters()
{
// Characters which mean something in a URL must neither end a parameter, start a fragment, nor change the path:
const string QUERY = "R&D #1 ../../admin?x=1";
var searchUrl = ConfluenceSearchTool.BuildSearchUrl(WIKI, QUERY, null);
var parameters = HttpUtility.ParseQueryString(searchUrl.Query);
Assert.That(searchUrl.AbsolutePath, Is.EqualTo("/confluence/dosearchsite.action"));
Assert.That(searchUrl.Fragment, Is.Empty);
Assert.That(parameters.AllKeys, Is.EquivalentTo(new[] { "cql", "queryString" }), "No word of the query may become a parameter of its own.");
Assert.That(parameters["queryString"], Is.EqualTo(QUERY));
}
[TestCase("https://wiki.example.org/confluence/dosearchsite.action?cql=x")]
[TestCase("https://wiki.example.org/confluence/display/DEV/Release+Plan")]
[TestCase("https://WIKI.example.org:443/confluence/pages/viewpage.action?pageId=1")]
[TestCase("https://wiki.example.org./confluence/display/DEV/")]
public void AnAddressBelowTheBaseUrlIsWithinTheWiki(string url)
{
Assert.That(ConfluenceSearchTool.IsWithinWiki(WIKI, new Uri(url)), Is.True, "The case of the host, the default port written out, and a trailing dot all name the same wiki.");
}
[TestCase("https://wiki.example.org/confluence-evil/dosearchsite.action")]
[TestCase("https://wiki.example.org/confluence/../admin/")]
[TestCase("https://wiki.example.org/")]
[TestCase("https://wiki.example.org:8443/confluence/")]
[TestCase("http://wiki.example.org/confluence/")]
[TestCase("https://wiki.example.org.evil.example/confluence/")]
[TestCase("https://id.atlassian.com/login")]
public void AnAddressOutsideTheBaseUrlIsNotWithinTheWiki(string url)
{
Assert.That(ConfluenceSearchTool.IsWithinWiki(WIKI, new Uri(url)), Is.False, "A neighbouring path, a path which climbs out, another port, plain HTTP, or another host is not the configured wiki, however similar it looks.");
}
[TestCase("https://wiki.example.org/confluence/login.action")]
[TestCase("https://wiki.example.org/confluence/login.action?os_destination=%2Fdosearchsite.action%3Fcql%3Dx")]
[TestCase("https://wiki.example.org/confluence/LOGIN.ACTION")]
[TestCase("https://wiki.example.org/confluence/signin?os_destination=%2Fdosearchsite.action")]
public void TheLoginPageIsRecognized(string url)
{
Assert.That(ConfluenceSearchTool.IsLoginPage(new Uri(url)), Is.True, "Confluence answers a search without a valid session with its login page, or with another page that carries the search as the destination to return to.");
}
[Test]
public void ASearchForTheWordsOfTheLoginPageIsNoLoginPage()
{
var searchUrl = ConfluenceSearchTool.BuildSearchUrl(WIKI, "login.action os_destination=", null);
Assert.That(ConfluenceSearchTool.IsLoginPage(searchUrl), Is.False, "Somebody looking up the login page in the wiki gets search results, not a message saying that the sign-in failed.");
}
private static Uri ParseBaseUrl(string value) => ConfluenceSearchTool.TryParseBaseUrl(value, out var baseUrl)
? baseUrl
: throw new AssertionException($"'{value}' should be a valid base URL.");
private static string? Cql(Uri searchUrl) => HttpUtility.ParseQueryString(searchUrl.Query)["cql"];
}
@@ -0,0 +1,55 @@
using AIStudio.Tools.ToolCallingSystem;
namespace AIStudio.Tests.Tools.ToolCalling;
/// <summary>
/// Checks how a selection of tools turns into the set which actually runs.
/// </summary>
/// <remarks>
/// Every tool selection in the app passes through this, and so do the audit of an assistant plugin
/// and its security card. Whatever it adds is therefore what the user and the audit get to see, so it
/// must neither add a tool nobody asked for nor keep adding each time it runs.
/// </remarks>
[TestFixture]
public sealed class ToolSelectionRulesTests
{
private const string SEARCH_CONFLUENCE = ToolSelectionRules.SEARCH_CONFLUENCE_TOOL_ID;
private const string READ_WEB_PAGE = ToolSelectionRules.READ_WEB_PAGE_TOOL_ID;
private const string WEB_SEARCH = ToolSelectionRules.WEB_SEARCH_TOOL_ID;
[Test]
public void SearchConfluenceBringsReadWebPageAlong()
{
Assert.That(ToolSelectionRules.NormalizeSelection([SEARCH_CONFLUENCE]), Is.EquivalentTo(new[] { SEARCH_CONFLUENCE, READ_WEB_PAGE }), "The search only finds pages; without Read Web Page the model could not open a single result.");
}
[TestCase(READ_WEB_PAGE)]
[TestCase(WEB_SEARCH)]
public void OtherToolsBringNothingAlong(string toolId)
{
Assert.That(ToolSelectionRules.NormalizeSelection([toolId]), Is.EquivalentTo(new[] { toolId }), "Only Search Confluence depends on another tool. Read Web Page in particular does not pull the search in.");
}
[Test]
public void NormalizingTwiceChangesNothing()
{
var once = ToolSelectionRules.NormalizeSelection([SEARCH_CONFLUENCE, WEB_SEARCH]);
Assert.That(ToolSelectionRules.NormalizeSelection(once), Is.EquivalentTo(once), "The selection fields normalize whatever they receive, including a selection they normalized themselves a moment ago.");
}
[Test]
public void ATwiceSelectedToolRunsOnce()
{
Assert.That(ToolSelectionRules.NormalizeSelection([WEB_SEARCH, WEB_SEARCH]), Has.Count.EqualTo(1));
}
[Test]
public void TheSelectionPassedInStaysUntouched()
{
HashSet<string> selected = [SEARCH_CONFLUENCE];
ToolSelectionRules.NormalizeSelection(selected);
Assert.That(selected, Is.EquivalentTo(new[] { SEARCH_CONFLUENCE }), "The caller's set, such as the tools of a stored chat template, must not change behind its back.");
}
}