mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-14 22:08:27 +00:00
Added the Model Comparison assistant for blind comparisons of two models (#1009)
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
1 parent
12f26445ba
commit
b7255694cd
221 files changed
+19989
-90
No files matched your search
@@ -0,0 +1,61 @@
|
||||
using AIStudio.Assistants.ModelComparison;
|
||||
|
||||
namespace AIStudio.Tests.Assistants.ModelComparison;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how the choices of several runs are counted.
|
||||
/// </summary>
|
||||
[TestFixture]
|
||||
public sealed class ModelComparisonTallyTests
|
||||
{
|
||||
[Test]
|
||||
public void NoRunsCountNothing()
|
||||
{
|
||||
var tally = ModelComparisonTally.Of([]);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(tally.TotalRunCount, Is.Zero);
|
||||
Assert.That(tally.ScoredRunCount, Is.Zero);
|
||||
Assert.That(tally.UnscoredRunCount, Is.Zero);
|
||||
Assert.That(tally.FirstModelWins, Is.Zero);
|
||||
Assert.That(tally.SecondModelWins, Is.Zero);
|
||||
Assert.That(tally.Ties, Is.Zero);
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void EveryChoiceCountsForItsModel()
|
||||
{
|
||||
var tally = ModelComparisonTally.Of(
|
||||
[
|
||||
ModelComparisonModelChoice.FIRST,
|
||||
ModelComparisonModelChoice.SECOND,
|
||||
ModelComparisonModelChoice.FIRST,
|
||||
ModelComparisonModelChoice.TIE,
|
||||
]);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(tally.TotalRunCount, Is.EqualTo(4));
|
||||
Assert.That(tally.ScoredRunCount, Is.EqualTo(4));
|
||||
Assert.That(tally.FirstModelWins, Is.EqualTo(2));
|
||||
Assert.That(tally.SecondModelWins, Is.EqualTo(1));
|
||||
Assert.That(tally.Ties, Is.EqualTo(1));
|
||||
}
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARunWithoutAChoiceCountsOnlyTowardTheTotal()
|
||||
{
|
||||
var tally = ModelComparisonTally.Of([ModelComparisonModelChoice.SECOND, null, null]);
|
||||
|
||||
using (Assert.EnterMultipleScope())
|
||||
{
|
||||
Assert.That(tally.TotalRunCount, Is.EqualTo(3), "Every run happened.");
|
||||
Assert.That(tally.ScoredRunCount, Is.EqualTo(1), "Only one run had a choice.");
|
||||
Assert.That(tally.UnscoredRunCount, Is.EqualTo(2));
|
||||
Assert.That(tally.SecondModelWins, Is.EqualTo(1));
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user