mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-09-27 00:53:37 +00:00
Guard the confidence threshold against an empty list
This commit is contained in:
parent
cd6a3f1daa
commit
48779ff144
@ -2,6 +2,8 @@ namespace AIStudio.Tools;
|
||||
|
||||
public static class IConfidenceExtensions
|
||||
{
|
||||
private static readonly ILogger<IConfidence> LOGGER = Program.LOGGER_FACTORY.CreateLogger<IConfidence>();
|
||||
|
||||
public static TargetWindow DetermineTargetWindow<T>(this IReadOnlyList<T> items, TargetWindowStrategy strategy, int numMaximumItems = 30) where T : IConfidence
|
||||
{
|
||||
switch (strategy)
|
||||
@ -53,8 +55,18 @@ public static class IConfidenceExtensions
|
||||
{
|
||||
if(!targetWindow.IsValid())
|
||||
{
|
||||
var logger = Program.SERVICE_PROVIDER.GetService<ILogger<IConfidence>>()!;
|
||||
logger.LogWarning("The target window is invalid. Returning 0f as threshold.");
|
||||
LOGGER.LogWarning("The target window is invalid. Returning 0f as threshold.");
|
||||
return 0f;
|
||||
}
|
||||
|
||||
//
|
||||
// Without items there is no threshold to find, and the Min and Max calls below would throw
|
||||
// on an empty sequence. Every caller checks this today, which is precisely how such a guard
|
||||
// goes missing once a new caller arrives. It belongs here, next to the calls it protects:
|
||||
//
|
||||
if(items.Count == 0)
|
||||
{
|
||||
LOGGER.LogWarning("There are no items to determine a confidence threshold for. Returning 0f as threshold.");
|
||||
return 0f;
|
||||
}
|
||||
|
||||
@ -91,10 +103,7 @@ public static class IConfidenceExtensions
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
var logger = Program.SERVICE_PROVIDER.GetService<ILogger<IConfidence>>()!;
|
||||
logger.LogWarning("The confidence values are too close. Returning 0f as threshold.");
|
||||
}
|
||||
LOGGER.LogWarning("The confidence values are too close. Returning 0f as threshold.");
|
||||
|
||||
return threshold;
|
||||
}
|
||||
|
||||
@ -50,7 +50,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
|
||||
}
|
||||
|
||||
// Log the selected data sources:
|
||||
var selectedDataSourceInfo = aiSelectedDataSources.Select(ds => $"[Id={ds.Id}, reason={ds.Reason}, confidence={ds.Confidence}]").Aggregate((a, b) => $"'{a}', '{b}'");
|
||||
var selectedDataSourceInfo = string.Join(", ", aiSelectedDataSources.Select(ds => $"'[Id={ds.Id}, reason={ds.Reason}, confidence={ds.Confidence}]'"));
|
||||
LOGGER.LogInformation($"The AI selected the data sources automatically. {aiSelectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}.");
|
||||
|
||||
//
|
||||
|
||||
58
app/Tests/Tools/ConfidenceThresholdTests.cs
Normal file
58
app/Tests/Tools/ConfidenceThresholdTests.cs
Normal file
@ -0,0 +1,58 @@
|
||||
using AIStudio.Tools;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Checks that determining a confidence threshold survives a list with nothing in it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// GetConfidenceThreshold reaches for Min and Max, both of which throw on an empty sequence. Until
|
||||
/// v26.9.1 the only thing standing between them and that exception was a count check at each of the
|
||||
/// two call sites. The RAG process already lost such a guard once -- a log line was written before
|
||||
/// the check that was meant to protect it, which is how every new chat produced an
|
||||
/// InvalidOperationException nobody noticed, because the RAG process catches and logs everything.
|
||||
/// The threshold is now asked to defend itself, and these tests hold it to that.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ConfidenceThresholdTests
|
||||
{
|
||||
private readonly record struct Decision(float Confidence) : IConfidence;
|
||||
|
||||
[Test]
|
||||
public void AnEmptyListYieldsAThresholdInsteadOfThrowing()
|
||||
{
|
||||
IReadOnlyList<Decision> nothingWasDecided = [];
|
||||
var targetWindow = new TargetWindow(1, 2, 3, 0f);
|
||||
|
||||
Assert.That(nothingWasDecided.GetConfidenceThreshold(targetWindow), Is.EqualTo(0f), "A threshold of zero keeps everything that follows, which is the harmless answer for a list that holds nothing to filter.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnEmptyListIsAnsweredEvenWhenTheWindowDemandsItems()
|
||||
{
|
||||
//
|
||||
// The window asks for between five and ten items while not a single one exists. The guard
|
||||
// has to hold regardless of what the window wants:
|
||||
//
|
||||
IReadOnlyList<Decision> nothingWasDecided = [];
|
||||
var demandingWindow = new TargetWindow(4, 5, 10, 0.5f);
|
||||
|
||||
Assert.That(nothingWasDecided.GetConfidenceThreshold(demandingWindow), Is.EqualTo(0f), "The number of items the window asks for cannot conjure items to measure.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ASpreadOfDecisionsIsNarrowedToTheTargetWindow()
|
||||
{
|
||||
//
|
||||
// Guarding the empty case must not change what the threshold does with actual items. Two
|
||||
// weak decisions and three strong ones, with a window asking for two to three:
|
||||
//
|
||||
IReadOnlyList<Decision> decisions = [new(0.1f), new(0.2f), new(0.9f), new(0.95f), new(1.0f)];
|
||||
var targetWindow = new TargetWindow(1, 2, 3, 0f);
|
||||
|
||||
var threshold = decisions.GetConfidenceThreshold(targetWindow);
|
||||
var survivors = decisions.Count(decision => decision.Confidence >= threshold);
|
||||
|
||||
Assert.That(survivors, Is.InRange(targetWindow.TargetWindowMin, targetWindow.TargetWindowMax), "The threshold exists to cut a list down to the size the window asks for, and the three strong decisions are what should be left.");
|
||||
}
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user