using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools;
using AIStudio.Tools.Services;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.Settings;
///
/// Checks which transcription providers may hear the audio and video files of a data source.
///
///
/// The transcription provider receives the content of a data source just like the embedding
/// provider. A data source which asks for a high confidence level must not send its recordings to a
/// provider it would never let embed its documents.
///
[TestFixture]
public sealed class TranscriptionConfidenceTests
{
private static IEnumerable DataSources()
{
foreach (var level in Enum.GetValues())
{
yield return new DataSourceLocalDirectory { ConfidenceLevel = level };
yield return new DataSourceLocalFile { ConfidenceLevel = level };
yield return new DataSourceMailbox { ConfidenceLevel = level };
}
}
[TestCaseSource(nameof(DataSources))]
public void TranscriptionFollowsTheRuleOfTheEmbedding(IIndexedDataSource dataSource)
{
foreach (var providerLevel in Enum.GetValues())
Assert.That(DataSourceEmbeddingService.AllowsTranscription(dataSource, providerLevel), Is.EqualTo(DataSourceEmbeddingService.AllowsEmbedding(dataSource, providerLevel)), $"A provider with the level {providerLevel} is treated differently for {dataSource.GetType().Name} with the level {dataSource.ConfidenceLevel}.");
}
[Test]
public void AFolderWhichAsksForAHighLevelKeepsItsRecordingsFromAMediumProvider()
{
var directory = new DataSourceLocalDirectory { ConfidenceLevel = ConfidenceLevel.HIGH };
Assert.Multiple(() =>
{
Assert.That(DataSourceEmbeddingService.AllowsTranscription(directory, ConfidenceLevel.MEDIUM), Is.False);
Assert.That(DataSourceEmbeddingService.AllowsTranscription(directory, ConfidenceLevel.HIGH), Is.True);
});
}
[Test]
public void AFolderWithoutALevelStaysOpenToEveryTranscriptionProvider()
{
var directory = new DataSourceLocalDirectory { ConfidenceLevel = ConfidenceLevel.NONE };
Assert.That(DataSourceEmbeddingService.AllowsTranscription(directory, ConfidenceLevel.UNTRUSTED), Is.True);
}
[Test]
public void ATranscriptionProviderHasTheLevelOfItsCompany()
{
var settingsManager = new SettingsManager(NullLogger.Instance, null!);
settingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.CUSTOM;
settingsManager.ConfigurationData.Confidence.CustomConfidenceScheme[LLMProviders.OPEN_AI] = ConfidenceLevel.LOW;
var provider = new TranscriptionProvider { Id = Guid.NewGuid().ToString(), UsedLLMProvider = LLMProviders.OPEN_AI };
Assert.That(provider.GetConfidenceLevel(settingsManager), Is.EqualTo(ConfidenceLevel.LOW));
}
[Test]
public void NoTranscriptionProviderHasNoConfidence()
{
var settingsManager = new SettingsManager(NullLogger.Instance, null!);
Assert.That(TranscriptionProvider.NONE.GetConfidenceLevel(settingsManager), Is.EqualTo(ConfidenceLevel.NONE));
}
}