using AIStudio.Settings; using AIStudio.Tools.RAG; using AIStudio.Tools.Security; using Microsoft.Extensions.Logging.Abstractions; namespace AIStudio.Tests.Tools; /// /// Checks how a passage too long for any page is shortened. /// /// /// A passage which does not fit even on its own would hold up every passage after it, so it is /// shortened instead. The cut should keep a line whole where that costs little, and it must never /// break a character in two. /// [TestFixture] public sealed class RetrievalContextShorteningTests { [Test] public void ATextWhichFitsStaysAsItIs() { const string TEXT = "first line\nsecond line"; Assert.That(IRetrievalContextExtensions.CutAtLineEnd(TEXT, TEXT.Length), Is.SameAs(TEXT)); } [Test] public void ACutEndsAtTheLastLineEndWhichFits() { Assert.That(IRetrievalContextExtensions.CutAtLineEnd("first\nsecond\nthird", 15), Is.EqualTo("first\nsecond")); } [Test] public void ACutKeepsNoHalfOfAWindowsLineEnd() { Assert.That(IRetrievalContextExtensions.CutAtLineEnd("first\r\nsecond\r\nthird", 17), Is.EqualTo("first\r\nsecond")); } [Test] public void ACutGivesUpAtMostHalfOfWhatFitsForALineEnd() { Assert.That(IRetrievalContextExtensions.CutAtLineEnd("a\nbcdefghijkl", 10), Is.EqualTo("a\nbcdefghi"), "With its only line end right at the start, the text would lose nearly everything."); } [Test] public void ACutNeverSplitsASurrogatePair() { Assert.That(IRetrievalContextExtensions.CutAtLineEnd("ab😀cd", 3), Is.EqualTo("ab")); } [Test] public void NothingIsLeftOfATextWithoutRoom() { Assert.That(IRetrievalContextExtensions.CutAtLineEnd("text", 0), Is.Empty); } [Test] public async Task AContextWhichFitsIsNeitherShortenedNorFilteredAgain() { var context = new RetrievalTextContext { DataSourceName = "Handbook", Category = RetrievalContentCategory.TEXT, Type = RetrievalContentType.TEXT_DOCUMENT, Path = "handbook.md", MatchedText = "The mixing console is described here." }; var prepared = new PreparedRetrievalContext(context, "The mixing console is described here.", [], null); // Without a runtime, filtering again would fail: var guardService = new PromptInjectionGuardService(null!, new SettingsManager(NullLogger.Instance, null!), NullLogger.Instance, NullLoggerFactory.Instance); var shortened = await prepared.ShortenToFitAsync(prepared.Body.Length, preparedContext => preparedContext.Body.Length, guardService); Assert.That(shortened, Is.SameAs(prepared)); } }