Fixed Semantic Search with external data sources (ERI servers) (#1090)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/Tests/Settings/DataSourceERICancellationTests.cs

@@ -0,0 +1,87 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.ERIClient.DataModel;
+using AIStudio.Tools.RAG;
+using AIStudio.Tools.Services;
+
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging.Abstractions;
+
+using ChatThread = AIStudio.Chat.ChatThread;
+
+namespace AIStudio.Tests.Settings;
+
+/// <summary>
+/// Checks what a search of an ERI data source does when it is cancelled, and when its server fails.
+/// </summary>
+/// <remarks>
+/// The ERI client answers a cancellation like any other failure, because its other callers pass a
+/// timeout of their own and expect an answer. A search the user cancelled has to stop, though,
+/// rather than report the data source as not searched and carry on. A server which fails without
+/// anybody cancelling still makes the data source count as not searched.
+/// </remarks>
+[TestFixture]
+public sealed class DataSourceERICancellationTests
+{
+    // Nothing listens on port 1 of the own machine, so a request fails right away:
+    private static readonly DataSourceERI_V1 UNREACHABLE = new()
+    {
+        Id = "33333333-3333-3333-3333-333333333333",
+        Name = "Intranet",
+        Type = DataSourceType.ERI_V1,
+        Hostname = "http://127.0.0.1",
+        Port = 1,
+        AuthMethod = AuthMethod.NONE,
+        SecurityPolicy = DataSourceSecurity.ALLOW_ANY,
+        MaxMatches = 10,
+    };
+
+    private RustService rustService = null!;
+    private ServiceProvider serviceProvider = null!;
+    private IServiceProvider previousServiceProvider = null!;
+
+    [SetUp]
+    public void PrepareServices()
+    {
+        this.rustService = new RustService("1", "unused");
+        var settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, this.rustService);
+
+        this.previousServiceProvider = Program.SERVICE_PROVIDER;
+        this.serviceProvider = new ServiceCollection()
+            .AddSingleton(this.rustService)
+            .AddSingleton(settingsManager)
+            .AddLogging()
+            .BuildServiceProvider();
+
+        Program.SERVICE_PROVIDER = this.serviceProvider;
+    }
+
+    [TearDown]
+    public void RestoreApplicationState()
+    {
+        Program.SERVICE_PROVIDER = this.previousServiceProvider;
+        this.serviceProvider.Dispose();
+        this.rustService.Dispose();
+    }
+
+    [Test]
+    public void ACancelledSearchStops()
+    {
+        using var cancellation = new CancellationTokenSource();
+        cancellation.Cancel();
+
+        Assert.CatchAsync<OperationCanceledException>(async () => await UNREACHABLE.RetrieveDataAsync("query", 1, new ChatThread(), cancellation.Token), "Reporting the data source as not searched would carry on after the user cancelled.");
+    }
+
+    [Test]
+    public async Task AFailingServerMakesTheDataSourceCountAsNotSearched()
+    {
+        var page = await UNREACHABLE.RetrieveDataAsync("query", 1, new ChatThread());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(page.Contexts, Is.Empty);
+            Assert.That(page.Gaps, Is.EqualTo(new[] { RetrievalGap.NOT_SEARCHED }), "Nobody cancelled, so the model learns that this data source is missing rather than empty.");
+        });
+    }
+}

app/Tests/Tools/ClassicRagUnreachableTests.cs

@@ -0,0 +1,81 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.RAG.RAGProcesses;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks how the classic RAG process tells the model about ERI data sources whose server could not be reached.
+/// </summary>
+/// <remarks>
+/// Such a data source is never searched. Without a word about it, the model would take its silence
+/// for finding nothing and answer from its own knowledge. Semantic Search names it in the
+/// description of its tool, the classic process next to the passages it found, and both name the
+/// same data sources of the chat.
+/// </remarks>
+[TestFixture]
+public sealed class ClassicRagUnreachableTests
+{
+    private static readonly IDataSource INTRANET = new DataSourceERI_V1 { Num = 3, Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10 };
+    private static readonly IDataSource WIKI = new DataSourceERI_V1 { Num = 4, Id = "44444444-4444-4444-4444-444444444444", Name = "Wiki", MaxMatches = 10 };
+
+    [Test]
+    public void TheNoteFollowsThePassages()
+    {
+        var augmentedData = AISrcSelWithRetCtxVal.WithUnreachableDataSources("# Retrieval context 1 of 1\n", [INTRANET]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(augmentedData, Does.StartWith("# Retrieval context 1 of 1"), "What was found stays first.");
+            Assert.That(augmentedData, Does.Contain("could not be reached or signed in to right now"));
+            Assert.That(augmentedData, Does.Contain("- name='Intranet'"), "The model has to know what is missing, so that it can tell the user.");
+        });
+    }
+
+    [Test]
+    public void WithoutPassagesTheNoteStandsAlone()
+    {
+        var augmentedData = AISrcSelWithRetCtxVal.WithUnreachableDataSources(string.Empty, [INTRANET]);
+
+        Assert.That(augmentedData, Does.StartWith("These external data sources of this chat could not be reached"), "Every selected data source may be out of reach, and then nothing was retrieved at all.");
+    }
+
+    [Test]
+    public void WhenEveryServerAnsweredNothingIsAdded()
+    {
+        const string PASSAGES = "# Retrieval context 1 of 1\n";
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(AISrcSelWithRetCtxVal.WithUnreachableDataSources(PASSAGES, []), Is.EqualTo(PASSAGES));
+            Assert.That(AISrcSelWithRetCtxVal.WithUnreachableDataSources(string.Empty, []), Is.Empty, "The system prompt stays as it was without data sources.");
+        });
+    }
+
+    [Test]
+    public void TheAutomaticSelectionNamesEveryUnreachableDataSource()
+    {
+        var unreachable = Checked([INTRANET, WIKI]).GetUnreachableOfTheChat(new DataSourceOptions { AutomaticDataSourceSelection = true });
+
+        Assert.That(unreachable, Is.EqualTo(new[] { INTRANET, WIKI }), "The selection agent might have picked any of them.");
+    }
+
+    [Test]
+    public void TheUserSelectionNamesOnlyTheSelectedUnreachableDataSources()
+    {
+        var options = new DataSourceOptions { AutomaticDataSourceSelection = false, PreselectedDataSourceIds = [INTRANET.Id] };
+
+        Assert.That(Checked([INTRANET, WIKI]).GetUnreachableOfTheChat(options), Is.EqualTo(new[] { INTRANET }), "The model must not learn about a data source the user did not select for this chat.");
+    }
+
+    [Test]
+    public void AnUnreachableDataSourceWithoutMatchesIsNotNamed()
+    {
+        var silent = new DataSourceERI_V1 { Num = 5, Id = "55555555-5555-5555-5555-555555555555", Name = "Archive", MaxMatches = 0 };
+
+        Assert.That(Checked([silent]).GetUnreachableOfTheChat(new DataSourceOptions { AutomaticDataSourceSelection = true }), Is.Empty, "It would add nothing even when its server answers.");
+    }
+
+    private static AllowedSelectedDataSources Checked(IReadOnlyList<IDataSource> unreachable) => new([], [], [], [], unreachable);
+}

app/Tests/Tools/DataSourceUnreachableTests.cs

@@ -0,0 +1,50 @@
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which ERI data sources whose server could not be reached may still be named to a provider.
+/// </summary>
+/// <remarks>
+/// Such a data source is never searched, but Semantic Search tells the model about it, so that the
+/// model does not take its silence for finding nothing. Its server's requirements are unknown then,
+/// so only the security policy configured in AI Studio decides, and a provider which that policy
+/// keeps away must not even learn the name.
+/// </remarks>
+[TestFixture]
+public sealed class DataSourceUnreachableTests
+{
+    [TestCase(true)]
+    [TestCase(false)]
+    public void ADataSourceForAnyProviderMayBeNamed(bool usingTrustedProvider)
+    {
+        Assert.That(DataSourceService.MayBeNamedWhileUnreachable(ERISource(DataSourceSecurity.ALLOW_ANY), usingTrustedProvider), Is.True);
+    }
+
+    [Test]
+    public void ADataSourceForSelfHostedProvidersIsNamedOnlyToATrustedOne()
+    {
+        var source = ERISource(DataSourceSecurity.SELF_HOSTED);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(DataSourceService.MayBeNamedWhileUnreachable(source, usingTrustedProvider: true), Is.True);
+            Assert.That(DataSourceService.MayBeNamedWhileUnreachable(source, usingTrustedProvider: false), Is.False, "The data source would be refused to this provider if its server answered.");
+        });
+    }
+
+    [Test]
+    public void ADataSourceWithoutASecurityPolicyIsNeverNamed()
+    {
+        Assert.That(DataSourceService.MayBeNamedWhileUnreachable(ERISource(DataSourceSecurity.NOT_SPECIFIED), usingTrustedProvider: true), Is.False, "Better safe than sorry, as with a server which answers.");
+    }
+
+    private static DataSourceERI_V1 ERISource(DataSourceSecurity securityPolicy) => new()
+    {
+        Id = "33333333-3333-3333-3333-333333333333",
+        Name = "Intranet",
+        Type = DataSourceType.ERI_V1,
+        SecurityPolicy = securityPolicy,
+    };
+}

app/Tests/Tools/ERISecurityCheckCacheTests.cs

@@ -0,0 +1,157 @@
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks how long the answer of an ERI server to the check of its security requirements is used.
+/// </summary>
+/// <remarks>
+/// One message checks the ERI data sources several times, and every check signs in. A server
+/// which limits sign-ins would start refusing, so an answer is kept for a minute. A server which
+/// did not answer is asked again sooner, and a check which is still running is joined.
+/// </remarks>
+[TestFixture]
+public sealed class ERISecurityCheckCacheTests
+{
+    private static readonly DataSourceERI_V1 INTRANET = new()
+    {
+        Id = "33333333-3333-3333-3333-333333333333",
+        Name = "Intranet",
+        Type = DataSourceType.ERI_V1,
+        Hostname = "https://intranet.example.com",
+        Port = 443,
+    };
+
+    private static readonly DataSourceERI_V1 WIKI = INTRANET with
+    {
+        Id = "44444444-4444-4444-4444-444444444444",
+        Name = "Wiki",
+    };
+
+    [Test]
+    public void ARunningCheckIsJoined()
+    {
+        var running = new TaskCompletionSource<ERISecurityCheckResult>();
+
+        Assert.That(ERISecurityCheckCache.IsValid(running.Task, TimeSpan.FromMinutes(5)), Is.True, "Starting a second check would sign in a second time.");
+    }
+
+    [Test]
+    public void AnAnswerIsUsedForOneMinute()
+    {
+        var answered = Answer(true);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(ERISecurityCheckCache.IsValid(answered, TimeSpan.FromSeconds(59)), Is.True, "The rounds of one answer share a single sign-in.");
+            Assert.That(ERISecurityCheckCache.IsValid(answered, TimeSpan.FromSeconds(61)), Is.False, "The operator may change the requirements at any time.");
+        });
+    }
+
+    [Test]
+    public void AServerWithoutAnswerIsAskedAgainSooner()
+    {
+        var unanswered = Answer(false);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(ERISecurityCheckCache.IsValid(unanswered, TimeSpan.FromSeconds(14)), Is.True, "The rounds of one answer do not wait six seconds each for the same server.");
+            Assert.That(ERISecurityCheckCache.IsValid(unanswered, TimeSpan.FromSeconds(16)), Is.False, "The server may well be back.");
+        });
+    }
+
+    [Test]
+    public void AFailedCheckIsNeverUsed()
+    {
+        var failed = Task.FromException<ERISecurityCheckResult>(new InvalidOperationException());
+
+        Assert.That(ERISecurityCheckCache.IsValid(failed, TimeSpan.Zero), Is.False);
+    }
+
+    [Test]
+    public void ACheckStillRunningIsSharedByEveryoneWhoAsks()
+    {
+        var running = new TaskCompletionSource<ERISecurityCheckResult>();
+        var fetches = 0;
+        var cache = new ERISecurityCheckCache(_ => { fetches++; return running.Task; }, new ManualTime());
+
+        var first = cache.GetAsync(INTRANET);
+        var second = cache.GetAsync(INTRANET);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(fetches, Is.EqualTo(1), "Semantic Search checks when it is offered and again before every search; one sign-in has to do for both.");
+            Assert.That(second, Is.SameAs(first));
+        });
+    }
+
+    [Test]
+    public void AnExpiredAnswerAsksTheServerAgain()
+    {
+        var time = new ManualTime();
+        var fetches = 0;
+        var cache = new ERISecurityCheckCache(_ => { fetches++; return Answer(true); }, time);
+
+        cache.GetAsync(INTRANET);
+        time.Advance(TimeSpan.FromSeconds(30));
+        cache.GetAsync(INTRANET);
+        Assert.That(fetches, Is.EqualTo(1), "Within a minute, the answer still holds.");
+
+        time.Advance(TimeSpan.FromSeconds(31));
+        cache.GetAsync(INTRANET);
+        Assert.That(fetches, Is.EqualTo(2), "The operator may have changed the requirements meanwhile.");
+    }
+
+    [Test]
+    public void AChangedConfigurationAsksTheServerRightAway()
+    {
+        var fetches = 0;
+        var cache = new ERISecurityCheckCache(_ => { fetches++; return Answer(true); }, new ManualTime());
+
+        cache.GetAsync(INTRANET);
+        cache.GetAsync(INTRANET with { Hostname = "https://elsewhere.example.com" });
+
+        Assert.That(fetches, Is.EqualTo(2), "Another server may require something else, although the data source kept its ID.");
+    }
+
+    [Test]
+    public void EachDataSourceIsCheckedOnItsOwn()
+    {
+        var asked = new List<string>();
+        var cache = new ERISecurityCheckCache(source => { asked.Add(source.Id); return Answer(true); }, new ManualTime());
+
+        cache.GetAsync(INTRANET);
+        cache.GetAsync(WIKI);
+        cache.GetAsync(INTRANET);
+
+        Assert.That(asked, Is.EqualTo(new[] { INTRANET.Id, WIKI.Id }));
+    }
+
+    [Test]
+    public void ExpiredChecksAreDroppedWhenAnotherOneStarts()
+    {
+        var time = new ManualTime();
+        var cache = new ERISecurityCheckCache(_ => Answer(true), time);
+
+        cache.GetAsync(INTRANET);
+        time.Advance(TimeSpan.FromMinutes(2));
+        cache.GetAsync(WIKI);
+
+        Assert.That(cache.Count, Is.EqualTo(1), "A deleted data source must not stay in the cache for as long as the app runs.");
+    }
+
+    private static Task<ERISecurityCheckResult> Answer(bool isReachable) => Task.FromResult(isReachable ? new ERISecurityCheckResult(true, null) : ERISecurityCheckResult.UNREACHABLE);
+
+    /// <summary>
+    /// A clock which only moves when a test moves it.
+    /// </summary>
+    private sealed class ManualTime : TimeProvider
+    {
+        private DateTimeOffset now = new(2026, 10, 19, 12, 0, 0, TimeSpan.Zero);
+
+        public void Advance(TimeSpan duration) => this.now += duration;
+
+        public override DateTimeOffset GetUtcNow() => this.now;
+    }
+}

app/Tests/Tools/ERIServerRulesTests.cs

@@ -0,0 +1,49 @@
+using AIStudio.Tools.ERIClient.DataModel;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which providers the server of an ERI data source allows to see its data.
+/// </summary>
+/// <remarks>
+/// The operator of the server owns the data, so the rule of the server applies on top of the
+/// security policy configured in AI Studio, and to a trusted provider as well. Until now, a trusted
+/// provider skipped the rule of the server altogether, so a server which allows no provider at all
+/// stayed open to it.
+/// </remarks>
+[TestFixture]
+public sealed class ERIServerRulesTests
+{
+    [TestCase(true)]
+    [TestCase(false)]
+    public void AServerForAnyProviderAllowsEveryProvider(bool usingTrustedProvider)
+    {
+        Assert.That(DataSourceService.ServerAllows(new SecurityRequirements(ProviderType.ANY), usingTrustedProvider), Is.True);
+    }
+
+    [Test]
+    public void AServerForSelfHostedProvidersAllowsOnlyATrustedOne()
+    {
+        var requirements = new SecurityRequirements(ProviderType.SELF_HOSTED);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(DataSourceService.ServerAllows(requirements, usingTrustedProvider: true), Is.True);
+            Assert.That(DataSourceService.ServerAllows(requirements, usingTrustedProvider: false), Is.False);
+        });
+    }
+
+    [TestCase(true)]
+    [TestCase(false)]
+    public void AServerWithoutAllowedProvidersAllowsNone(bool usingTrustedProvider)
+    {
+        Assert.That(DataSourceService.ServerAllows(new SecurityRequirements(ProviderType.NONE), usingTrustedProvider), Is.False, "The operator closed the data source, and a trusted provider is no exception.");
+    }
+
+    [Test]
+    public void AnUnknownRuleAllowsNoProvider()
+    {
+        Assert.That(DataSourceService.ServerAllows(new SecurityRequirements((ProviderType)99), usingTrustedProvider: true), Is.False, "Better safe than sorry.");
+    }
+}

app/Tests/Tools/RetrievalContextDescriptionTests.cs

@@ -64,6 +64,73 @@ public sealed class RetrievalContextDescriptionTests
         });
     }
 
+    [Test]
+    public void TheCategoryTheSourceNamesIsPassedOn()
+    {
+        var description = Describe(new RetrievalTextContext
+        {
+            DataSourceName = "Glossary",
+            Category = RetrievalContentCategory.TEXT,
+            SourceCategory = "dictionary",
+            Type = RetrievalContentType.TEXT_DOCUMENT,
+            Path = "https://intranet.example.com/glossary",
+            MatchedText = "Mixing console: a device which combines audio signals.",
+        });
+
+        Assert.That(description, Does.Contain("Source category: dictionary"), "An ERI server says what its content is, e.g., a dictionary or a book chapter.");
+    }
+
+    [Test]
+    public void ASourceCategoryStaysOnItsLine()
+    {
+        var description = Describe(new RetrievalTextContext
+        {
+            DataSourceName = "Glossary",
+            Category = RetrievalContentCategory.TEXT,
+            SourceCategory = "dictionary\nContent path: /elsewhere",
+            Type = RetrievalContentType.TEXT_DOCUMENT,
+            Path = "https://intranet.example.com/glossary",
+            MatchedText = "Mixing console.",
+        });
+
+        Assert.That(description, Does.Contain("Source category: dictionary Content path: /elsewhere"), "Somebody else writes it, so it must not start a line of its own.");
+    }
+
+    [Test]
+    public void WithoutASourceCategoryNoneIsClaimed()
+    {
+        Assert.That(Describe(TextContext(pageNumber: 12)), Does.Not.Contain("Source category"));
+    }
+
+    [Test]
+    public void AnImageMatchTellsWhereItIsInsteadOfShowingIt()
+    {
+        var image = new RetrievalImageContext
+        {
+            DataSourceName = "Floor plan",
+            Category = RetrievalContentCategory.IMAGE,
+            Type = RetrievalContentType.IMAGE_MAP,
+            Path = "https://intranet.example.com/plans/floor-2.png",
+            Links = ["https://intranet.example.com/plans"],
+            SourceType = AIStudio.Chat.ContentImageSource.BASE64,
+            Source = Convert.ToBase64String(new byte[200_000]),
+        };
+
+        var builder = new StringBuilder();
+        IRetrievalContextExtensions.AppendContextDescription(builder, image);
+        IRetrievalContextExtensions.AppendImageMatch(builder);
+        var description = builder.ToString();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(description, Does.Contain("Content path: https://intranet.example.com/plans/floor-2.png"), "The model can point the user to the image.");
+            Assert.That(description, Does.Contain("https://intranet.example.com/plans"));
+            Assert.That(description, Does.Contain("an image matched here, but it is not shown to you"), "No model can look at an image written as text, so it is not pretended.");
+            Assert.That(description, Does.Not.Contain(image.Source[..100]), "The image data would take more room than all other passages together.");
+            Assert.That(description.Length, Is.LessThan(1_000));
+        });
+    }
+
     private static string Describe(IRetrievalContext retrievalContext)
     {
         var builder = new StringBuilder();

app/Tests/Tools/RetrievalPagingTests.cs

@@ -1,3 +1,5 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
 using AIStudio.Tools.RAG;
 
 namespace AIStudio.Tests.Tools;
@@ -138,10 +140,36 @@ public sealed class RetrievalPagingTests
             Assert.That(secondPage.Matches, Is.EqualTo(new[] { "c", "d" }));
             Assert.That(secondPage.HasMore, Is.True);
             Assert.That(thirdPage.Matches, Is.EqualTo(new[] { "e" }));
-            Assert.That(thirdPage.HasMore, Is.False, "An ERI server which found fewer than asked for ends the paging.");
+            Assert.That(thirdPage.HasMore, Is.False, "A channel which found fewer than asked for ends the paging.");
         });
     }
 
+    [TestCase((ushort)1)]
+    [TestCase((ushort)10)]
+    [TestCase((ushort)500)]
+    public void AnERIDataSourceHasASinglePage(ushort maxMatches)
+    {
+        IDataSource source = new DataSourceERI_V1 { Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = maxMatches };
+
+        Assert.That(source.GetLastPage(), Is.EqualTo(1), "ERI v1 knows no offset, and a server which allows only the configured number of matches would refuse every further page.");
+    }
+
+    [Test]
+    public void ALocalDataSourcePagesAsItsPageSizeAllows()
+    {
+        IDataSource source = new DataSourceLocalDirectory { Id = "11111111-1111-1111-1111-111111111111", Name = "Handbook", MaxMatches = 10 };
+
+        Assert.That(source.GetLastPage(), Is.EqualTo(RetrievalPaging.GetLastPage(10)));
+    }
+
+    [Test]
+    public void AnERIDataSourceRefusesAFurtherPage()
+    {
+        var source = new DataSourceERI_V1 { Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10 };
+
+        Assert.ThrowsAsync<ArgumentOutOfRangeException>(async () => await source.RetrieveDataAsync("query", 2, new AIStudio.Chat.ChatThread()));
+    }
+
     private static (IReadOnlyList<string> Matches, bool HasMore) Merge(string[] first, string[] second, int page)
     {
         // A channel never returns more than it is asked for:

app/Tests/Tools/ToolCalling/ProviderDataSourceTrustTests.cs

@@ -0,0 +1,63 @@
+using AIStudio.Provider;
+using AIStudio.Settings;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks that a provider is trusted with a data source the same way in both forms a request knows it by.
+/// </summary>
+/// <remarks>
+/// The classic RAG process asks the provider instance whether it may use a data source which asks
+/// for a self-hosted or organization-trusted provider. Semantic Search decides which data sources
+/// it offers with the settings the instance creates for the request. When the two answer
+/// differently, a self-hosted provider searches such a data source in one process and does not even
+/// learn about it in the other.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class ProviderDataSourceTrustTests : ToolRegistryTestBase
+{
+    [Test]
+    public void ASelfHostedProviderIsTrustedInBothForms()
+    {
+        var (instance, settings) = BothForms(ToolCapableProvider());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(instance.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.True);
+            Assert.That(settings.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.True, "Semantic Search would otherwise not offer a data source for self-hosted providers.");
+        });
+    }
+
+    [Test]
+    public void AProviderTheOrganizationTrustsIsTrustedInBothForms()
+    {
+        var cloud = LessTrustedProvider();
+        this.SettingsManager.ConfigurationData.DataSourceSecurity.TrustedProviderIds.Add(cloud.Id);
+        var (instance, settings) = BothForms(cloud);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(instance.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.True);
+            Assert.That(settings.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.True, "The organization trusts the provider by its ID, so the settings have to keep it.");
+        });
+    }
+
+    [Test]
+    public void AnUntrustedProviderStaysUntrustedInBothForms()
+    {
+        var (instance, settings) = BothForms(LessTrustedProvider());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(instance.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.False);
+            Assert.That(settings.IsTrustedForDataSourceSecurityChecks(this.SettingsManager), Is.False);
+        });
+    }
+
+    private static (IProvider Instance, AIStudio.Settings.Provider Settings) BothForms(AIStudio.Settings.Provider configured)
+    {
+        var instance = configured.CreateProvider();
+        return (instance, instance.CreateSettingsProvider(configured.Model));
+    }
+}

app/Tests/Tools/ToolCalling/ProviderSettingsModelProfileTests.cs

@@ -0,0 +1,50 @@
+using AIStudio.Models;
+using AIStudio.Models.Live;
+using AIStudio.Provider;
+using AIStudio.Settings;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks that the settings a provider creates for a request know its model as the configured provider does.
+/// </summary>
+/// <remarks>
+/// The settings of a request carry the ID of the configured provider, see
+/// IProvider.CreateSettingsProvider. Through it, what the provider's own model list reported, such as
+/// the context window of a self-hosted model, reaches everything which asks the settings about the
+/// model: the tools offered in the request, and how images are passed on. Before the ID came along,
+/// the request knew the model only by the rules, while the provider dialog showed what the machine
+/// reported.<br/><br/>
+/// What a provider reported lands in the store the app shares, so these tests must not run next to
+/// anything else touching it.
+/// </remarks>
+[TestFixture]
+[NonParallelizable]
+public sealed class ProviderSettingsModelProfileTests : ToolRegistryTestBase
+{
+    private const string MACHINE = "55555555-5555-5555-5555-555555555555";
+
+    /// <summary>
+    /// A window no rule would ever state, so that finding it proves where the answer came from.
+    /// </summary>
+    private const int WHAT_THE_MACHINE_REPORTS = 55_555;
+
+    private static readonly Model MODEL = new("qwen3-32b", null);
+
+    [TearDown]
+    public void ForgetWhatTheMachineSaid() => ListedModels.Shared.Report(MACHINE, []);
+
+    [Test]
+    public void TheSettingsOfARequestKnowWhatTheMachineReported()
+    {
+        ListedModels.Shared.Report(MACHINE, [new(MODEL.Id, ContextWindow.Of(WHAT_THE_MACHINE_REPORTS))]);
+        var configured = new AIStudio.Settings.Provider(1, MACHINE, "A machine of my own", LLMProviders.SELF_HOSTED, MODEL, IsSelfHosted: true);
+        var settings = configured.CreateProvider().CreateSettingsProvider(MODEL);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(settings.GetModelProfile().Context.DefaultTokens, Is.EqualTo(WHAT_THE_MACHINE_REPORTS));
+            Assert.That(settings.GetModelProfile(), Is.EqualTo(configured.GetModelProfile()), "A request has to know the model as the configured provider does.");
+        });
+    }
+}

app/Tests/Tools/ToolCalling/SemanticSearchToolDescriptionTests.cs

@@ -30,7 +30,7 @@ public sealed class SemanticSearchToolDescriptionTests
         Assert.Multiple(() =>
         {
             Assert.That(function.DescriptionForLLM, Does.Contain($"id={HANDBOOK.Id}, name='Handbook', type=local folder, results per page=10, last page=9, description='Our processes.'"));
-            Assert.That(function.DescriptionForLLM, Does.Contain($"id={INTRANET.Id}, name='Intranet', type=external data source, results per page=10, last page=9"));
+            Assert.That(function.DescriptionForLLM, Does.Contain($"id={INTRANET.Id}, name='Intranet', type=external data source, results per page=10, last page=1"), "ERI v1 knows no offset, so an ERI data source has a single page.");
             Assert.That(function.DescriptionForLLM, Does.Not.Contain(MINUTES.Id).And.Not.Contain("Minutes"), "A data source which is not offered must not even be named.");
             Assert.That(OfferedIds(function), Is.EqualTo(new[] { HANDBOOK.Id, INTRANET.Id }), "The schema lets the model name exactly the data sources the description lists.");
         });
@@ -86,11 +86,49 @@ public sealed class SemanticSearchToolDescriptionTests
         Assert.That(function.DescriptionForLLM, Does.Contain($"description='{new string('x', 499)}...'"));
     }
 
-    private static ToolFunctionDefinition Describe(params (IDataSource DataSource, string Description)[] dataSources)
+    [Test]
+    public void AnUnreachableDataSourceIsNamedButCannotBeSearched()
+    {
+        var function = DescribeWithUnreachable([(HANDBOOK, "Our processes.")], [INTRANET]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(function.DescriptionForLLM, Does.Contain("could not be reached or signed in to right now"), "A wrong password keeps the data source away just as well as a server which is down.");
+            Assert.That(function.DescriptionForLLM, Does.Contain("- name='Intranet'"), "The model has to know what is missing, so that it can tell the user.");
+            Assert.That(function.DescriptionForLLM, Does.Not.Contain(INTRANET.Id), "A data source which cannot be searched gets no ID to pass.");
+            Assert.That(OfferedIds(function), Is.EqualTo(new[] { HANDBOOK.Id }), "The schema still lets the model name only what it may search.");
+        });
+    }
+
+    [Test]
+    public void OnlyUnreachableDataSourcesStillMakeAFunction()
+    {
+        var function = DescribeWithUnreachable([], [INTRANET]);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(function.DescriptionForLLM, Does.Contain("None of the data sources of this chat can be searched right now."));
+            Assert.That(function.DescriptionForLLM, Does.Contain("- name='Intranet'"));
+            Assert.That(function.DescriptionForLLM, Does.Not.Contain("The data sources you may search"));
+        });
+    }
+
+    [Test]
+    public void WithoutUnreachableDataSourcesTheDescriptionStaysAsBefore()
+    {
+        var withoutList = Describe((HANDBOOK, "Our processes."));
+        var withEmptyList = DescribeWithUnreachable([(HANDBOOK, "Our processes.")], []);
+
+        Assert.That(withEmptyList.DescriptionForLLM, Is.EqualTo(withoutList.DescriptionForLLM), "The providers cache a request from its beginning, so nothing may change while every server answers.");
+    }
+
+    private static ToolFunctionDefinition Describe(params (IDataSource DataSource, string Description)[] dataSources) => DescribeWithUnreachable(dataSources, []);
+
+    private static ToolFunctionDefinition DescribeWithUnreachable((IDataSource DataSource, string Description)[] dataSources, IDataSource[] unreachableDataSources)
     {
         // Stating its definition needs none of the services the tool searches with:
         var registered = new SemanticSearchTool(null!, null!, null!, null!, null!).GetDefinition().Function;
-        return SemanticSearchTool.DescribeDataSources(registered, dataSources);
+        return SemanticSearchTool.DescribeDataSources(registered, dataSources, unreachableDataSources);
     }
 
     private static IReadOnlyList<string?> OfferedIds(ToolFunctionDefinition function) => function.Parameters

app/Tests/Tools/ToolCalling/SemanticSearchToolRequestTests.cs

@@ -83,12 +83,12 @@ public sealed class SemanticSearchToolRequestTests
     [Test]
     public void APageAfterTheFirstComesThroughForOneDataSource()
     {
-        var named = SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{INTRANET.Id}}"],"page":2}"""), OFFERED);
+        var named = SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{HANDBOOK.Id}}"],"page":2}"""), OFFERED);
         var onlyOneOffered = SemanticSearchTool.ReadRequest(Arguments("""{"query":"travel expenses","page":2}"""), [HANDBOOK]);
 
         Assert.Multiple(() =>
         {
-            Assert.That(named.DataSources, Is.EqualTo(new[] { INTRANET }));
+            Assert.That(named.DataSources, Is.EqualTo(new[] { HANDBOOK }));
             Assert.That(named.Page, Is.EqualTo(2));
             Assert.That(onlyOneOffered.Page, Is.EqualTo(2), "With a single data source offered, leaving it unnamed still means that one.");
         });
@@ -101,6 +101,13 @@ public sealed class SemanticSearchToolRequestTests
         Assert.That(message, Does.Contain("at most 9").And.Contain("but was 10").And.Contain("Rephrase the query"));
     }
 
+    [Test]
+    public void AnERIDataSourceHasNoFurtherPage()
+    {
+        var message = Refusal(() => SemanticSearchTool.ReadRequest(Arguments($$"""{"query":"travel expenses","data_source_ids":["{{INTRANET.Id}}"],"page":2}"""), OFFERED));
+        Assert.That(message, Does.Contain("at most 1").And.Contain("but was 2").And.Contain("Rephrase the query"), "A server which allows only the configured number of matches would refuse the page, and the data source would count as not searched.");
+    }
+
     private static JsonElement Arguments(string json) => JsonSerializer.Deserialize<JsonElement>(json);
 
     private static JsonElement Query(string query) => JsonSerializer.SerializeToElement(new Dictionary<string, string> { ["query"] = query });

app/Tests/Tools/ToolCalling/SemanticSearchToolUnreachableTests.cs

@@ -0,0 +1,109 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.SemanticSearch;
+
+namespace AIStudio.Tests.Tools.ToolCalling;
+
+/// <summary>
+/// Checks how Semantic Search deals with ERI data sources whose server could not be reached.
+/// </summary>
+/// <remarks>
+/// Such a data source is never searched, but the model learns about it, so that it does not take
+/// its silence for finding nothing. It learns only about those it would have been offered, and it
+/// gets no ID to pass for them, since there is nothing it could search.
+/// </remarks>
+[TestFixture]
+public sealed class SemanticSearchToolUnreachableTests
+{
+    private static readonly IDataSource HANDBOOK = new DataSourceLocalDirectory { Num = 1, Id = "11111111-1111-1111-1111-111111111111", Name = "Handbook", MaxMatches = 10 };
+    private static readonly IDataSource MINUTES = new DataSourceLocalFile { Num = 2, Id = "22222222-2222-2222-2222-222222222222", Name = "Minutes", MaxMatches = 10 };
+    private static readonly IDataSource INTRANET = new DataSourceERI_V1 { Num = 3, Id = "33333333-3333-3333-3333-333333333333", Name = "Intranet", MaxMatches = 10 };
+    private static readonly IDataSource WIKI = new DataSourceERI_V1 { Num = 4, Id = "44444444-4444-4444-4444-444444444444", Name = "Wiki", MaxMatches = 10 };
+
+    [Test]
+    public void TheAutomaticSelectionNamesEveryUnreachableDataSource()
+    {
+        var offered = SemanticSearchTool.SelectOffered(new DataSourceOptions { AutomaticDataSourceSelection = true }, Checked([HANDBOOK], [HANDBOOK], [WIKI, INTRANET]));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(offered.Searchable, Is.EqualTo(new[] { HANDBOOK }));
+            Assert.That(offered.Unreachable, Is.EqualTo(new[] { INTRANET, WIKI }), "In the order they are offered, so that the function stays the same for the same data sources.");
+        });
+    }
+
+    [Test]
+    public void TheUserSelectionNamesOnlyTheSelectedUnreachableDataSources()
+    {
+        var options = new DataSourceOptions { AutomaticDataSourceSelection = false, PreselectedDataSourceIds = [HANDBOOK.Id, INTRANET.Id] };
+        var offered = SemanticSearchTool.SelectOffered(options, Checked([HANDBOOK, MINUTES], [HANDBOOK], [INTRANET, WIKI]));
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(offered.Searchable, Is.EqualTo(new[] { HANDBOOK }));
+            Assert.That(offered.Unreachable, Is.EqualTo(new[] { INTRANET }), "The model must not learn about a data source the user did not select for this chat.");
+        });
+    }
+
+    [Test]
+    public void AnUnreachableDataSourceWithoutMatchesIsNotNamed()
+    {
+        var silent = new DataSourceERI_V1 { Num = 5, Id = "55555555-5555-5555-5555-555555555555", Name = "Archive", MaxMatches = 0 };
+        var offered = SemanticSearchTool.SelectOffered(new DataSourceOptions { AutomaticDataSourceSelection = true }, Checked([], [], [silent]));
+
+        Assert.That(offered.IsEmpty, Is.True, "A data source configured to return no matches is not offered when it answers, either.");
+    }
+
+    [Test]
+    public void ASearchOfAllDataSourcesReportsTheUnreachableOnes()
+    {
+        var offered = new OfferedDataSources([HANDBOOK, MINUTES], [INTRANET]);
+        var request = new SemanticSearchRequest("travel expenses", [HANDBOOK, MINUTES], 1);
+
+        Assert.That(SemanticSearchTool.GetUnreachableToReport(request, offered), Is.EqualTo(new[] { INTRANET }));
+    }
+
+    [Test]
+    public void ASearchWithOnlyUnreachableDataSourcesReportsThem()
+    {
+        var offered = new OfferedDataSources([], [INTRANET]);
+        var request = new SemanticSearchRequest("travel expenses", [], 1);
+
+        Assert.That(SemanticSearchTool.GetUnreachableToReport(request, offered), Is.EqualTo(new[] { INTRANET }), "Otherwise, the model would answer as though the chat had no data sources.");
+    }
+
+    [Test]
+    public void ASearchOfSomeDataSourcesLeavesTheUnreachableOnesOut()
+    {
+        var offered = new OfferedDataSources([HANDBOOK, MINUTES], [INTRANET]);
+        var request = new SemanticSearchRequest("travel expenses", [HANDBOOK], 1);
+
+        Assert.That(SemanticSearchTool.GetUnreachableToReport(request, offered), Is.Empty, "The model asked for the handbook alone.");
+    }
+
+    [Test]
+    public void AFurtherPageLeavesTheUnreachableOnesOut()
+    {
+        var offered = new OfferedDataSources([HANDBOOK], [INTRANET]);
+        var request = new SemanticSearchRequest("travel expenses", [HANDBOOK], 2);
+
+        Assert.That(SemanticSearchTool.GetUnreachableToReport(request, offered), Is.Empty, "The first page reported them already.");
+    }
+
+    [Test]
+    public void TheResultNamesAnUnreachableDataSourceWithoutItsID()
+    {
+        var result = SemanticSearchTool.DescribeUnreachableResult(INTRANET);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.ContainsKey("id"), Is.False, "The model cannot search it, so an ID would only invite a call which is refused.");
+            Assert.That(result["name"]!.GetValue<string>(), Is.EqualTo("Intranet"));
+            Assert.That(result["result_count"]!.GetValue<int>(), Is.Zero);
+            Assert.That(result["issues"]!.AsArray(), Has.Count.EqualTo(1), "A result count of zero alone reads as finding nothing.");
+        });
+    }
+
+    private static AllowedSelectedDataSources Checked(IReadOnlyList<IDataSource> allowed, IReadOnlyList<IDataSource> selected, IReadOnlyList<IDataSource> unreachable) => new(allowed, selected, [], [], unreachable);
+}