using System.Text.Json; using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; namespace AIStudio.Tools.ToolCallingSystem; /// /// Holds the tools AI Studio knows and decides which of them a request may use. /// /// /// Definitions arrive through tool definition sources — the app's own tools from code, later the /// ones plugin authors write. Every definition passes the same validation regardless of where it /// came from, which matters most for the ones AI Studio does not control. /// public sealed class ToolRegistry { private readonly ILogger logger; private readonly SettingsManager settingsManager; private readonly ToolSettingsService toolSettingsService; private readonly Dictionary definitionsById = new(StringComparer.Ordinal); private readonly Dictionary implementationsByKey = new(StringComparer.Ordinal); /// /// What the checks of a single tool found. /// /// What keeps the tool from being offered, or none. /// The tool's implementation, once it was found. /// The confidence the tool requires and where that requirement came from, once it was read. private readonly record struct ToolCheck(ToolOfferBlockReason BlockReason, IToolImplementation? Implementation, SettingsManager.ToolMinimumProviderConfidenceResolution? MinimumConfidence); public ToolRegistry( IEnumerable implementations, IEnumerable definitionSources, SettingsManager settingsManager, ToolSettingsService toolSettingsService, ILogger logger) { this.logger = logger; this.settingsManager = settingsManager; this.toolSettingsService = toolSettingsService; foreach (var implementation in implementations) { if (string.IsNullOrWhiteSpace(implementation.ImplementationKey)) { this.logger.LogWarning("Skipping a tool implementation with an empty implementation key."); continue; } if (!this.implementationsByKey.TryAdd(implementation.ImplementationKey, implementation)) this.logger.LogWarning("Skipping duplicate tool implementation key '{ImplementationKey}'.", implementation.ImplementationKey); } // // Function names are checked across all sources together: two tools offering the same // name would be indistinguishable to a model, no matter who defined them. // var functionNames = new HashSet(StringComparer.Ordinal); foreach (var source in definitionSources) { foreach (var definition in source.GetDefinitions()) { if (!TryValidateDefinition(definition, out var validationIssue)) { this.logger.LogWarning("Skipping tool definition '{ToolId}' from source '{SourceName}': {ValidationIssue}", definition.Id, source.SourceName, validationIssue); continue; } if (!this.implementationsByKey.ContainsKey(definition.ImplementationKey)) { this.logger.LogWarning("Skipping tool definition '{ToolId}' because implementation key '{ImplementationKey}' is not registered.", definition.Id, definition.ImplementationKey); continue; } if (!this.definitionsById.TryAdd(definition.Id, definition)) { this.logger.LogWarning("Skipping duplicate tool definition ID '{ToolId}' from source '{SourceName}'.", definition.Id, source.SourceName); continue; } if (!functionNames.Add(definition.Function.Name)) { this.logger.LogWarning("Skipping tool definition '{ToolId}' because function name '{FunctionName}' is already registered.", definition.Id, definition.Function.Name); this.definitionsById.Remove(definition.Id); } } } } /// /// Whether a tool definition is complete enough to register. /// /// /// What a definition cannot be is null in its parts: definitions are C# objects whose members /// are non-nullable and initialized, so only their content is checked here. Should definitions /// one day arrive from outside as data — a tool plugin, say — that assumption ends at the point /// where the data becomes a definition, and it is there that null has to be caught. /// private static bool TryValidateDefinition(ToolDefinition definition, out string issue) { issue = string.Empty; if (definition.SchemaVersion != 1) { issue = $"unsupported schema version '{definition.SchemaVersion}'"; return false; } if (string.IsNullOrWhiteSpace(definition.Id)) { issue = "the definition ID is empty"; return false; } if (string.IsNullOrWhiteSpace(definition.ImplementationKey)) { issue = "the implementation key is empty"; return false; } if (!IsValidFunctionName(definition.Function.Name)) { issue = "the function name must contain 1-64 ASCII letters, digits, underscores, or hyphens"; return false; } if (definition.Function.Parameters.ValueKind is not JsonValueKind.Object) { issue = "the function parameters schema must be a JSON object"; return false; } if (definition.VisibleIn.AllowedComponents.Any(component => !Enum.IsDefined(component)) || definition.VisibleIn.DeniedComponents.Any(component => !Enum.IsDefined(component))) { issue = "the visibility definition must contain valid component lists"; return false; } if (!string.Equals(definition.SettingsSchema.Type, "object", StringComparison.OrdinalIgnoreCase)) { issue = "the settings schema must have type 'object'"; return false; } if (definition.SettingsSchema.Properties.Any(x => string.IsNullOrWhiteSpace(x.Key) || !string.Equals(x.Value.Type, "string", StringComparison.OrdinalIgnoreCase))) { issue = "settings properties must be named string fields"; return false; } // // An empty group is how a field says it belongs to no group. Whitespace looks the // same in the settings file but is a different string, so it would open a second, // nameless group next to the ungrouped fields: // var fieldsWithBlankGroup = definition.SettingsSchema.Properties .Where(x => x.Value.Group.Length > 0 && string.IsNullOrWhiteSpace(x.Value.Group)) .Select(x => x.Key) .ToList(); if (fieldsWithBlankGroup.Count > 0) { issue = $"these settings declare a blank group name: {string.Join(", ", fieldsWithBlankGroup)}"; return false; } var fieldsWithBothOptionKinds = definition.SettingsSchema.Properties .Where(x => !string.IsNullOrWhiteSpace(x.Value.OptionSource) && x.Value.EnumValues.Count > 0) .Select(x => x.Key) .ToList(); if (fieldsWithBothOptionKinds.Count > 0) { issue = $"these settings declare both an option source and an enum list: {string.Join(", ", fieldsWithBothOptionKinds)}"; return false; } var fieldsWithUnknownOptionSource = definition.SettingsSchema.Properties .Where(x => !string.IsNullOrWhiteSpace(x.Value.OptionSource) && !ToolSettingsOptionSources.IsKnown(x.Value.OptionSource)) .Select(x => $"{x.Key} ('{x.Value.OptionSource}')") .ToList(); if (fieldsWithUnknownOptionSource.Count > 0) { issue = $"these settings reference an unknown option source: {string.Join(", ", fieldsWithUnknownOptionSource)}"; return false; } if (definition.SettingsSchema.Required.Any(string.IsNullOrWhiteSpace)) { issue = "required setting names cannot be empty"; return false; } var missingRequiredProperties = definition.SettingsSchema.Required .Where(x => !definition.SettingsSchema.Properties.ContainsKey(x)) .ToList(); if (missingRequiredProperties.Count > 0) { issue = $"required settings are missing definitions: {string.Join(", ", missingRequiredProperties)}"; return false; } return true; } private static bool IsValidFunctionName(string? functionName) => !string.IsNullOrWhiteSpace(functionName) && functionName.Length <= 64 && functionName.All(character => char.IsAsciiLetterOrDigit(character) || character is '_' or '-'); public IReadOnlyList GetDefinitionsForComponent(Components component) { return this.definitionsById.Values .Where(x => x.VisibleIn.IsVisibleIn(component)) .OrderBy(x => this.implementationsByKey.GetValueOrDefault(x.ImplementationKey)?.GetDisplayName(), StringComparer.OrdinalIgnoreCase) .ToList(); } public IReadOnlyList GetAllDefinitions() => this.definitionsById.Values .OrderBy(x => this.implementationsByKey.GetValueOrDefault(x.ImplementationKey)?.GetDisplayName(), StringComparer.OrdinalIgnoreCase) .ToList(); public ToolDefinition? GetDefinition(string toolId) => this.definitionsById.GetValueOrDefault(toolId); public IToolImplementation? GetImplementation(string implementationKey) => this.implementationsByKey.GetValueOrDefault(implementationKey); /// /// The provider confidence a tool needs: its own minimum, unless the user or an administrator /// raised or lowered it. /// /// /// This is the place that knows both halves — the definition's own minimum and the stored /// overrides — so callers holding only a tool ID come here instead of to the settings. /// public ConfidenceLevel GetMinimumProviderConfidence(string toolId) => this.GetDefinition(toolId) is { } definition ? this.GetMinimumProviderConfidence(definition) : ConfidenceLevel.NONE; public ConfidenceLevel GetMinimumProviderConfidence(ToolDefinition definition) => this.settingsManager.GetMinimumProviderConfidenceForTool(definition.Id, definition.MinimumProviderConfidence); /// /// Narrows a selection of tool IDs to those the given provider may actually use. /// /// /// Used before a request is sent, so the chat records what will really be available rather /// than what the user once ticked. Lives here because judging a tool needs its definition: /// the settings know the overrides, the definition knows the tool's own minimum. /// /// The provider the request goes to. /// The tools the user selected. /// The subset that is enabled, active, and allowed by the provider's confidence. public HashSet FilterToolIdsForProvider(AIStudio.Settings.Provider provider, IEnumerable selectedToolIds) { if (!this.settingsManager.AreToolsEnabled()) return []; if (!provider.GetToolCallingAvailability().IsAvailable) return []; var providerConfidence = provider.UsedLLMProvider.GetConfidence(this.settingsManager).Level; var filtered = ToolSelectionRules.NormalizeSelection(selectedToolIds); foreach (var toolId in filtered.ToList()) { if (!this.settingsManager.IsToolActive(toolId)) { filtered.Remove(toolId); continue; } if (!ToolSelectionRules.IsProviderConfidenceAllowed(providerConfidence, this.GetMinimumProviderConfidence(toolId))) filtered.Remove(toolId); } return filtered; } /// /// The tools somebody can select in this component. /// /// /// Every selection in the app is built from this list: the one below the message field, the /// defaults, the templates, and the tools the AI picks for a new assistant. A tool which offers /// itself from the context of a chat is left out, because selecting it would change nothing. /// The tool list of the app settings asks for all definitions instead, so an organization can /// still switch such a tool off or set the trust it requires. /// public async Task> GetCatalogAsync(Components component) { var definitions = this.GetDefinitionsForComponent(component).Where(x => x.Activation is ToolActivation.SELECTION); return await this.GetCatalogAsync(definitions); } /// /// Reduces a set of tool IDs to the tools a user could switch on themselves in this component. /// /// /// For preselecting tools on someone's behalf, such as when a launcher opens a chat. A tool /// this installation does not know, one an organization switched off, or one whose settings are /// incomplete cannot be enabled by hand either, so handing it over as enabled would show the /// user a state they could not have produced and could not fix from where they are. The /// provider confidence stays out of this: it belongs to the moment a message is sent, not to /// the selection, and it may well be a different provider by then. /// public async Task> FilterSelectableToolIdsAsync(Components component, IEnumerable toolIds) { var wantedToolIds = ToolSelectionRules.NormalizeSelection(toolIds); if (wantedToolIds.Count is 0 || !this.settingsManager.AreToolsEnabled()) return []; var catalog = await this.GetCatalogAsync(component); return catalog .Where(x => wantedToolIds.Contains(x.Definition.Id) && x is { IsActive: true, ConfigurationState.IsConfigured: true }) .Select(x => x.Definition.Id) .ToHashSet(StringComparer.Ordinal); } public async Task> GetCatalogAsync(IEnumerable definitions) { var definitionList = definitions.ToList(); var items = new List(definitionList.Count); foreach (var definition in definitionList) { if (!this.implementationsByKey.TryGetValue(definition.ImplementationKey, out var implementation)) continue; items.Add(new ToolCatalogItem { Definition = definition, Implementation = implementation, ConfigurationState = await this.toolSettingsService.GetConfigurationStateAsync(definition, implementation), IsActive = this.settingsManager.IsToolActive(definition.Id), MinimumProviderConfidence = this.GetMinimumProviderConfidence(definition), }); } return items; } /// /// The tools a request offers the model, each with the function it offers in this request. /// /// /// Model capabilities are not a parameter on purpose: they are read from the given provider, /// which carries the user's expert capability overrides. Passing them in separately allowed a /// caller to gate tools on capabilities that differed from the ones the availability check saw.

/// The candidates are the selected tools and every tool which offers itself from the context of /// the chat, see ToolActivation. Each one passes the same checks, and only then is it asked what /// it offers in this request, see IToolImplementation.ResolveFunctionAsync and /// IToolImplementation.ResolveSystemPromptInstructionsAsync. ///
/// The request being prepared. /// The tools selected for the request. /// Whether the request may run tools at all, as its caller decides. /// The cancellation token of the request. /// The runnable tools, with their definitions as offered in this request. public async Task> GetRunnableToolsAsync(ToolResolutionContext context, IEnumerable selectedToolIds, bool mayRunTools, CancellationToken token = default) { var provider = context.Provider; var component = context.Component; var providerConfidence = context.ProviderConfidence; if (!this.settingsManager.AreToolsEnabled()) { this.logger.LogDebug("Tool calling is skipped because tools are disabled by managed configuration."); return []; } // // Where the user selects the tools, they must be able to see that selection; where the // assistant's own rules name them, there is nothing to see. Which of the two applies is // decided by the caller, because only it knows where its tools came from: // if (!mayRunTools) { this.logger.LogDebug("Tool calling is skipped for component '{Component}' because its tool selection is hidden and no assistant rule names the tools.", component); return []; } var toolCallingAvailability = provider.GetToolCallingAvailability(); if (!toolCallingAvailability.IsAvailable) { this.logger.LogDebug("Tool calling is unavailable for provider '{Provider}' with model '{ModelId}': {Reason}", provider.InstanceName, provider.Model.Id, toolCallingAvailability.Message); return []; } var selectedToolIdSet = ToolSelectionRules.NormalizeSelection(selectedToolIds); this.logger.LogDebug("Resolving runnable tools for provider '{Provider}' with model '{ModelId}'. Selected tool IDs: [{ToolIds}].", provider.InstanceName, provider.Model.Id, string.Join(", ", selectedToolIdSet.OrderBy(x => x, StringComparer.Ordinal))); var definitions = this.GetDefinitionsForComponent(component) .Where(x => x.Activation is ToolActivation.CONTEXT || selectedToolIdSet.Contains(x.Id)) .ToList(); var result = new List<(ToolDefinition, IToolImplementation)>(definitions.Count); foreach (var definition in definitions) { var check = await this.CheckToolAsync(definition, providerConfidence); if (check.MinimumConfidence is { } minimumConfidence) this.logger.LogDebug("Tool '{ToolId}' uses minimum provider confidence '{ConfidenceLevel}' from {Source}.", definition.Id, minimumConfidence.ConfidenceLevel, minimumConfidence.Source); switch (check) { case { BlockReason: ToolOfferBlockReason.NONE, Implementation: { } implementation }: if (await this.ResolveAsync(definition, implementation, context, token) is { } offeredDefinition) result.Add((offeredDefinition, implementation)); break; case { BlockReason: ToolOfferBlockReason.TOOL_SWITCHED_OFF }: this.logger.LogDebug("Skipping tool '{ToolId}' because it is disabled by managed configuration.", definition.Id); break; case { BlockReason: ToolOfferBlockReason.NOT_CONFIGURED }: this.logger.LogDebug("Skipping tool '{ToolId}' because it is not configured.", definition.Id); break; case { BlockReason: ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW }: this.logger.LogInformation("Skipping tool '{ToolId}' because provider confidence '{ProviderConfidence}' is below the required minimum '{MinimumConfidence}'.", definition.Id, providerConfidence, check.MinimumConfidence?.ConfidenceLevel); break; case { BlockReason: ToolOfferBlockReason.NOT_AVAILABLE_HERE }: this.logger.LogWarning("Skipping tool '{ToolId}' because no implementation is registered.", definition.Id); break; } } foreach (var selectedToolId in selectedToolIdSet.Where(selectedToolId => definitions.All(definition => !definition.Id.Equals(selectedToolId, StringComparison.Ordinal)))) this.logger.LogDebug("Skipping tool '{ToolId}' because it is not selected in this component or not available in this context.", selectedToolId); return result; } /// /// Whether a tool can be offered to a provider in this component, and if not, what is in the way. /// /// /// Asks the same questions, in the same order, as the preparation of a request does, because /// whoever decides something on the tool's behalf must not come to another answer than the /// request will. The RAG process, for instance, leaves the searching of the data sources to /// Semantic Search only when this says it can be offered; checks of its own which forgot one /// of these would leave a chat without its data sources.

/// Two questions stay out. Whether the tool is selected is the caller's business, and whether /// the tool has anything to offer right now depends on the chat, so only the preparation of a /// request can answer it. ///
/// The tool to check. /// The provider the request would go to. /// Where the request would come from. /// ToolOfferBlockReason.NONE when nothing is in the way, otherwise the first obstacle found. public async Task GetOfferBlockReasonAsync(string toolId, AIStudio.Settings.Provider provider, Components component) { if (!this.settingsManager.AreToolsEnabled()) return ToolOfferBlockReason.TOOLS_SWITCHED_OFF; if (!provider.GetToolCallingAvailability().IsAvailable) return ToolOfferBlockReason.MODEL_CANNOT_USE_TOOLS; if (this.GetDefinition(toolId) is not { } definition || !definition.VisibleIn.IsVisibleIn(component)) return ToolOfferBlockReason.NOT_AVAILABLE_HERE; var providerConfidence = provider.UsedLLMProvider.GetConfidence(this.settingsManager).Level; return (await this.CheckToolAsync(definition, providerConfidence)).BlockReason; } /// /// How the data sources of a chat are actually searched: the way the user wants, or with every /// message when Semantic Search cannot be offered. /// /// /// The one place which decides between the two. The RAG process, the data source selection, /// and the check of what a launched chat may search all ask here, so that none of them counts /// the agents of the RAG process in or out while another does the opposite. The chat searches /// itself when the user prefers it and GetOfferBlockReasonAsync has nothing against it. Its /// answer comes along whatever the user prefers, so that the user interface can leave out a /// choice which is none and say why the chat searches with every message.

/// Whether Semantic Search has data sources to offer right now is no question here. Without /// them, the RAG process would find nothing to search either: it asks the same checks, and more /// providers have to pass them. ///
/// The data source options of the chat. /// The provider the chat runs with. /// Where the chat runs. /// How the data sources are searched, and why Semantic Search cannot be used, if so. public async Task GetEffectiveRetrievalModeAsync(DataSourceOptions options, AIStudio.Settings.Provider provider, Components component) { var blockReason = await this.GetOfferBlockReasonAsync(ToolSelectionRules.SEMANTIC_SEARCH_TOOL_ID, provider, component); var searchesItself = options.RetrievalMode is DataSourceRetrievalMode.SEMANTIC_SEARCH && blockReason is ToolOfferBlockReason.NONE; return new(searchesItself ? DataSourceRetrievalMode.SEMANTIC_SEARCH : DataSourceRetrievalMode.EVERY_MESSAGE, blockReason); } /// /// Checks one tool on its own, apart from what applies to all tools of a request. /// /// /// Shared by the preparation of a request and by GetOfferBlockReasonAsync, so the two cannot /// drift apart. It reports rather than logs: the preparation of a request writes down why a /// tool was left out, while a question asked by the user interface on every render must not. /// private async Task CheckToolAsync(ToolDefinition definition, ConfidenceLevel providerConfidence) { if (!this.settingsManager.IsToolActive(definition.Id)) return new(ToolOfferBlockReason.TOOL_SWITCHED_OFF, null, null); if (!this.implementationsByKey.TryGetValue(definition.ImplementationKey, out var implementation)) return new(ToolOfferBlockReason.NOT_AVAILABLE_HERE, null, null); var configurationState = await this.toolSettingsService.GetConfigurationStateAsync(definition, implementation); if (!configurationState.IsConfigured) return new(ToolOfferBlockReason.NOT_CONFIGURED, implementation, null); var minimumConfidence = this.settingsManager.GetMinimumProviderConfidenceResolutionForTool(definition.Id, definition.MinimumProviderConfidence); if (!ToolSelectionRules.IsProviderConfidenceAllowed(providerConfidence, minimumConfidence.ConfidenceLevel)) return new(ToolOfferBlockReason.PROVIDER_CONFIDENCE_TOO_LOW, implementation, minimumConfidence); return new(ToolOfferBlockReason.NONE, implementation, minimumConfidence); } /// /// Asks a tool which passed every check what it offers in this request. /// /// /// Two answers are taken: the function, of which only the description and the parameters count, /// and the instructions for the system prompt. The name and the strict mode stay as registered, /// because the model's calls find their tool by that name, and the rest of the definition was /// checked a moment ago and must not change after that. The instructions are asked for only once /// the tool has a function to offer, since they would otherwise describe a tool the model never /// gets to see. /// /// The definition as offered in this request, or null when the tool has nothing to offer or could not say what. private async Task ResolveAsync(ToolDefinition definition, IToolImplementation implementation, ToolResolutionContext context, CancellationToken token) { ToolFunctionDefinition? function; string instructions; try { function = await implementation.ResolveFunctionAsync(definition, context, token); if (function is null) { this.logger.LogDebug("Skipping tool '{ToolId}' because it has nothing to offer in this request.", definition.Id); return null; } instructions = await implementation.ResolveSystemPromptInstructionsAsync(definition, context, token); } catch (OperationCanceledException) when (token.IsCancellationRequested) { throw; } catch (Exception exception) { this.logger.LogError(exception, "Skipping tool '{ToolId}' because it could not say what it offers in this request.", definition.Id); return null; } var offeredFunction = this.GetOfferedFunction(definition, function); if (ReferenceEquals(offeredFunction, definition.Function) && string.Equals(instructions, definition.SystemPromptInstructions, StringComparison.Ordinal)) return definition; return definition with { Function = offeredFunction, SystemPromptInstructions = instructions, }; } /// /// The function a tool offers in this request, made of what it answered and what it registered. /// /// The registered function when the tool offers it unchanged or offers something unusable, otherwise the tailored one with the registered name and strict mode. private ToolFunctionDefinition GetOfferedFunction(ToolDefinition definition, ToolFunctionDefinition function) { if (ReferenceEquals(function, definition.Function)) return definition.Function; if (function.Parameters.ValueKind is not JsonValueKind.Object) { this.logger.LogWarning("Tool '{ToolId}' offered parameters which are not a JSON object schema. Its function is offered as registered instead.", definition.Id); return definition.Function; } if (!string.Equals(function.Name, definition.Function.Name, StringComparison.Ordinal) || function.Strict != definition.Function.Strict) this.logger.LogWarning("Tool '{ToolId}' changed the name or the strict mode of its function for a request. Both stay as registered.", definition.Id); return function with { Name = definition.Function.Name, Strict = definition.Function.Strict, }; } }