Added a way to see the complete system prompt of a chat (#1001)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Chat/ChatThread.cs

@@ -390,43 +390,31 @@ public sealed record ChatThread
     /// request which is never sent.
     ///
     /// Nothing here writes to the thread and nothing logs, because this runs while somebody types.
+    ///
+    /// Each part is recorded as it is added, in the order of the text, so that the dialog showing
+    /// the prompt names the parts from the decisions made here rather than repeating them.
     /// </remarks>
     /// <param name="settingsManager">The settings manager instance to use.</param>
     /// <param name="runnableToolDefinitions">The tools which may run in this thread. Null when the thread runs without tools.</param>
     /// <returns>The system prompt and what building it decided.</returns>
     public PreparedSystemPrompt BuildSystemPrompt(SettingsManager settingsManager, IEnumerable<ToolDefinition>? runnableToolDefinitions = null)
     {
-        var allowProfile = true;
+        var parts = new List<SystemPromptPart>();
 
         //
         // Use the information from the chat template, if provided. Otherwise, use the default system prompt
         //
-        string systemPromptTextWithChatTemplate;
-        var logMessage = $"Using no chat template for chat thread '{this.Name}'.";
-        if (string.IsNullOrWhiteSpace(this.SelectedChatTemplate))
-            systemPromptTextWithChatTemplate = this.SystemPrompt;
-        else
-        {
-            if(!Guid.TryParse(this.SelectedChatTemplate, out var chatTemplateId))
-                systemPromptTextWithChatTemplate = this.SystemPrompt;
-            else
-            {
-                if(this.SelectedChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE.Id || chatTemplateId == Guid.Empty)
-                    systemPromptTextWithChatTemplate = this.SystemPrompt;
-                else
-                {
-                    var chatTemplate = settingsManager.GetChatTemplateById(this.SelectedChatTemplate);
-                    if(chatTemplate == ChatTemplate.NO_CHAT_TEMPLATE)
-                        systemPromptTextWithChatTemplate = this.SystemPrompt;
-                    else
-                    {
-                        logMessage = $"Using chat template '{chatTemplate.Name}' for chat thread '{this.Name}'.";
-                        allowProfile = chatTemplate.AllowProfileUsage;
-                        systemPromptTextWithChatTemplate = chatTemplate.ToSystemPrompt();
-                    }
-                }
-            }
-        }
+        var chatTemplate = this.GetSelectedChatTemplate(settingsManager);
+        var usesChatTemplate = chatTemplate != ChatTemplate.NO_CHAT_TEMPLATE;
+        var allowProfile = !usesChatTemplate || chatTemplate.AllowProfileUsage;
+        var systemPromptTextWithChatTemplate = usesChatTemplate ? chatTemplate.ToSystemPrompt() : this.SystemPrompt;
+        var logMessage = usesChatTemplate
+            ? $"Using chat template '{chatTemplate.Name}' for chat thread '{this.Name}'."
+            : $"Using no chat template for chat thread '{this.Name}'.";
+
+        parts.Add(usesChatTemplate
+            ? new(SystemPromptPartKind.CHAT_TEMPLATE, systemPromptTextWithChatTemplate) { SourceName = chatTemplate.Name }
+            : new(SystemPromptPartKind.BASE_PROMPT, systemPromptTextWithChatTemplate));
 
         //
         // Add augmented data, if available:
@@ -436,51 +424,50 @@ public sealed record ChatThread
         {
             true => $"""
                      {systemPromptTextWithChatTemplate}
-                     
+
                      {this.AugmentedData}
                      """,
 
             false => systemPromptTextWithChatTemplate,
         };
-        
+
+        if (isAugmentedDataAvailable)
+            parts.Add(new(SystemPromptPartKind.RETRIEVED_DATA, this.AugmentedData));
+
         logMessage = isAugmentedDataAvailable
             ? $"{logMessage} Augmented data is available for the chat thread."
             : $"{logMessage} No augmented data is available for the chat thread.";
 
         //
-        // Add information from the profile if available and allowed:
+        // Add information from the profile if available and allowed. A selected profile which the
+        // chat template does not allow is reported, so that nobody wonders where it went:
         //
         string systemPromptText;
         var profileNote = $"Using no profile for chat thread '{this.Name}'.";
-        if (string.IsNullOrWhiteSpace(this.SelectedProfile) || !allowProfile)
+        var profile = this.GetSelectedProfile(settingsManager);
+        var switchedOffProfile = Profile.NO_PROFILE;
+        if (profile == Profile.NO_PROFILE)
             systemPromptText = systemPromptWithAugmentedData;
+        else if (!allowProfile)
+        {
+            systemPromptText = systemPromptWithAugmentedData;
+            switchedOffProfile = profile;
+        }
         else
         {
-            if(!Guid.TryParse(this.SelectedProfile, out var profileId))
-                systemPromptText = systemPromptWithAugmentedData;
-            else
-            {
-                if(this.SelectedProfile == Profile.NO_PROFILE.Id || profileId == Guid.Empty)
-                    systemPromptText = systemPromptWithAugmentedData;
-                else
-                {
-                    var profile = settingsManager.GetProfileById(this.SelectedProfile);
-                    if(profile == Profile.NO_PROFILE)
-                        systemPromptText = systemPromptWithAugmentedData;
-                    else
-                    {
-                        profileNote = $"Using profile '{profile.Name}' for chat thread '{this.Name}'.";
-                        systemPromptText = $"""
-                                            {systemPromptWithAugmentedData}
+            var profilePrompt = profile.ToSystemPrompt();
+            profileNote = $"Using profile '{profile.Name}' for chat thread '{this.Name}'.";
+            systemPromptText = $"""
+                                {systemPromptWithAugmentedData}
 
-                                            {profile.ToSystemPrompt()}
-                                            """;
-                    }
-                }
-            }
+                                {profilePrompt}
+                                """;
+
+            parts.Add(new(SystemPromptPartKind.PROFILE, profilePrompt) { SourceName = profile.Name });
         }
 
-        var toolPolicy = ToolSelectionRules.BuildToolPolicyPrompt(runnableToolDefinitions ?? []);
+        var instructedTools = ToolSelectionRules.GetInstructedTools(runnableToolDefinitions ?? []);
+        var toolPolicy = ToolSelectionRules.BuildToolPolicyPrompt(instructedTools);
         if (!string.IsNullOrWhiteSpace(toolPolicy))
         {
             systemPromptText = $"""
@@ -488,11 +475,13 @@ public sealed record ChatThread
 
                                 {toolPolicy}
                                 """;
+
+            parts.Add(new(SystemPromptPartKind.TOOL_INSTRUCTIONS, toolPolicy) { Tools = instructedTools });
         }
 
         var explanation = $"{logMessage} {profileNote}";
         if(!this.IncludeDateTime)
-            return new(systemPromptText, systemPromptTextWithChatTemplate, allowProfile, explanation);
+            return new(systemPromptText, systemPromptTextWithChatTemplate, explanation, parts, switchedOffProfile);
 
         //
         // Prepend the current date and time to the system prompt:
@@ -510,7 +499,49 @@ public sealed record ChatThread
                             {systemPromptText}
                             """;
 
-        return new(withDateTime, systemPromptTextWithChatTemplate, allowProfile, explanation);
+        parts.Insert(0, new(SystemPromptPartKind.DATE_TIME, currentDateTime));
+        return new(withDateTime, systemPromptTextWithChatTemplate, explanation, parts, switchedOffProfile);
+    }
+
+    /// <summary>
+    /// The chat template selected for this thread, or ChatTemplate.NO_CHAT_TEMPLATE when there is none.
+    /// </summary>
+    /// <remarks>
+    /// A selection which is no ID, the ID of no chat template, or that of a chat template which no
+    /// longer exists counts as none, so the thread falls back to its own system prompt.
+    /// </remarks>
+    /// <param name="settingsManager">The settings manager instance to use.</param>
+    /// <returns>The selected chat template, or ChatTemplate.NO_CHAT_TEMPLATE.</returns>
+    private ChatTemplate GetSelectedChatTemplate(SettingsManager settingsManager)
+    {
+        if (string.IsNullOrWhiteSpace(this.SelectedChatTemplate) || !Guid.TryParse(this.SelectedChatTemplate, out var chatTemplateId))
+            return ChatTemplate.NO_CHAT_TEMPLATE;
+
+        if (this.SelectedChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE.Id || chatTemplateId == Guid.Empty)
+            return ChatTemplate.NO_CHAT_TEMPLATE;
+
+        return settingsManager.GetChatTemplateById(this.SelectedChatTemplate);
+    }
+
+    /// <summary>
+    /// The profile selected for this thread, or Profile.NO_PROFILE when there is none.
+    /// </summary>
+    /// <remarks>
+    /// The same rules as for the chat template: a selection which is no ID, the ID of no profile, or
+    /// that of a profile which no longer exists counts as none. Whether the chat template lets the
+    /// profile take part is decided by BuildSystemPrompt.
+    /// </remarks>
+    /// <param name="settingsManager">The settings manager instance to use.</param>
+    /// <returns>The selected profile, or Profile.NO_PROFILE.</returns>
+    private Profile GetSelectedProfile(SettingsManager settingsManager)
+    {
+        if (string.IsNullOrWhiteSpace(this.SelectedProfile) || !Guid.TryParse(this.SelectedProfile, out var profileId))
+            return Profile.NO_PROFILE;
+
+        if (this.SelectedProfile == Profile.NO_PROFILE.Id || profileId == Guid.Empty)
+            return Profile.NO_PROFILE;
+
+        return settingsManager.GetProfileById(this.SelectedProfile);
     }
 
     /// <summary>

app/MindWork AI Studio/Chat/PreparedSystemPrompt.cs

@@ -1,3 +1,5 @@
+using AIStudio.Settings;
+
 namespace AIStudio.Chat;
 
 /// <summary>
@@ -7,13 +9,15 @@ namespace AIStudio.Chat;
 /// The system prompt is not the text a person typed into it. A chat template may replace it, the
 /// retrieved data of a data source is appended to it, a profile adds its own paragraph, the tool
 /// policy adds another, and the current date goes in front of everything. Whoever wants to know how
-/// long the next request is has to ask the same question the request does.
+/// long the next request is has to ask the same question the request does, and whoever shows the
+/// prompt reads its parts from here rather than repeating the decisions behind them.
 /// </remarks>
 /// <param name="Text">The whole system prompt, as the provider receives it.</param>
 /// <param name="BasePrompt">
 /// The prompt without any of the parts added around it. The thread keeps this one, so that it can
 /// still say which prompt it was configured with rather than the assembled result.
 /// </param>
-/// <param name="ProfileIsAllowed">Whether the chat template let a profile take part.</param>
 /// <param name="Explanation">What was used, in one sentence, for the log.</param>
-public sealed record PreparedSystemPrompt(string Text, string BasePrompt, bool ProfileIsAllowed, string Explanation);
+/// <param name="Parts">The parts the text is made of, in the order in which they stand in it.</param>
+/// <param name="SwitchedOffProfile">The profile the user selected but the chat template does not let take part, or Profile.NO_PROFILE when nothing was switched off.</param>
+public sealed record PreparedSystemPrompt(string Text, string BasePrompt, string Explanation, IReadOnlyList<SystemPromptPart> Parts, Profile SwitchedOffProfile);

app/MindWork AI Studio/Chat/SystemPromptPart.cs

@@ -0,0 +1,26 @@
+using AIStudio.Tools.ToolCallingSystem;
+
+namespace AIStudio.Chat;
+
+/// <summary>
+/// One part of the system prompt, as it stands in the text sent to the provider.
+/// </summary>
+/// <remarks>
+/// Recorded by ChatThread.BuildSystemPrompt while it assembles the text, so that whoever shows the
+/// prompt learns where each paragraph came from from the one place which decided it, instead of
+/// working it out again.
+/// </remarks>
+/// <param name="Kind">Where the part comes from.</param>
+/// <param name="Text">The text of the part, exactly as it stands in the system prompt.</param>
+public sealed record SystemPromptPart(SystemPromptPartKind Kind, string Text)
+{
+    /// <summary>
+    /// The name of the chat template or the profile the part comes from; empty for every other part.
+    /// </summary>
+    public string SourceName { get; init; } = string.Empty;
+
+    /// <summary>
+    /// The tools whose usage policy the part states; empty for every other part.
+    /// </summary>
+    public IReadOnlyList<ToolDefinition> Tools { get; init; } = [];
+}

app/MindWork AI Studio/Chat/SystemPromptPartKind.cs

@@ -0,0 +1,41 @@
+namespace AIStudio.Chat;
+
+/// <summary>
+/// Where a part of the system prompt comes from.
+/// </summary>
+/// <remarks>
+/// The members are listed in the order in which the parts stand in the system prompt, see
+/// ChatThread.BuildSystemPrompt.
+/// </remarks>
+public enum SystemPromptPartKind
+{
+    /// <summary>
+    /// The current date and time, in front of everything else.
+    /// </summary>
+    DATE_TIME,
+
+    /// <summary>
+    /// The prompt the chat was set up with, used when no chat template replaces it.
+    /// </summary>
+    BASE_PROMPT,
+
+    /// <summary>
+    /// The prompt of the chat template, which replaces the one the chat was set up with.
+    /// </summary>
+    CHAT_TEMPLATE,
+
+    /// <summary>
+    /// What the last request retrieved from the data sources of the chat.
+    /// </summary>
+    RETRIEVED_DATA,
+
+    /// <summary>
+    /// What the selected profile tells about the user.
+    /// </summary>
+    PROFILE,
+
+    /// <summary>
+    /// How to use the tools the request offers.
+    /// </summary>
+    TOOL_INSTRUCTIONS,
+}