added images as a descriptive component for the assistant builder

This commit is contained in:
Nils Kruthoff 2026-02-24 11:31:16 +01:00
parent 93e0fb4842
commit 49746a2c07
No known key found for this signature in database
GPG Key ID: A5C0151B4DDB172C
8 changed files with 131 additions and 4 deletions

View File

@ -15,6 +15,23 @@
<MudTextField T="string" @bind-Text="@this.inputFields[textArea.Name]" Label="@textArea.Label" ReadOnly="@textArea.ReadOnly" AdornmentIcon="@Icons.Material.Filled.DocumentScanner" Adornment="Adornment.Start" Variant="Variant.Outlined" Lines="@lines" AutoGrow="@true" MaxLines="12" Class="mb-3"/> <MudTextField T="string" @bind-Text="@this.inputFields[textArea.Name]" Label="@textArea.Label" ReadOnly="@textArea.ReadOnly" AdornmentIcon="@Icons.Material.Filled.DocumentScanner" Adornment="Adornment.Start" Variant="Variant.Outlined" Lines="@lines" AutoGrow="@true" MaxLines="12" Class="mb-3"/>
} }
break; break;
case AssistantUiCompontentType.IMAGE:
if (component is AssistantImage assistantImage)
{
var resolvedSource = this.ResolveImageSource(assistantImage);
if (!string.IsNullOrWhiteSpace(resolvedSource))
{
var image = assistantImage;
<div Class="mb-4">
<MudImage Fluid="true" Src="@resolvedSource" Alt="@image.Alt" Class="rounded-lg mb-2" Elevation="20"/>
@if (!string.IsNullOrWhiteSpace(image.Caption))
{
<MudText Typo="Typo.caption" Align="Align.Center">@image.Caption</MudText>
}
</div>
}
}
break;
case AssistantUiCompontentType.WEB_CONTENT_READER: case AssistantUiCompontentType.WEB_CONTENT_READER:
if (component is AssistantWebContentReader webContent && this.webContentFields.TryGetValue(webContent.Name, out var webState)) if (component is AssistantWebContentReader webContent && this.webContentFields.TryGetValue(webContent.Name, out var webState))
{ {

View File

@ -1,4 +1,7 @@
using AIStudio.Dialogs.Settings; using System;
using System.IO;
using AIStudio.Dialogs.Settings;
using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem;
using AIStudio.Settings; using AIStudio.Settings;
using AIStudio.Tools.PluginSystem.Assistants; using AIStudio.Tools.PluginSystem.Assistants;
@ -36,6 +39,9 @@ public partial class AssistantDynamic : AssistantBaseCore<SettingsDialogDynamic>
private Dictionary<string, bool> switchFields = new(); private Dictionary<string, bool> switchFields = new();
private Dictionary<string, WebContentState> webContentFields = new(); private Dictionary<string, WebContentState> webContentFields = new();
private Dictionary<string, FileContentState> fileContentFields = new(); private Dictionary<string, FileContentState> fileContentFields = new();
private readonly Dictionary<string, string> imageCache = new();
private string pluginPath = string.Empty;
const string PLUGIN_SCHEME = "plugin://";
protected override void OnInitialized() protected override void OnInitialized()
{ {
@ -50,6 +56,7 @@ public partial class AssistantDynamic : AssistantBaseCore<SettingsDialogDynamic>
this.submitText = assistantPlugin.SubmitText; this.submitText = assistantPlugin.SubmitText;
this.allowProfiles = assistantPlugin.AllowProfiles; this.allowProfiles = assistantPlugin.AllowProfiles;
this.showFooterProfileSelection = !assistantPlugin.HasEmbeddedProfileSelection; this.showFooterProfileSelection = !assistantPlugin.HasEmbeddedProfileSelection;
this.pluginPath = assistantPlugin.PluginPath;
} }
foreach (var component in this.RootComponent!.Children) foreach (var component in this.RootComponent!.Children)
@ -118,6 +125,61 @@ public partial class AssistantDynamic : AssistantBaseCore<SettingsDialogDynamic>
return false; return false;
} }
private string ResolveImageSource(AssistantImage image)
{
if (string.IsNullOrWhiteSpace(image.Src))
return string.Empty;
if (this.imageCache.TryGetValue(image.Src, out var cached) && !string.IsNullOrWhiteSpace(cached))
return cached;
var resolved = image.Src;
if (resolved.StartsWith(PLUGIN_SCHEME, StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(this.pluginPath))
{
var relative = resolved[PLUGIN_SCHEME.Length..].TrimStart('/', '\\').Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar);
var filePath = Path.Join(this.pluginPath, relative);
if (File.Exists(filePath))
{
var mime = GetImageMimeType(filePath);
var data = Convert.ToBase64String(File.ReadAllBytes(filePath));
resolved = $"data:{mime};base64,{data}";
}
else
{
resolved = string.Empty;
}
}
else if (Uri.TryCreate(resolved, UriKind.Absolute, out var uri))
{
if (uri.Scheme is not ("http" or "https" or "data"))
resolved = string.Empty;
}
else
{
resolved = string.Empty;
}
this.imageCache[image.Src] = resolved;
return resolved;
}
private static string GetImageMimeType(string path)
{
var extension = Path.GetExtension(path).TrimStart('.').ToLowerInvariant();
return extension switch
{
"svg" => "image/svg+xml",
"png" => "image/png",
"jpg" => "image/jpeg",
"jpeg" => "image/jpeg",
"gif" => "image/gif",
"webp" => "image/webp",
"bmp" => "image/bmp",
_ => "image/png",
};
}
private string? ValidateCustomLanguage(string value) => string.Empty; private string? ValidateCustomLanguage(string value) => string.Empty;
private string CollectUserPrompt() private string CollectUserPrompt()

View File

@ -15,8 +15,11 @@ Supported types (matching the Blazor UI components):
- `PROVIDER_SELECTION` / `PROFILE_SELECTION`: hooks into the shared provider/profile selectors. - `PROVIDER_SELECTION` / `PROFILE_SELECTION`: hooks into the shared provider/profile selectors.
- `WEB_CONTENT_READER`: renders `ReadWebContent`; include `Name`, `UserPrompt`, `Preselect`, `PreselectContentCleanerAgent`. - `WEB_CONTENT_READER`: renders `ReadWebContent`; include `Name`, `UserPrompt`, `Preselect`, `PreselectContentCleanerAgent`.
- `FILE_CONTENT_READER`: renders `ReadFileContent`; include `Name`, `UserPrompt`. - `FILE_CONTENT_READER`: renders `ReadFileContent`; include `Name`, `UserPrompt`.
- `IMAGE`: embeds a static illustration; `Props` must include `Src` plus optionally `Alt` and `Caption`. `Src` can be an HTTP/HTTPS URL, a `data:` URI, or a plugin-relative path (`plugin://assets/your-image.png`). The runtime will convert plugin-relative paths into `data:` URLs (base64).
- `HEADING`, `TEXT`, `LIST`: descriptive helpers. - `HEADING`, `TEXT`, `LIST`: descriptive helpers.
Images referenced via the `plugin://` scheme must exist in the plugin directory (e.g., `assets/example.png`). Drop the file there and point `Src` at it. The component will read the file at runtime, encode it as Base64, and render it inside the assistant UI.
## Prompt Assembly ## Prompt Assembly
Each component exposes a `UserPrompt` string. When the assistant runs, `AssistantDynamic` iterates over `RootComponent.Children` and, for each component that has a prompt, emits: Each component exposes a `UserPrompt` string. When the assistant runs, `AssistantDynamic` iterates over `RootComponent.Children` and, for each component that has a prompt, emits:
@ -37,5 +40,3 @@ For switches the “value” is the boolean `true/false`; for readers it is the
3. When you expect default content (e.g., a textarea with instructions), keep `UserPrompt` but also set `PrefillText` so the user starts with a hint. 3. When you expect default content (e.g., a textarea with instructions), keep `UserPrompt` but also set `PrefillText` so the user starts with a hint.
4. If you need extra explanatory text (before or after the interactive controls), use `TEXT` or `HEADING` components. 4. If you need extra explanatory text (before or after the interactive controls), use `TEXT` or `HEADING` components.
5. Keep `Preselect`/`PreselectContentCleanerAgent` flags in `WEB_CONTENT_READER` to simplify the initial UI for the user. 5. Keep `Preselect`/`PreselectContentCleanerAgent` flags in `WEB_CONTENT_READER` to simplify the initial UI for the user.
The sample `plugin.lua` in this directory is the live reference. Adjust it, reload the assistant plugin via the desktop app, and verify that the prompt log contains the blocked `context`/`user prompt` pairs that you expect.

View File

@ -144,6 +144,14 @@ ASSISTANT = {
} }
} }
}, },
{
["Type"] = "IMAGE",
["Props"] = {
["Src"] = "plugin://assets/example.png",
["Alt"] = "SVG-inspired placeholder",
["Caption"] = "Static illustration via the IMAGE component."
}
},
{ {
["Type"] = "WEB_CONTENT_READER", -- allows the user to fetch a URL and clean it ["Type"] = "WEB_CONTENT_READER", -- allows the user to fetch a URL and clean it
["Props"] = { ["Props"] = {

View File

@ -37,6 +37,8 @@ public class AssistantComponentFactory
return new AssistantWebContentReader { Props = props, Children = children }; return new AssistantWebContentReader { Props = props, Children = children };
case AssistantUiCompontentType.FILE_CONTENT_READER: case AssistantUiCompontentType.FILE_CONTENT_READER:
return new AssistantFileContentReader { Props = props, Children = children }; return new AssistantFileContentReader { Props = props, Children = children };
case AssistantUiCompontentType.IMAGE:
return new AssistantImage { Props = props, Children = children };
default: default:
LOGGER.LogError($"Unknown assistant component type!\n{type} is not a supported assistant component type"); LOGGER.LogError($"Unknown assistant component type!\n{type} is not a supported assistant component type");
throw new Exception($"Unknown assistant component type: {type}"); throw new Exception($"Unknown assistant component type: {type}");

View File

@ -0,0 +1,32 @@
namespace AIStudio.Tools.PluginSystem.Assistants.DataModel;
public class AssistantImage : AssistantComponentBase
{
public override AssistantUiCompontentType Type => AssistantUiCompontentType.IMAGE;
public Dictionary<string, object> Props { get; set; } = new();
public List<IAssistantComponent> Children { get; set; } = new();
public string Src
{
get => this.Props.TryGetValue(nameof(this.Src), out var v)
? v.ToString() ?? string.Empty
: string.Empty;
set => this.Props[nameof(this.Src)] = value;
}
public string Alt
{
get => this.Props.TryGetValue(nameof(this.Alt), out var v)
? v.ToString() ?? string.Empty
: string.Empty;
set => this.Props[nameof(this.Alt)] = value;
}
public string Caption
{
get => this.Props.TryGetValue(nameof(this.Caption), out var v)
? v.ToString() ?? string.Empty
: string.Empty;
set => this.Props[nameof(this.Caption)] = value;
}
}

View File

@ -14,4 +14,5 @@ public enum AssistantUiCompontentType
LIST, LIST,
WEB_CONTENT_READER, WEB_CONTENT_READER,
FILE_CONTENT_READER, FILE_CONTENT_READER,
IMAGE,
} }

View File

@ -53,5 +53,9 @@ public static class ComponentPropSpecs
required: ["Name"], required: ["Name"],
optional: ["UserPrompt"] optional: ["UserPrompt"]
), ),
[AssistantUiCompontentType.IMAGE] = new(
required: ["Src"],
optional: ["Alt", "Caption"]
),
}; };
} }