mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-03-29 19:11:38 +00:00
added images as a descriptive component for the assistant builder
This commit is contained in:
parent
93e0fb4842
commit
49746a2c07
@ -15,6 +15,23 @@
|
||||
<MudTextField T="string" @bind-Text="@this.inputFields[textArea.Name]" Label="@textArea.Label" ReadOnly="@textArea.ReadOnly" AdornmentIcon="@Icons.Material.Filled.DocumentScanner" Adornment="Adornment.Start" Variant="Variant.Outlined" Lines="@lines" AutoGrow="@true" MaxLines="12" Class="mb-3"/>
|
||||
}
|
||||
break;
|
||||
case AssistantUiCompontentType.IMAGE:
|
||||
if (component is AssistantImage assistantImage)
|
||||
{
|
||||
var resolvedSource = this.ResolveImageSource(assistantImage);
|
||||
if (!string.IsNullOrWhiteSpace(resolvedSource))
|
||||
{
|
||||
var image = assistantImage;
|
||||
<div Class="mb-4">
|
||||
<MudImage Fluid="true" Src="@resolvedSource" Alt="@image.Alt" Class="rounded-lg mb-2" Elevation="20"/>
|
||||
@if (!string.IsNullOrWhiteSpace(image.Caption))
|
||||
{
|
||||
<MudText Typo="Typo.caption" Align="Align.Center">@image.Caption</MudText>
|
||||
}
|
||||
</div>
|
||||
}
|
||||
}
|
||||
break;
|
||||
case AssistantUiCompontentType.WEB_CONTENT_READER:
|
||||
if (component is AssistantWebContentReader webContent && this.webContentFields.TryGetValue(webContent.Name, out var webState))
|
||||
{
|
||||
|
||||
@ -1,4 +1,7 @@
|
||||
using AIStudio.Dialogs.Settings;
|
||||
using System;
|
||||
using System.IO;
|
||||
|
||||
using AIStudio.Dialogs.Settings;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools.PluginSystem.Assistants;
|
||||
@ -36,6 +39,9 @@ public partial class AssistantDynamic : AssistantBaseCore<SettingsDialogDynamic>
|
||||
private Dictionary<string, bool> switchFields = new();
|
||||
private Dictionary<string, WebContentState> webContentFields = new();
|
||||
private Dictionary<string, FileContentState> fileContentFields = new();
|
||||
private readonly Dictionary<string, string> imageCache = new();
|
||||
private string pluginPath = string.Empty;
|
||||
const string PLUGIN_SCHEME = "plugin://";
|
||||
|
||||
protected override void OnInitialized()
|
||||
{
|
||||
@ -50,6 +56,7 @@ public partial class AssistantDynamic : AssistantBaseCore<SettingsDialogDynamic>
|
||||
this.submitText = assistantPlugin.SubmitText;
|
||||
this.allowProfiles = assistantPlugin.AllowProfiles;
|
||||
this.showFooterProfileSelection = !assistantPlugin.HasEmbeddedProfileSelection;
|
||||
this.pluginPath = assistantPlugin.PluginPath;
|
||||
}
|
||||
|
||||
foreach (var component in this.RootComponent!.Children)
|
||||
@ -118,6 +125,61 @@ public partial class AssistantDynamic : AssistantBaseCore<SettingsDialogDynamic>
|
||||
return false;
|
||||
}
|
||||
|
||||
private string ResolveImageSource(AssistantImage image)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(image.Src))
|
||||
return string.Empty;
|
||||
|
||||
if (this.imageCache.TryGetValue(image.Src, out var cached) && !string.IsNullOrWhiteSpace(cached))
|
||||
return cached;
|
||||
|
||||
var resolved = image.Src;
|
||||
|
||||
if (resolved.StartsWith(PLUGIN_SCHEME, StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(this.pluginPath))
|
||||
{
|
||||
var relative = resolved[PLUGIN_SCHEME.Length..].TrimStart('/', '\\').Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar);
|
||||
var filePath = Path.Join(this.pluginPath, relative);
|
||||
if (File.Exists(filePath))
|
||||
{
|
||||
var mime = GetImageMimeType(filePath);
|
||||
var data = Convert.ToBase64String(File.ReadAllBytes(filePath));
|
||||
resolved = $"data:{mime};base64,{data}";
|
||||
}
|
||||
else
|
||||
{
|
||||
resolved = string.Empty;
|
||||
}
|
||||
}
|
||||
else if (Uri.TryCreate(resolved, UriKind.Absolute, out var uri))
|
||||
{
|
||||
if (uri.Scheme is not ("http" or "https" or "data"))
|
||||
resolved = string.Empty;
|
||||
}
|
||||
else
|
||||
{
|
||||
resolved = string.Empty;
|
||||
}
|
||||
|
||||
this.imageCache[image.Src] = resolved;
|
||||
return resolved;
|
||||
}
|
||||
|
||||
private static string GetImageMimeType(string path)
|
||||
{
|
||||
var extension = Path.GetExtension(path).TrimStart('.').ToLowerInvariant();
|
||||
return extension switch
|
||||
{
|
||||
"svg" => "image/svg+xml",
|
||||
"png" => "image/png",
|
||||
"jpg" => "image/jpeg",
|
||||
"jpeg" => "image/jpeg",
|
||||
"gif" => "image/gif",
|
||||
"webp" => "image/webp",
|
||||
"bmp" => "image/bmp",
|
||||
_ => "image/png",
|
||||
};
|
||||
}
|
||||
|
||||
private string? ValidateCustomLanguage(string value) => string.Empty;
|
||||
|
||||
private string CollectUserPrompt()
|
||||
|
||||
@ -15,8 +15,11 @@ Supported types (matching the Blazor UI components):
|
||||
- `PROVIDER_SELECTION` / `PROFILE_SELECTION`: hooks into the shared provider/profile selectors.
|
||||
- `WEB_CONTENT_READER`: renders `ReadWebContent`; include `Name`, `UserPrompt`, `Preselect`, `PreselectContentCleanerAgent`.
|
||||
- `FILE_CONTENT_READER`: renders `ReadFileContent`; include `Name`, `UserPrompt`.
|
||||
- `IMAGE`: embeds a static illustration; `Props` must include `Src` plus optionally `Alt` and `Caption`. `Src` can be an HTTP/HTTPS URL, a `data:` URI, or a plugin-relative path (`plugin://assets/your-image.png`). The runtime will convert plugin-relative paths into `data:` URLs (base64).
|
||||
- `HEADING`, `TEXT`, `LIST`: descriptive helpers.
|
||||
|
||||
Images referenced via the `plugin://` scheme must exist in the plugin directory (e.g., `assets/example.png`). Drop the file there and point `Src` at it. The component will read the file at runtime, encode it as Base64, and render it inside the assistant UI.
|
||||
|
||||
## Prompt Assembly
|
||||
Each component exposes a `UserPrompt` string. When the assistant runs, `AssistantDynamic` iterates over `RootComponent.Children` and, for each component that has a prompt, emits:
|
||||
|
||||
@ -37,5 +40,3 @@ For switches the “value” is the boolean `true/false`; for readers it is the
|
||||
3. When you expect default content (e.g., a textarea with instructions), keep `UserPrompt` but also set `PrefillText` so the user starts with a hint.
|
||||
4. If you need extra explanatory text (before or after the interactive controls), use `TEXT` or `HEADING` components.
|
||||
5. Keep `Preselect`/`PreselectContentCleanerAgent` flags in `WEB_CONTENT_READER` to simplify the initial UI for the user.
|
||||
|
||||
The sample `plugin.lua` in this directory is the live reference. Adjust it, reload the assistant plugin via the desktop app, and verify that the prompt log contains the blocked `context`/`user prompt` pairs that you expect.
|
||||
|
||||
@ -144,6 +144,14 @@ ASSISTANT = {
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
["Type"] = "IMAGE",
|
||||
["Props"] = {
|
||||
["Src"] = "plugin://assets/example.png",
|
||||
["Alt"] = "SVG-inspired placeholder",
|
||||
["Caption"] = "Static illustration via the IMAGE component."
|
||||
}
|
||||
},
|
||||
{
|
||||
["Type"] = "WEB_CONTENT_READER", -- allows the user to fetch a URL and clean it
|
||||
["Props"] = {
|
||||
|
||||
@ -37,6 +37,8 @@ public class AssistantComponentFactory
|
||||
return new AssistantWebContentReader { Props = props, Children = children };
|
||||
case AssistantUiCompontentType.FILE_CONTENT_READER:
|
||||
return new AssistantFileContentReader { Props = props, Children = children };
|
||||
case AssistantUiCompontentType.IMAGE:
|
||||
return new AssistantImage { Props = props, Children = children };
|
||||
default:
|
||||
LOGGER.LogError($"Unknown assistant component type!\n{type} is not a supported assistant component type");
|
||||
throw new Exception($"Unknown assistant component type: {type}");
|
||||
|
||||
@ -0,0 +1,32 @@
|
||||
namespace AIStudio.Tools.PluginSystem.Assistants.DataModel;
|
||||
|
||||
public class AssistantImage : AssistantComponentBase
|
||||
{
|
||||
public override AssistantUiCompontentType Type => AssistantUiCompontentType.IMAGE;
|
||||
public Dictionary<string, object> Props { get; set; } = new();
|
||||
public List<IAssistantComponent> Children { get; set; } = new();
|
||||
|
||||
public string Src
|
||||
{
|
||||
get => this.Props.TryGetValue(nameof(this.Src), out var v)
|
||||
? v.ToString() ?? string.Empty
|
||||
: string.Empty;
|
||||
set => this.Props[nameof(this.Src)] = value;
|
||||
}
|
||||
|
||||
public string Alt
|
||||
{
|
||||
get => this.Props.TryGetValue(nameof(this.Alt), out var v)
|
||||
? v.ToString() ?? string.Empty
|
||||
: string.Empty;
|
||||
set => this.Props[nameof(this.Alt)] = value;
|
||||
}
|
||||
|
||||
public string Caption
|
||||
{
|
||||
get => this.Props.TryGetValue(nameof(this.Caption), out var v)
|
||||
? v.ToString() ?? string.Empty
|
||||
: string.Empty;
|
||||
set => this.Props[nameof(this.Caption)] = value;
|
||||
}
|
||||
}
|
||||
@ -14,4 +14,5 @@ public enum AssistantUiCompontentType
|
||||
LIST,
|
||||
WEB_CONTENT_READER,
|
||||
FILE_CONTENT_READER,
|
||||
IMAGE,
|
||||
}
|
||||
|
||||
@ -53,5 +53,9 @@ public static class ComponentPropSpecs
|
||||
required: ["Name"],
|
||||
optional: ["UserPrompt"]
|
||||
),
|
||||
[AssistantUiCompontentType.IMAGE] = new(
|
||||
required: ["Src"],
|
||||
optional: ["Alt", "Caption"]
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user