mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-03-29 23:31:40 +00:00
added images as a descriptive component for the assistant builder
This commit is contained in:
parent
93e0fb4842
commit
49746a2c07
@ -15,6 +15,23 @@
|
|||||||
<MudTextField T="string" @bind-Text="@this.inputFields[textArea.Name]" Label="@textArea.Label" ReadOnly="@textArea.ReadOnly" AdornmentIcon="@Icons.Material.Filled.DocumentScanner" Adornment="Adornment.Start" Variant="Variant.Outlined" Lines="@lines" AutoGrow="@true" MaxLines="12" Class="mb-3"/>
|
<MudTextField T="string" @bind-Text="@this.inputFields[textArea.Name]" Label="@textArea.Label" ReadOnly="@textArea.ReadOnly" AdornmentIcon="@Icons.Material.Filled.DocumentScanner" Adornment="Adornment.Start" Variant="Variant.Outlined" Lines="@lines" AutoGrow="@true" MaxLines="12" Class="mb-3"/>
|
||||||
}
|
}
|
||||||
break;
|
break;
|
||||||
|
case AssistantUiCompontentType.IMAGE:
|
||||||
|
if (component is AssistantImage assistantImage)
|
||||||
|
{
|
||||||
|
var resolvedSource = this.ResolveImageSource(assistantImage);
|
||||||
|
if (!string.IsNullOrWhiteSpace(resolvedSource))
|
||||||
|
{
|
||||||
|
var image = assistantImage;
|
||||||
|
<div Class="mb-4">
|
||||||
|
<MudImage Fluid="true" Src="@resolvedSource" Alt="@image.Alt" Class="rounded-lg mb-2" Elevation="20"/>
|
||||||
|
@if (!string.IsNullOrWhiteSpace(image.Caption))
|
||||||
|
{
|
||||||
|
<MudText Typo="Typo.caption" Align="Align.Center">@image.Caption</MudText>
|
||||||
|
}
|
||||||
|
</div>
|
||||||
|
}
|
||||||
|
}
|
||||||
|
break;
|
||||||
case AssistantUiCompontentType.WEB_CONTENT_READER:
|
case AssistantUiCompontentType.WEB_CONTENT_READER:
|
||||||
if (component is AssistantWebContentReader webContent && this.webContentFields.TryGetValue(webContent.Name, out var webState))
|
if (component is AssistantWebContentReader webContent && this.webContentFields.TryGetValue(webContent.Name, out var webState))
|
||||||
{
|
{
|
||||||
|
|||||||
@ -1,4 +1,7 @@
|
|||||||
using AIStudio.Dialogs.Settings;
|
using System;
|
||||||
|
using System.IO;
|
||||||
|
|
||||||
|
using AIStudio.Dialogs.Settings;
|
||||||
using AIStudio.Tools.PluginSystem;
|
using AIStudio.Tools.PluginSystem;
|
||||||
using AIStudio.Settings;
|
using AIStudio.Settings;
|
||||||
using AIStudio.Tools.PluginSystem.Assistants;
|
using AIStudio.Tools.PluginSystem.Assistants;
|
||||||
@ -36,6 +39,9 @@ public partial class AssistantDynamic : AssistantBaseCore<SettingsDialogDynamic>
|
|||||||
private Dictionary<string, bool> switchFields = new();
|
private Dictionary<string, bool> switchFields = new();
|
||||||
private Dictionary<string, WebContentState> webContentFields = new();
|
private Dictionary<string, WebContentState> webContentFields = new();
|
||||||
private Dictionary<string, FileContentState> fileContentFields = new();
|
private Dictionary<string, FileContentState> fileContentFields = new();
|
||||||
|
private readonly Dictionary<string, string> imageCache = new();
|
||||||
|
private string pluginPath = string.Empty;
|
||||||
|
const string PLUGIN_SCHEME = "plugin://";
|
||||||
|
|
||||||
protected override void OnInitialized()
|
protected override void OnInitialized()
|
||||||
{
|
{
|
||||||
@ -50,6 +56,7 @@ public partial class AssistantDynamic : AssistantBaseCore<SettingsDialogDynamic>
|
|||||||
this.submitText = assistantPlugin.SubmitText;
|
this.submitText = assistantPlugin.SubmitText;
|
||||||
this.allowProfiles = assistantPlugin.AllowProfiles;
|
this.allowProfiles = assistantPlugin.AllowProfiles;
|
||||||
this.showFooterProfileSelection = !assistantPlugin.HasEmbeddedProfileSelection;
|
this.showFooterProfileSelection = !assistantPlugin.HasEmbeddedProfileSelection;
|
||||||
|
this.pluginPath = assistantPlugin.PluginPath;
|
||||||
}
|
}
|
||||||
|
|
||||||
foreach (var component in this.RootComponent!.Children)
|
foreach (var component in this.RootComponent!.Children)
|
||||||
@ -118,6 +125,61 @@ public partial class AssistantDynamic : AssistantBaseCore<SettingsDialogDynamic>
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private string ResolveImageSource(AssistantImage image)
|
||||||
|
{
|
||||||
|
if (string.IsNullOrWhiteSpace(image.Src))
|
||||||
|
return string.Empty;
|
||||||
|
|
||||||
|
if (this.imageCache.TryGetValue(image.Src, out var cached) && !string.IsNullOrWhiteSpace(cached))
|
||||||
|
return cached;
|
||||||
|
|
||||||
|
var resolved = image.Src;
|
||||||
|
|
||||||
|
if (resolved.StartsWith(PLUGIN_SCHEME, StringComparison.OrdinalIgnoreCase) && !string.IsNullOrWhiteSpace(this.pluginPath))
|
||||||
|
{
|
||||||
|
var relative = resolved[PLUGIN_SCHEME.Length..].TrimStart('/', '\\').Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar);
|
||||||
|
var filePath = Path.Join(this.pluginPath, relative);
|
||||||
|
if (File.Exists(filePath))
|
||||||
|
{
|
||||||
|
var mime = GetImageMimeType(filePath);
|
||||||
|
var data = Convert.ToBase64String(File.ReadAllBytes(filePath));
|
||||||
|
resolved = $"data:{mime};base64,{data}";
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
resolved = string.Empty;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
else if (Uri.TryCreate(resolved, UriKind.Absolute, out var uri))
|
||||||
|
{
|
||||||
|
if (uri.Scheme is not ("http" or "https" or "data"))
|
||||||
|
resolved = string.Empty;
|
||||||
|
}
|
||||||
|
else
|
||||||
|
{
|
||||||
|
resolved = string.Empty;
|
||||||
|
}
|
||||||
|
|
||||||
|
this.imageCache[image.Src] = resolved;
|
||||||
|
return resolved;
|
||||||
|
}
|
||||||
|
|
||||||
|
private static string GetImageMimeType(string path)
|
||||||
|
{
|
||||||
|
var extension = Path.GetExtension(path).TrimStart('.').ToLowerInvariant();
|
||||||
|
return extension switch
|
||||||
|
{
|
||||||
|
"svg" => "image/svg+xml",
|
||||||
|
"png" => "image/png",
|
||||||
|
"jpg" => "image/jpeg",
|
||||||
|
"jpeg" => "image/jpeg",
|
||||||
|
"gif" => "image/gif",
|
||||||
|
"webp" => "image/webp",
|
||||||
|
"bmp" => "image/bmp",
|
||||||
|
_ => "image/png",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
private string? ValidateCustomLanguage(string value) => string.Empty;
|
private string? ValidateCustomLanguage(string value) => string.Empty;
|
||||||
|
|
||||||
private string CollectUserPrompt()
|
private string CollectUserPrompt()
|
||||||
|
|||||||
@ -15,8 +15,11 @@ Supported types (matching the Blazor UI components):
|
|||||||
- `PROVIDER_SELECTION` / `PROFILE_SELECTION`: hooks into the shared provider/profile selectors.
|
- `PROVIDER_SELECTION` / `PROFILE_SELECTION`: hooks into the shared provider/profile selectors.
|
||||||
- `WEB_CONTENT_READER`: renders `ReadWebContent`; include `Name`, `UserPrompt`, `Preselect`, `PreselectContentCleanerAgent`.
|
- `WEB_CONTENT_READER`: renders `ReadWebContent`; include `Name`, `UserPrompt`, `Preselect`, `PreselectContentCleanerAgent`.
|
||||||
- `FILE_CONTENT_READER`: renders `ReadFileContent`; include `Name`, `UserPrompt`.
|
- `FILE_CONTENT_READER`: renders `ReadFileContent`; include `Name`, `UserPrompt`.
|
||||||
|
- `IMAGE`: embeds a static illustration; `Props` must include `Src` plus optionally `Alt` and `Caption`. `Src` can be an HTTP/HTTPS URL, a `data:` URI, or a plugin-relative path (`plugin://assets/your-image.png`). The runtime will convert plugin-relative paths into `data:` URLs (base64).
|
||||||
- `HEADING`, `TEXT`, `LIST`: descriptive helpers.
|
- `HEADING`, `TEXT`, `LIST`: descriptive helpers.
|
||||||
|
|
||||||
|
Images referenced via the `plugin://` scheme must exist in the plugin directory (e.g., `assets/example.png`). Drop the file there and point `Src` at it. The component will read the file at runtime, encode it as Base64, and render it inside the assistant UI.
|
||||||
|
|
||||||
## Prompt Assembly
|
## Prompt Assembly
|
||||||
Each component exposes a `UserPrompt` string. When the assistant runs, `AssistantDynamic` iterates over `RootComponent.Children` and, for each component that has a prompt, emits:
|
Each component exposes a `UserPrompt` string. When the assistant runs, `AssistantDynamic` iterates over `RootComponent.Children` and, for each component that has a prompt, emits:
|
||||||
|
|
||||||
@ -37,5 +40,3 @@ For switches the “value” is the boolean `true/false`; for readers it is the
|
|||||||
3. When you expect default content (e.g., a textarea with instructions), keep `UserPrompt` but also set `PrefillText` so the user starts with a hint.
|
3. When you expect default content (e.g., a textarea with instructions), keep `UserPrompt` but also set `PrefillText` so the user starts with a hint.
|
||||||
4. If you need extra explanatory text (before or after the interactive controls), use `TEXT` or `HEADING` components.
|
4. If you need extra explanatory text (before or after the interactive controls), use `TEXT` or `HEADING` components.
|
||||||
5. Keep `Preselect`/`PreselectContentCleanerAgent` flags in `WEB_CONTENT_READER` to simplify the initial UI for the user.
|
5. Keep `Preselect`/`PreselectContentCleanerAgent` flags in `WEB_CONTENT_READER` to simplify the initial UI for the user.
|
||||||
|
|
||||||
The sample `plugin.lua` in this directory is the live reference. Adjust it, reload the assistant plugin via the desktop app, and verify that the prompt log contains the blocked `context`/`user prompt` pairs that you expect.
|
|
||||||
|
|||||||
@ -144,6 +144,14 @@ ASSISTANT = {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
["Type"] = "IMAGE",
|
||||||
|
["Props"] = {
|
||||||
|
["Src"] = "plugin://assets/example.png",
|
||||||
|
["Alt"] = "SVG-inspired placeholder",
|
||||||
|
["Caption"] = "Static illustration via the IMAGE component."
|
||||||
|
}
|
||||||
|
},
|
||||||
{
|
{
|
||||||
["Type"] = "WEB_CONTENT_READER", -- allows the user to fetch a URL and clean it
|
["Type"] = "WEB_CONTENT_READER", -- allows the user to fetch a URL and clean it
|
||||||
["Props"] = {
|
["Props"] = {
|
||||||
|
|||||||
@ -37,6 +37,8 @@ public class AssistantComponentFactory
|
|||||||
return new AssistantWebContentReader { Props = props, Children = children };
|
return new AssistantWebContentReader { Props = props, Children = children };
|
||||||
case AssistantUiCompontentType.FILE_CONTENT_READER:
|
case AssistantUiCompontentType.FILE_CONTENT_READER:
|
||||||
return new AssistantFileContentReader { Props = props, Children = children };
|
return new AssistantFileContentReader { Props = props, Children = children };
|
||||||
|
case AssistantUiCompontentType.IMAGE:
|
||||||
|
return new AssistantImage { Props = props, Children = children };
|
||||||
default:
|
default:
|
||||||
LOGGER.LogError($"Unknown assistant component type!\n{type} is not a supported assistant component type");
|
LOGGER.LogError($"Unknown assistant component type!\n{type} is not a supported assistant component type");
|
||||||
throw new Exception($"Unknown assistant component type: {type}");
|
throw new Exception($"Unknown assistant component type: {type}");
|
||||||
|
|||||||
@ -0,0 +1,32 @@
|
|||||||
|
namespace AIStudio.Tools.PluginSystem.Assistants.DataModel;
|
||||||
|
|
||||||
|
public class AssistantImage : AssistantComponentBase
|
||||||
|
{
|
||||||
|
public override AssistantUiCompontentType Type => AssistantUiCompontentType.IMAGE;
|
||||||
|
public Dictionary<string, object> Props { get; set; } = new();
|
||||||
|
public List<IAssistantComponent> Children { get; set; } = new();
|
||||||
|
|
||||||
|
public string Src
|
||||||
|
{
|
||||||
|
get => this.Props.TryGetValue(nameof(this.Src), out var v)
|
||||||
|
? v.ToString() ?? string.Empty
|
||||||
|
: string.Empty;
|
||||||
|
set => this.Props[nameof(this.Src)] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public string Alt
|
||||||
|
{
|
||||||
|
get => this.Props.TryGetValue(nameof(this.Alt), out var v)
|
||||||
|
? v.ToString() ?? string.Empty
|
||||||
|
: string.Empty;
|
||||||
|
set => this.Props[nameof(this.Alt)] = value;
|
||||||
|
}
|
||||||
|
|
||||||
|
public string Caption
|
||||||
|
{
|
||||||
|
get => this.Props.TryGetValue(nameof(this.Caption), out var v)
|
||||||
|
? v.ToString() ?? string.Empty
|
||||||
|
: string.Empty;
|
||||||
|
set => this.Props[nameof(this.Caption)] = value;
|
||||||
|
}
|
||||||
|
}
|
||||||
@ -14,4 +14,5 @@ public enum AssistantUiCompontentType
|
|||||||
LIST,
|
LIST,
|
||||||
WEB_CONTENT_READER,
|
WEB_CONTENT_READER,
|
||||||
FILE_CONTENT_READER,
|
FILE_CONTENT_READER,
|
||||||
|
IMAGE,
|
||||||
}
|
}
|
||||||
|
|||||||
@ -53,5 +53,9 @@ public static class ComponentPropSpecs
|
|||||||
required: ["Name"],
|
required: ["Name"],
|
||||||
optional: ["UserPrompt"]
|
optional: ["UserPrompt"]
|
||||||
),
|
),
|
||||||
|
[AssistantUiCompontentType.IMAGE] = new(
|
||||||
|
required: ["Src"],
|
||||||
|
optional: ["Alt", "Caption"]
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user