MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-11-11 03:47:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Moved all extension methods into extension blocks (#1155)

Browse Source
This commit is contained in:
Thorsten Sommer authored and GitHub committed 2026-11-04 10:46:18 +01:00
1 parent acc7d09fa2
commit c0a75f39ac
138 files changed
+5884 -4954
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Chat/ChatProvenance.cs
+23 -20
View File
@@ -16,31 +16,34 @@ public static class ChatProvenance
{
private static readonly ILogger LOGGER = Program.LOGGER_FACTORY.CreateLogger(nameof(ChatProvenance));
/// <summary>
/// Collects the entries of the provenance store which a chat refers to.
/// </summary>
/// <remarks>
/// The passages of its answers, the documents attached to its messages, and those attached to
/// the message being written, which may come from a message sent before, e.g., one being edited,
/// together with the texts a preview read of them.
/// </remarks>
/// <param name="thread">The chat.</param>
/// <returns>The ids of the entries.</returns>
public static IReadOnlySet<Guid> CollectProvenanceIds(this ChatThread thread)
extension(ChatThread thread)
{
var provenanceIds = new HashSet<Guid>();
foreach (var block in thread.Blocks)
/// <summary>
/// Collects the entries of the provenance store which a chat refers to.
/// </summary>
/// <remarks>
/// The passages of its answers, the documents attached to its messages, and those attached to
/// the message being written, which may come from a message sent before, e.g., one being edited,
/// together with the texts a preview read of them.
/// </remarks>
/// <returns>The ids of the entries.</returns>
public IReadOnlySet<Guid> CollectProvenanceIds()
{
if (block.Content is ContentText text)
provenanceIds.UnionWith(text.SourcePassages.Select(passage => passage.ProvenanceId).OfType<Guid>());
var provenanceIds = new HashSet<Guid>();
foreach (var block in thread.Blocks)
{
if (block.Content is ContentText text)
provenanceIds.UnionWith(text.SourcePassages.Select(passage => passage.ProvenanceId).OfType<Guid>());
if (block.Content is not null)
provenanceIds.UnionWith(block.Content.FileAttachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>());
if (block.Content is not null)
provenanceIds.UnionWith(block.Content.FileAttachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>());
}
provenanceIds.UnionWith(thread.Draft.FileAttachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>());
provenanceIds.UnionWith(thread.Draft.PreviewedTexts.Select(text => text.ProvenanceId));
return provenanceIds;
}
provenanceIds.UnionWith(thread.Draft.FileAttachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>());
provenanceIds.UnionWith(thread.Draft.PreviewedTexts.Select(text => text.ProvenanceId));
return provenanceIds;
}
/// <summary>
app/MindWork AI Studio/Chat/ChatRestrictionsExtensions.cs
+59 -56
View File
@@ -6,72 +6,75 @@ namespace AIStudio.Chat;
public static class ChatRestrictionsExtensions
{
/// <summary>
/// Checks if the specified provider may see content with these restrictions.
/// </summary>
/// <remarks>
/// A provider the organization prohibited is never allowed, whatever the restrictions.<br/><br/>
///
/// We don't check if the provider is allowed to use the data sources the content came from.
/// That kind of check is done when the available data sources are resolved. What counts here is
/// what the content demands since it holds data of them, see ChatThreadExtensions.IsLLMProviderAllowed.
/// </remarks>
/// <param name="restrictions">What the content demands.</param>
/// <param name="provider">The provider to check.</param>
/// <returns>True, when the provider may see the content. False, otherwise.</returns>
public static bool IsLLMProviderAllowed<T>(this ChatRestrictions restrictions, T provider)
extension(ChatRestrictions restrictions)
{
var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
// A provider the organization prohibited is allowed nowhere, not even for content which demands nothing:
if (settingsManager.IsProviderProhibited(GetLLMProvider(provider)))
return false;
var providerConfidence = provider switch
/// <summary>
/// Checks if the specified provider may see content with these restrictions.
/// </summary>
/// <remarks>
/// A provider the organization prohibited is never allowed, whatever the restrictions.<br/><br/>
///
/// We don't check if the provider is allowed to use the data sources the content came from.
/// That kind of check is done when the available data sources are resolved. What counts here is
/// what the content demands since it holds data of them, see ChatThreadExtensions.IsLLMProviderAllowed.
/// </remarks>
/// <param name="provider">The provider to check.</param>
/// <returns>True, when the provider may see the content. False, otherwise.</returns>
public bool IsLLMProviderAllowed<T>(T provider)
{
IProvider p => p.GetConfidenceLevel(settingsManager),
AIStudio.Settings.Provider p => p.GetConfidenceLevel(settingsManager),
var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
_ => ConfidenceLevel.UNKNOWN,
};
// A provider the organization prohibited is allowed nowhere, not even for content which demands nothing:
if (settingsManager.IsProviderProhibited(GetLLMProvider(provider)))
return false;
//
// The confidence axis is checked on its own: a provider trusted by configuration counts as
// self-hosted for data-source security, which is the check further down, but that trust
// says nothing about how confidential the provider is. An organization which wants its
// contractually covered cloud provider to pass here raises its level through the custom
// confidence scheme instead.
//
if (providerConfidence < restrictions.ProviderConfidence)
return false;
var providerConfidence = provider switch
{
IProvider p => p.GetConfidenceLevel(settingsManager),
AIStudio.Settings.Provider p => p.GetConfidenceLevel(settingsManager),
// The data security is not specified. Means, the content never came from RAG, or RAG was
// enabled, but no data sources were selected. That's fine:
if (restrictions.DataSecurity is DataSourceSecurity.NOT_SPECIFIED)
return true;
_ => ConfidenceLevel.UNKNOWN,
};
//
// Is the provider trusted for data-source security checks?
//
var isTrustedProvider = provider switch
{
IProvider p => p.IsTrustedForDataSourceSecurityChecks(settingsManager),
AIStudio.Settings.Provider p => p.IsTrustedForDataSourceSecurityChecks(settingsManager),
//
// The confidence axis is checked on its own: a provider trusted by configuration counts as
// self-hosted for data-source security, which is the check further down, but that trust
// says nothing about how confidential the provider is. An organization which wants its
// contractually covered cloud provider to pass here raises its level through the custom
// confidence scheme instead.
//
if (providerConfidence < restrictions.ProviderConfidence)
return false;
_ => false,
};
// The data security is not specified. Means, the content never came from RAG, or RAG was
// enabled, but no data sources were selected. That's fine:
if (restrictions.DataSecurity is DataSourceSecurity.NOT_SPECIFIED)
return true;
//
// Check the data security of the content against the selected provider:
//
return isTrustedProvider switch
{
// The provider is trusted -- it may see data of any data source:
true => true,
//
// Is the provider trusted for data-source security checks?
//
var isTrustedProvider = provider switch
{
IProvider p => p.IsTrustedForDataSourceSecurityChecks(settingsManager),
AIStudio.Settings.Provider p => p.IsTrustedForDataSourceSecurityChecks(settingsManager),
// The provider is not trusted -- it depends on the data security of the content:
false => restrictions.DataSecurity is not DataSourceSecurity.SELF_HOSTED,
};
_ => false,
};
//
// Check the data security of the content against the selected provider:
//
return isTrustedProvider switch
{
// The provider is trusted -- it may see data of any data source:
true => true,
// The provider is not trusted -- it depends on the data security of the content:
false => restrictions.DataSecurity is not DataSourceSecurity.SELF_HOSTED,
};
}
}
private static LLMProviders GetLLMProvider<T>(T provider) => provider switch
app/MindWork AI Studio/Chat/ChatRoleExtensions.cs
+60 -61
View File
@@ -6,70 +6,69 @@ public static class ChatRoleExtensions
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ChatRoleExtensions).Namespace, nameof(ChatRoleExtensions));
/// <summary>
/// Returns the name of the role.
/// </summary>
/// <param name="role">The role.</param>
/// <returns>The name of the role.</returns>
public static string ToName(this ChatRole role) => role switch
extension(ChatRole role)
{
ChatRole.SYSTEM => TB("System"),
ChatRole.USER => TB("You"),
ChatRole.AI => TB("AI"),
/// <summary>
/// Returns the name of the role.
/// </summary>
/// <returns>The name of the role.</returns>
public string ToName() => role switch
{
ChatRole.SYSTEM => TB("System"),
ChatRole.USER => TB("You"),
ChatRole.AI => TB("AI"),
_ => TB("Unknown"),
};
/// <summary>
/// Returns the color of the role.
/// </summary>
/// <param name="role">The role.</param>
/// <returns>The color of the role.</returns>
public static Color ToColor(this ChatRole role) => role switch
{
ChatRole.SYSTEM => Color.Info,
ChatRole.USER => Color.Primary,
ChatRole.AI => Color.Tertiary,
_ => Color.Error,
};
/// <summary>
/// Returns the icon of the role.
/// </summary>
/// <param name="role">The role.</param>
/// <returns>The icon of the role.</returns>
public static string ToIcon(this ChatRole role) => role switch
{
ChatRole.SYSTEM => Icons.Material.Filled.Settings,
ChatRole.USER => Icons.Material.Filled.Person,
ChatRole.AI => Icons.Material.Filled.AutoAwesome,
_ => Icons.Material.Filled.Help,
};
_ => TB("Unknown"),
};
/// <summary>
/// Returns the specific name of the role for the chat template.
/// </summary>
/// <param name="role">The role.</param>
/// <returns>The name of the role.</returns>
public static string ToChatTemplateName(this ChatRole role) => role switch
{
ChatRole.SYSTEM => TB("System"),
ChatRole.USER => TB("User"),
ChatRole.AI => TB("Assistant"),
/// <summary>
/// Returns the color of the role.
/// </summary>
/// <returns>The color of the role.</returns>
public Color ToColor() => role switch
{
ChatRole.SYSTEM => Color.Info,
ChatRole.USER => Color.Primary,
ChatRole.AI => Color.Tertiary,
_ => Color.Error,
};
_ => TB("Unknown"),
};
/// <summary>
/// Selects the next role for a chat template.
/// </summary>
/// <param name="currentRole">The current role.</param>
/// <returns>The next role for the chat template.</returns>
public static ChatRole SelectNextRoleForTemplate(this ChatRole currentRole) => currentRole switch
{
ChatRole.USER => ChatRole.AI,
_ => ChatRole.USER,
};
/// <summary>
/// Returns the icon of the role.
/// </summary>
/// <returns>The icon of the role.</returns>
public string ToIcon() => role switch
{
ChatRole.SYSTEM => Icons.Material.Filled.Settings,
ChatRole.USER => Icons.Material.Filled.Person,
ChatRole.AI => Icons.Material.Filled.AutoAwesome,
_ => Icons.Material.Filled.Help,
};
/// <summary>
/// Returns the specific name of the role for the chat template.
/// </summary>
/// <returns>The name of the role.</returns>
public string ToChatTemplateName() => role switch
{
ChatRole.SYSTEM => TB("System"),
ChatRole.USER => TB("User"),
ChatRole.AI => TB("Assistant"),
_ => TB("Unknown"),
};
/// <summary>
/// Selects the next role for a chat template.
/// </summary>
/// <returns>The next role for the chat template.</returns>
public ChatRole SelectNextRoleForTemplate() => role switch
{
ChatRole.USER => ChatRole.AI,
_ => ChatRole.USER,
};
}
}
app/MindWork AI Studio/Chat/ChatThreadExtensions.cs
+22 -19
View File
@@ -2,24 +2,27 @@ namespace AIStudio.Chat;
public static class ChatThreadExtensions
{
/// <summary>
/// Checks if the specified provider is allowed for the chat thread.
/// </summary>
/// <remarks>
/// A provider the organization prohibited is never allowed, whatever the chat thread.<br/><br/>
///
/// We don't check if the provider is allowed to use the data sources of the chat thread.
/// That kind of check is done when the available data sources are resolved.<br/><br/>
///
/// One thing which is not so obvious: after RAG was used on this thread, the entire chat
/// thread is kind of a data source by itself. Why? Because the augmentation data collected
/// from the data sources is stored in the chat thread. This means we must check if the
/// selected provider is allowed to use this thread's data security and confidence level.
/// The check itself is ChatRestrictionsExtensions.IsLLMProviderAllowed, which content that left
/// the thread is checked with as well.
/// </remarks>
/// <param name="chatThread">The chat thread to check; null for a new chat, which demands nothing yet.</param>
/// <param name="provider">The provider to check.</param>
/// <returns>True, when the provider is allowed for the chat thread. False, otherwise.</returns>
public static bool IsLLMProviderAllowed<T>(this ChatThread? chatThread, T provider) => (chatThread?.CaptureRestrictions() ?? ChatRestrictions.NONE).IsLLMProviderAllowed(provider);
extension(ChatThread? chatThread)
{
/// <summary>
/// Checks if the specified provider is allowed for the chat thread.
/// </summary>
/// <remarks>
/// A provider the organization prohibited is never allowed, whatever the chat thread.<br/><br/>
///
/// We don't check if the provider is allowed to use the data sources of the chat thread.
/// That kind of check is done when the available data sources are resolved.<br/><br/>
///
/// One thing which is not so obvious: after RAG was used on this thread, the entire chat
/// thread is kind of a data source by itself. Why? Because the augmentation data collected
/// from the data sources is stored in the chat thread. This means we must check if the
/// selected provider is allowed to use this thread's data security and confidence level.
/// The check itself is ChatRestrictionsExtensions.IsLLMProviderAllowed, which content that left
/// the thread is checked with as well.
/// </remarks>
/// <param name="provider">The provider to check.</param>
/// <returns>True, when the provider is allowed for the chat thread. False, otherwise.</returns>
public bool IsLLMProviderAllowed<T>(T provider) => (chatThread?.CaptureRestrictions() ?? ChatRestrictions.NONE).IsLLMProviderAllowed(provider);
}
}
app/MindWork AI Studio/Chat/IContentExtensions.cs
+94 -94
View File
@@ -2,114 +2,114 @@ namespace AIStudio.Chat;
public static class IContentExtensions
{
/// <summary>
/// Detaches whoever listens to the stream of this content.
/// </summary>
/// <remarks>
/// The streaming handlers are closures over the component which registered them. A content
/// object belongs to the chat thread and therefore outlives every component which renders it,
/// so handlers left behind would keep those components alive for as long as the thread exists.
/// Whoever registers a handler calls this when it is no longer needed.
/// </remarks>
/// <param name="content">The content whose streaming handlers you want to detach.</param>
public static void ResetStreamingHandlers(this IContent content)
/// <param name="content">The content of a message in the chat.</param>
extension(IContent content)
{
content.StreamingEvent = IContent.NO_STREAMING_HANDLER;
content.StreamingDone = IContent.NO_STREAMING_HANDLER;
}
/// <summary>
/// Reads this content as the Markdown text the AI produced.
/// </summary>
/// <remarks>
/// Only text content carries Markdown. Everything else, an image for example, has no text
/// representation at all, which is why this reports failure instead of returning a placeholder:
/// a caller which writes files must not put an excuse into the file it writes. This is the text
/// the model wrote and nothing else: whoever reads a table out of a message wants exactly that,
/// while whoever writes a file wants the sources along with it and asks for the export reading.
/// </remarks>
/// <param name="content">The content to read.</param>
/// <param name="markdown">The Markdown text, or an empty string when there is none.</param>
/// <returns>True, when this content carries Markdown text.</returns>
public static bool TryGetMarkdownText(this IContent content, out string markdown)
{
if (content is ContentText text)
/// <summary>
/// Detaches whoever listens to the stream of this content.
/// </summary>
/// <remarks>
/// The streaming handlers are closures over the component which registered them. A content
/// object belongs to the chat thread and therefore outlives every component which renders it,
/// so handlers left behind would keep those components alive for as long as the thread exists.
/// Whoever registers a handler calls this when it is no longer needed.
/// </remarks>
public void ResetStreamingHandlers()
{
markdown = text.Text;
return true;
content.StreamingEvent = IContent.NO_STREAMING_HANDLER;
content.StreamingDone = IContent.NO_STREAMING_HANDLER;
}
markdown = string.Empty;
return false;
}
/// <summary>
/// Reads this content the way it leaves AI Studio, as a file or through the clipboard.
/// </summary>
/// <remarks>
/// What the user sees is the answer together with the sources AI Studio collected for it, and
/// that is what a document has to hold as well: an answer built on a web page a tool read, or on
/// a document of the user, is worth little when the reader cannot tell which one it was. Those
/// sources are not part of the text the model wrote, they hang on the content, which is why
/// every path out of the app asks for this and not for the text alone.
/// </remarks>
/// <param name="content">The content to read.</param>
/// <param name="markdown">The Markdown text including its sources, or an empty string when there is none.</param>
/// <param name="keepPageAnchors">Whether a link into a local file may name its page. Only a
/// format whose reader stumbles over such a link says no here; the clipboard and every text
/// format keep the page.</param>
/// <returns>True, when this content carries Markdown text.</returns>
public static bool TryGetExportMarkdown(this IContent content, out string markdown, bool keepPageAnchors = true)
{
if (content is not ContentText text)
/// <summary>
/// Reads this content as the Markdown text the AI produced.
/// </summary>
/// <remarks>
/// Only text content carries Markdown. Everything else, an image for example, has no text
/// representation at all, which is why this reports failure instead of returning a placeholder:
/// a caller which writes files must not put an excuse into the file it writes. This is the text
/// the model wrote and nothing else: whoever reads a table out of a message wants exactly that,
/// while whoever writes a file wants the sources along with it and asks for the export reading.
/// </remarks>
/// <param name="markdown">The Markdown text, or an empty string when there is none.</param>
/// <returns>True, when this content carries Markdown text.</returns>
public bool TryGetMarkdownText(out string markdown)
{
if (content is ContentText text)
{
markdown = text.Text;
return true;
}
markdown = string.Empty;
return false;
}
markdown = AppendSources(text.Text.Trim(), text.Sources.ToExportMarkdown(keepPageAnchors));
return true;
}
/// <summary>
/// Reads this content the way it leaves AI Studio, as a file or through the clipboard.
/// </summary>
/// <remarks>
/// What the user sees is the answer together with the sources AI Studio collected for it, and
/// that is what a document has to hold as well: an answer built on a web page a tool read, or on
/// a document of the user, is worth little when the reader cannot tell which one it was. Those
/// sources are not part of the text the model wrote, they hang on the content, which is why
/// every path out of the app asks for this and not for the text alone.
/// </remarks>
/// <param name="markdown">The Markdown text including its sources, or an empty string when there is none.</param>
/// <param name="keepPageAnchors">Whether a link into a local file may name its page. Only a
/// format whose reader stumbles over such a link says no here; the clipboard and every text
/// format keep the page.</param>
/// <returns>True, when this content carries Markdown text.</returns>
public bool TryGetExportMarkdown(out string markdown, bool keepPageAnchors = true)
{
if (content is not ContentText text)
{
markdown = string.Empty;
return false;
}
/// <summary>
/// Reads one file out of this content the way it leaves AI Studio, together with the sources
/// the answer rests on.
/// </summary>
/// <remarks>
/// A document saved on its own came out of the same answer, so it rests on the same sources
/// and takes them along. How depends on the format. Markdown is what the source list is written
/// in, so a Markdown text gets it just as the entire answer does. A web page or a LaTeX document
/// gets it as a comment at its end: anything visible would have to be woven into markup the
/// model wrote. A fragment has no body to put it in, a page may hide whatever lies outside its
/// layout, and one underscore in a title is enough to stop a LaTeX run. A comment breaks
/// neither, and whoever opens the file finds it. A table gets no sources at all, since it has
/// no column a link list would fit into. Neither does code: a program, a script, or a
/// configuration is meant for a compiler or a tool, and a comment of ours would be the one part
/// of it nobody asked for. In some languages, such as JSON, it would even break the file.
///
/// Apart from that, the file is what the model wrote, scripts of a web page included. Saving it
/// is what the user chose to do; the chat still never renders it. Code only gets what its tools
/// expect of a file, see ToSourceCodeFile.
/// </remarks>
/// <param name="content">The content the file was found in.</param>
/// <param name="file">The file, as PlainFileExport.ExtractFiles read it out of this content.</param>
/// <returns>The content of the file to write.</returns>
public static string ToExportContent(this IContent content, MessageFile file)
{
if (file.Format.IsTabular())
return file.Content;
markdown = AppendSources(text.Text.Trim(), text.Sources.ToExportMarkdown(keepPageAnchors));
return true;
}
if (file.Format is FileExportFormat.SOURCE_CODE)
return ToSourceCodeFile(file);
/// <summary>
/// Reads one file out of this content the way it leaves AI Studio, together with the sources
/// the answer rests on.
/// </summary>
/// <remarks>
/// A document saved on its own came out of the same answer, so it rests on the same sources
/// and takes them along. How depends on the format. Markdown is what the source list is written
/// in, so a Markdown text gets it just as the entire answer does. A web page or a LaTeX document
/// gets it as a comment at its end: anything visible would have to be woven into markup the
/// model wrote. A fragment has no body to put it in, a page may hide whatever lies outside its
/// layout, and one underscore in a title is enough to stop a LaTeX run. A comment breaks
/// neither, and whoever opens the file finds it. A table gets no sources at all, since it has
/// no column a link list would fit into. Neither does code: a program, a script, or a
/// configuration is meant for a compiler or a tool, and a comment of ours would be the one part
/// of it nobody asked for. In some languages, such as JSON, it would even break the file.
///
/// Apart from that, the file is what the model wrote, scripts of a web page included. Saving it
/// is what the user chose to do; the chat still never renders it. Code only gets what its tools
/// expect of a file, see ToSourceCodeFile.
/// </remarks>
/// <param name="file">The file, as PlainFileExport.ExtractFiles read it out of this content.</param>
/// <returns>The content of the file to write.</returns>
public string ToExportContent(MessageFile file)
{
if (file.Format.IsTabular())
return file.Content;
var sources = content.Sources.ToExportMarkdown(file.Format.FollowsPageAnchors());
if (file.Format is FileExportFormat.MARKDOWN)
return AppendSources(file.Content, sources);
if (file.Format is FileExportFormat.SOURCE_CODE)
return ToSourceCodeFile(file);
if (sources.Length is 0 || !file.Format.TryToComment(sources, out var comment))
return file.Content;
var sources = content.Sources.ToExportMarkdown(file.Format.FollowsPageAnchors());
if (file.Format is FileExportFormat.MARKDOWN)
return AppendSources(file.Content, sources);
return $"{file.Content}{Environment.NewLine}{Environment.NewLine}{comment}";
if (sources.Length is 0 || !file.Format.TryToComment(sources, out var comment))
return file.Content;
return $"{file.Content}{Environment.NewLine}{Environment.NewLine}{comment}";
}
}
/// <summary>
app/MindWork AI Studio/Chat/IImageSourceExtensions.cs
+68 -65
View File
@@ -30,76 +30,79 @@ public static class IImageSourceExtensions
return Math.Min(Math.Min(engineLimit ?? MAX_IMAGE_SIZE_BYTES, providerLimit ?? MAX_IMAGE_SIZE_BYTES), MAX_IMAGE_SIZE_BYTES);
}
/// <summary>
/// Read the image content as a base64 string.
/// </summary>
/// <remarks>
/// This is for the images inside the content of a chat, which go to external data sources and
/// agents as they are. Attached images, which go to a model, take ModelImageService instead: it
/// scales them down and encodes them anew within the limit of the provider.<br/>
/// <br/>
/// The images are directly converted to base64 strings. The maximum
/// size of the image is around 10 MB. If the image is larger, the method
/// returns an empty string.<br/>
/// <br/>
/// Note as well that this method returns just the base64 string without
/// any data URI prefix (like "data:image/png;base64,"). The caller has
/// to take care of that if needed.
/// </remarks>
/// <param name="image">The image source.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The image content as a base64 string; might be empty.</returns>
public static async Task<(bool success, string base64Content)> TryAsBase64(this IImageSource image, CancellationToken token = default)
extension(IImageSource image)
{
switch (image.SourceType)
/// <summary>
/// Read the image content as a base64 string.
/// </summary>
/// <remarks>
/// This is for the images inside the content of a chat, which go to external data sources and
/// agents as they are. Attached images, which go to a model, take ModelImageService instead: it
/// scales them down and encodes them anew within the limit of the provider.<br/>
/// <br/>
/// The images are directly converted to base64 strings. The maximum
/// size of the image is around 10 MB. If the image is larger, the method
/// returns an empty string.<br/>
/// <br/>
/// Note as well that this method returns just the base64 string without
/// any data URI prefix (like "data:image/png;base64,"). The caller has
/// to take care of that if needed.
/// </remarks>
/// <param name="token">The cancellation token.</param>
/// <returns>The image content as a base64 string; might be empty.</returns>
public async Task<(bool success, string base64Content)> TryAsBase64(CancellationToken token = default)
{
case ContentImageSource.BASE64:
return (success: true, image.Source);
case ContentImageSource.URL:
switch (image.SourceType)
{
using var httpClient = ExternalHttpClientTimeout.CreateHttpClient(ExternalHttpTrustPolicy.ALLOW_CUSTOM_ROOTS_WHEN_HOST_WHITELISTED);
using var timeoutTokenSource = ExternalHttpClientTimeout.CreateTimeoutTokenSource(token);
var timeoutToken = timeoutTokenSource.Token;
using var response = await httpClient.GetAsync(image.Source, HttpCompletionOption.ResponseHeadersRead, timeoutToken);
if(response.IsSuccessStatusCode)
{
// Read the length of the content:
var lengthBytes = response.Content.Headers.ContentLength;
if(lengthBytes > MAX_IMAGE_SIZE_BYTES)
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The image at the URL is too large (>10 MB). Skipping the image.")));
return (success: false, string.Empty);
}
var bytes = await response.Content.ReadAsByteArrayAsync(timeoutToken);
return (success: true, Convert.ToBase64String(bytes));
}
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("Failed to download the image from the URL. Skipping the image.")));
return (success: false, string.Empty);
}
case ContentImageSource.LOCAL_PATH:
if(File.Exists(image.Source))
{
// Read the content length:
var length = new FileInfo(image.Source).Length;
if(length > MAX_IMAGE_SIZE_BYTES)
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The local image file is too large (>10 MB). Skipping the image.")));
return (success: false, string.Empty);
}
var bytes = await File.ReadAllBytesAsync(image.Source, token);
return (success: true, Convert.ToBase64String(bytes));
}
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The local image file does not exist. Skipping the image.")));
return (success: false, string.Empty);
case ContentImageSource.BASE64:
return (success: true, image.Source);
default:
return (success: false, string.Empty);
case ContentImageSource.URL:
{
using var httpClient = ExternalHttpClientTimeout.CreateHttpClient(ExternalHttpTrustPolicy.ALLOW_CUSTOM_ROOTS_WHEN_HOST_WHITELISTED);
using var timeoutTokenSource = ExternalHttpClientTimeout.CreateTimeoutTokenSource(token);
var timeoutToken = timeoutTokenSource.Token;
using var response = await httpClient.GetAsync(image.Source, HttpCompletionOption.ResponseHeadersRead, timeoutToken);
if(response.IsSuccessStatusCode)
{
// Read the length of the content:
var lengthBytes = response.Content.Headers.ContentLength;
if(lengthBytes > MAX_IMAGE_SIZE_BYTES)
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The image at the URL is too large (>10 MB). Skipping the image.")));
return (success: false, string.Empty);
}
var bytes = await response.Content.ReadAsByteArrayAsync(timeoutToken);
return (success: true, Convert.ToBase64String(bytes));
}
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("Failed to download the image from the URL. Skipping the image.")));
return (success: false, string.Empty);
}
case ContentImageSource.LOCAL_PATH:
if(File.Exists(image.Source))
{
// Read the content length:
var length = new FileInfo(image.Source).Length;
if(length > MAX_IMAGE_SIZE_BYTES)
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The local image file is too large (>10 MB). Skipping the image.")));
return (success: false, string.Empty);
}
var bytes = await File.ReadAllBytesAsync(image.Source, token);
return (success: true, Convert.ToBase64String(bytes));
}
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The local image file does not exist. Skipping the image.")));
return (success: false, string.Empty);
default:
return (success: false, string.Empty);
}
}
}
}
app/MindWork AI Studio/Chat/ListContentBlockExtensions.cs
+141 -143
View File
@@ -9,161 +9,159 @@ namespace AIStudio.Chat;
public static class ListContentBlockExtensions
{
/// <summary>
/// Processes a list of content blocks by transforming them into a collection of message results asynchronously.
/// </summary>
/// <param name="blocks">The list of content blocks to process.</param>
/// <param name="roleTransformer">A function that transforms each content block into a message result asynchronously.</param>
/// <param name="provider">The configured provider, whose model is being written to.</param>
/// <param name="textSubContentFactory">A factory function to create text sub-content.</param>
/// <param name="imageSubContentFactory">A factory function to create image sub-content from an image prepared for the model.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>An asynchronous task that resolves to a list of transformed results.</returns>
public static async Task<IList<IMessageBase>> BuildMessagesAsync(
this List<ContentBlock> blocks,
AIStudio.Settings.Provider provider,
Func<ChatRole, string> roleTransformer,
Func<string, ISubContent> textSubContentFactory,
Func<PreparedModelImage, ISubContent> imageSubContentFactory,
CancellationToken token = default)
extension(List<ContentBlock> blocks)
{
//
// Asked through the configured provider, so that what a person set in their expert settings
// counts here too. It did not: this path read the automatic answer alone, so somebody who
// switched image input on saw it work while attaching the picture and saw it ignored while
// the message was built -- every chat round and every tool round.
//
var canProcessImages = provider.SupportsImageInput();
//
// One report for the whole request: every message of the conversation reads its attached
// documents anew, all at the same time. Twenty documents must produce one dialog listing
// all of them, not twenty dialogs in a row. A request without documents has nothing to
// report, and needs no scope.
//
await using var reportingScope = blocks.Any(block => block.Content is ContentText text && text.FileAttachments.MayContainDocuments())
? Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>().BeginAction()
: null;
var messageTaskList = new List<Task<IMessageBase>>(blocks.Count);
foreach (var block in blocks)
/// <summary>
/// Processes a list of content blocks by transforming them into a collection of message results asynchronously.
/// </summary>
/// <param name="roleTransformer">A function that transforms each content block into a message result asynchronously.</param>
/// <param name="provider">The configured provider, whose model is being written to.</param>
/// <param name="textSubContentFactory">A factory function to create text sub-content.</param>
/// <param name="imageSubContentFactory">A factory function to create image sub-content from an image prepared for the model.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>An asynchronous task that resolves to a list of transformed results.</returns>
public async Task<IList<IMessageBase>> BuildMessagesAsync(
AIStudio.Settings.Provider provider,
Func<ChatRole, string> roleTransformer,
Func<string, ISubContent> textSubContentFactory,
Func<PreparedModelImage, ISubContent> imageSubContentFactory,
CancellationToken token = default)
{
switch (block.Content)
//
// Asked through the configured provider, so that what a person set in their expert settings
// counts here too. It did not: this path read the automatic answer alone, so somebody who
// switched image input on saw it work while attaching the picture and saw it ignored while
// the message was built -- every chat round and every tool round.
//
var canProcessImages = provider.SupportsImageInput();
//
// One report for the whole request: every message of the conversation reads its attached
// documents anew, all at the same time. Twenty documents must produce one dialog listing
// all of them, not twenty dialogs in a row. A request without documents has nothing to
// report, and needs no scope.
//
await using var reportingScope = blocks.Any(block => block.Content is ContentText text && text.FileAttachments.MayContainDocuments())
? Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>().BeginAction()
: null;
var messageTaskList = new List<Task<IMessageBase>>(blocks.Count);
foreach (var block in blocks)
{
// The prompt may or may not contain image(s), but the provider/model cannot process images.
// Thus, we treat it as a regular text message.
case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && !canProcessImages:
messageTaskList.Add(CreateTextMessageAsync(block, text));
break;
switch (block.Content)
{
// The prompt may or may not contain image(s), but the provider/model cannot process images.
// Thus, we treat it as a regular text message.
case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && !canProcessImages:
messageTaskList.Add(CreateTextMessageAsync(block, text));
break;
// The regular case for text content without images:
case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && !text.FileAttachments.ContainsImages():
messageTaskList.Add(CreateTextMessageAsync(block, text));
break;
// The regular case for text content without images:
case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && !text.FileAttachments.ContainsImages():
messageTaskList.Add(CreateTextMessageAsync(block, text));
break;
// Text prompt with images as attachments, and the provider/model can process images:
case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && text.FileAttachments.ContainsImages():
messageTaskList.Add(CreateMultimodalMessageAsync(block, text));
break;
// Text prompt with images as attachments, and the provider/model can process images:
case ContentText text when block.ContentType is ContentType.TEXT && !string.IsNullOrWhiteSpace(text.Text) && text.FileAttachments.ContainsImages():
messageTaskList.Add(CreateMultimodalMessageAsync(block, text));
break;
}
}
// Await all messages:
await Task.WhenAll(messageTaskList);
// Select all results:
return messageTaskList.Select(n => n.Result).ToList();
// Local function to create a text message asynchronously.
Task<IMessageBase> CreateTextMessageAsync(ContentBlock block, ContentText text)
{
return Task.Run(async () => new TextMessage
{
Role = roleTransformer(block.Role),
Content = await text.PrepareTextContentForAI(reportingScope),
} as IMessageBase);
}
// Local function to create a multimodal message asynchronously.
Task<IMessageBase> CreateMultimodalMessageAsync(ContentBlock block, ContentText text)
{
return Task.Run(async () =>
{
var images = await Task.WhenAll(text.FileAttachments
.Where(x => x is { IsImage: true, Exists: true })
.Cast<FileAttachmentImage>()
.Select(CreateImageSubContentAsync));
return new MultimodalMessage
{
Role = roleTransformer(block.Role),
Content =
[
textSubContentFactory(await text.PrepareTextContentForAI(reportingScope)),
..images.OfType<ISubContent>(),
]
} as IMessageBase;
});
}
//
// Local function to prepare one image for the model. An image which cannot be prepared
// stays out of the message, and the user learns why. It used to go along as an image
// without data, which the providers refuse together with the whole request.
//
async Task<ISubContent?> CreateImageSubContentAsync(FileAttachmentImage image)
{
var prepared = await Program.SERVICE_PROVIDER.GetRequiredService<ModelImageService>().PrepareAsync(image, provider, token);
if (prepared.IsPrepared)
return imageSubContentFactory(prepared);
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, prepared.ErrorCode.ToUserMessage(image.FileName)));
return null;
}
}
// Await all messages:
await Task.WhenAll(messageTaskList);
// Select all results:
return messageTaskList.Select(n => n.Result).ToList();
// Local function to create a text message asynchronously.
Task<IMessageBase> CreateTextMessageAsync(ContentBlock block, ContentText text)
{
return Task.Run(async () => new TextMessage
{
Role = roleTransformer(block.Role),
Content = await text.PrepareTextContentForAI(reportingScope),
} as IMessageBase);
}
// Local function to create a multimodal message asynchronously.
Task<IMessageBase> CreateMultimodalMessageAsync(ContentBlock block, ContentText text)
{
return Task.Run(async () =>
{
var images = await Task.WhenAll(text.FileAttachments
.Where(x => x is { IsImage: true, Exists: true })
.Cast<FileAttachmentImage>()
.Select(CreateImageSubContentAsync));
return new MultimodalMessage
{
Role = roleTransformer(block.Role),
Content =
[
textSubContentFactory(await text.PrepareTextContentForAI(reportingScope)),
..images.OfType<ISubContent>(),
]
} as IMessageBase;
});
}
/// <summary>
/// Processes a list of content blocks using direct image URL format to create message results asynchronously.
/// </summary>
/// <param name="provider">The configured provider, whose model is being written to.</param>
/// <returns>An asynchronous task that resolves to a list of transformed message results.</returns>
/// <remarks>
/// Uses direct image URL format where the image data is placed directly in the image_url field:
/// <code>
/// { "type": "image_url", "image_url": "data:image/jpeg;base64,..." }
/// </code>
/// This format is used by OpenAI, Mistral, and Ollama.
/// </remarks>
public async Task<IList<IMessageBase>> BuildMessagesUsingDirectImageUrlAsync(
AIStudio.Settings.Provider provider) => await blocks.BuildMessagesAsync(
provider,
StandardRoleTransformer,
StandardTextSubContentFactory,
DirectImageSubContentFactory);
//
// Local function to prepare one image for the model. An image which cannot be prepared
// stays out of the message, and the user learns why. It used to go along as an image
// without data, which the providers refuse together with the whole request.
//
async Task<ISubContent?> CreateImageSubContentAsync(FileAttachmentImage image)
{
var prepared = await Program.SERVICE_PROVIDER.GetRequiredService<ModelImageService>().PrepareAsync(image, provider, token);
if (prepared.IsPrepared)
return imageSubContentFactory(prepared);
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, prepared.ErrorCode.ToUserMessage(image.FileName)));
return null;
}
/// <summary>
/// Processes a list of content blocks using nested image URL format to create message results asynchronously.
/// </summary>
/// <param name="provider">The configured provider, whose model is being written to.</param>
/// <returns>An asynchronous task that resolves to a list of transformed message results.</returns>
/// <remarks>
/// Uses nested image URL format where the image data is wrapped in an object:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// This format is used by LM Studio, VLLM, llama.cpp, and other OpenAI-compatible providers.
/// </remarks>
public async Task<IList<IMessageBase>> BuildMessagesUsingNestedImageUrlAsync(
AIStudio.Settings.Provider provider) => await blocks.BuildMessagesAsync(
provider,
StandardRoleTransformer,
StandardTextSubContentFactory,
NestedImageSubContentFactory);
}
/// <summary>
/// Processes a list of content blocks using direct image URL format to create message results asynchronously.
/// </summary>
/// <param name="blocks">The list of content blocks to process.</param>
/// <param name="provider">The configured provider, whose model is being written to.</param>
/// <returns>An asynchronous task that resolves to a list of transformed message results.</returns>
/// <remarks>
/// Uses direct image URL format where the image data is placed directly in the image_url field:
/// <code>
/// { "type": "image_url", "image_url": "data:image/jpeg;base64,..." }
/// </code>
/// This format is used by OpenAI, Mistral, and Ollama.
/// </remarks>
public static async Task<IList<IMessageBase>> BuildMessagesUsingDirectImageUrlAsync(
this List<ContentBlock> blocks,
AIStudio.Settings.Provider provider) => await blocks.BuildMessagesAsync(
provider,
StandardRoleTransformer,
StandardTextSubContentFactory,
DirectImageSubContentFactory);
/// <summary>
/// Processes a list of content blocks using nested image URL format to create message results asynchronously.
/// </summary>
/// <param name="blocks">The list of content blocks to process.</param>
/// <param name="provider">The configured provider, whose model is being written to.</param>
/// <returns>An asynchronous task that resolves to a list of transformed message results.</returns>
/// <remarks>
/// Uses nested image URL format where the image data is wrapped in an object:
/// <code>
/// { "type": "image_url", "image_url": { "url": "data:image/jpeg;base64,..." } }
/// </code>
/// This format is used by LM Studio, VLLM, llama.cpp, and other OpenAI-compatible providers.
/// </remarks>
public static async Task<IList<IMessageBase>> BuildMessagesUsingNestedImageUrlAsync(
this List<ContentBlock> blocks,
AIStudio.Settings.Provider provider) => await blocks.BuildMessagesAsync(
provider,
StandardRoleTransformer,
StandardTextSubContentFactory,
NestedImageSubContentFactory);
private static ISubContent StandardTextSubContentFactory(string text) => new SubContentText
{
Text = text,
app/MindWork AI Studio/Chat/ListFileAttachmentExtensions.cs
+15 -12
View File
@@ -2,17 +2,20 @@ namespace AIStudio.Chat;
public static class ListFileAttachmentExtensions
{
public static bool ContainsImages(this List<FileAttachment> attachments) => attachments.Any(attachment => attachment.IsImage);
/// <summary>
/// Whether preparing the message for the AI may read a document through the runtime.
/// </summary>
/// <remarks>
/// Every attachment but a picture counts: its type is checked again with the latest rules
/// before it is read, cf. FileAttachment.Normalize, and may turn out to be a document then.
/// Checking that here would touch every file twice, which takes long on a network drive.
/// </remarks>
/// <param name="attachments">The attachments of the message.</param>
/// <returns>True when a document may be read.</returns>
public static bool MayContainDocuments(this List<FileAttachment> attachments) => attachments.Any(attachment => !attachment.IsImage);
extension(List<FileAttachment> attachments)
{
public bool ContainsImages() => attachments.Any(attachment => attachment.IsImage);
/// <summary>
/// Whether preparing the message for the AI may read a document through the runtime.
/// </summary>
/// <remarks>
/// Every attachment but a picture counts: its type is checked again with the latest rules
/// before it is read, cf. FileAttachment.Normalize, and may turn out to be a document then.
/// Checking that here would touch every file twice, which takes long on a network drive.
/// </remarks>
/// <returns>True when a document may be read.</returns>
public bool MayContainDocuments() => attachments.Any(attachment => !attachment.IsImage);
}
}
app/MindWork AI Studio/Chat/SetFileAttachmentExtensions.cs
+16 -13
View File
@@ -2,20 +2,23 @@ namespace AIStudio.Chat;
public static class SetFileAttachmentExtensions
{
/// <summary>
/// Attaches a file in place of the attachment of the same file the set held before.
/// </summary>
/// <remarks>
/// The attachments differ by more than their files: one of a message being edited holds the
/// entry of the text it was sent with, and the size of the file may have changed since. Both
/// would stay in the set, and the message would go with the old and the new text of the file.
/// Attaching a file again means the file as it is now.
/// </remarks>
/// <param name="attachments">The attachments of a message.</param>
/// <param name="attachment">The attachment of the file just attached.</param>
public static void AttachInPlaceOfSameFile(this HashSet<FileAttachment> attachments, FileAttachment attachment)
extension(HashSet<FileAttachment> attachments)
{
attachments.RemoveWhere(existing => string.Equals(existing.FilePath, attachment.FilePath, StringComparison.Ordinal));
attachments.Add(attachment);
/// <summary>
/// Attaches a file in place of the attachment of the same file the set held before.
/// </summary>
/// <remarks>
/// The attachments differ by more than their files: one of a message being edited holds the
/// entry of the text it was sent with, and the size of the file may have changed since. Both
/// would stay in the set, and the message would go with the old and the new text of the file.
/// Attaching a file again means the file as it is now.
/// </remarks>
/// <param name="attachment">The attachment of the file just attached.</param>
public void AttachInPlaceOfSameFile(FileAttachment attachment)
{
attachments.RemoveWhere(existing => string.Equals(existing.FilePath, attachment.FilePath, StringComparison.Ordinal));
attachments.Add(attachment);
}
}
}
app/MindWork AI Studio/Chat/StringExtension.cs
+32 -29
View File
@@ -2,37 +2,40 @@
public static class StringExtensions
{
public static string RemoveThinkTags(this string input)
{
const string OPEN_TAG = "<think>";
const string CLOSE_TAG = "</think>";
if (string.IsNullOrWhiteSpace(input) || !input.StartsWith(OPEN_TAG, StringComparison.Ordinal))
return input;
var endIndex = input.IndexOf(CLOSE_TAG, StringComparison.Ordinal);
if (endIndex == -1)
return string.Empty;
return input[(endIndex + CLOSE_TAG.Length)..];
}
/// <summary>
/// Shortens a text to the given number of characters, and marks the cut with three dots.
/// </summary>
/// <remarks>
/// Never cuts between the two halves of a surrogate pair, which no JSON writer takes.<br/><br/>
/// Takes and returns a string rather than a span on purpose: most texts are short enough, and
/// those come back as the very same instance. Only a text which is cut costs one new string.
/// </remarks>
/// <param name="text">The text.</param>
/// <param name="maxCharacters">How many characters of the text to keep at most, the dots not counted.</param>
/// <returns>The text as it was when it is short enough, otherwise its beginning followed by three dots.</returns>
public static string Shorten(this string text, int maxCharacters)
extension(string text)
{
if (text.Length <= maxCharacters)
return text;
public string RemoveThinkTags()
{
const string OPEN_TAG = "<think>";
const string CLOSE_TAG = "</think>";
if (string.IsNullOrWhiteSpace(text) || !text.StartsWith(OPEN_TAG, StringComparison.Ordinal))
return text;
var endIndex = text.IndexOf(CLOSE_TAG, StringComparison.Ordinal);
if (endIndex == -1)
return string.Empty;
return text[(endIndex + CLOSE_TAG.Length)..];
}
var end = char.IsHighSurrogate(text[maxCharacters - 1]) ? maxCharacters - 1 : maxCharacters;
return string.Concat(text.AsSpan(0, end).TrimEnd(), "...");
/// <summary>
/// Shortens a text to the given number of characters, and marks the cut with three dots.
/// </summary>
/// <remarks>
/// Never cuts between the two halves of a surrogate pair, which no JSON writer takes.<br/><br/>
/// Takes and returns a string rather than a span on purpose: most texts are short enough, and
/// those come back as the very same instance. Only a text which is cut costs one new string.
/// </remarks>
/// <param name="maxCharacters">How many characters of the text to keep at most, the dots not counted.</param>
/// <returns>The text as it was when it is short enough, otherwise its beginning followed by three dots.</returns>
public string Shorten(int maxCharacters)
{
if (text.Length <= maxCharacters)
return text;
var end = char.IsHighSurrogate(text[maxCharacters - 1]) ? maxCharacters - 1 : maxCharacters;
return string.Concat(text.AsSpan(0, end).TrimEnd(), "...");
}
}
}