Add prompt injection detection framework and UI alert dialog

This commit is contained in:
hart_s3 2026-08-07 14:54:34 +02:00
parent d1a6781ea6
commit 9adf50c71b
31 changed files with 827 additions and 36 deletions

View File

@ -4,6 +4,7 @@ using AIStudio.Chat;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools.RAG;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services;
namespace AIStudio.Agents;
@ -237,7 +238,20 @@ public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalConte
// 2. Prepare the retrieval context for the agent:
//
var additionalData = new Dictionary<string, string>();
var markdownRetrievalContext = await retrievalContext.AsMarkdown(token: token);
string markdownRetrievalContext;
try
{
markdownRetrievalContext = await retrievalContext.AsMarkdown(token: token);
}
catch (PromptInjectionBlockedException exception)
{
logger.LogWarning(
exception,
"Blocked retrieval context '{DataSourceName}' at '{Path}' before validation.",
retrievalContext.DataSourceName,
retrievalContext.Path);
return new(false, "The retrieval context was blocked due to suspected prompt injection.", 1.0f, retrievalContext);
}
additionalData.Add("retrievalContext", markdownRetrievalContext);
//

View File

@ -5,6 +5,7 @@ using AIStudio.Dialogs.Settings;
using AIStudio.Tools.AIJobs;
using AIStudio.Tools.AssistantSessions;
using AIStudio.Tools.Media;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
@ -273,6 +274,12 @@ public abstract partial class AssistantBase<TSettings> : AssistantLowerBase wher
this.Logger.LogError(e, "The provider request failed for assistant '{AssistantTitle}'. Status={StatusCode}, Reason='{ReasonPhrase}', Body='{ResponseBody}'", this.Title, e.StatusCode, e.ReasonPhrase, e.ResponseBody);
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, e.UserMessage));
}
catch (PromptInjectionBlockedException e)
{
sessionStatus = AssistantSessionStatus.FAILED;
errorMessage = e.Message;
this.Logger.LogWarning(e, "Blocked prompt-injection content during assistant session '{AssistantTitle}'.", this.Title);
}
catch (Exception e)
{
sessionStatus = AssistantSessionStatus.FAILED;
@ -461,6 +468,18 @@ public abstract partial class AssistantBase<TSettings> : AssistantLowerBase wher
return string.Empty;
}
catch (PromptInjectionBlockedException e)
{
this.Logger.LogWarning(e, "Blocked prompt-injection content before sending assistant request for '{AssistantTitle}'.", this.Title);
if (this.ResultingContentBlock is not null && string.IsNullOrWhiteSpace(aiText.Text))
{
this.ChatThread?.Blocks.Remove(this.ResultingContentBlock);
this.ResultingContentBlock = null;
}
return string.Empty;
}
finally
{
this.IsProcessing = this.assistantSessionId is not null && (this.AssistantSessionService.TryGetSnapshot(this.assistantSessionKey)?.IsActive ?? false);

View File

@ -716,7 +716,7 @@ public partial class DocumentAnalysisAssistant : AssistantBaseCore<NoSettingsPan
continue;
}
var fileContent = await this.RustService.ReadArbitraryFileData(document.FilePath, int.MaxValue);
var fileContent = await UserFile.LoadFileData(document.FilePath, this.RustService, this.DialogService);
sb.AppendLine($"""
## DOCUMENT {numDocuments}:

View File

@ -5572,6 +5572,27 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINGGUIDELINEDIALOG::T384594633"] = "Th
-- Prompting Guideline
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINGGUIDELINEDIALOG::T4250996615"] = "Prompting Guideline"
-- Source
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1642243064"] = "Source"
-- AI Studio blocked this content before it reached a model or agent.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2378027194"] = "AI Studio blocked this content before it reached a model or agent."
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3448155331"] = "Close"
-- Prompt Injection Detected
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3580322580"] = "Prompt Injection Detected"
-- Source kind
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T47437466"] = "Source kind"
-- More information
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T475337262"] = "More information"
-- Detected signals
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T556618402"] = "Detected signals"
-- Hugging Face Inference Provider
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1085481431"] = "Hugging Face Inference Provider"
@ -6052,6 +6073,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T6790
-- When enabled, you can preselect options. This is might be useful when you prefer a specific language or LLM model.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T711745239"] = "When enabled, you can preselect options. This is might be useful when you prefer a specific language or LLM model."
-- Protect against prompt injection in external content?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1040385578"] = "Protect against prompt injection in external content?"
-- Preselect one of your chat templates?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1402022556"] = "Preselect one of your chat templates?"
@ -6073,12 +6097,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1915793195"]
-- Preselect a profile
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2322771068"] = "Preselect a profile"
-- A blocking alert explains the detected attack pattern
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2341755932"] = "A blocking alert explains the detected attack pattern"
-- Apply default data source option when sending assistant results to chat
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2510376349"] = "Apply default data source option when sending assistant results to chat"
-- Control how the LLM provider for added chats is selected.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T263621180"] = "Control how the LLM provider for added chats is selected."
-- Show a learning alert when prompt injection is detected?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2754886869"] = "Show a learning alert when prompt injection is detected?"
-- Provider selection when loading a chat and sending assistant results to chat
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2868379953"] = "Provider selection when loading a chat and sending assistant results to chat"
@ -6088,6 +6118,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2913693228"]
-- Do you want to use any shortcut to send your input?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2936560092"] = "Do you want to use any shortcut to send your input?"
-- Shows an explanation dialog with an external reference when AI Studio blocks suspicious content.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3129787554"] = "Shows an explanation dialog with an external reference when AI Studio blocks suspicious content."
-- Would you like to set one of your chat templates as the default for chats?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3234927721"] = "Would you like to set one of your chat templates as the default for chats?"
@ -6097,6 +6130,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3383186996"]
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3448155331"] = "Close"
-- Only the block notification is shown
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3491678837"] = "Only the block notification is shown"
-- First (oldest) message is shown, after loading a chat
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3507181366"] = "First (oldest) message is shown, after loading a chat"
@ -6109,9 +6145,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3730599555"]
-- Latest message is shown, after loading a chat
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3755993611"] = "Latest message is shown, after loading a chat"
-- Potential prompt injections are blocked before they reach an LLM
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3982766677"] = "Potential prompt injections are blocked before they reach an LLM"
-- Do you want to apply the default data source options when sending assistant results to chat?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4033153439"] = "Do you want to apply the default data source options when sending assistant results to chat?"
-- External content is passed through without prompt-injection checks
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4245463281"] = "External content is passed through without prompt-injection checks"
-- When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T477675197"] = "When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider."
@ -6121,6 +6163,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T492357592"] =
-- When enabled, the latest message is shown after loading a chat. When disabled, the first (oldest) message is shown.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T582516016"] = "When enabled, the latest message is shown after loading a chat. When disabled, the first (oldest) message is shown."
-- Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T69192586"] = "Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent."
-- Customize your AI experience with chat templates. Whether you want to experiment with prompt engineering, simply use a custom system prompt in the standard chat interface, or create a specialized assistant, our templates give you full control. Similar to common AI companies' playgrounds, you can define your own system prompts and leverage assistant prompts for providers that support them.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHATTEMPLATE::T1172171653"] = "Customize your AI experience with chat templates. Whether you want to experiment with prompt engineering, simply use a custom system prompt in the standard chat interface, or create a specialized assistant, our templates give you full control. Similar to common AI companies' playgrounds, you can define your own system prompts and leverage assistant prompts for providers that support them."
@ -7243,6 +7288,9 @@ UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2936083926"] = "AI Studio could
-- Writer
UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2979224202"] = "Writer"
-- Prompt Injection Detected
UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T3580322580"] = "Prompt Injection Detected"
-- Show details
UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T3692372066"] = "Show details"
@ -9487,6 +9535,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document"
-- Plugin archive
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T927001356"] = "Plugin archive"
-- AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONGUARDSERVICE::T3560909296"] = "AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt."
-- The Assistant Builder context could not be loaded.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGINGENERATIONSERVICE::T108292972"] = "The Assistant Builder context could not be loaded."

View File

@ -4,6 +4,7 @@ using System.Text.RegularExpressions;
using AIStudio.Chat;
using AIStudio.Dialogs;
using AIStudio.Dialogs.Settings;
using AIStudio.Tools.Security;
using AIStudio.Tools.AssistantSessions;
using Microsoft.AspNetCore.Components;
@ -583,6 +584,10 @@ public partial class AssistantPromptOptimizer : AssistantBaseCore<SettingsDialog
if (string.IsNullOrWhiteSpace(this.customPromptingGuidelineContent))
await this.MessageBus.SendWarning(new(Icons.Material.Filled.Description, T("The custom prompt guide file is empty or could not be read.")));
}
catch (PromptInjectionBlockedException)
{
this.customPromptingGuidelineContent = string.Empty;
}
catch
{
this.customPromptingGuidelineContent = string.Empty;

View File

@ -2,11 +2,15 @@
using AIStudio.Chat;
using AIStudio.Dialogs.Settings;
using AIStudio.Tools.AssistantSessions;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.SlideBuilder;
public partial class SlideAssistant : AssistantBaseCore<SettingsDialogSlideBuilder>
{
[Inject]
private IDialogService DialogService { get; init; } = null!;
protected override Tools.Components Component => Tools.Components.SLIDE_BUILDER_ASSISTANT;
protected override string Title => T("Slide Planner Assistant");
@ -382,7 +386,7 @@ public partial class SlideAssistant : AssistantBaseCore<SettingsDialogSlideBuild
continue;
}
var fileContent = await this.RustService.ReadArbitraryFileData(document.FilePath, int.MaxValue);
var fileContent = await UserFile.LoadFileData(document.FilePath, this.RustService, this.DialogService);
sb.AppendLine($"""
## DOCUMENT {numDocuments}:

View File

@ -5,6 +5,7 @@ using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG.RAGProcesses;
using AIStudio.Tools.Security;
namespace AIStudio.Chat;
@ -293,13 +294,17 @@ public sealed class ContentText : IContent
LOGGER.LogWarning("File attachment '{FilePath}' has a forbidden file type and will be skipped.", document.FilePath);
continue;
}
var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
var safeFileContent = await guardService.EnsureSafeForLlmAsync(
await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue),
PromptInjectionSource.ChatAttachment(document.FilePath));
sb.AppendLine();
sb.AppendLine("---------------------------------------");
sb.AppendLine($"File path: {document.FilePath}");
sb.AppendLine("File content:");
sb.AppendLine("````");
sb.AppendLine(await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue));
sb.AppendLine(safeFileContent);
sb.AppendLine("````");
}

View File

@ -3,6 +3,7 @@ using AIStudio.Tools.Media;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
using AIStudio.Tools.Security;
using Microsoft.AspNetCore.Components;
@ -329,6 +330,11 @@ public partial class ReadFileContent : MSGComponentBase
this.Logger.LogInformation("Successfully loaded file content: {FilePath}", filePath);
return true;
}
catch (PromptInjectionBlockedException)
{
this.Logger.LogWarning("Blocked suspected prompt injection while loading file content: {FilePath}", filePath);
return false;
}
catch (Exception ex)
{
this.Logger.LogError(ex, "Failed to load file content: {FilePath}", filePath);

View File

@ -1,5 +1,6 @@
using AIStudio.Agents;
using AIStudio.Chat;
using AIStudio.Tools.Security;
using Microsoft.AspNetCore.Components;
@ -7,6 +8,9 @@ namespace AIStudio.Components;
public partial class ReadWebContent : MSGComponentBase
{
[Inject]
private PromptInjectionGuardService PromptInjectionGuardService { get; init; } = null!;
[Inject]
private HTMLParser HTMLParser { get; init; } = null!;
@ -87,6 +91,7 @@ public partial class ReadWebContent : MSGComponentBase
this.processStep = this.process[ReadWebContentSteps.PARSING];
this.StateHasChanged();
markdown = this.HTMLParser.ParseToMarkdown(html);
markdown = await this.PromptInjectionGuardService.EnsureSafeForLlmAsync(markdown, PromptInjectionSource.WebContent(this.providedURL));
if (this.PreselectContentCleanerAgent && this.providerSettings != AIStudio.Settings.Provider.NONE)
{
@ -120,6 +125,10 @@ public partial class ReadWebContent : MSGComponentBase
this.StateHasChanged();
}
}
catch (PromptInjectionBlockedException)
{
markdown = string.Empty;
}
catch
{
if (this.AgentIsRunning)

View File

@ -2,6 +2,7 @@
using AIStudio.Components;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
using AIStudio.Tools.Security;
namespace AIStudio.Dialogs;
@ -42,6 +43,11 @@ public partial class DocumentCheckDialog : MSGComponentBase
this.FileContent = fileContent;
}
}
catch (PromptInjectionBlockedException exception)
{
this.Logger.LogWarning(exception, "Blocked suspected prompt injection while previewing '{FilePath}'", this.Document?.FilePath);
this.FileContent = string.Empty;
}
catch (Exception ex)
{
this.Logger.LogError(ex, "Failed to load file content from '{FilePath}'", this.Document);

View File

@ -0,0 +1,50 @@
@using AIStudio.Tools.Security
@inherits MSGComponentBase
<MudDialog>
<TitleContent>
<MudText Typo="Typo.h6" Class="d-flex align-center">
<MudIcon Icon="@Icons.Material.Filled.GppMaybe" Class="mr-2" Color="Color.Warning" />
@T("Prompt Injection Detected")
</MudText>
</TitleContent>
<DialogContent>
@if (this.Result is not null)
{
<MudAlert Severity="Severity.Warning" Variant="Variant.Filled" Dense="true" Class="mb-4">
@T("AI Studio blocked this content before it reached a model or agent.")
</MudAlert>
<MudText Typo="Typo.body1" Class="mb-2">
@T("Source kind"): @this.Result.Source.Kind
</MudText>
<MudText Typo="Typo.body2" Class="mb-4">
@T("Source"): @this.Result.Source.Label
</MudText>
<MudText Typo="Typo.subtitle1" Class="mb-2">
@T("Detected signals")
</MudText>
<MudList T="string" Dense="true" Class="mb-4">
@foreach (var finding in this.Result.Findings)
{
<MudListItem T="string" Icon="@Icons.Material.Filled.ReportProblem">
@($"{finding.Category} / {finding.DetectionStage}: {finding.Snippet}")
</MudListItem>
}
</MudList>
<MudText Typo="Typo.body2">
@T("More information"):
<MudLink Href="@PromptInjectionGuardService.WIKI_URL" Target="_blank">
@PromptInjectionGuardService.WIKI_URL
</MudLink>
</MudText>
}
</DialogContent>
<DialogActions>
<MudButton OnClick="@this.Close" Variant="Variant.Filled">
@T("Close")
</MudButton>
</DialogActions>
</MudDialog>

View File

@ -0,0 +1,17 @@
using AIStudio.Components;
using AIStudio.Tools.Security;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Dialogs;
public partial class PromptInjectionAlertDialog : MSGComponentBase
{
[CascadingParameter]
private IMudDialogInstance MudDialog { get; set; } = null!;
[Parameter]
public PromptInjectionScanResult Result { get; set; } = null!;
private void Close() => this.MudDialog.Close();
}

View File

@ -15,6 +15,11 @@
<ConfigurationSelect OptionDescription="@T("Provider selection when creating new chats")" SelectedValue="@(() => this.SettingsManager.ConfigurationData.Chat.AddChatProviderBehavior)" Data="@ConfigurationSelectDataFactory.GetAddChatProviderBehavior()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.Chat.AddChatProviderBehavior = selectedValue)" OptionHelp="@T("Control how the LLM provider for added chats is selected.")"/>
<ConfigurationSelect OptionDescription="@T("Provider selection when loading a chat and sending assistant results to chat")" SelectedValue="@(() => this.SettingsManager.ConfigurationData.Chat.LoadingProviderBehavior)" Data="@ConfigurationSelectDataFactory.GetLoadingChatProviderBehavior()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.Chat.LoadingProviderBehavior = selectedValue)" OptionHelp="@T("Control how the LLM provider for loaded chats is selected and when assistant results are sent to chat.")"/>
<MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
<ConfigurationOption OptionDescription="@T("Protect against prompt injection in external content?")" LabelOn="@T("Potential prompt injections are blocked before they reach an LLM")" LabelOff="@T("External content is passed through without prompt-injection checks")" State="@(() => this.SettingsManager.ConfigurationData.Chat.EnablePromptInjectionProtection)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.Chat.EnablePromptInjectionProtection = updatedState)" OptionHelp="@T("Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.Chat, x => x.EnablePromptInjectionProtection, out var meta) && meta.IsLocked"/>
<ConfigurationOption OptionDescription="@T("Show a learning alert when prompt injection is detected?")" LabelOn="@T("A blocking alert explains the detected attack pattern")" LabelOff="@T("Only the block notification is shown")" Disabled="@(() => !this.SettingsManager.ConfigurationData.Chat.EnablePromptInjectionProtection)" State="@(() => this.SettingsManager.ConfigurationData.Chat.ShowPromptInjectionAlert)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.Chat.ShowPromptInjectionAlert = updatedState)" OptionHelp="@T("Shows an explanation dialog with an external reference when AI Studio blocks suspicious content.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.Chat, x => x.ShowPromptInjectionAlert, out var meta) && meta.IsLocked"/>
</MudPaper>
<MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
<ConfigurationOption OptionDescription="@T("Preselect chat options?")" LabelOn="@T("Chat options are preselected")" LabelOff="@T("No chat options are preselected")" State="@(() => this.SettingsManager.ConfigurationData.Chat.PreselectOptions)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.Chat.PreselectOptions = updatedState)" OptionHelp="@T("When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.Chat, x => x.PreselectOptions, out var meta) && meta.IsLocked"/>
<ConfigurationProviderSelection Component="Components.CHAT" Data="@this.AvailableLLMProviders" Disabled="@(() => !this.SettingsManager.ConfigurationData.Chat.PreselectOptions)" SelectedValue="@(() => this.SettingsManager.ConfigurationData.Chat.PreselectedProvider)" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.Chat.PreselectedProvider = selectedValue)" IsLocked="() => ManagedConfiguration.TryGet(x => x.Chat, x => x.PreselectedProvider, out var meta) && meta.IsLocked"/>

View File

@ -7,6 +7,7 @@ using AIStudio.Tools.Media;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
using AIStudio.Tools.Security;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Routing;
@ -71,6 +72,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
private bool startupCompleted;
private bool settingsWriteProtectionWarningShown;
private readonly SemaphoreSlim mandatoryInfoDialogSemaphore = new(1, 1);
private readonly SemaphoreSlim promptInjectionDialogSemaphore = new(1, 1);
private IReadOnlyCollection<NavBarItem> navItems = [];
@ -112,7 +114,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
this.MessageBus.ApplyFilters(this, [],
[
Event.UPDATE_AVAILABLE, Event.CONFIGURATION_CHANGED, Event.COLOR_THEME_CHANGED, Event.SHOW_ERROR,
Event.SHOW_WARNING, Event.SHOW_SUCCESS, Event.SHOW_INFO, Event.STARTUP_PLUGIN_SYSTEM, Event.PLUGINS_RELOADED,
Event.SHOW_WARNING, Event.SHOW_SUCCESS, Event.SHOW_PROMPT_INJECTION_ALERT, Event.STARTUP_PLUGIN_SYSTEM, Event.PLUGINS_RELOADED,
Event.INSTALL_UPDATE, Event.STARTUP_COMPLETED, Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED,
Event.CHAT_GENERATION_CHANGED, Event.ASSISTANT_SESSION_CHANGED, Event.ASSISTANT_SESSION_FINISHED,
]);
@ -253,6 +255,12 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
success.Show(this.Snackbar);
break;
case Event.SHOW_PROMPT_INJECTION_ALERT:
if (data is PromptInjectionAlertMessage promptInjectionAlert)
await this.ShowPromptInjectionAlertAsync(promptInjectionAlert);
break;
case Event.SHOW_ERROR:
if (data is DataErrorMessage error)
@ -347,6 +355,29 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
}
});
}
private async Task ShowPromptInjectionAlertAsync(PromptInjectionAlertMessage alert)
{
await this.promptInjectionDialogSemaphore.WaitAsync();
try
{
var dialogParameters = new DialogParameters<PromptInjectionAlertDialog>
{
{ x => x.Result, alert.Result },
};
var dialogReference = await this.DialogService.ShowAsync<PromptInjectionAlertDialog>(
T("Prompt Injection Detected"),
dialogParameters,
DialogOptions.BLOCKING_FULLSCREEN);
await dialogReference.Result;
}
finally
{
this.promptInjectionDialogSemaphore.Release();
}
}
public Task<TResult?> ProcessMessageWithResult<TPayload, TResult>(ComponentBase? sendingComponent, Event triggeredEvent, TPayload? data)
{

View File

@ -259,6 +259,14 @@ CONFIG["SETTINGS"] = {}
-- Examples are PRE_WRITER_MODE_2024 and PRE_RAG_2024.
-- CONFIG["SETTINGS"]["DataApp.EnabledPreviewFeatures"] = { "PRE_RAG_2024" }
-- Configure prompt-injection protection for external content such as webpages,
-- retrieved context, and file attachments before it is sent to an LLM.
-- CONFIG["SETTINGS"]["DataChat.EnablePromptInjectionProtection"] = true
-- Configure whether AI Studio shows a blocking explanation dialog when
-- suspicious content is detected and blocked.
-- CONFIG["SETTINGS"]["DataChat.ShowPromptInjectionAlert"] = true
-- Configure the preselected provider.
-- It must be one of the provider IDs defined in CONFIG["LLM_PROVIDERS"].
-- Please note: using an empty string ("") will lock the preselected provider selection, even though no valid preselected provider is found.

View File

@ -10,6 +10,7 @@ using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.PluginSystem.Assistants;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
using AIStudio.Tools.Security;
using Microsoft.AspNetCore.DataProtection;
using Microsoft.AspNetCore.Server.Kestrel.Core;
@ -193,6 +194,8 @@ internal sealed class Program
builder.Services.AddHostedService<RustAvailabilityMonitorService>();
builder.Services.AddScoped<NativeShareService>();
builder.Services.AddScoped<PluginShareService>();
builder.Services.AddSingleton<PromptInjectionScanner>();
builder.Services.AddSingleton<PromptInjectionGuardService>();
// ReSharper disable AccessToDisposedClosure
builder.Services.AddHostedService<RustService>(_ => rust);
@ -300,4 +303,4 @@ internal sealed class Program
PluginFactory.Dispose();
programLogger.LogInformation("The AI Studio server was stopped.");
}
}
}

View File

@ -92,7 +92,16 @@ public sealed class DataChat(Expression<Func<Data, DataChat>>? configSelection =
this.PreselectedDataSourceIds = [..value.PreselectedDataSourceIds];
}
}
/// <summary>
/// Whether prompt-injection protection is enabled for external and attached content.
/// </summary>
public bool EnablePromptInjectionProtection { get; set; } = ManagedConfiguration.Register(configSelection, n => n.EnablePromptInjectionProtection, true);
/// <summary>
/// Whether an alert dialog should be shown when prompt-injection content is blocked.
/// </summary>
public bool ShowPromptInjectionAlert { get; set; } = ManagedConfiguration.Register(configSelection, n => n.ShowPromptInjectionAlert, true);
/// <summary>
/// Should we show the latest message after loading? When false, we show the first (aka oldest) message.
/// </summary>

View File

@ -5,6 +5,7 @@ using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG.RAGProcesses;
using AIStudio.Tools.Security;
namespace AIStudio.Tools.AIJobs;
@ -266,6 +267,12 @@ public sealed class AIJobService(
await this.CompleteChatGenerationAsync(state, AIJobStatus.FAILED, e.UserMessage);
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, e.UserMessage));
}
catch (PromptInjectionBlockedException e)
{
logger.LogWarning(e, "Blocked prompt-injection content during chat generation job '{JobId}'.", state.Snapshot.JobId);
RemoveEmptyAIResponse(state);
await this.CompleteChatGenerationAsync(state, AIJobStatus.FAILED, e.Message);
}
catch (Exception e)
{
logger.LogError(e, "The chat generation job '{JobId}' failed.", state.Snapshot.JobId);

View File

@ -72,6 +72,11 @@ public enum Event
/// Requests display of a success notification.
/// </summary>
SHOW_SUCCESS,
/// <summary>
/// Requests display of a prompt-injection alert dialog.
/// </summary>
SHOW_PROMPT_INJECTION_ALERT,
/// <summary>
/// Requests display of an informational notification.

View File

@ -281,6 +281,8 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectedDataSourcesAutomaticValidation, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectedDataSourceIds, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.SendToChatDataSourceBehavior, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.EnablePromptInjectionProtection, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.ShowPromptInjectionAlert, this.Id, settingsTable, dryRun);
// Config: transcription provider?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.UseTranscriptionProvider, Guid.Empty, this.Id, settingsTable, dryRun);

View File

@ -1,5 +1,5 @@
using System.Text;
using AIStudio.Tools.Security;
using AIStudio.Chat;
namespace AIStudio.Tools.RAG;
@ -16,7 +16,18 @@ public static class IRetrievalContextExtensions
foreach(var retrievalContext in retrievalContexts)
{
index++;
await retrievalContext.AsMarkdown(sb, index, retrievalContexts.Count, token);
try
{
await retrievalContext.AsMarkdown(sb, index, retrievalContexts.Count, token);
}
catch (PromptInjectionBlockedException exception)
{
LOGGER.LogWarning(
exception,
"Skipping retrieval context '{DataSourceName}' at '{Path}' because it was blocked by prompt-injection protection.",
retrievalContext.DataSourceName,
retrievalContext.Path);
}
}
return sb.ToString();
@ -25,74 +36,81 @@ public static class IRetrievalContextExtensions
public static async Task<string> AsMarkdown(this IRetrievalContext retrievalContext, StringBuilder? sb = null, int index = -1, int numTotalRetrievalContexts = -1, CancellationToken token = default)
{
sb ??= new StringBuilder();
var contextBuilder = new StringBuilder();
switch (index)
{
case > 0 when numTotalRetrievalContexts is -1:
sb.AppendLine($"# Retrieval context {index}");
contextBuilder.AppendLine($"# Retrieval context {index}");
break;
case > 0 when numTotalRetrievalContexts > 0:
sb.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}");
contextBuilder.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}");
break;
default:
sb.AppendLine("# Retrieval context");
contextBuilder.AppendLine("# Retrieval context");
break;
}
sb.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
sb.AppendLine($"Content category: {retrievalContext.Category}");
sb.AppendLine($"Content type: {retrievalContext.Type}");
sb.AppendLine($"Content path: {retrievalContext.Path}");
contextBuilder.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
contextBuilder.AppendLine($"Content category: {retrievalContext.Category}");
contextBuilder.AppendLine($"Content type: {retrievalContext.Type}");
contextBuilder.AppendLine($"Content path: {retrievalContext.Path}");
if(retrievalContext.Links.Count > 0)
{
sb.AppendLine("Additional links:");
contextBuilder.AppendLine("Additional links:");
foreach(var link in retrievalContext.Links)
sb.AppendLine($"- {link}");
contextBuilder.AppendLine($"- {link}");
}
var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
var source = PromptInjectionSource.RetrievalContext(retrievalContext.DataSourceName, retrievalContext.Path);
switch(retrievalContext)
{
case RetrievalTextContext textContext:
sb.AppendLine();
sb.AppendLine("Matched text content:");
sb.AppendLine("````");
sb.AppendLine(textContext.MatchedText);
sb.AppendLine("````");
contextBuilder.AppendLine();
contextBuilder.AppendLine("Matched text content:");
contextBuilder.AppendLine("````");
contextBuilder.AppendLine(textContext.MatchedText);
contextBuilder.AppendLine("````");
if(textContext.SurroundingContent.Count > 0)
{
sb.AppendLine();
sb.AppendLine("Surrounding text content:");
contextBuilder.AppendLine();
contextBuilder.AppendLine("Surrounding text content:");
foreach(var surrounding in textContext.SurroundingContent)
{
sb.AppendLine();
sb.AppendLine("````");
sb.AppendLine(surrounding);
sb.AppendLine("````");
contextBuilder.AppendLine();
contextBuilder.AppendLine("````");
contextBuilder.AppendLine(surrounding);
contextBuilder.AppendLine("````");
}
}
await guardService.EnsureSafeForLlmAsync(contextBuilder.ToString(), source);
break;
case RetrievalImageContext imageContext:
sb.AppendLine();
sb.AppendLine("Matched image content as base64-encoded data:");
sb.AppendLine("````");
sb.AppendLine(await imageContext.TryAsBase64(token) is (success: true, { } base64Image)
await guardService.EnsureSafeForLlmAsync(contextBuilder.ToString(), source);
contextBuilder.AppendLine();
contextBuilder.AppendLine("Matched image content as base64-encoded data:");
contextBuilder.AppendLine("````");
contextBuilder.AppendLine(await imageContext.TryAsBase64(token) is (success: true, { } base64Image)
? base64Image
: string.Empty);
sb.AppendLine("````");
contextBuilder.AppendLine("````");
break;
default:
await guardService.EnsureSafeForLlmAsync(contextBuilder.ToString(), source);
LOGGER.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet.");
break;
}
sb.AppendLine();
contextBuilder.AppendLine();
sb.Append(contextBuilder);
return sb.ToString();
}
}

View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Security;
public sealed record PromptInjectionAlertMessage(PromptInjectionScanResult Result);

View File

@ -0,0 +1,6 @@
namespace AIStudio.Tools.Security;
public sealed class PromptInjectionBlockedException(PromptInjectionScanResult result, string message) : Exception(message)
{
public PromptInjectionScanResult Result { get; } = result;
}

View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Security;
public sealed record PromptInjectionFinding(string RuleId, string Category, string DetectionStage, string Snippet);

View File

@ -0,0 +1,44 @@
using AIStudio.Settings;
using AIStudio.Tools.PluginSystem;
namespace AIStudio.Tools.Security;
public sealed class PromptInjectionGuardService(
PromptInjectionScanner scanner,
SettingsManager settingsManager,
ILogger<PromptInjectionGuardService> logger)
{
public const string WIKI_URL = "https://de.wikipedia.org/wiki/Prompt-Engineering#Prompt_Injection";
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PromptInjectionGuardService).Namespace, nameof(PromptInjectionGuardService));
public async Task<string> EnsureSafeForLlmAsync(string text, PromptInjectionSource source)
{
if (!settingsManager.ConfigurationData.Chat.EnablePromptInjectionProtection || string.IsNullOrWhiteSpace(text))
return text;
var result = scanner.Scan(text, source);
if (!result.IsBlocked)
return text;
var message = string.Format(TB("AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt."), source.Label);
await this.HandleDetectionAsync(result, message);
throw new PromptInjectionBlockedException(result, message);
}
private async Task HandleDetectionAsync(PromptInjectionScanResult result, string message)
{
logger.LogWarning(
"Blocked suspected prompt injection in {SourceKind} '{SourceLabel}'. RuleIds={RuleIds}",
result.Source.Kind,
result.Source.Label,
string.Join(", ", result.RuleIds));
await MessageBus.INSTANCE.SendWarning(new(
Icons.Material.Filled.GppMaybe,
message));
if (settingsManager.ConfigurationData.Chat.ShowPromptInjectionAlert)
await MessageBus.INSTANCE.SendMessage<PromptInjectionAlertMessage>(null, Event.SHOW_PROMPT_INJECTION_ALERT, new(result));
}
}

View File

@ -0,0 +1,141 @@
using System.Text.RegularExpressions;
namespace AIStudio.Tools.Security;
internal readonly record struct PromptInjectionRegexRule(string Id, string Category, Regex Regex);
internal static partial class PromptInjectionPatterns
{
internal static readonly IReadOnlyList<PromptInjectionRegexRule> RULES =
[
new("instruction_override", "override", InstructionOverrideRegex()),
new("instruction_priority_override", "override", InstructionPriorityOverrideRegex()),
new("system_prompt_spoofing", "role_override", SystemPromptSpoofingRegex()),
new("system_prompt_exfiltration", "exfiltration", SystemPromptExfiltrationRegex()),
new("prompt_echo_exfiltration", "exfiltration", PromptEchoExfiltrationRegex()),
new("policy_bypass", "override", PolicyBypassRegex()),
new("role_reassignment", "role_override", RoleReassignmentRegex()),
new("privileged_persona_activation", "jailbreak", PrivilegedPersonaActivationRegex()),
new("tool_or_secret_exfiltration", "exfiltration", ToolOrSecretExfiltrationRegex()),
new("conversation_memory_exfiltration", "exfiltration", ConversationMemoryExfiltrationRegex()),
new("tool_call_manipulation", "agent_manipulation", ToolCallManipulationRegex()),
new("agent_thought_injection", "agent_manipulation", AgentThoughtInjectionRegex()),
new("delimiter_wrapped_attack", "delimiter_evasion", DelimiterWrappedAttackRegex()),
new("hidden_markup_injection", "markup_evasion", HiddenMarkupInjectionRegex()),
new("latex_invisible_text", "markup_evasion", LatexInvisibleTextRegex()),
new("unicode_smuggling", "encoding_evasion", UnicodeSmugglingRegex()),
new("ignore_safety_after_data", "override", IgnoreSafetyAfterDataRegex()),
new("persistent_or_delayed_trigger", "persistence", PersistentOrDelayedTriggerRegex()),
new("jailbreak_marker", "jailbreak", JailbreakMarkerRegex()),
];
private const RegexOptions RULE_OPTIONS = RegexOptions.IgnoreCase | RegexOptions.CultureInvariant;
private const int MATCH_TIMEOUT_MILLISECONDS = 100;
private const string INSTRUCTION_OVERRIDE_PATTERN = """(?:ignore|disregard|forget|bypass|override|replace|drop)\s+(?:all\s+)?(?:previous|prior|above|earlier)\s+(?:instructions?|prompts?|messages?|rules?)""";
private const string INSTRUCTION_PRIORITY_OVERRIDE_PATTERN = """(?:(?:new|following|these)\s+(?:instructions?|rules?|prompts?)\s+(?:are|is)\s+(?:now\s+)?(?:the\s+)?(?:highest|top|only)\s+priority|(?:take|takes|treat)\s+(?:the\s+)?(?:following|these|this)\s+as\s+(?:the\s+)?(?:new\s+)?(?:system|developer)\s+(?:prompt|message|instructions?)|(?:supersede|replace|override)\s+(?:the\s+)?(?:system|developer|previous|prior|earlier)\s+(?:prompt|message|instructions?|rules?))""";
private const string SYSTEM_PROMPT_SPOOFING_PATTERN = """(?:(?:this|the\s+following)\s+is\s+(?:a\s+)?(?:system|developer)\s+(?:prompt|message|instruction)|(?:prepend|insert|write)\s+(?:a\s+)?(?:system|developer)\s+(?:prompt|message|instruction)|(?:system|developer|assistant)\s*[:>#-]\s*(?:ignore|bypass|override|reveal|you\s+are\s+now))""";
private const string SYSTEM_PROMPT_EXFILTRATION_PATTERN = """(?:reveal|show|print|display|dump|expose|leak|tell\s+me|return|quote|repeat\s+back)\s+(?:the\s+)?(?:hidden\s+|full\s+|exact\s+|verbatim\s+|initial\s+|original\s+)?(?:system|developer|assistant)\s+(?:prompt|message|instructions?)""";
private const string PROMPT_ECHO_EXFILTRATION_PATTERN = """(?:(?:what\s+(?:were|are))\s+your\s+(?:exact|full|hidden|original)\s+(?:instructions?|prompt)|(?:repeat|quote|print|output|display)\s+(?:the\s+)?text\s+(?:above|before|from\s+the\s+top)\s+(?:verbatim|exactly)?|starting\s+with\s+["']?you\s+are)""";
private const string POLICY_BYPASS_PATTERN = """(?:do\s+not|don't|stop\s+to|never)\s+(?:follow|obey|respect|apply|enforce)\s+(?:the\s+)?(?:system|developer|safety|security|content|usage)\s+(?:prompt|message|instructions?|policy|policies|guardrails?|restrictions?)""";
private const string ROLE_REASSIGNMENT_PATTERN = """(?:you\s+are\s+now|you\s+are\s+no\s+longer|act\s+as|pretend\s+to\s+be|simulate\s+being|assume\s+the\s+role\s+of|from\s+now\s+on\s+you\s+are)\s+(?:an\s+)?(?:unfiltered|unrestricted|developer|system|root|admin|jailbroken|evil|dan|do\s+anything\s+now)""";
private const string PRIVILEGED_PERSONA_ACTIVATION_PATTERN = """\b(?:developer\s+mode|debug\s+mode|admin\s+mode|root\s+mode|god\s+mode|maintenance\s+mode|dan\s*(?:mode)?|do\s+anything\s+now|grandmother\s+trick)\b""";
private const string TOOL_OR_SECRET_EXFILTRATION_PATTERN = """(?:export|send|return|reveal|show|print|list|dump|exfiltrate)\s+(?:all\s+)?(?:tools?|functions?|plugins?|api\s*keys?|keys?|tokens?|credentials?|secrets?|passwords?|hidden\s+instructions?|environment\s+variables?|system\s+information)""";
private const string CONVERSATION_MEMORY_EXFILTRATION_PATTERN = """(?:(?:show|print|reveal|return|dump|list)\s+(?:the\s+)?(?:conversation\s+history|chat\s+history|memory|scratchpad|chain\s+of\s+thought|reasoning|previous\s+user\s+messages?|prior\s+messages?)|(?:what\s+did\s+(?:the\s+)?previous\s+user\s+say))""";
private const string TOOL_CALL_MANIPULATION_PATTERN = """(?:(?:call|invoke|execute|run|use|trigger)\s+(?:the\s+)?(?:tool|function|plugin|api|browser|web|shell|terminal|command)[^\n]{0,120}(?:with|using|to)\s+(?:these\s+)?(?:arguments|params?|parameters)|(?:do\s+not|don't)\s+ask\s+for\s+(?:confirmation|approval|permission)|(?:silently|secretly|without\s+asking)\s+(?:call|invoke|execute|run|use))""";
private const string AGENT_THOUGHT_INJECTION_PATTERN = """(?:(?:thought|observation|reasoning|scratchpad|tool\s+output|assistant|system|developer)\s*[:=]\s*(?:ignore|bypass|override|reveal|call|execute)|forge\s+(?:an\s+)?(?:observation|tool\s+output|assistant\s+message)|pretend\s+(?:the\s+)?tool\s+(?:returned|said))""";
private const string DELIMITER_WRAPPED_ATTACK_PATTERN = """(?:^|\n)\s*(?:<{2,}|>{2,}|`{3,}|#{1,6}\s*)(?:\s*(?:system|developer|assistant|instructions?|prompt)\b)""";
private const string HIDDEN_MARKUP_INJECTION_PATTERN = """(?:<!--[^>\r\n]{0,300}(?:ignore|bypass|override|reveal|system\s+prompt)[^>\r\n]{0,300}-->|<(?:span|div|p|font|section)[^>]{0,200}(?:display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0|font-size\s*:\s*0|color\s*:\s*(?:white|#fff(?:fff)?|rgb\(\s*255\s*,\s*255\s*,\s*255\s*\)))[^>]{0,200}>)""";
private const string LATEX_INVISIBLE_TEXT_PATTERN = """(?:\\(?:color|textcolor)\s*\{\s*white\s*\}\s*\{[^}]{0,300}\}|\\(?:fontsize|tiny|scriptsize)\b[^\r\n]{0,120}(?:ignore|bypass|override|reveal))""";
private const string UNICODE_SMUGGLING_PATTERN = """[\u200B-\u200F\u2060-\u2064\u2066-\u2069\uFEFF]""";
private const string IGNORE_SAFETY_AFTER_DATA_PATTERN = """(?:after\s+reading|once\s+you\s+read|when\s+you\s+see)\s+.*?(?:ignore|bypass|override)\s+.*?(?:instructions?|safety|rules?)""";
private const string PERSISTENT_OR_DELAYED_TRIGGER_PATTERN = """(?:(?:remember|store|save|persist|memorize)\s+(?:this|these|the\s+following)\s+(?:instructions?|rules?|message)|(?:later|in\s+the\s+next\s+message|when\s+you\s+see|whenever\s+you\s+read|if\s+you\s+encounter)\s+.{0,120}(?:ignore|bypass|override|reveal|exfiltrate))""";
private const string JAILBREAK_MARKER_PATTERN = """\b(?:jailbreak|prompt\s+injection|ignore\s+your\s+guardrails?|bypass\s+(?:your\s+)?(?:guardrails?|safety)|unfiltered\s+mode|do\s+anything\s+now|developer\s+mode|admin\s+mode|root\s+mode)\b""";
private const string ANY_RULE_PATTERN =
"(?:" + INSTRUCTION_OVERRIDE_PATTERN + ")|(?:" +
INSTRUCTION_PRIORITY_OVERRIDE_PATTERN + ")|(?:" + SYSTEM_PROMPT_SPOOFING_PATTERN + ")|(?:" +
SYSTEM_PROMPT_EXFILTRATION_PATTERN + ")|(?:" + PROMPT_ECHO_EXFILTRATION_PATTERN + ")|(?:" +
POLICY_BYPASS_PATTERN + ")|(?:" + ROLE_REASSIGNMENT_PATTERN + ")|(?:" +
PRIVILEGED_PERSONA_ACTIVATION_PATTERN + ")|(?:" + TOOL_OR_SECRET_EXFILTRATION_PATTERN + ")|(?:" +
CONVERSATION_MEMORY_EXFILTRATION_PATTERN + ")|(?:" + TOOL_CALL_MANIPULATION_PATTERN + ")|(?:" +
AGENT_THOUGHT_INJECTION_PATTERN + ")|(?:" + DELIMITER_WRAPPED_ATTACK_PATTERN + ")|(?:" +
HIDDEN_MARKUP_INJECTION_PATTERN + ")|(?:" + LATEX_INVISIBLE_TEXT_PATTERN + ")|(?:" +
UNICODE_SMUGGLING_PATTERN + ")|(?:" + IGNORE_SAFETY_AFTER_DATA_PATTERN + ")|(?:" +
PERSISTENT_OR_DELAYED_TRIGGER_PATTERN + ")|(?:" + JAILBREAK_MARKER_PATTERN + ")";
[GeneratedRegex(ANY_RULE_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
internal static partial Regex AnyRuleRegex();
[GeneratedRegex(@"\b[a-zA-Z](?:[\s._:/\\|-]+[a-zA-Z]){2,}\b", RegexOptions.CultureInvariant)]
internal static partial Regex SpacedLetterSequenceRegex();
[GeneratedRegex(@"\b[a-zA-Z]{5,12}\b", RegexOptions.CultureInvariant)]
internal static partial Regex WordRegex();
[GeneratedRegex(@"(?<![A-Za-z0-9+/=])[A-Za-z0-9+/]{16,}={0,2}(?![A-Za-z0-9+/=])", RegexOptions.CultureInvariant)]
internal static partial Regex Base64Regex();
[GeneratedRegex(@"(?<![0-9A-Fa-f])(?:[0-9A-Fa-f]{2}(?:[\s:-]+|$)){8,}", RegexOptions.CultureInvariant)]
internal static partial Regex HexPairRegex();
[GeneratedRegex(@"(?<![0-9A-Fa-f])(?:[0-9A-Fa-f]{2}){8,}(?![0-9A-Fa-f])", RegexOptions.CultureInvariant)]
internal static partial Regex HexCompactRegex();
[GeneratedRegex(INSTRUCTION_OVERRIDE_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex InstructionOverrideRegex();
[GeneratedRegex(INSTRUCTION_PRIORITY_OVERRIDE_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex InstructionPriorityOverrideRegex();
[GeneratedRegex(SYSTEM_PROMPT_SPOOFING_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex SystemPromptSpoofingRegex();
[GeneratedRegex(SYSTEM_PROMPT_EXFILTRATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex SystemPromptExfiltrationRegex();
[GeneratedRegex(PROMPT_ECHO_EXFILTRATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex PromptEchoExfiltrationRegex();
[GeneratedRegex(POLICY_BYPASS_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex PolicyBypassRegex();
[GeneratedRegex(ROLE_REASSIGNMENT_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex RoleReassignmentRegex();
[GeneratedRegex(PRIVILEGED_PERSONA_ACTIVATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex PrivilegedPersonaActivationRegex();
[GeneratedRegex(TOOL_OR_SECRET_EXFILTRATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex ToolOrSecretExfiltrationRegex();
[GeneratedRegex(CONVERSATION_MEMORY_EXFILTRATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex ConversationMemoryExfiltrationRegex();
[GeneratedRegex(TOOL_CALL_MANIPULATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex ToolCallManipulationRegex();
[GeneratedRegex(AGENT_THOUGHT_INJECTION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex AgentThoughtInjectionRegex();
[GeneratedRegex(DELIMITER_WRAPPED_ATTACK_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex DelimiterWrappedAttackRegex();
[GeneratedRegex(HIDDEN_MARKUP_INJECTION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex HiddenMarkupInjectionRegex();
[GeneratedRegex(LATEX_INVISIBLE_TEXT_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex LatexInvisibleTextRegex();
[GeneratedRegex(UNICODE_SMUGGLING_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex UnicodeSmugglingRegex();
[GeneratedRegex(IGNORE_SAFETY_AFTER_DATA_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex IgnoreSafetyAfterDataRegex();
[GeneratedRegex(PERSISTENT_OR_DELAYED_TRIGGER_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex PersistentOrDelayedTriggerRegex();
[GeneratedRegex(JAILBREAK_MARKER_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex JailbreakMarkerRegex();
}

View File

@ -0,0 +1,12 @@
namespace AIStudio.Tools.Security;
public sealed class PromptInjectionScanResult(PromptInjectionSource source, IReadOnlyList<PromptInjectionFinding> findings)
{
public PromptInjectionSource Source { get; } = source;
public IReadOnlyList<PromptInjectionFinding> Findings { get; } = findings;
public bool IsBlocked => this.Findings.Count > 0;
public IReadOnlyList<string> RuleIds => this.Findings.Select(finding => finding.RuleId).Distinct(StringComparer.Ordinal).ToList();
}

View File

@ -0,0 +1,293 @@
using System.Buffers;
using System.Text;
using System.Text.RegularExpressions;
namespace AIStudio.Tools.Security;
public sealed class PromptInjectionScanner(ILogger<PromptInjectionScanner> logger)
{
private const int MAX_DECODED_CANDIDATES_PER_ENCODING = 12;
private const int MAX_DECODED_TEXT_LENGTH = 12_000;
private const int MAX_FINDINGS = 8;
private const int SNIPPET_RADIUS = 80;
private static readonly IReadOnlyDictionary<(int Length, char First, char Last), string[]> TYPOGLYCEMIA_KEYWORDS =
CreateTypoglycemiaKeywordIndex();
public PromptInjectionScanResult Scan(string text, PromptInjectionSource source)
{
if (string.IsNullOrWhiteSpace(text))
return new(source, []);
var findings = new List<PromptInjectionFinding>();
var findingKeys = new HashSet<string>(StringComparer.Ordinal);
this.ScanVariant(text, "raw", findings, findingKeys);
if (findings.Count >= MAX_FINDINGS)
return new(source, findings);
var collapsed = CollapseCharacterSpacedContent(text);
if (!string.Equals(text, collapsed, StringComparison.Ordinal))
this.ScanVariant(collapsed, "character_spacing", findings, findingKeys);
if (findings.Count < MAX_FINDINGS)
this.ScanDecodedCandidates(text, findings, findingKeys);
if (findings.Count < MAX_FINDINGS)
this.ScanTypoglycemia(text, findings, findingKeys);
return new(source, findings);
}
private void ScanVariant(string text, string stage, List<PromptInjectionFinding> findings, HashSet<string> findingKeys)
{
try
{
if (!PromptInjectionPatterns.AnyRuleRegex().IsMatch(text))
return;
}
catch (RegexMatchTimeoutException exception)
{
logger.LogWarning(exception, "Prompt-injection regex prefilter timed out during stage '{Stage}'. Falling back to individual rules.", stage);
}
foreach (var rule in PromptInjectionPatterns.RULES)
{
if (findings.Count >= MAX_FINDINGS)
return;
Match match;
try
{
match = rule.Regex.Match(text);
}
catch (RegexMatchTimeoutException exception)
{
logger.LogWarning(exception, "Prompt-injection regex '{RuleId}' timed out during stage '{Stage}'.", rule.Id, stage);
continue;
}
if (!match.Success)
continue;
var snippet = ExtractSnippet(text, match.Index, match.Length, SNIPPET_RADIUS);
AddFinding(findings, findingKeys, new(rule.Id, rule.Category, stage, snippet));
}
}
private void ScanDecodedCandidates(string text, List<PromptInjectionFinding> findings, HashSet<string> findingKeys)
{
var processed = 0;
var seenDecodedTexts = new HashSet<string>(StringComparer.Ordinal);
foreach (var match in PromptInjectionPatterns.Base64Regex().EnumerateMatches(text))
{
if (processed++ >= MAX_DECODED_CANDIDATES_PER_ENCODING || findings.Count >= MAX_FINDINGS)
break;
var decoded = TryDecodeBase64(text.AsSpan(match.Index, match.Length));
if (decoded is not null && seenDecodedTexts.Add(decoded))
this.ScanVariant(decoded, "decoded_base64", findings, findingKeys);
}
processed = 0;
seenDecodedTexts.Clear();
foreach (var match in PromptInjectionPatterns.HexPairRegex().EnumerateMatches(text))
{
if (processed++ >= MAX_DECODED_CANDIDATES_PER_ENCODING || findings.Count >= MAX_FINDINGS)
break;
var decoded = TryDecodeHex(text.AsSpan(match.Index, match.Length));
if (decoded is not null && seenDecodedTexts.Add(decoded))
this.ScanVariant(decoded, "decoded_hex_pairs", findings, findingKeys);
}
processed = 0;
seenDecodedTexts.Clear();
foreach (var match in PromptInjectionPatterns.HexCompactRegex().EnumerateMatches(text))
{
if (processed++ >= MAX_DECODED_CANDIDATES_PER_ENCODING || findings.Count >= MAX_FINDINGS)
break;
var decoded = TryDecodeHex(text.AsSpan(match.Index, match.Length));
if (decoded is not null && seenDecodedTexts.Add(decoded))
this.ScanVariant(decoded, "decoded_hex", findings, findingKeys);
}
}
private void ScanTypoglycemia(string text, List<PromptInjectionFinding> findings, HashSet<string> findingKeys)
{
foreach (var match in PromptInjectionPatterns.WordRegex().EnumerateMatches(text))
{
if (findings.Count >= MAX_FINDINGS)
return;
var token = text.AsSpan(match.Index, match.Length);
var key = (token.Length, char.ToLowerInvariant(token[0]), char.ToLowerInvariant(token[^1]));
if (!TYPOGLYCEMIA_KEYWORDS.TryGetValue(key, out var keywords))
continue;
foreach (var keyword in keywords)
{
if (!IsTypoglycemiaVariant(token, keyword))
continue;
var snippet = ExtractSnippet(text, match.Index, match.Length, SNIPPET_RADIUS);
AddFinding(findings, findingKeys, new($"typoglycemia:{keyword}", "evasion", "typoglycemia", snippet));
break;
}
}
}
private static bool IsTypoglycemiaVariant(ReadOnlySpan<char> token, string keyword)
{
if (token.Equals(keyword, StringComparison.OrdinalIgnoreCase))
return false;
Span<int> characterCounts = stackalloc int[26];
for (var index = 1; index < token.Length - 1; index++)
{
characterCounts[char.ToLowerInvariant(token[index]) - 'a']++;
characterCounts[keyword[index] - 'a']--;
}
foreach (var count in characterCounts)
{
if (count != 0)
return false;
}
return true;
}
private static string CollapseCharacterSpacedContent(string text)
{
return PromptInjectionPatterns.SpacedLetterSequenceRegex().Replace(text, static match =>
{
var builder = new StringBuilder(match.Value.Length);
foreach (var character in match.Value)
{
if (char.IsLetter(character))
builder.Append(character);
}
return builder.ToString();
});
}
private static string? TryDecodeBase64(ReadOnlySpan<char> candidate)
{
var encodedLength = Math.Min(candidate.Length, ((MAX_DECODED_TEXT_LENGTH + 2) / 3) * 4);
encodedLength -= encodedLength % 4;
if (encodedLength == 0)
return null;
var bytes = ArrayPool<byte>.Shared.Rent(MAX_DECODED_TEXT_LENGTH);
try
{
if (!Convert.TryFromBase64Chars(candidate[..encodedLength], bytes, out var bytesWritten))
return null;
return ConvertDecodedBytesToText(bytes.AsSpan(0, bytesWritten));
}
finally
{
ArrayPool<byte>.Shared.Return(bytes);
}
}
private static string? TryDecodeHex(ReadOnlySpan<char> candidate)
{
var bytes = ArrayPool<byte>.Shared.Rent(MAX_DECODED_TEXT_LENGTH);
try
{
var bytesWritten = 0;
var highNibble = -1;
foreach (var character in candidate)
{
var nibble = HexValue(character);
if (nibble < 0)
continue;
if (highNibble < 0)
{
highNibble = nibble;
continue;
}
bytes[bytesWritten++] = (byte)((highNibble << 4) | nibble);
highNibble = -1;
if (bytesWritten >= MAX_DECODED_TEXT_LENGTH)
break;
}
return bytesWritten == 0 ? null : ConvertDecodedBytesToText(bytes.AsSpan(0, bytesWritten));
}
finally
{
ArrayPool<byte>.Shared.Return(bytes);
}
}
private static int HexValue(char character)
{
if (character is >= '0' and <= '9')
return character - '0';
if (character is >= 'A' and <= 'F')
return character - 'A' + 10;
if (character is >= 'a' and <= 'f')
return character - 'a' + 10;
return -1;
}
private static string? ConvertDecodedBytesToText(ReadOnlySpan<byte> bytes)
{
if (bytes.IsEmpty)
return null;
var text = Encoding.UTF8.GetString(bytes);
return LooksTextLike(text) ? text : null;
}
private static bool LooksTextLike(string text)
{
if (string.IsNullOrWhiteSpace(text))
return false;
var printableCount = 0;
foreach (var character in text)
{
if (!char.IsControl(character) || character is '\r' or '\n' or '\t')
printableCount++;
}
return printableCount >= text.Length * 0.85;
}
private static void AddFinding(List<PromptInjectionFinding> findings, HashSet<string> findingKeys, PromptInjectionFinding finding)
{
var key = $"{finding.RuleId}|{finding.DetectionStage}|{finding.Snippet}";
if (findingKeys.Add(key))
findings.Add(finding);
}
private static string ExtractSnippet(string text, int index, int length, int radius)
{
var start = Math.Max(0, index - radius);
var end = Math.Min(text.Length, index + length + radius);
var snippet = text[start..end].Replace('\r', ' ').Replace('\n', ' ').Trim();
return snippet.Length > radius * 2 ? $"{snippet[..(radius * 2)]}..." : snippet;
}
private static IReadOnlyDictionary<(int Length, char First, char Last), string[]> CreateTypoglycemiaKeywordIndex()
{
string[] keywords =
[
"ignore", "bypass", "override", "reveal", "forget", "disregard", "delete", "reset", "expose",
"system", "prompt", "policy", "safety", "developer", "instructions", "admin", "secret", "token", "credential",
];
return keywords
.GroupBy(keyword => (keyword.Length, keyword[0], keyword[^1]))
.ToDictionary(group => group.Key, group => group.ToArray());
}
}

View File

@ -0,0 +1,12 @@
namespace AIStudio.Tools.Security;
public readonly record struct PromptInjectionSource(string Kind, string Label)
{
public static PromptInjectionSource WebContent(string url) => new("Web content", url);
public static PromptInjectionSource FileContent(string filePath) => new("File content", filePath);
public static PromptInjectionSource ChatAttachment(string filePath) => new("Chat attachment", filePath);
public static PromptInjectionSource RetrievalContext(string dataSourceName, string path) => new("Retrieval context", $"{dataSourceName}: {path}");
}

View File

@ -2,6 +2,7 @@
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Services;
using DialogOptions = AIStudio.Dialogs.DialogOptions;
using AIStudio.Tools.Security;
namespace AIStudio.Tools;
@ -46,6 +47,7 @@ public static class UserFile
}
var fileContent = await rustService.ReadArbitraryFileData(filePath, int.MaxValue);
return fileContent;
var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
return await guardService.EnsureSafeForLlmAsync(fileContent, PromptInjectionSource.FileContent(filePath));
}
}

View File

@ -6,6 +6,7 @@
- Added the option to import plugins by dropping a plugin archive onto the plugin page.
- Added the dedicated file extension `.mwplugin` for plugin archives.
- Added an option for organizations to disable importing, sharing, and exporting plugins.
- Added checks for prompt injections in file attachments, images and web pages to process large documents faster and with lower memory usage.
- Improved how your organization's configuration behaves when a configuration plugin is present but cannot be loaded, e.g. because of an error in the plugin. Such a plugin still manages your app, so its settings, providers, data sources, profiles, and chat templates now stay in place instead of being removed.
- Fixed reset buttons in assistants. As you may have noticed in the Document Analysis Assistant, resetting it could leave content from the previous analysis visible. Reset buttons now clear previous results completely.
- Fixed dropping files after you closed a dialog that accepts files itself. Such a dialog takes over dropped files while it is open, but never handed that role back when you closed it. Afterwards, the chat and the assistants silently ignored dropped files until you switched to another page. Each time you opened such a dialog again, the problem got worse.