Add prompt injection detection framework and UI alert dialog

This commit is contained in:
hart_s3 2026-07-14 14:37:11 +02:00
parent 5890b3734a
commit 9e645d1fdb
32 changed files with 836 additions and 37 deletions

View File

@ -4,6 +4,7 @@ using AIStudio.Chat;
using AIStudio.Provider; using AIStudio.Provider;
using AIStudio.Settings; using AIStudio.Settings;
using AIStudio.Tools.RAG; using AIStudio.Tools.RAG;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services; using AIStudio.Tools.Services;
namespace AIStudio.Agents; namespace AIStudio.Agents;
@ -237,7 +238,20 @@ public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalConte
// 2. Prepare the retrieval context for the agent: // 2. Prepare the retrieval context for the agent:
// //
var additionalData = new Dictionary<string, string>(); var additionalData = new Dictionary<string, string>();
var markdownRetrievalContext = await retrievalContext.AsMarkdown(token: token); string markdownRetrievalContext;
try
{
markdownRetrievalContext = await retrievalContext.AsMarkdown(token: token);
}
catch (PromptInjectionBlockedException exception)
{
logger.LogWarning(
exception,
"Blocked retrieval context '{DataSourceName}' at '{Path}' before validation.",
retrievalContext.DataSourceName,
retrievalContext.Path);
return new(false, "The retrieval context was blocked due to suspected prompt injection.", 1.0f, retrievalContext);
}
additionalData.Add("retrievalContext", markdownRetrievalContext); additionalData.Add("retrievalContext", markdownRetrievalContext);
// //

View File

@ -4,6 +4,7 @@ using AIStudio.Settings;
using AIStudio.Dialogs.Settings; using AIStudio.Dialogs.Settings;
using AIStudio.Tools.AIJobs; using AIStudio.Tools.AIJobs;
using AIStudio.Tools.AssistantSessions; using AIStudio.Tools.AssistantSessions;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services; using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components;
@ -263,6 +264,12 @@ public abstract partial class AssistantBase<TSettings> : AssistantLowerBase wher
this.Logger.LogError(e, "The provider request failed for assistant '{AssistantTitle}'. Status={StatusCode}, Reason='{ReasonPhrase}', Body='{ResponseBody}'", this.Title, e.StatusCode, e.ReasonPhrase, e.ResponseBody); this.Logger.LogError(e, "The provider request failed for assistant '{AssistantTitle}'. Status={StatusCode}, Reason='{ReasonPhrase}', Body='{ResponseBody}'", this.Title, e.StatusCode, e.ReasonPhrase, e.ResponseBody);
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, e.UserMessage)); await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, e.UserMessage));
} }
catch (PromptInjectionBlockedException e)
{
sessionStatus = AssistantSessionStatus.FAILED;
errorMessage = e.Message;
this.Logger.LogWarning(e, "Blocked prompt-injection content during assistant session '{AssistantTitle}'.", this.Title);
}
catch (Exception e) catch (Exception e)
{ {
sessionStatus = AssistantSessionStatus.FAILED; sessionStatus = AssistantSessionStatus.FAILED;
@ -451,6 +458,18 @@ public abstract partial class AssistantBase<TSettings> : AssistantLowerBase wher
return string.Empty; return string.Empty;
} }
catch (PromptInjectionBlockedException e)
{
this.Logger.LogWarning(e, "Blocked prompt-injection content before sending assistant request for '{AssistantTitle}'.", this.Title);
if (this.ResultingContentBlock is not null && string.IsNullOrWhiteSpace(aiText.Text))
{
this.ChatThread?.Blocks.Remove(this.ResultingContentBlock);
this.ResultingContentBlock = null;
}
return string.Empty;
}
finally finally
{ {
this.IsProcessing = this.assistantSessionId is not null && (this.AssistantSessionService.TryGetSnapshot(this.assistantSessionKey)?.IsActive ?? false); this.IsProcessing = this.assistantSessionId is not null && (this.AssistantSessionService.TryGetSnapshot(this.assistantSessionKey)?.IsActive ?? false);

View File

@ -705,7 +705,7 @@ public partial class DocumentAnalysisAssistant : AssistantBaseCore<NoSettingsPan
continue; continue;
} }
var fileContent = await this.RustService.ReadArbitraryFileData(document.FilePath, int.MaxValue); var fileContent = await UserFile.LoadFileData(document.FilePath, this.RustService, this.DialogService);
sb.AppendLine($""" sb.AppendLine($"""
## DOCUMENT {numDocuments}: ## DOCUMENT {numDocuments}:

View File

@ -4738,6 +4738,27 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINGGUIDELINEDIALOG::T384594633"] = "Th
-- Prompting Guideline -- Prompting Guideline
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINGGUIDELINEDIALOG::T4250996615"] = "Prompting Guideline" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINGGUIDELINEDIALOG::T4250996615"] = "Prompting Guideline"
-- Source
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1642243064"] = "Source"
-- AI Studio blocked this content before it reached a model or agent.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2378027194"] = "AI Studio blocked this content before it reached a model or agent."
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3448155331"] = "Close"
-- Prompt Injection Detected
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3580322580"] = "Prompt Injection Detected"
-- Source kind
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T47437466"] = "Source kind"
-- More information
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T475337262"] = "More information"
-- Detected signals
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T556618402"] = "Detected signals"
-- Hugging Face Inference Provider -- Hugging Face Inference Provider
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1085481431"] = "Hugging Face Inference Provider" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1085481431"] = "Hugging Face Inference Provider"
@ -5218,6 +5239,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T6790
-- When enabled, you can preselect options. This is might be useful when you prefer a specific language or LLM model. -- When enabled, you can preselect options. This is might be useful when you prefer a specific language or LLM model.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T711745239"] = "When enabled, you can preselect options. This is might be useful when you prefer a specific language or LLM model." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T711745239"] = "When enabled, you can preselect options. This is might be useful when you prefer a specific language or LLM model."
-- Protect against prompt injection in external content?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1040385578"] = "Protect against prompt injection in external content?"
-- Preselect one of your chat templates? -- Preselect one of your chat templates?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1402022556"] = "Preselect one of your chat templates?" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1402022556"] = "Preselect one of your chat templates?"
@ -5239,12 +5263,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1915793195"]
-- Preselect a profile -- Preselect a profile
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2322771068"] = "Preselect a profile" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2322771068"] = "Preselect a profile"
-- A blocking alert explains the detected attack pattern
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2341755932"] = "A blocking alert explains the detected attack pattern"
-- Apply default data source option when sending assistant results to chat -- Apply default data source option when sending assistant results to chat
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2510376349"] = "Apply default data source option when sending assistant results to chat" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2510376349"] = "Apply default data source option when sending assistant results to chat"
-- Control how the LLM provider for added chats is selected. -- Control how the LLM provider for added chats is selected.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T263621180"] = "Control how the LLM provider for added chats is selected." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T263621180"] = "Control how the LLM provider for added chats is selected."
-- Show a learning alert when prompt injection is detected?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2754886869"] = "Show a learning alert when prompt injection is detected?"
-- Provider selection when loading a chat and sending assistant results to chat -- Provider selection when loading a chat and sending assistant results to chat
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2868379953"] = "Provider selection when loading a chat and sending assistant results to chat" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2868379953"] = "Provider selection when loading a chat and sending assistant results to chat"
@ -5254,6 +5284,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2913693228"]
-- Do you want to use any shortcut to send your input? -- Do you want to use any shortcut to send your input?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2936560092"] = "Do you want to use any shortcut to send your input?" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2936560092"] = "Do you want to use any shortcut to send your input?"
-- Shows an explanation dialog with an external reference when AI Studio blocks suspicious content.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3129787554"] = "Shows an explanation dialog with an external reference when AI Studio blocks suspicious content."
-- Would you like to set one of your chat templates as the default for chats? -- Would you like to set one of your chat templates as the default for chats?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3234927721"] = "Would you like to set one of your chat templates as the default for chats?" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3234927721"] = "Would you like to set one of your chat templates as the default for chats?"
@ -5263,6 +5296,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3383186996"]
-- Close -- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3448155331"] = "Close" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3448155331"] = "Close"
-- Only the block notification is shown
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3491678837"] = "Only the block notification is shown"
-- First (oldest) message is shown, after loading a chat -- First (oldest) message is shown, after loading a chat
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3507181366"] = "First (oldest) message is shown, after loading a chat" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3507181366"] = "First (oldest) message is shown, after loading a chat"
@ -5275,9 +5311,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3730599555"]
-- Latest message is shown, after loading a chat -- Latest message is shown, after loading a chat
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3755993611"] = "Latest message is shown, after loading a chat" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3755993611"] = "Latest message is shown, after loading a chat"
-- Potential prompt injections are blocked before they reach an LLM
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3982766677"] = "Potential prompt injections are blocked before they reach an LLM"
-- Do you want to apply the default data source options when sending assistant results to chat? -- Do you want to apply the default data source options when sending assistant results to chat?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4033153439"] = "Do you want to apply the default data source options when sending assistant results to chat?" UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4033153439"] = "Do you want to apply the default data source options when sending assistant results to chat?"
-- External content is passed through without prompt-injection checks
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4245463281"] = "External content is passed through without prompt-injection checks"
-- When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider. -- When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T477675197"] = "When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T477675197"] = "When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider."
@ -5287,6 +5329,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T492357592"] =
-- When enabled, the latest message is shown after loading a chat. When disabled, the first (oldest) message is shown. -- When enabled, the latest message is shown after loading a chat. When disabled, the first (oldest) message is shown.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T582516016"] = "When enabled, the latest message is shown after loading a chat. When disabled, the first (oldest) message is shown." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T582516016"] = "When enabled, the latest message is shown after loading a chat. When disabled, the first (oldest) message is shown."
-- Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T69192586"] = "Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent."
-- Customize your AI experience with chat templates. Whether you want to experiment with prompt engineering, simply use a custom system prompt in the standard chat interface, or create a specialized assistant, our templates give you full control. Similar to common AI companies' playgrounds, you can define your own system prompts and leverage assistant prompts for providers that support them. -- Customize your AI experience with chat templates. Whether you want to experiment with prompt engineering, simply use a custom system prompt in the standard chat interface, or create a specialized assistant, our templates give you full control. Similar to common AI companies' playgrounds, you can define your own system prompts and leverage assistant prompts for providers that support them.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHATTEMPLATE::T1172171653"] = "Customize your AI experience with chat templates. Whether you want to experiment with prompt engineering, simply use a custom system prompt in the standard chat interface, or create a specialized assistant, our templates give you full control. Similar to common AI companies' playgrounds, you can define your own system prompts and leverage assistant prompts for providers that support them." UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHATTEMPLATE::T1172171653"] = "Customize your AI experience with chat templates. Whether you want to experiment with prompt engineering, simply use a custom system prompt in the standard chat interface, or create a specialized assistant, our templates give you full control. Similar to common AI companies' playgrounds, you can define your own system prompts and leverage assistant prompts for providers that support them."
@ -6358,6 +6403,9 @@ UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2936083926"] = "AI Studio could
-- Writer -- Writer
UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2979224202"] = "Writer" UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T2979224202"] = "Writer"
-- Prompt Injection Detected
UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T3580322580"] = "Prompt Injection Detected"
-- Show details -- Show details
UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T3692372066"] = "Show details" UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T3692372066"] = "Show details"
@ -8455,6 +8503,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source like p
-- Document -- Document
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document" UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document"
-- AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONGUARDSERVICE::T3560909296"] = "AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt."
-- Pandoc Installation -- Pandoc Installation
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::PANDOCAVAILABILITYSERVICE::T185447014"] = "Pandoc Installation" UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::PANDOCAVAILABILITYSERVICE::T185447014"] = "Pandoc Installation"

View File

@ -4,6 +4,7 @@ using System.Text.RegularExpressions;
using AIStudio.Chat; using AIStudio.Chat;
using AIStudio.Dialogs; using AIStudio.Dialogs;
using AIStudio.Dialogs.Settings; using AIStudio.Dialogs.Settings;
using AIStudio.Tools.Security;
using AIStudio.Tools.AssistantSessions; using AIStudio.Tools.AssistantSessions;
using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components;
@ -583,6 +584,10 @@ public partial class AssistantPromptOptimizer : AssistantBaseCore<SettingsDialog
if (string.IsNullOrWhiteSpace(this.customPromptingGuidelineContent)) if (string.IsNullOrWhiteSpace(this.customPromptingGuidelineContent))
this.Snackbar.Add(T("The custom prompt guide file is empty or could not be read."), Severity.Warning); this.Snackbar.Add(T("The custom prompt guide file is empty or could not be read."), Severity.Warning);
} }
catch (PromptInjectionBlockedException)
{
this.customPromptingGuidelineContent = string.Empty;
}
catch catch
{ {
this.customPromptingGuidelineContent = string.Empty; this.customPromptingGuidelineContent = string.Empty;

View File

@ -3,10 +3,15 @@ using AIStudio.Chat;
using AIStudio.Dialogs.Settings; using AIStudio.Dialogs.Settings;
using AIStudio.Tools.AssistantSessions; using AIStudio.Tools.AssistantSessions;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.SlideBuilder; namespace AIStudio.Assistants.SlideBuilder;
public partial class SlideAssistant : AssistantBaseCore<SettingsDialogSlideBuilder> public partial class SlideAssistant : AssistantBaseCore<SettingsDialogSlideBuilder>
{ {
[Inject]
private IDialogService DialogService { get; init; } = null!;
protected override Tools.Components Component => Tools.Components.SLIDE_BUILDER_ASSISTANT; protected override Tools.Components Component => Tools.Components.SLIDE_BUILDER_ASSISTANT;
protected override string Title => T("Slide Planner Assistant"); protected override string Title => T("Slide Planner Assistant");
@ -382,7 +387,7 @@ public partial class SlideAssistant : AssistantBaseCore<SettingsDialogSlideBuild
continue; continue;
} }
var fileContent = await this.RustService.ReadArbitraryFileData(document.FilePath, int.MaxValue); var fileContent = await UserFile.LoadFileData(document.FilePath, this.RustService, this.DialogService);
sb.AppendLine($""" sb.AppendLine($"""
## DOCUMENT {numDocuments}: ## DOCUMENT {numDocuments}:

View File

@ -5,6 +5,7 @@ using AIStudio.Provider;
using AIStudio.Settings; using AIStudio.Settings;
using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG.RAGProcesses; using AIStudio.Tools.RAG.RAGProcesses;
using AIStudio.Tools.Security;
namespace AIStudio.Chat; namespace AIStudio.Chat;
@ -294,12 +295,17 @@ public sealed class ContentText : IContent
continue; continue;
} }
var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
var safeFileContent = await guardService.EnsureSafeForLlmAsync(
await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue),
PromptInjectionSource.ChatAttachment(document.FilePath));
sb.AppendLine(); sb.AppendLine();
sb.AppendLine("---------------------------------------"); sb.AppendLine("---------------------------------------");
sb.AppendLine($"File path: {document.FilePath}"); sb.AppendLine($"File path: {document.FilePath}");
sb.AppendLine("File content:"); sb.AppendLine("File content:");
sb.AppendLine("````"); sb.AppendLine("````");
sb.AppendLine(await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue)); sb.AppendLine(safeFileContent);
sb.AppendLine("````"); sb.AppendLine("````");
} }

View File

@ -1,4 +1,5 @@
using AIStudio.Tools.Rust; using AIStudio.Tools.Rust;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services; using AIStudio.Tools.Services;
using AIStudio.Tools.Validation; using AIStudio.Tools.Validation;
@ -192,6 +193,11 @@ public partial class ReadFileContent : MSGComponentBase
this.Logger.LogInformation("Successfully loaded file content: {FilePath}", filePath); this.Logger.LogInformation("Successfully loaded file content: {FilePath}", filePath);
return true; return true;
} }
catch (PromptInjectionBlockedException)
{
this.Logger.LogWarning("Blocked suspected prompt injection while loading file content: {FilePath}", filePath);
return false;
}
catch (Exception ex) catch (Exception ex)
{ {
this.Logger.LogError(ex, "Failed to load file content: {FilePath}", filePath); this.Logger.LogError(ex, "Failed to load file content: {FilePath}", filePath);

View File

@ -1,5 +1,6 @@
using AIStudio.Agents; using AIStudio.Agents;
using AIStudio.Chat; using AIStudio.Chat;
using AIStudio.Tools.Security;
using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components;
@ -13,6 +14,9 @@ public partial class ReadWebContent : MSGComponentBase
[Inject] [Inject]
private AgentTextContentCleaner AgentTextContentCleaner { get; init; } = null!; private AgentTextContentCleaner AgentTextContentCleaner { get; init; } = null!;
[Inject]
private PromptInjectionGuardService PromptInjectionGuardService { get; init; } = null!;
[Parameter] [Parameter]
public string Content { get; set; } = string.Empty; public string Content { get; set; } = string.Empty;
@ -87,6 +91,7 @@ public partial class ReadWebContent : MSGComponentBase
this.processStep = this.process[ReadWebContentSteps.PARSING]; this.processStep = this.process[ReadWebContentSteps.PARSING];
this.StateHasChanged(); this.StateHasChanged();
markdown = this.HTMLParser.ParseToMarkdown(html); markdown = this.HTMLParser.ParseToMarkdown(html);
markdown = await this.PromptInjectionGuardService.EnsureSafeForLlmAsync(markdown, PromptInjectionSource.WebContent(this.providedURL));
if (this.PreselectContentCleanerAgent && this.providerSettings != AIStudio.Settings.Provider.NONE) if (this.PreselectContentCleanerAgent && this.providerSettings != AIStudio.Settings.Provider.NONE)
{ {
@ -120,6 +125,10 @@ public partial class ReadWebContent : MSGComponentBase
this.StateHasChanged(); this.StateHasChanged();
} }
} }
catch (PromptInjectionBlockedException)
{
markdown = string.Empty;
}
catch catch
{ {
if (this.AgentIsRunning) if (this.AgentIsRunning)

View File

@ -1,5 +1,6 @@
using AIStudio.Chat; using AIStudio.Chat;
using AIStudio.Components; using AIStudio.Components;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services; using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components;
@ -42,6 +43,11 @@ public partial class DocumentCheckDialog : MSGComponentBase
this.FileContent = fileContent; this.FileContent = fileContent;
} }
} }
catch (PromptInjectionBlockedException exception)
{
this.Logger.LogWarning(exception, "Blocked suspected prompt injection while previewing '{FilePath}'", this.Document?.FilePath);
this.FileContent = string.Empty;
}
catch (Exception ex) catch (Exception ex)
{ {
this.Logger.LogError(ex, "Failed to load file content from '{FilePath}'", this.Document); this.Logger.LogError(ex, "Failed to load file content from '{FilePath}'", this.Document);

View File

@ -0,0 +1,50 @@
@using AIStudio.Tools.Security
@inherits MSGComponentBase
<MudDialog>
<TitleContent>
<MudText Typo="Typo.h6" Class="d-flex align-center">
<MudIcon Icon="@Icons.Material.Filled.GppMaybe" Class="mr-2" Color="Color.Warning" />
@T("Prompt Injection Detected")
</MudText>
</TitleContent>
<DialogContent>
@if (this.Result is not null)
{
<MudAlert Severity="Severity.Warning" Variant="Variant.Filled" Dense="true" Class="mb-4">
@T("AI Studio blocked this content before it reached a model or agent.")
</MudAlert>
<MudText Typo="Typo.body1" Class="mb-2">
@T("Source kind"): @this.Result.Source.Kind
</MudText>
<MudText Typo="Typo.body2" Class="mb-4">
@T("Source"): @this.Result.Source.Label
</MudText>
<MudText Typo="Typo.subtitle1" Class="mb-2">
@T("Detected signals")
</MudText>
<MudList T="string" Dense="true" Class="mb-4">
@foreach (var finding in this.Result.Findings)
{
<MudListItem T="string" Icon="@Icons.Material.Filled.ReportProblem">
@($"{finding.Category} / {finding.DetectionStage}: {finding.Snippet}")
</MudListItem>
}
</MudList>
<MudText Typo="Typo.body2">
@T("More information"):
<MudLink Href="@PromptInjectionGuardService.WIKI_URL" Target="_blank">
@PromptInjectionGuardService.WIKI_URL
</MudLink>
</MudText>
}
</DialogContent>
<DialogActions>
<MudButton OnClick="@this.Close" Variant="Variant.Filled">
@T("Close")
</MudButton>
</DialogActions>
</MudDialog>

View File

@ -0,0 +1,17 @@
using AIStudio.Components;
using AIStudio.Tools.Security;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Dialogs;
public partial class PromptInjectionAlertDialog : MSGComponentBase
{
[CascadingParameter]
private IMudDialogInstance MudDialog { get; set; } = null!;
[Parameter]
public PromptInjectionScanResult Result { get; set; } = null!;
private void Close() => this.MudDialog.Close();
}

View File

@ -14,6 +14,10 @@
<ConfigurationOption OptionDescription="@T("Show the latest message after loading?")" LabelOn="@T("Latest message is shown, after loading a chat")" LabelOff="@T("First (oldest) message is shown, after loading a chat")" State="@(() => this.SettingsManager.ConfigurationData.Chat.ShowLatestMessageAfterLoading)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.Chat.ShowLatestMessageAfterLoading = updatedState)" OptionHelp="@T("When enabled, the latest message is shown after loading a chat. When disabled, the first (oldest) message is shown.")"/> <ConfigurationOption OptionDescription="@T("Show the latest message after loading?")" LabelOn="@T("Latest message is shown, after loading a chat")" LabelOff="@T("First (oldest) message is shown, after loading a chat")" State="@(() => this.SettingsManager.ConfigurationData.Chat.ShowLatestMessageAfterLoading)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.Chat.ShowLatestMessageAfterLoading = updatedState)" OptionHelp="@T("When enabled, the latest message is shown after loading a chat. When disabled, the first (oldest) message is shown.")"/>
<ConfigurationSelect OptionDescription="@T("Provider selection when creating new chats")" SelectedValue="@(() => this.SettingsManager.ConfigurationData.Chat.AddChatProviderBehavior)" Data="@ConfigurationSelectDataFactory.GetAddChatProviderBehavior()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.Chat.AddChatProviderBehavior = selectedValue)" OptionHelp="@T("Control how the LLM provider for added chats is selected.")"/> <ConfigurationSelect OptionDescription="@T("Provider selection when creating new chats")" SelectedValue="@(() => this.SettingsManager.ConfigurationData.Chat.AddChatProviderBehavior)" Data="@ConfigurationSelectDataFactory.GetAddChatProviderBehavior()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.Chat.AddChatProviderBehavior = selectedValue)" OptionHelp="@T("Control how the LLM provider for added chats is selected.")"/>
<ConfigurationSelect OptionDescription="@T("Provider selection when loading a chat and sending assistant results to chat")" SelectedValue="@(() => this.SettingsManager.ConfigurationData.Chat.LoadingProviderBehavior)" Data="@ConfigurationSelectDataFactory.GetLoadingChatProviderBehavior()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.Chat.LoadingProviderBehavior = selectedValue)" OptionHelp="@T("Control how the LLM provider for loaded chats is selected and when assistant results are sent to chat.")"/> <ConfigurationSelect OptionDescription="@T("Provider selection when loading a chat and sending assistant results to chat")" SelectedValue="@(() => this.SettingsManager.ConfigurationData.Chat.LoadingProviderBehavior)" Data="@ConfigurationSelectDataFactory.GetLoadingChatProviderBehavior()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.Chat.LoadingProviderBehavior = selectedValue)" OptionHelp="@T("Control how the LLM provider for loaded chats is selected and when assistant results are sent to chat.")"/>
<MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
<ConfigurationOption OptionDescription="@T("Protect against prompt injection in external content?")" LabelOn="@T("Potential prompt injections are blocked before they reach an LLM")" LabelOff="@T("External content is passed through without prompt-injection checks")" State="@(() => this.SettingsManager.ConfigurationData.Chat.EnablePromptInjectionProtection)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.Chat.EnablePromptInjectionProtection = updatedState)" OptionHelp="@T("Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.Chat, x => x.EnablePromptInjectionProtection, out var meta) && meta.IsLocked"/>
<ConfigurationOption OptionDescription="@T("Show a learning alert when prompt injection is detected?")" LabelOn="@T("A blocking alert explains the detected attack pattern")" LabelOff="@T("Only the block notification is shown")" Disabled="@(() => !this.SettingsManager.ConfigurationData.Chat.EnablePromptInjectionProtection)" State="@(() => this.SettingsManager.ConfigurationData.Chat.ShowPromptInjectionAlert)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.Chat.ShowPromptInjectionAlert = updatedState)" OptionHelp="@T("Shows an explanation dialog with an external reference when AI Studio blocks suspicious content.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.Chat, x => x.ShowPromptInjectionAlert, out var meta) && meta.IsLocked"/>
</MudPaper>
<MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg"> <MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
<ConfigurationOption OptionDescription="@T("Preselect chat options?")" LabelOn="@T("Chat options are preselected")" LabelOff="@T("No chat options are preselected")" State="@(() => this.SettingsManager.ConfigurationData.Chat.PreselectOptions)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.Chat.PreselectOptions = updatedState)" OptionHelp="@T("When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.Chat, x => x.PreselectOptions, out var meta) && meta.IsLocked"/> <ConfigurationOption OptionDescription="@T("Preselect chat options?")" LabelOn="@T("Chat options are preselected")" LabelOff="@T("No chat options are preselected")" State="@(() => this.SettingsManager.ConfigurationData.Chat.PreselectOptions)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.Chat.PreselectOptions = updatedState)" OptionHelp="@T("When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.Chat, x => x.PreselectOptions, out var meta) && meta.IsLocked"/>

View File

@ -4,6 +4,7 @@ using AIStudio.Settings.DataModel;
using AIStudio.Tools.AIJobs; using AIStudio.Tools.AIJobs;
using AIStudio.Tools.AssistantSessions; using AIStudio.Tools.AssistantSessions;
using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Security;
using AIStudio.Tools.Rust; using AIStudio.Tools.Rust;
using AIStudio.Tools.Services; using AIStudio.Tools.Services;
@ -64,6 +65,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
private bool startupCompleted; private bool startupCompleted;
private bool settingsWriteProtectionWarningShown; private bool settingsWriteProtectionWarningShown;
private readonly SemaphoreSlim mandatoryInfoDialogSemaphore = new(1, 1); private readonly SemaphoreSlim mandatoryInfoDialogSemaphore = new(1, 1);
private readonly SemaphoreSlim promptInjectionDialogSemaphore = new(1, 1);
private IReadOnlyCollection<NavBarItem> navItems = []; private IReadOnlyCollection<NavBarItem> navItems = [];
@ -104,7 +106,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
this.MessageBus.ApplyFilters(this, [], this.MessageBus.ApplyFilters(this, [],
[ [
Event.UPDATE_AVAILABLE, Event.CONFIGURATION_CHANGED, Event.COLOR_THEME_CHANGED, Event.SHOW_ERROR, Event.UPDATE_AVAILABLE, Event.CONFIGURATION_CHANGED, Event.COLOR_THEME_CHANGED, Event.SHOW_ERROR,
Event.SHOW_WARNING, Event.SHOW_SUCCESS, Event.STARTUP_PLUGIN_SYSTEM, Event.PLUGINS_RELOADED, Event.SHOW_WARNING, Event.SHOW_SUCCESS, Event.SHOW_PROMPT_INJECTION_ALERT, Event.STARTUP_PLUGIN_SYSTEM, Event.PLUGINS_RELOADED,
Event.INSTALL_UPDATE, Event.STARTUP_COMPLETED, Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED, Event.INSTALL_UPDATE, Event.STARTUP_COMPLETED, Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED,
Event.CHAT_GENERATION_CHANGED, Event.ASSISTANT_SESSION_CHANGED, Event.ASSISTANT_SESSION_FINISHED, Event.CHAT_GENERATION_CHANGED, Event.ASSISTANT_SESSION_CHANGED, Event.ASSISTANT_SESSION_FINISHED,
]); ]);
@ -244,6 +246,12 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
break; break;
case Event.SHOW_PROMPT_INJECTION_ALERT:
if (data is PromptInjectionAlertMessage promptInjectionAlert)
await this.ShowPromptInjectionAlertAsync(promptInjectionAlert);
break;
case Event.SHOW_ERROR: case Event.SHOW_ERROR:
if (data is DataErrorMessage error) if (data is DataErrorMessage error)
error.Show(this.Snackbar); error.Show(this.Snackbar);
@ -332,6 +340,29 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan
}); });
} }
private async Task ShowPromptInjectionAlertAsync(PromptInjectionAlertMessage alert)
{
await this.promptInjectionDialogSemaphore.WaitAsync();
try
{
var dialogParameters = new DialogParameters<PromptInjectionAlertDialog>
{
{ x => x.Result, alert.Result },
};
var dialogReference = await this.DialogService.ShowAsync<PromptInjectionAlertDialog>(
T("Prompt Injection Detected"),
dialogParameters,
DialogOptions.BLOCKING_FULLSCREEN);
await dialogReference.Result;
}
finally
{
this.promptInjectionDialogSemaphore.Release();
}
}
public Task<TResult?> ProcessMessageWithResult<TPayload, TResult>(ComponentBase? sendingComponent, Event triggeredEvent, TPayload? data) public Task<TResult?> ProcessMessageWithResult<TPayload, TResult>(ComponentBase? sendingComponent, Event triggeredEvent, TPayload? data)
{ {
return Task.FromResult<TResult?>(default); return Task.FromResult<TResult?>(default);

View File

@ -255,6 +255,14 @@ CONFIG["SETTINGS"] = {}
-- This must be enabled for the chat-specific provider, profile, and chat template to take effect. -- This must be enabled for the chat-specific provider, profile, and chat template to take effect.
-- CONFIG["SETTINGS"]["DataChat.PreselectOptions"] = true -- CONFIG["SETTINGS"]["DataChat.PreselectOptions"] = true
-- --
-- Configure prompt-injection protection for external content such as webpages,
-- retrieved context, and file attachments before it is sent to an LLM.
-- CONFIG["SETTINGS"]["DataChat.EnablePromptInjectionProtection"] = true
--
-- Configure whether AI Studio shows a blocking explanation dialog when
-- suspicious content is detected and blocked.
-- CONFIG["SETTINGS"]["DataChat.ShowPromptInjectionAlert"] = true
--
-- Configure the preselected provider for chats. -- Configure the preselected provider for chats.
-- It must be one of the provider IDs defined in CONFIG["LLM_PROVIDERS"]. -- It must be one of the provider IDs defined in CONFIG["LLM_PROVIDERS"].
-- CONFIG["SETTINGS"]["DataChat.PreselectedProvider"] = "00000000-0000-0000-0000-000000000000" -- CONFIG["SETTINGS"]["DataChat.PreselectedProvider"] = "00000000-0000-0000-0000-000000000000"

View File

@ -6,6 +6,7 @@ using AIStudio.Tools.AIJobs;
using AIStudio.Tools.AssistantSessions; using AIStudio.Tools.AssistantSessions;
using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.PluginSystem.Assistants; using AIStudio.Tools.PluginSystem.Assistants;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services; using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Server.Kestrel.Core; using Microsoft.AspNetCore.Server.Kestrel.Core;
@ -129,6 +130,8 @@ internal sealed class Program
builder.Services.AddSingleton(rust); builder.Services.AddSingleton(rust);
builder.Services.AddMudMarkdownClipboardService<MarkdownClipboardService>(); builder.Services.AddMudMarkdownClipboardService<MarkdownClipboardService>();
builder.Services.AddSingleton<SettingsManager>(); builder.Services.AddSingleton<SettingsManager>();
builder.Services.AddSingleton<PromptInjectionScanner>();
builder.Services.AddSingleton<PromptInjectionGuardService>();
builder.Services.AddSingleton<ThreadSafeRandom>(); builder.Services.AddSingleton<ThreadSafeRandom>();
builder.Services.AddSingleton<AIJobService>(); builder.Services.AddSingleton<AIJobService>();
builder.Services.AddSingleton<AssistantSessionService>(); builder.Services.AddSingleton<AssistantSessionService>();

View File

@ -93,6 +93,16 @@ public sealed class DataChat(Expression<Func<Data, DataChat>>? configSelection =
} }
} }
/// <summary>
/// Whether prompt-injection protection is enabled for external and attached content.
/// </summary>
public bool EnablePromptInjectionProtection { get; set; } = ManagedConfiguration.Register(configSelection, n => n.EnablePromptInjectionProtection, true);
/// <summary>
/// Whether an alert dialog should be shown when prompt-injection content is blocked.
/// </summary>
public bool ShowPromptInjectionAlert { get; set; } = ManagedConfiguration.Register(configSelection, n => n.ShowPromptInjectionAlert, true);
/// <summary> /// <summary>
/// Should we show the latest message after loading? When false, we show the first (aka oldest) message. /// Should we show the latest message after loading? When false, we show the first (aka oldest) message.
/// </summary> /// </summary>

View File

@ -5,6 +5,7 @@ using AIStudio.Provider;
using AIStudio.Settings; using AIStudio.Settings;
using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG.RAGProcesses; using AIStudio.Tools.RAG.RAGProcesses;
using AIStudio.Tools.Security;
namespace AIStudio.Tools.AIJobs; namespace AIStudio.Tools.AIJobs;
@ -266,6 +267,12 @@ public sealed class AIJobService(
await this.CompleteChatGenerationAsync(state, AIJobStatus.FAILED, e.UserMessage); await this.CompleteChatGenerationAsync(state, AIJobStatus.FAILED, e.UserMessage);
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, e.UserMessage)); await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.CloudOff, e.UserMessage));
} }
catch (PromptInjectionBlockedException e)
{
logger.LogWarning(e, "Blocked prompt-injection content during chat generation job '{JobId}'.", state.Snapshot.JobId);
RemoveEmptyAIResponse(state);
await this.CompleteChatGenerationAsync(state, AIJobStatus.FAILED, e.Message);
}
catch (Exception e) catch (Exception e)
{ {
logger.LogError(e, "The chat generation job '{JobId}' failed.", state.Snapshot.JobId); logger.LogError(e, "The chat generation job '{JobId}' failed.", state.Snapshot.JobId);

View File

@ -73,6 +73,11 @@ public enum Event
/// </summary> /// </summary>
SHOW_SUCCESS, SHOW_SUCCESS,
/// <summary>
/// Requests display of a prompt-injection alert dialog.
/// </summary>
SHOW_PROMPT_INJECTION_ALERT,
/// <summary> /// <summary>
/// Carries an event received from the Tauri runtime. /// Carries an event received from the Tauri runtime.
/// </summary> /// </summary>

View File

@ -269,6 +269,8 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectedDataSourcesAutomaticValidation, this.Id, settingsTable, dryRun); ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectedDataSourcesAutomaticValidation, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectedDataSourceIds, this.Id, settingsTable, dryRun); ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.PreselectedDataSourceIds, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.SendToChatDataSourceBehavior, this.Id, settingsTable, dryRun); ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.SendToChatDataSourceBehavior, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.EnablePromptInjectionProtection, this.Id, settingsTable, dryRun);
ManagedConfiguration.TryProcessConfiguration(x => x.Chat, x => x.ShowPromptInjectionAlert, this.Id, settingsTable, dryRun);
// Config: transcription provider? // Config: transcription provider?
ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.UseTranscriptionProvider, Guid.Empty, this.Id, settingsTable, dryRun); ManagedConfiguration.TryProcessConfiguration(x => x.App, x => x.UseTranscriptionProvider, Guid.Empty, this.Id, settingsTable, dryRun);

View File

@ -230,6 +230,12 @@ public static partial class PluginFactory
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.SendToChatDataSourceBehavior, AVAILABLE_PLUGINS)) if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.SendToChatDataSourceBehavior, AVAILABLE_PLUGINS))
wasConfigurationChanged = true; wasConfigurationChanged = true;
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.EnablePromptInjectionProtection, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.Chat, x => x.ShowPromptInjectionAlert, AVAILABLE_PLUGINS))
wasConfigurationChanged = true;
// Check for the update interval: // Check for the update interval:
if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.UpdateInterval, AVAILABLE_PLUGINS)) if(ManagedConfiguration.IsConfigurationLeftOver(x => x.App, x => x.UpdateInterval, AVAILABLE_PLUGINS))
wasConfigurationChanged = true; wasConfigurationChanged = true;

View File

@ -1,6 +1,7 @@
using System.Text; using System.Text;
using AIStudio.Chat; using AIStudio.Chat;
using AIStudio.Tools.Security;
namespace AIStudio.Tools.RAG; namespace AIStudio.Tools.RAG;
@ -16,7 +17,18 @@ public static class IRetrievalContextExtensions
foreach(var retrievalContext in retrievalContexts) foreach(var retrievalContext in retrievalContexts)
{ {
index++; index++;
await retrievalContext.AsMarkdown(sb, index, retrievalContexts.Count, token); try
{
await retrievalContext.AsMarkdown(sb, index, retrievalContexts.Count, token);
}
catch (PromptInjectionBlockedException exception)
{
LOGGER.LogWarning(
exception,
"Skipping retrieval context '{DataSourceName}' at '{Path}' because it was blocked by prompt-injection protection.",
retrievalContext.DataSourceName,
retrievalContext.Path);
}
} }
return sb.ToString(); return sb.ToString();
@ -25,74 +37,80 @@ public static class IRetrievalContextExtensions
public static async Task<string> AsMarkdown(this IRetrievalContext retrievalContext, StringBuilder? sb = null, int index = -1, int numTotalRetrievalContexts = -1, CancellationToken token = default) public static async Task<string> AsMarkdown(this IRetrievalContext retrievalContext, StringBuilder? sb = null, int index = -1, int numTotalRetrievalContexts = -1, CancellationToken token = default)
{ {
sb ??= new StringBuilder(); sb ??= new StringBuilder();
var contextBuilder = new StringBuilder();
switch (index) switch (index)
{ {
case > 0 when numTotalRetrievalContexts is -1: case > 0 when numTotalRetrievalContexts is -1:
sb.AppendLine($"# Retrieval context {index}"); contextBuilder.AppendLine($"# Retrieval context {index}");
break; break;
case > 0 when numTotalRetrievalContexts > 0: case > 0 when numTotalRetrievalContexts > 0:
sb.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}"); contextBuilder.AppendLine($"# Retrieval context {index} of {numTotalRetrievalContexts}");
break; break;
default: default:
sb.AppendLine("# Retrieval context"); contextBuilder.AppendLine("# Retrieval context");
break; break;
} }
sb.AppendLine($"Data source name: {retrievalContext.DataSourceName}"); contextBuilder.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
sb.AppendLine($"Content category: {retrievalContext.Category}"); contextBuilder.AppendLine($"Content category: {retrievalContext.Category}");
sb.AppendLine($"Content type: {retrievalContext.Type}"); contextBuilder.AppendLine($"Content type: {retrievalContext.Type}");
sb.AppendLine($"Content path: {retrievalContext.Path}"); contextBuilder.AppendLine($"Content path: {retrievalContext.Path}");
if(retrievalContext.Links.Count > 0) if(retrievalContext.Links.Count > 0)
{ {
sb.AppendLine("Additional links:"); contextBuilder.AppendLine("Additional links:");
foreach(var link in retrievalContext.Links) foreach(var link in retrievalContext.Links)
sb.AppendLine($"- {link}"); contextBuilder.AppendLine($"- {link}");
} }
var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
var source = PromptInjectionSource.RetrievalContext(retrievalContext.DataSourceName, retrievalContext.Path);
switch(retrievalContext) switch(retrievalContext)
{ {
case RetrievalTextContext textContext: case RetrievalTextContext textContext:
sb.AppendLine(); contextBuilder.AppendLine();
sb.AppendLine("Matched text content:"); contextBuilder.AppendLine("Matched text content:");
sb.AppendLine("````"); contextBuilder.AppendLine("````");
sb.AppendLine(textContext.MatchedText); contextBuilder.AppendLine(textContext.MatchedText);
sb.AppendLine("````"); contextBuilder.AppendLine("````");
if(textContext.SurroundingContent.Count > 0) if(textContext.SurroundingContent.Count > 0)
{ {
sb.AppendLine(); contextBuilder.AppendLine();
sb.AppendLine("Surrounding text content:"); contextBuilder.AppendLine("Surrounding text content:");
foreach(var surrounding in textContext.SurroundingContent) foreach(var surrounding in textContext.SurroundingContent)
{ {
sb.AppendLine(); contextBuilder.AppendLine();
sb.AppendLine("````"); contextBuilder.AppendLine("````");
sb.AppendLine(surrounding); contextBuilder.AppendLine(surrounding);
sb.AppendLine("````"); contextBuilder.AppendLine("````");
} }
} }
await guardService.EnsureSafeForLlmAsync(contextBuilder.ToString(), source);
break; break;
case RetrievalImageContext imageContext: case RetrievalImageContext imageContext:
sb.AppendLine(); await guardService.EnsureSafeForLlmAsync(contextBuilder.ToString(), source);
sb.AppendLine("Matched image content as base64-encoded data:"); contextBuilder.AppendLine();
sb.AppendLine("````"); contextBuilder.AppendLine("Matched image content as base64-encoded data:");
sb.AppendLine(await imageContext.TryAsBase64(token) is (success: true, { } base64Image) contextBuilder.AppendLine("````");
contextBuilder.AppendLine(await imageContext.TryAsBase64(token) is (success: true, { } base64Image)
? base64Image ? base64Image
: string.Empty); : string.Empty);
sb.AppendLine("````"); contextBuilder.AppendLine("````");
break; break;
default: default:
await guardService.EnsureSafeForLlmAsync(contextBuilder.ToString(), source);
LOGGER.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet."); LOGGER.LogWarning($"The retrieval content type '{retrievalContext.Type}' of data source '{retrievalContext.DataSourceName}' at location '{retrievalContext.Path}' is not supported yet.");
break; break;
} }
sb.AppendLine(); contextBuilder.AppendLine();
sb.Append(contextBuilder);
return sb.ToString(); return sb.ToString();
} }
} }

View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Security;
public sealed record PromptInjectionAlertMessage(PromptInjectionScanResult Result);

View File

@ -0,0 +1,6 @@
namespace AIStudio.Tools.Security;
public sealed class PromptInjectionBlockedException(PromptInjectionScanResult result, string message) : Exception(message)
{
public PromptInjectionScanResult Result { get; } = result;
}

View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Security;
public sealed record PromptInjectionFinding(string RuleId, string Category, string DetectionStage, string Snippet);

View File

@ -0,0 +1,44 @@
using AIStudio.Settings;
using AIStudio.Tools.PluginSystem;
namespace AIStudio.Tools.Security;
public sealed class PromptInjectionGuardService(
PromptInjectionScanner scanner,
SettingsManager settingsManager,
ILogger<PromptInjectionGuardService> logger)
{
public const string WIKI_URL = "https://de.wikipedia.org/wiki/Prompt-Engineering#Prompt_Injection";
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PromptInjectionGuardService).Namespace, nameof(PromptInjectionGuardService));
public async Task<string> EnsureSafeForLlmAsync(string text, PromptInjectionSource source)
{
if (!settingsManager.ConfigurationData.Chat.EnablePromptInjectionProtection || string.IsNullOrWhiteSpace(text))
return text;
var result = scanner.Scan(text, source);
if (!result.IsBlocked)
return text;
var message = string.Format(TB("AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt."), source.Label);
await this.HandleDetectionAsync(result, message);
throw new PromptInjectionBlockedException(result, message);
}
private async Task HandleDetectionAsync(PromptInjectionScanResult result, string message)
{
logger.LogWarning(
"Blocked suspected prompt injection in {SourceKind} '{SourceLabel}'. RuleIds={RuleIds}",
result.Source.Kind,
result.Source.Label,
string.Join(", ", result.RuleIds));
await MessageBus.INSTANCE.SendWarning(new(
Icons.Material.Filled.GppMaybe,
message));
if (settingsManager.ConfigurationData.Chat.ShowPromptInjectionAlert)
await MessageBus.INSTANCE.SendMessage<PromptInjectionAlertMessage>(null, Event.SHOW_PROMPT_INJECTION_ALERT, new(result));
}
}

View File

@ -0,0 +1,141 @@
using System.Text.RegularExpressions;
namespace AIStudio.Tools.Security;
internal readonly record struct PromptInjectionRegexRule(string Id, string Category, Regex Regex);
internal static partial class PromptInjectionPatterns
{
internal static readonly IReadOnlyList<PromptInjectionRegexRule> RULES =
[
new("instruction_override", "override", InstructionOverrideRegex()),
new("instruction_priority_override", "override", InstructionPriorityOverrideRegex()),
new("system_prompt_spoofing", "role_override", SystemPromptSpoofingRegex()),
new("system_prompt_exfiltration", "exfiltration", SystemPromptExfiltrationRegex()),
new("prompt_echo_exfiltration", "exfiltration", PromptEchoExfiltrationRegex()),
new("policy_bypass", "override", PolicyBypassRegex()),
new("role_reassignment", "role_override", RoleReassignmentRegex()),
new("privileged_persona_activation", "jailbreak", PrivilegedPersonaActivationRegex()),
new("tool_or_secret_exfiltration", "exfiltration", ToolOrSecretExfiltrationRegex()),
new("conversation_memory_exfiltration", "exfiltration", ConversationMemoryExfiltrationRegex()),
new("tool_call_manipulation", "agent_manipulation", ToolCallManipulationRegex()),
new("agent_thought_injection", "agent_manipulation", AgentThoughtInjectionRegex()),
new("delimiter_wrapped_attack", "delimiter_evasion", DelimiterWrappedAttackRegex()),
new("hidden_markup_injection", "markup_evasion", HiddenMarkupInjectionRegex()),
new("latex_invisible_text", "markup_evasion", LatexInvisibleTextRegex()),
new("unicode_smuggling", "encoding_evasion", UnicodeSmugglingRegex()),
new("ignore_safety_after_data", "override", IgnoreSafetyAfterDataRegex()),
new("persistent_or_delayed_trigger", "persistence", PersistentOrDelayedTriggerRegex()),
new("jailbreak_marker", "jailbreak", JailbreakMarkerRegex()),
];
private const RegexOptions RULE_OPTIONS = RegexOptions.IgnoreCase | RegexOptions.CultureInvariant;
private const int MATCH_TIMEOUT_MILLISECONDS = 100;
private const string INSTRUCTION_OVERRIDE_PATTERN = """(?:ignore|disregard|forget|bypass|override|replace|drop)\s+(?:all\s+)?(?:previous|prior|above|earlier)\s+(?:instructions?|prompts?|messages?|rules?)""";
private const string INSTRUCTION_PRIORITY_OVERRIDE_PATTERN = """(?:(?:new|following|these)\s+(?:instructions?|rules?|prompts?)\s+(?:are|is)\s+(?:now\s+)?(?:the\s+)?(?:highest|top|only)\s+priority|(?:take|takes|treat)\s+(?:the\s+)?(?:following|these|this)\s+as\s+(?:the\s+)?(?:new\s+)?(?:system|developer)\s+(?:prompt|message|instructions?)|(?:supersede|replace|override)\s+(?:the\s+)?(?:system|developer|previous|prior|earlier)\s+(?:prompt|message|instructions?|rules?))""";
private const string SYSTEM_PROMPT_SPOOFING_PATTERN = """(?:(?:this|the\s+following)\s+is\s+(?:a\s+)?(?:system|developer)\s+(?:prompt|message|instruction)|(?:prepend|insert|write)\s+(?:a\s+)?(?:system|developer)\s+(?:prompt|message|instruction)|(?:system|developer|assistant)\s*[:>#-]\s*(?:ignore|bypass|override|reveal|you\s+are\s+now))""";
private const string SYSTEM_PROMPT_EXFILTRATION_PATTERN = """(?:reveal|show|print|display|dump|expose|leak|tell\s+me|return|quote|repeat\s+back)\s+(?:the\s+)?(?:hidden\s+|full\s+|exact\s+|verbatim\s+|initial\s+|original\s+)?(?:system|developer|assistant)\s+(?:prompt|message|instructions?)""";
private const string PROMPT_ECHO_EXFILTRATION_PATTERN = """(?:(?:what\s+(?:were|are))\s+your\s+(?:exact|full|hidden|original)\s+(?:instructions?|prompt)|(?:repeat|quote|print|output|display)\s+(?:the\s+)?text\s+(?:above|before|from\s+the\s+top)\s+(?:verbatim|exactly)?|starting\s+with\s+["']?you\s+are)""";
private const string POLICY_BYPASS_PATTERN = """(?:do\s+not|don't|stop\s+to|never)\s+(?:follow|obey|respect|apply|enforce)\s+(?:the\s+)?(?:system|developer|safety|security|content|usage)\s+(?:prompt|message|instructions?|policy|policies|guardrails?|restrictions?)""";
private const string ROLE_REASSIGNMENT_PATTERN = """(?:you\s+are\s+now|you\s+are\s+no\s+longer|act\s+as|pretend\s+to\s+be|simulate\s+being|assume\s+the\s+role\s+of|from\s+now\s+on\s+you\s+are)\s+(?:an\s+)?(?:unfiltered|unrestricted|developer|system|root|admin|jailbroken|evil|dan|do\s+anything\s+now)""";
private const string PRIVILEGED_PERSONA_ACTIVATION_PATTERN = """\b(?:developer\s+mode|debug\s+mode|admin\s+mode|root\s+mode|god\s+mode|maintenance\s+mode|dan\s*(?:mode)?|do\s+anything\s+now|grandmother\s+trick)\b""";
private const string TOOL_OR_SECRET_EXFILTRATION_PATTERN = """(?:export|send|return|reveal|show|print|list|dump|exfiltrate)\s+(?:all\s+)?(?:tools?|functions?|plugins?|api\s*keys?|keys?|tokens?|credentials?|secrets?|passwords?|hidden\s+instructions?|environment\s+variables?|system\s+information)""";
private const string CONVERSATION_MEMORY_EXFILTRATION_PATTERN = """(?:(?:show|print|reveal|return|dump|list)\s+(?:the\s+)?(?:conversation\s+history|chat\s+history|memory|scratchpad|chain\s+of\s+thought|reasoning|previous\s+user\s+messages?|prior\s+messages?)|(?:what\s+did\s+(?:the\s+)?previous\s+user\s+say))""";
private const string TOOL_CALL_MANIPULATION_PATTERN = """(?:(?:call|invoke|execute|run|use|trigger)\s+(?:the\s+)?(?:tool|function|plugin|api|browser|web|shell|terminal|command)[^\n]{0,120}(?:with|using|to)\s+(?:these\s+)?(?:arguments|params?|parameters)|(?:do\s+not|don't)\s+ask\s+for\s+(?:confirmation|approval|permission)|(?:silently|secretly|without\s+asking)\s+(?:call|invoke|execute|run|use))""";
private const string AGENT_THOUGHT_INJECTION_PATTERN = """(?:(?:thought|observation|reasoning|scratchpad|tool\s+output|assistant|system|developer)\s*[:=]\s*(?:ignore|bypass|override|reveal|call|execute)|forge\s+(?:an\s+)?(?:observation|tool\s+output|assistant\s+message)|pretend\s+(?:the\s+)?tool\s+(?:returned|said))""";
private const string DELIMITER_WRAPPED_ATTACK_PATTERN = """(?:^|\n)\s*(?:<{2,}|>{2,}|`{3,}|#{1,6}\s*)(?:\s*(?:system|developer|assistant|instructions?|prompt)\b)""";
private const string HIDDEN_MARKUP_INJECTION_PATTERN = """(?:<!--[^>\r\n]{0,300}(?:ignore|bypass|override|reveal|system\s+prompt)[^>\r\n]{0,300}-->|<(?:span|div|p|font|section)[^>]{0,200}(?:display\s*:\s*none|visibility\s*:\s*hidden|opacity\s*:\s*0|font-size\s*:\s*0|color\s*:\s*(?:white|#fff(?:fff)?|rgb\(\s*255\s*,\s*255\s*,\s*255\s*\)))[^>]{0,200}>)""";
private const string LATEX_INVISIBLE_TEXT_PATTERN = """(?:\\(?:color|textcolor)\s*\{\s*white\s*\}\s*\{[^}]{0,300}\}|\\(?:fontsize|tiny|scriptsize)\b[^\r\n]{0,120}(?:ignore|bypass|override|reveal))""";
private const string UNICODE_SMUGGLING_PATTERN = """[\u200B-\u200F\u2060-\u2064\u2066-\u2069\uFEFF]""";
private const string IGNORE_SAFETY_AFTER_DATA_PATTERN = """(?:after\s+reading|once\s+you\s+read|when\s+you\s+see)\s+.*?(?:ignore|bypass|override)\s+.*?(?:instructions?|safety|rules?)""";
private const string PERSISTENT_OR_DELAYED_TRIGGER_PATTERN = """(?:(?:remember|store|save|persist|memorize)\s+(?:this|these|the\s+following)\s+(?:instructions?|rules?|message)|(?:later|in\s+the\s+next\s+message|when\s+you\s+see|whenever\s+you\s+read|if\s+you\s+encounter)\s+.{0,120}(?:ignore|bypass|override|reveal|exfiltrate))""";
private const string JAILBREAK_MARKER_PATTERN = """\b(?:jailbreak|prompt\s+injection|ignore\s+your\s+guardrails?|bypass\s+(?:your\s+)?(?:guardrails?|safety)|unfiltered\s+mode|do\s+anything\s+now|developer\s+mode|admin\s+mode|root\s+mode)\b""";
private const string ANY_RULE_PATTERN =
"(?:" + INSTRUCTION_OVERRIDE_PATTERN + ")|(?:" +
INSTRUCTION_PRIORITY_OVERRIDE_PATTERN + ")|(?:" + SYSTEM_PROMPT_SPOOFING_PATTERN + ")|(?:" +
SYSTEM_PROMPT_EXFILTRATION_PATTERN + ")|(?:" + PROMPT_ECHO_EXFILTRATION_PATTERN + ")|(?:" +
POLICY_BYPASS_PATTERN + ")|(?:" + ROLE_REASSIGNMENT_PATTERN + ")|(?:" +
PRIVILEGED_PERSONA_ACTIVATION_PATTERN + ")|(?:" + TOOL_OR_SECRET_EXFILTRATION_PATTERN + ")|(?:" +
CONVERSATION_MEMORY_EXFILTRATION_PATTERN + ")|(?:" + TOOL_CALL_MANIPULATION_PATTERN + ")|(?:" +
AGENT_THOUGHT_INJECTION_PATTERN + ")|(?:" + DELIMITER_WRAPPED_ATTACK_PATTERN + ")|(?:" +
HIDDEN_MARKUP_INJECTION_PATTERN + ")|(?:" + LATEX_INVISIBLE_TEXT_PATTERN + ")|(?:" +
UNICODE_SMUGGLING_PATTERN + ")|(?:" + IGNORE_SAFETY_AFTER_DATA_PATTERN + ")|(?:" +
PERSISTENT_OR_DELAYED_TRIGGER_PATTERN + ")|(?:" + JAILBREAK_MARKER_PATTERN + ")";
[GeneratedRegex(ANY_RULE_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
internal static partial Regex AnyRuleRegex();
[GeneratedRegex(@"\b[a-zA-Z](?:[\s._:/\\|-]+[a-zA-Z]){2,}\b", RegexOptions.CultureInvariant)]
internal static partial Regex SpacedLetterSequenceRegex();
[GeneratedRegex(@"\b[a-zA-Z]{5,12}\b", RegexOptions.CultureInvariant)]
internal static partial Regex WordRegex();
[GeneratedRegex(@"(?<![A-Za-z0-9+/=])[A-Za-z0-9+/]{16,}={0,2}(?![A-Za-z0-9+/=])", RegexOptions.CultureInvariant)]
internal static partial Regex Base64Regex();
[GeneratedRegex(@"(?<![0-9A-Fa-f])(?:[0-9A-Fa-f]{2}(?:[\s:-]+|$)){8,}", RegexOptions.CultureInvariant)]
internal static partial Regex HexPairRegex();
[GeneratedRegex(@"(?<![0-9A-Fa-f])(?:[0-9A-Fa-f]{2}){8,}(?![0-9A-Fa-f])", RegexOptions.CultureInvariant)]
internal static partial Regex HexCompactRegex();
[GeneratedRegex(INSTRUCTION_OVERRIDE_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex InstructionOverrideRegex();
[GeneratedRegex(INSTRUCTION_PRIORITY_OVERRIDE_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex InstructionPriorityOverrideRegex();
[GeneratedRegex(SYSTEM_PROMPT_SPOOFING_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex SystemPromptSpoofingRegex();
[GeneratedRegex(SYSTEM_PROMPT_EXFILTRATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex SystemPromptExfiltrationRegex();
[GeneratedRegex(PROMPT_ECHO_EXFILTRATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex PromptEchoExfiltrationRegex();
[GeneratedRegex(POLICY_BYPASS_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex PolicyBypassRegex();
[GeneratedRegex(ROLE_REASSIGNMENT_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex RoleReassignmentRegex();
[GeneratedRegex(PRIVILEGED_PERSONA_ACTIVATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex PrivilegedPersonaActivationRegex();
[GeneratedRegex(TOOL_OR_SECRET_EXFILTRATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex ToolOrSecretExfiltrationRegex();
[GeneratedRegex(CONVERSATION_MEMORY_EXFILTRATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex ConversationMemoryExfiltrationRegex();
[GeneratedRegex(TOOL_CALL_MANIPULATION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex ToolCallManipulationRegex();
[GeneratedRegex(AGENT_THOUGHT_INJECTION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex AgentThoughtInjectionRegex();
[GeneratedRegex(DELIMITER_WRAPPED_ATTACK_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex DelimiterWrappedAttackRegex();
[GeneratedRegex(HIDDEN_MARKUP_INJECTION_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex HiddenMarkupInjectionRegex();
[GeneratedRegex(LATEX_INVISIBLE_TEXT_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex LatexInvisibleTextRegex();
[GeneratedRegex(UNICODE_SMUGGLING_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex UnicodeSmugglingRegex();
[GeneratedRegex(IGNORE_SAFETY_AFTER_DATA_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex IgnoreSafetyAfterDataRegex();
[GeneratedRegex(PERSISTENT_OR_DELAYED_TRIGGER_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex PersistentOrDelayedTriggerRegex();
[GeneratedRegex(JAILBREAK_MARKER_PATTERN, RULE_OPTIONS, MATCH_TIMEOUT_MILLISECONDS)]
private static partial Regex JailbreakMarkerRegex();
}

View File

@ -0,0 +1,12 @@
namespace AIStudio.Tools.Security;
public sealed class PromptInjectionScanResult(PromptInjectionSource source, IReadOnlyList<PromptInjectionFinding> findings)
{
public PromptInjectionSource Source { get; } = source;
public IReadOnlyList<PromptInjectionFinding> Findings { get; } = findings;
public bool IsBlocked => this.Findings.Count > 0;
public IReadOnlyList<string> RuleIds => this.Findings.Select(finding => finding.RuleId).Distinct(StringComparer.Ordinal).ToList();
}

View File

@ -0,0 +1,293 @@
using System.Buffers;
using System.Text;
using System.Text.RegularExpressions;
namespace AIStudio.Tools.Security;
public sealed class PromptInjectionScanner(ILogger<PromptInjectionScanner> logger)
{
private const int MAX_DECODED_CANDIDATES_PER_ENCODING = 12;
private const int MAX_DECODED_TEXT_LENGTH = 12_000;
private const int MAX_FINDINGS = 8;
private const int SNIPPET_RADIUS = 80;
private static readonly IReadOnlyDictionary<(int Length, char First, char Last), string[]> TYPOGLYCEMIA_KEYWORDS =
CreateTypoglycemiaKeywordIndex();
public PromptInjectionScanResult Scan(string text, PromptInjectionSource source)
{
if (string.IsNullOrWhiteSpace(text))
return new(source, []);
var findings = new List<PromptInjectionFinding>();
var findingKeys = new HashSet<string>(StringComparer.Ordinal);
this.ScanVariant(text, "raw", findings, findingKeys);
if (findings.Count >= MAX_FINDINGS)
return new(source, findings);
var collapsed = CollapseCharacterSpacedContent(text);
if (!string.Equals(text, collapsed, StringComparison.Ordinal))
this.ScanVariant(collapsed, "character_spacing", findings, findingKeys);
if (findings.Count < MAX_FINDINGS)
this.ScanDecodedCandidates(text, findings, findingKeys);
if (findings.Count < MAX_FINDINGS)
this.ScanTypoglycemia(text, findings, findingKeys);
return new(source, findings);
}
private void ScanVariant(string text, string stage, List<PromptInjectionFinding> findings, HashSet<string> findingKeys)
{
try
{
if (!PromptInjectionPatterns.AnyRuleRegex().IsMatch(text))
return;
}
catch (RegexMatchTimeoutException exception)
{
logger.LogWarning(exception, "Prompt-injection regex prefilter timed out during stage '{Stage}'. Falling back to individual rules.", stage);
}
foreach (var rule in PromptInjectionPatterns.RULES)
{
if (findings.Count >= MAX_FINDINGS)
return;
Match match;
try
{
match = rule.Regex.Match(text);
}
catch (RegexMatchTimeoutException exception)
{
logger.LogWarning(exception, "Prompt-injection regex '{RuleId}' timed out during stage '{Stage}'.", rule.Id, stage);
continue;
}
if (!match.Success)
continue;
var snippet = ExtractSnippet(text, match.Index, match.Length, SNIPPET_RADIUS);
AddFinding(findings, findingKeys, new(rule.Id, rule.Category, stage, snippet));
}
}
private void ScanDecodedCandidates(string text, List<PromptInjectionFinding> findings, HashSet<string> findingKeys)
{
var processed = 0;
var seenDecodedTexts = new HashSet<string>(StringComparer.Ordinal);
foreach (var match in PromptInjectionPatterns.Base64Regex().EnumerateMatches(text))
{
if (processed++ >= MAX_DECODED_CANDIDATES_PER_ENCODING || findings.Count >= MAX_FINDINGS)
break;
var decoded = TryDecodeBase64(text.AsSpan(match.Index, match.Length));
if (decoded is not null && seenDecodedTexts.Add(decoded))
this.ScanVariant(decoded, "decoded_base64", findings, findingKeys);
}
processed = 0;
seenDecodedTexts.Clear();
foreach (var match in PromptInjectionPatterns.HexPairRegex().EnumerateMatches(text))
{
if (processed++ >= MAX_DECODED_CANDIDATES_PER_ENCODING || findings.Count >= MAX_FINDINGS)
break;
var decoded = TryDecodeHex(text.AsSpan(match.Index, match.Length));
if (decoded is not null && seenDecodedTexts.Add(decoded))
this.ScanVariant(decoded, "decoded_hex_pairs", findings, findingKeys);
}
processed = 0;
seenDecodedTexts.Clear();
foreach (var match in PromptInjectionPatterns.HexCompactRegex().EnumerateMatches(text))
{
if (processed++ >= MAX_DECODED_CANDIDATES_PER_ENCODING || findings.Count >= MAX_FINDINGS)
break;
var decoded = TryDecodeHex(text.AsSpan(match.Index, match.Length));
if (decoded is not null && seenDecodedTexts.Add(decoded))
this.ScanVariant(decoded, "decoded_hex", findings, findingKeys);
}
}
private void ScanTypoglycemia(string text, List<PromptInjectionFinding> findings, HashSet<string> findingKeys)
{
foreach (var match in PromptInjectionPatterns.WordRegex().EnumerateMatches(text))
{
if (findings.Count >= MAX_FINDINGS)
return;
var token = text.AsSpan(match.Index, match.Length);
var key = (token.Length, char.ToLowerInvariant(token[0]), char.ToLowerInvariant(token[^1]));
if (!TYPOGLYCEMIA_KEYWORDS.TryGetValue(key, out var keywords))
continue;
foreach (var keyword in keywords)
{
if (!IsTypoglycemiaVariant(token, keyword))
continue;
var snippet = ExtractSnippet(text, match.Index, match.Length, SNIPPET_RADIUS);
AddFinding(findings, findingKeys, new($"typoglycemia:{keyword}", "evasion", "typoglycemia", snippet));
break;
}
}
}
private static bool IsTypoglycemiaVariant(ReadOnlySpan<char> token, string keyword)
{
if (token.Equals(keyword, StringComparison.OrdinalIgnoreCase))
return false;
Span<int> characterCounts = stackalloc int[26];
for (var index = 1; index < token.Length - 1; index++)
{
characterCounts[char.ToLowerInvariant(token[index]) - 'a']++;
characterCounts[keyword[index] - 'a']--;
}
foreach (var count in characterCounts)
{
if (count != 0)
return false;
}
return true;
}
private static string CollapseCharacterSpacedContent(string text)
{
return PromptInjectionPatterns.SpacedLetterSequenceRegex().Replace(text, static match =>
{
var builder = new StringBuilder(match.Value.Length);
foreach (var character in match.Value)
{
if (char.IsLetter(character))
builder.Append(character);
}
return builder.ToString();
});
}
private static string? TryDecodeBase64(ReadOnlySpan<char> candidate)
{
var encodedLength = Math.Min(candidate.Length, ((MAX_DECODED_TEXT_LENGTH + 2) / 3) * 4);
encodedLength -= encodedLength % 4;
if (encodedLength == 0)
return null;
var bytes = ArrayPool<byte>.Shared.Rent(MAX_DECODED_TEXT_LENGTH);
try
{
if (!Convert.TryFromBase64Chars(candidate[..encodedLength], bytes, out var bytesWritten))
return null;
return ConvertDecodedBytesToText(bytes.AsSpan(0, bytesWritten));
}
finally
{
ArrayPool<byte>.Shared.Return(bytes);
}
}
private static string? TryDecodeHex(ReadOnlySpan<char> candidate)
{
var bytes = ArrayPool<byte>.Shared.Rent(MAX_DECODED_TEXT_LENGTH);
try
{
var bytesWritten = 0;
var highNibble = -1;
foreach (var character in candidate)
{
var nibble = HexValue(character);
if (nibble < 0)
continue;
if (highNibble < 0)
{
highNibble = nibble;
continue;
}
bytes[bytesWritten++] = (byte)((highNibble << 4) | nibble);
highNibble = -1;
if (bytesWritten >= MAX_DECODED_TEXT_LENGTH)
break;
}
return bytesWritten == 0 ? null : ConvertDecodedBytesToText(bytes.AsSpan(0, bytesWritten));
}
finally
{
ArrayPool<byte>.Shared.Return(bytes);
}
}
private static int HexValue(char character)
{
if (character is >= '0' and <= '9')
return character - '0';
if (character is >= 'A' and <= 'F')
return character - 'A' + 10;
if (character is >= 'a' and <= 'f')
return character - 'a' + 10;
return -1;
}
private static string? ConvertDecodedBytesToText(ReadOnlySpan<byte> bytes)
{
if (bytes.IsEmpty)
return null;
var text = Encoding.UTF8.GetString(bytes);
return LooksTextLike(text) ? text : null;
}
private static bool LooksTextLike(string text)
{
if (string.IsNullOrWhiteSpace(text))
return false;
var printableCount = 0;
foreach (var character in text)
{
if (!char.IsControl(character) || character is '\r' or '\n' or '\t')
printableCount++;
}
return printableCount >= text.Length * 0.85;
}
private static void AddFinding(List<PromptInjectionFinding> findings, HashSet<string> findingKeys, PromptInjectionFinding finding)
{
var key = $"{finding.RuleId}|{finding.DetectionStage}|{finding.Snippet}";
if (findingKeys.Add(key))
findings.Add(finding);
}
private static string ExtractSnippet(string text, int index, int length, int radius)
{
var start = Math.Max(0, index - radius);
var end = Math.Min(text.Length, index + length + radius);
var snippet = text[start..end].Replace('\r', ' ').Replace('\n', ' ').Trim();
return snippet.Length > radius * 2 ? $"{snippet[..(radius * 2)]}..." : snippet;
}
private static IReadOnlyDictionary<(int Length, char First, char Last), string[]> CreateTypoglycemiaKeywordIndex()
{
string[] keywords =
[
"ignore", "bypass", "override", "reveal", "forget", "disregard", "delete", "reset", "expose",
"system", "prompt", "policy", "safety", "developer", "instructions", "admin", "secret", "token", "credential",
];
return keywords
.GroupBy(keyword => (keyword.Length, keyword[0], keyword[^1]))
.ToDictionary(group => group.Key, group => group.ToArray());
}
}

View File

@ -0,0 +1,12 @@
namespace AIStudio.Tools.Security;
public readonly record struct PromptInjectionSource(string Kind, string Label)
{
public static PromptInjectionSource WebContent(string url) => new("Web content", url);
public static PromptInjectionSource FileContent(string filePath) => new("File content", filePath);
public static PromptInjectionSource ChatAttachment(string filePath) => new("Chat attachment", filePath);
public static PromptInjectionSource RetrievalContext(string dataSourceName, string path) => new("Retrieval context", $"{dataSourceName}: {path}");
}

View File

@ -1,4 +1,5 @@
using AIStudio.Dialogs; using AIStudio.Dialogs;
using AIStudio.Tools.Security;
using AIStudio.Tools.PluginSystem; using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Services; using AIStudio.Tools.Services;
using DialogOptions = AIStudio.Dialogs.DialogOptions; using DialogOptions = AIStudio.Dialogs.DialogOptions;
@ -46,6 +47,7 @@ public static class UserFile
} }
var fileContent = await rustService.ReadArbitraryFileData(filePath, int.MaxValue); var fileContent = await rustService.ReadArbitraryFileData(filePath, int.MaxValue);
return fileContent; var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
return await guardService.EnsureSafeForLlmAsync(fileContent, PromptInjectionSource.FileContent(filePath));
} }
} }

View File

@ -1,5 +1,6 @@
# v26.7.3, build 245 (2026-07-xx xx:xx UTC) # v26.7.3, build 245 (2026-07-xx xx:xx UTC)
- Improved the "My Tasks Assistant": you can now provide one or more documents in addition to text or use documents alone when asking to identify tasks. - Improved the "My Tasks Assistant": you can now provide one or more documents in addition to text or use documents alone when asking to identify tasks.
- Added checks for prompt injections in file attachments, images and web pages to process large documents faster and with lower memory usage.
- Fixed an issue that could leave AI Studio unresponsive after waking the computer from sleep. - Fixed an issue that could leave AI Studio unresponsive after waking the computer from sleep.
- Fixed enterprise configuration plugins from Windows-created ZIP files not loading correctly on Linux when the ZIP contained plugin files inside a folder. - Fixed enterprise configuration plugins from Windows-created ZIP files not loading correctly on Linux when the ZIP contained plugin files inside a folder.
- Upgraded Rust to v1.97.0. - Upgraded Rust to v1.97.0.