From ac3f90373350b9d81caf724f614c032f09b3a1b3 Mon Sep 17 00:00:00 2001 From: hart_s3 Date: Fri, 7 Aug 2026 15:22:42 +0200 Subject: [PATCH] Refactored prompt injection handling logic with enhanced blocking patterns --- .../Assistants/I18N/allTexts.lua | 67 +++++++- .../Dialogs/PromptInjectionAlertDialog.razor | 152 ++++++++++++++---- .../PromptInjectionAlertDialog.razor.cs | 28 ++++ .../plugin.lua | 102 ++++++++++++ .../plugin.lua | 67 ++++++++ .../Tools/Security/PromptInjectionFinding.cs | 2 +- .../Security/PromptInjectionGuardService.cs | 2 +- .../Tools/Security/PromptInjectionPatterns.cs | 2 +- 8 files changed, 376 insertions(+), 46 deletions(-) diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 10ee1882..687ee48c 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -5572,26 +5572,77 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINGGUIDELINEDIALOG::T384594633"] = "Th -- Prompting Guideline UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINGGUIDELINEDIALOG::T4250996615"] = "Prompting Guideline" --- Source -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1642243064"] = "Source" +-- Chat attachment +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1071345316"] = "Chat attachment" + +-- Danger detected +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1081126836"] = "Danger detected" + +-- Content source +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1129278507"] = "Content source" + +-- Attempt to override instructions +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T161976090"] = "Attempt to override instructions" + +-- Attempt to expose protected data +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2050274293"] = "Attempt to expose protected data" + +-- Attempt to bypass safeguards +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2260642992"] = "Attempt to bypass safeguards" + +-- Attempt to change the AI's role +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2340508370"] = "Attempt to change the AI's role" -- AI Studio blocked this content before it reached a model or agent. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2378027194"] = "AI Studio blocked this content before it reached a model or agent." +-- Hidden instructions using markup +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2526538070"] = "Hidden instructions using markup" + +-- Web content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2626468388"] = "Web content" + +-- Source type +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T280442848"] = "Source type" + +-- Prompt injection is a method used to manipulate AI systems such as chatbots. An attacker places misleading instructions in content so that the AI treats them as legitimate. This can cause the AI to ignore safeguards, expose private information, or generate harmful content. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3122726298"] = "Prompt injection is a method used to manipulate AI systems such as chatbots. An attacker places misleading instructions in content so that the AI treats them as legitimate. This can cause the AI to ignore safeguards, expose private information, or generate harmful content." + +-- Hidden instructions using delimiters +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T329656456"] = "Hidden instructions using delimiters" + +-- Retrieved context +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3347144620"] = "Retrieved context" + -- Close UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3448155331"] = "Close" --- Prompt Injection Detected -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3580322580"] = "Prompt Injection Detected" +-- Attempt to manipulate an agent +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T355252317"] = "Attempt to manipulate an agent" --- Source kind -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T47437466"] = "Source kind" +-- File content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3788064862"] = "File content" + +-- Persistent or delayed instruction +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4169123215"] = "Persistent or delayed instruction" + +-- Typical attacks on AI systems (e.g. prompt injection) hide instructions within untrusted content to trick an AI model into ignoring its intended rules or performing unintended actions. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4221674400"] = "Typical attacks on AI systems (e.g. prompt injection) hide instructions within untrusted content to trick an AI model into ignoring its intended rules or performing unintended actions." + +-- Detected content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4223810750"] = "Detected content" -- More information UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T475337262"] = "More information" --- Detected signals -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T556618402"] = "Detected signals" +-- Hidden instructions using encoding +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T49495195"] = "Hidden instructions using encoding" + +-- Hide more information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T808738984"] = "Hide more information" + +-- Obfuscated instruction +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T87316699"] = "Obfuscated instruction" -- Hugging Face Inference Provider UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROVIDERDIALOG::T1085481431"] = "Hugging Face Inference Provider" diff --git a/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor b/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor index e51b7a63..f2fbc798 100644 --- a/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor +++ b/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor @@ -1,49 +1,131 @@ @using AIStudio.Tools.Security @inherits MSGComponentBase - - - - - @T("Prompt Injection Detected") - - + + - @if (this.Result is not null) + + @if (Result is not null) { - - @T("AI Studio blocked this content before it reached a model or agent.") - + + + + + - - @T("Source kind"): @this.Result.Source.Kind - - - @T("Source"): @this.Result.Source.Label - + + + @T("Danger detected") + - - @T("Detected signals") - - - @foreach (var finding in this.Result.Findings) - { - - @($"{finding.Category} / {finding.DetectionStage}: {finding.Snippet}") - - } - + + @T("AI Studio blocked this content before it reached a model or agent.") + + + + - - @T("More information"): - - @PromptInjectionGuardService.WIKI_URL - - + + + + + @(showPromptInjectionInformation + ? T("Hide more information") + : T("More information")) + + + + + + + @T("Typical attacks on AI systems (e.g. prompt injection) hide instructions within untrusted content to trick an AI model into ignoring its intended rules or performing unintended actions.") + + + + + + + + + + + @T("Source type") + + + + @this.GetSourceKindLabel(Result.Source.Kind) + + + + + + + + + + @T("Content source") + + + + @Result.Source.Label + + + + + + + + + + @T("Detected content") + + + @foreach (var finding in Result.Findings) + { + + + + @($"{this.GetFindingCategoryLabel(finding.Category)}") + + + + + @finding.Snippet + + + } + + + + + + + + + @T("Prompt injection is a method used to manipulate AI systems such as chatbots. An attacker places misleading instructions in content so that the AI treats them as legitimate. This can cause the AI to ignore safeguards, expose private information, or generate harmful content.") + + + + @PromptInjectionGuardService.WIKI_URL + + + } - + + @T("Close") diff --git a/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor.cs b/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor.cs index 4e87fe2e..c4168c89 100644 --- a/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/PromptInjectionAlertDialog.razor.cs @@ -7,6 +7,8 @@ namespace AIStudio.Dialogs; public partial class PromptInjectionAlertDialog : MSGComponentBase { + private bool showPromptInjectionInformation; + [CascadingParameter] private IMudDialogInstance MudDialog { get; set; } = null!; @@ -14,4 +16,30 @@ public partial class PromptInjectionAlertDialog : MSGComponentBase public PromptInjectionScanResult Result { get; set; } = null!; private void Close() => this.MudDialog.Close(); + + private void TogglePromptInjectionInformation() => this.showPromptInjectionInformation = !this.showPromptInjectionInformation; + + private string GetSourceKindLabel(string sourceKind) => sourceKind switch + { + "Web content" => T("Web content"), + "File content" => T("File content"), + "Chat attachment" => T("Chat attachment"), + "Retrieval context" => T("Retrieved context"), + _ => sourceKind, + }; + + private string GetFindingCategoryLabel(string category) => category switch + { + "override" => T("Attempt to override instructions"), + "role_override" => T("Attempt to change the AI's role"), + "exfiltration" => T("Attempt to expose protected data"), + "jailbreak" => T("Attempt to bypass safeguards"), + "agent_manipulation" => T("Attempt to manipulate an agent"), + "delimiter_evasion" => T("Hidden instructions using delimiters"), + "markup_evasion" => T("Hidden instructions using markup"), + "encoding_evasion" => T("Hidden instructions using encoding"), + "persistence" => T("Persistent or delayed instruction"), + "evasion" => T("Obfuscated instruction"), + _ => category, + }; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index bae6a489..0361d1ee 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -4275,6 +4275,108 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T974954792"] = "Das Ergebn -- Are you sure you want to delete the chat '{0}' in the workspace '{1}'? UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::WORKSPACES::T1016188706"] = "Möchten Sie den Chat „{0}“ im Arbeitsbereich „{1}“ wirklich löschen?" +-- Chat attachment +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1071345316"] = "Dateianhang" + +-- Danger detected +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1081126836"] = "Gefahr erkannt" + +-- Content source +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1129278507"] = "Datei" + +-- Prompt injection hides instructions in untrusted content to make an AI model ignore its intended rules or perform unintended actions. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4221674400"] = "Typische Angriffe auf KI-Systeme (z.B. Prompt Injection) verbergen Anweisungen in nicht vertrauenswürdigen Inhalten, um ein KI-Modell dazu zu bringen, seine vorgesehenen Regeln zu ignorieren oder ungewollte Aktionen auszuführen." + +-- Attempt to override instructions +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T161976090"] = "Versuch, Anweisungen zu überschreiben" + +-- Attempt to expose protected data +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2050274293"] = "Versuch, geschützte Daten offenzulegen" + +-- Attempt to bypass safeguards +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2260642992"] = "Versuch, Sicherheitsvorkehrungen zu umgehen" + +-- Attempt to change the AI's role +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2340508370"] = "Versuch, die Rolle der KI zu ändern" + +-- AI Studio blocked this content before it reached a model or agent. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2378027194"] = "AI Studio hat diesen Inhalt blockiert, bevor er ein Modell oder einen Agenten erreicht hat." + +-- Hidden instructions using markup +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2526538070"] = "Versteckte Anweisungen mit Markup" + +-- Web content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2626468388"] = "Web-Inhalt" + +-- Source type +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T280442848"] = "Quelle" + +-- Prompt injection is a method used to manipulate AI systems such as chatbots. An attacker places misleading instructions in content so that the AI treats them as legitimate. This can cause the AI to ignore safeguards, expose private information, or generate harmful content. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3122726298"] = "Prompt Injection ist eine Methode, um KI-Systeme wie Chatbots zu manipulieren. Dabei platziert ein Angreifer irreführende Anweisungen in Inhalten, sodass die KI diese als legitim betrachtet. Dadurch kann die KI Sicherheitsvorkehrungen ignorieren, private Informationen preisgeben oder schädliche Inhalte erzeugen." + +-- Hidden instructions using delimiters +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T329656456"] = "Versteckte Anweisungen mit Trennzeichen" + +-- Retrieved context +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3347144620"] = "Es scheint, dass der Kontext bereits bereitgestellt wurde. Bitte geben Sie den Text an, den Sie aus dem Englischen (US) ins Deutsche (Deutschland) übersetzen möchten. Ich werde sicherstellen, dass die Übersetzung präzise, natürlich klingt und für die Zielgruppe leicht verständlich ist." + +-- Close +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3448155331"] = "Schließen" + +-- Attempt to manipulate an agent +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T355252317"] = "Versuch, einen Agenten zu manipulieren" + +-- File content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3788064862"] = "Inhalt der Datei" + +-- Persistent or delayed instruction +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4169123215"] = "Dauerhafte oder verzögerte Anweisung" + +-- Detected content +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T4223810750"] = "Erkannter Inhalt" + +-- More information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T475337262"] = "Weitere Informationen" + +-- Hidden instructions using encoding +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T49495195"] = "Versteckte Anweisungen mit Kodierung" + +-- Hide more information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T808738984"] = "Weniger Informationen anzeigen" + +-- Obfuscated instruction +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T87316699"] = "Verschleierte Anweisung" + +-- Protect against prompt injection in external content? +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1040385578"] = "Soll der Zugriff auf externe Inhalte vor Prompt-Injection geschützt werden?" + +-- A blocking alert explains the detected attack pattern +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2341755932"] = "Eine blockierende Warnung erklärt das erkannte Angriffsmuster." + +-- Show a learning alert when prompt injection is detected? +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2754886869"] = "Lernwarnung anzeigen, wenn Prompt-Injection erkannt wird?" + +-- Shows an explanation dialog with an external reference when AI Studio blocks suspicious content. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3129787554"] = "Zeigt einen Erklärungsdialog mit einem externen Verweis an, wenn AI Studio verdächtige Inhalte blockiert." + +-- Only the block notification is shown +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3491678837"] = "Nur die Blockbenachrichtigung wird angezeigt" + +-- Potential prompt injections are blocked before they reach an LLM +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3982766677"] = "Potenzielle Prompt-Injections werden blockiert, bevor sie ein LLM erreichen." + +-- External content is passed through without prompt-injection checks +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4245463281"] = "Externer Inhalt wird ohne Prompt-Injection-Prüfungen durchgereicht" + +-- Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T69192586"] = "Prüft Webinhalte, Dateianhänge, Abrufkontext und ähnliche externe Eingaben auf Prompt-Injection-Muster, bevor sie an ein Modell oder einen Agenten gesendet werden." + +-- Prompt Injection Detected +UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T3580322580"] = "Prompt-Injection erkannt" + +-- AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONGUARDSERVICE::T3560909296"] = "AI Studio hat Inhalte von '{0}' blockiert, da es sich um einen möglichen Prompt-Injection-Versuch handelt." + -- Move chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::WORKSPACES::T1133040906"] = "Chat verschieben" diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 1809e2a8..1b9105c4 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -4176,6 +4176,73 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VISION::T1061000046"] = "We hope this vis -- Integration of enterprise data UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VISION::T1127694951"] = "Integration of enterprise data" +-- Danger detected +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1081126836"] = "Danger detected" + +-- More Information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1266371726"] = "More Information" + +-- Prompt injection is an attempt to hide instructions in untrusted content so that an AI model ignores its intended rules or performs unintended actions. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T163682410"] = "Prompt injection is an attempt to hide instructions in untrusted content so that an AI model ignores its intended rules or performs unintended actions." + +-- Source +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1642243064"] = "Source" + +-- Hide More Information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T1648537896"] = "Hide More Information" + +-- AI Studio blocked this content before it reached a model or agent. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T2378027194"] = "AI Studio blocked this content before it reached a model or agent." + +-- Prompt Injection detected +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3052247348"] = "Prompt Injection detected" + +-- Close +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3448155331"] = "Close" + +-- Prompt injection is a trick used to manipulate AI systems like chatbots. Normally, these systems follow the rules set by their developers—such as being helpful and safe. But with prompt injection, an attacker crafts a clever input that makes the AI think it’s receiving a new, legitimate instruction. This can cause the AI to ignore its usual safeguards and do something unintended, like sharing private information or generating harmful content. The issue arises because the AI can’t always tell the difference between a trusted command and a deceptive one. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T3548442822"] = "Prompt injection is a trick used to manipulate AI systems like chatbots. Normally, these systems follow the rules set by their developers—such as being helpful and safe. But with prompt injection, an attacker crafts a clever input that makes the AI think it’s receiving a new, legitimate instruction. This can cause the AI to ignore its usual safeguards and do something unintended, like sharing private information or generating harmful content. The issue arises because the AI can’t always tell the difference between a trusted command and a deceptive one." + +-- Source kind +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T47437466"] = "Source kind" + +-- More information +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T475337262"] = "More information" + +-- Detected signals +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::PROMPTINJECTIONALERTDIALOG::T556618402"] = "Detected signals" + + +-- Protect against prompt injection in external content? +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1040385578"] = "Protect against prompt injection in external content?" + +-- A blocking alert explains the detected attack pattern +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2341755932"] = "A blocking alert explains the detected attack pattern" + +-- Show a learning alert when prompt injection is detected? +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2754886869"] = "Show a learning alert when prompt injection is detected?" + +-- Shows an explanation dialog with an external reference when AI Studio blocks suspicious content. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3129787554"] = "Shows an explanation dialog with an external reference when AI Studio blocks suspicious content." + +-- Only the block notification is shown +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3491678837"] = "Only the block notification is shown" + +-- Potential prompt injections are blocked before they reach an LLM +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3982766677"] = "Potential prompt injections are blocked before they reach an LLM" + +-- External content is passed through without prompt-injection checks +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4245463281"] = "External content is passed through without prompt-injection checks" + +-- Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T69192586"] = "Checks web content, file attachments, retrieval context, and similar external input for prompt-injection patterns before it is sent to a model or agent." + +-- Prompt Injection Detected +UI_TEXT_CONTENT["AISTUDIO::LAYOUT::MAINLAYOUT::T3580322580"] = "Prompt Injection Detected" + +-- AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SECURITY::PROMPTINJECTIONGUARDSERVICE::T3560909296"] = "AI Studio blocked content from '{0}' because it looks like a prompt-injection attempt." + -- Meet your needs UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VISION::T127032776"] = "Meet your needs" diff --git a/app/MindWork AI Studio/Tools/Security/PromptInjectionFinding.cs b/app/MindWork AI Studio/Tools/Security/PromptInjectionFinding.cs index 20c95167..864820c4 100644 --- a/app/MindWork AI Studio/Tools/Security/PromptInjectionFinding.cs +++ b/app/MindWork AI Studio/Tools/Security/PromptInjectionFinding.cs @@ -1,3 +1,3 @@ namespace AIStudio.Tools.Security; -public sealed record PromptInjectionFinding(string RuleId, string Category, string DetectionStage, string Snippet); \ No newline at end of file +public sealed record PromptInjectionFinding(string RuleId, string Category, string Snippet); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Security/PromptInjectionGuardService.cs b/app/MindWork AI Studio/Tools/Security/PromptInjectionGuardService.cs index 5f612247..28d1d6b8 100644 --- a/app/MindWork AI Studio/Tools/Security/PromptInjectionGuardService.cs +++ b/app/MindWork AI Studio/Tools/Security/PromptInjectionGuardService.cs @@ -8,7 +8,7 @@ public sealed class PromptInjectionGuardService( SettingsManager settingsManager, ILogger logger) { - public const string WIKI_URL = "https://de.wikipedia.org/wiki/Prompt-Engineering#Prompt_Injection"; + public const string WIKI_URL = "https://en.wikipedia.org/wiki/Prompt_engineering#Prompt_injection"; private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PromptInjectionGuardService).Namespace, nameof(PromptInjectionGuardService)); diff --git a/app/MindWork AI Studio/Tools/Security/PromptInjectionPatterns.cs b/app/MindWork AI Studio/Tools/Security/PromptInjectionPatterns.cs index ad56775c..6c92c3a2 100644 --- a/app/MindWork AI Studio/Tools/Security/PromptInjectionPatterns.cs +++ b/app/MindWork AI Studio/Tools/Security/PromptInjectionPatterns.cs @@ -40,7 +40,7 @@ internal static partial class PromptInjectionPatterns private const string POLICY_BYPASS_PATTERN = """(?:do\s+not|don't|stop\s+to|never)\s+(?:follow|obey|respect|apply|enforce)\s+(?:the\s+)?(?:system|developer|safety|security|content|usage)\s+(?:prompt|message|instructions?|policy|policies|guardrails?|restrictions?)"""; private const string ROLE_REASSIGNMENT_PATTERN = """(?:you\s+are\s+now|you\s+are\s+no\s+longer|act\s+as|pretend\s+to\s+be|simulate\s+being|assume\s+the\s+role\s+of|from\s+now\s+on\s+you\s+are)\s+(?:an\s+)?(?:unfiltered|unrestricted|developer|system|root|admin|jailbroken|evil|dan|do\s+anything\s+now)"""; private const string PRIVILEGED_PERSONA_ACTIVATION_PATTERN = """\b(?:developer\s+mode|debug\s+mode|admin\s+mode|root\s+mode|god\s+mode|maintenance\s+mode|dan\s*(?:mode)?|do\s+anything\s+now|grandmother\s+trick)\b"""; - private const string TOOL_OR_SECRET_EXFILTRATION_PATTERN = """(?:export|send|return|reveal|show|print|list|dump|exfiltrate)\s+(?:all\s+)?(?:tools?|functions?|plugins?|api\s*keys?|keys?|tokens?|credentials?|secrets?|passwords?|hidden\s+instructions?|environment\s+variables?|system\s+information)"""; + private const string TOOL_OR_SECRET_EXFILTRATION_PATTERN = """(?:export|send|return|reveal|show|print|output|list|dump|exfiltrate)\s+(?:all\s+)?(?:tools?|functions?|plugins?|api\s*keys?|keys?|tokens?|credentials?|secrets?|passwords?|hidden\s+instructions?|environment\s+variables?|system\s+information|internal\s+data)"""; private const string CONVERSATION_MEMORY_EXFILTRATION_PATTERN = """(?:(?:show|print|reveal|return|dump|list)\s+(?:the\s+)?(?:conversation\s+history|chat\s+history|memory|scratchpad|chain\s+of\s+thought|reasoning|previous\s+user\s+messages?|prior\s+messages?)|(?:what\s+did\s+(?:the\s+)?previous\s+user\s+say))"""; private const string TOOL_CALL_MANIPULATION_PATTERN = """(?:(?:call|invoke|execute|run|use|trigger)\s+(?:the\s+)?(?:tool|function|plugin|api|browser|web|shell|terminal|command)[^\n]{0,120}(?:with|using|to)\s+(?:these\s+)?(?:arguments|params?|parameters)|(?:do\s+not|don't)\s+ask\s+for\s+(?:confirmation|approval|permission)|(?:silently|secretly|without\s+asking)\s+(?:call|invoke|execute|run|use))"""; private const string AGENT_THOUGHT_INJECTION_PATTERN = """(?:(?:thought|observation|reasoning|scratchpad|tool\s+output|assistant|system|developer)\s*[:=]\s*(?:ignore|bypass|override|reveal|call|execute)|forge\s+(?:an\s+)?(?:observation|tool\s+output|assistant\s+message)|pretend\s+(?:the\s+)?tool\s+(?:returned|said))""";