using AIStudio.Settings; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Services; namespace AIStudio.Tools.Security; /// /// Filters prompt injections out of external content before it reaches a model. /// /// /// The detection itself lives in the Rust runtime. File content is filtered while the runtime /// streams it, so it never passes through here; what this service adds is the path for content /// the runtime does not read itself — web pages and retrieval contexts — and the reporting the /// user sees.

/// The reporting works per user action, cf. BeginAction: each action tells the user once, when /// it ends, about what was filtered for it. One service serves the whole app, so the action has /// to travel with the content instead of being kept here. ///
public sealed class PromptInjectionGuardService( RustService rustService, SettingsManager settingsManager, ILogger logger, ILoggerFactory loggerFactory) { public const string WIKI_URL = "https://en.wikipedia.org/wiki/Prompt_engineering#Prompt_injection"; private const string DETECTION_LOG_CATEGORY = "PromptInjectionProtection"; private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PromptInjectionGuardService).Namespace, nameof(PromptInjectionGuardService)); private readonly ILogger detectionLogger = loggerFactory.CreateLogger(DETECTION_LOG_CATEGORY); private readonly Lock reportLock = new(); /// /// Filters prompt injections out of a text the runtime did not read itself, such as a web /// page or a retrieval context. /// /// /// Returns usable text in every case. When the runtime cannot be reached, the text is passed /// through unchanged: refusing the user's content because a check could not run would cost /// them their work over a check that is best-effort anyway. The failure is logged and shown, /// so it does not pass silently. /// /// The content to filter. /// Where the content came from, for the report shown to the user. /// The action the content belongs to, or null when it belongs to none. /// The content with any suspicious passages removed. public async Task SanitizeAsync(string text, PromptInjectionSource source, PromptInjectionReportingScope? reportingScope = null) { if (string.IsNullOrWhiteSpace(text)) return text; if (await rustService.SanitizePromptInjections(text) is not { } response) { logger.LogError("Could not check {SourceKind} '{SourceLabel}' for prompt injections. The content is used unchanged.", source.Kind, source.Label); await MessageBus.INSTANCE.SendWarning(new( Icons.Material.Filled.GppMaybe, string.Format(TB("AI Studio could not check '{0}' for prompt injections. The content is used as it is."), source.NotificationLabel))); return text; } if (response.RedactedCount > 0) await this.ReportAsync(new(source, response.Findings, response.RedactedCount), reportingScope); return response.SanitizedText; } /// /// Filters prompt injections out of several texts in one runtime request. /// /// /// For content that belongs to one user action, such as every page a web search returned. /// The user gets a single report for the whole action, and texts sharing a source are /// reported as that one source.

/// Returns usable text in every case, for the reason given on the single-text overload. When /// the check cannot run, every text is passed through unchanged. ///
/// The contents to filter, each with its source. /// The action the contents belong to, or null when they belong to none. /// The contents with any suspicious passages removed, in the order they came in. public async Task> SanitizeAsync(IReadOnlyList texts, PromptInjectionReportingScope? reportingScope = null) { if (texts.Count is 0) return []; // // Empty fields are common — many pages have no description or authors — and the runtime // has nothing to do with them. Only the texts with content are sent, and their positions // are remembered so the answer can be put back in the caller's order. // var sanitizedTexts = texts.Select(x => x.Text).ToArray(); List indicesToScan = []; for (var index = 0; index < texts.Count; index++) { if (!string.IsNullOrWhiteSpace(texts[index].Text)) indicesToScan.Add(index); } if (indicesToScan.Count is 0) return sanitizedTexts; var responses = await rustService.SanitizePromptInjectionsBatch(indicesToScan.Select(index => texts[index].Text).ToList()); if (responses is null) { var sources = texts.Select(x => x.Source).Distinct().ToList(); logger.LogError("Could not check {SourceCount} content source(s) for prompt injections. The content is used unchanged. Sources: {SourceLabels}", sources.Count, string.Join(", ", sources.Select(x => $"{x.Kind} '{x.Label}'"))); await MessageBus.INSTANCE.SendWarning(new( Icons.Material.Filled.GppMaybe, sources.Count is 1 ? string.Format(TB("AI Studio could not check '{0}' for prompt injections. The content is used as it is."), sources[0].NotificationLabel) : string.Format(TB("AI Studio could not check {0} sources for prompt injections. The content is used as it is."), sources.Count))); return sanitizedTexts; } // // Findings are collected per source, not per text: a page whose content and title were // both filtered is one thing that happened to the user, not two. Doing so here rather than // in the report keeps both counts: two texts with the same findings would look like one // text filtered twice there. // var findingsBySource = new Dictionary Findings, int RedactedCount)>(); for (var responseIndex = 0; responseIndex < indicesToScan.Count; responseIndex++) { var response = responses[responseIndex]; var textIndex = indicesToScan[responseIndex]; sanitizedTexts[textIndex] = response.SanitizedText; if (response.RedactedCount is 0) continue; var source = texts[textIndex].Source; if (!findingsBySource.TryGetValue(source, out var aggregate)) aggregate = ([], 0); aggregate.Findings.AddRange(response.Findings); findingsBySource[source] = (aggregate.Findings, aggregate.RedactedCount + response.RedactedCount); } if (findingsBySource.Count is 0) return sanitizedTexts; // // One scope around all sources, so a search across five pages reports once instead of // five times: // await using var batchScope = this.BeginAction(within: reportingScope); foreach (var (source, aggregate) in findingsBySource) await this.ReportAsync(new(source, aggregate.Findings, aggregate.RedactedCount), batchScope); return sanitizedTexts; } /// /// Records what was filtered out of one piece of content and tells the user about it. /// /// /// Within an action, the result is collected and reported together with the rest of that /// action once it ends. Without one it is reported right away, and so is a result whose /// action has ended already: waiting for the next action would either never reach the user, /// or reach them as part of an unrelated action later on.

/// Only the action passed in counts, never what else is running. A run over a large data /// source takes minutes, and a message sent meanwhile must not wait for it, nor end up in the /// same report. ///
/// What was filtered out of the content. /// The action the content belongs to, or null when it belongs to none. public async Task ReportAsync(PromptInjectionScanResult result, PromptInjectionReportingScope? reportingScope = null) { if (!result.WasFiltered) return; lock (this.reportLock) { if (FindCollectingScope(reportingScope) is { } collectingScope) { collectingScope.PendingResults.Add(result); return; } } await this.ReportNowAsync([result]); } /// /// Marks the start of one user action, such as attaching a batch of files or sending a /// message. /// /// /// Results are collected until the action finishes, so the user gets one report about /// twenty documents instead of twenty reports. An action begun within another one is part /// of it, and only the outermost one reports. Unrelated actions report on their own, each /// as soon as it ends, however much they overlap.

/// The scope has to travel with the content: hand it to wherever that content is read or /// filtered, such as ReportAsync, SanitizeAsync, or RustService.ReadArbitraryFileData. ///
/// The action this one is part of, or null when it is an action of its own. /// A scope that reports what was filtered once it is disposed. public PromptInjectionReportingScope BeginAction(PromptInjectionReportingScope? within = null) => new(this, within); /// /// Ends an action and reports what was collected for it. /// /// /// A scope within an action which still runs has collected nothing, since its results went to /// the outermost scope. It collects only once the outer one has ended before it, e.g., for work /// which outlived the action that started it. Ending a scope a second time does nothing. /// /// The scope which ends. internal async Task EndActionAsync(PromptInjectionReportingScope scope) { List results; lock (this.reportLock) { if (!scope.IsOpen) return; scope.IsOpen = false; results = [..scope.PendingResults]; scope.PendingResults.Clear(); } if (results.Count > 0) await this.ReportNowAsync(results); } /// /// Finds the scope which collects a result of the given one: the outermost scope of its action /// which is still running. /// /// /// Call under the report lock only. /// /// The scope the result was reported for, or null. /// The collecting scope, or null when the result has to be reported right away. private static PromptInjectionReportingScope? FindCollectingScope(PromptInjectionReportingScope? scope) { PromptInjectionReportingScope? collectingScope = null; for (; scope is not null; scope = scope.Parent) { if (scope.IsOpen) collectingScope = scope; } return collectingScope; } /// /// Combines the results of one report into one result per source, in the order the sources came in. /// /// /// An action may filter the same content twice: the classic data source search filters a passage when /// it checks whether the passage fits the question, and once more when it hands the passage to the /// model. Both times, the result is the same, so a result equal to one already seen for its source is /// dropped; the user hears about the passage once. The findings carry no position, though, so two /// passages of one file with exactly the same findings count as one as well. That mostly happens /// where two neighboring chunks overlap, which is the same passage indeed.

/// Other results of one source, such as two passages of one file or the mails of one mailbox, /// become a single result with all their findings and the sum of the removed passages. The report /// counts sources, and one file is one source, however many of its passages were filtered. ///
/// The results of one report. /// One result per source. private static IReadOnlyList CombineBySource(IReadOnlyList results) { if (results.Count < 2) return results; return results.GroupBy(result => result.Source).Select(CombineOneSource).ToList(); static PromptInjectionScanResult CombineOneSource(IGrouping resultsOfSource) { List distinctResults = []; foreach (var result in resultsOfSource) { if (!distinctResults.Any(known => known.RedactedCount == result.RedactedCount && known.Findings.SequenceEqual(result.Findings))) distinctResults.Add(result); } return distinctResults.Count is 1 ? distinctResults[0] : new(resultsOfSource.Key, distinctResults.SelectMany(result => result.Findings).ToList(), distinctResults.Sum(result => result.RedactedCount)); } } private async Task ReportNowAsync(IReadOnlyList reportedResults) { var results = CombineBySource(reportedResults); var totalCount = results.Sum(result => result.RedactedCount); this.detectionLogger.LogWarning( "Detected and removed {PassageCount} potentially dangerous passage(s) in {SourceCount} content source(s).", totalCount, results.Count); await MessageBus.INSTANCE.SendWarning(new( Icons.Material.Filled.GppMaybe, results.Count is 1 ? string.Format(TB("AI Studio removed suspicious instructions from '{0}' before using it."), results[0].Source.NotificationLabel) : string.Format(TB("AI Studio removed suspicious instructions from {0} sources before using them."), results.Count))); if (settingsManager.ConfigurationData.App.ShowPromptInjectionAlert) await MessageBus.INSTANCE.SendMessage(null, Event.SHOW_PROMPT_INJECTION_ALERT, new(results)); } }