MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-08-11 17:32:11 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added security checks for confidence levels

Browse Source
This commit is contained in:
Paul Koudelka 2026-08-05 15:05:17 +02:00
parent 3eca153f83
commit 185a99c9ed
22 changed files with 322 additions and 125 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
app/MindWork AI Studio/Agents/AgentDataSourceSelection.cs
View File

@ -140,6 +140,8 @@ public sealed class AgentDataSourceSelection (ILogger<AgentDataSourceSelection>
//
// We start with the provider currently selected by the user:
var requiredDataSecurity = dataSources.AllowedDataSources.GetRequiredSecurityPolicy();
var requiredComplianceLevel = dataSources.AllowedDataSources.GetRequiredComplianceLevel();
var agentProvider = this.SettingsManager.GetPreselectedProvider(Tools.Components.AGENT_DATA_SOURCE_SELECTION, provider.Id, true);
if (agentProvider == Settings.Provider.NONE)
{
@ -147,6 +149,12 @@ public sealed class AgentDataSourceSelection (ILogger<AgentDataSourceSelection>
return [];
}
if (!agentProvider.AllowsDataSourceAccess(this.SettingsManager, requiredDataSecurity, requiredComplianceLevel))
{
logger.LogWarning($"The agent for data source selection uses provider '{agentProvider.InstanceName}' with confidence '{agentProvider.GetConfidenceLevel(this.SettingsManager).GetName()}', but the available data sources require data security '{requiredDataSecurity}' and provider confidence '{requiredComplianceLevel.GetName()}'. The agent cannot select data sources.");
return [];
}
// Assign the provider settings to the agent:
logger.LogInformation($"The agent for the data source selection uses the provider '{agentProvider.InstanceName}' ({agentProvider.UsedLLMProvider.ToName()}, confidence={agentProvider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level.GetName()}).");
this.ProviderSettings = agentProvider;

16
app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs
View File

@ -3,6 +3,7 @@ using System.Text.Json;
using AIStudio.Chat;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.RAG;
using AIStudio.Tools.Services;
@ -129,19 +130,30 @@ public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalConte
/// you can set the provider once and then call the validation method in parallel.
/// </remarks>
/// <param name="provider">The current LLM provider. When the user doesn't preselect an agent provider, the agent uses this provider.</param>
public void SetLLMProvider(IProvider provider)
/// <param name="requiredDataSecurity">The data security required by the retrieved data.</param>
/// <param name="requiredComplianceLevel">The minimum provider confidence required by the retrieved data.</param>
public bool SetLLMProvider(IProvider provider, DataSourceSecurity requiredDataSecurity = DataSourceSecurity.NOT_SPECIFIED, ConfidenceLevel requiredComplianceLevel = ConfidenceLevel.NONE)
{
// We start with the provider currently selected by the user:
var agentProvider = this.SettingsManager.GetPreselectedProvider(Tools.Components.AGENT_RETRIEVAL_CONTEXT_VALIDATION, provider.Id, true);
if (agentProvider == Settings.Provider.NONE)
{
logger.LogWarning("No provider is selected for the agent.");
return;
this.ProviderSettings = Settings.Provider.NONE;
return false;
}
if (!agentProvider.AllowsDataSourceAccess(this.SettingsManager, requiredDataSecurity, requiredComplianceLevel))
{
logger.LogWarning($"The agent for retrieval context validation uses provider '{agentProvider.InstanceName}' with confidence '{agentProvider.GetConfidenceLevel(this.SettingsManager).GetName()}', but the retrieved data requires data security '{requiredDataSecurity}' and provider confidence '{requiredComplianceLevel.GetName()}'. The agent cannot validate retrieval contexts.");
this.ProviderSettings = Settings.Provider.NONE;
return false;
}
// Assign the provider settings to the agent:
logger.LogInformation($"The agent for the retrieval context validation uses the provider '{agentProvider.InstanceName}' ({agentProvider.UsedLLMProvider.ToName()}, confidence={agentProvider.UsedLLMProvider.GetConfidence(this.SettingsManager).Level.GetName()}).");
this.ProviderSettings = agentProvider;
return true;
}
/// <summary>

6
app/MindWork AI Studio/Chat/ChatThread.cs
View File

@ -1,6 +1,7 @@
using System.Globalization;
using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ERIClient.DataModel;
@ -76,6 +77,11 @@ public sealed record ChatThread
/// </summary>
public DataSourceSecurity DataSecurity { get; set; } = DataSourceSecurity.NOT_SPECIFIED;
/// <summary>
/// The minimum provider confidence required by data sources used so far.
/// </summary>
public ConfidenceLevel DataComplianceLevel { get; set; } = ConfidenceLevel.NONE;
/// <summary>
/// The name of the chat thread. Usually generated by an AI model or manually edited by the user.
/// </summary>

19
app/MindWork AI Studio/Chat/ChatThreadExtensions.cs
View File

@ -11,12 +11,12 @@ public static class ChatThreadExtensions
/// </summary>
/// <remarks>
/// We don't check if the provider is allowed to use the data sources of the chat thread.
/// That kind of check is done in the RAG process itself.<br/><br/>
/// That kind of check is done when the available data sources are resolved.<br/><br/>
///
/// One thing which is not so obvious: after RAG was used on this thread, the entire chat
/// thread is kind of a data source by itself. Why? Because the augmentation data collected
/// from the data sources is stored in the chat thread. This means we must check if the
/// selected provider is allowed to use this thread's data.
/// selected provider is allowed to use this thread's data security and compliance level.
/// </remarks>
/// <param name="chatThread">The chat thread to check.</param>
/// <param name="provider">The provider to check.</param>
@ -26,7 +26,19 @@ public static class ChatThreadExtensions
// No chat thread available means we have a new chat. That's fine:
if (chatThread is null)
return true;
var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
var providerConfidenceLevel = provider switch
{
IProvider p => p.GetConfidenceLevel(settingsManager),
AIStudio.Settings.Provider p => p.GetConfidenceLevel(settingsManager),
_ => ConfidenceLevel.NONE,
};
if (!providerConfidenceLevel.AllowsDataSourceComplianceLevel(chatThread.DataComplianceLevel))
return false;
// The chat thread is available, but the data security is not specified.
// Means, we never used RAG or RAG was enabled, but no data sources were selected.
// That's fine as well:
@ -36,7 +48,6 @@ public static class ChatThreadExtensions
//
// Is the provider trusted for data-source security checks?
//
var settingsManager = Program.SERVICE_PROVIDER.GetRequiredService<SettingsManager>();
var isTrustedProvider = provider switch
{
IProvider p => p.IsTrustedForDataSourceSecurityChecks(settingsManager),

2
app/MindWork AI Studio/Chat/ContentText.cs
View File

@ -57,7 +57,7 @@ public sealed class ContentText : IContent
if(!chatThread.IsLLMProviderAllowed(provider))
{
LOGGER.LogError("The provider is not allowed for this chat thread due to data security reasons. Skipping the AI process.");
LOGGER.LogError("The provider is not allowed for this chat thread due to data security or compliance reasons. Skipping the AI process.");
await this.CompleteWithoutStreaming();
return chatThread;
}

2
app/MindWork AI Studio/Components/ChatComponent.razor
View File

@ -147,7 +147,7 @@
@if (!this.ChatThread.IsLLMProviderAllowed(this.Provider))
{
<MudTooltip Text="@T("The selected provider is not allowed in this chat due to data security reasons.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
<MudTooltip Text="@T("The selected provider is not allowed in this chat due to data security or compliance reasons.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
<MudIconButton Icon="@Icons.Material.Filled.Error" Color="Color.Error"/>
</MudTooltip>
}

6
app/MindWork AI Studio/Components/DataSourceSelection.razor
View File

@ -70,7 +70,7 @@
{
case true when this.availableDataSources.Count == 0:
<MudText Typo="Typo.body1" Class="mb-3">
@T("Your data sources cannot be used with the LLM provider you selected due to data privacy, or they are currently unavailable.")
@T("Your data sources cannot be used with the LLM provider you selected due to data privacy or compliance requirements, or they are currently unavailable.")
</MudText>
break;
@ -82,7 +82,7 @@
case false when this.availableDataSources.Count == 0:
<MudText Typo="Typo.body1" Class="mb-3">
@T("Your data sources cannot be used with the LLM provider you selected due to data privacy, or they are currently unavailable.")
@T("Your data sources cannot be used with the LLM provider you selected due to data privacy or compliance requirements, or they are currently unavailable.")
</MudText>
break;
@ -177,4 +177,4 @@ else if (this.SelectionMode is DataSourceSelectionMode.CONFIGURATION_MODE)
</MudField>
}
</MudPaper>
}
}

115
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor
View File

@ -73,7 +73,7 @@
</MudJustifiedText>
@if (this.CanChangeSourceAndEmbedding)
{
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
@foreach (var embedding in this.AvailableEmbeddings)
{
<MudSelectItem Value="@embedding.Value">
@ -99,18 +99,6 @@
@if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
{
<MudTextField
T="string"
Text="@this.SelectedEmbeddingTokenizerText"
Label="@T("Tokenizer")"
Class="mb-3"
ReadOnly="@true"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.DataObject"
AdornmentColor="Color.Info"
/>
if (this.SelectedCloudEmbedding)
{
<DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
@ -121,59 +109,11 @@
@T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
</MudJustifiedText>
}
<MudStack Class="mb-3">
<MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
@(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
</MudButton>
<MudDivider/>
<MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("Optional expert settings for how this data source is split before embedding.")
</MudJustifiedText>
<MudNumericField
T="int"
@bind-Value="@this.dataMaxChunkTokenLength"
Label="@T("Token limit")"
Class="mb-3"
Min="0"
Immediate="@true"
Validation="@this.ValidateMaxChunkTokenLength"
HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.DataObject"
AdornmentColor="Color.Info"
/>
<MudNumericField
T="int"
@bind-Value="@this.dataChunkOverlapTokenLength"
Label="@T("Token overlap")"
Min="0"
Immediate="@true"
Validation="@this.ValidateChunkOverlapTokenLength"
HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap.")"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.CompareArrows"
AdornmentColor="Color.Info"
/>
</MudCollapse>
</MudStack>
}
<ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
<MudSelect @bind-Value="@this.dataSecurityPolicy" Text="@this.dataSecurityPolicy.ToSelectionText()" Label="@T("Your security policy")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateSecurityPolicy">
@foreach (var policy in Enum.GetValues<DataSourceSecurity>())
{
<MudSelectItem Value="@policy">
@policy.ToSelectionText()
</MudSelectItem>
}
</MudSelect>
<MudSelect @bind-Value="@this.dataComplianceLevel" Text="@this.dataComplianceLevel.GetName()" Label="@T("Compliance level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start">
<MudSelect @bind-Value="@this.dataComplianceLevel" Text="@this.dataComplianceLevel.GetName()" Label="@T("Compliance level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateDataSourceComplianceLevel">
@foreach (var level in this.ComplianceLevels)
{
<MudSelectItem Value="@level.Value">
@ -182,7 +122,56 @@
}
</MudSelect>
<MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />
<MudStack Class="mb-3">
<MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
@(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
</MudButton>
<MudDivider/>
<MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
@if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
{
<MudTextField
T="string"
Text="@this.SelectedEmbeddingTokenizerText"
Label="@T("Tokenizer")"
Class="mb-3"
ReadOnly="@true"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.DataObject"
AdornmentColor="Color.Info"/>
}
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("Optional expert settings for how this data source is split before embedding.")
</MudJustifiedText>
<MudNumericField
T="int"
@bind-Value="@this.dataMaxChunkTokenLength"
Label="@T("Token limit")"
Class="mb-3"
Min="0"
Immediate="@true"
Validation="@this.ValidateMaxChunkTokenLength"
HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.DataObject"
AdornmentColor="Color.Info"/>
<MudNumericField
T="int"
@bind-Value="@this.dataChunkOverlapTokenLength"
Label="@T("Token overlap")"
Min="0"
Immediate="@true"
Validation="@this.ValidateChunkOverlapTokenLength"
HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap.")"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.CompareArrows"
AdornmentColor="Color.Info"/>
<MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />
</MudCollapse>
</MudStack>
</MudForm>
<Issues IssuesData="@this.dataIssues"/>
</DialogContent>

8
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs
View File

@ -49,7 +49,6 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
private int dataChunkOverlapTokenLength;
private ushort dataMaxMatches = 10;
private bool showExpertSettings;
private DataSourceSecurity dataSecurityPolicy;
private ConfidenceLevel dataComplianceLevel = ConfidenceLevel.UNKNOWN;
// We get the form reference from Blazor code to validate it manually:
@ -60,6 +59,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
this.dataSourceValidation = new()
{
GetSelectedCloudEmbedding = () => this.SelectedCloudEmbedding,
GetSelectedEmbeddingProvider = () => this.SelectedEmbedding,
GetSecurityPolicy = () => DataSourceSecurity.ALLOW_ANY,
GetComplianceLevel = () => this.dataComplianceLevel,
GetSettingsManager = () => this.SettingsManager,
GetPreviousDataSourceName = () => this.dataEditingPreviousInstanceName,
GetUsedDataSourceNames = () => this.UsedDataSourcesNames,
};
@ -87,7 +90,6 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
this.dataPath = this.DataSource.Path;
this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
this.dataComplianceLevel = this.DataSource.ComplianceLevel;
this.dataMaxMatches = this.DataSource.MaxMatches;
this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
@ -143,7 +145,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path,
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
SecurityPolicy = this.dataSecurityPolicy,
SecurityPolicy = DataSourceSecurity.ALLOW_ANY,
ComplianceLevel = this.dataComplianceLevel,
MaxMatches = this.dataMaxMatches,
};

1
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor
View File

@ -38,7 +38,6 @@
</MudJustifiedText>
}
<TextInfoLines Label="@T("Your security policy")" MaxLines="3" Value="@this.DataSource.SecurityPolicy.ToInfoText()" Color="@this.DataSource.SecurityPolicy.GetColor()" ClipboardTooltipSubject="@T("your security policy")"/>
<TextInfoLine Label="@T("Compliance level")" Value="@this.DataSource.ComplianceLevel.GetName()" ClipboardTooltipSubject="@T("the compliance level")"/>
<TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>

13
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor
View File

@ -73,7 +73,7 @@
</MudJustifiedText>
@if (this.CanChangeSourceAndEmbedding)
{
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingId">
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
@foreach (var embedding in this.AvailableEmbeddings)
{
<MudSelectItem Value="@embedding.Value">
@ -164,16 +164,7 @@
<ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
<MudSelect @bind-Value="@this.dataSecurityPolicy" Text="@this.dataSecurityPolicy.ToSelectionText()" Label="@T("Your security policy")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateSecurityPolicy">
@foreach (var policy in Enum.GetValues<DataSourceSecurity>())
{
<MudSelectItem Value="@policy">
@policy.ToSelectionText()
</MudSelectItem>
}
</MudSelect>
<MudSelect @bind-Value="@this.dataComplianceLevel" Text="@this.dataComplianceLevel.GetName()" Label="@T("Compliance level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start">
<MudSelect @bind-Value="@this.dataComplianceLevel" Text="@this.dataComplianceLevel.GetName()" Label="@T("Compliance level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateDataSourceComplianceLevel">
@foreach (var level in this.ComplianceLevels)
{
<MudSelectItem Value="@level.Value">

8
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs
View File

@ -49,7 +49,6 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
private int dataChunkOverlapTokenLength;
private ushort dataMaxMatches = 10;
private bool showExpertSettings;
private DataSourceSecurity dataSecurityPolicy;
private ConfidenceLevel dataComplianceLevel = ConfidenceLevel.UNKNOWN;
// We get the form reference from Blazor code to validate it manually:
@ -60,6 +59,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
this.dataSourceValidation = new()
{
GetSelectedCloudEmbedding = () => this.SelectedCloudEmbedding,
GetSelectedEmbeddingProvider = () => this.SelectedEmbedding,
GetSecurityPolicy = () => DataSourceSecurity.ALLOW_ANY,
GetComplianceLevel = () => this.dataComplianceLevel,
GetSettingsManager = () => this.SettingsManager,
GetPreviousDataSourceName = () => this.dataEditingPreviousInstanceName,
GetUsedDataSourceNames = () => this.UsedDataSourcesNames,
};
@ -87,7 +90,6 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
this.dataFilePath = this.DataSource.FilePath;
this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
this.dataComplianceLevel = this.DataSource.ComplianceLevel;
this.dataMaxMatches = this.DataSource.MaxMatches;
this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
@ -143,7 +145,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath,
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
SecurityPolicy = this.dataSecurityPolicy,
SecurityPolicy = DataSourceSecurity.ALLOW_ANY,
ComplianceLevel = this.dataComplianceLevel,
MaxMatches = this.dataMaxMatches,
};

1
app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor
View File

@ -38,7 +38,6 @@
</MudJustifiedText>
}
<TextInfoLines Label="@T("Your security policy")" MaxLines="3" Value="@this.DataSource.SecurityPolicy.ToInfoText()" Color="@this.DataSource.SecurityPolicy.GetColor()" ClipboardTooltipSubject="@T("your security policy")"/>
<TextInfoLine Label="@T("Compliance level")" Value="@this.DataSource.ComplianceLevel.GetName()" ClipboardTooltipSubject="@T("the compliance level")"/>
<TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
<TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("File size")" Value="@this.FileSize" ClipboardTooltipSubject="@T("the file size")"/>

2
app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs
View File

@ -42,7 +42,7 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
public int ChunkOverlapTokenLength { get; init; }
/// <inheritdoc />
public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.ALLOW_ANY;
/// <inheritdoc />
public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN;

2
app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs
View File

@ -42,7 +42,7 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
public int ChunkOverlapTokenLength { get; init; }
/// <inheritdoc />
public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.ALLOW_ANY;
/// <inheritdoc />
public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN;

92
app/MindWork AI Studio/Settings/DataSourceSecurityTrustExtensions.cs
View File

@ -1,4 +1,5 @@
using AIStudio.Provider;
using AIStudio.Settings.DataModel;
namespace AIStudio.Settings;
@ -36,6 +37,97 @@ public static class DataSourceSecurityTrustExtensions
return provider.Provider is LLMProviders.SELF_HOSTED || IsTrustedProviderId(provider.ConfiguredProviderId, settingsManager);
}
public static ConfidenceLevel GetConfidenceLevel(this Provider provider, SettingsManager settingsManager)
{
if (provider == Provider.NONE)
return ConfidenceLevel.NONE;
return provider.UsedLLMProvider.GetConfidence(settingsManager).Level;
}
public static ConfidenceLevel GetConfidenceLevel(this EmbeddingProvider provider, SettingsManager settingsManager)
{
if (provider == EmbeddingProvider.NONE)
return ConfidenceLevel.NONE;
return provider.UsedLLMProvider.GetConfidence(settingsManager).Level;
}
public static ConfidenceLevel GetConfidenceLevel(this IProvider provider, SettingsManager settingsManager)
{
if (provider is NoProvider)
return ConfidenceLevel.NONE;
return provider.Provider.GetConfidence(settingsManager).Level;
}
public static bool AllowsDataSourceAccess(this Provider provider, SettingsManager settingsManager, DataSourceSecurity dataSourceSecurity, ConfidenceLevel dataSourceComplianceLevel)
{
return provider.AllowsDataSourceSecurity(dataSourceSecurity, settingsManager)
&& provider.GetConfidenceLevel(settingsManager).AllowsDataSourceComplianceLevel(dataSourceComplianceLevel);
}
public static bool AllowsDataSourceAccess(this EmbeddingProvider provider, SettingsManager settingsManager, DataSourceSecurity dataSourceSecurity, ConfidenceLevel dataSourceComplianceLevel)
{
return provider.AllowsDataSourceSecurity(dataSourceSecurity, settingsManager)
&& provider.GetConfidenceLevel(settingsManager).AllowsDataSourceComplianceLevel(dataSourceComplianceLevel);
}
public static bool AllowsDataSourceAccess(this IProvider provider, SettingsManager settingsManager, DataSourceSecurity dataSourceSecurity, ConfidenceLevel dataSourceComplianceLevel)
{
return provider.AllowsDataSourceSecurity(dataSourceSecurity, settingsManager)
&& provider.GetConfidenceLevel(settingsManager).AllowsDataSourceComplianceLevel(dataSourceComplianceLevel);
}
public static bool AllowsDataSourceSecurity(this Provider provider, DataSourceSecurity dataSourceSecurity, SettingsManager settingsManager)
=> provider.IsTrustedForDataSourceSecurityChecks(settingsManager).AllowsDataSourceSecurity(dataSourceSecurity);
public static bool AllowsDataSourceSecurity(this EmbeddingProvider provider, DataSourceSecurity dataSourceSecurity, SettingsManager settingsManager)
=> provider.IsTrustedForDataSourceSecurityChecks(settingsManager).AllowsDataSourceSecurity(dataSourceSecurity);
public static bool AllowsDataSourceSecurity(this IProvider provider, DataSourceSecurity dataSourceSecurity, SettingsManager settingsManager)
=> provider.IsTrustedForDataSourceSecurityChecks(settingsManager).AllowsDataSourceSecurity(dataSourceSecurity);
public static bool AllowsDataSourceSecurity(this bool usingTrustedProvider, DataSourceSecurity dataSourceSecurity) => dataSourceSecurity switch
{
DataSourceSecurity.ALLOW_ANY => true,
DataSourceSecurity.SELF_HOSTED => usingTrustedProvider,
_ => false,
};
public static bool AllowsDataSourceComplianceLevel(this ConfidenceLevel providerConfidenceLevel, ConfidenceLevel dataSourceComplianceLevel)
{
if (dataSourceComplianceLevel is ConfidenceLevel.NONE)
return true;
return providerConfidenceLevel >= dataSourceComplianceLevel;
}
public static ConfidenceLevel GetRequiredComplianceLevel(this IEnumerable<IDataSource> dataSources)
{
var requiredComplianceLevel = ConfidenceLevel.NONE;
foreach (var dataSource in dataSources)
if (dataSource.ComplianceLevel > requiredComplianceLevel)
requiredComplianceLevel = dataSource.ComplianceLevel;
return requiredComplianceLevel;
}
public static DataSourceSecurity GetRequiredSecurityPolicy(this IEnumerable<IDataSource> dataSources)
{
var requiredSecurityPolicy = DataSourceSecurity.ALLOW_ANY;
foreach (var dataSource in dataSources)
{
if (dataSource.SecurityPolicy is DataSourceSecurity.NOT_SPECIFIED)
return DataSourceSecurity.NOT_SPECIFIED;
if (dataSource.SecurityPolicy is DataSourceSecurity.SELF_HOSTED)
requiredSecurityPolicy = DataSourceSecurity.SELF_HOSTED;
}
return requiredSecurityPolicy;
}
public static bool IsTrustedByConfiguration(this Provider provider, SettingsManager settingsManager) => IsTrustedProviderId(provider.Id, settingsManager);
public static bool IsTrustedByConfiguration(this EmbeddingProvider provider, SettingsManager settingsManager) => IsTrustedProviderId(provider.Id, settingsManager);

27
app/MindWork AI Studio/Tools/RAG/AugmentationProcesses/AugmentationOne.cs
View File

@ -43,19 +43,22 @@ public sealed class AugmentationOne : IAugmentationProcess
{
// Let's get the validation agent & set up its provider:
var validationAgent = Program.SERVICE_PROVIDER.GetService<AgentRetrievalContextValidation>()!;
validationAgent.SetLLMProvider(provider);
if (validationAgent.SetLLMProvider(provider, chatThread.DataSecurity, chatThread.DataComplianceLevel))
{
// Let's validate all retrieval contexts:
var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastUserPrompt, chatThread, retrievalContexts, token);
//
// Now, filter the retrieval contexts to the most relevant ones:
//
var targetWindow = validationResults.DetermineTargetWindow(TargetWindowStrategy.TOP10_BETTER_THAN_GUESSING);
var threshold = validationResults.GetConfidenceThreshold(targetWindow);
// Let's validate all retrieval contexts:
var validationResults = await validationAgent.ValidateRetrievalContextsAsync(lastUserPrompt, chatThread, retrievalContexts, token);
//
// Now, filter the retrieval contexts to the most relevant ones:
//
var targetWindow = validationResults.DetermineTargetWindow(TargetWindowStrategy.TOP10_BETTER_THAN_GUESSING);
var threshold = validationResults.GetConfidenceThreshold(targetWindow);
// Filter the retrieval contexts:
retrievalContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList();
// Filter the retrieval contexts:
retrievalContexts = validationResults.Where(x => x.RetrievalContext is not null && x.Confidence >= threshold).Select(x => x.RetrievalContext!).ToList();
}
else
LOGGER.LogWarning("Skipping retrieval context validation because no compliant validation agent provider is available.");
}
LOGGER.LogInformation($"Starting the augmentation process over {numTotalRetrievalContexts:###,###,###,###} retrieval contexts.");

14
app/MindWork AI Studio/Tools/RAG/DataSourceSelectionProcesses/AgenticSrcSelWithDynHeur.cs
View File

@ -31,11 +31,9 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
IReadOnlyList<IDataSource> selectedDataSources = [];
IReadOnlyList<DataSourceAgentSelected> finalAISelection = [];
// Get the settings manager:
var settings = Program.SERVICE_PROVIDER.GetService<SettingsManager>()!;
// Get the agent for the data source selection:
var selectionAgent = Program.SERVICE_PROVIDER.GetService<AgentDataSourceSelection>()!;
var allowedDataSources = dataSources.AllowedDataSources.ToDictionary(ds => ds.Id, StringComparer.Ordinal);
try
{
@ -61,14 +59,14 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
var totalAISelectedDataSources = aiSelectedDataSources.Count;
// Filter out the data sources that are not available:
aiSelectedDataSources = aiSelectedDataSources.Where(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id) is not null).ToList();
aiSelectedDataSources = aiSelectedDataSources.Where(x => allowedDataSources.ContainsKey(x.Id)).ToList();
// Store the real AI-selected data sources:
finalAISelection = aiSelectedDataSources.Select(x => new DataSourceAgentSelected { DataSource = settings.ConfigurationData.DataSources.First(ds => ds.Id == x.Id), AIDecision = x, Selected = false }).ToList();
finalAISelection = aiSelectedDataSources.Select(x => new DataSourceAgentSelected { DataSource = allowedDataSources[x.Id], AIDecision = x, Selected = false }).ToList();
var numHallucinatedSources = totalAISelectedDataSources - aiSelectedDataSources.Count;
if (numHallucinatedSources > 0)
LOGGER.LogWarning($"The AI hallucinated {numHallucinatedSources} data source(s). We ignore them.");
LOGGER.LogWarning($"The AI selected {numHallucinatedSources} unavailable data source(s). We ignore them.");
if (aiSelectedDataSources.Count > 3)
{
@ -87,7 +85,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
LOGGER.LogInformation($"The AI selected {aiSelectedDataSources.Count} data source(s) with a confidence of at least {threshold}.");
// Transform the final data sources to the actual data sources:
selectedDataSources = aiSelectedDataSources.Select(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id)).Where(ds => ds is not null).ToList()!;
selectedDataSources = aiSelectedDataSources.Select(x => allowedDataSources[x.Id]).ToList();
return new(proceedWithRAG, selectedDataSources);
}
@ -96,7 +94,7 @@ public class AgenticSrcSelWithDynHeur : IDataSourceSelectionProcess
//
// Transform the selected data sources to the actual data sources:
selectedDataSources = aiSelectedDataSources.Select(x => settings.ConfigurationData.DataSources.FirstOrDefault(ds => ds.Id == x.Id)).Where(ds => ds is not null).ToList()!;
selectedDataSources = aiSelectedDataSources.Select(x => allowedDataSources[x.Id]).ToList();
// Mark the data sources as selected:
foreach (var dataSource in finalAISelection)

12
app/MindWork AI Studio/Tools/RAG/RAGProcesses/AISrcSelWithRetCtxVal.cs
View File

@ -92,7 +92,7 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
//
// No, the user made the choice manually:
//
var selectedDataSourceInfo = selectedDataSources.Select(ds => ds.Name).Aggregate((a, b) => $"'{a}', '{b}'");
var selectedDataSourceInfo = string.Join(", ", selectedDataSources.Select(ds => $"'{ds.Name}'"));
LOGGER.LogInformation($"The user selected the data sources manually. {selectedDataSources.Count} data source(s) are selected: {selectedDataSourceInfo}.");
}
@ -104,12 +104,13 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
else
{
var previousDataSecurity = chatThread.DataSecurity;
var previousDataComplianceLevel = chatThread.DataComplianceLevel;
//
// Update the data security of the chat thread. We consider the current data security
// of the chat thread and the data security of the selected data sources:
//
var dataSecurityRestrictedToSelfHosted = selectedDataSources.Any(x => x.SecurityPolicy is DataSourceSecurity.SELF_HOSTED);
var dataSecurityRestrictedToSelfHosted = selectedDataSources.Any(x => x is not IInternalDataSource && x.SecurityPolicy is DataSourceSecurity.SELF_HOSTED);
chatThread.DataSecurity = dataSecurityRestrictedToSelfHosted switch
{
//
@ -150,6 +151,13 @@ public sealed class AISrcSelWithRetCtxVal : IRagProcess
if (previousDataSecurity != chatThread.DataSecurity)
LOGGER.LogInformation($"The data security of the chat thread was updated from '{previousDataSecurity}' to '{chatThread.DataSecurity}'.");
foreach (var dataSource in selectedDataSources)
if (dataSource.ComplianceLevel > chatThread.DataComplianceLevel)
chatThread.DataComplianceLevel = dataSource.ComplianceLevel;
if (previousDataComplianceLevel != chatThread.DataComplianceLevel)
LOGGER.LogInformation($"The data compliance level of the chat thread was updated from '{previousDataComplianceLevel.GetName()}' to '{chatThread.DataComplianceLevel.GetName()}'.");
}
//

19
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
View File

@ -434,7 +434,24 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
this.UpsertStatus(this.GetFallbackStatus(dataSource, "The selected embedding provider is not available."));
return;
}
if (!embeddingProvider.AllowsDataSourceAccess(settingsManager, dataSource.SecurityPolicy, dataSource.ComplianceLevel))
{
var errorMessage = $"The selected embedding provider is not allowed to embed this data source. The data source requires provider confidence '{dataSource.ComplianceLevel.GetName()}'. The embedding provider has confidence '{embeddingProvider.GetConfidenceLevel(settingsManager).GetName()}'.";
logger.LogWarning(
"Skipping background embeddings for data source '{DataSourceName}' ({DataSourceId}) because embedding provider '{EmbeddingProviderName}' ({EmbeddingProviderId}) is not allowed. RequiredDataSecurity={RequiredDataSecurity}, RequiredCompliance={RequiredCompliance}, EmbeddingProviderConfidence={EmbeddingProviderConfidence}.",
dataSource.Name,
dataSource.Id,
embeddingProvider.Name,
embeddingProvider.Id,
dataSource.SecurityPolicy,
dataSource.ComplianceLevel.GetName(),
embeddingProvider.GetConfidenceLevel(settingsManager).GetName());
token.ThrowIfCancellationRequested();
this.UpsertStatus(this.GetFallbackStatus(dataSource, errorMessage));
return;
}
logger.LogInformation(
"Using embedding provider '{EmbeddingProviderId}' with model '{EmbeddingModelId}' for data source '{DataSourceName}' ({DataSourceId}).",

27
app/MindWork AI Studio/Tools/Services/DataSourceService.cs
View File

@ -42,7 +42,10 @@ public sealed class DataSourceService
return new([], []);
}
return await this.GetDataSources(selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), previousSelectedDataSources);
return await this.GetDataSources(
selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager),
selectedLLMProvider.GetConfidenceLevel(this.settingsManager),
previousSelectedDataSources);
}
/// <summary>
@ -65,10 +68,13 @@ public sealed class DataSourceService
return new([], []);
}
return await this.GetDataSources(selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager), previousSelectedDataSources);
return await this.GetDataSources(
selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager),
selectedLLMProvider.GetConfidenceLevel(this.settingsManager),
previousSelectedDataSources);
}
private async Task<AllowedSelectedDataSources> GetDataSources(bool usingTrustedProvider, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
private async Task<AllowedSelectedDataSources> GetDataSources(bool usingTrustedProvider, ConfidenceLevel providerConfidenceLevel, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
{
var allDataSources = this.settingsManager.ConfigurationData.DataSources.ToList();
var previousSelectedDataSourceIds = previousSelectedDataSources?.Select(source => source.Id).ToHashSet(StringComparer.Ordinal) ?? [];
@ -78,7 +84,7 @@ public sealed class DataSourceService
// Start all checks in parallel:
foreach (var source in allDataSources)
tasks.Add(this.CheckOneDataSource(source, usingTrustedProvider));
tasks.Add(this.CheckOneDataSource(source, usingTrustedProvider, providerConfidenceLevel));
// Wait for all checks and collect the results:
foreach (var task in tasks)
@ -95,8 +101,17 @@ public sealed class DataSourceService
return new(filteredDataSources, filteredSelectedDataSources);
}
private async Task<IDataSource?> CheckOneDataSource(IDataSource source, bool usingTrustedProvider)
private async Task<IDataSource?> CheckOneDataSource(IDataSource source, bool usingTrustedProvider, ConfidenceLevel providerConfidenceLevel)
{
if (!providerConfidenceLevel.AllowsDataSourceComplianceLevel(source.ComplianceLevel))
{
this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) requires provider confidence '{source.ComplianceLevel.GetName()}'. The selected provider only has confidence '{providerConfidenceLevel.GetName()}'. We skip this source.");
return null;
}
if (source is IInternalDataSource)
return source;
//
// Unfortunately, we have to live-check any ERI source for its security requirements.
// Because the ERI server operator might change the security requirements at any time.
@ -206,4 +221,4 @@ public sealed class DataSourceService
return null;
}
}
}
}

47
app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs
View File

@ -1,3 +1,5 @@
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ERIClient.DataModel;
using AIStudio.Tools.PluginSystem;
@ -19,6 +21,14 @@ public sealed class DataSourceValidation
public Func<SecurityRequirements?> GetSecurityRequirements { get; init; } = () => null;
public Func<bool> GetSelectedCloudEmbedding { get; init; } = () => false;
public Func<EmbeddingProvider?> GetSelectedEmbeddingProvider { get; init; } = () => null;
public Func<DataSourceSecurity> GetSecurityPolicy { get; init; } = () => DataSourceSecurity.ALLOW_ANY;
public Func<ConfidenceLevel> GetComplianceLevel { get; init; } = () => ConfidenceLevel.NONE;
public Func<SettingsManager?> GetSettingsManager { get; init; } = () => null;
public Func<bool> GetTestedConnection { get; init; } = () => false;
@ -149,6 +159,20 @@ public sealed class DataSourceValidation
return null;
}
public string? ValidateEmbeddingProviderAccess(string embeddingId)
{
var embeddingIssue = this.ValidateEmbeddingId(embeddingId);
return embeddingIssue ?? this.ValidateSelectedEmbeddingProviderAccess();
}
public string? ValidateDataSourceComplianceLevel(ConfidenceLevel complianceLevel)
{
if(complianceLevel is ConfidenceLevel.NONE)
return TB("Please select a compliance level.");
return this.ValidateSelectedEmbeddingProviderAccess();
}
public string? ValidateUserAcknowledgedCloudEmbedding(bool value)
{
if(this.GetSelectedCloudEmbedding() && !value)
@ -175,4 +199,25 @@ public sealed class DataSourceValidation
return null;
}
}
private string? ValidateSelectedEmbeddingProviderAccess()
{
var selectedEmbedding = this.GetSelectedEmbeddingProvider();
var settingsManager = this.GetSettingsManager();
if(selectedEmbedding is null || settingsManager is null)
return null;
var dataSecurity = this.GetSecurityPolicy();
var complianceLevel = this.GetComplianceLevel();
if(selectedEmbedding.AllowsDataSourceAccess(settingsManager, dataSecurity, complianceLevel))
return null;
if(!selectedEmbedding.AllowsDataSourceSecurity(dataSecurity, settingsManager))
return TB("The selected embedding provider is not allowed to process this data source due to its data security policy. Select a self-hosted or organization-trusted embedding provider.");
return string.Format(
TB("The selected embedding provider has confidence '{0}', but this data source requires provider confidence '{1}'. Select an embedding provider with equal or higher confidence or lower the compliance level."),
selectedEmbedding.GetConfidenceLevel(settingsManager).GetName(),
complianceLevel.GetName());
}
}