MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2025-04-28 08:19:47 +00:00
Code Issues Actions 2 Packages Projects Releases Wiki Activity

Added a data source service

Browse Source
This commit is contained in:
Thorsten Sommer 2025-02-13 14:30:24 +01:00
parent 03ac9af273
commit e78e74fbc3
Signed by: tsommer
GPG Key ID: 371BBA77A02C0108
3 changed files with 175 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/MindWork AI Studio/Program.cs
View File

@ -115,6 +115,7 @@ internal sealed class Program
builder.Services.AddMudMarkdownClipboardService<MarkdownClipboardService>();
builder.Services.AddSingleton<SettingsManager>();
builder.Services.AddSingleton<ThreadSafeRandom>();
builder.Services.AddSingleton<DataSourceService>();
builder.Services.AddTransient<HTMLParser>();
builder.Services.AddTransient<AgentTextContentCleaner>();
builder.Services.AddHostedService<UpdateService>();

13
app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs Normal file
View File

@ -0,0 +1,13 @@
using AIStudio.Settings;
namespace AIStudio.Tools;
/// <summary>
/// Contains both the allowed and selected data sources.
/// </summary>
/// <remarks>
/// The selected data sources are a subset of the allowed data sources.
/// </remarks>
/// <param name="AllowedDataSources">The allowed data sources.</param>
/// <param name="SelectedDataSources">The selected data sources, which are a subset of the allowed data sources.</param>
public readonly record struct AllowedSelectedDataSources(IReadOnlyList<IDataSource> AllowedDataSources, IReadOnlyList<IDataSource> SelectedDataSources);

161
app/MindWork AI Studio/Tools/Services/DataSourceService.cs Normal file
View File

@ -0,0 +1,161 @@
using AIStudio.Assistants.ERI;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ERIClient;
using AIStudio.Tools.ERIClient.DataModel;
namespace AIStudio.Tools.Services;
public sealed class DataSourceService
{
private readonly RustService rustService;
private readonly SettingsManager settingsManager;
private readonly ILogger<DataSourceService> logger;
public DataSourceService(SettingsManager settingsManager, ILogger<DataSourceService> logger, RustService rustService)
{
this.logger = logger;
this.rustService = rustService;
this.settingsManager = settingsManager;
this.logger.LogInformation("The data source service has been initialized.");
}
public async Task<AllowedSelectedDataSources> GetDataSources(AIStudio.Settings.Provider selectedLLMProvider, IReadOnlyCollection<IDataSource>? previousSelectedDataSources = null)
{
var allDataSources = this.settingsManager.ConfigurationData.DataSources;
var filteredDataSources = new List<IDataSource>(allDataSources.Count);
var filteredSelectedDataSources = new List<IDataSource>(previousSelectedDataSources?.Count ?? 0);
var tasks = new List<Task<IDataSource?>>(allDataSources.Count);
// Start all checks in parallel:
foreach (var source in allDataSources)
tasks.Add(this.CheckOneDataSource(source, selectedLLMProvider));
// Wait for all checks and collect the results:
foreach (var task in tasks)
{
var source = await task;
if (source is not null)
{
filteredDataSources.Add(source);
if (previousSelectedDataSources is not null && previousSelectedDataSources.Contains(source))
filteredSelectedDataSources.Add(source);
}
}
return new(filteredDataSources, filteredSelectedDataSources);
}
private async Task<IDataSource?> CheckOneDataSource(IDataSource source, AIStudio.Settings.Provider selectedLLMProvider)
{
//
// Unfortunately, we have to live-check any ERI source for its security requirements.
// Because the ERI server operator might change the security requirements at any time.
//
SecurityRequirements? eriSourceRequirements = null;
if (source is DataSourceERI_V1 eriSource)
{
using var cancellationTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(6));
using var client = ERIClientFactory.Get(ERIVersion.V1, eriSource);
if(client is null)
{
this.logger.LogError($"Could not create ERI client for source '{source.Name}' (id={source.Id}). We skip this source.");
return null;
}
this.logger.LogInformation($"Authenticating with ERI source '{source.Name}' (id={source.Id})...");
var loginResult = await client.AuthenticateAsync(eriSource, this.rustService, cancellationTokenSource.Token);
if (!loginResult.Successful)
{
this.logger.LogWarning($"Authentication with ERI source '{source.Name}' (id={source.Id}) failed. We skip this source. Reason: {loginResult.Message}");
return null;
}
this.logger.LogInformation($"Checking security requirements for ERI source '{source.Name}' (id={source.Id})...");
var securityRequest = await client.GetSecurityRequirementsAsync(cancellationTokenSource.Token);
if (!securityRequest.Successful)
{
this.logger.LogWarning($"Could not retrieve security requirements for ERI source '{source.Name}' (id={source.Id}). We skip this source. Reason: {loginResult.Message}");
return null;
}
eriSourceRequirements = securityRequest.Data;
this.logger.LogInformation($"Security requirements for ERI source '{source.Name}' (id={source.Id}) retrieved successfully.");
}
switch (source.SecurityPolicy)
{
case DataSourceSecurity.ALLOW_ANY:
//
// Case: The data source allows any provider type. We want to use a self-hosted provider.
// There is no issue with this source. Accept it.
//
if(selectedLLMProvider.IsSelfHosted)
return source;
//
// Case: This is a local data source. When the source allows any provider type, we can use it.
// Accept it.
//
if(eriSourceRequirements is null)
return source;
//
// Case: The ERI source requires a self-hosted provider. This misconfiguration happens
// when the ERI server operator changes the security requirements. The ERI server
// operator owns the data -- we have to respect their rules. We skip this source.
//
if (eriSourceRequirements is { AllowedProviderType: ProviderType.SELF_HOSTED })
{
this.logger.LogWarning($"The ERI source '{source.Name}' (id={source.Id}) requires a self-hosted provider. We skip this source.");
return null;
}
//
// Case: The ERI source allows any provider type. The data source configuration is correct.
// Accept it.
//
if(eriSourceRequirements is { AllowedProviderType: ProviderType.ANY })
return source;
//
// Case: Missing rules. We skip this source. Better safe than sorry.
//
this.logger.LogDebug($"The ERI source '{source.Name}' (id={source.Id}) was filtered out due to missing rules.");
return null;
//
// Case: The data source requires a self-hosted provider. We want to use a self-hosted provider.
// There is no issue with this source. Accept it.
//
case DataSourceSecurity.SELF_HOSTED when selectedLLMProvider.IsSelfHosted:
return source;
//
// Case: The data source requires a self-hosted provider. We want to use a cloud provider.
// We skip this source.
//
case DataSourceSecurity.SELF_HOSTED when !selectedLLMProvider.IsSelfHosted:
this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) requires a self-hosted provider. We skip this source.");
return null;
//
// Case: The data source did not specify a security policy. We skip this source.
// Better safe than sorry.
//
case DataSourceSecurity.NOT_SPECIFIED:
this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) has no security policy. We skip this source.");
return null;
//
// Case: Some developer forgot to implement a security policy. We skip this source.
// Better safe than sorry.
//
default:
this.logger.LogWarning($"The data source '{source.Name}' (id={source.Id}) was filtered out due unknown security policy.");
return null;
}
}
}