Renamed EDI to ERI

This commit is contained in:
tsommer committed 2024-12-14 19:48:47 +01:00
1 parent 9bc6d04b22
commit 3f5ebc53c3
21 files changed
+99 -98

No files matched your search

@@ -0,0 +1,118 @@
@attribute [Route(Routes.ASSISTANT_ERI)]
@using MudExtensions
@inherits AssistantBaseCore
<MudText Typo="Typo.body1" Class="mb-3">
You can imagine it like this: Hypothetically, when Wikipedia implemented the ERI, it would vectorize
all pages using an embedding method. All of Wikipedia’s data would remain with Wikipedia, including the
vector database (decentralized approach). Then, any AI Studio user could add Wikipedia as a data source to
significantly reduce the hallucination of the LLM in knowledge questions.
</MudText>
<MudText Typo="Typo.body1">
<b>Related links:</b>
</MudText>
<MudList T="string" Class="mb-6">
<MudListItem T="string" Icon="@Icons.Material.Filled.Link" Target="_blank" Href="https://github.com/MindWorkAI/ERI">ERI repository with example implementation in .NET and C#</MudListItem>
<MudListItem T="string" Icon="@Icons.Material.Filled.Link" Target="_blank" Href="https://mindworkai.org/swagger-ui.html">Interactive documentation aka Swagger UI</MudListItem>
</MudList>
<PreviewPrototype/>
<div class="mb-6"></div>
<MudTextField T="string" @bind-Text="@this.serverName" Validation="@this.ValidateServerName" Immediate="@true" Label="ERI server name" HelperText="Please give your ERI server a name that provides information about the data source and/or its intended purpose. The name will be displayed to users in AI Studio." Counter="60" MaxLength="60" Variant="Variant.Outlined" Margin="Margin.Normal" UserAttributes="@USER_INPUT_ATTRIBUTES" Class="mb-3"/>
<MudTextField T="string" @bind-Text="@this.serverDescription" Validation="@this.ValidateServerDescription" Immediate="@true" Label="ERI server description" HelperText="Please provide a brief description of your ERI server. Describe or explain what your ERI server does and what data it uses for this purpose. This description will be shown to users in AI Studio." Counter="512" MaxLength="512" Variant="Variant.Outlined" Margin="Margin.Normal" Lines="3" AutoGrow="@true" MaxLines="6" UserAttributes="@USER_INPUT_ATTRIBUTES" Class="mb-3"/>
<MudStack Row="@true" AlignItems="AlignItems.Center" Class="mb-3">
<MudSelect T="ERIVersion" @bind-Value="@this.selectedERIVersion" Label="ERI specification version" Variant="Variant.Outlined" Margin="Margin.Dense" Validation="@this.ValidateERIVersion">
@foreach (var version in Enum.GetValues<ERIVersion>())
{
<MudSelectItem Value="@version">@version</MudSelectItem>
}
</MudSelect>
<MudButton Variant="Variant.Outlined" Size="Size.Small" Disabled="@(!this.selectedERIVersion.WasSpecificationSelected())" Href="@this.selectedERIVersion.SpecificationURL()" Target="_blank">
<MudIcon Icon="@Icons.Material.Filled.Link" Class="mr-2"/> Download specification
</MudButton>
</MudStack>
<MudStack Row="@true" Class="mb-3">
<MudSelect T="ProgrammingLanguages" @bind-Value="@this.selectedProgrammingLanguage" AdornmentIcon="@Icons.Material.Filled.Code" Adornment="Adornment.Start" Label="Programming language" Variant="Variant.Outlined" Margin="Margin.Dense" Validation="@this.ValidateProgrammingLanguage">
@foreach (var language in Enum.GetValues<ProgrammingLanguages>())
{
<MudSelectItem Value="@language">@language.Name()</MudSelectItem>
}
</MudSelect>
@if (this.selectedProgrammingLanguage is ProgrammingLanguages.OTHER)
{
<MudTextField T="string" @bind-Text="@this.otherProgrammingLanguage" Validation="@this.ValidateOtherLanguage" Label="Other language" Variant="Variant.Outlined" Margin="Margin.Dense" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
}
</MudStack>
<MudStack Row="@false" Spacing="1" Class="mb-3">
<MudSelect T="DataSources" @bind-Value="@this.selectedDataSource" AdornmentIcon="@Icons.Material.Filled.Dataset" Adornment="Adornment.Start" Label="Data source" Variant="Variant.Outlined" Margin="Margin.Dense" Validation="@this.ValidateDataSource" SelectedValuesChanged="@this.DataSourceWasChanged">
@foreach (var dataSource in Enum.GetValues<DataSources>())
{
<MudSelectItem Value="@dataSource">@dataSource.Name()</MudSelectItem>
}
</MudSelect>
@if (this.selectedDataSource is DataSources.CUSTOM)
{
<MudTextField T="string" @bind-Text="@this.otherDataSource" Validation="@this.ValidateOtherDataSource" Label="Describe your data source" Variant="Variant.Outlined" Margin="Margin.Normal" Lines="3" AutoGrow="@true" MaxLines="6" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
}
</MudStack>
@if(this.selectedDataSource > DataSources.FILE_SYSTEM)
{
<MudTextField T="string" @bind-Text="@this.dataSourceProductName" Label="Data source: product name" Validation="@this.ValidateDataSourceProductName" Variant="Variant.Outlined" Margin="Margin.Dense" UserAttributes="@USER_INPUT_ATTRIBUTES" Class="mb-3"/>
}
@if (this.NeedHostnamePort())
{
<div class="mb-3">
<MudStack Row="@true">
<MudTextField T="string" @bind-Text="@this.dataSourceHostname" Label="Data source: hostname" Validation="@this.ValidateHostname" Variant="Variant.Outlined" Margin="Margin.Dense" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
<MudNumericField Label="Data source: port" Immediate="@true" Min="1" Max="65535" Validation="@this.ValidatePort" @bind-Value="@this.dataSourcePort" Variant="Variant.Outlined" Margin="Margin.Dense"/>
</MudStack>
@if (this.dataSourcePort < 1024)
{
<MudText Typo="Typo.body2">
<b>Warning:</b> Ports below 1024 are reserved for system services. Your ERI server need to run with elevated permissions (root user).
</MudText>
}
</div>
}
<MudStack Row="@false" Spacing="1" Class="mb-1">
<MudSelectExtended
T="Auth"
ShrinkLabel="@true"
MultiSelection="@true"
MultiSelectionTextFunc="@this.GetMultiSelectionAuthText"
SelectedValues="@this.selectedAuthenticationMethods"
Validation="@this.ValidateAuthenticationMethods"
SelectedValuesChanged="@this.AuthenticationMethodWasChanged"
Label="Authentication method(s)"
Variant="Variant.Outlined"
Margin="Margin.Dense">
@foreach (var authMethod in Enum.GetValues<Auth>())
{
<MudSelectItemExtended Value="@authMethod">@authMethod.Name()</MudSelectItemExtended>
}
</MudSelectExtended>
<MudTextField T="string" @bind-Text="@this.authDescription" Label="@this.AuthDescriptionTitle()" Validation="@this.ValidateAuthDescription" Variant="Variant.Outlined" Margin="Margin.Normal" Lines="3" AutoGrow="@true" MaxLines="6" UserAttributes="@USER_INPUT_ATTRIBUTES"/>
</MudStack>
@if (this.selectedAuthenticationMethods.Contains(Auth.KERBEROS))
{
<MudSelect T="OperatingSystem" @bind-Value="@this.selectedOperatingSystem" Label="Operating system on which your ERI will run" Variant="Variant.Outlined" Margin="Margin.Dense" Validation="@this.ValidateOperatingSystem" Class="mb-1">
@foreach (var os in Enum.GetValues<OperatingSystem>())
{
<MudSelectItem Value="@os">@os.Name()</MudSelectItem>
}
</MudSelect>
}
<MudTextField T="string" @bind-Text="@this.retrievalDescription" Validation="@this.ValidateRetrievalDescription" Label="Describe your data retrieval process" Variant="Variant.Outlined" Margin="Margin.Normal" Lines="6" AutoGrow="@true" MaxLines="12" UserAttributes="@USER_INPUT_ATTRIBUTES" Class="mb-3"/>
<MudTextField T="string" @bind-Text="@this.additionalLibraries" Label="(Optional) Additional libraries" HelperText="Do you want to include additional libraries? Then name them and briefly describe what you want to achieve with them." Variant="Variant.Outlined" Margin="Margin.Normal" Lines="3" AutoGrow="@true" MaxLines="12" UserAttributes="@USER_INPUT_ATTRIBUTES" Class="mb-3"/>
<ProviderSelection @bind-ProviderSettings="@this.providerSettings" ValidateProvider="@this.ValidatingProvider"/>
@@ -0,0 +1,361 @@
using AIStudio.Chat;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Assistants.ERI;
public partial class AssistantERI : AssistantBaseCore
{
[Inject]
private HttpClient HttpClient { get; set; } = null!;
public override Tools.Components Component => Tools.Components.ERI_ASSISTANT;
protected override string Title => "ERI Server";
protected override string Description =>
"""
The ERI is the External Retrieval Interface for AI Studio and other tools. The ERI acts as a contract
between decentralized data sources and, e.g., AI Studio. The ERI is implemented by the data sources,
allowing them to be integrated into AI Studio later. This means that the data sources assume the server
role and AI Studio (or any other LLM tool) assumes the client role of the API. This approach serves to
realize a Retrieval-Augmented Generation (RAG) process with external data.
""";
protected override string SystemPrompt =>
$"""
""";
protected override IReadOnlyList<IButtonData> FooterButtons => [];
protected override string SubmitText => "Create the ERI server";
protected override Func<Task> SubmitAction => this.GenerateServer;
protected override ChatThread ConvertToChatThread => (this.chatThread ?? new()) with
{
SystemPrompt = SystemPrompts.DEFAULT,
};
protected override void ResetFrom()
{
if (!this.MightPreselectValues())
{
this.serverName = string.Empty;
this.serverDescription = string.Empty;
this.selectedERIVersion = ERIVersion.V1;
this.selectedProgrammingLanguage = ProgrammingLanguages.NONE;
this.otherProgrammingLanguage = string.Empty;
this.selectedDataSource = DataSources.NONE;
this.dataSourceProductName = string.Empty;
this.otherDataSource = string.Empty;
this.dataSourceHostname = string.Empty;
this.dataSourcePort = null;
this.selectedAuthenticationMethods = [];
this.authDescription = string.Empty;
this.selectedOperatingSystem = OperatingSystem.NONE;
this.retrievalDescription = string.Empty;
this.additionalLibraries = string.Empty;
}
}
protected override bool MightPreselectValues()
{
if (this.SettingsManager.ConfigurationData.ERI.PreselectOptions)
{
this.serverName = this.SettingsManager.ConfigurationData.ERI.PreselectedServerName;
this.serverDescription = this.SettingsManager.ConfigurationData.ERI.PreselectedServerDescription;
this.selectedERIVersion = this.SettingsManager.ConfigurationData.ERI.PreselectedERIVersion;
this.selectedProgrammingLanguage = this.SettingsManager.ConfigurationData.ERI.PreselectedProgrammingLanguage;
this.otherProgrammingLanguage = this.SettingsManager.ConfigurationData.ERI.PreselectedOtherProgrammingLanguage;
this.selectedDataSource = this.SettingsManager.ConfigurationData.ERI.PreselectedDataSource;
this.dataSourceProductName = this.SettingsManager.ConfigurationData.ERI.PreselectedDataSourceProductName;
this.otherDataSource = this.SettingsManager.ConfigurationData.ERI.PreselectedOtherDataSource;
this.dataSourceHostname = this.SettingsManager.ConfigurationData.ERI.PreselectedDataSourceHostname;
this.dataSourcePort = this.SettingsManager.ConfigurationData.ERI.PreselectedDataSourcePort;
var authMethods = new HashSet<Auth>(this.SettingsManager.ConfigurationData.ERI.PreselectedAuthMethods);
this.selectedAuthenticationMethods = authMethods;
this.authDescription = this.SettingsManager.ConfigurationData.ERI.PreselectedAuthDescription;
this.selectedOperatingSystem = this.SettingsManager.ConfigurationData.ERI.PreselectedOperatingSystem;
this.retrievalDescription = this.SettingsManager.ConfigurationData.ERI.PreselectedRetrievalDescription;
this.additionalLibraries = this.SettingsManager.ConfigurationData.ERI.PreselectedAdditionalLibraries;
return true;
}
return false;
}
private string serverName = string.Empty;
private string serverDescription = string.Empty;
private ERIVersion selectedERIVersion = ERIVersion.V1;
private ProgrammingLanguages selectedProgrammingLanguage = ProgrammingLanguages.NONE;
private string otherProgrammingLanguage = string.Empty;
private DataSources selectedDataSource = DataSources.NONE;
private string otherDataSource = string.Empty;
private string dataSourceProductName = string.Empty;
private string dataSourceHostname = string.Empty;
private int? dataSourcePort;
private IEnumerable<Auth> selectedAuthenticationMethods = new HashSet<Auth>();
private string authDescription = string.Empty;
private OperatingSystem selectedOperatingSystem = OperatingSystem.NONE;
private string retrievalDescription = string.Empty;
private string additionalLibraries = string.Empty;
private string? ValidateServerName(string name)
{
if(string.IsNullOrWhiteSpace(name))
return "Please provide a name for your ERI server. This name will be used to identify the server in AI Studio.";
if(name.Length is > 60 or < 6)
return "The name of your ERI server must be between 6 and 60 characters long.";
return null;
}
private string? ValidateServerDescription(string description)
{
if(string.IsNullOrWhiteSpace(description))
return "Please provide a description for your ERI server. What data will the server retrieve? This description will be used to inform users about the purpose of your ERI server.";
if(description.Length is < 32 or > 512)
return "The description of your ERI server must be between 32 and 512 characters long.";
return null;
}
private string? ValidateERIVersion(ERIVersion version)
{
if (version == ERIVersion.NONE)
return "Please select an ERI specification version for the ERI server.";
return null;
}
private string? ValidateProgrammingLanguage(ProgrammingLanguages language)
{
if (language == ProgrammingLanguages.OTHER)
return null;
if (language == ProgrammingLanguages.NONE)
return "Please select a programming language for the ERI server.";
return null;
}
private string? ValidateOtherLanguage(string language)
{
if(this.selectedProgrammingLanguage != ProgrammingLanguages.OTHER)
return null;
if(string.IsNullOrWhiteSpace(language))
return "Please specify the custom programming language for the ERI server.";
return null;
}
private string? ValidateDataSource(DataSources dataSource)
{
if (dataSource == DataSources.CUSTOM)
return null;
if (dataSource == DataSources.NONE)
return "Please select a data source for the ERI server.";
return null;
}
private string? ValidateDataSourceProductName(string productName)
{
if(this.selectedDataSource is DataSources.CUSTOM or DataSources.NONE or DataSources.FILE_SYSTEM)
return null;
if(string.IsNullOrWhiteSpace(productName))
return "Please specify the product name of the data source, e.g., 'MongoDB', 'Redis', 'PostgreSQL', 'Neo4j', or 'MinIO', etc.";
return null;
}
private string? ValidateOtherDataSource(string dataSource)
{
if(this.selectedDataSource != DataSources.CUSTOM)
return null;
if(string.IsNullOrWhiteSpace(dataSource))
return "Please describe the data source of your ERI server.";
return null;
}
private string? ValidateHostname(string hostname)
{
if(!this.NeedHostnamePort())
return null;
// When using a custom data source, the hostname is optional:
if(this.selectedDataSource is DataSources.CUSTOM)
return null;
if(string.IsNullOrWhiteSpace(hostname))
return "Please provide the hostname of the data source. Use 'localhost' if the data source is on the same machine as the ERI server.";
if(hostname.Length > 255)
return "The hostname of the data source must not exceed 255 characters.";
return null;
}
private string? ValidatePort(int? port)
{
if(!this.NeedHostnamePort())
return null;
// When using a custom data source, the port is optional:
if(this.selectedDataSource is DataSources.CUSTOM)
return null;
if(port is null)
return "Please provide the port of the data source.";
if(port is < 1 or > 65535)
return "The port of the data source must be between 1 and 65535.";
return null;
}
private void DataSourceWasChanged()
{
if(this.SettingsManager.ConfigurationData.ERI.PreselectedDataSourcePort is not null)
return;
//
// Preselect the default port for the selected data source
//
this.dataSourcePort = this.selectedDataSource switch
{
DataSources.DOCUMENT_STORE => 27017,
DataSources.KEY_VALUE_STORE => 6379,
DataSources.OBJECT_STORAGE => 9000,
DataSources.RELATIONAL_DATABASE => 5432,
DataSources.GRAPH_DATABASE => 7687,
_ => null
};
}
private string? ValidateAuthenticationMethods(Auth _)
{
var authenticationMethods = (this.selectedAuthenticationMethods as HashSet<Auth>)!;
if(authenticationMethods.Count == 0)
return "Please select at least one authentication method for the ERI server.";
return null;
}
private void AuthenticationMethodWasChanged(IEnumerable<Auth>? selectedValues)
{
if(selectedValues is null)
{
this.selectedAuthenticationMethods = [];
this.selectedOperatingSystem = OperatingSystem.NONE;
return;
}
this.selectedAuthenticationMethods = selectedValues;
if(!this.IsUsingKerberos())
this.selectedOperatingSystem = OperatingSystem.NONE;
}
private bool IsUsingKerberos()
{
return this.selectedAuthenticationMethods.Contains(Auth.KERBEROS);
}
private string? ValidateOperatingSystem(OperatingSystem os)
{
if(!this.IsUsingKerberos())
return null;
if(os is OperatingSystem.NONE)
return "Please select the operating system on which the ERI server will run. This is necessary when using SSO with Kerberos.";
return null;
}
private string AuthDescriptionTitle()
{
const string TITLE = "Describe how you planned the authentication process";
return this.IsAuthDescriptionOptional() ? $"(Optional) {TITLE}" : TITLE;
}
private bool IsAuthDescriptionOptional()
{
var authenticationMethods = (this.selectedAuthenticationMethods as HashSet<Auth>)!;
if(authenticationMethods.Count > 1)
return false;
if (authenticationMethods.Any(n => n == Auth.NONE) && authenticationMethods.Count > 1)
return false;
return true;
}
private string? ValidateAuthDescription(string description)
{
var authenticationMethods = (this.selectedAuthenticationMethods as HashSet<Auth>)!;
if(authenticationMethods.Any(n => n == Auth.NONE) && authenticationMethods.Count > 1 && string.IsNullOrWhiteSpace(this.authDescription))
return "Please describe how the selected authentication methods should be used. Especially, explain for what data the NONE method (public access) is used.";
if(authenticationMethods.Count > 1 && string.IsNullOrWhiteSpace(this.authDescription))
return "Please describe how the selected authentication methods should be used.";
return null;
}
private string GetMultiSelectionAuthText(List<Auth> selectedValues)
{
if(selectedValues.Count == 0)
return "Please select at least one authentication method";
if(selectedValues.Count == 1)
return $"You have selected 1 authentication method";
return $"You have selected {selectedValues.Count} authentication methods";
}
private string? ValidateRetrievalDescription(string description)
{
if(string.IsNullOrWhiteSpace(description))
return "Please describe how the data retrieval process should work. This is important for the integration of the data source into AI Studio by means of the ERI.";
return null;
}
private bool NeedHostnamePort()
{
switch (this.selectedDataSource)
{
case DataSources.NONE:
case DataSources.FILE_SYSTEM:
return false;
default:
return true;
}
}
private async Task GenerateServer()
{
await this.form!.Validate();
if (!this.inputIsValid)
return;
var ediSpecification = await this.selectedERIVersion.ReadSpecification(this.HttpClient);
if (string.IsNullOrWhiteSpace(ediSpecification))
{
// TODO: Show an error message
}
}
}
@@ -0,0 +1,9 @@
namespace AIStudio.Assistants.ERI;
public enum Auth
{
NONE,
KERBEROS,
USERNAME_PASSWORD,
TOKEN,
}
@@ -0,0 +1,15 @@
namespace AIStudio.Assistants.ERI;
public static class AuthExtensions
{
public static string Name(this Auth auth) => auth switch
{
Auth.NONE => "No login necessary: useful for public data sources",
Auth.KERBEROS => "Login by single-sign-on (SSO) using Kerberos: very complex to implement and to operate, useful for many users",
Auth.USERNAME_PASSWORD => "Login by username and password: simple to implement and to operate, useful for few users; easy to use for users",
Auth.TOKEN => "Login by token: simple to implement and to operate, useful for few users; unusual for many users",
_ => "Unknown login method"
};
}
@@ -0,0 +1,15 @@
namespace AIStudio.Assistants.ERI;
public enum DataSources
{
NONE,
CUSTOM,
FILE_SYSTEM,
OBJECT_STORAGE,
KEY_VALUE_STORE,
DOCUMENT_STORE,
RELATIONAL_DATABASE,
GRAPH_DATABASE,
}
@@ -0,0 +1,19 @@
namespace AIStudio.Assistants.ERI;
public static class DataSourcesExtensions
{
public static string Name(this DataSources dataSource) => dataSource switch
{
DataSources.NONE => "No data source selected",
DataSources.CUSTOM => "Custom description",
DataSources.FILE_SYSTEM => "File system (local or network share)",
DataSources.OBJECT_STORAGE => "Object storage, like Amazon S3, MinIO, etc.",
DataSources.KEY_VALUE_STORE => "Key-Value store, like Redis, etc.",
DataSources.DOCUMENT_STORE => "Document store, like MongoDB, etc.",
DataSources.RELATIONAL_DATABASE => "Relational database, like MySQL, PostgreSQL, etc.",
DataSources.GRAPH_DATABASE => "Graph database, like Neo4j, ArangoDB, etc.",
_ => "Unknown data source"
};
}
@@ -0,0 +1,8 @@
namespace AIStudio.Assistants.ERI;
public enum ERIVersion
{
NONE,
V1,
}
@@ -0,0 +1,27 @@
namespace AIStudio.Assistants.ERI;
public static class ERIVersionExtensions
{
public static async Task<string> ReadSpecification(this ERIVersion version, HttpClient httpClient)
{
try
{
var url = version.SpecificationURL();
var response = await httpClient.GetAsync(url);
return await response.Content.ReadAsStringAsync();
}
catch
{
return string.Empty;
}
}
public static string SpecificationURL(this ERIVersion version)
{
var nameLower = version.ToString().ToLowerInvariant();
var filename = $"{nameLower}.json";
return $"specs/eri/{filename}";
}
public static bool WasSpecificationSelected(this ERIVersion version) => version != ERIVersion.NONE;
}
@@ -0,0 +1,9 @@
namespace AIStudio.Assistants.ERI;
public enum OperatingSystem
{
NONE,
WINDOWS,
LINUX,
}
@@ -0,0 +1,14 @@
namespace AIStudio.Assistants.ERI;
public static class OperatingSystemExtensions
{
public static string Name(this OperatingSystem os) => os switch
{
OperatingSystem.NONE => "No operating system specified",
OperatingSystem.WINDOWS => "Windows",
OperatingSystem.LINUX => "Linux",
_ => "Unknown operating system"
};
}
@@ -0,0 +1,20 @@
namespace AIStudio.Assistants.ERI;
public enum ProgrammingLanguages
{
NONE,
C,
CPP,
CSHARP,
GO,
JAVA,
JAVASCRIPT,
JULIA,
MATLAB,
PHP,
PYTHON,
RUST,
OTHER,
}
@@ -0,0 +1,24 @@
namespace AIStudio.Assistants.ERI;
public static class ProgrammingLanguagesExtensions
{
public static string Name(this ProgrammingLanguages language) => language switch
{
ProgrammingLanguages.NONE => "No programming language selected",
ProgrammingLanguages.C => "C",
ProgrammingLanguages.CPP => "C++",
ProgrammingLanguages.CSHARP => "C#",
ProgrammingLanguages.GO => "Go",
ProgrammingLanguages.JAVA => "Java",
ProgrammingLanguages.JAVASCRIPT => "JavaScript",
ProgrammingLanguages.JULIA => "Julia",
ProgrammingLanguages.MATLAB => "MATLAB",
ProgrammingLanguages.PHP => "PHP",
ProgrammingLanguages.PYTHON => "Python",
ProgrammingLanguages.RUST => "Rust",
ProgrammingLanguages.OTHER => "Other",
_ => "Unknown"
};
}