MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-11-02 09:07:21 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixed the Agenda Planner and ERI Server assistants losing parts of your input (#1142)

Browse Source
This commit is contained in:
Thorsten Sommer authored and GitHub committed 2026-10-31 15:24:33 +01:00
1 parent 8a4443ab4b
commit 652d3a509f
19 files changed
+1159 -337
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Assistants/Agenda/AssistantAgenda.razor.cs
+1 -1
View File
@@ -480,7 +480,7 @@ public partial class AssistantAgenda : AssistantBaseCore<SettingsDialogAgenda>
private string PromptSocialActivity()
{
if (this.doingSocialActivity)
return "- The participants will engage in a social activity after the meeting or seminar. This can be a team-building exercise, a sightseeing tour, or a visit to a local attraction. Please make a recommendation for an activity at '{this.inputLocation}'.";
return $"- The participants will engage in a social activity after the meeting or seminar. This can be a team-building exercise, a sightseeing tour, or a visit to a local attraction. Please make a recommendation for an activity at '{this.inputLocation}'.";
return "- The participants will not engage in a social activity after the meeting or seminar.";
}
app/MindWork AI Studio/Assistants/ERI/AssistantERI.razor.cs
+33 -336
View File
@@ -1,6 +1,3 @@
using System.Text;
using System.Text.RegularExpressions;
using AIStudio.Chat;
using AIStudio.Dialogs;
using AIStudio.Dialogs.Settings;
@@ -27,270 +24,25 @@ public partial class AssistantERI : AssistantBaseCore<SettingsDialogERIServer>
protected override string Description => T("The ERI is the External Retrieval Interface for AI Studio and other tools. The ERI acts as a contract between decentralized data sources and, e.g., AI Studio. The ERI is implemented by the data sources, allowing them to be integrated into AI Studio later. This means that the data sources assume the server role and AI Studio (or any other LLM tool) assumes the client role of the API. This approach serves to realize a Retrieval-Augmented Generation (RAG) process with external data.");
protected override string SystemPrompt
protected override string SystemPrompt => new ERIServerPrompt
{
get
{
var sb = new StringBuilder();
//
// ---------------------------------
// Introduction
// ---------------------------------
//
var programmingLanguage = this.selectedProgrammingLanguage is ProgrammingLanguages.OTHER ? this.otherProgrammingLanguage : this.selectedProgrammingLanguage.ToPrompt();
sb.Append($"""
# Introduction
You are an experienced {programmingLanguage} developer. Your task is to implement an API server in
the {programmingLanguage} language according to the following OpenAPI Description (OAD):
```
{this.eriSpecification}
```
The server realizes the data retrieval component for a "Retrieval-Augmentation Generation" (RAG) process.
The server is called "{this.serverName}" and is described as follows:
```
{this.serverDescription}
```
""");
//
// ---------------------------------
// Data Source
// ---------------------------------
//
sb.Append("""
# Data Source
""");
switch (this.selectedDataSource)
{
case DataSources.CUSTOM:
sb.Append($"""
The data source for the retrieval process is described as follows:
```
{this.otherDataSource}
```
""");
if(!string.IsNullOrWhiteSpace(this.dataSourceHostname))
{
sb.Append($"""
The data source is accessible via the hostname `{this.dataSourceHostname}`.
""");
}
if(this.dataSourcePort is not null)
{
sb.Append($"""
The data source is accessible via port `{this.dataSourcePort}`.
""");
}
break;
case DataSources.FILE_SYSTEM:
sb.Append("""
The data source for the retrieval process is the local file system. Use a placeholder for the data source path.
""");
break;
default:
case DataSources.OBJECT_STORAGE:
case DataSources.KEY_VALUE_STORE:
case DataSources.DOCUMENT_STORE:
case DataSources.RELATIONAL_DATABASE:
case DataSources.GRAPH_DATABASE:
sb.Append($"""
The data source for the retrieval process is an "{this.dataSourceProductName}" database running on the
host `{this.dataSourceHostname}` and is accessible via port `{this.dataSourcePort}`.
""");
break;
}
//
// ---------------------------------
// Authentication and Authorization
// ---------------------------------
//
sb.Append("""
# Authentication and Authorization
The process for authentication and authorization is two-step. Step 1: Users must authenticate
with the API server using a `POST` call on `/auth` with the chosen method. If this step is
successful, the API server returns a token. Step 2: This token is then required for all
other API calls.
Important notes:
- Calls to `/auth` and `/auth/methods` are accessible without authentication. All other API
endpoints require a valid step 2 token.
- It is possible that a token (step 1 token) is desired as `authMethod`. These step 1 tokens
must never be accepted as valid tokens for step 2.
- The OpenAPI Description (OAD) for `/auth` is not complete. This is because, at the time of
writing the OAD, it is not known what kind of authentication is desired. Therefore, you must
supplement the corresponding fields or data in the implementation. Example: If username/password
is desired, you must expect and read both. If both token and username/password are desired, you
must dynamically read the `authMethod` and expect and evaluate different fields accordingly.
The following authentications and authorizations should be implemented for the API server:
""");
foreach (var auth in this.selectedAuthenticationMethods)
sb.Append($"- {auth.ToPrompt()}");
if(this.IsUsingKerberos())
{
sb.Append($"""
The server will run on {this.selectedOperatingSystem.ToPrompt()} operating systems. Keep
this in mind when implementing the SSO with Kerberos.
""");
}
//
// ---------------------------------
// Security
// ---------------------------------
//
sb.Append($"""
# Security
The following security requirement for `allowedProviderType` was chosen: `{this.allowedLLMProviders}`
""");
//
// ---------------------------------
// Retrieval Processes
// ---------------------------------
//
sb.Append($"""
# Retrieval Processes
You are implementing the following data retrieval processes:
""");
var retrievalProcessCounter = 1;
foreach (var retrievalProcess in this.retrievalProcesses)
{
sb.Append($"""
## {retrievalProcessCounter++}. Retrieval Process
- Name: {retrievalProcess.Name}
- Description:
```
{retrievalProcess.Description}
```
""");
if(retrievalProcess.ParametersDescription?.Count > 0)
{
sb.Append("""
This retrieval process recognizes the following parameters:
""");
var parameterCounter = 1;
foreach (var (parameter, description) in retrievalProcess.ParametersDescription)
{
sb.Append($"""
- The {parameterCounter++} parameter is named "{parameter}":
```
{description}
```
""");
}
sb.Append("""
Please use sensible default values for the parameters. They are optional
for the user.
""");
}
if(retrievalProcess.Embeddings?.Count > 0)
{
sb.Append("""
The following embeddings are implemented for this retrieval process:
""");
var embeddingCounter = 1;
foreach (var embedding in retrievalProcess.Embeddings)
{
sb.Append($"""
- {embeddingCounter++}. Embedding
- Name: {embedding.EmbeddingName}
- Type: {embedding.EmbeddingType}
- Description:
```
{embedding.Description}
```
- When used:
```
{embedding.UsedWhen}
```
""");
}
}
}
//
// ---------------------------------
// Additional Libraries
// ---------------------------------
//
if (!string.IsNullOrWhiteSpace(this.additionalLibraries))
{
sb.Append($"""
# Additional Libraries
You use the following libraries for your implementation:
{this.additionalLibraries}
""");
}
//
// ---------------------------------
// Remarks
// ---------------------------------
//
sb.Append("""
# Remarks
- You do not ask follow-up questions.
- You consider the security of the implementation by applying the Security by Design principle.
- Your output is formatted as Markdown. Code is formatted as code blocks. For every file, you
create a separate code block with its file path and name as chapter title.
- Important: The JSON objects of the API messages use camel case for the data field names.
""");
return sb.ToString();
}
}
ProgrammingLanguage = this.selectedProgrammingLanguage,
OtherProgrammingLanguage = this.otherProgrammingLanguage,
ERISpecification = this.eriSpecification,
ServerName = this.serverName,
ServerDescription = this.serverDescription,
DataSource = this.selectedDataSource,
OtherDataSource = this.otherDataSource,
DataSourceProductName = this.dataSourceProductName,
DataSourceHostname = this.dataSourceHostname,
DataSourcePort = this.dataSourcePort,
AuthenticationMethods = this.selectedAuthenticationMethods,
AuthDescription = this.authDescription,
OperatingSystem = this.selectedOperatingSystem,
AllowedLLMProviders = this.allowedLLMProviders,
RetrievalProcesses = this.retrievalProcesses,
AdditionalLibraries = this.additionalLibraries,
}.ToSystemPrompt();
protected override IReadOnlyList<IButtonData> FooterButtons =>
[
@@ -1071,36 +823,6 @@ public partial class AssistantERI : AssistantBaseCore<SettingsDialogERIServer>
await this.AutoSave();
}
[GeneratedRegex("""
"([\\/._\w-]+)"
""", RegexOptions.NonBacktracking)]
private static partial Regex FileExtractRegex();
private IEnumerable<string> ExtractFiles(string fileListAnswer)
{
//
// We asked the LLM for answering using a specific JSON scheme.
// However, the LLM might not follow this scheme. Therefore, we
// need to parse the answer and extract the files.
// The parsing strategy is to look for all strings.
//
var matches = FileExtractRegex().Matches(fileListAnswer);
foreach (Match match in matches)
if(match.Groups[1].Value is {} file && !string.IsNullOrWhiteSpace(file) && !file.Equals("files", StringComparison.OrdinalIgnoreCase))
yield return file;
}
[GeneratedRegex("""
\s*#+\s+[\w\\/.]+\s*```\w*\s+([\s\w\W]+)\s*```\s*
""", RegexOptions.Singleline)]
private static partial Regex CodeExtractRegex();
private string ExtractCode(string markdown)
{
var match = CodeExtractRegex().Match(markdown);
return match.Success ? match.Groups[1].Value : string.Empty;
}
private async Task GenerateServer()
{
if(this.IsNoneERIServerSelected)
@@ -1152,13 +874,19 @@ public partial class AssistantERI : AssistantBaseCore<SettingsDialogERIServer>
Use the following JSON schema:
{
[
"files": [
"path/to/file1",
"path/to/file2"
]
}
""", true);
var fileListAnswer = await this.AddAIResponseAsync(time, true);
if (!ERIServerFileList.TryParse(fileListAnswer, out var filesToGenerate))
{
this.Logger.LogWarning("The list of files could not be read from the answer of the model: '{Answer}'", fileListAnswer);
await this.MessageBus.SendWarning(new(Icons.Material.Filled.SaveAs, T("The list of files in the answer of the model could not be read, so no files were created. Please try again.")));
return;
}
//
// Is this an update of the ERI server? If so, we need to delete the previously generated files.
@@ -1166,7 +894,7 @@ public partial class AssistantERI : AssistantBaseCore<SettingsDialogERIServer>
// and it outlives a change of the base directory, as well as a restore on another computer.
// There, the same path may belong to files of somebody else:
//
if (writeToFilesystemSnapshot && previouslyGeneratedFilesSnapshot.Length > 0 && !string.IsNullOrWhiteSpace(fileListAnswer))
if (writeToFilesystemSnapshot && previouslyGeneratedFilesSnapshot.Length > 0)
{
foreach (var file in previouslyGeneratedFilesSnapshot)
{
@@ -1196,7 +924,6 @@ public partial class AssistantERI : AssistantBaseCore<SettingsDialogERIServer>
}
var generatedFiles = new List<string>();
var filesToGenerate = this.ExtractFiles(fileListAnswer).ToArray();
foreach (var file in filesToGenerate)
{
this.Logger.LogInformation($"The LLM want to create the file: '{file}'");
@@ -1215,48 +942,18 @@ public partial class AssistantERI : AssistantBaseCore<SettingsDialogERIServer>
```language
content of the file
```
When the file itself contains code blocks, enclose its content in a longer
fence, such as four backticks.
""", true);
var generatedCodeMarkdown = await this.AddAIResponseAsync(time);
if (writeToFilesystemSnapshot)
{
var desiredFilePath = Path.Join(baseDirectorySnapshot, file);
// Security check: ensure that the desired file path is inside the base directory.
// We cannot trust the beginning of the file path because it would be possible
// to escape by using `..` in the file path.
if (!PathTools.IsInsideDirectory(baseDirectorySnapshot, desiredFilePath))
this.Logger.LogWarning($"The file path '{desiredFilePath}' is may not inside the base directory '{baseDirectorySnapshot}'.");
var outcome = await ERIServerFiles.WriteAsync(baseDirectorySnapshot, file, generatedCodeMarkdown, this.Logger);
if (outcome is ERIServerFileOutcome.WRITTEN)
generatedFiles.Add(ERIServerFiles.PathOf(baseDirectorySnapshot, file));
else
{
var code = this.ExtractCode(generatedCodeMarkdown);
if (string.IsNullOrWhiteSpace(code))
this.Logger.LogWarning($"The file content for '{desiredFilePath}' is empty or was not found.");
else
{
// Ensure that the directory exists:
var fileDirectory = Path.GetDirectoryName(desiredFilePath);
if (fileDirectory is null)
this.Logger.LogWarning($"The file path '{desiredFilePath}' does not contain a directory.");
else
{
generatedFiles.Add(desiredFilePath);
var fileDirectoryInfo = new DirectoryInfo(fileDirectory);
if(!fileDirectoryInfo.Exists)
{
fileDirectoryInfo.Create();
this.Logger.LogInformation($"The directory '{fileDirectory}' was created.");
}
// Save the file to the file system:
await File.WriteAllTextAsync(desiredFilePath, code, Encoding.UTF8);
this.Logger.LogInformation($"The file '{desiredFilePath}' was created.");
}
}
}
await this.MessageBus.SendWarning(new(Icons.Material.Filled.SaveAs, outcome.ToUserMessage(file)));
}
}
app/MindWork AI Studio/Assistants/ERI/ERIServerFileList.cs
+60
View File
@@ -0,0 +1,60 @@
using System.Text.Json;
namespace AIStudio.Assistants.ERI;
/// <summary>
/// The files which the model of the ERI Server assistant wants to create.
/// </summary>
/// <remarks>
/// The model answers with a JSON object of the form {"files": ["path/to/file1", "path/to/file2"]}. Up to
/// v26.10.1, the assistant asked for invalid JSON, an object holding a bare list, and took every string of
/// the answer for a file, apart from the word "files". A path with a space was lost, while any other key
/// the model came up with became a file.
/// </remarks>
internal sealed class ERIServerFileList
{
private static readonly JsonSerializerOptions JSON_OPTIONS = new()
{
PropertyNamingPolicy = JsonNamingPolicy.CamelCase,
AllowTrailingCommas = false,
ReadCommentHandling = JsonCommentHandling.Disallow,
MaxDepth = 8,
};
/// <summary>
/// The paths of the files, relative to the base directory.
/// </summary>
public List<string>? Files { get; init; }
/// <summary>
/// Reads the list of files out of the answer of the model.
/// </summary>
/// <param name="answer">The answer of the model to the request for the list of files.</param>
/// <param name="files">The paths of the files, without empty ones, or an empty list when the answer
/// holds none.</param>
/// <returns>True, when the answer holds a list with at least one file.</returns>
public static bool TryParse(string answer, out IReadOnlyList<string> files)
{
files = [];
var json = ModelAnswerJson.ExtractObject(answer);
if (json.IsWhiteSpace())
return false;
ERIServerFileList? parsed;
try
{
parsed = JsonSerializer.Deserialize<ERIServerFileList>(json, JSON_OPTIONS);
}
catch (JsonException)
{
return false;
}
var paths = parsed?.Files?.Where(file => !string.IsNullOrWhiteSpace(file)).ToList() ?? [];
if (paths.Count is 0)
return false;
files = paths;
return true;
}
}
app/MindWork AI Studio/Assistants/ERI/ERIServerFileOutcome.cs
+27
View File
@@ -0,0 +1,27 @@
namespace AIStudio.Assistants.ERI;
/// <summary>
/// What became of a file which the model of the ERI Server assistant created.
/// </summary>
internal enum ERIServerFileOutcome
{
/// <summary>
/// The file was written below the base directory.
/// </summary>
WRITTEN,
/// <summary>
/// The path of the file leads outside the base directory, so the file was not written.
/// </summary>
OUTSIDE_BASE_DIRECTORY,
/// <summary>
/// The answer holds no complete code block, so there was nothing to write.
/// </summary>
NO_COMPLETE_CODE_BLOCK,
/// <summary>
/// Writing the file failed, e.g., for lack of permission.
/// </summary>
WRITE_FAILED,
}
app/MindWork AI Studio/Assistants/ERI/ERIServerFileOutcomeExtensions.cs
+30
View File
@@ -0,0 +1,30 @@
namespace AIStudio.Assistants.ERI;
/// <summary>
/// Tells the user why the ERI Server assistant did not write a file.
/// </summary>
/// <remarks>
/// Up to v26.10.1, such a file only showed up in the log, and the user found out by comparing the
/// base directory with the list of files in the answer.
/// </remarks>
internal static class ERIServerFileOutcomeExtensions
{
private static string TB(string fallbackEN) => Tools.PluginSystem.I18N.I.T(fallbackEN, typeof(ERIServerFileOutcomeExtensions).Namespace, nameof(ERIServerFileOutcomeExtensions));
/// <summary>
/// Gets the localized message which explains why a file was not written.
/// </summary>
/// <param name="outcome">What became of the file.</param>
/// <param name="file">The path of the file, as the model named it.</param>
/// <returns>The localized message.</returns>
internal static string ToUserMessage(this ERIServerFileOutcome outcome, string file) => string.Format(ToUserMessageFormat(outcome), file);
private static string ToUserMessageFormat(ERIServerFileOutcome outcome) => outcome switch
{
ERIServerFileOutcome.OUTSIDE_BASE_DIRECTORY => TB("The file '{0}' was not written, because its path leads outside the base directory."),
ERIServerFileOutcome.NO_COMPLETE_CODE_BLOCK => TB("The file '{0}' was not written, because the answer of the model contains no complete code block for it. You find the answer in the conversation."),
ERIServerFileOutcome.WRITE_FAILED => TB("The file '{0}' could not be written. The log of AI Studio names the reason."),
_ => TB("The file '{0}' was not written."),
};
}
app/MindWork AI Studio/Assistants/ERI/ERIServerFiles.cs
+111
View File
@@ -0,0 +1,111 @@
using System.Text;
using Markdig.Extensions.Mathematics;
using Markdig.Syntax;
namespace AIStudio.Assistants.ERI;
/// <summary>
/// Reads the files of an ERI server out of the answers of the model and writes them to disk.
/// </summary>
internal static class ERIServerFiles
{
/// <summary>
/// Reads the content of a file out of the answer which wrote it.
/// </summary>
/// <remarks>
/// The model is asked for a heading with the path of the file, followed by a code block with its
/// content. Only the code block counts: models write the heading as they please, e.g., with the
/// path in backticks, with a "File:" in front, or not at all. Up to v26.10.1, a regular expression
/// insisted on a heading without a hyphen, which left out files such as docker-compose.yml, and it
/// read up to the last fence of the answer, which merged a usage example into the file.
///
/// Markdig reads the answer the way the chat shows it. So we take the first complete code block,
/// and a file which holds code blocks itself stays whole when the model fences it with more
/// backticks. A block the model never closed is left out: the answer broke off, and half a file
/// is nothing to write to disk.
/// </remarks>
/// <param name="answer">The answer of the model to the request for one file.</param>
/// <returns>The content of the file, ending with a line break, or an empty string when the
/// answer holds no complete code block.</returns>
public static string ReadFileContent(string answer)
{
if (string.IsNullOrWhiteSpace(answer))
return string.Empty;
//
// Markdig reads a formula between $$ as a MathBlock, which is a fenced code block as well:
//
var document = Markdig.Markdown.Parse(answer, Markdown.SAFE_MARKDOWN_PIPELINE);
var codeBlock = document.Descendants<FencedCodeBlock>().FirstOrDefault(block => block is not MathBlock && block.ClosingFencedCharCount > 0);
if (codeBlock is null)
return string.Empty;
var content = codeBlock.Lines.ToString();
return content.Length is 0 ? string.Empty : $"{content}\n";
}
/// <summary>
/// The path to which a file of the model is written.
/// </summary>
/// <param name="baseDirectory">The base directory of the ERI server.</param>
/// <param name="file">The path of the file, as the model named it.</param>
/// <returns>The path of the file below the base directory.</returns>
public static string PathOf(string baseDirectory, string file) => Path.Join(baseDirectory, file);
/// <summary>
/// Writes a file of the model below the base directory.
/// </summary>
/// <remarks>
/// A file which cannot be written ends nothing but its own attempt: the caller tells the user,
/// and the server gets its other files. Up to v26.10.1, an error while writing ended the whole
/// run, and the files written so far were never recorded for their deletion on the next run.
/// </remarks>
/// <param name="baseDirectory">The base directory of the ERI server.</param>
/// <param name="file">The path of the file, as the model named it.</param>
/// <param name="answer">The answer of the model to the request for this file.</param>
/// <param name="logger">The logger which records the details.</param>
/// <returns>What became of the file.</returns>
public static async Task<ERIServerFileOutcome> WriteAsync(string baseDirectory, string file, string answer, ILogger logger)
{
var desiredFilePath = PathOf(baseDirectory, file);
//
// Security check: ensure that the desired file path is inside the base directory.
// We cannot trust the beginning of the file path because it would be possible
// to escape by using `..` in the file path.
//
if (!PathTools.IsInsideDirectory(baseDirectory, desiredFilePath))
{
logger.LogWarning("The file path '{FilePath}' is not inside the base directory '{BaseDirectory}'.", desiredFilePath, baseDirectory);
return ERIServerFileOutcome.OUTSIDE_BASE_DIRECTORY;
}
var content = ReadFileContent(answer);
if (string.IsNullOrWhiteSpace(content))
{
logger.LogWarning("The content of the file '{FilePath}' is empty or was not found.", desiredFilePath);
return ERIServerFileOutcome.NO_COMPLETE_CODE_BLOCK;
}
try
{
// A path inside the base directory always has a directory:
var fileDirectory = Path.GetDirectoryName(desiredFilePath)!;
if (!Directory.Exists(fileDirectory))
{
Directory.CreateDirectory(fileDirectory);
logger.LogInformation("The directory '{Directory}' was created.", fileDirectory);
}
await File.WriteAllTextAsync(desiredFilePath, content, Encoding.UTF8);
logger.LogInformation("The file '{FilePath}' was created.", desiredFilePath);
return ERIServerFileOutcome.WRITTEN;
}
catch (Exception e) when (e is IOException or UnauthorizedAccessException or ArgumentException or NotSupportedException)
{
logger.LogWarning(e, "The file '{FilePath}' could not be written.", desiredFilePath);
return ERIServerFileOutcome.WRITE_FAILED;
}
}
}
app/MindWork AI Studio/Assistants/ERI/ERIServerPrompt.cs
+332
View File
@@ -0,0 +1,332 @@
using System.Text;
namespace AIStudio.Assistants.ERI;
/// <summary>
/// What the ERI Server assistant tells the model about the server it has to implement.
/// </summary>
/// <remarks>
/// Kept apart from the assistant, so that the system prompt can be checked without a running app:
/// every input of the form has to arrive in it, and in a shape the model can read.
/// </remarks>
internal sealed class ERIServerPrompt
{
public ProgrammingLanguages ProgrammingLanguage { get; init; } = ProgrammingLanguages.NONE;
public string OtherProgrammingLanguage { get; init; } = string.Empty;
/// <summary>
/// The OpenAPI description of the ERI, or null while it is not loaded yet.
/// </summary>
public string? ERISpecification { get; init; }
public string ServerName { get; init; } = string.Empty;
public string ServerDescription { get; init; } = string.Empty;
public DataSources DataSource { get; init; } = DataSources.NONE;
public string OtherDataSource { get; init; } = string.Empty;
public string DataSourceProductName { get; init; } = string.Empty;
public string DataSourceHostname { get; init; } = string.Empty;
public int? DataSourcePort { get; init; }
public IEnumerable<Auth> AuthenticationMethods { get; init; } = [];
/// <summary>
/// How the authentication methods are meant to be used, e.g., which data is public.
/// </summary>
public string AuthDescription { get; init; } = string.Empty;
public OperatingSystem OperatingSystem { get; init; } = OperatingSystem.NONE;
public AllowedLLMProviders AllowedLLMProviders { get; init; } = AllowedLLMProviders.NONE;
public IReadOnlyList<RetrievalInfo> RetrievalProcesses { get; init; } = [];
public string AdditionalLibraries { get; init; } = string.Empty;
public string ToSystemPrompt()
{
var sb = new StringBuilder();
//
// ---------------------------------
// Introduction
// ---------------------------------
//
var programmingLanguage = this.ProgrammingLanguage is ProgrammingLanguages.OTHER ? this.OtherProgrammingLanguage : this.ProgrammingLanguage.ToPrompt();
sb.Append($"""
# Introduction
You are an experienced {programmingLanguage} developer. Your task is to implement an API server in
the {programmingLanguage} language according to the following OpenAPI Description (OAD):
```
{this.ERISpecification}
```
The server realizes the data retrieval component for a "Retrieval-Augmentation Generation" (RAG) process.
The server is called "{this.ServerName}" and is described as follows:
```
{this.ServerDescription}
```
""");
//
// ---------------------------------
// Data Source
// ---------------------------------
//
sb.Append("""
# Data Source
""");
switch (this.DataSource)
{
case DataSources.CUSTOM:
sb.Append($"""
The data source for the retrieval process is described as follows:
```
{this.OtherDataSource}
```
""");
if(!string.IsNullOrWhiteSpace(this.DataSourceHostname))
{
sb.Append($"""
The data source is accessible via the hostname `{this.DataSourceHostname}`.
""");
}
if(this.DataSourcePort is not null)
{
sb.Append($"""
The data source is accessible via port `{this.DataSourcePort}`.
""");
}
break;
case DataSources.FILE_SYSTEM:
sb.Append("""
The data source for the retrieval process is the local file system. Use a placeholder for the data source path.
""");
break;
default:
case DataSources.OBJECT_STORAGE:
case DataSources.KEY_VALUE_STORE:
case DataSources.DOCUMENT_STORE:
case DataSources.RELATIONAL_DATABASE:
case DataSources.GRAPH_DATABASE:
sb.Append($"""
The data source for the retrieval process is an "{this.DataSourceProductName}" database running on the
host `{this.DataSourceHostname}` and is accessible via port `{this.DataSourcePort}`.
""");
break;
}
//
// ---------------------------------
// Authentication and Authorization
// ---------------------------------
//
sb.Append("""
# Authentication and Authorization
The process for authentication and authorization is two-step. Step 1: Users must authenticate
with the API server using a `POST` call on `/auth` with the chosen method. If this step is
successful, the API server returns a token. Step 2: This token is then required for all
other API calls.
Important notes:
- Calls to `/auth` and `/auth/methods` are accessible without authentication. All other API
endpoints require a valid step 2 token.
- It is possible that a token (step 1 token) is desired as `authMethod`. These step 1 tokens
must never be accepted as valid tokens for step 2.
- The OpenAPI Description (OAD) for `/auth` is not complete. This is because, at the time of
writing the OAD, it is not known what kind of authentication is desired. Therefore, you must
supplement the corresponding fields or data in the implementation. Example: If username/password
is desired, you must expect and read both. If both token and username/password are desired, you
must dynamically read the `authMethod` and expect and evaluate different fields accordingly.
The following authentications and authorizations should be implemented for the API server:
""");
foreach (var auth in this.AuthenticationMethods)
{
sb.Append($"""
- {auth.ToPrompt()}
""");
}
if(!string.IsNullOrWhiteSpace(this.AuthDescription))
{
sb.Append($"""
The authentication process is planned as follows:
```
{this.AuthDescription}
```
""");
}
if(this.AuthenticationMethods.Contains(Auth.KERBEROS))
{
sb.Append($"""
The server will run on {this.OperatingSystem.ToPrompt()} operating systems. Keep
this in mind when implementing the SSO with Kerberos.
""");
}
//
// ---------------------------------
// Security
// ---------------------------------
//
sb.Append($"""
# Security
The following security requirement for `allowedProviderType` was chosen: `{this.AllowedLLMProviders}`
""");
//
// ---------------------------------
// Retrieval Processes
// ---------------------------------
//
sb.Append($"""
# Retrieval Processes
You are implementing the following data retrieval processes:
""");
var retrievalProcessCounter = 1;
foreach (var retrievalProcess in this.RetrievalProcesses)
{
sb.Append($"""
## {retrievalProcessCounter++}. Retrieval Process
- Name: {retrievalProcess.Name}
- Description:
```
{retrievalProcess.Description}
```
""");
if(retrievalProcess.ParametersDescription?.Count > 0)
{
sb.Append("""
This retrieval process recognizes the following parameters:
""");
var parameterCounter = 1;
foreach (var (parameter, description) in retrievalProcess.ParametersDescription)
{
sb.Append($"""
- The {parameterCounter++} parameter is named "{parameter}":
```
{description}
```
""");
}
sb.Append("""
Please use sensible default values for the parameters. They are optional
for the user.
""");
}
if(retrievalProcess.Embeddings?.Count > 0)
{
sb.Append("""
The following embeddings are implemented for this retrieval process:
""");
var embeddingCounter = 1;
foreach (var embedding in retrievalProcess.Embeddings)
{
sb.Append($"""
- {embeddingCounter++}. Embedding
- Name: {embedding.EmbeddingName}
- Type: {embedding.EmbeddingType}
- Description:
```
{embedding.Description}
```
- When used:
```
{embedding.UsedWhen}
```
""");
}
}
}
//
// ---------------------------------
// Additional Libraries
// ---------------------------------
//
if (!string.IsNullOrWhiteSpace(this.AdditionalLibraries))
{
sb.Append($"""
# Additional Libraries
You use the following libraries for your implementation:
{this.AdditionalLibraries}
""");
}
//
// ---------------------------------
// Remarks
// ---------------------------------
//
sb.Append("""
# Remarks
- You do not ask follow-up questions.
- You consider the security of the implementation by applying the Security by Design principle.
- Your output is formatted as Markdown. Code is formatted as code blocks. For every file, you
create a separate code block with its file path and name as chapter title.
- Important: The JSON objects of the API messages use camel case for the data field names.
""");
return sb.ToString();
}
}
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
+15
View File
@@ -1624,6 +1624,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3315518334"] = "Data
-- No, just show me the code
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3316699247"] = "No, just show me the code"
-- The list of files in the answer of the model could not be read, so no files were created. Please try again.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3342789164"] = "The list of files in the answer of the model could not be read, so no files were created. Please try again."
-- Other language
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3374524465"] = "Other language"
@@ -1792,6 +1795,18 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::DATASOURCESEXTENSIONS::T3394837922"]
-- Unknown data source
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::DATASOURCESEXTENSIONS::T752426884"] = "Unknown data source"
-- The file '{0}' was not written, because the answer of the model contains no complete code block for it. You find the answer in the conversation.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T1485612676"] = "The file '{0}' was not written, because the answer of the model contains no complete code block for it. You find the answer in the conversation."
-- The file '{0}' was not written.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T1622509075"] = "The file '{0}' was not written."
-- The file '{0}' could not be written. The log of AI Studio names the reason.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T2881284775"] = "The file '{0}' could not be written. The log of AI Studio names the reason."
-- The file '{0}' was not written, because its path leads outside the base directory.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T3820652397"] = "The file '{0}' was not written, because its path leads outside the base directory."
-- Windows
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::OPERATINGSYSTEMEXTENSIONS::T104858250"] = "Windows"
app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
+15
View File
@@ -1626,6 +1626,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3315518334"] = "Daten
-- No, just show me the code
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3316699247"] = "Nein, zeig mir einfach den Code."
-- The list of files in the answer of the model could not be read, so no files were created. Please try again.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3342789164"] = "Die Liste der Dateien in der Antwort des Modells konnte nicht ausgelesen werden. Daher wurden keine Dateien erstellt. Bitte versuchen Sie es erneut."
-- Other language
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3374524465"] = "Andere Sprache"
@@ -1794,6 +1797,18 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::DATASOURCESEXTENSIONS::T3394837922"]
-- Unknown data source
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::DATASOURCESEXTENSIONS::T752426884"] = "Unbekannte Datenquelle"
-- The file '{0}' was not written, because the answer of the model contains no complete code block for it. You find the answer in the conversation.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T1485612676"] = "Die Datei „{0}“ wurde nicht geschrieben, weil die Antwort des Modells keinen vollständigen Codeblock für diese Datei enthält. Die Antwort finden Sie in der Unterhaltung."
-- The file '{0}' was not written.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T1622509075"] = "Die Datei „{0}“ wurde nicht geschrieben."
-- The file '{0}' could not be written. The log of AI Studio names the reason.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T2881284775"] = "Die Datei „{0}“ konnte nicht geschrieben werden. Das Protokoll von AI Studio nennt den Grund."
-- The file '{0}' was not written, because its path leads outside the base directory.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T3820652397"] = "Die Datei „{0}“ wurde nicht geschrieben, da ihr Pfad außerhalb des Basisverzeichnisses liegt."
-- Windows
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::OPERATINGSYSTEMEXTENSIONS::T104858250"] = "Windows"
app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
+15
View File
@@ -1626,6 +1626,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3315518334"] = "Data
-- No, just show me the code
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3316699247"] = "No, just show me the code"
-- The list of files in the answer of the model could not be read, so no files were created. Please try again.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3342789164"] = "The list of files in the answer of the model could not be read, so no files were created. Please try again."
-- Other language
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ASSISTANTERI::T3374524465"] = "Other language"
@@ -1794,6 +1797,18 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::DATASOURCESEXTENSIONS::T3394837922"]
-- Unknown data source
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::DATASOURCESEXTENSIONS::T752426884"] = "Unknown data source"
-- The file '{0}' was not written, because the answer of the model contains no complete code block for it. You find the answer in the conversation.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T1485612676"] = "The file '{0}' was not written, because the answer of the model contains no complete code block for it. You find the answer in the conversation."
-- The file '{0}' was not written.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T1622509075"] = "The file '{0}' was not written."
-- The file '{0}' could not be written. The log of AI Studio names the reason.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T2881284775"] = "The file '{0}' could not be written. The log of AI Studio names the reason."
-- The file '{0}' was not written, because its path leads outside the base directory.
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::ERISERVERFILEOUTCOMEEXTENSIONS::T3820652397"] = "The file '{0}' was not written, because its path leads outside the base directory."
-- Windows
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ERI::OPERATINGSYSTEMEXTENSIONS::T104858250"] = "Windows"
app/MindWork AI Studio/wwwroot/changelog/v26.11.1.md
+3
View File
@@ -5,6 +5,9 @@
- Fixed AI Studio becoming unusable when an update failed after its download. The app now restarts with the version you have installed, so you can simply try the update again.
- Fixed configuration plugins you installed yourself being able to change the tool settings of your organization, such as the minimum provider confidence of a tool. These settings now follow the same rules as all other settings, which also gives you back your own values once your organization removes its configuration.
- Fixed the log of AI Studio reporting that a configuration plugin you installed yourself tried to take over a setting of your organization, even when the plugin does not contain that setting.
- Fixed the Agenda Planner assistant leaving out the location of your event when it suggests a social activity.
- Fixed the ERI Server assistant leaving out your description of the authentication process when it generates a server.
- Fixed the ERI Server assistant not writing some generated files to disk, such as docker-compose.yml. It now tells you about every file it could not write.
- Upgraded several libraries to improve security.
- Upgraded Tauri to v2.12.5.
- Upgraded MudBlazor to v9.12.0.
app/SourceCodeRules/SourceCodeRules/AnalyzerReleases.Shipped.md
+1
View File
@@ -17,3 +17,4 @@
MWAIS0011 | Usage | Error | CanonicalJsonShapeAnalyzer
MWAIS0012 | Usage | Error | DirectI18NGetTextAnalyzer
MWAIS0013 | Usage | Error | ModelPatternLiteralAnalyzer
MWAIS0014 | Usage | Error | MissingInterpolationAnalyzer
app/SourceCodeRules/SourceCodeRules/Identifier.cs
+1
View File
@@ -15,4 +15,5 @@ public static class Identifier
public const string CANONICAL_JSON_SHAPE_ANALYZER = $"{Tools.ID_PREFIX}0011";
public const string DIRECT_I18N_GET_TEXT_ANALYZER = $"{Tools.ID_PREFIX}0012";
public const string MODEL_PATTERN_LITERAL_ANALYZER = $"{Tools.ID_PREFIX}0013";
public const string MISSING_INTERPOLATION_ANALYZER = $"{Tools.ID_PREFIX}0014";
}
app/SourceCodeRules/SourceCodeRules/UsageAnalyzers/MissingInterpolationAnalyzer.cs
+96
View File
@@ -0,0 +1,96 @@
using System;
using System.Collections.Immutable;
using Microsoft.CodeAnalysis;
using Microsoft.CodeAnalysis.CSharp;
using Microsoft.CodeAnalysis.CSharp.Syntax;
using Microsoft.CodeAnalysis.Diagnostics;
namespace SourceCodeRules.UsageAnalyzers;
/// <summary>
/// Reports a string which names a member in braces, such as {this.location}, without being interpolated.
/// </summary>
/// <remarks>
/// A string without a $ in front keeps its braces as text, and so does a single pair of braces in a raw
/// string written with $$. The compiler accepts both. At runtime, nothing looks wrong either: the model, the
/// user, or the file simply receives the text {this.location} instead of the location. Because every member
/// is accessed through this. (see MWAIS0005), looking for {this. finds each member which was meant to be
/// interpolated, and leaves placeholders like {0} for string.Format alone.
/// </remarks>
#pragma warning disable RS1038
[DiagnosticAnalyzer(LanguageNames.CSharp)]
#pragma warning restore RS1038
public sealed class MissingInterpolationAnalyzer : DiagnosticAnalyzer
{
private const string DIAGNOSTIC_ID = Identifier.MISSING_INTERPOLATION_ANALYZER;
private const string CATEGORY = "Usage";
private const string MEMBER_IN_BRACES = "{this.";
private const string TITLE = "A member in braces needs an interpolated string";
private const string MESSAGE_FORMAT = "The text \"{0}\" is not interpolated and stays as it is: put a $ in front of the string, or one more $ in front of a raw string";
private const string DESCRIPTION = "A string which names a member in braces, such as {this.location}, without being interpolated passes the braces on as text instead of the value of the member.";
private static readonly DiagnosticDescriptor RULE = new(DIAGNOSTIC_ID, TITLE, MESSAGE_FORMAT, CATEGORY, DiagnosticSeverity.Error, isEnabledByDefault: true, description: DESCRIPTION);
public override ImmutableArray<DiagnosticDescriptor> SupportedDiagnostics => [RULE];
public override void Initialize(AnalysisContext context)
{
context.ConfigureGeneratedCodeAnalysis(GeneratedCodeAnalysisFlags.None);
context.EnableConcurrentExecution();
context.RegisterSyntaxNodeAction(AnalyzeStringLiteral, SyntaxKind.StringLiteralExpression);
context.RegisterSyntaxNodeAction(AnalyzeInterpolatedText, SyntaxKind.InterpolatedStringText);
}
private static void AnalyzeStringLiteral(SyntaxNodeAnalysisContext context)
{
var stringLiteral = (LiteralExpressionSyntax)context.Node;
Check(context, stringLiteral.Token.ValueText, stringLiteral.GetLocation());
}
/// <summary>
/// Checks the text between the holes of an interpolated string.
/// </summary>
/// <remarks>
/// There, braces end up as text when they are escaped, like {{this.location}} in $"...", or when they
/// are fewer than the dollar signs, like {this.location} in $$"""...""".
/// </remarks>
private static void AnalyzeInterpolatedText(SyntaxNodeAnalysisContext context)
{
var interpolatedText = (InterpolatedStringTextSyntax)context.Node;
Check(context, interpolatedText.TextToken.ValueText, interpolatedText.GetLocation());
}
private static void Check(SyntaxNodeAnalysisContext context, string text, Location location)
{
var start = FindMemberInBraces(text);
if (start < 0)
return;
var end = text.IndexOf('}', start);
var memberInBraces = end < 0 ? text.Substring(start) : text.Substring(start, end - start + 1);
context.ReportDiagnostic(Diagnostic.Create(RULE, location, memberInBraces));
}
/// <summary>
/// Finds {this. followed by the first character of a member name.
/// </summary>
/// <returns>The position of the opening brace, or -1 when the text names no member in braces.</returns>
private static int FindMemberInBraces(string text)
{
var position = text.IndexOf(MEMBER_IN_BRACES, StringComparison.Ordinal);
while (position >= 0)
{
var next = position + MEMBER_IN_BRACES.Length;
if (next < text.Length && (char.IsLetter(text[next]) || text[next] == '_'))
return position;
position = text.IndexOf(MEMBER_IN_BRACES, next, StringComparison.Ordinal);
}
return -1;
}
}
app/Tests/Analyzers/MissingInterpolationAnalyzerTests.cs
+78
View File
@@ -0,0 +1,78 @@
using AIStudio.Tests.Models.Generation;
using Microsoft.CodeAnalysis;
using SourceCodeRules.UsageAnalyzers;
namespace AIStudio.Tests.Analyzers;
/// <summary>
/// Checks that a string which names a member in braces without being interpolated is refused while compiling.
/// </summary>
/// <remarks>
/// The Agenda Planner asked the model for an activity at '{this.inputLocation}', word for word, because a $
/// was missing in front of the string. Nothing at runtime gives such a string away, so MWAIS0014 turns it into
/// a build error, and these tests say which strings it recognizes and which it leaves alone.
/// </remarks>
[TestFixture]
public sealed class MissingInterpolationAnalyzerTests
{
[TestCase(""" "at '{this.location}'" """, TestName = "A regular string")]
[TestCase(""" @"at '{this.location}'" """, TestName = "A verbatim string")]
[TestCase("""" """at '{this.location}'""" """", TestName = "A raw string")]
[TestCase("""" $$"""at '{this.location}'""" """", TestName = "A raw string with two dollar signs and one pair of braces")]
[TestCase(""" $"at '{{this.location}}'" """, TestName = "An interpolated string with escaped braces")]
public async Task AMemberInBracesWhichStaysTextIsRefused(string expression)
{
var reported = await AnalyzeAsync(expression);
Assert.Multiple(() =>
{
Assert.That(reported, Has.Count.EqualTo(1));
Assert.That(reported.FirstOrDefault()?.Id, Is.EqualTo("MWAIS0014"));
Assert.That(reported.FirstOrDefault()?.GetMessage(), Does.Contain("\"{this.location}\""));
});
}
[Test]
public async Task AMultiLineRawStringIsRefused()
{
var reported = await AnalyzeAsync(
""""
"""
The participants meet at '{this.location}'.
"""
"""");
Assert.That(reported, Has.Count.EqualTo(1));
}
[TestCase(""" $"at '{this.location}'" """, TestName = "An interpolated string")]
[TestCase("""" $$"""at '{{this.location}}'""" """", TestName = "A raw string with two dollar signs and two pairs of braces")]
[TestCase(""" string.Format("at '{0}'", this.location) """, TestName = "A placeholder for string.Format")]
[TestCase(""" "the {this} keyword" """, TestName = "The keyword in braces")]
[TestCase(""" "at this.location" """, TestName = "A member without braces")]
public async Task TextWhichIsNoForgottenInterpolationIsLeftAlone(string expression)
{
var reported = await AnalyzeAsync(expression);
Assert.That(reported, Is.Empty);
}
private static async Task<IReadOnlyList<Diagnostic>> AnalyzeAsync(string expression)
{
var source =
$$"""
namespace Sample;
public sealed class Meeting
{
private readonly string location = "Cologne";
public string Describe() => {{expression}};
}
""";
return await CompilationHarness.AnalyzeAsync(source, new MissingInterpolationAnalyzer());
}
}
app/Tests/Assistants/ERI/ERIServerFileListTests.cs
+82
View File
@@ -0,0 +1,82 @@
using AIStudio.Assistants.ERI;
namespace AIStudio.Tests.Assistants.ERI;
/// <summary>
/// Checks how the ERI Server assistant reads which files the model wants to create.
/// </summary>
/// <remarks>
/// Up to v26.10.1, the assistant asked for invalid JSON and took every string of the answer for a file,
/// apart from the word "files". A path with a space was lost that way, and any other key the model came
/// up with became a file of its own.
/// </remarks>
[TestFixture]
public sealed class ERIServerFileListTests
{
private static readonly string[] FILES = ["docker-compose.yml", "src/my-server/main.py", "docs/read me.md"];
private static readonly string[] PATH_WITH_BACKSLASH = [@"src\main.py"];
private static readonly string[] MAIN_PY = ["main.py"];
[Test]
public void AListBetweenProseAndAFenceIsRead()
{
const string ANSWER = """
Here are the files:
```json
{ "files": [ "docker-compose.yml", "src/my-server/main.py", "docs/read me.md" ] }
```
""";
var isRead = ERIServerFileList.TryParse(ANSWER, out var files);
Assert.Multiple(() =>
{
Assert.That(isRead, Is.True);
Assert.That(files, Is.EqualTo(FILES));
});
}
[Test]
public void EscapedCharactersAreDecoded()
{
var isRead = ERIServerFileList.TryParse("""{ "files": [ "src\\main.py" ] }""", out var files);
Assert.Multiple(() =>
{
Assert.That(isRead, Is.True);
Assert.That(files, Is.EqualTo(PATH_WITH_BACKSLASH));
});
}
[Test]
public void EmptyPathsAreLeftOut()
{
var isRead = ERIServerFileList.TryParse("""{ "files": [ "main.py", "", " " ] }""", out var files);
Assert.Multiple(() =>
{
Assert.That(isRead, Is.True);
Assert.That(files, Is.EqualTo(MAIN_PY));
});
}
[TestCase("""[ "main.py" ]""", TestName = "A list without its object")]
[TestCase("""{ "paths": [ "main.py" ] }""", TestName = "A key other than files")]
[TestCase("""{ "files": [] }""", TestName = "An empty list")]
[TestCase("""{ "files": [ "main.py", }""", TestName = "Broken JSON")]
[TestCase("""{ "files": "main.py" }""", TestName = "A single path instead of a list")]
[TestCase("The files are main.py and README.md.", TestName = "No JSON at all")]
public void AnAnswerWithoutAListOfFilesIsRefused(string answer)
{
var isRead = ERIServerFileList.TryParse(answer, out var files);
Assert.Multiple(() =>
{
Assert.That(isRead, Is.False);
Assert.That(files, Is.Empty);
});
}
}
app/Tests/Assistants/ERI/ERIServerFileWritingTests.cs
+90
View File
@@ -0,0 +1,90 @@
using AIStudio.Assistants.ERI;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.Assistants.ERI;
/// <summary>
/// Checks what the ERI Server assistant reports for each file it is asked to write below the base directory.
/// </summary>
/// <remarks>
/// Up to v26.10.1, a file which was not written only showed up in the log, and an error while writing
/// ended the whole run. Now every file gets an outcome, which the assistant tells the user about.
/// </remarks>
[TestFixture]
public sealed class ERIServerFileWritingTests
{
private const string MAIN_PY = "print('ERI server')\n";
private const string ANSWER = $"## main.py\n\n```python\n{MAIN_PY}```";
private string baseDirectory = string.Empty;
[SetUp]
public void CreateBaseDirectory()
{
this.baseDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-eri-server-{Guid.NewGuid():N}");
Directory.CreateDirectory(this.baseDirectory);
}
[TearDown]
public void RemoveBaseDirectory()
{
try
{
Directory.Delete(this.baseDirectory, true);
}
catch (IOException)
{
// A temporary directory we could not remove says nothing about the code under test.
}
}
[Test]
public async Task AFileIsWrittenBelowTheBaseDirectory()
{
var outcome = await ERIServerFiles.WriteAsync(this.baseDirectory, "src/my-server/main.py", ANSWER, NullLogger.Instance);
Assert.Multiple(() =>
{
Assert.That(outcome, Is.EqualTo(ERIServerFileOutcome.WRITTEN));
Assert.That(File.ReadAllText(Path.Join(this.baseDirectory, "src", "my-server", "main.py")), Is.EqualTo(MAIN_PY));
});
}
[Test]
public async Task APathLeadingOutsideTheBaseDirectoryIsNotWritten()
{
var file = $"../outside-{Guid.NewGuid():N}.py";
var outcome = await ERIServerFiles.WriteAsync(this.baseDirectory, file, ANSWER, NullLogger.Instance);
Assert.Multiple(() =>
{
Assert.That(outcome, Is.EqualTo(ERIServerFileOutcome.OUTSIDE_BASE_DIRECTORY));
Assert.That(File.Exists(Path.GetFullPath(Path.Join(this.baseDirectory, file))), Is.False);
});
}
[Test]
public async Task AnAnswerWhichBreaksOffWritesNothing()
{
var outcome = await ERIServerFiles.WriteAsync(this.baseDirectory, "main.py", "## main.py\n\n```python\nprint(", NullLogger.Instance);
Assert.Multiple(() =>
{
Assert.That(outcome, Is.EqualTo(ERIServerFileOutcome.NO_COMPLETE_CODE_BLOCK));
Assert.That(File.Exists(Path.Join(this.baseDirectory, "main.py")), Is.False);
});
}
[Test]
public async Task AFileWhichCannotBeWrittenIsReportedInsteadOfEndingTheRun()
{
// A directory of the same name stands where the file would go:
Directory.CreateDirectory(Path.Join(this.baseDirectory, "main.py"));
var outcome = await ERIServerFiles.WriteAsync(this.baseDirectory, "main.py", ANSWER, NullLogger.Instance);
Assert.That(outcome, Is.EqualTo(ERIServerFileOutcome.WRITE_FAILED));
}
}
app/Tests/Assistants/ERI/ERIServerFilesTests.cs
+88
View File
@@ -0,0 +1,88 @@
using AIStudio.Assistants.ERI;
namespace AIStudio.Tests.Assistants.ERI;
/// <summary>
/// Checks how the ERI Server assistant reads the files of a server out of the answers of the model.
/// </summary>
/// <remarks>
/// Up to v26.10.1, a file such as docker-compose.yml never reached the disk: the regular expression which
/// read its content expected a heading without a hyphen, and the file only showed up in the log. The same
/// expression read up to the last fence of an answer, so a usage example after the file ended up in it.
/// </remarks>
[TestFixture]
public sealed class ERIServerFilesTests
{
private const string COMPOSE_FILE = "services:\n eri:\n build: .\n ports:\n - \"8080:8080\"\n";
[TestCase("## docker-compose.yml", TestName = "A heading with a hyphen")]
[TestCase("## `docker-compose.yml`", TestName = "A heading with the path in backticks")]
[TestCase("### File: deploy/my-server/docker-compose.yml", TestName = "A heading with a label in front of the path")]
[TestCase("Here is the file:", TestName = "No heading at all")]
public void TheContentIsReadWhateverStandsAboveIt(string introduction)
{
var answer = Answer(introduction, string.Empty, "```yaml", COMPOSE_FILE.TrimEnd('\n'), "```");
Assert.That(ERIServerFiles.ReadFileContent(answer), Is.EqualTo(COMPOSE_FILE));
}
[Test]
public void ACodeBlockAfterTheFileStaysOutOfIt()
{
var answer = Answer(
"## main.py",
string.Empty,
"```python",
"print('ERI server')",
"```",
string.Empty,
"Start the server with:",
string.Empty,
"```bash",
"python main.py",
"```");
Assert.That(ERIServerFiles.ReadFileContent(answer), Is.EqualTo("print('ERI server')\n"));
}
[Test]
public void AFileWithCodeBlocksOfItsOwnStaysWholeInALongerFence()
{
var answer = Answer(
"## README.md",
string.Empty,
"````markdown",
"# ERI Server",
string.Empty,
"```bash",
"python main.py",
"```",
"````");
Assert.That(ERIServerFiles.ReadFileContent(answer), Is.EqualTo("# ERI Server\n\n```bash\npython main.py\n```\n"));
}
[Test]
public void AFenceOfTildesIsReadAsWell()
{
var answer = Answer("## main.py", "~~~python", "print('ERI server')", "~~~");
Assert.That(ERIServerFiles.ReadFileContent(answer), Is.EqualTo("print('ERI server')\n"));
}
[Test]
public void ACodeBlockWhichBreaksOffIsNoFile()
{
var answer = Answer("## main.py", "```python", "print('ERI server')");
Assert.That(ERIServerFiles.ReadFileContent(answer), Is.Empty);
}
[Test]
public void AnAnswerWithoutACodeBlockIsNoFile()
{
Assert.That(ERIServerFiles.ReadFileContent("I cannot create this file."), Is.Empty);
}
private static string Answer(params string[] lines) => string.Join('\n', lines);
}
app/Tests/Assistants/ERI/ERIServerPromptTests.cs
+81
View File
@@ -0,0 +1,81 @@
using AIStudio.Assistants.ERI;
using OperatingSystem = AIStudio.Assistants.ERI.OperatingSystem;
namespace AIStudio.Tests.Assistants.ERI;
/// <summary>
/// Checks that the system prompt of the ERI Server assistant carries every input of the form, in a shape
/// the model can read.
/// </summary>
/// <remarks>
/// Up to v26.10.1, the description of the planned authentication never reached the model. Besides, the
/// heading of the data source ran into the text after it, and all authentication methods stood on a
/// single line.
/// </remarks>
[TestFixture]
public sealed class ERIServerPromptTests
{
private const string AUTH_DESCRIPTION = "Editors log in with their username and password; the public catalog needs no login.";
private static readonly string[] HEADINGS =
[
"# Introduction",
"# Data Source",
"# Authentication and Authorization",
"# Security",
"# Retrieval Processes",
"# Remarks"
];
private static readonly string[] TOKEN_AND_PASSWORD_LINES = ["- Login by static token per user.", "- Login by username and password."];
[Test]
public void EveryHeadingStandsOnItsOwnLine([Values] DataSources dataSource)
{
var lines = Lines(new ERIServerPrompt { DataSource = dataSource, AuthenticationMethods = [Auth.TOKEN] });
Assert.That(lines, Is.SupersetOf(HEADINGS));
}
[Test]
public void EachAuthenticationMethodStandsOnItsOwnLine()
{
var lines = Lines(new ERIServerPrompt { AuthenticationMethods = [Auth.TOKEN, Auth.USERNAME_PASSWORD] });
Assert.That(lines, Is.SupersetOf(TOKEN_AND_PASSWORD_LINES));
}
[Test]
public void TheDescriptionOfTheAuthenticationReachesTheModel()
{
var lines = Lines(new ERIServerPrompt { AuthenticationMethods = [Auth.NONE, Auth.USERNAME_PASSWORD], AuthDescription = AUTH_DESCRIPTION });
Assert.Multiple(() =>
{
Assert.That(lines, Does.Contain("The authentication process is planned as follows:"));
Assert.That(lines, Does.Contain(AUTH_DESCRIPTION));
});
}
[Test]
public void WithoutADescriptionOfTheAuthenticationItsSectionIsLeftOut()
{
var prompt = new ERIServerPrompt { AuthenticationMethods = [Auth.TOKEN] }.ToSystemPrompt();
Assert.That(prompt, Does.Not.Contain("The authentication process is planned as follows:"));
}
[Test]
public void KerberosStillNamesTheOperatingSystemAfterTheDescription()
{
var lines = Lines(new ERIServerPrompt { AuthenticationMethods = [Auth.KERBEROS, Auth.TOKEN], AuthDescription = AUTH_DESCRIPTION, OperatingSystem = OperatingSystem.LINUX });
Assert.That(lines, Does.Contain("The server will run on Linux operating systems. Keep"));
}
/// <summary>
/// The lines of the system prompt, whatever line endings the source of the prompt was checked out with.
/// </summary>
private static string[] Lines(ERIServerPrompt prompt) => prompt.ToSystemPrompt().ReplaceLineEndings("\n").Split('\n');
}