improved code and made sure components are reused if possible

This commit is contained in:
Paul Koudelka committed 2026-08-12 14:19:32 +02:00
1 parent d95487aa6e
commit 647c428ebf
13 files changed
+142 -212

No files matched your search

@@ -9598,12 +9598,18 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text"
-- Office Files
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office Files"
-- Spreadsheet
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1313839225"] = "Spreadsheet"
-- Executable
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1364437037"] = "Executable"
-- Mail
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1399880782"] = "Mail"
-- Delimited table
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1405737676"] = "Delimited table"
-- Source like
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1487238587"] = "Source like"
@@ -9631,6 +9637,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2502277006"] = "Custom"
-- Visual briefing image
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2505088878"] = "Visual briefing image"
-- Shortcut
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2547828883"] = "Shortcut"
-- Media
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T3507473059"] = "Media"
@@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Tools.Validation
@using AIStudio.Provider
@using AIStudio.Tools.ERIClient.DataModel
@inherits MSGComponentBase
@@ -12,8 +13,8 @@
@bind-Text="@this.dataName"
Label="@T("Data Source Name")"
Class="mb-6"
MaxLength="40"
Counter="40"
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
Immediate="@true"
Validation="@this.dataSourceValidation.ValidatingName"
Adornment="Adornment.Start"
@@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Tools.Validation
@using AIStudio.Provider
@inherits MSGComponentBase
@@ -11,8 +12,8 @@
@bind-Text="@this.dataName"
Label="@T("Data Source Name")"
Class="mb-6"
MaxLength="40"
Counter="40"
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
Immediate="@true"
Validation="@this.dataSourceValidation.ValidatingName"
Adornment="Adornment.Start"
@@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Tools.Validation
@using AIStudio.Provider
@inherits MSGComponentBase
@@ -11,8 +12,8 @@
@bind-Text="@this.dataName"
Label="@T("Data Source Name")"
Class="mb-6"
MaxLength="40"
Counter="40"
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
Immediate="@true"
Validation="@this.dataSourceValidation.ValidatingName"
Adornment="Adornment.Start"
@@ -8,6 +8,7 @@ using AIStudio.Tools.ERIClient.DataModel;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG;
using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
using SharedTools;
@@ -168,9 +169,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
return false;
}
if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || string.IsNullOrWhiteSpace(name) || name.Length > 40 || name.Any(char.IsControl))
if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || !DataSourceValidation.IsNameValid(name))
{
LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most 40 characters without control characters. (Plugin ID: {configPluginId})");
LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most {DataSourceValidation.MAX_NAME_LENGTH} characters without control characters. (Plugin ID: {configPluginId})");
return false;
}
@@ -83,7 +83,7 @@ public sealed class QdrantEdgeClientImplementation(
}
public override async Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>( DATABASE_NAME, ENSURE_PATH,
await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>(DATABASE_NAME, ENSURE_PATH,
new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token) ?? throw new InvalidOperationException("The vector store ensure response was empty.");
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
+20 -9
View File
@@ -30,7 +30,9 @@ public static class FileTypes
public static readonly FileTypeFilter RUST = FileTypeFilter.Leaf("Rust", "rs");
public static readonly FileTypeFilter LUA = FileTypeFilter.Leaf("Lua", "lua");
public static readonly FileTypeFilter PHP = FileTypeFilter.Leaf("PHP", "php");
public static readonly FileTypeFilter WEB = FileTypeFilter.Leaf("HTML/CSS", "html", "css");
public static readonly FileTypeFilter HTML = FileTypeFilter.Leaf("HTML", "html", "htm");
public static readonly FileTypeFilter CSS = FileTypeFilter.Leaf("CSS", "css");
public static readonly FileTypeFilter WEB = FileTypeFilter.Parent("HTML/CSS", HTML, CSS);
/// <summary>
/// Gets the standalone HTML filter used for visual briefing import and export.
@@ -52,15 +54,18 @@ public static class FileTypes
public static readonly FileTypeFilter TEXT = FileTypeFilter.Leaf(TB("Text"), "txt", "md", "rtf");
public static readonly FileTypeFilter MS_WORD = FileTypeFilter.Leaf("Microsoft Word", "docx");
public static readonly FileTypeFilter WORD = FileTypeFilter.Composite("Word", ["odt"], MS_WORD);
public static readonly FileTypeFilter EXCEL = FileTypeFilter.Leaf("Excel", "xls", "xlsx");
public static readonly FileTypeFilter EXCEL = FileTypeFilter.Leaf("Excel", "xls", "xlsx", "xlsm", "xlsb", "xla", "xlam");
public static readonly FileTypeFilter OPEN_DOCUMENT_SPREADSHEET = FileTypeFilter.Leaf("OpenDocument Spreadsheet", "ods");
public static readonly FileTypeFilter SPREADSHEET = FileTypeFilter.Parent(TB("Spreadsheet"), EXCEL, OPEN_DOCUMENT_SPREADSHEET);
public static readonly FileTypeFilter DELIMITED_TABLE = FileTypeFilter.Leaf(TB("Delimited table"), "csv", "tsv");
public static readonly FileTypeFilter POWER_POINT = FileTypeFilter.Leaf("PowerPoint", "ppt", "pptx", "odp");
public static readonly FileTypeFilter MAIL = FileTypeFilter.Leaf(TB("Mail"), "eml", "msg", "mbox");
public static readonly FileTypeFilter LATEX = FileTypeFilter.Leaf("LaTeX", "tex", "bib", "sty", "cls", "log");
public static readonly FileTypeFilter OFFICE_FILES = FileTypeFilter.Parent(TB("Office Files"),
WORD, EXCEL, POWER_POINT, PDF);
WORD, SPREADSHEET, POWER_POINT, PDF);
public static readonly FileTypeFilter DOCUMENT = FileTypeFilter.Parent(TB("Document"),
TEXT, OFFICE_FILES, SOURCE_CODE, LATEX);
TEXT, OFFICE_FILES, SOURCE_CODE, LATEX, DELIMITED_TABLE);
// Media hierarchy
public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"),
@@ -81,6 +86,7 @@ public static class FileTypes
// Other standalone types
public static readonly FileTypeFilter CERTIFICATE_BUNDLE = FileTypeFilter.Leaf(TB("Certificate bundle"), "pem", "crt", "cer");
public static readonly FileTypeFilter EXECUTABLES = FileTypeFilter.Leaf(TB("Executable"), "exe", "app", "bin", "appimage");
public static readonly FileTypeFilter SHORTCUT = FileTypeFilter.Leaf(TB("Shortcut"), "lnk");
public static FileTypeFilter? AsOneFileType(params FileTypeFilter[]? types)
{
@@ -106,6 +112,14 @@ public static class FileTypes
.ToArray();
}
public static bool IsAllowedExtension(string extension, params FileTypeFilter[]? types)
{
if (types == null || types.Length == 0 || string.IsNullOrWhiteSpace(extension))
return false;
return OnlyAllowTypes(types).Contains(extension.TrimStart('.'), StringComparer.OrdinalIgnoreCase);
}
/// <summary>
/// Validates a file path against the provided filters.
/// Supports extension-based matching and source-like file names (e.g. Dockerfile).
@@ -116,11 +130,8 @@ public static class FileTypes
return false;
var extension = Path.GetExtension(filePath).TrimStart('.');
if (!string.IsNullOrWhiteSpace(extension))
{
if (OnlyAllowTypes(types).Contains(extension, StringComparer.OrdinalIgnoreCase))
return true;
}
if (IsAllowedExtension(extension, types))
return true;
var fileName = Path.GetFileName(filePath);
if (string.IsNullOrWhiteSpace(fileName))
@@ -17,11 +17,6 @@ public sealed partial class DataSourceEmbeddingService
private const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300;
private const bool IMAGE_EMBEDDING_ENABLED = false;
private static readonly string[] RAG_DELIMITED_TABLE_FILE_EXTENSIONS = ["csv", "tsv"];
private static readonly string[] RAG_SPREADSHEET_FILE_EXTENSIONS = ["ods", "xlsm", "xlsb"];
private static readonly string[] RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS = ["xla", "xlam"];
private static readonly string[] SKIPPED_RAG_FILE_EXTENSIONS = ["lnk"];
private enum RagFileIndexingDecision
{
INDEXABLE,
@@ -575,7 +570,7 @@ public sealed partial class DataSourceEmbeddingService
]);
if (this.IsSourceCodeFilePath(filePath))
return GetSourceCodeChunkingStrategy(filePath);
return GetSourceCodeChunkingStrategy();
return new("document", [
new("Page or extracted section", SplitBySourceSegments, true),
@@ -587,24 +582,13 @@ public sealed partial class DataSourceEmbeddingService
]);
}
private static ChunkingStrategy GetSourceCodeChunkingStrategy(string filePath)
{
var rules = new List<ChunkingRule>
{
private static ChunkingStrategy GetSourceCodeChunkingStrategy() =>
new("source-code", [
new("Extracted section", SplitBySourceSegments, true),
};
rules.AddRange(GetSourceCodeDelimiterRules(filePath));
rules.Add(new("Line break", SplitByLineBreaks));
rules.Add(new("Whitespace", SplitByWhitespace));
rules.Add(new("Hard cut", null));
return new("source-code", rules);
}
private static IReadOnlyList<ChunkingRule> GetSourceCodeDelimiterRules(string filePath) => Path.GetExtension(filePath).TrimStart('.') switch
{
_ => [],
};
new("Line break", SplitByLineBreaks),
new("Whitespace", SplitByWhitespace),
new("Hard cut", null),
]);
private static List<string> NormalizeSplitUnits(IReadOnlyList<string> units, string fallbackText)
{
@@ -875,16 +859,12 @@ public sealed partial class DataSourceEmbeddingService
private bool IsDelimitedTableFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
return RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
return FileTypes.IsAllowedPath(filePath, FileTypes.DELIMITED_TABLE);
}
private bool IsSpreadsheetFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
return FileTypes.IsAllowedPath(filePath, FileTypes.EXCEL)
|| RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|| RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
return FileTypes.IsAllowedPath(filePath, FileTypes.SPREADSHEET);
}
private bool IsSourceCodeFilePath(string filePath)
@@ -894,18 +874,12 @@ public sealed partial class DataSourceEmbeddingService
private bool IsHtmlFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
return extension.Equals("html", StringComparison.OrdinalIgnoreCase)
|| extension.Equals("htm", StringComparison.OrdinalIgnoreCase);
return FileTypes.IsAllowedPath(filePath, FileTypes.HTML);
}
private bool IsSupportedRagFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT)
|| RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|| RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|| RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT);
}
private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file)
@@ -942,8 +916,7 @@ public sealed partial class DataSourceEmbeddingService
private static bool IsSkippedRagFileName(string fileName)
{
var extension = Path.GetExtension(fileName).TrimStart('.');
return SKIPPED_RAG_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
return FileTypes.IsAllowedPath(fileName, FileTypes.SHORTCUT)
|| fileName.StartsWith(OFFICE_LOCK_FILE_PREFIX, StringComparison.Ordinal);
}
@@ -1126,9 +1099,6 @@ public sealed partial class DataSourceEmbeddingService
: null;
}
private string GetCollectionName(string dataSourceId) =>
DataSourceEmbeddingNames.GetCollectionName(dataSourceId);
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) =>
CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
@@ -11,7 +11,7 @@ public sealed partial class DataSourceEmbeddingService
EmbeddingStateClient? embeddingState,
CancellationToken token)
{
await this.DeleteCollectionAsync(this.GetCollectionName(dataSourceId), vectorStore, token);
await this.DeleteCollectionAsync(DataSourceEmbeddingNames.GetCollectionName(dataSourceId), vectorStore, token);
embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token);
if (!embeddingState.IsAvailable)
@@ -428,11 +428,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
return;
}
var collectionName = this.GetCollectionName(dataSource.Id);
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
var persistedManifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
if (persistedManifest.VectorSize > 0)
{
var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, persistedManifest.VectorSize, token);
var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedManifest.VectorSize, token);
if (ensureResult.Created)
{
logger.LogWarning(
@@ -685,7 +685,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
VectorStoreOptimizationTracker optimizationTracker,
CancellationToken token)
{
var collectionName = this.GetCollectionName(dataSource.Id);
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
logger.LogDebug(
"Resetting stored embeddings for file '{FilePath}' in collection '{CollectionName}' before re-indexing.",
file.FullName,
@@ -783,7 +783,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
if (manifest.VectorSize == 0)
{
token.ThrowIfCancellationRequested();
var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, vectorSize, token);
if (!ensureResult.Created)
{
logger.LogWarning(
@@ -792,7 +792,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Name,
dataSource.Id);
await vectorStore.DeleteVectorStore(collectionName, token);
ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, vectorSize, token);
if (!ensureResult.Created)
throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly.");
}
@@ -845,11 +845,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
batch.Clear();
}
private async Task<VectorStoreEnsureResult> EnsureCollectionExistsAsync(VectorStoreClient vectorStore, string collectionName, string dataSourceName, int vectorSize, CancellationToken token)
{
return await vectorStore.EnsureVectorStoreExists(collectionName, dataSourceName, vectorSize, token);
}
private async Task UpsertPointsAsync(
VectorStoreClient vectorStore,
string collectionName,
@@ -6,6 +6,7 @@ using AIStudio.Tools.Databases;
using AIStudio.Tools.Databases.EmbeddingState;
using AIStudio.Tools.Databases.VectorStore;
using AIStudio.Tools.RAG;
using AIStudio.Tools.Rust;
namespace AIStudio.Tools.Services;
@@ -371,13 +372,18 @@ public sealed class DataSourceLocalRetrievalService(
}
}
private static RetrievalContentType GetRetrievalContentType(string fileType) => fileType.TrimStart('.').ToLowerInvariant() switch
private static RetrievalContentType GetRetrievalContentType(string fileType)
{
"csv" or "tsv" or "ods" or "xls" or "xlsx" or "xlsm" or "xlsb" => RetrievalContentType.TEXT_SPREADSHEET,
"odp" or "ppt" or "pptx" => RetrievalContentType.TEXT_PRESENTATION,
"htm" or "html" => RetrievalContentType.TEXT_WEBSITE,
_ => RetrievalContentType.TEXT_DOCUMENT
};
if (FileTypes.IsAllowedExtension(fileType, FileTypes.DELIMITED_TABLE, FileTypes.SPREADSHEET))
return RetrievalContentType.TEXT_SPREADSHEET;
if (FileTypes.IsAllowedExtension(fileType, FileTypes.POWER_POINT))
return RetrievalContentType.TEXT_PRESENTATION;
return FileTypes.IsAllowedExtension(fileType, FileTypes.HTML)
? RetrievalContentType.TEXT_WEBSITE
: RetrievalContentType.TEXT_DOCUMENT;
}
private static string GetQueryText(IContent lastUserPrompt) => lastUserPrompt switch
{
@@ -8,7 +8,11 @@ namespace AIStudio.Tools.Validation;
public sealed class DataSourceValidation
{
public const int MAX_NAME_LENGTH = 40;
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceValidation).Namespace, nameof(DataSourceValidation));
public static bool IsNameValid(string name) => !string.IsNullOrWhiteSpace(name) && name.Length <= MAX_NAME_LENGTH && !name.Any(char.IsControl);
public Func<string> GetSecretStorageIssue { get; init; } = () => string.Empty;
@@ -116,10 +120,10 @@ public sealed class DataSourceValidation
public string? ValidatingName(string dataSourceName)
{
if(string.IsNullOrWhiteSpace(dataSourceName))
if (string.IsNullOrWhiteSpace(dataSourceName))
return TB("The name must not be empty.");
if (dataSourceName.Length > 40)
if (dataSourceName.Length > MAX_NAME_LENGTH)
return TB("The name must not exceed 40 characters.");
if (dataSourceName.Any(char.IsControl))