mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-08-16 00:02:12 +00:00
improved code and made sure components are reused if possible
This commit is contained in:
parent
d95487aa6e
commit
647c428ebf
@ -9598,12 +9598,18 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text"
|
||||
-- Office Files
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office Files"
|
||||
|
||||
-- Spreadsheet
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1313839225"] = "Spreadsheet"
|
||||
|
||||
-- Executable
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1364437037"] = "Executable"
|
||||
|
||||
-- Mail
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1399880782"] = "Mail"
|
||||
|
||||
-- Delimited table
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1405737676"] = "Delimited table"
|
||||
|
||||
-- Source like
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1487238587"] = "Source like"
|
||||
|
||||
@ -9631,6 +9637,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2502277006"] = "Custom"
|
||||
-- Visual briefing image
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2505088878"] = "Visual briefing image"
|
||||
|
||||
-- Shortcut
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2547828883"] = "Shortcut"
|
||||
|
||||
-- Media
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T3507473059"] = "Media"
|
||||
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
@using AIStudio.Settings.DataModel
|
||||
@using AIStudio.Tools.Validation
|
||||
@using AIStudio.Provider
|
||||
@using AIStudio.Tools.ERIClient.DataModel
|
||||
@inherits MSGComponentBase
|
||||
@ -12,8 +13,8 @@
|
||||
@bind-Text="@this.dataName"
|
||||
Label="@T("Data Source Name")"
|
||||
Class="mb-6"
|
||||
MaxLength="40"
|
||||
Counter="40"
|
||||
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||
Immediate="@true"
|
||||
Validation="@this.dataSourceValidation.ValidatingName"
|
||||
Adornment="Adornment.Start"
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
@using AIStudio.Settings.DataModel
|
||||
@using AIStudio.Tools.Validation
|
||||
@using AIStudio.Provider
|
||||
@inherits MSGComponentBase
|
||||
|
||||
@ -11,8 +12,8 @@
|
||||
@bind-Text="@this.dataName"
|
||||
Label="@T("Data Source Name")"
|
||||
Class="mb-6"
|
||||
MaxLength="40"
|
||||
Counter="40"
|
||||
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||
Immediate="@true"
|
||||
Validation="@this.dataSourceValidation.ValidatingName"
|
||||
Adornment="Adornment.Start"
|
||||
|
||||
@ -1,4 +1,5 @@
|
||||
@using AIStudio.Settings.DataModel
|
||||
@using AIStudio.Tools.Validation
|
||||
@using AIStudio.Provider
|
||||
@inherits MSGComponentBase
|
||||
|
||||
@ -11,8 +12,8 @@
|
||||
@bind-Text="@this.dataName"
|
||||
Label="@T("Data Source Name")"
|
||||
Class="mb-6"
|
||||
MaxLength="40"
|
||||
Counter="40"
|
||||
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||
Immediate="@true"
|
||||
Validation="@this.dataSourceValidation.ValidatingName"
|
||||
Adornment="Adornment.Start"
|
||||
|
||||
@ -8,6 +8,7 @@ using AIStudio.Tools.ERIClient.DataModel;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.RAG;
|
||||
using AIStudio.Tools.Services;
|
||||
using AIStudio.Tools.Validation;
|
||||
|
||||
using SharedTools;
|
||||
|
||||
@ -168,9 +169,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || string.IsNullOrWhiteSpace(name) || name.Length > 40 || name.Any(char.IsControl))
|
||||
if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || !DataSourceValidation.IsNameValid(name))
|
||||
{
|
||||
LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most 40 characters without control characters. (Plugin ID: {configPluginId})");
|
||||
LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most {DataSourceValidation.MAX_NAME_LENGTH} characters without control characters. (Plugin ID: {configPluginId})");
|
||||
return false;
|
||||
}
|
||||
|
||||
|
||||
@ -83,7 +83,7 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
}
|
||||
|
||||
public override async Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
|
||||
await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>( DATABASE_NAME, ENSURE_PATH,
|
||||
await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>(DATABASE_NAME, ENSURE_PATH,
|
||||
new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token) ?? throw new InvalidOperationException("The vector store ensure response was empty.");
|
||||
|
||||
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
|
||||
|
||||
@ -30,7 +30,9 @@ public static class FileTypes
|
||||
public static readonly FileTypeFilter RUST = FileTypeFilter.Leaf("Rust", "rs");
|
||||
public static readonly FileTypeFilter LUA = FileTypeFilter.Leaf("Lua", "lua");
|
||||
public static readonly FileTypeFilter PHP = FileTypeFilter.Leaf("PHP", "php");
|
||||
public static readonly FileTypeFilter WEB = FileTypeFilter.Leaf("HTML/CSS", "html", "css");
|
||||
public static readonly FileTypeFilter HTML = FileTypeFilter.Leaf("HTML", "html", "htm");
|
||||
public static readonly FileTypeFilter CSS = FileTypeFilter.Leaf("CSS", "css");
|
||||
public static readonly FileTypeFilter WEB = FileTypeFilter.Parent("HTML/CSS", HTML, CSS);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the standalone HTML filter used for visual briefing import and export.
|
||||
@ -52,15 +54,18 @@ public static class FileTypes
|
||||
public static readonly FileTypeFilter TEXT = FileTypeFilter.Leaf(TB("Text"), "txt", "md", "rtf");
|
||||
public static readonly FileTypeFilter MS_WORD = FileTypeFilter.Leaf("Microsoft Word", "docx");
|
||||
public static readonly FileTypeFilter WORD = FileTypeFilter.Composite("Word", ["odt"], MS_WORD);
|
||||
public static readonly FileTypeFilter EXCEL = FileTypeFilter.Leaf("Excel", "xls", "xlsx");
|
||||
public static readonly FileTypeFilter EXCEL = FileTypeFilter.Leaf("Excel", "xls", "xlsx", "xlsm", "xlsb", "xla", "xlam");
|
||||
public static readonly FileTypeFilter OPEN_DOCUMENT_SPREADSHEET = FileTypeFilter.Leaf("OpenDocument Spreadsheet", "ods");
|
||||
public static readonly FileTypeFilter SPREADSHEET = FileTypeFilter.Parent(TB("Spreadsheet"), EXCEL, OPEN_DOCUMENT_SPREADSHEET);
|
||||
public static readonly FileTypeFilter DELIMITED_TABLE = FileTypeFilter.Leaf(TB("Delimited table"), "csv", "tsv");
|
||||
public static readonly FileTypeFilter POWER_POINT = FileTypeFilter.Leaf("PowerPoint", "ppt", "pptx", "odp");
|
||||
public static readonly FileTypeFilter MAIL = FileTypeFilter.Leaf(TB("Mail"), "eml", "msg", "mbox");
|
||||
public static readonly FileTypeFilter LATEX = FileTypeFilter.Leaf("LaTeX", "tex", "bib", "sty", "cls", "log");
|
||||
|
||||
public static readonly FileTypeFilter OFFICE_FILES = FileTypeFilter.Parent(TB("Office Files"),
|
||||
WORD, EXCEL, POWER_POINT, PDF);
|
||||
WORD, SPREADSHEET, POWER_POINT, PDF);
|
||||
public static readonly FileTypeFilter DOCUMENT = FileTypeFilter.Parent(TB("Document"),
|
||||
TEXT, OFFICE_FILES, SOURCE_CODE, LATEX);
|
||||
TEXT, OFFICE_FILES, SOURCE_CODE, LATEX, DELIMITED_TABLE);
|
||||
|
||||
// Media hierarchy
|
||||
public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"),
|
||||
@ -81,6 +86,7 @@ public static class FileTypes
|
||||
// Other standalone types
|
||||
public static readonly FileTypeFilter CERTIFICATE_BUNDLE = FileTypeFilter.Leaf(TB("Certificate bundle"), "pem", "crt", "cer");
|
||||
public static readonly FileTypeFilter EXECUTABLES = FileTypeFilter.Leaf(TB("Executable"), "exe", "app", "bin", "appimage");
|
||||
public static readonly FileTypeFilter SHORTCUT = FileTypeFilter.Leaf(TB("Shortcut"), "lnk");
|
||||
|
||||
public static FileTypeFilter? AsOneFileType(params FileTypeFilter[]? types)
|
||||
{
|
||||
@ -106,6 +112,14 @@ public static class FileTypes
|
||||
.ToArray();
|
||||
}
|
||||
|
||||
public static bool IsAllowedExtension(string extension, params FileTypeFilter[]? types)
|
||||
{
|
||||
if (types == null || types.Length == 0 || string.IsNullOrWhiteSpace(extension))
|
||||
return false;
|
||||
|
||||
return OnlyAllowTypes(types).Contains(extension.TrimStart('.'), StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Validates a file path against the provided filters.
|
||||
/// Supports extension-based matching and source-like file names (e.g. Dockerfile).
|
||||
@ -116,11 +130,8 @@ public static class FileTypes
|
||||
return false;
|
||||
|
||||
var extension = Path.GetExtension(filePath).TrimStart('.');
|
||||
if (!string.IsNullOrWhiteSpace(extension))
|
||||
{
|
||||
if (OnlyAllowTypes(types).Contains(extension, StringComparer.OrdinalIgnoreCase))
|
||||
return true;
|
||||
}
|
||||
if (IsAllowedExtension(extension, types))
|
||||
return true;
|
||||
|
||||
var fileName = Path.GetFileName(filePath);
|
||||
if (string.IsNullOrWhiteSpace(fileName))
|
||||
|
||||
@ -17,11 +17,6 @@ public sealed partial class DataSourceEmbeddingService
|
||||
private const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300;
|
||||
private const bool IMAGE_EMBEDDING_ENABLED = false;
|
||||
|
||||
private static readonly string[] RAG_DELIMITED_TABLE_FILE_EXTENSIONS = ["csv", "tsv"];
|
||||
private static readonly string[] RAG_SPREADSHEET_FILE_EXTENSIONS = ["ods", "xlsm", "xlsb"];
|
||||
private static readonly string[] RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS = ["xla", "xlam"];
|
||||
private static readonly string[] SKIPPED_RAG_FILE_EXTENSIONS = ["lnk"];
|
||||
|
||||
private enum RagFileIndexingDecision
|
||||
{
|
||||
INDEXABLE,
|
||||
@ -575,7 +570,7 @@ public sealed partial class DataSourceEmbeddingService
|
||||
]);
|
||||
|
||||
if (this.IsSourceCodeFilePath(filePath))
|
||||
return GetSourceCodeChunkingStrategy(filePath);
|
||||
return GetSourceCodeChunkingStrategy();
|
||||
|
||||
return new("document", [
|
||||
new("Page or extracted section", SplitBySourceSegments, true),
|
||||
@ -587,24 +582,13 @@ public sealed partial class DataSourceEmbeddingService
|
||||
]);
|
||||
}
|
||||
|
||||
private static ChunkingStrategy GetSourceCodeChunkingStrategy(string filePath)
|
||||
{
|
||||
var rules = new List<ChunkingRule>
|
||||
{
|
||||
private static ChunkingStrategy GetSourceCodeChunkingStrategy() =>
|
||||
new("source-code", [
|
||||
new("Extracted section", SplitBySourceSegments, true),
|
||||
};
|
||||
rules.AddRange(GetSourceCodeDelimiterRules(filePath));
|
||||
rules.Add(new("Line break", SplitByLineBreaks));
|
||||
rules.Add(new("Whitespace", SplitByWhitespace));
|
||||
rules.Add(new("Hard cut", null));
|
||||
|
||||
return new("source-code", rules);
|
||||
}
|
||||
|
||||
private static IReadOnlyList<ChunkingRule> GetSourceCodeDelimiterRules(string filePath) => Path.GetExtension(filePath).TrimStart('.') switch
|
||||
{
|
||||
_ => [],
|
||||
};
|
||||
new("Line break", SplitByLineBreaks),
|
||||
new("Whitespace", SplitByWhitespace),
|
||||
new("Hard cut", null),
|
||||
]);
|
||||
|
||||
private static List<string> NormalizeSplitUnits(IReadOnlyList<string> units, string fallbackText)
|
||||
{
|
||||
@ -875,16 +859,12 @@ public sealed partial class DataSourceEmbeddingService
|
||||
|
||||
private bool IsDelimitedTableFilePath(string filePath)
|
||||
{
|
||||
var extension = Path.GetExtension(filePath).TrimStart('.');
|
||||
return RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
|
||||
return FileTypes.IsAllowedPath(filePath, FileTypes.DELIMITED_TABLE);
|
||||
}
|
||||
|
||||
private bool IsSpreadsheetFilePath(string filePath)
|
||||
{
|
||||
var extension = Path.GetExtension(filePath).TrimStart('.');
|
||||
return FileTypes.IsAllowedPath(filePath, FileTypes.EXCEL)
|
||||
|| RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|
||||
|| RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
|
||||
return FileTypes.IsAllowedPath(filePath, FileTypes.SPREADSHEET);
|
||||
}
|
||||
|
||||
private bool IsSourceCodeFilePath(string filePath)
|
||||
@ -894,18 +874,12 @@ public sealed partial class DataSourceEmbeddingService
|
||||
|
||||
private bool IsHtmlFilePath(string filePath)
|
||||
{
|
||||
var extension = Path.GetExtension(filePath).TrimStart('.');
|
||||
return extension.Equals("html", StringComparison.OrdinalIgnoreCase)
|
||||
|| extension.Equals("htm", StringComparison.OrdinalIgnoreCase);
|
||||
return FileTypes.IsAllowedPath(filePath, FileTypes.HTML);
|
||||
}
|
||||
|
||||
private bool IsSupportedRagFilePath(string filePath)
|
||||
{
|
||||
var extension = Path.GetExtension(filePath).TrimStart('.');
|
||||
return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT)
|
||||
|| RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|
||||
|| RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|
||||
|| RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
|
||||
return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT);
|
||||
}
|
||||
|
||||
private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file)
|
||||
@ -942,8 +916,7 @@ public sealed partial class DataSourceEmbeddingService
|
||||
|
||||
private static bool IsSkippedRagFileName(string fileName)
|
||||
{
|
||||
var extension = Path.GetExtension(fileName).TrimStart('.');
|
||||
return SKIPPED_RAG_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|
||||
return FileTypes.IsAllowedPath(fileName, FileTypes.SHORTCUT)
|
||||
|| fileName.StartsWith(OFFICE_LOCK_FILE_PREFIX, StringComparison.Ordinal);
|
||||
}
|
||||
|
||||
@ -1126,9 +1099,6 @@ public sealed partial class DataSourceEmbeddingService
|
||||
: null;
|
||||
}
|
||||
|
||||
private string GetCollectionName(string dataSourceId) =>
|
||||
DataSourceEmbeddingNames.GetCollectionName(dataSourceId);
|
||||
|
||||
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) =>
|
||||
CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@ public sealed partial class DataSourceEmbeddingService
|
||||
EmbeddingStateClient? embeddingState,
|
||||
CancellationToken token)
|
||||
{
|
||||
await this.DeleteCollectionAsync(this.GetCollectionName(dataSourceId), vectorStore, token);
|
||||
await this.DeleteCollectionAsync(DataSourceEmbeddingNames.GetCollectionName(dataSourceId), vectorStore, token);
|
||||
|
||||
embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token);
|
||||
if (!embeddingState.IsAvailable)
|
||||
|
||||
@ -428,11 +428,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
return;
|
||||
}
|
||||
|
||||
var collectionName = this.GetCollectionName(dataSource.Id);
|
||||
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
|
||||
var persistedManifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
|
||||
if (persistedManifest.VectorSize > 0)
|
||||
{
|
||||
var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, persistedManifest.VectorSize, token);
|
||||
var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedManifest.VectorSize, token);
|
||||
if (ensureResult.Created)
|
||||
{
|
||||
logger.LogWarning(
|
||||
@ -685,7 +685,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
VectorStoreOptimizationTracker optimizationTracker,
|
||||
CancellationToken token)
|
||||
{
|
||||
var collectionName = this.GetCollectionName(dataSource.Id);
|
||||
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
|
||||
logger.LogDebug(
|
||||
"Resetting stored embeddings for file '{FilePath}' in collection '{CollectionName}' before re-indexing.",
|
||||
file.FullName,
|
||||
@ -783,7 +783,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
if (manifest.VectorSize == 0)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
|
||||
var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, vectorSize, token);
|
||||
if (!ensureResult.Created)
|
||||
{
|
||||
logger.LogWarning(
|
||||
@ -792,7 +792,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
dataSource.Name,
|
||||
dataSource.Id);
|
||||
await vectorStore.DeleteVectorStore(collectionName, token);
|
||||
ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
|
||||
ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, vectorSize, token);
|
||||
if (!ensureResult.Created)
|
||||
throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly.");
|
||||
}
|
||||
@ -845,11 +845,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
batch.Clear();
|
||||
}
|
||||
|
||||
private async Task<VectorStoreEnsureResult> EnsureCollectionExistsAsync(VectorStoreClient vectorStore, string collectionName, string dataSourceName, int vectorSize, CancellationToken token)
|
||||
{
|
||||
return await vectorStore.EnsureVectorStoreExists(collectionName, dataSourceName, vectorSize, token);
|
||||
}
|
||||
|
||||
private async Task UpsertPointsAsync(
|
||||
VectorStoreClient vectorStore,
|
||||
string collectionName,
|
||||
|
||||
@ -6,6 +6,7 @@ using AIStudio.Tools.Databases;
|
||||
using AIStudio.Tools.Databases.EmbeddingState;
|
||||
using AIStudio.Tools.Databases.VectorStore;
|
||||
using AIStudio.Tools.RAG;
|
||||
using AIStudio.Tools.Rust;
|
||||
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
@ -371,13 +372,18 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
}
|
||||
}
|
||||
|
||||
private static RetrievalContentType GetRetrievalContentType(string fileType) => fileType.TrimStart('.').ToLowerInvariant() switch
|
||||
private static RetrievalContentType GetRetrievalContentType(string fileType)
|
||||
{
|
||||
"csv" or "tsv" or "ods" or "xls" or "xlsx" or "xlsm" or "xlsb" => RetrievalContentType.TEXT_SPREADSHEET,
|
||||
"odp" or "ppt" or "pptx" => RetrievalContentType.TEXT_PRESENTATION,
|
||||
"htm" or "html" => RetrievalContentType.TEXT_WEBSITE,
|
||||
_ => RetrievalContentType.TEXT_DOCUMENT
|
||||
};
|
||||
if (FileTypes.IsAllowedExtension(fileType, FileTypes.DELIMITED_TABLE, FileTypes.SPREADSHEET))
|
||||
return RetrievalContentType.TEXT_SPREADSHEET;
|
||||
|
||||
if (FileTypes.IsAllowedExtension(fileType, FileTypes.POWER_POINT))
|
||||
return RetrievalContentType.TEXT_PRESENTATION;
|
||||
|
||||
return FileTypes.IsAllowedExtension(fileType, FileTypes.HTML)
|
||||
? RetrievalContentType.TEXT_WEBSITE
|
||||
: RetrievalContentType.TEXT_DOCUMENT;
|
||||
}
|
||||
|
||||
private static string GetQueryText(IContent lastUserPrompt) => lastUserPrompt switch
|
||||
{
|
||||
|
||||
@ -8,7 +8,11 @@ namespace AIStudio.Tools.Validation;
|
||||
|
||||
public sealed class DataSourceValidation
|
||||
{
|
||||
public const int MAX_NAME_LENGTH = 40;
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceValidation).Namespace, nameof(DataSourceValidation));
|
||||
|
||||
public static bool IsNameValid(string name) => !string.IsNullOrWhiteSpace(name) && name.Length <= MAX_NAME_LENGTH && !name.Any(char.IsControl);
|
||||
|
||||
public Func<string> GetSecretStorageIssue { get; init; } = () => string.Empty;
|
||||
|
||||
@ -116,10 +120,10 @@ public sealed class DataSourceValidation
|
||||
|
||||
public string? ValidatingName(string dataSourceName)
|
||||
{
|
||||
if(string.IsNullOrWhiteSpace(dataSourceName))
|
||||
if (string.IsNullOrWhiteSpace(dataSourceName))
|
||||
return TB("The name must not be empty.");
|
||||
|
||||
if (dataSourceName.Length > 40)
|
||||
|
||||
if (dataSourceName.Length > MAX_NAME_LENGTH)
|
||||
return TB("The name must not exceed 40 characters.");
|
||||
|
||||
if (dataSourceName.Any(char.IsControl))
|
||||
|
||||
@ -127,16 +127,10 @@ pub struct DeleteQdrantEdgeStoreRequest {
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct QdrantEdgeOperationResponse {
|
||||
pub struct QdrantEdgeResponse<T> {
|
||||
pub success: bool,
|
||||
pub issue: String,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct QdrantEdgeEnsureStoreResponse {
|
||||
pub success: bool,
|
||||
pub issue: String,
|
||||
pub data: Option<QdrantEdgeEnsureStoreResult>,
|
||||
pub data: Option<T>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
@ -144,13 +138,6 @@ pub struct QdrantEdgeEnsureStoreResult {
|
||||
pub created: bool,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct QdrantEdgeSearchResponse {
|
||||
pub success: bool,
|
||||
pub issue: String,
|
||||
pub data: Vec<QdrantEdgeSearchResult>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct QdrantEdgeSearchResult {
|
||||
pub point_id: String,
|
||||
@ -203,23 +190,10 @@ impl QdrantEdgeDatabase {
|
||||
}
|
||||
|
||||
// To ensure a shard exists and that you can insert a vector
|
||||
fn get_or_create_store(&mut self, store_name: &str, vector_size: usize) -> QdrantEdgeResult<&EdgeShard> {
|
||||
let path = self.store_path(store_name)?;
|
||||
let is_initialized = store_is_initialized(&path, store_name)?;
|
||||
fn get_or_create_store(&mut self, store_name: &str, vector_size: usize) -> QdrantEdgeResult<(&EdgeShard, bool)> {
|
||||
let (path, is_initialized) = self.reconcile_store_state(store_name)?;
|
||||
if self.shards.contains_key(store_name) {
|
||||
if is_initialized {
|
||||
return Ok(self.shards.get(store_name).unwrap());
|
||||
}
|
||||
|
||||
warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name);
|
||||
self.shards.remove(store_name);
|
||||
}
|
||||
|
||||
if path.exists() && !is_initialized {
|
||||
warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before recreating it.", store_name);
|
||||
fs::remove_dir_all(&path).map_err(|error| {
|
||||
format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display())
|
||||
})?;
|
||||
return Ok((self.shards.get(store_name).unwrap(), false));
|
||||
}
|
||||
|
||||
let shard = if is_initialized {
|
||||
@ -248,27 +222,14 @@ impl QdrantEdgeDatabase {
|
||||
};
|
||||
|
||||
self.shards.insert(store_name.to_string(), shard);
|
||||
Ok(self.shards.get(store_name).unwrap())
|
||||
Ok((self.shards.get(store_name).unwrap(), !is_initialized))
|
||||
}
|
||||
|
||||
// To check whether a shard exists so you can delete a file from it
|
||||
fn get_existing_store(&mut self, store_name: &str) -> QdrantEdgeResult<Option<&EdgeShard>> {
|
||||
let path = self.store_path(store_name)?;
|
||||
let is_initialized = store_is_initialized(&path, store_name)?;
|
||||
let (path, is_initialized) = self.reconcile_store_state(store_name)?;
|
||||
if self.shards.contains_key(store_name) {
|
||||
if is_initialized {
|
||||
return Ok(self.shards.get(store_name));
|
||||
}
|
||||
|
||||
warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name);
|
||||
self.shards.remove(store_name);
|
||||
}
|
||||
|
||||
if path.exists() && !is_initialized {
|
||||
warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before continuing.", store_name);
|
||||
fs::remove_dir_all(&path).map_err(|error| {
|
||||
format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display())
|
||||
})?;
|
||||
return Ok(self.shards.get(store_name));
|
||||
}
|
||||
|
||||
if !is_initialized {
|
||||
@ -282,6 +243,25 @@ impl QdrantEdgeDatabase {
|
||||
Ok(self.shards.get(store_name))
|
||||
}
|
||||
|
||||
fn reconcile_store_state(&mut self, store_name: &str) -> QdrantEdgeResult<(PathBuf, bool)> {
|
||||
let path = self.store_path(store_name)?;
|
||||
let is_initialized = store_is_initialized(&path, store_name)?;
|
||||
|
||||
if self.shards.contains_key(store_name) && !is_initialized {
|
||||
warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name);
|
||||
self.shards.remove(store_name);
|
||||
}
|
||||
|
||||
if path.exists() && !is_initialized {
|
||||
warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before continuing.", store_name);
|
||||
fs::remove_dir_all(&path).map_err(|error| {
|
||||
format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display())
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok((path, is_initialized))
|
||||
}
|
||||
|
||||
fn info(&self) -> QdrantEdgeResult<QdrantEdgeInfo> {
|
||||
let stores_path = self.base_path.join("stores");
|
||||
let stores_count = if stores_path.exists() {
|
||||
@ -305,11 +285,10 @@ impl QdrantEdgeDatabase {
|
||||
validate_vector_size(vector_size)?;
|
||||
validate_data_source_name(data_source_name)?;
|
||||
let store_path = self.store_path(store_name)?;
|
||||
let store_existed = store_is_initialized(&store_path, store_name)?;
|
||||
self.get_or_create_store(store_name, vector_size)?;
|
||||
let (_, created) = self.get_or_create_store(store_name, vector_size)?;
|
||||
write_store_display_name(&store_path, data_source_name)?;
|
||||
Ok(QdrantEdgeEnsureStoreResult {
|
||||
created: !store_existed,
|
||||
created,
|
||||
})
|
||||
}
|
||||
|
||||
@ -331,7 +310,7 @@ impl QdrantEdgeDatabase {
|
||||
}
|
||||
|
||||
let store_path = self.store_path(store_name)?;
|
||||
let shard = self.get_or_create_store(store_name, vector_size)?;
|
||||
let (shard, _) = self.get_or_create_store(store_name, vector_size)?;
|
||||
write_store_display_name(&store_path, &data_source_name)?;
|
||||
let points = points
|
||||
.into_iter()
|
||||
@ -454,60 +433,38 @@ pub async fn qdrant_edge_info(_token: APIToken) -> Json<QdrantEdgeServiceInfo> {
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn ensure_qdrant_edge_store(_token: APIToken, Json(request): Json<EnsureQdrantEdgeStoreRequest>) -> Json<QdrantEdgeEnsureStoreResponse> {
|
||||
let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap();
|
||||
let Some(database) = database_guard.as_mut() else {
|
||||
return Json(QdrantEdgeEnsureStoreResponse {
|
||||
success: false,
|
||||
issue: "Qdrant Edge is not available.".to_string(),
|
||||
data: None,
|
||||
});
|
||||
};
|
||||
|
||||
match database.ensure_store_exists(&request.store_name, &request.data_source_name, request.vector_size) {
|
||||
Ok(result) => Json(QdrantEdgeEnsureStoreResponse {
|
||||
success: true,
|
||||
issue: String::new(),
|
||||
data: Some(result),
|
||||
}),
|
||||
Err(error) => {
|
||||
let issue = error.to_string();
|
||||
error!(Source = "Qdrant Edge"; "Qdrant Edge operation failed: {issue}");
|
||||
Json(QdrantEdgeEnsureStoreResponse {
|
||||
success: false,
|
||||
issue,
|
||||
data: None,
|
||||
})
|
||||
},
|
||||
}
|
||||
pub async fn ensure_qdrant_edge_store(_token: APIToken, Json(request): Json<EnsureQdrantEdgeStoreRequest>) -> Json<QdrantEdgeResponse<QdrantEdgeEnsureStoreResult>> {
|
||||
execute_qdrant_edge_request(|database| {
|
||||
database.ensure_store_exists(&request.store_name, &request.data_source_name, request.vector_size)
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn insert_qdrant_edge_embedding(_token: APIToken, Json(request): Json<InsertQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeOperationResponse> {
|
||||
execute_qdrant_edge_operation(|database| {
|
||||
pub async fn insert_qdrant_edge_embedding(_token: APIToken, Json(request): Json<InsertQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeResponse<()>> {
|
||||
execute_qdrant_edge_request(|database| {
|
||||
database.insert_embedding(&request.store_name, request.points)
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn search_qdrant_edge_embeddings(_token: APIToken, Json(request): Json<SearchQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeSearchResponse> {
|
||||
execute_qdrant_edge_query(|database| {
|
||||
pub async fn search_qdrant_edge_embeddings(_token: APIToken, Json(request): Json<SearchQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeResponse<Vec<QdrantEdgeSearchResult>>> {
|
||||
execute_qdrant_edge_request(|database| {
|
||||
database.search_embedding(&request.store_name, request.vector, request.max_matches)
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn delete_qdrant_edge_embedding_by_file(_token: APIToken, Json(request): Json<DeleteQdrantEdgeEmbeddingByFileRequest>) -> Json<QdrantEdgeOperationResponse> {
|
||||
execute_qdrant_edge_operation(|database| {
|
||||
pub async fn delete_qdrant_edge_embedding_by_file(_token: APIToken, Json(request): Json<DeleteQdrantEdgeEmbeddingByFileRequest>) -> Json<QdrantEdgeResponse<()>> {
|
||||
execute_qdrant_edge_request(|database| {
|
||||
database.delete_embedding_by_file(&request.store_name, &request.file_path)
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn optimize_qdrant_edge_store(_token: APIToken, Json(request): Json<OptimizeQdrantEdgeStoreRequest>) -> Json<QdrantEdgeOperationResponse> {
|
||||
execute_qdrant_edge_operation(|database| {
|
||||
pub async fn optimize_qdrant_edge_store(_token: APIToken, Json(request): Json<OptimizeQdrantEdgeStoreRequest>) -> Json<QdrantEdgeResponse<()>> {
|
||||
execute_qdrant_edge_request(|database| {
|
||||
database.optimize_store(&request.store_name)
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn delete_qdrant_edge_store(_token: APIToken, Json(request): Json<DeleteQdrantEdgeStoreRequest>) -> Json<QdrantEdgeOperationResponse> {
|
||||
execute_qdrant_edge_operation(|database| {
|
||||
pub async fn delete_qdrant_edge_store(_token: APIToken, Json(request): Json<DeleteQdrantEdgeStoreRequest>) -> Json<QdrantEdgeResponse<()>> {
|
||||
execute_qdrant_edge_request(|database| {
|
||||
database.delete_store(&request.store_name)
|
||||
})
|
||||
}
|
||||
@ -550,60 +507,33 @@ pub fn stop_qdrant_edge_database() {
|
||||
set_qdrant_edge_unavailable("Qdrant Edge was stopped.".to_string());
|
||||
}
|
||||
|
||||
fn execute_qdrant_edge_operation<F>(operation: F) -> Json<QdrantEdgeOperationResponse>
|
||||
fn execute_qdrant_edge_request<T, F>(operation: F) -> Json<QdrantEdgeResponse<T>>
|
||||
where
|
||||
F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult<()>,
|
||||
T: Serialize,
|
||||
F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult<T>,
|
||||
{
|
||||
let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap();
|
||||
let Some(database) = database_guard.as_mut() else {
|
||||
return Json(QdrantEdgeOperationResponse {
|
||||
return Json(QdrantEdgeResponse {
|
||||
success: false,
|
||||
issue: "Qdrant Edge is not available.".to_string(),
|
||||
data: None,
|
||||
});
|
||||
};
|
||||
|
||||
match operation(database) {
|
||||
Ok(_) => Json(QdrantEdgeOperationResponse {
|
||||
Ok(data) => Json(QdrantEdgeResponse {
|
||||
success: true,
|
||||
issue: String::new(),
|
||||
data: Some(data),
|
||||
}),
|
||||
Err(e) => {
|
||||
let issue = e.to_string();
|
||||
error!(Source = "Qdrant Edge"; "Qdrant Edge operation failed: {issue}");
|
||||
Json(QdrantEdgeOperationResponse {
|
||||
error!(Source = "Qdrant Edge"; "Qdrant Edge request failed: {issue}");
|
||||
Json(QdrantEdgeResponse {
|
||||
success: false,
|
||||
issue,
|
||||
})
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
fn execute_qdrant_edge_query<F>(operation: F) -> Json<QdrantEdgeSearchResponse>
|
||||
where
|
||||
F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult<Vec<QdrantEdgeSearchResult>>,
|
||||
{
|
||||
let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap();
|
||||
let Some(database) = database_guard.as_mut() else {
|
||||
return Json(QdrantEdgeSearchResponse {
|
||||
success: false,
|
||||
issue: "Qdrant Edge is not available.".to_string(),
|
||||
data: vec![],
|
||||
});
|
||||
};
|
||||
|
||||
match operation(database) {
|
||||
Ok(data) => Json(QdrantEdgeSearchResponse {
|
||||
success: true,
|
||||
issue: String::new(),
|
||||
data,
|
||||
}),
|
||||
Err(e) => {
|
||||
let issue = e.to_string();
|
||||
error!(Source = "Qdrant Edge"; "Qdrant Edge query failed: {issue}");
|
||||
Json(QdrantEdgeSearchResponse {
|
||||
success: false,
|
||||
issue,
|
||||
data: vec![],
|
||||
data: None,
|
||||
})
|
||||
},
|
||||
}
|
||||
@ -741,20 +671,21 @@ fn store_is_initialized(path: &Path, store_name: &str) -> QdrantEdgeResult<bool>
|
||||
}
|
||||
|
||||
fn write_store_initialization_marker(path: &Path, store_name: &str) -> std::io::Result<()> {
|
||||
let marker_path = path.join(STORE_INITIALIZATION_MARKER);
|
||||
let temporary_marker_path = path.join(STORE_INITIALIZATION_MARKER_TEMP);
|
||||
fs::write(&temporary_marker_path, store_name)?;
|
||||
fs::rename(temporary_marker_path, marker_path)
|
||||
write_store_marker(path, STORE_INITIALIZATION_MARKER, STORE_INITIALIZATION_MARKER_TEMP, store_name)
|
||||
}
|
||||
|
||||
fn write_store_display_name(path: &Path, data_source_name: &str) -> std::io::Result<()> {
|
||||
let marker_path = path.join(STORE_DISPLAY_NAME_MARKER);
|
||||
if fs::read_to_string(&marker_path).is_ok_and(|current_name| current_name == data_source_name) {
|
||||
write_store_marker(path, STORE_DISPLAY_NAME_MARKER, STORE_DISPLAY_NAME_MARKER_TEMP, data_source_name)
|
||||
}
|
||||
|
||||
fn write_store_marker(path: &Path, marker_name: &str, temporary_marker_name: &str, value: &str) -> std::io::Result<()> {
|
||||
let marker_path = path.join(marker_name);
|
||||
if fs::read_to_string(&marker_path).is_ok_and(|current_value| current_value == value) {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let temporary_marker_path = path.join(STORE_DISPLAY_NAME_MARKER_TEMP);
|
||||
fs::write(&temporary_marker_path, data_source_name)?;
|
||||
let temporary_marker_path = path.join(temporary_marker_name);
|
||||
fs::write(&temporary_marker_path, value)?;
|
||||
if marker_path.exists() {
|
||||
fs::remove_file(&marker_path)?;
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user