mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-08-24 20:52:11 +00:00
improved code and made sure components are reused if possible
This commit is contained in:
parent
d95487aa6e
commit
647c428ebf
@ -9598,12 +9598,18 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text"
|
|||||||
-- Office Files
|
-- Office Files
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office Files"
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office Files"
|
||||||
|
|
||||||
|
-- Spreadsheet
|
||||||
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1313839225"] = "Spreadsheet"
|
||||||
|
|
||||||
-- Executable
|
-- Executable
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1364437037"] = "Executable"
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1364437037"] = "Executable"
|
||||||
|
|
||||||
-- Mail
|
-- Mail
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1399880782"] = "Mail"
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1399880782"] = "Mail"
|
||||||
|
|
||||||
|
-- Delimited table
|
||||||
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1405737676"] = "Delimited table"
|
||||||
|
|
||||||
-- Source like
|
-- Source like
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1487238587"] = "Source like"
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1487238587"] = "Source like"
|
||||||
|
|
||||||
@ -9631,6 +9637,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2502277006"] = "Custom"
|
|||||||
-- Visual briefing image
|
-- Visual briefing image
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2505088878"] = "Visual briefing image"
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2505088878"] = "Visual briefing image"
|
||||||
|
|
||||||
|
-- Shortcut
|
||||||
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2547828883"] = "Shortcut"
|
||||||
|
|
||||||
-- Media
|
-- Media
|
||||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T3507473059"] = "Media"
|
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T3507473059"] = "Media"
|
||||||
|
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
@using AIStudio.Settings.DataModel
|
@using AIStudio.Settings.DataModel
|
||||||
|
@using AIStudio.Tools.Validation
|
||||||
@using AIStudio.Provider
|
@using AIStudio.Provider
|
||||||
@using AIStudio.Tools.ERIClient.DataModel
|
@using AIStudio.Tools.ERIClient.DataModel
|
||||||
@inherits MSGComponentBase
|
@inherits MSGComponentBase
|
||||||
@ -12,8 +13,8 @@
|
|||||||
@bind-Text="@this.dataName"
|
@bind-Text="@this.dataName"
|
||||||
Label="@T("Data Source Name")"
|
Label="@T("Data Source Name")"
|
||||||
Class="mb-6"
|
Class="mb-6"
|
||||||
MaxLength="40"
|
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||||
Counter="40"
|
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||||
Immediate="@true"
|
Immediate="@true"
|
||||||
Validation="@this.dataSourceValidation.ValidatingName"
|
Validation="@this.dataSourceValidation.ValidatingName"
|
||||||
Adornment="Adornment.Start"
|
Adornment="Adornment.Start"
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
@using AIStudio.Settings.DataModel
|
@using AIStudio.Settings.DataModel
|
||||||
|
@using AIStudio.Tools.Validation
|
||||||
@using AIStudio.Provider
|
@using AIStudio.Provider
|
||||||
@inherits MSGComponentBase
|
@inherits MSGComponentBase
|
||||||
|
|
||||||
@ -11,8 +12,8 @@
|
|||||||
@bind-Text="@this.dataName"
|
@bind-Text="@this.dataName"
|
||||||
Label="@T("Data Source Name")"
|
Label="@T("Data Source Name")"
|
||||||
Class="mb-6"
|
Class="mb-6"
|
||||||
MaxLength="40"
|
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||||
Counter="40"
|
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||||
Immediate="@true"
|
Immediate="@true"
|
||||||
Validation="@this.dataSourceValidation.ValidatingName"
|
Validation="@this.dataSourceValidation.ValidatingName"
|
||||||
Adornment="Adornment.Start"
|
Adornment="Adornment.Start"
|
||||||
|
|||||||
@ -1,4 +1,5 @@
|
|||||||
@using AIStudio.Settings.DataModel
|
@using AIStudio.Settings.DataModel
|
||||||
|
@using AIStudio.Tools.Validation
|
||||||
@using AIStudio.Provider
|
@using AIStudio.Provider
|
||||||
@inherits MSGComponentBase
|
@inherits MSGComponentBase
|
||||||
|
|
||||||
@ -11,8 +12,8 @@
|
|||||||
@bind-Text="@this.dataName"
|
@bind-Text="@this.dataName"
|
||||||
Label="@T("Data Source Name")"
|
Label="@T("Data Source Name")"
|
||||||
Class="mb-6"
|
Class="mb-6"
|
||||||
MaxLength="40"
|
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||||
Counter="40"
|
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
|
||||||
Immediate="@true"
|
Immediate="@true"
|
||||||
Validation="@this.dataSourceValidation.ValidatingName"
|
Validation="@this.dataSourceValidation.ValidatingName"
|
||||||
Adornment="Adornment.Start"
|
Adornment="Adornment.Start"
|
||||||
|
|||||||
@ -8,6 +8,7 @@ using AIStudio.Tools.ERIClient.DataModel;
|
|||||||
using AIStudio.Tools.PluginSystem;
|
using AIStudio.Tools.PluginSystem;
|
||||||
using AIStudio.Tools.RAG;
|
using AIStudio.Tools.RAG;
|
||||||
using AIStudio.Tools.Services;
|
using AIStudio.Tools.Services;
|
||||||
|
using AIStudio.Tools.Validation;
|
||||||
|
|
||||||
using SharedTools;
|
using SharedTools;
|
||||||
|
|
||||||
@ -168,9 +169,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
|
|||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || string.IsNullOrWhiteSpace(name) || name.Length > 40 || name.Any(char.IsControl))
|
if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || !DataSourceValidation.IsNameValid(name))
|
||||||
{
|
{
|
||||||
LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most 40 characters without control characters. (Plugin ID: {configPluginId})");
|
LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most {DataSourceValidation.MAX_NAME_LENGTH} characters without control characters. (Plugin ID: {configPluginId})");
|
||||||
return false;
|
return false;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -83,7 +83,7 @@ public sealed class QdrantEdgeClientImplementation(
|
|||||||
}
|
}
|
||||||
|
|
||||||
public override async Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
|
public override async Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
|
||||||
await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>( DATABASE_NAME, ENSURE_PATH,
|
await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>(DATABASE_NAME, ENSURE_PATH,
|
||||||
new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token) ?? throw new InvalidOperationException("The vector store ensure response was empty.");
|
new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token) ?? throw new InvalidOperationException("The vector store ensure response was empty.");
|
||||||
|
|
||||||
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
|
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
|
||||||
|
|||||||
@ -30,7 +30,9 @@ public static class FileTypes
|
|||||||
public static readonly FileTypeFilter RUST = FileTypeFilter.Leaf("Rust", "rs");
|
public static readonly FileTypeFilter RUST = FileTypeFilter.Leaf("Rust", "rs");
|
||||||
public static readonly FileTypeFilter LUA = FileTypeFilter.Leaf("Lua", "lua");
|
public static readonly FileTypeFilter LUA = FileTypeFilter.Leaf("Lua", "lua");
|
||||||
public static readonly FileTypeFilter PHP = FileTypeFilter.Leaf("PHP", "php");
|
public static readonly FileTypeFilter PHP = FileTypeFilter.Leaf("PHP", "php");
|
||||||
public static readonly FileTypeFilter WEB = FileTypeFilter.Leaf("HTML/CSS", "html", "css");
|
public static readonly FileTypeFilter HTML = FileTypeFilter.Leaf("HTML", "html", "htm");
|
||||||
|
public static readonly FileTypeFilter CSS = FileTypeFilter.Leaf("CSS", "css");
|
||||||
|
public static readonly FileTypeFilter WEB = FileTypeFilter.Parent("HTML/CSS", HTML, CSS);
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Gets the standalone HTML filter used for visual briefing import and export.
|
/// Gets the standalone HTML filter used for visual briefing import and export.
|
||||||
@ -52,15 +54,18 @@ public static class FileTypes
|
|||||||
public static readonly FileTypeFilter TEXT = FileTypeFilter.Leaf(TB("Text"), "txt", "md", "rtf");
|
public static readonly FileTypeFilter TEXT = FileTypeFilter.Leaf(TB("Text"), "txt", "md", "rtf");
|
||||||
public static readonly FileTypeFilter MS_WORD = FileTypeFilter.Leaf("Microsoft Word", "docx");
|
public static readonly FileTypeFilter MS_WORD = FileTypeFilter.Leaf("Microsoft Word", "docx");
|
||||||
public static readonly FileTypeFilter WORD = FileTypeFilter.Composite("Word", ["odt"], MS_WORD);
|
public static readonly FileTypeFilter WORD = FileTypeFilter.Composite("Word", ["odt"], MS_WORD);
|
||||||
public static readonly FileTypeFilter EXCEL = FileTypeFilter.Leaf("Excel", "xls", "xlsx");
|
public static readonly FileTypeFilter EXCEL = FileTypeFilter.Leaf("Excel", "xls", "xlsx", "xlsm", "xlsb", "xla", "xlam");
|
||||||
|
public static readonly FileTypeFilter OPEN_DOCUMENT_SPREADSHEET = FileTypeFilter.Leaf("OpenDocument Spreadsheet", "ods");
|
||||||
|
public static readonly FileTypeFilter SPREADSHEET = FileTypeFilter.Parent(TB("Spreadsheet"), EXCEL, OPEN_DOCUMENT_SPREADSHEET);
|
||||||
|
public static readonly FileTypeFilter DELIMITED_TABLE = FileTypeFilter.Leaf(TB("Delimited table"), "csv", "tsv");
|
||||||
public static readonly FileTypeFilter POWER_POINT = FileTypeFilter.Leaf("PowerPoint", "ppt", "pptx", "odp");
|
public static readonly FileTypeFilter POWER_POINT = FileTypeFilter.Leaf("PowerPoint", "ppt", "pptx", "odp");
|
||||||
public static readonly FileTypeFilter MAIL = FileTypeFilter.Leaf(TB("Mail"), "eml", "msg", "mbox");
|
public static readonly FileTypeFilter MAIL = FileTypeFilter.Leaf(TB("Mail"), "eml", "msg", "mbox");
|
||||||
public static readonly FileTypeFilter LATEX = FileTypeFilter.Leaf("LaTeX", "tex", "bib", "sty", "cls", "log");
|
public static readonly FileTypeFilter LATEX = FileTypeFilter.Leaf("LaTeX", "tex", "bib", "sty", "cls", "log");
|
||||||
|
|
||||||
public static readonly FileTypeFilter OFFICE_FILES = FileTypeFilter.Parent(TB("Office Files"),
|
public static readonly FileTypeFilter OFFICE_FILES = FileTypeFilter.Parent(TB("Office Files"),
|
||||||
WORD, EXCEL, POWER_POINT, PDF);
|
WORD, SPREADSHEET, POWER_POINT, PDF);
|
||||||
public static readonly FileTypeFilter DOCUMENT = FileTypeFilter.Parent(TB("Document"),
|
public static readonly FileTypeFilter DOCUMENT = FileTypeFilter.Parent(TB("Document"),
|
||||||
TEXT, OFFICE_FILES, SOURCE_CODE, LATEX);
|
TEXT, OFFICE_FILES, SOURCE_CODE, LATEX, DELIMITED_TABLE);
|
||||||
|
|
||||||
// Media hierarchy
|
// Media hierarchy
|
||||||
public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"),
|
public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"),
|
||||||
@ -81,6 +86,7 @@ public static class FileTypes
|
|||||||
// Other standalone types
|
// Other standalone types
|
||||||
public static readonly FileTypeFilter CERTIFICATE_BUNDLE = FileTypeFilter.Leaf(TB("Certificate bundle"), "pem", "crt", "cer");
|
public static readonly FileTypeFilter CERTIFICATE_BUNDLE = FileTypeFilter.Leaf(TB("Certificate bundle"), "pem", "crt", "cer");
|
||||||
public static readonly FileTypeFilter EXECUTABLES = FileTypeFilter.Leaf(TB("Executable"), "exe", "app", "bin", "appimage");
|
public static readonly FileTypeFilter EXECUTABLES = FileTypeFilter.Leaf(TB("Executable"), "exe", "app", "bin", "appimage");
|
||||||
|
public static readonly FileTypeFilter SHORTCUT = FileTypeFilter.Leaf(TB("Shortcut"), "lnk");
|
||||||
|
|
||||||
public static FileTypeFilter? AsOneFileType(params FileTypeFilter[]? types)
|
public static FileTypeFilter? AsOneFileType(params FileTypeFilter[]? types)
|
||||||
{
|
{
|
||||||
@ -106,6 +112,14 @@ public static class FileTypes
|
|||||||
.ToArray();
|
.ToArray();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public static bool IsAllowedExtension(string extension, params FileTypeFilter[]? types)
|
||||||
|
{
|
||||||
|
if (types == null || types.Length == 0 || string.IsNullOrWhiteSpace(extension))
|
||||||
|
return false;
|
||||||
|
|
||||||
|
return OnlyAllowTypes(types).Contains(extension.TrimStart('.'), StringComparer.OrdinalIgnoreCase);
|
||||||
|
}
|
||||||
|
|
||||||
/// <summary>
|
/// <summary>
|
||||||
/// Validates a file path against the provided filters.
|
/// Validates a file path against the provided filters.
|
||||||
/// Supports extension-based matching and source-like file names (e.g. Dockerfile).
|
/// Supports extension-based matching and source-like file names (e.g. Dockerfile).
|
||||||
@ -116,11 +130,8 @@ public static class FileTypes
|
|||||||
return false;
|
return false;
|
||||||
|
|
||||||
var extension = Path.GetExtension(filePath).TrimStart('.');
|
var extension = Path.GetExtension(filePath).TrimStart('.');
|
||||||
if (!string.IsNullOrWhiteSpace(extension))
|
if (IsAllowedExtension(extension, types))
|
||||||
{
|
return true;
|
||||||
if (OnlyAllowTypes(types).Contains(extension, StringComparer.OrdinalIgnoreCase))
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
var fileName = Path.GetFileName(filePath);
|
var fileName = Path.GetFileName(filePath);
|
||||||
if (string.IsNullOrWhiteSpace(fileName))
|
if (string.IsNullOrWhiteSpace(fileName))
|
||||||
|
|||||||
@ -17,11 +17,6 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
private const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300;
|
private const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300;
|
||||||
private const bool IMAGE_EMBEDDING_ENABLED = false;
|
private const bool IMAGE_EMBEDDING_ENABLED = false;
|
||||||
|
|
||||||
private static readonly string[] RAG_DELIMITED_TABLE_FILE_EXTENSIONS = ["csv", "tsv"];
|
|
||||||
private static readonly string[] RAG_SPREADSHEET_FILE_EXTENSIONS = ["ods", "xlsm", "xlsb"];
|
|
||||||
private static readonly string[] RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS = ["xla", "xlam"];
|
|
||||||
private static readonly string[] SKIPPED_RAG_FILE_EXTENSIONS = ["lnk"];
|
|
||||||
|
|
||||||
private enum RagFileIndexingDecision
|
private enum RagFileIndexingDecision
|
||||||
{
|
{
|
||||||
INDEXABLE,
|
INDEXABLE,
|
||||||
@ -575,7 +570,7 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
]);
|
]);
|
||||||
|
|
||||||
if (this.IsSourceCodeFilePath(filePath))
|
if (this.IsSourceCodeFilePath(filePath))
|
||||||
return GetSourceCodeChunkingStrategy(filePath);
|
return GetSourceCodeChunkingStrategy();
|
||||||
|
|
||||||
return new("document", [
|
return new("document", [
|
||||||
new("Page or extracted section", SplitBySourceSegments, true),
|
new("Page or extracted section", SplitBySourceSegments, true),
|
||||||
@ -587,24 +582,13 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
]);
|
]);
|
||||||
}
|
}
|
||||||
|
|
||||||
private static ChunkingStrategy GetSourceCodeChunkingStrategy(string filePath)
|
private static ChunkingStrategy GetSourceCodeChunkingStrategy() =>
|
||||||
{
|
new("source-code", [
|
||||||
var rules = new List<ChunkingRule>
|
|
||||||
{
|
|
||||||
new("Extracted section", SplitBySourceSegments, true),
|
new("Extracted section", SplitBySourceSegments, true),
|
||||||
};
|
new("Line break", SplitByLineBreaks),
|
||||||
rules.AddRange(GetSourceCodeDelimiterRules(filePath));
|
new("Whitespace", SplitByWhitespace),
|
||||||
rules.Add(new("Line break", SplitByLineBreaks));
|
new("Hard cut", null),
|
||||||
rules.Add(new("Whitespace", SplitByWhitespace));
|
]);
|
||||||
rules.Add(new("Hard cut", null));
|
|
||||||
|
|
||||||
return new("source-code", rules);
|
|
||||||
}
|
|
||||||
|
|
||||||
private static IReadOnlyList<ChunkingRule> GetSourceCodeDelimiterRules(string filePath) => Path.GetExtension(filePath).TrimStart('.') switch
|
|
||||||
{
|
|
||||||
_ => [],
|
|
||||||
};
|
|
||||||
|
|
||||||
private static List<string> NormalizeSplitUnits(IReadOnlyList<string> units, string fallbackText)
|
private static List<string> NormalizeSplitUnits(IReadOnlyList<string> units, string fallbackText)
|
||||||
{
|
{
|
||||||
@ -875,16 +859,12 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
|
|
||||||
private bool IsDelimitedTableFilePath(string filePath)
|
private bool IsDelimitedTableFilePath(string filePath)
|
||||||
{
|
{
|
||||||
var extension = Path.GetExtension(filePath).TrimStart('.');
|
return FileTypes.IsAllowedPath(filePath, FileTypes.DELIMITED_TABLE);
|
||||||
return RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool IsSpreadsheetFilePath(string filePath)
|
private bool IsSpreadsheetFilePath(string filePath)
|
||||||
{
|
{
|
||||||
var extension = Path.GetExtension(filePath).TrimStart('.');
|
return FileTypes.IsAllowedPath(filePath, FileTypes.SPREADSHEET);
|
||||||
return FileTypes.IsAllowedPath(filePath, FileTypes.EXCEL)
|
|
||||||
|| RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|
|
||||||
|| RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool IsSourceCodeFilePath(string filePath)
|
private bool IsSourceCodeFilePath(string filePath)
|
||||||
@ -894,18 +874,12 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
|
|
||||||
private bool IsHtmlFilePath(string filePath)
|
private bool IsHtmlFilePath(string filePath)
|
||||||
{
|
{
|
||||||
var extension = Path.GetExtension(filePath).TrimStart('.');
|
return FileTypes.IsAllowedPath(filePath, FileTypes.HTML);
|
||||||
return extension.Equals("html", StringComparison.OrdinalIgnoreCase)
|
|
||||||
|| extension.Equals("htm", StringComparison.OrdinalIgnoreCase);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private bool IsSupportedRagFilePath(string filePath)
|
private bool IsSupportedRagFilePath(string filePath)
|
||||||
{
|
{
|
||||||
var extension = Path.GetExtension(filePath).TrimStart('.');
|
return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT);
|
||||||
return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT)
|
|
||||||
|| RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|
|
||||||
|| RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|
|
||||||
|| RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file)
|
private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file)
|
||||||
@ -942,8 +916,7 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
|
|
||||||
private static bool IsSkippedRagFileName(string fileName)
|
private static bool IsSkippedRagFileName(string fileName)
|
||||||
{
|
{
|
||||||
var extension = Path.GetExtension(fileName).TrimStart('.');
|
return FileTypes.IsAllowedPath(fileName, FileTypes.SHORTCUT)
|
||||||
return SKIPPED_RAG_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|
|
||||||
|| fileName.StartsWith(OFFICE_LOCK_FILE_PREFIX, StringComparison.Ordinal);
|
|| fileName.StartsWith(OFFICE_LOCK_FILE_PREFIX, StringComparison.Ordinal);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -1126,9 +1099,6 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
: null;
|
: null;
|
||||||
}
|
}
|
||||||
|
|
||||||
private string GetCollectionName(string dataSourceId) =>
|
|
||||||
DataSourceEmbeddingNames.GetCollectionName(dataSourceId);
|
|
||||||
|
|
||||||
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) =>
|
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) =>
|
||||||
CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
|
CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
|
||||||
|
|
||||||
|
|||||||
@ -11,7 +11,7 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
EmbeddingStateClient? embeddingState,
|
EmbeddingStateClient? embeddingState,
|
||||||
CancellationToken token)
|
CancellationToken token)
|
||||||
{
|
{
|
||||||
await this.DeleteCollectionAsync(this.GetCollectionName(dataSourceId), vectorStore, token);
|
await this.DeleteCollectionAsync(DataSourceEmbeddingNames.GetCollectionName(dataSourceId), vectorStore, token);
|
||||||
|
|
||||||
embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token);
|
embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token);
|
||||||
if (!embeddingState.IsAvailable)
|
if (!embeddingState.IsAvailable)
|
||||||
|
|||||||
@ -428,11 +428,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
var collectionName = this.GetCollectionName(dataSource.Id);
|
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
|
||||||
var persistedManifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
|
var persistedManifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
|
||||||
if (persistedManifest.VectorSize > 0)
|
if (persistedManifest.VectorSize > 0)
|
||||||
{
|
{
|
||||||
var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, persistedManifest.VectorSize, token);
|
var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedManifest.VectorSize, token);
|
||||||
if (ensureResult.Created)
|
if (ensureResult.Created)
|
||||||
{
|
{
|
||||||
logger.LogWarning(
|
logger.LogWarning(
|
||||||
@ -685,7 +685,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
VectorStoreOptimizationTracker optimizationTracker,
|
VectorStoreOptimizationTracker optimizationTracker,
|
||||||
CancellationToken token)
|
CancellationToken token)
|
||||||
{
|
{
|
||||||
var collectionName = this.GetCollectionName(dataSource.Id);
|
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
|
||||||
logger.LogDebug(
|
logger.LogDebug(
|
||||||
"Resetting stored embeddings for file '{FilePath}' in collection '{CollectionName}' before re-indexing.",
|
"Resetting stored embeddings for file '{FilePath}' in collection '{CollectionName}' before re-indexing.",
|
||||||
file.FullName,
|
file.FullName,
|
||||||
@ -783,7 +783,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
if (manifest.VectorSize == 0)
|
if (manifest.VectorSize == 0)
|
||||||
{
|
{
|
||||||
token.ThrowIfCancellationRequested();
|
token.ThrowIfCancellationRequested();
|
||||||
var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
|
var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, vectorSize, token);
|
||||||
if (!ensureResult.Created)
|
if (!ensureResult.Created)
|
||||||
{
|
{
|
||||||
logger.LogWarning(
|
logger.LogWarning(
|
||||||
@ -792,7 +792,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
dataSource.Name,
|
dataSource.Name,
|
||||||
dataSource.Id);
|
dataSource.Id);
|
||||||
await vectorStore.DeleteVectorStore(collectionName, token);
|
await vectorStore.DeleteVectorStore(collectionName, token);
|
||||||
ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
|
ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, vectorSize, token);
|
||||||
if (!ensureResult.Created)
|
if (!ensureResult.Created)
|
||||||
throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly.");
|
throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly.");
|
||||||
}
|
}
|
||||||
@ -845,11 +845,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
batch.Clear();
|
batch.Clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<VectorStoreEnsureResult> EnsureCollectionExistsAsync(VectorStoreClient vectorStore, string collectionName, string dataSourceName, int vectorSize, CancellationToken token)
|
|
||||||
{
|
|
||||||
return await vectorStore.EnsureVectorStoreExists(collectionName, dataSourceName, vectorSize, token);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async Task UpsertPointsAsync(
|
private async Task UpsertPointsAsync(
|
||||||
VectorStoreClient vectorStore,
|
VectorStoreClient vectorStore,
|
||||||
string collectionName,
|
string collectionName,
|
||||||
|
|||||||
@ -6,6 +6,7 @@ using AIStudio.Tools.Databases;
|
|||||||
using AIStudio.Tools.Databases.EmbeddingState;
|
using AIStudio.Tools.Databases.EmbeddingState;
|
||||||
using AIStudio.Tools.Databases.VectorStore;
|
using AIStudio.Tools.Databases.VectorStore;
|
||||||
using AIStudio.Tools.RAG;
|
using AIStudio.Tools.RAG;
|
||||||
|
using AIStudio.Tools.Rust;
|
||||||
|
|
||||||
namespace AIStudio.Tools.Services;
|
namespace AIStudio.Tools.Services;
|
||||||
|
|
||||||
@ -371,13 +372,18 @@ public sealed class DataSourceLocalRetrievalService(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
private static RetrievalContentType GetRetrievalContentType(string fileType) => fileType.TrimStart('.').ToLowerInvariant() switch
|
private static RetrievalContentType GetRetrievalContentType(string fileType)
|
||||||
{
|
{
|
||||||
"csv" or "tsv" or "ods" or "xls" or "xlsx" or "xlsm" or "xlsb" => RetrievalContentType.TEXT_SPREADSHEET,
|
if (FileTypes.IsAllowedExtension(fileType, FileTypes.DELIMITED_TABLE, FileTypes.SPREADSHEET))
|
||||||
"odp" or "ppt" or "pptx" => RetrievalContentType.TEXT_PRESENTATION,
|
return RetrievalContentType.TEXT_SPREADSHEET;
|
||||||
"htm" or "html" => RetrievalContentType.TEXT_WEBSITE,
|
|
||||||
_ => RetrievalContentType.TEXT_DOCUMENT
|
if (FileTypes.IsAllowedExtension(fileType, FileTypes.POWER_POINT))
|
||||||
};
|
return RetrievalContentType.TEXT_PRESENTATION;
|
||||||
|
|
||||||
|
return FileTypes.IsAllowedExtension(fileType, FileTypes.HTML)
|
||||||
|
? RetrievalContentType.TEXT_WEBSITE
|
||||||
|
: RetrievalContentType.TEXT_DOCUMENT;
|
||||||
|
}
|
||||||
|
|
||||||
private static string GetQueryText(IContent lastUserPrompt) => lastUserPrompt switch
|
private static string GetQueryText(IContent lastUserPrompt) => lastUserPrompt switch
|
||||||
{
|
{
|
||||||
|
|||||||
@ -8,7 +8,11 @@ namespace AIStudio.Tools.Validation;
|
|||||||
|
|
||||||
public sealed class DataSourceValidation
|
public sealed class DataSourceValidation
|
||||||
{
|
{
|
||||||
|
public const int MAX_NAME_LENGTH = 40;
|
||||||
|
|
||||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceValidation).Namespace, nameof(DataSourceValidation));
|
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceValidation).Namespace, nameof(DataSourceValidation));
|
||||||
|
|
||||||
|
public static bool IsNameValid(string name) => !string.IsNullOrWhiteSpace(name) && name.Length <= MAX_NAME_LENGTH && !name.Any(char.IsControl);
|
||||||
|
|
||||||
public Func<string> GetSecretStorageIssue { get; init; } = () => string.Empty;
|
public Func<string> GetSecretStorageIssue { get; init; } = () => string.Empty;
|
||||||
|
|
||||||
@ -116,10 +120,10 @@ public sealed class DataSourceValidation
|
|||||||
|
|
||||||
public string? ValidatingName(string dataSourceName)
|
public string? ValidatingName(string dataSourceName)
|
||||||
{
|
{
|
||||||
if(string.IsNullOrWhiteSpace(dataSourceName))
|
if (string.IsNullOrWhiteSpace(dataSourceName))
|
||||||
return TB("The name must not be empty.");
|
return TB("The name must not be empty.");
|
||||||
|
|
||||||
if (dataSourceName.Length > 40)
|
if (dataSourceName.Length > MAX_NAME_LENGTH)
|
||||||
return TB("The name must not exceed 40 characters.");
|
return TB("The name must not exceed 40 characters.");
|
||||||
|
|
||||||
if (dataSourceName.Any(char.IsControl))
|
if (dataSourceName.Any(char.IsControl))
|
||||||
|
|||||||
@ -127,16 +127,10 @@ pub struct DeleteQdrantEdgeStoreRequest {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub struct QdrantEdgeOperationResponse {
|
pub struct QdrantEdgeResponse<T> {
|
||||||
pub success: bool,
|
pub success: bool,
|
||||||
pub issue: String,
|
pub issue: String,
|
||||||
}
|
pub data: Option<T>,
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub struct QdrantEdgeEnsureStoreResponse {
|
|
||||||
pub success: bool,
|
|
||||||
pub issue: String,
|
|
||||||
pub data: Option<QdrantEdgeEnsureStoreResult>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
@ -144,13 +138,6 @@ pub struct QdrantEdgeEnsureStoreResult {
|
|||||||
pub created: bool,
|
pub created: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Serialize)]
|
|
||||||
pub struct QdrantEdgeSearchResponse {
|
|
||||||
pub success: bool,
|
|
||||||
pub issue: String,
|
|
||||||
pub data: Vec<QdrantEdgeSearchResult>,
|
|
||||||
}
|
|
||||||
|
|
||||||
#[derive(Serialize)]
|
#[derive(Serialize)]
|
||||||
pub struct QdrantEdgeSearchResult {
|
pub struct QdrantEdgeSearchResult {
|
||||||
pub point_id: String,
|
pub point_id: String,
|
||||||
@ -203,23 +190,10 @@ impl QdrantEdgeDatabase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// To ensure a shard exists and that you can insert a vector
|
// To ensure a shard exists and that you can insert a vector
|
||||||
fn get_or_create_store(&mut self, store_name: &str, vector_size: usize) -> QdrantEdgeResult<&EdgeShard> {
|
fn get_or_create_store(&mut self, store_name: &str, vector_size: usize) -> QdrantEdgeResult<(&EdgeShard, bool)> {
|
||||||
let path = self.store_path(store_name)?;
|
let (path, is_initialized) = self.reconcile_store_state(store_name)?;
|
||||||
let is_initialized = store_is_initialized(&path, store_name)?;
|
|
||||||
if self.shards.contains_key(store_name) {
|
if self.shards.contains_key(store_name) {
|
||||||
if is_initialized {
|
return Ok((self.shards.get(store_name).unwrap(), false));
|
||||||
return Ok(self.shards.get(store_name).unwrap());
|
|
||||||
}
|
|
||||||
|
|
||||||
warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name);
|
|
||||||
self.shards.remove(store_name);
|
|
||||||
}
|
|
||||||
|
|
||||||
if path.exists() && !is_initialized {
|
|
||||||
warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before recreating it.", store_name);
|
|
||||||
fs::remove_dir_all(&path).map_err(|error| {
|
|
||||||
format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display())
|
|
||||||
})?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
let shard = if is_initialized {
|
let shard = if is_initialized {
|
||||||
@ -248,27 +222,14 @@ impl QdrantEdgeDatabase {
|
|||||||
};
|
};
|
||||||
|
|
||||||
self.shards.insert(store_name.to_string(), shard);
|
self.shards.insert(store_name.to_string(), shard);
|
||||||
Ok(self.shards.get(store_name).unwrap())
|
Ok((self.shards.get(store_name).unwrap(), !is_initialized))
|
||||||
}
|
}
|
||||||
|
|
||||||
// To check whether a shard exists so you can delete a file from it
|
// To check whether a shard exists so you can delete a file from it
|
||||||
fn get_existing_store(&mut self, store_name: &str) -> QdrantEdgeResult<Option<&EdgeShard>> {
|
fn get_existing_store(&mut self, store_name: &str) -> QdrantEdgeResult<Option<&EdgeShard>> {
|
||||||
let path = self.store_path(store_name)?;
|
let (path, is_initialized) = self.reconcile_store_state(store_name)?;
|
||||||
let is_initialized = store_is_initialized(&path, store_name)?;
|
|
||||||
if self.shards.contains_key(store_name) {
|
if self.shards.contains_key(store_name) {
|
||||||
if is_initialized {
|
return Ok(self.shards.get(store_name));
|
||||||
return Ok(self.shards.get(store_name));
|
|
||||||
}
|
|
||||||
|
|
||||||
warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name);
|
|
||||||
self.shards.remove(store_name);
|
|
||||||
}
|
|
||||||
|
|
||||||
if path.exists() && !is_initialized {
|
|
||||||
warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before continuing.", store_name);
|
|
||||||
fs::remove_dir_all(&path).map_err(|error| {
|
|
||||||
format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display())
|
|
||||||
})?;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if !is_initialized {
|
if !is_initialized {
|
||||||
@ -282,6 +243,25 @@ impl QdrantEdgeDatabase {
|
|||||||
Ok(self.shards.get(store_name))
|
Ok(self.shards.get(store_name))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn reconcile_store_state(&mut self, store_name: &str) -> QdrantEdgeResult<(PathBuf, bool)> {
|
||||||
|
let path = self.store_path(store_name)?;
|
||||||
|
let is_initialized = store_is_initialized(&path, store_name)?;
|
||||||
|
|
||||||
|
if self.shards.contains_key(store_name) && !is_initialized {
|
||||||
|
warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name);
|
||||||
|
self.shards.remove(store_name);
|
||||||
|
}
|
||||||
|
|
||||||
|
if path.exists() && !is_initialized {
|
||||||
|
warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before continuing.", store_name);
|
||||||
|
fs::remove_dir_all(&path).map_err(|error| {
|
||||||
|
format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display())
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok((path, is_initialized))
|
||||||
|
}
|
||||||
|
|
||||||
fn info(&self) -> QdrantEdgeResult<QdrantEdgeInfo> {
|
fn info(&self) -> QdrantEdgeResult<QdrantEdgeInfo> {
|
||||||
let stores_path = self.base_path.join("stores");
|
let stores_path = self.base_path.join("stores");
|
||||||
let stores_count = if stores_path.exists() {
|
let stores_count = if stores_path.exists() {
|
||||||
@ -305,11 +285,10 @@ impl QdrantEdgeDatabase {
|
|||||||
validate_vector_size(vector_size)?;
|
validate_vector_size(vector_size)?;
|
||||||
validate_data_source_name(data_source_name)?;
|
validate_data_source_name(data_source_name)?;
|
||||||
let store_path = self.store_path(store_name)?;
|
let store_path = self.store_path(store_name)?;
|
||||||
let store_existed = store_is_initialized(&store_path, store_name)?;
|
let (_, created) = self.get_or_create_store(store_name, vector_size)?;
|
||||||
self.get_or_create_store(store_name, vector_size)?;
|
|
||||||
write_store_display_name(&store_path, data_source_name)?;
|
write_store_display_name(&store_path, data_source_name)?;
|
||||||
Ok(QdrantEdgeEnsureStoreResult {
|
Ok(QdrantEdgeEnsureStoreResult {
|
||||||
created: !store_existed,
|
created,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -331,7 +310,7 @@ impl QdrantEdgeDatabase {
|
|||||||
}
|
}
|
||||||
|
|
||||||
let store_path = self.store_path(store_name)?;
|
let store_path = self.store_path(store_name)?;
|
||||||
let shard = self.get_or_create_store(store_name, vector_size)?;
|
let (shard, _) = self.get_or_create_store(store_name, vector_size)?;
|
||||||
write_store_display_name(&store_path, &data_source_name)?;
|
write_store_display_name(&store_path, &data_source_name)?;
|
||||||
let points = points
|
let points = points
|
||||||
.into_iter()
|
.into_iter()
|
||||||
@ -454,60 +433,38 @@ pub async fn qdrant_edge_info(_token: APIToken) -> Json<QdrantEdgeServiceInfo> {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn ensure_qdrant_edge_store(_token: APIToken, Json(request): Json<EnsureQdrantEdgeStoreRequest>) -> Json<QdrantEdgeEnsureStoreResponse> {
|
pub async fn ensure_qdrant_edge_store(_token: APIToken, Json(request): Json<EnsureQdrantEdgeStoreRequest>) -> Json<QdrantEdgeResponse<QdrantEdgeEnsureStoreResult>> {
|
||||||
let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap();
|
execute_qdrant_edge_request(|database| {
|
||||||
let Some(database) = database_guard.as_mut() else {
|
database.ensure_store_exists(&request.store_name, &request.data_source_name, request.vector_size)
|
||||||
return Json(QdrantEdgeEnsureStoreResponse {
|
})
|
||||||
success: false,
|
|
||||||
issue: "Qdrant Edge is not available.".to_string(),
|
|
||||||
data: None,
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
match database.ensure_store_exists(&request.store_name, &request.data_source_name, request.vector_size) {
|
|
||||||
Ok(result) => Json(QdrantEdgeEnsureStoreResponse {
|
|
||||||
success: true,
|
|
||||||
issue: String::new(),
|
|
||||||
data: Some(result),
|
|
||||||
}),
|
|
||||||
Err(error) => {
|
|
||||||
let issue = error.to_string();
|
|
||||||
error!(Source = "Qdrant Edge"; "Qdrant Edge operation failed: {issue}");
|
|
||||||
Json(QdrantEdgeEnsureStoreResponse {
|
|
||||||
success: false,
|
|
||||||
issue,
|
|
||||||
data: None,
|
|
||||||
})
|
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn insert_qdrant_edge_embedding(_token: APIToken, Json(request): Json<InsertQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeOperationResponse> {
|
pub async fn insert_qdrant_edge_embedding(_token: APIToken, Json(request): Json<InsertQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeResponse<()>> {
|
||||||
execute_qdrant_edge_operation(|database| {
|
execute_qdrant_edge_request(|database| {
|
||||||
database.insert_embedding(&request.store_name, request.points)
|
database.insert_embedding(&request.store_name, request.points)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn search_qdrant_edge_embeddings(_token: APIToken, Json(request): Json<SearchQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeSearchResponse> {
|
pub async fn search_qdrant_edge_embeddings(_token: APIToken, Json(request): Json<SearchQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeResponse<Vec<QdrantEdgeSearchResult>>> {
|
||||||
execute_qdrant_edge_query(|database| {
|
execute_qdrant_edge_request(|database| {
|
||||||
database.search_embedding(&request.store_name, request.vector, request.max_matches)
|
database.search_embedding(&request.store_name, request.vector, request.max_matches)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn delete_qdrant_edge_embedding_by_file(_token: APIToken, Json(request): Json<DeleteQdrantEdgeEmbeddingByFileRequest>) -> Json<QdrantEdgeOperationResponse> {
|
pub async fn delete_qdrant_edge_embedding_by_file(_token: APIToken, Json(request): Json<DeleteQdrantEdgeEmbeddingByFileRequest>) -> Json<QdrantEdgeResponse<()>> {
|
||||||
execute_qdrant_edge_operation(|database| {
|
execute_qdrant_edge_request(|database| {
|
||||||
database.delete_embedding_by_file(&request.store_name, &request.file_path)
|
database.delete_embedding_by_file(&request.store_name, &request.file_path)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn optimize_qdrant_edge_store(_token: APIToken, Json(request): Json<OptimizeQdrantEdgeStoreRequest>) -> Json<QdrantEdgeOperationResponse> {
|
pub async fn optimize_qdrant_edge_store(_token: APIToken, Json(request): Json<OptimizeQdrantEdgeStoreRequest>) -> Json<QdrantEdgeResponse<()>> {
|
||||||
execute_qdrant_edge_operation(|database| {
|
execute_qdrant_edge_request(|database| {
|
||||||
database.optimize_store(&request.store_name)
|
database.optimize_store(&request.store_name)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub async fn delete_qdrant_edge_store(_token: APIToken, Json(request): Json<DeleteQdrantEdgeStoreRequest>) -> Json<QdrantEdgeOperationResponse> {
|
pub async fn delete_qdrant_edge_store(_token: APIToken, Json(request): Json<DeleteQdrantEdgeStoreRequest>) -> Json<QdrantEdgeResponse<()>> {
|
||||||
execute_qdrant_edge_operation(|database| {
|
execute_qdrant_edge_request(|database| {
|
||||||
database.delete_store(&request.store_name)
|
database.delete_store(&request.store_name)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
@ -550,60 +507,33 @@ pub fn stop_qdrant_edge_database() {
|
|||||||
set_qdrant_edge_unavailable("Qdrant Edge was stopped.".to_string());
|
set_qdrant_edge_unavailable("Qdrant Edge was stopped.".to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
fn execute_qdrant_edge_operation<F>(operation: F) -> Json<QdrantEdgeOperationResponse>
|
fn execute_qdrant_edge_request<T, F>(operation: F) -> Json<QdrantEdgeResponse<T>>
|
||||||
where
|
where
|
||||||
F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult<()>,
|
T: Serialize,
|
||||||
|
F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult<T>,
|
||||||
{
|
{
|
||||||
let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap();
|
let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap();
|
||||||
let Some(database) = database_guard.as_mut() else {
|
let Some(database) = database_guard.as_mut() else {
|
||||||
return Json(QdrantEdgeOperationResponse {
|
return Json(QdrantEdgeResponse {
|
||||||
success: false,
|
success: false,
|
||||||
issue: "Qdrant Edge is not available.".to_string(),
|
issue: "Qdrant Edge is not available.".to_string(),
|
||||||
|
data: None,
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
match operation(database) {
|
match operation(database) {
|
||||||
Ok(_) => Json(QdrantEdgeOperationResponse {
|
Ok(data) => Json(QdrantEdgeResponse {
|
||||||
success: true,
|
success: true,
|
||||||
issue: String::new(),
|
issue: String::new(),
|
||||||
|
data: Some(data),
|
||||||
}),
|
}),
|
||||||
Err(e) => {
|
Err(e) => {
|
||||||
let issue = e.to_string();
|
let issue = e.to_string();
|
||||||
error!(Source = "Qdrant Edge"; "Qdrant Edge operation failed: {issue}");
|
error!(Source = "Qdrant Edge"; "Qdrant Edge request failed: {issue}");
|
||||||
Json(QdrantEdgeOperationResponse {
|
Json(QdrantEdgeResponse {
|
||||||
success: false,
|
success: false,
|
||||||
issue,
|
issue,
|
||||||
})
|
data: None,
|
||||||
},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
fn execute_qdrant_edge_query<F>(operation: F) -> Json<QdrantEdgeSearchResponse>
|
|
||||||
where
|
|
||||||
F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult<Vec<QdrantEdgeSearchResult>>,
|
|
||||||
{
|
|
||||||
let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap();
|
|
||||||
let Some(database) = database_guard.as_mut() else {
|
|
||||||
return Json(QdrantEdgeSearchResponse {
|
|
||||||
success: false,
|
|
||||||
issue: "Qdrant Edge is not available.".to_string(),
|
|
||||||
data: vec![],
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
match operation(database) {
|
|
||||||
Ok(data) => Json(QdrantEdgeSearchResponse {
|
|
||||||
success: true,
|
|
||||||
issue: String::new(),
|
|
||||||
data,
|
|
||||||
}),
|
|
||||||
Err(e) => {
|
|
||||||
let issue = e.to_string();
|
|
||||||
error!(Source = "Qdrant Edge"; "Qdrant Edge query failed: {issue}");
|
|
||||||
Json(QdrantEdgeSearchResponse {
|
|
||||||
success: false,
|
|
||||||
issue,
|
|
||||||
data: vec![],
|
|
||||||
})
|
})
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
@ -741,20 +671,21 @@ fn store_is_initialized(path: &Path, store_name: &str) -> QdrantEdgeResult<bool>
|
|||||||
}
|
}
|
||||||
|
|
||||||
fn write_store_initialization_marker(path: &Path, store_name: &str) -> std::io::Result<()> {
|
fn write_store_initialization_marker(path: &Path, store_name: &str) -> std::io::Result<()> {
|
||||||
let marker_path = path.join(STORE_INITIALIZATION_MARKER);
|
write_store_marker(path, STORE_INITIALIZATION_MARKER, STORE_INITIALIZATION_MARKER_TEMP, store_name)
|
||||||
let temporary_marker_path = path.join(STORE_INITIALIZATION_MARKER_TEMP);
|
|
||||||
fs::write(&temporary_marker_path, store_name)?;
|
|
||||||
fs::rename(temporary_marker_path, marker_path)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
fn write_store_display_name(path: &Path, data_source_name: &str) -> std::io::Result<()> {
|
fn write_store_display_name(path: &Path, data_source_name: &str) -> std::io::Result<()> {
|
||||||
let marker_path = path.join(STORE_DISPLAY_NAME_MARKER);
|
write_store_marker(path, STORE_DISPLAY_NAME_MARKER, STORE_DISPLAY_NAME_MARKER_TEMP, data_source_name)
|
||||||
if fs::read_to_string(&marker_path).is_ok_and(|current_name| current_name == data_source_name) {
|
}
|
||||||
|
|
||||||
|
fn write_store_marker(path: &Path, marker_name: &str, temporary_marker_name: &str, value: &str) -> std::io::Result<()> {
|
||||||
|
let marker_path = path.join(marker_name);
|
||||||
|
if fs::read_to_string(&marker_path).is_ok_and(|current_value| current_value == value) {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
|
|
||||||
let temporary_marker_path = path.join(STORE_DISPLAY_NAME_MARKER_TEMP);
|
let temporary_marker_path = path.join(temporary_marker_name);
|
||||||
fs::write(&temporary_marker_path, data_source_name)?;
|
fs::write(&temporary_marker_path, value)?;
|
||||||
if marker_path.exists() {
|
if marker_path.exists() {
|
||||||
fs::remove_file(&marker_path)?;
|
fs::remove_file(&marker_path)?;
|
||||||
}
|
}
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user