improved code and made sure components are reused if possible

This commit is contained in:
Paul Koudelka 2026-08-12 14:19:32 +02:00
parent d95487aa6e
commit 647c428ebf
13 changed files with 142 additions and 212 deletions

View File

@ -9598,12 +9598,18 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text"
-- Office Files
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1063218378"] = "Office Files"
-- Spreadsheet
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1313839225"] = "Spreadsheet"
-- Executable
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1364437037"] = "Executable"
-- Mail
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1399880782"] = "Mail"
-- Delimited table
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1405737676"] = "Delimited table"
-- Source like
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1487238587"] = "Source like"
@ -9631,6 +9637,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2502277006"] = "Custom"
-- Visual briefing image
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2505088878"] = "Visual briefing image"
-- Shortcut
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T2547828883"] = "Shortcut"
-- Media
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T3507473059"] = "Media"

View File

@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Tools.Validation
@using AIStudio.Provider
@using AIStudio.Tools.ERIClient.DataModel
@inherits MSGComponentBase
@ -12,8 +13,8 @@
@bind-Text="@this.dataName"
Label="@T("Data Source Name")"
Class="mb-6"
MaxLength="40"
Counter="40"
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
Immediate="@true"
Validation="@this.dataSourceValidation.ValidatingName"
Adornment="Adornment.Start"

View File

@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Tools.Validation
@using AIStudio.Provider
@inherits MSGComponentBase
@ -11,8 +12,8 @@
@bind-Text="@this.dataName"
Label="@T("Data Source Name")"
Class="mb-6"
MaxLength="40"
Counter="40"
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
Immediate="@true"
Validation="@this.dataSourceValidation.ValidatingName"
Adornment="Adornment.Start"

View File

@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Tools.Validation
@using AIStudio.Provider
@inherits MSGComponentBase
@ -11,8 +12,8 @@
@bind-Text="@this.dataName"
Label="@T("Data Source Name")"
Class="mb-6"
MaxLength="40"
Counter="40"
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
Immediate="@true"
Validation="@this.dataSourceValidation.ValidatingName"
Adornment="Adornment.Start"

View File

@ -8,6 +8,7 @@ using AIStudio.Tools.ERIClient.DataModel;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG;
using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
using SharedTools;
@ -168,9 +169,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
return false;
}
if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || string.IsNullOrWhiteSpace(name) || name.Length > 40 || name.Any(char.IsControl))
if (!table.TryGetValue("Name", out var nameValue) || !nameValue.TryRead<string>(out var name) || !DataSourceValidation.IsNameValid(name))
{
LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most 40 characters without control characters. (Plugin ID: {configPluginId})");
LOGGER.LogWarning($"The configured data source {idx} does not contain a valid name of at most {DataSourceValidation.MAX_NAME_LENGTH} characters without control characters. (Plugin ID: {configPluginId})");
return false;
}

View File

@ -83,7 +83,7 @@ public sealed class QdrantEdgeClientImplementation(
}
public override async Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>( DATABASE_NAME, ENSURE_PATH,
await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>(DATABASE_NAME, ENSURE_PATH,
new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token) ?? throw new InvalidOperationException("The vector store ensure response was empty.");
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>

View File

@ -30,7 +30,9 @@ public static class FileTypes
public static readonly FileTypeFilter RUST = FileTypeFilter.Leaf("Rust", "rs");
public static readonly FileTypeFilter LUA = FileTypeFilter.Leaf("Lua", "lua");
public static readonly FileTypeFilter PHP = FileTypeFilter.Leaf("PHP", "php");
public static readonly FileTypeFilter WEB = FileTypeFilter.Leaf("HTML/CSS", "html", "css");
public static readonly FileTypeFilter HTML = FileTypeFilter.Leaf("HTML", "html", "htm");
public static readonly FileTypeFilter CSS = FileTypeFilter.Leaf("CSS", "css");
public static readonly FileTypeFilter WEB = FileTypeFilter.Parent("HTML/CSS", HTML, CSS);
/// <summary>
/// Gets the standalone HTML filter used for visual briefing import and export.
@ -52,15 +54,18 @@ public static class FileTypes
public static readonly FileTypeFilter TEXT = FileTypeFilter.Leaf(TB("Text"), "txt", "md", "rtf");
public static readonly FileTypeFilter MS_WORD = FileTypeFilter.Leaf("Microsoft Word", "docx");
public static readonly FileTypeFilter WORD = FileTypeFilter.Composite("Word", ["odt"], MS_WORD);
public static readonly FileTypeFilter EXCEL = FileTypeFilter.Leaf("Excel", "xls", "xlsx");
public static readonly FileTypeFilter EXCEL = FileTypeFilter.Leaf("Excel", "xls", "xlsx", "xlsm", "xlsb", "xla", "xlam");
public static readonly FileTypeFilter OPEN_DOCUMENT_SPREADSHEET = FileTypeFilter.Leaf("OpenDocument Spreadsheet", "ods");
public static readonly FileTypeFilter SPREADSHEET = FileTypeFilter.Parent(TB("Spreadsheet"), EXCEL, OPEN_DOCUMENT_SPREADSHEET);
public static readonly FileTypeFilter DELIMITED_TABLE = FileTypeFilter.Leaf(TB("Delimited table"), "csv", "tsv");
public static readonly FileTypeFilter POWER_POINT = FileTypeFilter.Leaf("PowerPoint", "ppt", "pptx", "odp");
public static readonly FileTypeFilter MAIL = FileTypeFilter.Leaf(TB("Mail"), "eml", "msg", "mbox");
public static readonly FileTypeFilter LATEX = FileTypeFilter.Leaf("LaTeX", "tex", "bib", "sty", "cls", "log");
public static readonly FileTypeFilter OFFICE_FILES = FileTypeFilter.Parent(TB("Office Files"),
WORD, EXCEL, POWER_POINT, PDF);
WORD, SPREADSHEET, POWER_POINT, PDF);
public static readonly FileTypeFilter DOCUMENT = FileTypeFilter.Parent(TB("Document"),
TEXT, OFFICE_FILES, SOURCE_CODE, LATEX);
TEXT, OFFICE_FILES, SOURCE_CODE, LATEX, DELIMITED_TABLE);
// Media hierarchy
public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"),
@ -81,6 +86,7 @@ public static class FileTypes
// Other standalone types
public static readonly FileTypeFilter CERTIFICATE_BUNDLE = FileTypeFilter.Leaf(TB("Certificate bundle"), "pem", "crt", "cer");
public static readonly FileTypeFilter EXECUTABLES = FileTypeFilter.Leaf(TB("Executable"), "exe", "app", "bin", "appimage");
public static readonly FileTypeFilter SHORTCUT = FileTypeFilter.Leaf(TB("Shortcut"), "lnk");
public static FileTypeFilter? AsOneFileType(params FileTypeFilter[]? types)
{
@ -106,6 +112,14 @@ public static class FileTypes
.ToArray();
}
public static bool IsAllowedExtension(string extension, params FileTypeFilter[]? types)
{
if (types == null || types.Length == 0 || string.IsNullOrWhiteSpace(extension))
return false;
return OnlyAllowTypes(types).Contains(extension.TrimStart('.'), StringComparer.OrdinalIgnoreCase);
}
/// <summary>
/// Validates a file path against the provided filters.
/// Supports extension-based matching and source-like file names (e.g. Dockerfile).
@ -116,11 +130,8 @@ public static class FileTypes
return false;
var extension = Path.GetExtension(filePath).TrimStart('.');
if (!string.IsNullOrWhiteSpace(extension))
{
if (OnlyAllowTypes(types).Contains(extension, StringComparer.OrdinalIgnoreCase))
return true;
}
if (IsAllowedExtension(extension, types))
return true;
var fileName = Path.GetFileName(filePath);
if (string.IsNullOrWhiteSpace(fileName))

View File

@ -17,11 +17,6 @@ public sealed partial class DataSourceEmbeddingService
private const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300;
private const bool IMAGE_EMBEDDING_ENABLED = false;
private static readonly string[] RAG_DELIMITED_TABLE_FILE_EXTENSIONS = ["csv", "tsv"];
private static readonly string[] RAG_SPREADSHEET_FILE_EXTENSIONS = ["ods", "xlsm", "xlsb"];
private static readonly string[] RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS = ["xla", "xlam"];
private static readonly string[] SKIPPED_RAG_FILE_EXTENSIONS = ["lnk"];
private enum RagFileIndexingDecision
{
INDEXABLE,
@ -575,7 +570,7 @@ public sealed partial class DataSourceEmbeddingService
]);
if (this.IsSourceCodeFilePath(filePath))
return GetSourceCodeChunkingStrategy(filePath);
return GetSourceCodeChunkingStrategy();
return new("document", [
new("Page or extracted section", SplitBySourceSegments, true),
@ -587,24 +582,13 @@ public sealed partial class DataSourceEmbeddingService
]);
}
private static ChunkingStrategy GetSourceCodeChunkingStrategy(string filePath)
{
var rules = new List<ChunkingRule>
{
private static ChunkingStrategy GetSourceCodeChunkingStrategy() =>
new("source-code", [
new("Extracted section", SplitBySourceSegments, true),
};
rules.AddRange(GetSourceCodeDelimiterRules(filePath));
rules.Add(new("Line break", SplitByLineBreaks));
rules.Add(new("Whitespace", SplitByWhitespace));
rules.Add(new("Hard cut", null));
return new("source-code", rules);
}
private static IReadOnlyList<ChunkingRule> GetSourceCodeDelimiterRules(string filePath) => Path.GetExtension(filePath).TrimStart('.') switch
{
_ => [],
};
new("Line break", SplitByLineBreaks),
new("Whitespace", SplitByWhitespace),
new("Hard cut", null),
]);
private static List<string> NormalizeSplitUnits(IReadOnlyList<string> units, string fallbackText)
{
@ -875,16 +859,12 @@ public sealed partial class DataSourceEmbeddingService
private bool IsDelimitedTableFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
return RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
return FileTypes.IsAllowedPath(filePath, FileTypes.DELIMITED_TABLE);
}
private bool IsSpreadsheetFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
return FileTypes.IsAllowedPath(filePath, FileTypes.EXCEL)
|| RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|| RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
return FileTypes.IsAllowedPath(filePath, FileTypes.SPREADSHEET);
}
private bool IsSourceCodeFilePath(string filePath)
@ -894,18 +874,12 @@ public sealed partial class DataSourceEmbeddingService
private bool IsHtmlFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
return extension.Equals("html", StringComparison.OrdinalIgnoreCase)
|| extension.Equals("htm", StringComparison.OrdinalIgnoreCase);
return FileTypes.IsAllowedPath(filePath, FileTypes.HTML);
}
private bool IsSupportedRagFilePath(string filePath)
{
var extension = Path.GetExtension(filePath).TrimStart('.');
return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT)
|| RAG_DELIMITED_TABLE_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|| RAG_SPREADSHEET_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
|| RAG_SPREADSHEET_ADD_IN_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase);
return FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT);
}
private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file)
@ -942,8 +916,7 @@ public sealed partial class DataSourceEmbeddingService
private static bool IsSkippedRagFileName(string fileName)
{
var extension = Path.GetExtension(fileName).TrimStart('.');
return SKIPPED_RAG_FILE_EXTENSIONS.Contains(extension, StringComparer.OrdinalIgnoreCase)
return FileTypes.IsAllowedPath(fileName, FileTypes.SHORTCUT)
|| fileName.StartsWith(OFFICE_LOCK_FILE_PREFIX, StringComparison.Ordinal);
}
@ -1126,9 +1099,6 @@ public sealed partial class DataSourceEmbeddingService
: null;
}
private string GetCollectionName(string dataSourceId) =>
DataSourceEmbeddingNames.GetCollectionName(dataSourceId);
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) =>
CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");

View File

@ -11,7 +11,7 @@ public sealed partial class DataSourceEmbeddingService
EmbeddingStateClient? embeddingState,
CancellationToken token)
{
await this.DeleteCollectionAsync(this.GetCollectionName(dataSourceId), vectorStore, token);
await this.DeleteCollectionAsync(DataSourceEmbeddingNames.GetCollectionName(dataSourceId), vectorStore, token);
embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token);
if (!embeddingState.IsAvailable)

View File

@ -428,11 +428,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
return;
}
var collectionName = this.GetCollectionName(dataSource.Id);
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
var persistedManifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
if (persistedManifest.VectorSize > 0)
{
var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, persistedManifest.VectorSize, token);
var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, persistedManifest.VectorSize, token);
if (ensureResult.Created)
{
logger.LogWarning(
@ -685,7 +685,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
VectorStoreOptimizationTracker optimizationTracker,
CancellationToken token)
{
var collectionName = this.GetCollectionName(dataSource.Id);
var collectionName = DataSourceEmbeddingNames.GetCollectionName(dataSource.Id);
logger.LogDebug(
"Resetting stored embeddings for file '{FilePath}' in collection '{CollectionName}' before re-indexing.",
file.FullName,
@ -783,7 +783,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
if (manifest.VectorSize == 0)
{
token.ThrowIfCancellationRequested();
var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
var ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, vectorSize, token);
if (!ensureResult.Created)
{
logger.LogWarning(
@ -792,7 +792,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Name,
dataSource.Id);
await vectorStore.DeleteVectorStore(collectionName, token);
ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
ensureResult = await vectorStore.EnsureVectorStoreExists(collectionName, dataSource.Name, vectorSize, token);
if (!ensureResult.Created)
throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly.");
}
@ -845,11 +845,6 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
batch.Clear();
}
private async Task<VectorStoreEnsureResult> EnsureCollectionExistsAsync(VectorStoreClient vectorStore, string collectionName, string dataSourceName, int vectorSize, CancellationToken token)
{
return await vectorStore.EnsureVectorStoreExists(collectionName, dataSourceName, vectorSize, token);
}
private async Task UpsertPointsAsync(
VectorStoreClient vectorStore,
string collectionName,

View File

@ -6,6 +6,7 @@ using AIStudio.Tools.Databases;
using AIStudio.Tools.Databases.EmbeddingState;
using AIStudio.Tools.Databases.VectorStore;
using AIStudio.Tools.RAG;
using AIStudio.Tools.Rust;
namespace AIStudio.Tools.Services;
@ -371,13 +372,18 @@ public sealed class DataSourceLocalRetrievalService(
}
}
private static RetrievalContentType GetRetrievalContentType(string fileType) => fileType.TrimStart('.').ToLowerInvariant() switch
private static RetrievalContentType GetRetrievalContentType(string fileType)
{
"csv" or "tsv" or "ods" or "xls" or "xlsx" or "xlsm" or "xlsb" => RetrievalContentType.TEXT_SPREADSHEET,
"odp" or "ppt" or "pptx" => RetrievalContentType.TEXT_PRESENTATION,
"htm" or "html" => RetrievalContentType.TEXT_WEBSITE,
_ => RetrievalContentType.TEXT_DOCUMENT
};
if (FileTypes.IsAllowedExtension(fileType, FileTypes.DELIMITED_TABLE, FileTypes.SPREADSHEET))
return RetrievalContentType.TEXT_SPREADSHEET;
if (FileTypes.IsAllowedExtension(fileType, FileTypes.POWER_POINT))
return RetrievalContentType.TEXT_PRESENTATION;
return FileTypes.IsAllowedExtension(fileType, FileTypes.HTML)
? RetrievalContentType.TEXT_WEBSITE
: RetrievalContentType.TEXT_DOCUMENT;
}
private static string GetQueryText(IContent lastUserPrompt) => lastUserPrompt switch
{

View File

@ -8,7 +8,11 @@ namespace AIStudio.Tools.Validation;
public sealed class DataSourceValidation
{
public const int MAX_NAME_LENGTH = 40;
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceValidation).Namespace, nameof(DataSourceValidation));
public static bool IsNameValid(string name) => !string.IsNullOrWhiteSpace(name) && name.Length <= MAX_NAME_LENGTH && !name.Any(char.IsControl);
public Func<string> GetSecretStorageIssue { get; init; } = () => string.Empty;
@ -116,10 +120,10 @@ public sealed class DataSourceValidation
public string? ValidatingName(string dataSourceName)
{
if(string.IsNullOrWhiteSpace(dataSourceName))
if (string.IsNullOrWhiteSpace(dataSourceName))
return TB("The name must not be empty.");
if (dataSourceName.Length > 40)
if (dataSourceName.Length > MAX_NAME_LENGTH)
return TB("The name must not exceed 40 characters.");
if (dataSourceName.Any(char.IsControl))

View File

@ -127,16 +127,10 @@ pub struct DeleteQdrantEdgeStoreRequest {
}
#[derive(Serialize)]
pub struct QdrantEdgeOperationResponse {
pub struct QdrantEdgeResponse<T> {
pub success: bool,
pub issue: String,
}
#[derive(Serialize)]
pub struct QdrantEdgeEnsureStoreResponse {
pub success: bool,
pub issue: String,
pub data: Option<QdrantEdgeEnsureStoreResult>,
pub data: Option<T>,
}
#[derive(Serialize)]
@ -144,13 +138,6 @@ pub struct QdrantEdgeEnsureStoreResult {
pub created: bool,
}
#[derive(Serialize)]
pub struct QdrantEdgeSearchResponse {
pub success: bool,
pub issue: String,
pub data: Vec<QdrantEdgeSearchResult>,
}
#[derive(Serialize)]
pub struct QdrantEdgeSearchResult {
pub point_id: String,
@ -203,23 +190,10 @@ impl QdrantEdgeDatabase {
}
// To ensure a shard exists and that you can insert a vector
fn get_or_create_store(&mut self, store_name: &str, vector_size: usize) -> QdrantEdgeResult<&EdgeShard> {
let path = self.store_path(store_name)?;
let is_initialized = store_is_initialized(&path, store_name)?;
fn get_or_create_store(&mut self, store_name: &str, vector_size: usize) -> QdrantEdgeResult<(&EdgeShard, bool)> {
let (path, is_initialized) = self.reconcile_store_state(store_name)?;
if self.shards.contains_key(store_name) {
if is_initialized {
return Ok(self.shards.get(store_name).unwrap());
}
warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name);
self.shards.remove(store_name);
}
if path.exists() && !is_initialized {
warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before recreating it.", store_name);
fs::remove_dir_all(&path).map_err(|error| {
format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display())
})?;
return Ok((self.shards.get(store_name).unwrap(), false));
}
let shard = if is_initialized {
@ -248,27 +222,14 @@ impl QdrantEdgeDatabase {
};
self.shards.insert(store_name.to_string(), shard);
Ok(self.shards.get(store_name).unwrap())
Ok((self.shards.get(store_name).unwrap(), !is_initialized))
}
// To check whether a shard exists so you can delete a file from it
fn get_existing_store(&mut self, store_name: &str) -> QdrantEdgeResult<Option<&EdgeShard>> {
let path = self.store_path(store_name)?;
let is_initialized = store_is_initialized(&path, store_name)?;
let (path, is_initialized) = self.reconcile_store_state(store_name)?;
if self.shards.contains_key(store_name) {
if is_initialized {
return Ok(self.shards.get(store_name));
}
warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name);
self.shards.remove(store_name);
}
if path.exists() && !is_initialized {
warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before continuing.", store_name);
fs::remove_dir_all(&path).map_err(|error| {
format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display())
})?;
return Ok(self.shards.get(store_name));
}
if !is_initialized {
@ -282,6 +243,25 @@ impl QdrantEdgeDatabase {
Ok(self.shards.get(store_name))
}
fn reconcile_store_state(&mut self, store_name: &str) -> QdrantEdgeResult<(PathBuf, bool)> {
let path = self.store_path(store_name)?;
let is_initialized = store_is_initialized(&path, store_name)?;
if self.shards.contains_key(store_name) && !is_initialized {
warn!(Source = "Qdrant Edge"; "Removing stale cached vector store '{}' because its initialized data directory no longer exists.", store_name);
self.shards.remove(store_name);
}
if path.exists() && !is_initialized {
warn!(Source = "Qdrant Edge"; "Removing incompletely initialized vector store '{}' before continuing.", store_name);
fs::remove_dir_all(&path).map_err(|error| {
format!("Failed to remove incomplete vector store '{store_name}' at '{}': {error}", path.display())
})?;
}
Ok((path, is_initialized))
}
fn info(&self) -> QdrantEdgeResult<QdrantEdgeInfo> {
let stores_path = self.base_path.join("stores");
let stores_count = if stores_path.exists() {
@ -305,11 +285,10 @@ impl QdrantEdgeDatabase {
validate_vector_size(vector_size)?;
validate_data_source_name(data_source_name)?;
let store_path = self.store_path(store_name)?;
let store_existed = store_is_initialized(&store_path, store_name)?;
self.get_or_create_store(store_name, vector_size)?;
let (_, created) = self.get_or_create_store(store_name, vector_size)?;
write_store_display_name(&store_path, data_source_name)?;
Ok(QdrantEdgeEnsureStoreResult {
created: !store_existed,
created,
})
}
@ -331,7 +310,7 @@ impl QdrantEdgeDatabase {
}
let store_path = self.store_path(store_name)?;
let shard = self.get_or_create_store(store_name, vector_size)?;
let (shard, _) = self.get_or_create_store(store_name, vector_size)?;
write_store_display_name(&store_path, &data_source_name)?;
let points = points
.into_iter()
@ -454,60 +433,38 @@ pub async fn qdrant_edge_info(_token: APIToken) -> Json<QdrantEdgeServiceInfo> {
})
}
pub async fn ensure_qdrant_edge_store(_token: APIToken, Json(request): Json<EnsureQdrantEdgeStoreRequest>) -> Json<QdrantEdgeEnsureStoreResponse> {
let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap();
let Some(database) = database_guard.as_mut() else {
return Json(QdrantEdgeEnsureStoreResponse {
success: false,
issue: "Qdrant Edge is not available.".to_string(),
data: None,
});
};
match database.ensure_store_exists(&request.store_name, &request.data_source_name, request.vector_size) {
Ok(result) => Json(QdrantEdgeEnsureStoreResponse {
success: true,
issue: String::new(),
data: Some(result),
}),
Err(error) => {
let issue = error.to_string();
error!(Source = "Qdrant Edge"; "Qdrant Edge operation failed: {issue}");
Json(QdrantEdgeEnsureStoreResponse {
success: false,
issue,
data: None,
})
},
}
pub async fn ensure_qdrant_edge_store(_token: APIToken, Json(request): Json<EnsureQdrantEdgeStoreRequest>) -> Json<QdrantEdgeResponse<QdrantEdgeEnsureStoreResult>> {
execute_qdrant_edge_request(|database| {
database.ensure_store_exists(&request.store_name, &request.data_source_name, request.vector_size)
})
}
pub async fn insert_qdrant_edge_embedding(_token: APIToken, Json(request): Json<InsertQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeOperationResponse> {
execute_qdrant_edge_operation(|database| {
pub async fn insert_qdrant_edge_embedding(_token: APIToken, Json(request): Json<InsertQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeResponse<()>> {
execute_qdrant_edge_request(|database| {
database.insert_embedding(&request.store_name, request.points)
})
}
pub async fn search_qdrant_edge_embeddings(_token: APIToken, Json(request): Json<SearchQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeSearchResponse> {
execute_qdrant_edge_query(|database| {
pub async fn search_qdrant_edge_embeddings(_token: APIToken, Json(request): Json<SearchQdrantEdgeEmbeddingRequest>) -> Json<QdrantEdgeResponse<Vec<QdrantEdgeSearchResult>>> {
execute_qdrant_edge_request(|database| {
database.search_embedding(&request.store_name, request.vector, request.max_matches)
})
}
pub async fn delete_qdrant_edge_embedding_by_file(_token: APIToken, Json(request): Json<DeleteQdrantEdgeEmbeddingByFileRequest>) -> Json<QdrantEdgeOperationResponse> {
execute_qdrant_edge_operation(|database| {
pub async fn delete_qdrant_edge_embedding_by_file(_token: APIToken, Json(request): Json<DeleteQdrantEdgeEmbeddingByFileRequest>) -> Json<QdrantEdgeResponse<()>> {
execute_qdrant_edge_request(|database| {
database.delete_embedding_by_file(&request.store_name, &request.file_path)
})
}
pub async fn optimize_qdrant_edge_store(_token: APIToken, Json(request): Json<OptimizeQdrantEdgeStoreRequest>) -> Json<QdrantEdgeOperationResponse> {
execute_qdrant_edge_operation(|database| {
pub async fn optimize_qdrant_edge_store(_token: APIToken, Json(request): Json<OptimizeQdrantEdgeStoreRequest>) -> Json<QdrantEdgeResponse<()>> {
execute_qdrant_edge_request(|database| {
database.optimize_store(&request.store_name)
})
}
pub async fn delete_qdrant_edge_store(_token: APIToken, Json(request): Json<DeleteQdrantEdgeStoreRequest>) -> Json<QdrantEdgeOperationResponse> {
execute_qdrant_edge_operation(|database| {
pub async fn delete_qdrant_edge_store(_token: APIToken, Json(request): Json<DeleteQdrantEdgeStoreRequest>) -> Json<QdrantEdgeResponse<()>> {
execute_qdrant_edge_request(|database| {
database.delete_store(&request.store_name)
})
}
@ -550,60 +507,33 @@ pub fn stop_qdrant_edge_database() {
set_qdrant_edge_unavailable("Qdrant Edge was stopped.".to_string());
}
fn execute_qdrant_edge_operation<F>(operation: F) -> Json<QdrantEdgeOperationResponse>
fn execute_qdrant_edge_request<T, F>(operation: F) -> Json<QdrantEdgeResponse<T>>
where
F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult<()>,
T: Serialize,
F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult<T>,
{
let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap();
let Some(database) = database_guard.as_mut() else {
return Json(QdrantEdgeOperationResponse {
return Json(QdrantEdgeResponse {
success: false,
issue: "Qdrant Edge is not available.".to_string(),
data: None,
});
};
match operation(database) {
Ok(_) => Json(QdrantEdgeOperationResponse {
Ok(data) => Json(QdrantEdgeResponse {
success: true,
issue: String::new(),
data: Some(data),
}),
Err(e) => {
let issue = e.to_string();
error!(Source = "Qdrant Edge"; "Qdrant Edge operation failed: {issue}");
Json(QdrantEdgeOperationResponse {
error!(Source = "Qdrant Edge"; "Qdrant Edge request failed: {issue}");
Json(QdrantEdgeResponse {
success: false,
issue,
})
},
}
}
fn execute_qdrant_edge_query<F>(operation: F) -> Json<QdrantEdgeSearchResponse>
where
F: FnOnce(&mut QdrantEdgeDatabase) -> QdrantEdgeResult<Vec<QdrantEdgeSearchResult>>,
{
let mut database_guard = QDRANT_EDGE_DATABASE.lock().unwrap();
let Some(database) = database_guard.as_mut() else {
return Json(QdrantEdgeSearchResponse {
success: false,
issue: "Qdrant Edge is not available.".to_string(),
data: vec![],
});
};
match operation(database) {
Ok(data) => Json(QdrantEdgeSearchResponse {
success: true,
issue: String::new(),
data,
}),
Err(e) => {
let issue = e.to_string();
error!(Source = "Qdrant Edge"; "Qdrant Edge query failed: {issue}");
Json(QdrantEdgeSearchResponse {
success: false,
issue,
data: vec![],
data: None,
})
},
}
@ -741,20 +671,21 @@ fn store_is_initialized(path: &Path, store_name: &str) -> QdrantEdgeResult<bool>
}
fn write_store_initialization_marker(path: &Path, store_name: &str) -> std::io::Result<()> {
let marker_path = path.join(STORE_INITIALIZATION_MARKER);
let temporary_marker_path = path.join(STORE_INITIALIZATION_MARKER_TEMP);
fs::write(&temporary_marker_path, store_name)?;
fs::rename(temporary_marker_path, marker_path)
write_store_marker(path, STORE_INITIALIZATION_MARKER, STORE_INITIALIZATION_MARKER_TEMP, store_name)
}
fn write_store_display_name(path: &Path, data_source_name: &str) -> std::io::Result<()> {
let marker_path = path.join(STORE_DISPLAY_NAME_MARKER);
if fs::read_to_string(&marker_path).is_ok_and(|current_name| current_name == data_source_name) {
write_store_marker(path, STORE_DISPLAY_NAME_MARKER, STORE_DISPLAY_NAME_MARKER_TEMP, data_source_name)
}
fn write_store_marker(path: &Path, marker_name: &str, temporary_marker_name: &str, value: &str) -> std::io::Result<()> {
let marker_path = path.join(marker_name);
if fs::read_to_string(&marker_path).is_ok_and(|current_value| current_value == value) {
return Ok(());
}
let temporary_marker_path = path.join(STORE_DISPLAY_NAME_MARKER_TEMP);
fs::write(&temporary_marker_path, data_source_name)?;
let temporary_marker_path = path.join(temporary_marker_name);
fs::write(&temporary_marker_path, value)?;
if marker_path.exists() {
fs::remove_file(&marker_path)?;
}