mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-08-24 20:52:11 +00:00
changed qdrant to qdrant edge
This commit is contained in:
parent
b7b18aa45c
commit
6989cbe3b2
@ -6211,11 +6211,12 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1107156991"] = "Browse AI Studio
|
|||||||
-- Vector store version
|
-- Vector store version
|
||||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1124039623"] = "Vector store version"
|
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1124039623"] = "Vector store version"
|
||||||
|
|
||||||
-- The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.
|
|
||||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer."
|
|
||||||
-- Qdrant Edge is an embedded vector database and vector similarity search engine. We use it to realize local RAG—retrieval-augmented generation—within AI Studio. Thanks for the effort and great work that has been and is being put into Qdrant.
|
-- Qdrant Edge is an embedded vector database and vector similarity search engine. We use it to realize local RAG—retrieval-augmented generation—within AI Studio. Thanks for the effort and great work that has been and is being put into Qdrant.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1126023000"] = "Qdrant Edge is an embedded vector database and vector similarity search engine. We use it to realize local RAG—retrieval-augmented generation—within AI Studio. Thanks for the effort and great work that has been and is being put into Qdrant."
|
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1126023000"] = "Qdrant Edge is an embedded vector database and vector similarity search engine. We use it to realize local RAG—retrieval-augmented generation—within AI Studio. Thanks for the effort and great work that has been and is being put into Qdrant."
|
||||||
|
|
||||||
|
-- The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.
|
||||||
|
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer."
|
||||||
|
|
||||||
-- ID mismatch: the plugin ID differs from the enterprise configuration ID.
|
-- ID mismatch: the plugin ID differs from the enterprise configuration ID.
|
||||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID mismatch: the plugin ID differs from the enterprise configuration ID."
|
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID mismatch: the plugin ID differs from the enterprise configuration ID."
|
||||||
|
|
||||||
|
|||||||
@ -1,80 +0,0 @@
|
|||||||
using AIStudio.Tools.Databases.VectorStore;
|
|
||||||
using AIStudio.Tools.Rust;
|
|
||||||
|
|
||||||
namespace AIStudio.Tools.Databases;
|
|
||||||
|
|
||||||
public sealed partial class DatabaseClientProvider
|
|
||||||
{
|
|
||||||
private async Task<DatabaseClient> CreateQdrantClientAsync(CancellationToken cancellationToken)
|
|
||||||
{
|
|
||||||
var qdrantInfo = await rustService.GetQdrantInfo(cancellationToken);
|
|
||||||
if (qdrantInfo.Status is QdrantStatus.STARTING)
|
|
||||||
{
|
|
||||||
return this.CreateNoDatabaseClient(
|
|
||||||
"Qdrant",
|
|
||||||
"Qdrant is starting. Details will appear shortly.",
|
|
||||||
DatabaseClientStatus.STARTING);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!qdrantInfo.IsAvailable || qdrantInfo.Status is QdrantStatus.UNAVAILABLE)
|
|
||||||
{
|
|
||||||
var reason = qdrantInfo.UnavailableReason ?? "unknown";
|
|
||||||
this.logger.LogWarning("Qdrant is not available. Starting without vector database. Reason: '{Reason}'.", reason);
|
|
||||||
return this.CreateNoDatabaseClient("Qdrant", qdrantInfo.UnavailableReason, DatabaseClientStatus.UNAVAILABLE);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!HasValidQdrantConnectionInfo(qdrantInfo, out var invalidReason))
|
|
||||||
return this.CreateNoDatabaseClient("Qdrant", invalidReason, DatabaseClientStatus.UNAVAILABLE);
|
|
||||||
|
|
||||||
var client = new QdrantClientImplementation("Qdrant", qdrantInfo.Path, qdrantInfo.PortHttp, qdrantInfo.PortGrpc, qdrantInfo.Fingerprint, qdrantInfo.ApiToken);
|
|
||||||
client.SetLogger(this.databaseClientLogger);
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await client.CheckAvailabilityAsync();
|
|
||||||
return client;
|
|
||||||
}
|
|
||||||
catch (Exception e)
|
|
||||||
{
|
|
||||||
client.Dispose();
|
|
||||||
this.logger.LogWarning(e, "Qdrant reported as available by Rust, but the health check failed.");
|
|
||||||
return this.CreateNoDatabaseClient("Qdrant", e.Message, DatabaseClientStatus.STARTING);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
private static bool HasValidQdrantConnectionInfo(QdrantInfo qdrantInfo, out string invalidReason)
|
|
||||||
{
|
|
||||||
if (qdrantInfo.Path == string.Empty)
|
|
||||||
{
|
|
||||||
invalidReason = "Failed to get the Qdrant path from Rust.";
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (qdrantInfo.PortHttp == 0)
|
|
||||||
{
|
|
||||||
invalidReason = "Failed to get the Qdrant HTTP port from Rust.";
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (qdrantInfo.PortGrpc == 0)
|
|
||||||
{
|
|
||||||
invalidReason = "Failed to get the Qdrant gRPC port from Rust.";
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (qdrantInfo.Fingerprint == string.Empty)
|
|
||||||
{
|
|
||||||
invalidReason = "Failed to get the Qdrant fingerprint from Rust.";
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (qdrantInfo.ApiToken == string.Empty)
|
|
||||||
{
|
|
||||||
invalidReason = "Failed to get the Qdrant API token from Rust.";
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
invalidReason = string.Empty;
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@ -44,10 +44,10 @@ public sealed partial class DatabaseClientProvider(RustService rustService, ILog
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
public async Task<IVectorStoreClient> GetVectorStoreAsync(CancellationToken cancellationToken = default)
|
public async Task<VectorStoreClient> GetVectorStoreAsync(CancellationToken cancellationToken = default)
|
||||||
{
|
{
|
||||||
var client = await this.GetClientAsync(DatabaseRole.VECTOR_STORE, cancellationToken);
|
var client = await this.GetClientAsync(DatabaseRole.VECTOR_STORE, cancellationToken);
|
||||||
if (client is IVectorStoreClient vectorStore)
|
if (client is VectorStoreClient vectorStore)
|
||||||
return vectorStore;
|
return vectorStore;
|
||||||
|
|
||||||
return new NoVectorStoreClient(
|
return new NoVectorStoreClient(
|
||||||
|
|||||||
@ -1,16 +0,0 @@
|
|||||||
namespace AIStudio.Tools.Databases;
|
|
||||||
|
|
||||||
public sealed record VectorStoragePoint(
|
|
||||||
string PointId,
|
|
||||||
IReadOnlyList<float> Vector,
|
|
||||||
string DataSourceId,
|
|
||||||
string DataSourceName,
|
|
||||||
string DataSourceType,
|
|
||||||
string FilePath,
|
|
||||||
string FileName,
|
|
||||||
string RelativePath,
|
|
||||||
int ChunkIndex,
|
|
||||||
string Text,
|
|
||||||
string Fingerprint,
|
|
||||||
DateTime LastWriteUtc,
|
|
||||||
DateTime EmbeddedAtUtc);
|
|
||||||
@ -1,12 +0,0 @@
|
|||||||
namespace AIStudio.Tools.Databases.VectorStore;
|
|
||||||
|
|
||||||
public interface IVectorStoreClient
|
|
||||||
{
|
|
||||||
Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token);
|
|
||||||
|
|
||||||
Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token);
|
|
||||||
|
|
||||||
Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token);
|
|
||||||
|
|
||||||
Task DeleteVectorStore(string storeName, CancellationToken token);
|
|
||||||
}
|
|
||||||
@ -2,7 +2,7 @@ using AIStudio.Tools.PluginSystem;
|
|||||||
|
|
||||||
namespace AIStudio.Tools.Databases.VectorStore;
|
namespace AIStudio.Tools.Databases.VectorStore;
|
||||||
|
|
||||||
public sealed class NoVectorStoreClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : DatabaseClient(name, string.Empty), IVectorStoreClient
|
public sealed class NoVectorStoreClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : VectorStoreClient(name, string.Empty)
|
||||||
{
|
{
|
||||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoVectorStoreClient).Namespace, nameof(NoVectorStoreClient));
|
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoVectorStoreClient).Namespace, nameof(NoVectorStoreClient));
|
||||||
|
|
||||||
@ -22,16 +22,16 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
|
|||||||
await Task.CompletedTask;
|
await Task.CompletedTask;
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token) =>
|
public override Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token) =>
|
||||||
Task.FromException(this.CreateUnavailableException());
|
Task.FromException(this.CreateUnavailableException());
|
||||||
|
|
||||||
public Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
|
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
|
||||||
Task.FromException(this.CreateUnavailableException());
|
Task.FromException(this.CreateUnavailableException());
|
||||||
|
|
||||||
public Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token) =>
|
public override Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token) =>
|
||||||
Task.FromException(this.CreateUnavailableException());
|
Task.FromException(this.CreateUnavailableException());
|
||||||
|
|
||||||
public Task DeleteVectorStore(string storeName, CancellationToken token) =>
|
public override Task DeleteVectorStore(string storeName, CancellationToken token) =>
|
||||||
Task.FromException(this.CreateUnavailableException());
|
Task.FromException(this.CreateUnavailableException());
|
||||||
|
|
||||||
private InvalidOperationException CreateUnavailableException() =>
|
private InvalidOperationException CreateUnavailableException() =>
|
||||||
|
|||||||
@ -1,142 +0,0 @@
|
|||||||
using Qdrant.Client;
|
|
||||||
using Qdrant.Client.Grpc;
|
|
||||||
using Grpc.Core;
|
|
||||||
using AIStudio.Tools.PluginSystem;
|
|
||||||
using static Qdrant.Client.Grpc.Conditions;
|
|
||||||
|
|
||||||
namespace AIStudio.Tools.Databases.VectorStore;
|
|
||||||
|
|
||||||
public class QdrantClientImplementation : DatabaseClient, IVectorStoreClient
|
|
||||||
{
|
|
||||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(QdrantClientImplementation).Namespace, nameof(QdrantClientImplementation));
|
|
||||||
|
|
||||||
private int HttpPort { get; }
|
|
||||||
|
|
||||||
private int GrpcPort { get; }
|
|
||||||
|
|
||||||
private QdrantClient GrpcClient { get; }
|
|
||||||
|
|
||||||
private string Fingerprint { get; }
|
|
||||||
|
|
||||||
private string ApiToken { get; }
|
|
||||||
|
|
||||||
public QdrantClientImplementation(string name, string path, int? httpPort, int? grpcPort, string? fingerprint, string? apiToken): base(name, path)
|
|
||||||
{
|
|
||||||
this.HttpPort = httpPort ?? 0;
|
|
||||||
this.GrpcPort = grpcPort ?? 0;
|
|
||||||
this.Fingerprint = fingerprint ?? string.Empty;
|
|
||||||
this.ApiToken = apiToken ?? string.Empty;
|
|
||||||
this.GrpcClient = this.CreateQdrantClient();
|
|
||||||
}
|
|
||||||
|
|
||||||
public override string CacheKey => $"{this.Name}:{this.HttpPort}:{this.GrpcPort}:{this.Fingerprint}";
|
|
||||||
|
|
||||||
private const string IP_ADDRESS = "localhost";
|
|
||||||
|
|
||||||
private QdrantClient CreateQdrantClient()
|
|
||||||
{
|
|
||||||
var address = "https://" + IP_ADDRESS + ":" + this.GrpcPort;
|
|
||||||
var channel = QdrantChannel.ForAddress(address, new ClientConfiguration
|
|
||||||
{
|
|
||||||
ApiKey = this.ApiToken,
|
|
||||||
CertificateThumbprint = this.Fingerprint
|
|
||||||
});
|
|
||||||
var grpcClient = new QdrantGrpcClient(channel);
|
|
||||||
return new QdrantClient(grpcClient);
|
|
||||||
}
|
|
||||||
|
|
||||||
private async Task<string> GetVersion()
|
|
||||||
{
|
|
||||||
var operation = await this.GrpcClient.HealthAsync();
|
|
||||||
return $"v{operation.Version}";
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task CheckAvailabilityAsync()
|
|
||||||
{
|
|
||||||
await this.GrpcClient.HealthAsync();
|
|
||||||
}
|
|
||||||
|
|
||||||
private async Task<string> GetCollectionsAmount()
|
|
||||||
{
|
|
||||||
var operation = await this.GrpcClient.ListCollectionsAsync();
|
|
||||||
return operation.Count.ToString();
|
|
||||||
}
|
|
||||||
|
|
||||||
public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo()
|
|
||||||
{
|
|
||||||
yield return (TB("HTTP port"), this.HttpPort.ToString());
|
|
||||||
yield return (TB("gRPC port"), this.GrpcPort.ToString());
|
|
||||||
yield return (TB("Reported version"), await this.GetVersion());
|
|
||||||
yield return (TB("Storage size"), $"{this.GetStorageSize()}");
|
|
||||||
yield return (TB("Number of collections"), await this.GetCollectionsAmount());
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task EnsureVectorStoreExists(string collectionName, int vectorSize, CancellationToken token)
|
|
||||||
{
|
|
||||||
var exists = await this.GrpcClient.CollectionExistsAsync(collectionName, token);
|
|
||||||
if (exists)
|
|
||||||
return;
|
|
||||||
|
|
||||||
await this.GrpcClient.CreateCollectionAsync(
|
|
||||||
collectionName,
|
|
||||||
new VectorParams
|
|
||||||
{
|
|
||||||
Size = (ulong)vectorSize,
|
|
||||||
Distance = Distance.Cosine,
|
|
||||||
},
|
|
||||||
cancellationToken: token);
|
|
||||||
}
|
|
||||||
|
|
||||||
public Task InsertEmbedding(string collectionName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token)
|
|
||||||
{
|
|
||||||
var qdrantPoints = points.Select(point => new PointStruct
|
|
||||||
{
|
|
||||||
Id = Guid.Parse(point.PointId),
|
|
||||||
Vectors = point.Vector.ToArray(),
|
|
||||||
Payload =
|
|
||||||
{
|
|
||||||
["data_source_id"] = point.DataSourceId,
|
|
||||||
["data_source_name"] = point.DataSourceName,
|
|
||||||
["data_source_type"] = point.DataSourceType,
|
|
||||||
["file_path"] = point.FilePath,
|
|
||||||
["file_name"] = point.FileName,
|
|
||||||
["relative_path"] = point.RelativePath,
|
|
||||||
["chunk_index"] = (long)point.ChunkIndex,
|
|
||||||
["text"] = point.Text,
|
|
||||||
["fingerprint"] = point.Fingerprint,
|
|
||||||
["last_write_utc"] = point.LastWriteUtc.ToString("O"),
|
|
||||||
["embedded_at_utc"] = point.EmbeddedAtUtc.ToString("O"),
|
|
||||||
}
|
|
||||||
}).ToList();
|
|
||||||
|
|
||||||
return this.GrpcClient.UpsertAsync(collectionName, qdrantPoints, true, null, null, token);
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task DeleteEmbeddingByFile(string collectionName, string filePath, CancellationToken token)
|
|
||||||
{
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await this.GrpcClient.DeleteAsync(collectionName, MatchKeyword("file_path", filePath), true, null, null, token);
|
|
||||||
}
|
|
||||||
catch (RpcException exception) when (exception.StatusCode is StatusCode.NotFound)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public async Task DeleteVectorStore(string collectionName, CancellationToken token)
|
|
||||||
{
|
|
||||||
var exists = await this.GrpcClient.CollectionExistsAsync(collectionName, token);
|
|
||||||
if (!exists)
|
|
||||||
return;
|
|
||||||
|
|
||||||
try
|
|
||||||
{
|
|
||||||
await this.GrpcClient.DeleteCollectionAsync(collectionName, cancellationToken: token);
|
|
||||||
}
|
|
||||||
catch (RpcException exception) when (exception.StatusCode is StatusCode.NotFound)
|
|
||||||
{
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
public override void Dispose() => this.GrpcClient.Dispose();
|
|
||||||
}
|
|
||||||
@ -9,7 +9,7 @@ public sealed class QdrantEdgeClientImplementation(
|
|||||||
string path,
|
string path,
|
||||||
string version,
|
string version,
|
||||||
int storesCount,
|
int storesCount,
|
||||||
RustService rustService) : DatabaseClient(name, path), IVectorStoreClient
|
RustService rustService) : VectorStoreClient(name, path)
|
||||||
{
|
{
|
||||||
private const string DATABASE_NAME = "Qdrant Edge";
|
private const string DATABASE_NAME = "Qdrant Edge";
|
||||||
private const string INFO_PATH = "/system/qdrant-edge/info";
|
private const string INFO_PATH = "/system/qdrant-edge/info";
|
||||||
@ -80,16 +80,16 @@ public sealed class QdrantEdgeClientImplementation(
|
|||||||
yield return (TB("Number of vector stores"), displayStoresCount.ToString());
|
yield return (TB("Number of vector stores"), displayStoresCount.ToString());
|
||||||
}
|
}
|
||||||
|
|
||||||
public Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token) =>
|
public override Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token) =>
|
||||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, ENSURE_PATH, new EnsureVectorStoreRequest(storeName, vectorSize), token);
|
rustService.ExecuteDatabaseOperation(DATABASE_NAME, ENSURE_PATH, new EnsureVectorStoreRequest(storeName, vectorSize), token);
|
||||||
|
|
||||||
public Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
|
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
|
||||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, INSERT_PATH, new InsertEmbeddingRequest(storeName, points), token);
|
rustService.ExecuteDatabaseOperation(DATABASE_NAME, INSERT_PATH, new InsertEmbeddingRequest(storeName, points), token);
|
||||||
|
|
||||||
public Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token) =>
|
public override Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token) =>
|
||||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_FILE_PATH, new DeleteEmbeddingByFileRequest(storeName, filePath), token);
|
rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_FILE_PATH, new DeleteEmbeddingByFileRequest(storeName, filePath), token);
|
||||||
|
|
||||||
public Task DeleteVectorStore(string storeName, CancellationToken token) =>
|
public override Task DeleteVectorStore(string storeName, CancellationToken token) =>
|
||||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_STORE_PATH, new DeleteVectorStoreRequest(storeName), token);
|
rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_STORE_PATH, new DeleteVectorStoreRequest(storeName), token);
|
||||||
|
|
||||||
public override void Dispose()
|
public override void Dispose()
|
||||||
|
|||||||
@ -0,0 +1,12 @@
|
|||||||
|
namespace AIStudio.Tools.Databases.VectorStore;
|
||||||
|
|
||||||
|
public abstract class VectorStoreClient(string name, string path): DatabaseClient(name, path)
|
||||||
|
{
|
||||||
|
public abstract Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token);
|
||||||
|
|
||||||
|
public abstract Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token);
|
||||||
|
|
||||||
|
public abstract Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token);
|
||||||
|
|
||||||
|
public abstract Task DeleteVectorStore(string storeName, CancellationToken token);
|
||||||
|
}
|
||||||
@ -219,13 +219,21 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
return Convert.ToHexString(bytes);
|
return Convert.ToHexString(bytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
private string GetCollectionName(string dataSourceId)
|
private string GetCollectionName(string dataSourceName, string dataSourceId)
|
||||||
{
|
{
|
||||||
var safeId = dataSourceId
|
var safeId = dataSourceId
|
||||||
.ToLowerInvariant()
|
.ToLowerInvariant()
|
||||||
.Replace("-", string.Empty, StringComparison.Ordinal);
|
.Replace("-", string.Empty, StringComparison.Ordinal);
|
||||||
|
|
||||||
return $"rag_{safeId}";
|
var safeName = new string(dataSourceName
|
||||||
|
.ToLowerInvariant()
|
||||||
|
.Where(c => c is >= 'a' and <= 'z' or >= '0' and <= '9')
|
||||||
|
.Take(32)
|
||||||
|
.ToArray());
|
||||||
|
|
||||||
|
safeName = string.IsNullOrWhiteSpace(safeName) ? "datasource" : safeName;
|
||||||
|
|
||||||
|
return $"rag_{safeName}_{safeId}";
|
||||||
}
|
}
|
||||||
|
|
||||||
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex)
|
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex)
|
||||||
|
|||||||
@ -71,11 +71,11 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
await File.WriteAllTextAsync(statePath, json, token);
|
await File.WriteAllTextAsync(statePath, json, token);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task ResetPersistedStateAsync(string dataSourceId, IVectorStoreClient? vectorStore, CancellationToken token)
|
private async Task ResetPersistedStateAsync(string dataSourcename, string dataSourceId, VectorStoreClient? vectorStore, CancellationToken token)
|
||||||
{
|
{
|
||||||
await this.EnsureStateLoadedAsync(token);
|
await this.EnsureStateLoadedAsync(token);
|
||||||
this.manifests.Remove(dataSourceId);
|
this.manifests.Remove(dataSourceId);
|
||||||
await this.DeleteCollectionAsync(this.GetCollectionName(dataSourceId), vectorStore, token);
|
await this.DeleteCollectionAsync(this.GetCollectionName(dataSourcename, dataSourceId), vectorStore, token);
|
||||||
await this.SaveStateAsync(token);
|
await this.SaveStateAsync(token);
|
||||||
logger.LogInformation("Reset persisted embedding state for data source '{DataSourceId}'.", dataSourceId);
|
logger.LogInformation("Reset persisted embedding state for data source '{DataSourceId}'.", dataSourceId);
|
||||||
}
|
}
|
||||||
|
|||||||
@ -114,7 +114,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
|
|
||||||
this.RemoveWatcher(dataSource.Id);
|
this.RemoveWatcher(dataSource.Id);
|
||||||
this.statuses.TryRemove(dataSource.Id, out _);
|
this.statuses.TryRemove(dataSource.Id, out _);
|
||||||
await this.ResetPersistedStateAsync(dataSource.Id, null, CancellationToken.None);
|
await this.ResetPersistedStateAsync(dataSource.Name, dataSource.Id, null, CancellationToken.None);
|
||||||
this.PublishStatusChanged();
|
this.PublishStatusChanged();
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -180,6 +180,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
logger.LogInformation(
|
logger.LogInformation(
|
||||||
"Using embedding provider '{EmbeddingProviderId}' with model '{EmbeddingModelId}' for data source '{DataSourceName}' ({DataSourceId}).",
|
"Using embedding provider '{EmbeddingProviderId}' with model '{EmbeddingModelId}' for data source '{DataSourceName}' ({DataSourceId}).",
|
||||||
embeddingProvider.Id,
|
embeddingProvider.Id,
|
||||||
@ -187,7 +188,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
dataSource.Name,
|
dataSource.Name,
|
||||||
dataSource.Id);
|
dataSource.Id);
|
||||||
|
|
||||||
var collectionName = this.GetCollectionName(dataSource.Id);
|
var collectionName = this.GetCollectionName(dataSource.Name, dataSource.Id);
|
||||||
var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, token);
|
var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, token);
|
||||||
var inputFiles = this.GetInputFiles(dataSource);
|
var inputFiles = this.GetInputFiles(dataSource);
|
||||||
var indexedFiles = inputFiles.Files;
|
var indexedFiles = inputFiles.Files;
|
||||||
@ -289,7 +290,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async Task<int> IndexOneFileAsync(
|
private async Task<int> IndexOneFileAsync(
|
||||||
IVectorStoreClient vectorStore,
|
VectorStoreClient vectorStore,
|
||||||
IDataSource dataSource,
|
IDataSource dataSource,
|
||||||
FileInfo file,
|
FileInfo file,
|
||||||
string fingerprint,
|
string fingerprint,
|
||||||
@ -298,7 +299,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
DataSourceEmbeddingManifest manifest,
|
DataSourceEmbeddingManifest manifest,
|
||||||
CancellationToken token)
|
CancellationToken token)
|
||||||
{
|
{
|
||||||
var collectionName = this.GetCollectionName(dataSource.Id);
|
var collectionName = this.GetCollectionName(dataSource.Name, dataSource.Id);
|
||||||
logger.LogDebug(
|
logger.LogDebug(
|
||||||
"Resetting stored embeddings for file '{FilePath}' in collection '{CollectionName}' before re-indexing.",
|
"Resetting stored embeddings for file '{FilePath}' in collection '{CollectionName}' before re-indexing.",
|
||||||
file.FullName,
|
file.FullName,
|
||||||
@ -334,7 +335,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async Task FlushBatchAsync(
|
private async Task FlushBatchAsync(
|
||||||
IVectorStoreClient vectorStore,
|
VectorStoreClient vectorStore,
|
||||||
IDataSource dataSource,
|
IDataSource dataSource,
|
||||||
FileInfo file,
|
FileInfo file,
|
||||||
string fingerprint,
|
string fingerprint,
|
||||||
@ -397,13 +398,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
batch.Clear();
|
batch.Clear();
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task EnsureCollectionExistsAsync(IVectorStoreClient vectorStore, string collectionName, int vectorSize, CancellationToken token)
|
private async Task EnsureCollectionExistsAsync(VectorStoreClient vectorStore, string collectionName, int vectorSize, CancellationToken token)
|
||||||
{
|
{
|
||||||
await vectorStore.EnsureVectorStoreExists(collectionName, vectorSize, token);
|
await vectorStore.EnsureVectorStoreExists(collectionName, vectorSize, token);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task UpsertPointsAsync(
|
private async Task UpsertPointsAsync(
|
||||||
IVectorStoreClient vectorStore,
|
VectorStoreClient vectorStore,
|
||||||
string collectionName,
|
string collectionName,
|
||||||
IDataSource dataSource,
|
IDataSource dataSource,
|
||||||
FileInfo file,
|
FileInfo file,
|
||||||
@ -432,12 +433,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
await vectorStore.InsertEmbedding(collectionName, points, token);
|
await vectorStore.InsertEmbedding(collectionName, points, token);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task DeleteFilePointsAsync(IVectorStoreClient vectorStore, string collectionName, string filePath, CancellationToken token)
|
private async Task DeleteFilePointsAsync(VectorStoreClient vectorStore, string collectionName, string filePath, CancellationToken token)
|
||||||
{
|
{
|
||||||
await vectorStore.DeleteEmbeddingByFile(collectionName, filePath, token);
|
await vectorStore.DeleteEmbeddingByFile(collectionName, filePath, token);
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task DeleteCollectionAsync(string collectionName, IVectorStoreClient? vectorStore, CancellationToken token)
|
private async Task DeleteCollectionAsync(string collectionName, VectorStoreClient? vectorStore, CancellationToken token)
|
||||||
{
|
{
|
||||||
vectorStore ??= await databaseClientProvider.GetVectorStoreAsync(token);
|
vectorStore ??= await databaseClientProvider.GetVectorStoreAsync(token);
|
||||||
if (!vectorStore.IsAvailable)
|
if (!vectorStore.IsAvailable)
|
||||||
@ -483,7 +484,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
return embeddingProvider != default && embeddingProvider.UsedLLMProvider is not LLMProviders.NONE;
|
return embeddingProvider != default && embeddingProvider.UsedLLMProvider is not LLMProviders.NONE;
|
||||||
}
|
}
|
||||||
|
|
||||||
private async Task<DataSourceEmbeddingManifest> EnsureCompatibleManifestAsync(IDataSource dataSource, EmbeddingProvider embeddingProvider, string collectionName, IVectorStoreClient vectorStore, CancellationToken token)
|
private async Task<DataSourceEmbeddingManifest> EnsureCompatibleManifestAsync(IDataSource dataSource, EmbeddingProvider embeddingProvider, string collectionName, VectorStoreClient vectorStore, CancellationToken token)
|
||||||
{
|
{
|
||||||
var embeddingSignature = this.BuildEmbeddingSignature(embeddingProvider);
|
var embeddingSignature = this.BuildEmbeddingSignature(embeddingProvider);
|
||||||
var manifest = await this.GetManifestAsync(dataSource.Id, token);
|
var manifest = await this.GetManifestAsync(dataSource.Id, token);
|
||||||
@ -495,7 +496,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
dataSource.Name,
|
dataSource.Name,
|
||||||
dataSource.Id,
|
dataSource.Id,
|
||||||
collectionName);
|
collectionName);
|
||||||
await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, token);
|
await this.ResetPersistedStateAsync(dataSource.Name, dataSource.Id, vectorStore, token);
|
||||||
manifest = await this.GetManifestAsync(dataSource.Id, token);
|
manifest = await this.GetManifestAsync(dataSource.Id, token);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -511,7 +512,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
}
|
}
|
||||||
|
|
||||||
private async Task RemoveMissingFileEmbeddingsAsync(
|
private async Task RemoveMissingFileEmbeddingsAsync(
|
||||||
IVectorStoreClient vectorStore,
|
VectorStoreClient vectorStore,
|
||||||
IDataSource dataSource,
|
IDataSource dataSource,
|
||||||
string collectionName,
|
string collectionName,
|
||||||
DataSourceEmbeddingManifest manifest,
|
DataSourceEmbeddingManifest manifest,
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user