current main is now successfully merged into branch

This commit is contained in:
Paul Koudelka committed 2026-05-27 20:02:43 +02:00
1 parent 00b6c98f3b
commit ea87d79ca9
32 files changed
+607 -478

No files matched your search

@@ -1,6 +1,6 @@
namespace AIStudio.Tools.Databases;
public abstract class EmbeddingStore(string name, string path)
public abstract class DatabaseClient(string name, string path)
{
public string Name => name;
@@ -12,7 +12,7 @@ public abstract class EmbeddingStore(string name, string path)
private string Path => path;
private ILogger<EmbeddingStore>? logger;
private ILogger<DatabaseClient>? logger;
public abstract IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo();
@@ -50,7 +50,7 @@ public abstract class EmbeddingStore(string name, string path)
return $"{convertedSize:0.##} {suffixes[suffixIndex]}";
}
public void SetLogger(ILogger<EmbeddingStore> logService)
public void SetLogger(ILogger<DatabaseClient> logService)
{
this.logger = logService;
}
@@ -1,4 +1,5 @@
using AIStudio.Tools.Rust;
using AIStudio.Tools.Databases.VectorStore;
using AIStudio.Tools.Rust;
namespace AIStudio.Tools.Databases;
@@ -6,7 +7,7 @@ public sealed partial class DatabaseClientProvider
{
private async Task<DatabaseClient> CreateQdrantClientAsync(CancellationToken cancellationToken)
{
var qdrantInfo = await this.rustService.GetQdrantInfo(cancellationToken);
var qdrantInfo = await rustService.GetQdrantInfo(cancellationToken);
if (qdrantInfo.Status is QdrantStatus.STARTING)
{
return this.CreateNoDatabaseClient(
@@ -1,15 +1,14 @@
using AIStudio.Tools.Databases.Qdrant;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
using AIStudio.Tools.Databases.VectorStore;
namespace AIStudio.Tools.Databases;
public sealed class EmbeddingStoreProvider(RustService rustService, ILoggerFactory loggerFactory) : IDisposable
public sealed partial class DatabaseClientProvider(RustService rustService, ILoggerFactory loggerFactory) : IDisposable
{
private readonly Dictionary<DatabaseRole, EmbeddingStore> clients = new();
private readonly Dictionary<DatabaseRole, DatabaseClient> clients = new();
private readonly Dictionary<DatabaseRole, SemaphoreSlim> locks = new();
private readonly Lock locksLock = new();
private readonly ILogger<EmbeddingStoreProvider> logger = loggerFactory.CreateLogger<EmbeddingStoreProvider>();
private readonly ILogger<DatabaseClientProvider> logger = loggerFactory.CreateLogger<DatabaseClientProvider>();
private readonly ILogger<DatabaseClient> databaseClientLogger = loggerFactory.CreateLogger<DatabaseClient>();
public async Task<DatabaseClient> GetClientAsync(DatabaseRole databaseRole, CancellationToken cancellationToken = default)
@@ -45,6 +44,18 @@ public sealed class EmbeddingStoreProvider(RustService rustService, ILoggerFacto
}
}
public async Task<IVectorStoreClient> GetVectorStoreAsync(CancellationToken cancellationToken = default)
{
var client = await this.GetClientAsync(DatabaseRole.VECTOR_STORE, cancellationToken);
if (client is IVectorStoreClient vectorStore)
return vectorStore;
return new NoVectorStoreClient(
client.Name,
"The configured database client does not support vector store operations.",
client.Status);
}
private DatabaseClient CacheIfAvailable(DatabaseRole databaseRole, DatabaseClient client)
{
if (!client.IsAvailable)
@@ -84,79 +95,6 @@ public sealed class EmbeddingStoreProvider(RustService rustService, ILoggerFacto
_ => new NoDatabaseClient(databaseRole.ToString(), "The requested database role is not supported.")
};
private async Task<DatabaseClient> CreateQdrantClientAsync(CancellationToken cancellationToken)
{
var qdrantInfo = await rustService.GetQdrantInfo(cancellationToken);
if (qdrantInfo.Status is QdrantStatus.STARTING)
{
return this.CreateNoDatabaseClient(
"Qdrant",
"Qdrant is starting. Details will appear shortly.",
DatabaseClientStatus.STARTING);
}
if (!qdrantInfo.IsAvailable || qdrantInfo.Status is QdrantStatus.UNAVAILABLE)
{
var reason = qdrantInfo.UnavailableReason ?? "unknown";
this.logger.LogWarning("Qdrant is not available. Starting without vector database. Reason: '{Reason}'.", reason);
return this.CreateNoDatabaseClient("Qdrant", qdrantInfo.UnavailableReason, DatabaseClientStatus.UNAVAILABLE);
}
if (!HasValidQdrantConnectionInfo(qdrantInfo, out var invalidReason))
return this.CreateNoDatabaseClient("Qdrant", invalidReason, DatabaseClientStatus.UNAVAILABLE);
var client = new QdrantClientImplementation("Qdrant", qdrantInfo.Path, qdrantInfo.PortHttp, qdrantInfo.PortGrpc, qdrantInfo.Fingerprint, qdrantInfo.ApiToken);
client.SetLogger(this.databaseClientLogger);
try
{
await client.CheckAvailabilityAsync();
return client;
}
catch (Exception e)
{
client.Dispose();
this.logger.LogWarning(e, "Qdrant reported as available by Rust, but the health check failed.");
return this.CreateNoDatabaseClient("Qdrant", e.Message, DatabaseClientStatus.STARTING);
}
}
private static bool HasValidQdrantConnectionInfo(QdrantInfo qdrantInfo, out string invalidReason)
{
if (qdrantInfo.Path == string.Empty)
{
invalidReason = "Failed to get the Qdrant path from Rust.";
return false;
}
if (qdrantInfo.PortHttp == 0)
{
invalidReason = "Failed to get the Qdrant HTTP port from Rust.";
return false;
}
if (qdrantInfo.PortGrpc == 0)
{
invalidReason = "Failed to get the Qdrant gRPC port from Rust.";
return false;
}
if (qdrantInfo.Fingerprint == string.Empty)
{
invalidReason = "Failed to get the Qdrant fingerprint from Rust.";
return false;
}
if (qdrantInfo.ApiToken == string.Empty)
{
invalidReason = "Failed to get the Qdrant API token from Rust.";
return false;
}
invalidReason = string.Empty;
return true;
}
private NoDatabaseClient CreateNoDatabaseClient(string name, string? unavailableReason, DatabaseClientStatus status)
{
var client = new NoDatabaseClient(name, unavailableReason, status);
@@ -1,33 +0,0 @@
using AIStudio.Tools.Databases.Qdrant;
namespace AIStudio.Tools.Databases;
public class EmbeddingStoreFactory
{
public static EmbeddingStore Create(EmbeddingStoreConfiguration configuration) => configuration.Kind switch
{
EmbeddingStoreKind.NONE => new NoEmbeddingStore(configuration.Name, configuration.UnavailableReason ?? "unknown"),
_ when configuration.Location is null => new NoEmbeddingStore(configuration.Name, $"No location specified for {configuration.Name}"),
EmbeddingStoreKind.QDRANT_REMOTE when configuration.Location is RemoteLocation location=> new QdrantClientImplementation(configuration.Name, location.Path, location.HttpPort, location.GrpcPort, location.Fingerprint, location.ApiToken),
_ => throw new ArgumentException("Invalid configuration for " + configuration.Name, nameof(configuration)),
};
}
public enum EmbeddingStoreKind
{
NONE,
QDRANT_EMBED,
QDRANT_REMOTE,
}
public abstract record EmbeddingStoreLocation;
public sealed record EmbeddedLocation(string Path) : EmbeddingStoreLocation;
public sealed record RemoteLocation(string Path, int? HttpPort, int? GrpcPort, string? Fingerprint, string? ApiToken) : EmbeddingStoreLocation;
public sealed record EmbeddingStoreConfiguration(
EmbeddingStoreKind Kind,
string Name,
EmbeddingStoreLocation? Location,
string? UnavailableReason);
@@ -2,9 +2,9 @@ using AIStudio.Tools.PluginSystem;
namespace AIStudio.Tools.Databases;
public sealed class NoEmbeddingStore(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : EmbeddingStore(name, string.Empty)
public sealed class NoDatabaseClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : DatabaseClient(name, string.Empty)
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoEmbeddingStore).Namespace, nameof(NoEmbeddingStore));
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoDatabaseClient).Namespace, nameof(NoDatabaseClient));
public override DatabaseClientStatus Status => status;
@@ -1,6 +1,6 @@
namespace AIStudio.Tools.Databases;
public sealed record EmbeddingStoragePoint(
public sealed record VectorStoragePoint(
string PointId,
IReadOnlyList<float> Vector,
string DataSourceId,
@@ -0,0 +1,20 @@
namespace AIStudio.Tools.Databases.VectorStore;
public interface IVectorStoreClient
{
string Name { get; }
DatabaseClientStatus Status { get; }
bool IsAvailable { get; }
IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo();
Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token);
Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token);
Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token);
Task DeleteVectorStore(string storeName, CancellationToken token);
}
@@ -0,0 +1,39 @@
using AIStudio.Tools.PluginSystem;
namespace AIStudio.Tools.Databases.VectorStore;
public sealed class NoVectorStoreClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : IVectorStoreClient
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoVectorStoreClient).Namespace, nameof(NoVectorStoreClient));
public string Name => name;
public DatabaseClientStatus Status => status;
public bool IsAvailable => false;
public async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo()
{
yield return (TB("Status"), TB("Unavailable"));
if (!string.IsNullOrWhiteSpace(unavailableReason))
yield return (TB("Reason"), unavailableReason);
await Task.CompletedTask;
}
public Task EnsureVectorStoreExists(string storeName, int vectorSize, CancellationToken token) =>
Task.FromException(this.CreateUnavailableException());
public Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
Task.FromException(this.CreateUnavailableException());
public Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token) =>
Task.FromException(this.CreateUnavailableException());
public Task DeleteVectorStore(string storeName, CancellationToken token) =>
Task.FromException(this.CreateUnavailableException());
private InvalidOperationException CreateUnavailableException() =>
new(unavailableReason ?? "The vector store is not available.");
}
@@ -4,9 +4,9 @@ using Grpc.Core;
using AIStudio.Tools.PluginSystem;
using static Qdrant.Client.Grpc.Conditions;
namespace AIStudio.Tools.Databases.Qdrant;
namespace AIStudio.Tools.Databases.VectorStore;
public class QdrantClientImplementation : EmbeddingStore
public class QdrantClientImplementation : DatabaseClient, IVectorStoreClient
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(QdrantClientImplementation).Namespace, nameof(QdrantClientImplementation));
@@ -71,7 +71,7 @@ public class QdrantClientImplementation : EmbeddingStore
yield return (TB("Number of collections"), await this.GetCollectionsAmount());
}
public override async Task EnsureEmbeddingStoreExists(string collectionName, int vectorSize, CancellationToken token)
public async Task EnsureVectorStoreExists(string collectionName, int vectorSize, CancellationToken token)
{
var exists = await this.GrpcClient.CollectionExistsAsync(collectionName, token);
if (exists)
@@ -87,7 +87,7 @@ public class QdrantClientImplementation : EmbeddingStore
cancellationToken: token);
}
public override Task InsertEmbedding(string collectionName, IReadOnlyList<EmbeddingStoragePoint> points, CancellationToken token)
public Task InsertEmbedding(string collectionName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token)
{
var qdrantPoints = points.Select(point => new PointStruct
{
@@ -112,7 +112,7 @@ public class QdrantClientImplementation : EmbeddingStore
return this.GrpcClient.UpsertAsync(collectionName, qdrantPoints, true, null, null, token);
}
public override async Task DeleteEmbeddingByFile(string collectionName, string filePath, CancellationToken token)
public async Task DeleteEmbeddingByFile(string collectionName, string filePath, CancellationToken token)
{
try
{
@@ -120,11 +120,10 @@ public class QdrantClientImplementation : EmbeddingStore
}
catch (RpcException exception) when (exception.StatusCode is StatusCode.NotFound)
{
return;
}
}
public override async Task DeleteEmbeddingStore(string collectionName, CancellationToken token)
public async Task DeleteVectorStore(string collectionName, CancellationToken token)
{
var exists = await this.GrpcClient.CollectionExistsAsync(collectionName, token);
if (!exists)
@@ -136,7 +135,6 @@ public class QdrantClientImplementation : EmbeddingStore
}
catch (RpcException exception) when (exception.StatusCode is StatusCode.NotFound)
{
return;
}
}