mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-05 06:49:40 +00:00
made check for vectordb more robust
This commit is contained in:
1 parent
92e986d9b9
commit
987f493848
5 files changed
+108
-17
No files matched your search
@@ -22,8 +22,8 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
public override Task EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
|
||||
Task.FromException(this.CreateUnavailableException());
|
||||
public override Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
|
||||
Task.FromException<VectorStoreEnsureResult>(this.CreateUnavailableException());
|
||||
|
||||
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
|
||||
Task.FromException(this.CreateUnavailableException());
|
||||
|
||||
+3
-2
@@ -82,8 +82,9 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
yield return (TB("Number of vector stores"), displayStoresCount.ToString());
|
||||
}
|
||||
|
||||
public override Task EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
|
||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, ENSURE_PATH, new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token);
|
||||
public override async Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
|
||||
await rustService.ExecuteDatabaseQuery<EnsureVectorStoreRequest, VectorStoreEnsureResult>( DATABASE_NAME, ENSURE_PATH,
|
||||
new EnsureVectorStoreRequest(storeName, dataSourceName, vectorSize), token) ?? throw new InvalidOperationException("The vector store ensure response was empty.");
|
||||
|
||||
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token) =>
|
||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, INSERT_PATH, new InsertEmbeddingRequest(storeName, points), token);
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
public abstract class VectorStoreClient(string name, string path): DatabaseClient(name, path)
|
||||
{
|
||||
public abstract Task EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token);
|
||||
public abstract Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token);
|
||||
|
||||
public abstract Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token);
|
||||
|
||||
@@ -14,3 +14,5 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
|
||||
|
||||
public abstract Task DeleteVectorStore(string storeName, CancellationToken token);
|
||||
}
|
||||
|
||||
public sealed record VectorStoreEnsureResult(bool Created);
|
||||
@@ -428,6 +428,22 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
return;
|
||||
}
|
||||
|
||||
var collectionName = this.GetCollectionName(dataSource.Id);
|
||||
var persistedManifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
|
||||
if (persistedManifest.VectorSize > 0)
|
||||
{
|
||||
var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, persistedManifest.VectorSize, token);
|
||||
if (ensureResult.Created)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Vector store '{CollectionName}' for data source '{DataSourceName}' ({DataSourceId}) was missing although persisted embedding state exists. Resetting the stale state so all vectors are rebuilt.",
|
||||
collectionName,
|
||||
dataSource.Name,
|
||||
dataSource.Id);
|
||||
await this.ResetPersistedStateAsync(dataSource.Id, vectorStore, embeddingState, token);
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.TryResolveEmbeddingProvider(dataSource, out var embeddingProvider))
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
@@ -460,11 +476,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
dataSource.Name,
|
||||
dataSource.Id);
|
||||
|
||||
var collectionName = this.GetCollectionName(dataSource.Id);
|
||||
var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, embeddingState, token);
|
||||
token.ThrowIfCancellationRequested();
|
||||
if (manifest.VectorSize > 0)
|
||||
await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, manifest.VectorSize, token);
|
||||
|
||||
var inputFiles = this.GetInputFiles(dataSource);
|
||||
var indexedFiles = inputFiles.Files;
|
||||
@@ -770,7 +783,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
if (manifest.VectorSize == 0)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
|
||||
var ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
|
||||
if (!ensureResult.Created)
|
||||
{
|
||||
logger.LogWarning(
|
||||
"Vector store '{CollectionName}' exists for data source '{DataSourceName}' ({DataSourceId}) although no persisted embedding state exists. Replacing the orphaned store before indexing.",
|
||||
collectionName,
|
||||
dataSource.Name,
|
||||
dataSource.Id);
|
||||
await vectorStore.DeleteVectorStore(collectionName, token);
|
||||
ensureResult = await this.EnsureCollectionExistsAsync(vectorStore, collectionName, dataSource.Name, vectorSize, token);
|
||||
if (!ensureResult.Created)
|
||||
throw new InvalidOperationException($"Vector store '{collectionName}' could not be recreated cleanly.");
|
||||
}
|
||||
|
||||
await embeddingState.UpdateVectorSizeAsync(dataSource.Id, vectorSize, token);
|
||||
manifest.VectorSize = vectorSize;
|
||||
logger.LogInformation(
|
||||
@@ -819,9 +845,9 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
batch.Clear();
|
||||
}
|
||||
|
||||
private async Task EnsureCollectionExistsAsync(VectorStoreClient vectorStore, string collectionName, string dataSourceName, int vectorSize, CancellationToken token)
|
||||
private async Task<VectorStoreEnsureResult> EnsureCollectionExistsAsync(VectorStoreClient vectorStore, string collectionName, string dataSourceName, int vectorSize, CancellationToken token)
|
||||
{
|
||||
await vectorStore.EnsureVectorStoreExists(collectionName, dataSourceName, vectorSize, token);
|
||||
return await vectorStore.EnsureVectorStoreExists(collectionName, dataSourceName, vectorSize, token);
|
||||
}
|
||||
|
||||
private async Task UpsertPointsAsync(
|
||||
|
||||
Reference in new issue
Block a user