mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-14 14:13:48 +00:00
Add a cheap total chunk count to the index store
This commit is contained in:
4 files changed
+53
-5
No files matched your search
@@ -8,11 +8,20 @@ public abstract class DatabaseClient(string name, string path)
|
|||||||
|
|
||||||
public virtual DatabaseClientStatus Status => DatabaseClientStatus.AVAILABLE;
|
public virtual DatabaseClientStatus Status => DatabaseClientStatus.AVAILABLE;
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// The version the running database reports about itself.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// Empty when the client cannot tell. Callers which want to show a version in a headline read it
|
||||||
|
/// from here instead of picking it out of the label-value pairs the display info yields.
|
||||||
|
/// </remarks>
|
||||||
|
public virtual string Version => string.Empty;
|
||||||
|
|
||||||
public bool IsAvailable => this.Status is DatabaseClientStatus.AVAILABLE;
|
public bool IsAvailable => this.Status is DatabaseClientStatus.AVAILABLE;
|
||||||
|
|
||||||
private string Path => path;
|
private string Path => path;
|
||||||
|
|
||||||
private ILogger<DatabaseClient>? logger;
|
protected ILogger<DatabaseClient>? Logger;
|
||||||
|
|
||||||
public abstract IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo();
|
public abstract IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo();
|
||||||
|
|
||||||
@@ -20,13 +29,13 @@ public abstract class DatabaseClient(string name, string path)
|
|||||||
{
|
{
|
||||||
if (string.IsNullOrWhiteSpace(this.Path))
|
if (string.IsNullOrWhiteSpace(this.Path))
|
||||||
{
|
{
|
||||||
this.logger!.LogError($"Error: Database path '{this.Path}' cannot be null or empty.");
|
this.Logger!.LogError($"Error: Database path '{this.Path}' cannot be null or empty.");
|
||||||
return "0 B";
|
return "0 B";
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!Directory.Exists(this.Path))
|
if (!Directory.Exists(this.Path))
|
||||||
{
|
{
|
||||||
this.logger!.LogError($"Error: Database path '{this.Path}' does not exist.");
|
this.Logger!.LogError($"Error: Database path '{this.Path}' does not exist.");
|
||||||
return "0 B";
|
return "0 B";
|
||||||
}
|
}
|
||||||
var files = Directory.EnumerateFiles(this.Path, "*", SearchOption.AllDirectories)
|
var files = Directory.EnumerateFiles(this.Path, "*", SearchOption.AllDirectories)
|
||||||
@@ -52,7 +61,7 @@ public abstract class DatabaseClient(string name, string path)
|
|||||||
|
|
||||||
public void SetLogger(ILogger<DatabaseClient> logService)
|
public void SetLogger(ILogger<DatabaseClient> logService)
|
||||||
{
|
{
|
||||||
this.logger = logService;
|
this.Logger = logService;
|
||||||
}
|
}
|
||||||
|
|
||||||
public abstract void Dispose();
|
public abstract void Dispose();
|
||||||
|
|||||||
@@ -40,4 +40,17 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
|
|||||||
public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
|
public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
|
||||||
|
|
||||||
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
|
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
|
||||||
|
|
||||||
|
/// <summary>
|
||||||
|
/// Counts the search chunks the index holds across all data sources.
|
||||||
|
/// </summary>
|
||||||
|
/// <remarks>
|
||||||
|
/// One chunk is one vector: every chunk becomes exactly one point carrying the single named
|
||||||
|
/// vector "embedding". The vector store reports its vector count from here, because counting
|
||||||
|
/// the points in Qdrant Edge would have to load every shard first and would hold the global
|
||||||
|
/// database mutex against ongoing inserts and searches while doing so.
|
||||||
|
/// </remarks>
|
||||||
|
/// <param name="token">The cancellation token.</param>
|
||||||
|
/// <returns>The number of chunks, or null when the index cannot tell. Null and zero mean different things here.</returns>
|
||||||
|
public abstract Task<long?> GetTotalChunkCountAsync(CancellationToken token);
|
||||||
}
|
}
|
||||||
@@ -55,6 +55,8 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
|
|||||||
|
|
||||||
public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask;
|
public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask;
|
||||||
|
|
||||||
|
public override Task<long?> GetTotalChunkCountAsync(CancellationToken token) => Task.FromResult<long?>(null);
|
||||||
|
|
||||||
public override void Dispose()
|
public override void Dispose()
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|||||||
+25
-1
@@ -25,6 +25,8 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
|||||||
|
|
||||||
public override string CacheKey => $"{this.Name}:{this.databasePath}:{version}";
|
public override string CacheKey => $"{this.Name}:{this.databasePath}:{version}";
|
||||||
|
|
||||||
|
public override string Version => version;
|
||||||
|
|
||||||
public static async Task<DatabaseClient> CreateAsync(
|
public static async Task<DatabaseClient> CreateAsync(
|
||||||
ILogger logger,
|
ILogger logger,
|
||||||
ILogger<DatabaseClient> databaseClientLogger,
|
ILogger<DatabaseClient> databaseClientLogger,
|
||||||
@@ -65,7 +67,8 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
|||||||
yield return (TB("Storage size"), this.GetStorageSize());
|
yield return (TB("Storage size"), this.GetStorageSize());
|
||||||
yield return (TB("Indexed data sources"), (await context.DataSources.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
yield return (TB("Indexed data sources"), (await context.DataSources.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||||
yield return (TB("Indexed files"), (await context.EmbeddedFiles.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
yield return (TB("Indexed files"), (await context.EmbeddedFiles.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||||
yield return (TB("Search chunks"), (await context.EmbeddingChunks.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
var searchChunks = await this.GetTotalChunkCountAsync(CancellationToken.None);
|
||||||
|
yield return (TB("Search chunks"), searchChunks?.ToString(CultureInfo.InvariantCulture) ?? TB("unknown"));
|
||||||
yield return (TB("Permanently skipped files"), (await context.PermanentIndexingFailures.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
yield return (TB("Permanently skipped files"), (await context.PermanentIndexingFailures.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -348,6 +351,27 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
|||||||
await transaction.CommitAsync(token);
|
await transaction.CommitAsync(token);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
public override async Task<long?> GetTotalChunkCountAsync(CancellationToken token)
|
||||||
|
{
|
||||||
|
try
|
||||||
|
{
|
||||||
|
await using var context = this.CreateContext();
|
||||||
|
|
||||||
|
//
|
||||||
|
// Sum the chunk counts the files carry instead of counting the rows of the chunk table:
|
||||||
|
// embedded_files holds one row per file, embedding_chunks one per chunk. On a large index
|
||||||
|
// that is a difference of two orders of magnitude, and the information page reads this on
|
||||||
|
// every visit.
|
||||||
|
//
|
||||||
|
return await context.EmbeddedFiles.SumAsync(file => (long)file.ChunkCount, token);
|
||||||
|
}
|
||||||
|
catch (Exception exception)
|
||||||
|
{
|
||||||
|
this.Logger?.LogWarning(exception, "Failed to count the search chunks of the local RAG index.");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
public override void Dispose()
|
public override void Dispose()
|
||||||
{
|
{
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user