mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-13 12:33:48 +00:00
Add a cheap total chunk count to the index store
This commit is contained in:
4 files changed
+53
-5
No files matched your search
@@ -40,4 +40,17 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
|
||||
public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
|
||||
|
||||
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
|
||||
|
||||
/// <summary>
|
||||
/// Counts the search chunks the index holds across all data sources.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// One chunk is one vector: every chunk becomes exactly one point carrying the single named
|
||||
/// vector "embedding". The vector store reports its vector count from here, because counting
|
||||
/// the points in Qdrant Edge would have to load every shard first and would hold the global
|
||||
/// database mutex against ongoing inserts and searches while doing so.
|
||||
/// </remarks>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The number of chunks, or null when the index cannot tell. Null and zero mean different things here.</returns>
|
||||
public abstract Task<long?> GetTotalChunkCountAsync(CancellationToken token);
|
||||
}
|
||||
@@ -55,6 +55,8 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
|
||||
|
||||
public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task<long?> GetTotalChunkCountAsync(CancellationToken token) => Task.FromResult<long?>(null);
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
+25
-1
@@ -25,6 +25,8 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
|
||||
public override string CacheKey => $"{this.Name}:{this.databasePath}:{version}";
|
||||
|
||||
public override string Version => version;
|
||||
|
||||
public static async Task<DatabaseClient> CreateAsync(
|
||||
ILogger logger,
|
||||
ILogger<DatabaseClient> databaseClientLogger,
|
||||
@@ -65,7 +67,8 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
yield return (TB("Storage size"), this.GetStorageSize());
|
||||
yield return (TB("Indexed data sources"), (await context.DataSources.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Indexed files"), (await context.EmbeddedFiles.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Search chunks"), (await context.EmbeddingChunks.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
var searchChunks = await this.GetTotalChunkCountAsync(CancellationToken.None);
|
||||
yield return (TB("Search chunks"), searchChunks?.ToString(CultureInfo.InvariantCulture) ?? TB("unknown"));
|
||||
yield return (TB("Permanently skipped files"), (await context.PermanentIndexingFailures.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
@@ -348,6 +351,27 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
await transaction.CommitAsync(token);
|
||||
}
|
||||
|
||||
public override async Task<long?> GetTotalChunkCountAsync(CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
|
||||
//
|
||||
// Sum the chunk counts the files carry instead of counting the rows of the chunk table:
|
||||
// embedded_files holds one row per file, embedding_chunks one per chunk. On a large index
|
||||
// that is a difference of two orders of magnitude, and the information page reads this on
|
||||
// every visit.
|
||||
//
|
||||
return await context.EmbeddedFiles.SumAsync(file => (long)file.ChunkCount, token);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
this.Logger?.LogWarning(exception, "Failed to count the search chunks of the local RAG index.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user