Add a cheap total chunk count to the index store

This commit is contained in:
Thorsten Sommer 2026-09-17 10:33:21 +02:00
parent b2f46a5611
commit c2634bbb21
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
4 changed files with 53 additions and 5 deletions

View File

@ -8,11 +8,20 @@ public abstract class DatabaseClient(string name, string path)
public virtual DatabaseClientStatus Status => DatabaseClientStatus.AVAILABLE;
/// <summary>
/// The version the running database reports about itself.
/// </summary>
/// <remarks>
/// Empty when the client cannot tell. Callers which want to show a version in a headline read it
/// from here instead of picking it out of the label-value pairs the display info yields.
/// </remarks>
public virtual string Version => string.Empty;
public bool IsAvailable => this.Status is DatabaseClientStatus.AVAILABLE;
private string Path => path;
private ILogger<DatabaseClient>? logger;
protected ILogger<DatabaseClient>? Logger;
public abstract IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo();
@ -20,13 +29,13 @@ public abstract class DatabaseClient(string name, string path)
{
if (string.IsNullOrWhiteSpace(this.Path))
{
this.logger!.LogError($"Error: Database path '{this.Path}' cannot be null or empty.");
this.Logger!.LogError($"Error: Database path '{this.Path}' cannot be null or empty.");
return "0 B";
}
if (!Directory.Exists(this.Path))
{
this.logger!.LogError($"Error: Database path '{this.Path}' does not exist.");
this.Logger!.LogError($"Error: Database path '{this.Path}' does not exist.");
return "0 B";
}
var files = Directory.EnumerateFiles(this.Path, "*", SearchOption.AllDirectories)
@ -52,7 +61,7 @@ public abstract class DatabaseClient(string name, string path)
public void SetLogger(ILogger<DatabaseClient> logService)
{
this.logger = logService;
this.Logger = logService;
}
public abstract void Dispose();

View File

@ -40,4 +40,17 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
/// <summary>
/// Counts the search chunks the index holds across all data sources.
/// </summary>
/// <remarks>
/// One chunk is one vector: every chunk becomes exactly one point carrying the single named
/// vector "embedding". The vector store reports its vector count from here, because counting
/// the points in Qdrant Edge would have to load every shard first and would hold the global
/// database mutex against ongoing inserts and searches while doing so.
/// </remarks>
/// <param name="token">The cancellation token.</param>
/// <returns>The number of chunks, or null when the index cannot tell. Null and zero mean different things here.</returns>
public abstract Task<long?> GetTotalChunkCountAsync(CancellationToken token);
}

View File

@ -55,6 +55,8 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask;
public override Task<long?> GetTotalChunkCountAsync(CancellationToken token) => Task.FromResult<long?>(null);
public override void Dispose()
{
}

View File

@ -25,6 +25,8 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
public override string CacheKey => $"{this.Name}:{this.databasePath}:{version}";
public override string Version => version;
public static async Task<DatabaseClient> CreateAsync(
ILogger logger,
ILogger<DatabaseClient> databaseClientLogger,
@ -65,7 +67,8 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
yield return (TB("Storage size"), this.GetStorageSize());
yield return (TB("Indexed data sources"), (await context.DataSources.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
yield return (TB("Indexed files"), (await context.EmbeddedFiles.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
yield return (TB("Search chunks"), (await context.EmbeddingChunks.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
var searchChunks = await this.GetTotalChunkCountAsync(CancellationToken.None);
yield return (TB("Search chunks"), searchChunks?.ToString(CultureInfo.InvariantCulture) ?? TB("unknown"));
yield return (TB("Permanently skipped files"), (await context.PermanentIndexingFailures.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
}
@ -348,6 +351,27 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
await transaction.CommitAsync(token);
}
public override async Task<long?> GetTotalChunkCountAsync(CancellationToken token)
{
try
{
await using var context = this.CreateContext();
//
// Sum the chunk counts the files carry instead of counting the rows of the chunk table:
// embedded_files holds one row per file, embedding_chunks one per chunk. On a large index
// that is a difference of two orders of magnitude, and the information page reads this on
// every visit.
//
return await context.EmbeddedFiles.SumAsync(file => (long)file.ChunkCount, token);
}
catch (Exception exception)
{
this.Logger?.LogWarning(exception, "Failed to count the search chunks of the local RAG index.");
return null;
}
}
public override void Dispose()
{
}