mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-09-27 00:53:37 +00:00
Add a cheap total chunk count to the index store
This commit is contained in:
parent
b2f46a5611
commit
c2634bbb21
@ -8,11 +8,20 @@ public abstract class DatabaseClient(string name, string path)
|
||||
|
||||
public virtual DatabaseClientStatus Status => DatabaseClientStatus.AVAILABLE;
|
||||
|
||||
/// <summary>
|
||||
/// The version the running database reports about itself.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Empty when the client cannot tell. Callers which want to show a version in a headline read it
|
||||
/// from here instead of picking it out of the label-value pairs the display info yields.
|
||||
/// </remarks>
|
||||
public virtual string Version => string.Empty;
|
||||
|
||||
public bool IsAvailable => this.Status is DatabaseClientStatus.AVAILABLE;
|
||||
|
||||
private string Path => path;
|
||||
|
||||
private ILogger<DatabaseClient>? logger;
|
||||
protected ILogger<DatabaseClient>? Logger;
|
||||
|
||||
public abstract IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo();
|
||||
|
||||
@ -20,13 +29,13 @@ public abstract class DatabaseClient(string name, string path)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(this.Path))
|
||||
{
|
||||
this.logger!.LogError($"Error: Database path '{this.Path}' cannot be null or empty.");
|
||||
this.Logger!.LogError($"Error: Database path '{this.Path}' cannot be null or empty.");
|
||||
return "0 B";
|
||||
}
|
||||
|
||||
if (!Directory.Exists(this.Path))
|
||||
{
|
||||
this.logger!.LogError($"Error: Database path '{this.Path}' does not exist.");
|
||||
this.Logger!.LogError($"Error: Database path '{this.Path}' does not exist.");
|
||||
return "0 B";
|
||||
}
|
||||
var files = Directory.EnumerateFiles(this.Path, "*", SearchOption.AllDirectories)
|
||||
@ -52,7 +61,7 @@ public abstract class DatabaseClient(string name, string path)
|
||||
|
||||
public void SetLogger(ILogger<DatabaseClient> logService)
|
||||
{
|
||||
this.logger = logService;
|
||||
this.Logger = logService;
|
||||
}
|
||||
|
||||
public abstract void Dispose();
|
||||
|
||||
@ -40,4 +40,17 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
|
||||
public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
|
||||
|
||||
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
|
||||
|
||||
/// <summary>
|
||||
/// Counts the search chunks the index holds across all data sources.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// One chunk is one vector: every chunk becomes exactly one point carrying the single named
|
||||
/// vector "embedding". The vector store reports its vector count from here, because counting
|
||||
/// the points in Qdrant Edge would have to load every shard first and would hold the global
|
||||
/// database mutex against ongoing inserts and searches while doing so.
|
||||
/// </remarks>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The number of chunks, or null when the index cannot tell. Null and zero mean different things here.</returns>
|
||||
public abstract Task<long?> GetTotalChunkCountAsync(CancellationToken token);
|
||||
}
|
||||
|
||||
@ -55,6 +55,8 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
|
||||
|
||||
public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task<long?> GetTotalChunkCountAsync(CancellationToken token) => Task.FromResult<long?>(null);
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
@ -25,6 +25,8 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
|
||||
public override string CacheKey => $"{this.Name}:{this.databasePath}:{version}";
|
||||
|
||||
public override string Version => version;
|
||||
|
||||
public static async Task<DatabaseClient> CreateAsync(
|
||||
ILogger logger,
|
||||
ILogger<DatabaseClient> databaseClientLogger,
|
||||
@ -65,7 +67,8 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
yield return (TB("Storage size"), this.GetStorageSize());
|
||||
yield return (TB("Indexed data sources"), (await context.DataSources.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Indexed files"), (await context.EmbeddedFiles.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Search chunks"), (await context.EmbeddingChunks.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
var searchChunks = await this.GetTotalChunkCountAsync(CancellationToken.None);
|
||||
yield return (TB("Search chunks"), searchChunks?.ToString(CultureInfo.InvariantCulture) ?? TB("unknown"));
|
||||
yield return (TB("Permanently skipped files"), (await context.PermanentIndexingFailures.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
}
|
||||
|
||||
@ -348,6 +351,27 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
await transaction.CommitAsync(token);
|
||||
}
|
||||
|
||||
public override async Task<long?> GetTotalChunkCountAsync(CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
|
||||
//
|
||||
// Sum the chunk counts the files carry instead of counting the rows of the chunk table:
|
||||
// embedded_files holds one row per file, embedding_chunks one per chunk. On a large index
|
||||
// that is a difference of two orders of magnitude, and the information page reads this on
|
||||
// every visit.
|
||||
//
|
||||
return await context.EmbeddedFiles.SumAsync(file => (long)file.ChunkCount, token);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
this.Logger?.LogWarning(exception, "Failed to count the search chunks of the local RAG index.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user