mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-17 10:18:40 +00:00
Show the details of both RAG databases on the information page (#980)
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
This commit is contained in:
1 parent
b2f46a5611
commit
557d0b1409
15 files changed
+764
-100
No files matched your search
@@ -8,11 +8,20 @@ public abstract class DatabaseClient(string name, string path)
|
||||
|
||||
public virtual DatabaseClientStatus Status => DatabaseClientStatus.AVAILABLE;
|
||||
|
||||
/// <summary>
|
||||
/// The version the running database reports about itself.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Empty when the client cannot tell. Callers which want to show a version in a headline read it
|
||||
/// from here instead of picking it out of the label-value pairs the display info yields.
|
||||
/// </remarks>
|
||||
public virtual string Version => string.Empty;
|
||||
|
||||
public bool IsAvailable => this.Status is DatabaseClientStatus.AVAILABLE;
|
||||
|
||||
private string Path => path;
|
||||
|
||||
private ILogger<DatabaseClient>? logger;
|
||||
protected ILogger<DatabaseClient>? Logger;
|
||||
|
||||
public abstract IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo();
|
||||
|
||||
@@ -20,13 +29,13 @@ public abstract class DatabaseClient(string name, string path)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(this.Path))
|
||||
{
|
||||
this.logger!.LogError($"Error: Database path '{this.Path}' cannot be null or empty.");
|
||||
this.Logger!.LogError($"Error: Database path '{this.Path}' cannot be null or empty.");
|
||||
return "0 B";
|
||||
}
|
||||
|
||||
if (!Directory.Exists(this.Path))
|
||||
{
|
||||
this.logger!.LogError($"Error: Database path '{this.Path}' does not exist.");
|
||||
this.Logger!.LogError($"Error: Database path '{this.Path}' does not exist.");
|
||||
return "0 B";
|
||||
}
|
||||
var files = Directory.EnumerateFiles(this.Path, "*", SearchOption.AllDirectories)
|
||||
@@ -52,7 +61,7 @@ public abstract class DatabaseClient(string name, string path)
|
||||
|
||||
public void SetLogger(ILogger<DatabaseClient> logService)
|
||||
{
|
||||
this.logger = logService;
|
||||
this.Logger = logService;
|
||||
}
|
||||
|
||||
public abstract void Dispose();
|
||||
|
||||
@@ -69,6 +69,25 @@ public sealed class DatabaseClientProvider(RustService rustService, ILoggerFacto
|
||||
client.Status);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Builds the client which stands in for a database role that cannot serve right now.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Callers outside this namespace get their stand-in from here instead of naming the concrete
|
||||
/// type themselves, so a new role does not have to be spelled out in every one of them.
|
||||
/// </remarks>
|
||||
/// <param name="databaseRole">The role the stand-in has to fill.</param>
|
||||
/// <param name="name">The name to show for the database.</param>
|
||||
/// <param name="reason">Why the database is not available.</param>
|
||||
/// <param name="status">Whether the database is starting or unavailable.</param>
|
||||
/// <returns>A client which answers every operation without a database behind it.</returns>
|
||||
public static DatabaseClient CreateUnavailableClient(DatabaseRole databaseRole, string name, string? reason, DatabaseClientStatus status) => databaseRole switch
|
||||
{
|
||||
DatabaseRole.VECTOR_STORE => new NoVectorStoreClient(name, reason, status),
|
||||
DatabaseRole.INDEX_STORE => new NoIndexStoreClient(name, reason, status),
|
||||
_ => new NoDatabaseClient(name, reason, status)
|
||||
};
|
||||
|
||||
private DatabaseClient CacheIfAvailable(DatabaseRole databaseRole, DatabaseClient client)
|
||||
{
|
||||
if (!client.IsAvailable)
|
||||
@@ -104,7 +123,7 @@ public sealed class DatabaseClientProvider(RustService rustService, ILoggerFacto
|
||||
|
||||
private async Task<DatabaseClient> CreateClientAsync(DatabaseRole databaseRole, CancellationToken cancellationToken) => databaseRole switch
|
||||
{
|
||||
DatabaseRole.VECTOR_STORE => await QdrantEdgeClientImplementation.CreateAsync(rustService, this.logger, this.databaseClientLogger, cancellationToken),
|
||||
DatabaseRole.VECTOR_STORE => await QdrantEdgeClientImplementation.CreateAsync(rustService, this.GetIndexStoreAsync, this.logger, this.databaseClientLogger, cancellationToken),
|
||||
DatabaseRole.INDEX_STORE => await SqliteIndexStoreClientImplementation.CreateAsync(this.logger, this.databaseClientLogger, cancellationToken),
|
||||
_ => new NoDatabaseClient(databaseRole.ToString(), "The requested database role is not supported.")
|
||||
};
|
||||
|
||||
@@ -40,4 +40,17 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
|
||||
public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
|
||||
|
||||
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
|
||||
|
||||
/// <summary>
|
||||
/// Counts the search chunks the index holds across all data sources.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// One chunk is one vector: every chunk becomes exactly one point carrying the single named
|
||||
/// vector "embedding". The vector store reports its vector count from here, because counting
|
||||
/// the points in Qdrant Edge would have to load every shard first and would hold the global
|
||||
/// database mutex against ongoing inserts and searches while doing so.
|
||||
/// </remarks>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The number of chunks, or null when the index cannot tell. Null and zero mean different things here.</returns>
|
||||
public abstract Task<long?> GetTotalChunkCountAsync(CancellationToken token);
|
||||
}
|
||||
@@ -20,9 +20,19 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
|
||||
if (!string.IsNullOrWhiteSpace(unavailableReason))
|
||||
yield return (TB("Reason"), unavailableReason);
|
||||
|
||||
//
|
||||
// Say which native library this process bound to even though the database itself is out of
|
||||
// reach. When SQLite cannot be loaded on a platform at all, this client is exactly what the
|
||||
// user sees, so this is the one place where those details matter most.
|
||||
//
|
||||
yield return (TB("Native library"), OrUnknown(SqliteRuntimeInfo.GetNativeLibraryName()));
|
||||
yield return (TB("Process architecture"), OrUnknown(SqliteRuntimeInfo.GetProcessArchitecture()));
|
||||
|
||||
await Task.CompletedTask;
|
||||
}
|
||||
|
||||
private static string OrUnknown(string value) => string.IsNullOrWhiteSpace(value) ? TB("unknown") : value;
|
||||
|
||||
public override Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token) => Task.FromResult(new DataSourceEmbeddingManifest());
|
||||
|
||||
public override Task<DataSourceIndexState?> GetDataSourceStateAsync(string dataSourceId, CancellationToken token) => Task.FromResult<DataSourceIndexState?>(null);
|
||||
@@ -55,6 +65,8 @@ public sealed class NoIndexStoreClient(string name, string? unavailableReason, D
|
||||
|
||||
public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task<long?> GetTotalChunkCountAsync(CancellationToken token) => Task.FromResult<long?>(null);
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
+186
-8
@@ -1,3 +1,4 @@
|
||||
using System.Data;
|
||||
using System.Globalization;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
@@ -11,7 +12,7 @@ namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
public sealed class SqliteIndexStoreClientImplementation(string name, string databasePath, string basePath, string version) : IndexStoreClient(name, basePath)
|
||||
{
|
||||
private const string DATABASE_NAME = "Local RAG Index";
|
||||
private const string DATABASE_NAME = "SQLite";
|
||||
private const string DATABASE_FILENAME = "rag-index.sqlite3";
|
||||
private const int MAX_FTS_QUERY_TERMS = 32;
|
||||
private const int CHUNK_UPSERT_BATCH_SIZE = 500;
|
||||
@@ -25,6 +26,8 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
|
||||
public override string CacheKey => $"{this.Name}:{this.databasePath}:{version}";
|
||||
|
||||
public override string Version => version;
|
||||
|
||||
public static async Task<DatabaseClient> CreateAsync(
|
||||
ILogger logger,
|
||||
ILogger<DatabaseClient> databaseClientLogger,
|
||||
@@ -58,15 +61,33 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
|
||||
public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo()
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
//
|
||||
// Read everything before yielding the first line: this is an iterator, and a try/catch
|
||||
// cannot wrap a yield return. Each probe therefore catches its own failure and answers
|
||||
// with an empty string, which shows up as "unknown" below. Without that, a single failing
|
||||
// PRAGMA would throw out of here and the information page would replace the entire block
|
||||
// with the fallback client.
|
||||
//
|
||||
var snapshot = await this.ReadDisplaySnapshotAsync();
|
||||
|
||||
yield return (TB("Reported version"), version);
|
||||
yield return (TB("Database path"), this.databasePath);
|
||||
yield return (TB("Native library"), OrUnknown(SqliteRuntimeInfo.GetNativeLibraryName()));
|
||||
yield return (TB("Wrapper version"), OrUnknown(SqliteRuntimeInfo.GetWrapperVersion()));
|
||||
yield return (TB("Process architecture"), OrUnknown(SqliteRuntimeInfo.GetProcessArchitecture()));
|
||||
|
||||
// Only worth a line when the process runs on a foreign architecture, Rosetta above all:
|
||||
var systemArchitecture = SqliteRuntimeInfo.GetSystemArchitecture();
|
||||
if (!string.IsNullOrWhiteSpace(systemArchitecture))
|
||||
yield return (TB("System architecture"), systemArchitecture);
|
||||
|
||||
yield return (TB("Full-text search (FTS5)"), OrUnknown(snapshot.FullTextSearch));
|
||||
yield return (TB("Journal mode"), OrUnknown(snapshot.JournalMode));
|
||||
yield return (TB("Schema version"), OrUnknown(snapshot.SchemaVersion));
|
||||
yield return (TB("Database tables"), OrUnknown(snapshot.TableCount));
|
||||
yield return (TB("Storage size"), this.GetStorageSize());
|
||||
yield return (TB("Indexed data sources"), (await context.DataSources.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Indexed files"), (await context.EmbeddedFiles.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Search chunks"), (await context.EmbeddingChunks.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Permanently skipped files"), (await context.PermanentIndexingFailures.CountAsync(CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
|
||||
yield return (TB("Indexed data sources"), OrUnknown(snapshot.DataSourceCount));
|
||||
yield return (TB("Indexed files"), OrUnknown(snapshot.FileCount));
|
||||
yield return (TB("Permanently skipped files"), OrUnknown(snapshot.FailureCount));
|
||||
}
|
||||
|
||||
public override async Task<DataSourceIndexState?> GetDataSourceStateAsync(string dataSourceId, CancellationToken token)
|
||||
@@ -348,10 +369,167 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
await transaction.CommitAsync(token);
|
||||
}
|
||||
|
||||
public override async Task<long?> GetTotalChunkCountAsync(CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
|
||||
//
|
||||
// Sum the chunk counts the files carry instead of counting the rows of the chunk table:
|
||||
// embedded_files holds one row per file, embedding_chunks one per chunk. On a large index
|
||||
// that is a difference of two orders of magnitude, and the information page reads this on
|
||||
// every visit.
|
||||
//
|
||||
return await context.EmbeddedFiles.SumAsync(file => (long)file.ChunkCount, token);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
this.Logger?.LogWarning(exception, "Failed to count the search chunks of the local RAG index.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Everything the display info reads out of the database in one go.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Every property is empty when its probe could not answer. The caller turns that into "unknown".
|
||||
/// </remarks>
|
||||
private sealed record DisplaySnapshot
|
||||
{
|
||||
public string FullTextSearch { get; init; } = string.Empty;
|
||||
|
||||
public string JournalMode { get; init; } = string.Empty;
|
||||
|
||||
public string SchemaVersion { get; init; } = string.Empty;
|
||||
|
||||
public string TableCount { get; init; } = string.Empty;
|
||||
|
||||
public string DataSourceCount { get; init; } = string.Empty;
|
||||
|
||||
public string FileCount { get; init; } = string.Empty;
|
||||
|
||||
public string FailureCount { get; init; } = string.Empty;
|
||||
}
|
||||
|
||||
private static string OrUnknown(string value) => string.IsNullOrWhiteSpace(value) ? TB("unknown") : value;
|
||||
|
||||
private async Task<DisplaySnapshot> ReadDisplaySnapshotAsync()
|
||||
{
|
||||
var token = CancellationToken.None;
|
||||
try
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
return new DisplaySnapshot
|
||||
{
|
||||
FullTextSearch = await GetFullTextSearchStateAsync(context, token),
|
||||
JournalMode = (await QueryScalarTextAsync(context, "PRAGMA journal_mode;", token)).ToUpperInvariant(),
|
||||
SchemaVersion = await GetSchemaVersionAsync(context, token),
|
||||
TableCount = await GetTableCountAsync(context, token),
|
||||
DataSourceCount = await FormatCountAsync(context.DataSources, token),
|
||||
FileCount = await FormatCountAsync(context.EmbeddedFiles, token),
|
||||
FailureCount = await FormatCountAsync(context.PermanentIndexingFailures, token),
|
||||
};
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
//
|
||||
// Opening the database failed altogether. The runtime details the caller shows next to
|
||||
// these values still say which library was loaded and for which architecture, which is
|
||||
// what a support case needs most in exactly this situation. So hand back an empty
|
||||
// snapshot instead of letting the whole block fall back.
|
||||
//
|
||||
this.Logger?.LogWarning(exception, "Failed to read the display details of the local RAG index.");
|
||||
return new DisplaySnapshot();
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<string> QueryScalarTextAsync(IndexStoreDbContext context, string sql, CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
//
|
||||
// Go through the raw connection rather than through SqlQueryRaw: that one expects a
|
||||
// column named "Value" and wraps the statement, neither of which works for a PRAGMA.
|
||||
//
|
||||
var connection = context.Database.GetDbConnection();
|
||||
if (connection.State is not ConnectionState.Open)
|
||||
await connection.OpenAsync(token);
|
||||
|
||||
await using var command = connection.CreateCommand();
|
||||
command.CommandText = sql;
|
||||
|
||||
var result = await command.ExecuteScalarAsync(token);
|
||||
return result?.ToString() ?? string.Empty;
|
||||
}
|
||||
catch
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<string> GetFullTextSearchStateAsync(IndexStoreDbContext context, CancellationToken token)
|
||||
{
|
||||
var compiledIn = await QueryScalarTextAsync(context, "SELECT sqlite_compileoption_used('ENABLE_FTS5')", token);
|
||||
return compiledIn switch
|
||||
{
|
||||
"1" => TB("available"),
|
||||
"0" => TB("not available"),
|
||||
_ => string.Empty
|
||||
};
|
||||
}
|
||||
|
||||
private static async Task<string> GetTableCountAsync(IndexStoreDbContext context, CancellationToken token)
|
||||
{
|
||||
//
|
||||
// Counts the migration history, the FTS5 virtual table and its shadow tables as well. That
|
||||
// is the point: a missing shadow table is a finding, not noise.
|
||||
//
|
||||
var tables = await QueryScalarTextAsync(context, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'", token);
|
||||
return int.TryParse(tables, NumberStyles.Integer, CultureInfo.InvariantCulture, out var tableCount) ? tableCount.CompactCount() : string.Empty;
|
||||
}
|
||||
|
||||
private static async Task<string> GetSchemaVersionAsync(IndexStoreDbContext context, CancellationToken token)
|
||||
{
|
||||
try
|
||||
{
|
||||
//
|
||||
// Reading the applied migrations only touches the history table, no assembly scan. The
|
||||
// pending ones do scan, but the schema migrator walks that same path on every start, so
|
||||
// the DynamicDependency attributes over there already keep the migration types alive.
|
||||
//
|
||||
var appliedMigrations = (await context.Database.GetAppliedMigrationsAsync(token)).ToList();
|
||||
if (appliedMigrations.Count == 0)
|
||||
return TB("no migration applied");
|
||||
|
||||
var pendingMigrations = (await context.Database.GetPendingMigrationsAsync(token)).ToList();
|
||||
return pendingMigrations.Count == 0
|
||||
? string.Format(I18N.I.Culture, TB("{0} ({1} applied)"), appliedMigrations[^1], appliedMigrations.Count.CompactCount())
|
||||
: string.Format(I18N.I.Culture, TB("{0} ({1} applied, {2} pending)"), appliedMigrations[^1], appliedMigrations.Count.CompactCount(), pendingMigrations.Count.CompactCount());
|
||||
}
|
||||
catch
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<string> FormatCountAsync<T>(IQueryable<T> query, CancellationToken token) where T : class
|
||||
{
|
||||
try
|
||||
{
|
||||
return (await query.CountAsync(token)).CompactCount();
|
||||
}
|
||||
catch
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task InitializeAsync(CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
@@ -468,4 +646,4 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
|
||||
client.SetLogger(databaseClientLogger);
|
||||
return client;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
using System.Runtime.InteropServices;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
/// <summary>
|
||||
/// What the running process can tell about the SQLite library it has loaded.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// These are methods instead of static fields on purpose: SQLitePCL.Batteries_V2.Init() runs when the
|
||||
/// index store client is created, and a type initializer could well run before that. Every member
|
||||
/// answers with an empty string when it cannot tell, so a single unavailable detail never costs the
|
||||
/// caller the rest of them.
|
||||
/// </remarks>
|
||||
internal static class SqliteRuntimeInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// The name of the native library SQLitePCLRaw has bound to.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// We ship our own build through the bundle_e_sqlite3 package, so this reads e_sqlite3 on every
|
||||
/// platform. Anything else means the process bound to a different library than we shipped, which
|
||||
/// is exactly the kind of thing a support case needs to show.
|
||||
/// </remarks>
|
||||
public static string GetNativeLibraryName()
|
||||
{
|
||||
try
|
||||
{
|
||||
return SQLitePCL.raw.GetNativeLibraryName();
|
||||
}
|
||||
catch
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The version of the managed SQLitePCLRaw wrapper, which is a different thing than the SQLite version.
|
||||
/// </summary>
|
||||
public static string GetWrapperVersion()
|
||||
{
|
||||
try
|
||||
{
|
||||
// Read the assembly name rather than an attribute: reflecting over members would not
|
||||
// survive trimming, the name does.
|
||||
var wrapperVersion = typeof(SQLitePCL.raw).Assembly.GetName().Version;
|
||||
return wrapperVersion is null ? string.Empty : $"SQLitePCLRaw.core {wrapperVersion.ToString(3)}";
|
||||
}
|
||||
catch
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The architecture this process runs as, together with the runtime identifier it was built for.
|
||||
/// </summary>
|
||||
public static string GetProcessArchitecture()
|
||||
{
|
||||
try
|
||||
{
|
||||
return $"{RuntimeInformation.ProcessArchitecture} ({RuntimeInformation.RuntimeIdentifier})";
|
||||
}
|
||||
catch
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The architecture of the machine, but only when it differs from the one of the process.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A difference means the process runs through an emulation layer, Rosetta above all. That is a
|
||||
/// classic reason for a native library failing to load, so it earns its own line when it happens
|
||||
/// and stays out of the way when it does not.
|
||||
/// </remarks>
|
||||
public static string GetSystemArchitecture()
|
||||
{
|
||||
try
|
||||
{
|
||||
return RuntimeInformation.OSArchitecture == RuntimeInformation.ProcessArchitecture ? string.Empty : RuntimeInformation.OSArchitecture.ToString();
|
||||
}
|
||||
catch
|
||||
{
|
||||
return string.Empty;
|
||||
}
|
||||
}
|
||||
}
|
||||
+39
-3
@@ -1,15 +1,24 @@
|
||||
using AIStudio.Tools.Databases.IndexStore;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.Rust;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
namespace AIStudio.Tools.Databases.VectorStore;
|
||||
|
||||
/// <param name="indexStoreAccessor">
|
||||
/// Resolves the index store, which is where the number of stored vectors comes from. Counting the
|
||||
/// points in Qdrant Edge itself would have to load every shard first and would hold the global
|
||||
/// database mutex against ongoing inserts and searches. The accessor is only called while building
|
||||
/// the display info, never while this client is created: creating it already holds the vector store
|
||||
/// lock, and the accessor takes the index store lock, so the two are never held at the same time.
|
||||
/// </param>
|
||||
public sealed class QdrantEdgeClientImplementation(
|
||||
string name,
|
||||
string path,
|
||||
string version,
|
||||
int storesCount,
|
||||
RustService rustService) : VectorStoreClient(name, path)
|
||||
RustService rustService,
|
||||
Func<CancellationToken, Task<IndexStoreClient>> indexStoreAccessor) : VectorStoreClient(name, path)
|
||||
{
|
||||
private const string DATABASE_NAME = "Qdrant Edge";
|
||||
private const string INFO_PATH = "/system/qdrant-edge/info";
|
||||
@@ -28,6 +37,7 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
|
||||
public static async Task<DatabaseClient> CreateAsync(
|
||||
RustService rustService,
|
||||
Func<CancellationToken, Task<IndexStoreClient>> indexStoreAccessor,
|
||||
ILogger logger,
|
||||
ILogger<DatabaseClient> databaseClientLogger,
|
||||
CancellationToken cancellationToken)
|
||||
@@ -60,7 +70,7 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
return CreateNoVectorStoreClient(DATABASE_NAME, $"Failed to get the {DATABASE_NAME} path from Rust.", DatabaseClientStatus.UNAVAILABLE, databaseClientLogger);
|
||||
|
||||
var name = string.IsNullOrWhiteSpace(qdrantEdgeInfo.Name) ? DATABASE_NAME : qdrantEdgeInfo.Name;
|
||||
var client = new QdrantEdgeClientImplementation(name, qdrantEdgeInfo.Path, qdrantEdgeInfo.Version, qdrantEdgeInfo.StoresCount, rustService);
|
||||
var client = new QdrantEdgeClientImplementation(name, qdrantEdgeInfo.Path, qdrantEdgeInfo.Version, qdrantEdgeInfo.StoresCount, rustService, indexStoreAccessor);
|
||||
client.SetLogger(databaseClientLogger);
|
||||
return client;
|
||||
}
|
||||
@@ -77,9 +87,35 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
if (!currentInfo.IsAvailable)
|
||||
yield return (TB("Status"), currentInfo.UnavailableReason ?? TB("Qdrant Edge is not available."));
|
||||
|
||||
var storedVectors = await this.GetStoredVectorCountAsync();
|
||||
|
||||
yield return (TB("Reported version"), displayVersion);
|
||||
yield return (TB("Storage size"), $"{this.GetStorageSize()}");
|
||||
yield return (TB("Number of vector stores"), displayStoresCount.ToString());
|
||||
yield return (TB("Number of vector stores"), displayStoresCount.CompactCount());
|
||||
yield return (TB("Stored vectors"), storedVectors?.CompactCount() ?? TB("unknown"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads how many vectors the vector stores hold in total.
|
||||
/// </summary>
|
||||
/// <returns>The number of vectors, or null when the index store cannot tell.</returns>
|
||||
private async Task<long?> GetStoredVectorCountAsync()
|
||||
{
|
||||
try
|
||||
{
|
||||
//
|
||||
// One chunk is one vector: every chunk becomes exactly one point carrying the single
|
||||
// named vector "embedding". So the index store knows this number without Qdrant Edge
|
||||
// having to load a single shard for it.
|
||||
//
|
||||
var indexStore = await indexStoreAccessor(CancellationToken.None);
|
||||
return await indexStore.GetTotalChunkCountAsync(CancellationToken.None);
|
||||
}
|
||||
catch (Exception exception)
|
||||
{
|
||||
this.Logger?.LogWarning(exception, "Failed to read the number of stored vectors from the index store.");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public override async Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) =>
|
||||
|
||||
@@ -1,7 +1,46 @@
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
public static class LongExtensions
|
||||
{
|
||||
private static readonly string[] COUNT_SUFFIXES = ["k", "M", "B", "T"];
|
||||
|
||||
/// <summary>
|
||||
/// Formats a count so that large numbers stay readable: 1456 becomes 1.46k, 4512900 becomes 4.51M.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Counts scale in thousands, not in steps of 1024 — that is what FileSize is for, and storage
|
||||
/// sizes keep using it. Numbers below 1000 stay exact, because shortening them would hide the
|
||||
/// difference between 4 and 999. The decimal separator follows the culture of the active language
|
||||
/// plugin rather than the one of the thread, which never moves along with the app's language.
|
||||
/// </remarks>
|
||||
/// <param name="count">The number to format.</param>
|
||||
/// <returns>The formatted number.</returns>
|
||||
public static string CompactCount(this long count)
|
||||
{
|
||||
var culture = I18N.I.Culture;
|
||||
if (count is > -1_000 and < 1_000)
|
||||
return count.ToString("N0", culture);
|
||||
|
||||
var order = -1;
|
||||
double value = count;
|
||||
while (Math.Abs(value) >= 1_000 && order < COUNT_SUFFIXES.Length - 1)
|
||||
{
|
||||
order++;
|
||||
value /= 1_000;
|
||||
}
|
||||
|
||||
return $"{value.ToString("0.##", culture)}{COUNT_SUFFIXES[order]}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Formats a count so that large numbers stay readable.
|
||||
/// </summary>
|
||||
/// <param name="count">The number to format.</param>
|
||||
/// <returns>The formatted number.</returns>
|
||||
public static string CompactCount(this int count) => ((long)count).CompactCount();
|
||||
|
||||
/// <summary>
|
||||
/// Formats the file size in a human-readable format.
|
||||
/// </summary>
|
||||
|
||||
Reference in new issue
Block a user