refactored the classes and functions

This commit is contained in:
PaulKoudelka 2026-08-14 14:25:53 +02:00
parent 1a263127f5
commit 15afe1c80c
29 changed files with 297 additions and 321 deletions

View File

@ -32,15 +32,7 @@ public sealed record EmbeddingProvider(
public static readonly EmbeddingProvider NONE = new();
public EmbeddingProvider() : this(
0,
Guid.Empty.ToString(),
string.Empty,
LLMProviders.NONE,
default,
false,
false,
Guid.Empty)
public EmbeddingProvider() : this(0, Guid.Empty.ToString(), string.Empty, LLMProviders.NONE, default, false, false, Guid.Empty)
{
}

View File

@ -41,15 +41,7 @@ public sealed record Provider(
public static readonly Provider NONE = new();
public Provider() : this(
0,
Guid.Empty.ToString(),
string.Empty,
LLMProviders.NONE,
default,
false,
false,
Guid.Empty)
public Provider() : this(0, Guid.Empty.ToString(), string.Empty, LLMProviders.NONE, default, false, false, Guid.Empty)
{
}

View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Databases.IndexStore;
public sealed record EmbeddingStateChunk(string ChunkId, string ParentFileId, int? PageNumber, int ChunkIndex, string ChunkText, DateTimeOffset EmbeddedAtUtc);

View File

@ -0,0 +1,20 @@
namespace AIStudio.Tools.Databases.IndexStore;
internal sealed class EmbeddingStateChunkEntity
{
public int Id { get; set; }
public string ChunkId { get; set; } = string.Empty;
public string ParentFileId { get; set; } = string.Empty;
public int? PageNumber { get; set; }
public int ChunkIndex { get; set; }
public string ChunkText { get; set; } = string.Empty;
public DateTimeOffset EmbeddedAtUtc { get; set; }
public EmbeddingStateFileEntity? File { get; set; }
}

View File

@ -0,0 +1,22 @@
namespace AIStudio.Tools.Databases.IndexStore;
internal sealed class EmbeddingStateDataSourceEntity
{
public string DataSourceId { get; set; } = string.Empty;
public string DataSourceName { get; set; } = string.Empty;
public string DataSourceType { get; set; } = string.Empty;
public string EmbeddingProviderId { get; set; } = string.Empty;
public string EmbeddingSignature { get; set; } = string.Empty;
public string SourceHash { get; set; } = string.Empty;
public int VectorSize { get; set; }
public DateTimeOffset UpdatedAtUtc { get; set; }
public List<EmbeddingStateFileEntity> Files { get; set; } = [];
}

View File

@ -0,0 +1,16 @@
namespace AIStudio.Tools.Databases.IndexStore;
public sealed record EmbeddingStateFile(
string ParentFileId,
string AbsolutePath,
string FileName,
string RelativePath,
string FileType,
string Fingerprint,
long FileSize,
DateTimeOffset CreationUtc,
DateTimeOffset LastWriteUtc,
DateTimeOffset EmbeddedAtUtc,
int ChunkCount,
string ConfidenceLevel,
int ConfidenceLevelRank);

View File

@ -0,0 +1,36 @@
namespace AIStudio.Tools.Databases.IndexStore;
internal sealed class EmbeddingStateFileEntity
{
public string ParentFileId { get; set; } = string.Empty;
public string DataSourceId { get; set; } = string.Empty;
public string AbsolutePath { get; set; } = string.Empty;
public string FileName { get; set; } = string.Empty;
public string RelativePath { get; set; } = string.Empty;
public string FileType { get; set; } = string.Empty;
public string Fingerprint { get; set; } = string.Empty;
public long FileSize { get; set; }
public DateTimeOffset CreationUtc { get; set; }
public DateTimeOffset LastWriteUtc { get; set; }
public DateTimeOffset EmbeddedAtUtc { get; set; }
public int ChunkCount { get; set; }
public string ConfidenceLevel { get; set; } = string.Empty;
public int ConfidenceLevelRank { get; set; }
public EmbeddingStateDataSourceEntity? DataSource { get; set; }
public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];
}

View File

@ -30,49 +30,3 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
}
public sealed record EmbeddingStateFile(
string ParentFileId,
string AbsolutePath,
string FileName,
string RelativePath,
string FileType,
string Fingerprint,
long FileSize,
DateTimeOffset CreationUtc,
DateTimeOffset LastWriteUtc,
DateTimeOffset EmbeddedAtUtc,
int ChunkCount,
string ConfidenceLevel,
int ConfidenceLevelRank);
public sealed record EmbeddingStateChunk(
string ChunkId,
string ParentFileId,
int? PageNumber,
int ChunkIndex,
string ChunkText,
DateTimeOffset EmbeddedAtUtc);
public sealed record IndexStoreSearchResult(
string ChunkId,
string ParentFileId,
string DataSourceId,
string DataSourceName,
string DataSourceType,
string AbsolutePath,
string FileName,
string RelativePath,
string FileType,
int? PageNumber,
int ChunkIndex,
string ChunkText,
double Score,
string Fingerprint,
long FileSize,
DateTimeOffset CreationUtc,
DateTimeOffset LastWriteUtc,
DateTimeOffset EmbeddedAtUtc,
int ChunkCount,
string ConfidenceLevel,
int ConfidenceLevelRank);

View File

@ -0,0 +1,18 @@
using System.Globalization;
namespace AIStudio.Tools.Databases.IndexStore;
internal static class IndexStoreDateTimeOffset
{
public static string ToUtcText(DateTimeOffset dateTime)
{
return dateTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
}
public static DateTimeOffset ParseUtc(string value)
{
return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateTime)
? dateTime.ToUniversalTime()
: DateTimeOffset.UnixEpoch;
}
}

View File

@ -0,0 +1,7 @@
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
namespace AIStudio.Tools.Databases.IndexStore;
internal sealed class IndexStoreDateTimeOffsetConverter() : ValueConverter<DateTimeOffset, string>(
value => IndexStoreDateTimeOffset.ToUtcText(value),
value => IndexStoreDateTimeOffset.ParseUtc(value));

View File

@ -1,8 +1,5 @@
using System.Globalization;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
namespace AIStudio.Tools.Databases.IndexStore;
@ -117,142 +114,3 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
DefaultTimeout = 30,
}.ToString();
}
internal sealed class EmbeddingStateDataSourceEntity
{
public string DataSourceId { get; set; } = string.Empty;
public string DataSourceName { get; set; } = string.Empty;
public string DataSourceType { get; set; } = string.Empty;
public string EmbeddingProviderId { get; set; } = string.Empty;
public string EmbeddingSignature { get; set; } = string.Empty;
public string SourceHash { get; set; } = string.Empty;
public int VectorSize { get; set; }
public DateTimeOffset UpdatedAtUtc { get; set; }
public List<EmbeddingStateFileEntity> Files { get; set; } = [];
}
internal sealed class EmbeddingStateFileEntity
{
public string ParentFileId { get; set; } = string.Empty;
public string DataSourceId { get; set; } = string.Empty;
public string AbsolutePath { get; set; } = string.Empty;
public string FileName { get; set; } = string.Empty;
public string RelativePath { get; set; } = string.Empty;
public string FileType { get; set; } = string.Empty;
public string Fingerprint { get; set; } = string.Empty;
public long FileSize { get; set; }
public DateTimeOffset CreationUtc { get; set; }
public DateTimeOffset LastWriteUtc { get; set; }
public DateTimeOffset EmbeddedAtUtc { get; set; }
public int ChunkCount { get; set; }
public string ConfidenceLevel { get; set; } = string.Empty;
public int ConfidenceLevelRank { get; set; }
public EmbeddingStateDataSourceEntity? DataSource { get; set; }
public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];
}
internal sealed class EmbeddingStateChunkEntity
{
public int Id { get; set; }
public string ChunkId { get; set; } = string.Empty;
public string ParentFileId { get; set; } = string.Empty;
public int? PageNumber { get; set; }
public int ChunkIndex { get; set; }
public string ChunkText { get; set; } = string.Empty;
public DateTimeOffset EmbeddedAtUtc { get; set; }
public EmbeddingStateFileEntity? File { get; set; }
}
internal sealed class IndexStoreSearchResultEntity
{
public string ChunkId { get; set; } = string.Empty;
public string ParentFileId { get; set; } = string.Empty;
public string DataSourceId { get; set; } = string.Empty;
public string DataSourceName { get; set; } = string.Empty;
public string DataSourceType { get; set; } = string.Empty;
public string AbsolutePath { get; set; } = string.Empty;
public string FileName { get; set; } = string.Empty;
public string RelativePath { get; set; } = string.Empty;
public string FileType { get; set; } = string.Empty;
public int? PageNumber { get; set; }
public int ChunkIndex { get; set; }
public string ChunkText { get; set; } = string.Empty;
public double Score { get; set; }
public string Fingerprint { get; set; } = string.Empty;
public long FileSize { get; set; }
public DateTimeOffset CreationUtc { get; set; }
public DateTimeOffset LastWriteUtc { get; set; }
public DateTimeOffset EmbeddedAtUtc { get; set; }
public int ChunkCount { get; set; }
public string ConfidenceLevel { get; set; } = string.Empty;
public int ConfidenceLevelRank { get; set; }
}
internal sealed class IndexStoreDateTimeOffsetConverter() : ValueConverter<DateTimeOffset, string>(
value => IndexStoreDateTimeOffset.ToUtcText(value),
value => IndexStoreDateTimeOffset.ParseUtc(value));
internal static class IndexStoreDateTimeOffset
{
public static string ToUtcText(DateTimeOffset dateTime)
{
return dateTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
}
public static DateTimeOffset ParseUtc(string value)
{
return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateTime)
? dateTime.ToUniversalTime()
: DateTimeOffset.UnixEpoch;
}
}

View File

@ -0,0 +1,24 @@
namespace AIStudio.Tools.Databases.IndexStore;
public sealed record IndexStoreSearchResult(
string ChunkId,
string ParentFileId,
string DataSourceId,
string DataSourceName,
string DataSourceType,
string AbsolutePath,
string FileName,
string RelativePath,
string FileType,
int? PageNumber,
int ChunkIndex,
string ChunkText,
double Score,
string Fingerprint,
long FileSize,
DateTimeOffset CreationUtc,
DateTimeOffset LastWriteUtc,
DateTimeOffset EmbeddedAtUtc,
int ChunkCount,
string ConfidenceLevel,
int ConfidenceLevelRank);

View File

@ -0,0 +1,46 @@
namespace AIStudio.Tools.Databases.IndexStore;
internal sealed class IndexStoreSearchResultEntity
{
public string ChunkId { get; set; } = string.Empty;
public string ParentFileId { get; set; } = string.Empty;
public string DataSourceId { get; set; } = string.Empty;
public string DataSourceName { get; set; } = string.Empty;
public string DataSourceType { get; set; } = string.Empty;
public string AbsolutePath { get; set; } = string.Empty;
public string FileName { get; set; } = string.Empty;
public string RelativePath { get; set; } = string.Empty;
public string FileType { get; set; } = string.Empty;
public int? PageNumber { get; set; }
public int ChunkIndex { get; set; }
public string ChunkText { get; set; } = string.Empty;
public double Score { get; set; }
public string Fingerprint { get; set; } = string.Empty;
public long FileSize { get; set; }
public DateTimeOffset CreationUtc { get; set; }
public DateTimeOffset LastWriteUtc { get; set; }
public DateTimeOffset EmbeddedAtUtc { get; set; }
public int ChunkCount { get; set; }
public string ConfidenceLevel { get; set; } = string.Empty;
public int ConfidenceLevelRank { get; set; }
}

View File

@ -9,11 +9,7 @@ using Microsoft.EntityFrameworkCore;
namespace AIStudio.Tools.Databases.IndexStore;
public sealed class SqliteIndexStoreClientImplementation(
string name,
string databasePath,
string basePath,
string version) : IndexStoreClient(name, basePath)
public sealed class SqliteIndexStoreClientImplementation(string name, string databasePath, string basePath, string version) : IndexStoreClient(name, basePath)
{
private const string DATABASE_NAME = "Local RAG Index";
private const string DATABASE_FILENAME = "rag-index.sqlite3";

View File

@ -14,5 +14,3 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
public abstract Task DeleteVectorStore(string storeName, CancellationToken token);
}
public sealed record VectorStoreEnsureResult(bool Created);

View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Databases.VectorStore;
public sealed record VectorStoreEnsureResult(bool Created);

View File

@ -1,7 +1,3 @@
namespace AIStudio.Tools.Rust;
public readonly record struct TokenizerResponse(
bool Success,
int TokenCount,
string Message,
string StoredPath = "");
public readonly record struct TokenizerResponse(bool Success, int TokenCount, string Message, string StoredPath = "");

View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Services;
public sealed record ArbitraryFileDataSegment(string Content, int TokenCount);

View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Services;
public sealed record DataSourceEmbeddingFailure(string FilePath, string Reason);

View File

@ -0,0 +1,14 @@
namespace AIStudio.Tools.Services;
public sealed class DataSourceEmbeddingManifest
{
public string EmbeddingProviderId { get; set; } = string.Empty;
public string EmbeddingSignature { get; set; } = string.Empty;
public string SourceHash { get; set; } = string.Empty;
public int VectorSize { get; set; }
public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);
}

View File

@ -1,97 +0,0 @@
using AIStudio.Settings.DataModel;
using AIStudio.Tools.PluginSystem;
namespace AIStudio.Tools.Services;
public sealed record DataSourceEmbeddingOverview(
bool IsVisible,
DataSourceEmbeddingState State,
int IndexedFiles,
int TotalFiles,
int FailedFiles);
public enum DataSourceEmbeddingState
{
IDLE,
QUEUED,
RUNNING,
COMPLETED,
FAILED,
}
public sealed record DataSourceEmbeddingStatus(
string DataSourceId,
string DataSourceName,
DataSourceType DataSourceType,
DataSourceEmbeddingState State,
int TotalFiles,
int IndexedFiles,
int FailedFiles,
string CurrentFile,
string LastError,
IReadOnlyList<DataSourceEmbeddingFailure> Failures)
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService));
public int ProgressPercent => this.TotalFiles <= 0 ? 0 : Math.Clamp((int)Math.Round(this.IndexedFiles * 100d / this.TotalFiles), 0, 100);
public string StateLabel => this.State switch
{
DataSourceEmbeddingState.QUEUED => TB("Queued"),
DataSourceEmbeddingState.RUNNING => TB("Running"),
DataSourceEmbeddingState.COMPLETED => TB("Completed"),
DataSourceEmbeddingState.FAILED => TB("Needs attention"),
_ => TB("Idle")
};
public int SortOrder => this.State switch
{
DataSourceEmbeddingState.RUNNING => 0,
DataSourceEmbeddingState.QUEUED => 1,
DataSourceEmbeddingState.FAILED => 2,
DataSourceEmbeddingState.COMPLETED => 3,
_ => 4,
};
}
public sealed record DataSourceEmbeddingFailure(
string FilePath,
string Reason);
public sealed class FileEnumerationResult
{
public List<FileInfo> Files { get; } = [];
public List<DataSourceEmbeddingFailure> Failures { get; } = [];
public int FailedFiles { get; set; }
public string LastError { get; set; } = string.Empty;
public void AddFailure(string filePath, string reason)
{
this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason));
this.FailedFiles = this.Failures.Count;
this.LastError = reason;
}
}
public sealed class DataSourceEmbeddingManifest
{
public string EmbeddingProviderId { get; set; } = string.Empty;
public string EmbeddingSignature { get; set; } = string.Empty;
public string SourceHash { get; set; } = string.Empty;
public int VectorSize { get; set; }
public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);
}
public sealed record EmbeddedFileRecord(
string Fingerprint,
long FileSize,
DateTimeOffset LastWriteUtc,
DateTimeOffset EmbeddedAtUtc,
int ChunkCount);

View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Services;
public sealed record DataSourceEmbeddingOverview(bool IsVisible, DataSourceEmbeddingState State, int IndexedFiles, int TotalFiles, int FailedFiles);

View File

@ -13,7 +13,9 @@ using AIStudio.Tools.PluginSystem;
namespace AIStudio.Tools.Services;
public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, ILogger<DataSourceEmbeddingService> logger)
public sealed partial class DataSourceEmbeddingService(
SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider,
ILogger<DataSourceEmbeddingService> logger)
: BackgroundService
{
private const int VECTOR_STORE_OPTIMIZATION_CHUNK_THRESHOLD = 100_000;

View File

@ -0,0 +1,10 @@
namespace AIStudio.Tools.Services;
public enum DataSourceEmbeddingState
{
IDLE,
QUEUED,
RUNNING,
COMPLETED,
FAILED,
}

View File

@ -0,0 +1,39 @@
using AIStudio.Settings.DataModel;
using AIStudio.Tools.PluginSystem;
namespace AIStudio.Tools.Services;
public sealed record DataSourceEmbeddingStatus(
string DataSourceId,
string DataSourceName,
DataSourceType DataSourceType,
DataSourceEmbeddingState State,
int TotalFiles,
int IndexedFiles,
int FailedFiles,
string CurrentFile,
string LastError,
IReadOnlyList<DataSourceEmbeddingFailure> Failures)
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService));
public int ProgressPercent => this.TotalFiles <= 0 ? 0 : Math.Clamp((int)Math.Round(this.IndexedFiles * 100d / this.TotalFiles), 0, 100);
public string StateLabel => this.State switch
{
DataSourceEmbeddingState.QUEUED => TB("Queued"),
DataSourceEmbeddingState.RUNNING => TB("Running"),
DataSourceEmbeddingState.COMPLETED => TB("Completed"),
DataSourceEmbeddingState.FAILED => TB("Needs attention"),
_ => TB("Idle")
};
public int SortOrder => this.State switch
{
DataSourceEmbeddingState.RUNNING => 0,
DataSourceEmbeddingState.QUEUED => 1,
DataSourceEmbeddingState.FAILED => 2,
DataSourceEmbeddingState.COMPLETED => 3,
_ => 4,
};
}

View File

@ -11,9 +11,7 @@ using AIStudio.Tools.Rust;
namespace AIStudio.Tools.Services;
public sealed class DataSourceLocalRetrievalService(
SettingsManager settingsManager,
RustService rustService,
DatabaseClientProvider databaseClientProvider,
SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider,
ILogger<DataSourceLocalRetrievalService> logger)
{
private enum RetrievalChannel

View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Services;
public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount);

View File

@ -0,0 +1,19 @@
namespace AIStudio.Tools.Services;
public sealed class FileEnumerationResult
{
public List<FileInfo> Files { get; } = [];
public List<DataSourceEmbeddingFailure> Failures { get; } = [];
public int FailedFiles { get; set; }
public string LastError { get; set; } = string.Empty;
public void AddFailure(string filePath, string reason)
{
this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason));
this.FailedFiles = this.Failures.Count;
this.LastError = reason;
}
}

View File

@ -7,8 +7,6 @@ using AIStudio.Tools.Rust;
namespace AIStudio.Tools.Services;
public sealed record ArbitraryFileDataSegment(string Content, int TokenCount);
public sealed partial class RustService
{
public async Task<string> ReadArbitraryFileData(string path, int maxChunks, bool extractImages = false)