From 15afe1c80cf1111e5fecef55e4294a71f9445db4 Mon Sep 17 00:00:00 2001 From: PaulKoudelka Date: Fri, 14 Aug 2026 14:25:53 +0200 Subject: [PATCH] refactored the classes and functions --- .../Settings/EmbeddingProvider.cs | 10 +- app/MindWork AI Studio/Settings/Provider.cs | 10 +- .../IndexStore/EmbeddingStateChunk.cs | 3 + .../IndexStore/EmbeddingStateChunkEntity.cs | 20 +++ .../EmbeddingStateDataSourceEntity.cs | 22 +++ .../IndexStore/EmbeddingStateFile.cs | 16 ++ .../IndexStore/EmbeddingStateFileEntity.cs | 36 +++++ .../Databases/IndexStore/IndexStoreClient.cs | 46 ------ .../IndexStore/IndexStoreDateTimeOffset.cs | 18 +++ .../IndexStoreDateTimeOffsetConverter.cs | 7 + .../IndexStore/IndexStoreDbContext.cs | 142 ------------------ .../IndexStore/IndexStoreSearchResult.cs | 24 +++ .../IndexStoreSearchResultEntity.cs | 46 ++++++ .../SqliteIndexStoreClientImplementation.cs | 6 +- .../VectorStore/VectorStoreClient.cs | 2 - .../VectorStore/VectorStoreEnsureResult.cs | 3 + .../Tools/Rust/TokenizerResponse.cs | 6 +- .../Services/ArbitraryFileDataSegment.cs | 3 + .../Services/DataSourceEmbeddingFailure.cs | 3 + .../Services/DataSourceEmbeddingManifest.cs | 14 ++ .../Services/DataSourceEmbeddingModels.cs | 97 ------------ .../Services/DataSourceEmbeddingOverview.cs | 3 + .../Services/DataSourceEmbeddingService.cs | 4 +- .../Services/DataSourceEmbeddingState.cs | 10 ++ .../Services/DataSourceEmbeddingStatus.cs | 39 +++++ .../DataSourceLocalRetrievalService.cs | 4 +- .../Tools/Services/EmbeddedFileRecord.cs | 3 + .../Tools/Services/FileEnumerationResult.cs | 19 +++ .../Tools/Services/RustService.Retrieval.cs | 2 - 29 files changed, 297 insertions(+), 321 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunk.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunkEntity.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDateTimeOffset.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDateTimeOffsetConverter.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreEnsureResult.cs create mode 100644 app/MindWork AI Studio/Tools/Services/ArbitraryFileDataSegment.cs create mode 100644 app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs create mode 100644 app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs delete mode 100644 app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs create mode 100644 app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingOverview.cs create mode 100644 app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingState.cs create mode 100644 app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs create mode 100644 app/MindWork AI Studio/Tools/Services/EmbeddedFileRecord.cs create mode 100644 app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs diff --git a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs index 5ae23759..6aa7d3c8 100644 --- a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs +++ b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs @@ -32,15 +32,7 @@ public sealed record EmbeddingProvider( public static readonly EmbeddingProvider NONE = new(); - public EmbeddingProvider() : this( - 0, - Guid.Empty.ToString(), - string.Empty, - LLMProviders.NONE, - default, - false, - false, - Guid.Empty) + public EmbeddingProvider() : this(0, Guid.Empty.ToString(), string.Empty, LLMProviders.NONE, default, false, false, Guid.Empty) { } diff --git a/app/MindWork AI Studio/Settings/Provider.cs b/app/MindWork AI Studio/Settings/Provider.cs index b63fb9df..d8e35260 100644 --- a/app/MindWork AI Studio/Settings/Provider.cs +++ b/app/MindWork AI Studio/Settings/Provider.cs @@ -41,15 +41,7 @@ public sealed record Provider( public static readonly Provider NONE = new(); - public Provider() : this( - 0, - Guid.Empty.ToString(), - string.Empty, - LLMProviders.NONE, - default, - false, - false, - Guid.Empty) + public Provider() : this(0, Guid.Empty.ToString(), string.Empty, LLMProviders.NONE, default, false, false, Guid.Empty) { } diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunk.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunk.cs new file mode 100644 index 00000000..8076c817 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunk.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Databases.IndexStore; + +public sealed record EmbeddingStateChunk(string ChunkId, string ParentFileId, int? PageNumber, int ChunkIndex, string ChunkText, DateTimeOffset EmbeddedAtUtc); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunkEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunkEntity.cs new file mode 100644 index 00000000..867c4252 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunkEntity.cs @@ -0,0 +1,20 @@ +namespace AIStudio.Tools.Databases.IndexStore; + +internal sealed class EmbeddingStateChunkEntity +{ + public int Id { get; set; } + + public string ChunkId { get; set; } = string.Empty; + + public string ParentFileId { get; set; } = string.Empty; + + public int? PageNumber { get; set; } + + public int ChunkIndex { get; set; } + + public string ChunkText { get; set; } = string.Empty; + + public DateTimeOffset EmbeddedAtUtc { get; set; } + + public EmbeddingStateFileEntity? File { get; set; } +} diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs new file mode 100644 index 00000000..660a897f --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateDataSourceEntity.cs @@ -0,0 +1,22 @@ +namespace AIStudio.Tools.Databases.IndexStore; + +internal sealed class EmbeddingStateDataSourceEntity +{ + public string DataSourceId { get; set; } = string.Empty; + + public string DataSourceName { get; set; } = string.Empty; + + public string DataSourceType { get; set; } = string.Empty; + + public string EmbeddingProviderId { get; set; } = string.Empty; + + public string EmbeddingSignature { get; set; } = string.Empty; + + public string SourceHash { get; set; } = string.Empty; + + public int VectorSize { get; set; } + + public DateTimeOffset UpdatedAtUtc { get; set; } + + public List Files { get; set; } = []; +} diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs new file mode 100644 index 00000000..cbf268b4 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs @@ -0,0 +1,16 @@ +namespace AIStudio.Tools.Databases.IndexStore; + +public sealed record EmbeddingStateFile( + string ParentFileId, + string AbsolutePath, + string FileName, + string RelativePath, + string FileType, + string Fingerprint, + long FileSize, + DateTimeOffset CreationUtc, + DateTimeOffset LastWriteUtc, + DateTimeOffset EmbeddedAtUtc, + int ChunkCount, + string ConfidenceLevel, + int ConfidenceLevelRank); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs new file mode 100644 index 00000000..bd116ba2 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs @@ -0,0 +1,36 @@ +namespace AIStudio.Tools.Databases.IndexStore; + +internal sealed class EmbeddingStateFileEntity +{ + public string ParentFileId { get; set; } = string.Empty; + + public string DataSourceId { get; set; } = string.Empty; + + public string AbsolutePath { get; set; } = string.Empty; + + public string FileName { get; set; } = string.Empty; + + public string RelativePath { get; set; } = string.Empty; + + public string FileType { get; set; } = string.Empty; + + public string Fingerprint { get; set; } = string.Empty; + + public long FileSize { get; set; } + + public DateTimeOffset CreationUtc { get; set; } + + public DateTimeOffset LastWriteUtc { get; set; } + + public DateTimeOffset EmbeddedAtUtc { get; set; } + + public int ChunkCount { get; set; } + + public string ConfidenceLevel { get; set; } = string.Empty; + + public int ConfidenceLevelRank { get; set; } + + public EmbeddingStateDataSourceEntity? DataSource { get; set; } + + public List Chunks { get; set; } = []; +} diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs index 9ea68584..6bef7bf9 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs @@ -30,49 +30,3 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token); } - -public sealed record EmbeddingStateFile( - string ParentFileId, - string AbsolutePath, - string FileName, - string RelativePath, - string FileType, - string Fingerprint, - long FileSize, - DateTimeOffset CreationUtc, - DateTimeOffset LastWriteUtc, - DateTimeOffset EmbeddedAtUtc, - int ChunkCount, - string ConfidenceLevel, - int ConfidenceLevelRank); - -public sealed record EmbeddingStateChunk( - string ChunkId, - string ParentFileId, - int? PageNumber, - int ChunkIndex, - string ChunkText, - DateTimeOffset EmbeddedAtUtc); - -public sealed record IndexStoreSearchResult( - string ChunkId, - string ParentFileId, - string DataSourceId, - string DataSourceName, - string DataSourceType, - string AbsolutePath, - string FileName, - string RelativePath, - string FileType, - int? PageNumber, - int ChunkIndex, - string ChunkText, - double Score, - string Fingerprint, - long FileSize, - DateTimeOffset CreationUtc, - DateTimeOffset LastWriteUtc, - DateTimeOffset EmbeddedAtUtc, - int ChunkCount, - string ConfidenceLevel, - int ConfidenceLevelRank); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDateTimeOffset.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDateTimeOffset.cs new file mode 100644 index 00000000..49c9355b --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDateTimeOffset.cs @@ -0,0 +1,18 @@ +using System.Globalization; + +namespace AIStudio.Tools.Databases.IndexStore; + +internal static class IndexStoreDateTimeOffset +{ + public static string ToUtcText(DateTimeOffset dateTime) + { + return dateTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture); + } + + public static DateTimeOffset ParseUtc(string value) + { + return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateTime) + ? dateTime.ToUniversalTime() + : DateTimeOffset.UnixEpoch; + } +} diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDateTimeOffsetConverter.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDateTimeOffsetConverter.cs new file mode 100644 index 00000000..c087464f --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDateTimeOffsetConverter.cs @@ -0,0 +1,7 @@ +using Microsoft.EntityFrameworkCore.Storage.ValueConversion; + +namespace AIStudio.Tools.Databases.IndexStore; + +internal sealed class IndexStoreDateTimeOffsetConverter() : ValueConverter( + value => IndexStoreDateTimeOffset.ToUtcText(value), + value => IndexStoreDateTimeOffset.ParseUtc(value)); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs index 726026f4..7d0d3ab5 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs @@ -1,8 +1,5 @@ -using System.Globalization; - using Microsoft.Data.Sqlite; using Microsoft.EntityFrameworkCore; -using Microsoft.EntityFrameworkCore.Storage.ValueConversion; namespace AIStudio.Tools.Databases.IndexStore; @@ -117,142 +114,3 @@ internal sealed class IndexStoreDbContext(DbContextOptions DefaultTimeout = 30, }.ToString(); } - -internal sealed class EmbeddingStateDataSourceEntity -{ - public string DataSourceId { get; set; } = string.Empty; - - public string DataSourceName { get; set; } = string.Empty; - - public string DataSourceType { get; set; } = string.Empty; - - public string EmbeddingProviderId { get; set; } = string.Empty; - - public string EmbeddingSignature { get; set; } = string.Empty; - - public string SourceHash { get; set; } = string.Empty; - - public int VectorSize { get; set; } - - public DateTimeOffset UpdatedAtUtc { get; set; } - - public List Files { get; set; } = []; -} - -internal sealed class EmbeddingStateFileEntity -{ - public string ParentFileId { get; set; } = string.Empty; - - public string DataSourceId { get; set; } = string.Empty; - - public string AbsolutePath { get; set; } = string.Empty; - - public string FileName { get; set; } = string.Empty; - - public string RelativePath { get; set; } = string.Empty; - - public string FileType { get; set; } = string.Empty; - - public string Fingerprint { get; set; } = string.Empty; - - public long FileSize { get; set; } - - public DateTimeOffset CreationUtc { get; set; } - - public DateTimeOffset LastWriteUtc { get; set; } - - public DateTimeOffset EmbeddedAtUtc { get; set; } - - public int ChunkCount { get; set; } - - public string ConfidenceLevel { get; set; } = string.Empty; - - public int ConfidenceLevelRank { get; set; } - - public EmbeddingStateDataSourceEntity? DataSource { get; set; } - - public List Chunks { get; set; } = []; -} - -internal sealed class EmbeddingStateChunkEntity -{ - public int Id { get; set; } - - public string ChunkId { get; set; } = string.Empty; - - public string ParentFileId { get; set; } = string.Empty; - - public int? PageNumber { get; set; } - - public int ChunkIndex { get; set; } - - public string ChunkText { get; set; } = string.Empty; - - public DateTimeOffset EmbeddedAtUtc { get; set; } - - public EmbeddingStateFileEntity? File { get; set; } -} - -internal sealed class IndexStoreSearchResultEntity -{ - public string ChunkId { get; set; } = string.Empty; - - public string ParentFileId { get; set; } = string.Empty; - - public string DataSourceId { get; set; } = string.Empty; - - public string DataSourceName { get; set; } = string.Empty; - - public string DataSourceType { get; set; } = string.Empty; - - public string AbsolutePath { get; set; } = string.Empty; - - public string FileName { get; set; } = string.Empty; - - public string RelativePath { get; set; } = string.Empty; - - public string FileType { get; set; } = string.Empty; - - public int? PageNumber { get; set; } - - public int ChunkIndex { get; set; } - - public string ChunkText { get; set; } = string.Empty; - - public double Score { get; set; } - - public string Fingerprint { get; set; } = string.Empty; - - public long FileSize { get; set; } - - public DateTimeOffset CreationUtc { get; set; } - - public DateTimeOffset LastWriteUtc { get; set; } - - public DateTimeOffset EmbeddedAtUtc { get; set; } - - public int ChunkCount { get; set; } - - public string ConfidenceLevel { get; set; } = string.Empty; - - public int ConfidenceLevelRank { get; set; } -} - -internal sealed class IndexStoreDateTimeOffsetConverter() : ValueConverter( - value => IndexStoreDateTimeOffset.ToUtcText(value), - value => IndexStoreDateTimeOffset.ParseUtc(value)); - -internal static class IndexStoreDateTimeOffset -{ - public static string ToUtcText(DateTimeOffset dateTime) - { - return dateTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture); - } - - public static DateTimeOffset ParseUtc(string value) - { - return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateTime) - ? dateTime.ToUniversalTime() - : DateTimeOffset.UnixEpoch; - } -} diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs new file mode 100644 index 00000000..0ae8d884 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs @@ -0,0 +1,24 @@ +namespace AIStudio.Tools.Databases.IndexStore; + +public sealed record IndexStoreSearchResult( + string ChunkId, + string ParentFileId, + string DataSourceId, + string DataSourceName, + string DataSourceType, + string AbsolutePath, + string FileName, + string RelativePath, + string FileType, + int? PageNumber, + int ChunkIndex, + string ChunkText, + double Score, + string Fingerprint, + long FileSize, + DateTimeOffset CreationUtc, + DateTimeOffset LastWriteUtc, + DateTimeOffset EmbeddedAtUtc, + int ChunkCount, + string ConfidenceLevel, + int ConfidenceLevelRank); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs new file mode 100644 index 00000000..1385c9f5 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs @@ -0,0 +1,46 @@ +namespace AIStudio.Tools.Databases.IndexStore; + +internal sealed class IndexStoreSearchResultEntity +{ + public string ChunkId { get; set; } = string.Empty; + + public string ParentFileId { get; set; } = string.Empty; + + public string DataSourceId { get; set; } = string.Empty; + + public string DataSourceName { get; set; } = string.Empty; + + public string DataSourceType { get; set; } = string.Empty; + + public string AbsolutePath { get; set; } = string.Empty; + + public string FileName { get; set; } = string.Empty; + + public string RelativePath { get; set; } = string.Empty; + + public string FileType { get; set; } = string.Empty; + + public int? PageNumber { get; set; } + + public int ChunkIndex { get; set; } + + public string ChunkText { get; set; } = string.Empty; + + public double Score { get; set; } + + public string Fingerprint { get; set; } = string.Empty; + + public long FileSize { get; set; } + + public DateTimeOffset CreationUtc { get; set; } + + public DateTimeOffset LastWriteUtc { get; set; } + + public DateTimeOffset EmbeddedAtUtc { get; set; } + + public int ChunkCount { get; set; } + + public string ConfidenceLevel { get; set; } = string.Empty; + + public int ConfidenceLevelRank { get; set; } +} diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs index 7ccb06d2..b0669e33 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs @@ -9,11 +9,7 @@ using Microsoft.EntityFrameworkCore; namespace AIStudio.Tools.Databases.IndexStore; -public sealed class SqliteIndexStoreClientImplementation( - string name, - string databasePath, - string basePath, - string version) : IndexStoreClient(name, basePath) +public sealed class SqliteIndexStoreClientImplementation(string name, string databasePath, string basePath, string version) : IndexStoreClient(name, basePath) { private const string DATABASE_NAME = "Local RAG Index"; private const string DATABASE_FILENAME = "rag-index.sqlite3"; diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs index 1fef76ab..faa20d66 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreClient.cs @@ -14,5 +14,3 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien public abstract Task DeleteVectorStore(string storeName, CancellationToken token); } - -public sealed record VectorStoreEnsureResult(bool Created); diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreEnsureResult.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreEnsureResult.cs new file mode 100644 index 00000000..0a9471f3 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreEnsureResult.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Databases.VectorStore; + +public sealed record VectorStoreEnsureResult(bool Created); diff --git a/app/MindWork AI Studio/Tools/Rust/TokenizerResponse.cs b/app/MindWork AI Studio/Tools/Rust/TokenizerResponse.cs index 4eb89c6c..6fe3c054 100644 --- a/app/MindWork AI Studio/Tools/Rust/TokenizerResponse.cs +++ b/app/MindWork AI Studio/Tools/Rust/TokenizerResponse.cs @@ -1,7 +1,3 @@ namespace AIStudio.Tools.Rust; -public readonly record struct TokenizerResponse( - bool Success, - int TokenCount, - string Message, - string StoredPath = ""); +public readonly record struct TokenizerResponse(bool Success, int TokenCount, string Message, string StoredPath = ""); diff --git a/app/MindWork AI Studio/Tools/Services/ArbitraryFileDataSegment.cs b/app/MindWork AI Studio/Tools/Services/ArbitraryFileDataSegment.cs new file mode 100644 index 00000000..99c1ce7f --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/ArbitraryFileDataSegment.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Services; + +public sealed record ArbitraryFileDataSegment(string Content, int TokenCount); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs new file mode 100644 index 00000000..e73bb5a4 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingFailure.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Services; + +public sealed record DataSourceEmbeddingFailure(string FilePath, string Reason); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs new file mode 100644 index 00000000..44b0d526 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingManifest.cs @@ -0,0 +1,14 @@ +namespace AIStudio.Tools.Services; + +public sealed class DataSourceEmbeddingManifest +{ + public string EmbeddingProviderId { get; set; } = string.Empty; + + public string EmbeddingSignature { get; set; } = string.Empty; + + public string SourceHash { get; set; } = string.Empty; + + public int VectorSize { get; set; } + + public Dictionary Files { get; init; } = new(StringComparer.OrdinalIgnoreCase); +} diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs deleted file mode 100644 index 917af1aa..00000000 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingModels.cs +++ /dev/null @@ -1,97 +0,0 @@ -using AIStudio.Settings.DataModel; -using AIStudio.Tools.PluginSystem; - -namespace AIStudio.Tools.Services; - -public sealed record DataSourceEmbeddingOverview( - bool IsVisible, - DataSourceEmbeddingState State, - int IndexedFiles, - int TotalFiles, - int FailedFiles); - -public enum DataSourceEmbeddingState -{ - IDLE, - QUEUED, - RUNNING, - COMPLETED, - FAILED, -} - -public sealed record DataSourceEmbeddingStatus( - string DataSourceId, - string DataSourceName, - DataSourceType DataSourceType, - DataSourceEmbeddingState State, - int TotalFiles, - int IndexedFiles, - int FailedFiles, - string CurrentFile, - string LastError, - IReadOnlyList Failures) -{ - private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService)); - - public int ProgressPercent => this.TotalFiles <= 0 ? 0 : Math.Clamp((int)Math.Round(this.IndexedFiles * 100d / this.TotalFiles), 0, 100); - - public string StateLabel => this.State switch - { - DataSourceEmbeddingState.QUEUED => TB("Queued"), - DataSourceEmbeddingState.RUNNING => TB("Running"), - DataSourceEmbeddingState.COMPLETED => TB("Completed"), - DataSourceEmbeddingState.FAILED => TB("Needs attention"), - _ => TB("Idle") - }; - - public int SortOrder => this.State switch - { - DataSourceEmbeddingState.RUNNING => 0, - DataSourceEmbeddingState.QUEUED => 1, - DataSourceEmbeddingState.FAILED => 2, - DataSourceEmbeddingState.COMPLETED => 3, - _ => 4, - }; -} - -public sealed record DataSourceEmbeddingFailure( - string FilePath, - string Reason); - -public sealed class FileEnumerationResult -{ - public List Files { get; } = []; - - public List Failures { get; } = []; - - public int FailedFiles { get; set; } - - public string LastError { get; set; } = string.Empty; - - public void AddFailure(string filePath, string reason) - { - this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason)); - this.FailedFiles = this.Failures.Count; - this.LastError = reason; - } -} - -public sealed class DataSourceEmbeddingManifest -{ - public string EmbeddingProviderId { get; set; } = string.Empty; - - public string EmbeddingSignature { get; set; } = string.Empty; - - public string SourceHash { get; set; } = string.Empty; - - public int VectorSize { get; set; } - - public Dictionary Files { get; init; } = new(StringComparer.OrdinalIgnoreCase); -} - -public sealed record EmbeddedFileRecord( - string Fingerprint, - long FileSize, - DateTimeOffset LastWriteUtc, - DateTimeOffset EmbeddedAtUtc, - int ChunkCount); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingOverview.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingOverview.cs new file mode 100644 index 00000000..604e7862 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingOverview.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Services; + +public sealed record DataSourceEmbeddingOverview(bool IsVisible, DataSourceEmbeddingState State, int IndexedFiles, int TotalFiles, int FailedFiles); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 42ce0de1..967f81ea 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -13,7 +13,9 @@ using AIStudio.Tools.PluginSystem; namespace AIStudio.Tools.Services; -public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, ILogger logger) +public sealed partial class DataSourceEmbeddingService( + SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, + ILogger logger) : BackgroundService { private const int VECTOR_STORE_OPTIMIZATION_CHUNK_THRESHOLD = 100_000; diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingState.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingState.cs new file mode 100644 index 00000000..9578c3f8 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingState.cs @@ -0,0 +1,10 @@ +namespace AIStudio.Tools.Services; + +public enum DataSourceEmbeddingState +{ + IDLE, + QUEUED, + RUNNING, + COMPLETED, + FAILED, +} diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs new file mode 100644 index 00000000..2047e81b --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs @@ -0,0 +1,39 @@ +using AIStudio.Settings.DataModel; +using AIStudio.Tools.PluginSystem; + +namespace AIStudio.Tools.Services; + +public sealed record DataSourceEmbeddingStatus( + string DataSourceId, + string DataSourceName, + DataSourceType DataSourceType, + DataSourceEmbeddingState State, + int TotalFiles, + int IndexedFiles, + int FailedFiles, + string CurrentFile, + string LastError, + IReadOnlyList Failures) +{ + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService)); + + public int ProgressPercent => this.TotalFiles <= 0 ? 0 : Math.Clamp((int)Math.Round(this.IndexedFiles * 100d / this.TotalFiles), 0, 100); + + public string StateLabel => this.State switch + { + DataSourceEmbeddingState.QUEUED => TB("Queued"), + DataSourceEmbeddingState.RUNNING => TB("Running"), + DataSourceEmbeddingState.COMPLETED => TB("Completed"), + DataSourceEmbeddingState.FAILED => TB("Needs attention"), + _ => TB("Idle") + }; + + public int SortOrder => this.State switch + { + DataSourceEmbeddingState.RUNNING => 0, + DataSourceEmbeddingState.QUEUED => 1, + DataSourceEmbeddingState.FAILED => 2, + DataSourceEmbeddingState.COMPLETED => 3, + _ => 4, + }; +} diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs index 973c6ecc..f8b2513c 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs @@ -11,9 +11,7 @@ using AIStudio.Tools.Rust; namespace AIStudio.Tools.Services; public sealed class DataSourceLocalRetrievalService( - SettingsManager settingsManager, - RustService rustService, - DatabaseClientProvider databaseClientProvider, + SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, ILogger logger) { private enum RetrievalChannel diff --git a/app/MindWork AI Studio/Tools/Services/EmbeddedFileRecord.cs b/app/MindWork AI Studio/Tools/Services/EmbeddedFileRecord.cs new file mode 100644 index 00000000..f4690151 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/EmbeddedFileRecord.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Services; + +public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount); diff --git a/app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs b/app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs new file mode 100644 index 00000000..e0837179 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/FileEnumerationResult.cs @@ -0,0 +1,19 @@ +namespace AIStudio.Tools.Services; + +public sealed class FileEnumerationResult +{ + public List Files { get; } = []; + + public List Failures { get; } = []; + + public int FailedFiles { get; set; } + + public string LastError { get; set; } = string.Empty; + + public void AddFailure(string filePath, string reason) + { + this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason)); + this.FailedFiles = this.Failures.Count; + this.LastError = reason; + } +} diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs index 3598db6e..3aabb505 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Retrieval.cs @@ -7,8 +7,6 @@ using AIStudio.Tools.Rust; namespace AIStudio.Tools.Services; -public sealed record ArbitraryFileDataSegment(string Content, int TokenCount); - public sealed partial class RustService { public async Task ReadArbitraryFileData(string path, int maxChunks, bool extractImages = false)