mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-08-16 00:42:12 +00:00
refactored the classes and functions
This commit is contained in:
parent
1a263127f5
commit
15afe1c80c
@ -32,15 +32,7 @@ public sealed record EmbeddingProvider(
|
||||
|
||||
public static readonly EmbeddingProvider NONE = new();
|
||||
|
||||
public EmbeddingProvider() : this(
|
||||
0,
|
||||
Guid.Empty.ToString(),
|
||||
string.Empty,
|
||||
LLMProviders.NONE,
|
||||
default,
|
||||
false,
|
||||
false,
|
||||
Guid.Empty)
|
||||
public EmbeddingProvider() : this(0, Guid.Empty.ToString(), string.Empty, LLMProviders.NONE, default, false, false, Guid.Empty)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@ -41,15 +41,7 @@ public sealed record Provider(
|
||||
|
||||
public static readonly Provider NONE = new();
|
||||
|
||||
public Provider() : this(
|
||||
0,
|
||||
Guid.Empty.ToString(),
|
||||
string.Empty,
|
||||
LLMProviders.NONE,
|
||||
default,
|
||||
false,
|
||||
false,
|
||||
Guid.Empty)
|
||||
public Provider() : this(0, Guid.Empty.ToString(), string.Empty, LLMProviders.NONE, default, false, false, Guid.Empty)
|
||||
{
|
||||
}
|
||||
|
||||
|
||||
@ -0,0 +1,3 @@
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
public sealed record EmbeddingStateChunk(string ChunkId, string ParentFileId, int? PageNumber, int ChunkIndex, string ChunkText, DateTimeOffset EmbeddedAtUtc);
|
||||
@ -0,0 +1,20 @@
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal sealed class EmbeddingStateChunkEntity
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public string ChunkId { get; set; } = string.Empty;
|
||||
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public int? PageNumber { get; set; }
|
||||
|
||||
public int ChunkIndex { get; set; }
|
||||
|
||||
public string ChunkText { get; set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||
|
||||
public EmbeddingStateFileEntity? File { get; set; }
|
||||
}
|
||||
@ -0,0 +1,22 @@
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal sealed class EmbeddingStateDataSourceEntity
|
||||
{
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceName { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceType { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingProviderId { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingSignature { get; set; } = string.Empty;
|
||||
|
||||
public string SourceHash { get; set; } = string.Empty;
|
||||
|
||||
public int VectorSize { get; set; }
|
||||
|
||||
public DateTimeOffset UpdatedAtUtc { get; set; }
|
||||
|
||||
public List<EmbeddingStateFileEntity> Files { get; set; } = [];
|
||||
}
|
||||
@ -0,0 +1,16 @@
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
public sealed record EmbeddingStateFile(
|
||||
string ParentFileId,
|
||||
string AbsolutePath,
|
||||
string FileName,
|
||||
string RelativePath,
|
||||
string FileType,
|
||||
string Fingerprint,
|
||||
long FileSize,
|
||||
DateTimeOffset CreationUtc,
|
||||
DateTimeOffset LastWriteUtc,
|
||||
DateTimeOffset EmbeddedAtUtc,
|
||||
int ChunkCount,
|
||||
string ConfidenceLevel,
|
||||
int ConfidenceLevelRank);
|
||||
@ -0,0 +1,36 @@
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal sealed class EmbeddingStateFileEntity
|
||||
{
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string AbsolutePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileName { get; set; } = string.Empty;
|
||||
|
||||
public string RelativePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileType { get; set; } = string.Empty;
|
||||
|
||||
public string Fingerprint { get; set; } = string.Empty;
|
||||
|
||||
public long FileSize { get; set; }
|
||||
|
||||
public DateTimeOffset CreationUtc { get; set; }
|
||||
|
||||
public DateTimeOffset LastWriteUtc { get; set; }
|
||||
|
||||
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||
|
||||
public int ChunkCount { get; set; }
|
||||
|
||||
public string ConfidenceLevel { get; set; } = string.Empty;
|
||||
|
||||
public int ConfidenceLevelRank { get; set; }
|
||||
|
||||
public EmbeddingStateDataSourceEntity? DataSource { get; set; }
|
||||
|
||||
public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];
|
||||
}
|
||||
@ -30,49 +30,3 @@ public abstract class IndexStoreClient(string name, string path) : DatabaseClien
|
||||
|
||||
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
|
||||
}
|
||||
|
||||
public sealed record EmbeddingStateFile(
|
||||
string ParentFileId,
|
||||
string AbsolutePath,
|
||||
string FileName,
|
||||
string RelativePath,
|
||||
string FileType,
|
||||
string Fingerprint,
|
||||
long FileSize,
|
||||
DateTimeOffset CreationUtc,
|
||||
DateTimeOffset LastWriteUtc,
|
||||
DateTimeOffset EmbeddedAtUtc,
|
||||
int ChunkCount,
|
||||
string ConfidenceLevel,
|
||||
int ConfidenceLevelRank);
|
||||
|
||||
public sealed record EmbeddingStateChunk(
|
||||
string ChunkId,
|
||||
string ParentFileId,
|
||||
int? PageNumber,
|
||||
int ChunkIndex,
|
||||
string ChunkText,
|
||||
DateTimeOffset EmbeddedAtUtc);
|
||||
|
||||
public sealed record IndexStoreSearchResult(
|
||||
string ChunkId,
|
||||
string ParentFileId,
|
||||
string DataSourceId,
|
||||
string DataSourceName,
|
||||
string DataSourceType,
|
||||
string AbsolutePath,
|
||||
string FileName,
|
||||
string RelativePath,
|
||||
string FileType,
|
||||
int? PageNumber,
|
||||
int ChunkIndex,
|
||||
string ChunkText,
|
||||
double Score,
|
||||
string Fingerprint,
|
||||
long FileSize,
|
||||
DateTimeOffset CreationUtc,
|
||||
DateTimeOffset LastWriteUtc,
|
||||
DateTimeOffset EmbeddedAtUtc,
|
||||
int ChunkCount,
|
||||
string ConfidenceLevel,
|
||||
int ConfidenceLevelRank);
|
||||
|
||||
@ -0,0 +1,18 @@
|
||||
using System.Globalization;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal static class IndexStoreDateTimeOffset
|
||||
{
|
||||
public static string ToUtcText(DateTimeOffset dateTime)
|
||||
{
|
||||
return dateTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
public static DateTimeOffset ParseUtc(string value)
|
||||
{
|
||||
return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateTime)
|
||||
? dateTime.ToUniversalTime()
|
||||
: DateTimeOffset.UnixEpoch;
|
||||
}
|
||||
}
|
||||
@ -0,0 +1,7 @@
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal sealed class IndexStoreDateTimeOffsetConverter() : ValueConverter<DateTimeOffset, string>(
|
||||
value => IndexStoreDateTimeOffset.ToUtcText(value),
|
||||
value => IndexStoreDateTimeOffset.ParseUtc(value));
|
||||
@ -1,8 +1,5 @@
|
||||
using System.Globalization;
|
||||
|
||||
using Microsoft.Data.Sqlite;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.EntityFrameworkCore.Storage.ValueConversion;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
@ -117,142 +114,3 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
|
||||
DefaultTimeout = 30,
|
||||
}.ToString();
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateDataSourceEntity
|
||||
{
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceName { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceType { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingProviderId { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingSignature { get; set; } = string.Empty;
|
||||
|
||||
public string SourceHash { get; set; } = string.Empty;
|
||||
|
||||
public int VectorSize { get; set; }
|
||||
|
||||
public DateTimeOffset UpdatedAtUtc { get; set; }
|
||||
|
||||
public List<EmbeddingStateFileEntity> Files { get; set; } = [];
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateFileEntity
|
||||
{
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string AbsolutePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileName { get; set; } = string.Empty;
|
||||
|
||||
public string RelativePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileType { get; set; } = string.Empty;
|
||||
|
||||
public string Fingerprint { get; set; } = string.Empty;
|
||||
|
||||
public long FileSize { get; set; }
|
||||
|
||||
public DateTimeOffset CreationUtc { get; set; }
|
||||
|
||||
public DateTimeOffset LastWriteUtc { get; set; }
|
||||
|
||||
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||
|
||||
public int ChunkCount { get; set; }
|
||||
|
||||
public string ConfidenceLevel { get; set; } = string.Empty;
|
||||
|
||||
public int ConfidenceLevelRank { get; set; }
|
||||
|
||||
public EmbeddingStateDataSourceEntity? DataSource { get; set; }
|
||||
|
||||
public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];
|
||||
}
|
||||
|
||||
internal sealed class EmbeddingStateChunkEntity
|
||||
{
|
||||
public int Id { get; set; }
|
||||
|
||||
public string ChunkId { get; set; } = string.Empty;
|
||||
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public int? PageNumber { get; set; }
|
||||
|
||||
public int ChunkIndex { get; set; }
|
||||
|
||||
public string ChunkText { get; set; } = string.Empty;
|
||||
|
||||
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||
|
||||
public EmbeddingStateFileEntity? File { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class IndexStoreSearchResultEntity
|
||||
{
|
||||
public string ChunkId { get; set; } = string.Empty;
|
||||
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceName { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceType { get; set; } = string.Empty;
|
||||
|
||||
public string AbsolutePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileName { get; set; } = string.Empty;
|
||||
|
||||
public string RelativePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileType { get; set; } = string.Empty;
|
||||
|
||||
public int? PageNumber { get; set; }
|
||||
|
||||
public int ChunkIndex { get; set; }
|
||||
|
||||
public string ChunkText { get; set; } = string.Empty;
|
||||
|
||||
public double Score { get; set; }
|
||||
|
||||
public string Fingerprint { get; set; } = string.Empty;
|
||||
|
||||
public long FileSize { get; set; }
|
||||
|
||||
public DateTimeOffset CreationUtc { get; set; }
|
||||
|
||||
public DateTimeOffset LastWriteUtc { get; set; }
|
||||
|
||||
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||
|
||||
public int ChunkCount { get; set; }
|
||||
|
||||
public string ConfidenceLevel { get; set; } = string.Empty;
|
||||
|
||||
public int ConfidenceLevelRank { get; set; }
|
||||
}
|
||||
|
||||
internal sealed class IndexStoreDateTimeOffsetConverter() : ValueConverter<DateTimeOffset, string>(
|
||||
value => IndexStoreDateTimeOffset.ToUtcText(value),
|
||||
value => IndexStoreDateTimeOffset.ParseUtc(value));
|
||||
|
||||
internal static class IndexStoreDateTimeOffset
|
||||
{
|
||||
public static string ToUtcText(DateTimeOffset dateTime)
|
||||
{
|
||||
return dateTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
|
||||
}
|
||||
|
||||
public static DateTimeOffset ParseUtc(string value)
|
||||
{
|
||||
return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateTime)
|
||||
? dateTime.ToUniversalTime()
|
||||
: DateTimeOffset.UnixEpoch;
|
||||
}
|
||||
}
|
||||
|
||||
@ -0,0 +1,24 @@
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
public sealed record IndexStoreSearchResult(
|
||||
string ChunkId,
|
||||
string ParentFileId,
|
||||
string DataSourceId,
|
||||
string DataSourceName,
|
||||
string DataSourceType,
|
||||
string AbsolutePath,
|
||||
string FileName,
|
||||
string RelativePath,
|
||||
string FileType,
|
||||
int? PageNumber,
|
||||
int ChunkIndex,
|
||||
string ChunkText,
|
||||
double Score,
|
||||
string Fingerprint,
|
||||
long FileSize,
|
||||
DateTimeOffset CreationUtc,
|
||||
DateTimeOffset LastWriteUtc,
|
||||
DateTimeOffset EmbeddedAtUtc,
|
||||
int ChunkCount,
|
||||
string ConfidenceLevel,
|
||||
int ConfidenceLevelRank);
|
||||
@ -0,0 +1,46 @@
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
internal sealed class IndexStoreSearchResultEntity
|
||||
{
|
||||
public string ChunkId { get; set; } = string.Empty;
|
||||
|
||||
public string ParentFileId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceId { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceName { get; set; } = string.Empty;
|
||||
|
||||
public string DataSourceType { get; set; } = string.Empty;
|
||||
|
||||
public string AbsolutePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileName { get; set; } = string.Empty;
|
||||
|
||||
public string RelativePath { get; set; } = string.Empty;
|
||||
|
||||
public string FileType { get; set; } = string.Empty;
|
||||
|
||||
public int? PageNumber { get; set; }
|
||||
|
||||
public int ChunkIndex { get; set; }
|
||||
|
||||
public string ChunkText { get; set; } = string.Empty;
|
||||
|
||||
public double Score { get; set; }
|
||||
|
||||
public string Fingerprint { get; set; } = string.Empty;
|
||||
|
||||
public long FileSize { get; set; }
|
||||
|
||||
public DateTimeOffset CreationUtc { get; set; }
|
||||
|
||||
public DateTimeOffset LastWriteUtc { get; set; }
|
||||
|
||||
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||
|
||||
public int ChunkCount { get; set; }
|
||||
|
||||
public string ConfidenceLevel { get; set; } = string.Empty;
|
||||
|
||||
public int ConfidenceLevelRank { get; set; }
|
||||
}
|
||||
@ -9,11 +9,7 @@ using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace AIStudio.Tools.Databases.IndexStore;
|
||||
|
||||
public sealed class SqliteIndexStoreClientImplementation(
|
||||
string name,
|
||||
string databasePath,
|
||||
string basePath,
|
||||
string version) : IndexStoreClient(name, basePath)
|
||||
public sealed class SqliteIndexStoreClientImplementation(string name, string databasePath, string basePath, string version) : IndexStoreClient(name, basePath)
|
||||
{
|
||||
private const string DATABASE_NAME = "Local RAG Index";
|
||||
private const string DATABASE_FILENAME = "rag-index.sqlite3";
|
||||
|
||||
@ -14,5 +14,3 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
|
||||
|
||||
public abstract Task DeleteVectorStore(string storeName, CancellationToken token);
|
||||
}
|
||||
|
||||
public sealed record VectorStoreEnsureResult(bool Created);
|
||||
|
||||
@ -0,0 +1,3 @@
|
||||
namespace AIStudio.Tools.Databases.VectorStore;
|
||||
|
||||
public sealed record VectorStoreEnsureResult(bool Created);
|
||||
@ -1,7 +1,3 @@
|
||||
namespace AIStudio.Tools.Rust;
|
||||
|
||||
public readonly record struct TokenizerResponse(
|
||||
bool Success,
|
||||
int TokenCount,
|
||||
string Message,
|
||||
string StoredPath = "");
|
||||
public readonly record struct TokenizerResponse(bool Success, int TokenCount, string Message, string StoredPath = "");
|
||||
|
||||
@ -0,0 +1,3 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed record ArbitraryFileDataSegment(string Content, int TokenCount);
|
||||
@ -0,0 +1,3 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed record DataSourceEmbeddingFailure(string FilePath, string Reason);
|
||||
@ -0,0 +1,14 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed class DataSourceEmbeddingManifest
|
||||
{
|
||||
public string EmbeddingProviderId { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingSignature { get; set; } = string.Empty;
|
||||
|
||||
public string SourceHash { get; set; } = string.Empty;
|
||||
|
||||
public int VectorSize { get; set; }
|
||||
|
||||
public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
@ -1,97 +0,0 @@
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed record DataSourceEmbeddingOverview(
|
||||
bool IsVisible,
|
||||
DataSourceEmbeddingState State,
|
||||
int IndexedFiles,
|
||||
int TotalFiles,
|
||||
int FailedFiles);
|
||||
|
||||
public enum DataSourceEmbeddingState
|
||||
{
|
||||
IDLE,
|
||||
QUEUED,
|
||||
RUNNING,
|
||||
COMPLETED,
|
||||
FAILED,
|
||||
}
|
||||
|
||||
public sealed record DataSourceEmbeddingStatus(
|
||||
string DataSourceId,
|
||||
string DataSourceName,
|
||||
DataSourceType DataSourceType,
|
||||
DataSourceEmbeddingState State,
|
||||
int TotalFiles,
|
||||
int IndexedFiles,
|
||||
int FailedFiles,
|
||||
string CurrentFile,
|
||||
string LastError,
|
||||
IReadOnlyList<DataSourceEmbeddingFailure> Failures)
|
||||
{
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService));
|
||||
|
||||
public int ProgressPercent => this.TotalFiles <= 0 ? 0 : Math.Clamp((int)Math.Round(this.IndexedFiles * 100d / this.TotalFiles), 0, 100);
|
||||
|
||||
public string StateLabel => this.State switch
|
||||
{
|
||||
DataSourceEmbeddingState.QUEUED => TB("Queued"),
|
||||
DataSourceEmbeddingState.RUNNING => TB("Running"),
|
||||
DataSourceEmbeddingState.COMPLETED => TB("Completed"),
|
||||
DataSourceEmbeddingState.FAILED => TB("Needs attention"),
|
||||
_ => TB("Idle")
|
||||
};
|
||||
|
||||
public int SortOrder => this.State switch
|
||||
{
|
||||
DataSourceEmbeddingState.RUNNING => 0,
|
||||
DataSourceEmbeddingState.QUEUED => 1,
|
||||
DataSourceEmbeddingState.FAILED => 2,
|
||||
DataSourceEmbeddingState.COMPLETED => 3,
|
||||
_ => 4,
|
||||
};
|
||||
}
|
||||
|
||||
public sealed record DataSourceEmbeddingFailure(
|
||||
string FilePath,
|
||||
string Reason);
|
||||
|
||||
public sealed class FileEnumerationResult
|
||||
{
|
||||
public List<FileInfo> Files { get; } = [];
|
||||
|
||||
public List<DataSourceEmbeddingFailure> Failures { get; } = [];
|
||||
|
||||
public int FailedFiles { get; set; }
|
||||
|
||||
public string LastError { get; set; } = string.Empty;
|
||||
|
||||
public void AddFailure(string filePath, string reason)
|
||||
{
|
||||
this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason));
|
||||
this.FailedFiles = this.Failures.Count;
|
||||
this.LastError = reason;
|
||||
}
|
||||
}
|
||||
|
||||
public sealed class DataSourceEmbeddingManifest
|
||||
{
|
||||
public string EmbeddingProviderId { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingSignature { get; set; } = string.Empty;
|
||||
|
||||
public string SourceHash { get; set; } = string.Empty;
|
||||
|
||||
public int VectorSize { get; set; }
|
||||
|
||||
public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
public sealed record EmbeddedFileRecord(
|
||||
string Fingerprint,
|
||||
long FileSize,
|
||||
DateTimeOffset LastWriteUtc,
|
||||
DateTimeOffset EmbeddedAtUtc,
|
||||
int ChunkCount);
|
||||
@ -0,0 +1,3 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed record DataSourceEmbeddingOverview(bool IsVisible, DataSourceEmbeddingState State, int IndexedFiles, int TotalFiles, int FailedFiles);
|
||||
@ -13,7 +13,9 @@ using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, ILogger<DataSourceEmbeddingService> logger)
|
||||
public sealed partial class DataSourceEmbeddingService(
|
||||
SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider,
|
||||
ILogger<DataSourceEmbeddingService> logger)
|
||||
: BackgroundService
|
||||
{
|
||||
private const int VECTOR_STORE_OPTIMIZATION_CHUNK_THRESHOLD = 100_000;
|
||||
|
||||
@ -0,0 +1,10 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public enum DataSourceEmbeddingState
|
||||
{
|
||||
IDLE,
|
||||
QUEUED,
|
||||
RUNNING,
|
||||
COMPLETED,
|
||||
FAILED,
|
||||
}
|
||||
@ -0,0 +1,39 @@
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed record DataSourceEmbeddingStatus(
|
||||
string DataSourceId,
|
||||
string DataSourceName,
|
||||
DataSourceType DataSourceType,
|
||||
DataSourceEmbeddingState State,
|
||||
int TotalFiles,
|
||||
int IndexedFiles,
|
||||
int FailedFiles,
|
||||
string CurrentFile,
|
||||
string LastError,
|
||||
IReadOnlyList<DataSourceEmbeddingFailure> Failures)
|
||||
{
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingService).Namespace, nameof(DataSourceEmbeddingService));
|
||||
|
||||
public int ProgressPercent => this.TotalFiles <= 0 ? 0 : Math.Clamp((int)Math.Round(this.IndexedFiles * 100d / this.TotalFiles), 0, 100);
|
||||
|
||||
public string StateLabel => this.State switch
|
||||
{
|
||||
DataSourceEmbeddingState.QUEUED => TB("Queued"),
|
||||
DataSourceEmbeddingState.RUNNING => TB("Running"),
|
||||
DataSourceEmbeddingState.COMPLETED => TB("Completed"),
|
||||
DataSourceEmbeddingState.FAILED => TB("Needs attention"),
|
||||
_ => TB("Idle")
|
||||
};
|
||||
|
||||
public int SortOrder => this.State switch
|
||||
{
|
||||
DataSourceEmbeddingState.RUNNING => 0,
|
||||
DataSourceEmbeddingState.QUEUED => 1,
|
||||
DataSourceEmbeddingState.FAILED => 2,
|
||||
DataSourceEmbeddingState.COMPLETED => 3,
|
||||
_ => 4,
|
||||
};
|
||||
}
|
||||
@ -11,9 +11,7 @@ using AIStudio.Tools.Rust;
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed class DataSourceLocalRetrievalService(
|
||||
SettingsManager settingsManager,
|
||||
RustService rustService,
|
||||
DatabaseClientProvider databaseClientProvider,
|
||||
SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider,
|
||||
ILogger<DataSourceLocalRetrievalService> logger)
|
||||
{
|
||||
private enum RetrievalChannel
|
||||
|
||||
@ -0,0 +1,3 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount);
|
||||
@ -0,0 +1,19 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed class FileEnumerationResult
|
||||
{
|
||||
public List<FileInfo> Files { get; } = [];
|
||||
|
||||
public List<DataSourceEmbeddingFailure> Failures { get; } = [];
|
||||
|
||||
public int FailedFiles { get; set; }
|
||||
|
||||
public string LastError { get; set; } = string.Empty;
|
||||
|
||||
public void AddFailure(string filePath, string reason)
|
||||
{
|
||||
this.Failures.Add(new DataSourceEmbeddingFailure(filePath, reason));
|
||||
this.FailedFiles = this.Failures.Count;
|
||||
this.LastError = reason;
|
||||
}
|
||||
}
|
||||
@ -7,8 +7,6 @@ using AIStudio.Tools.Rust;
|
||||
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed record ArbitraryFileDataSegment(string Content, int TokenCount);
|
||||
|
||||
public sealed partial class RustService
|
||||
{
|
||||
public async Task<string> ReadArbitraryFileData(string path, int maxChunks, bool extractImages = false)
|
||||
|
||||
Loading…
Reference in New Issue
Block a user