mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-08-11 17:32:11 +00:00
changed DateTime to DateTimeOffset
This commit is contained in:
parent
9ab196c13c
commit
6985678d2f
@ -39,9 +39,9 @@ public sealed record EmbeddingStateFile(
|
|||||||
string FileType,
|
string FileType,
|
||||||
string Fingerprint,
|
string Fingerprint,
|
||||||
long FileSize,
|
long FileSize,
|
||||||
DateTime CreationUtc,
|
DateTimeOffset CreationUtc,
|
||||||
DateTime LastWriteUtc,
|
DateTimeOffset LastWriteUtc,
|
||||||
DateTime EmbeddedAtUtc,
|
DateTimeOffset EmbeddedAtUtc,
|
||||||
int ChunkCount,
|
int ChunkCount,
|
||||||
string ComplianceLevel,
|
string ComplianceLevel,
|
||||||
int ComplianceLevelRank);
|
int ComplianceLevelRank);
|
||||||
@ -52,7 +52,7 @@ public sealed record EmbeddingStateChunk(
|
|||||||
int? PageNumber,
|
int? PageNumber,
|
||||||
int ChunkIndex,
|
int ChunkIndex,
|
||||||
string ChunkText,
|
string ChunkText,
|
||||||
DateTime EmbeddedAtUtc);
|
DateTimeOffset EmbeddedAtUtc);
|
||||||
|
|
||||||
public sealed record EmbeddingStateSearchResult(
|
public sealed record EmbeddingStateSearchResult(
|
||||||
string ChunkId,
|
string ChunkId,
|
||||||
@ -70,9 +70,9 @@ public sealed record EmbeddingStateSearchResult(
|
|||||||
double Score,
|
double Score,
|
||||||
string Fingerprint,
|
string Fingerprint,
|
||||||
long FileSize,
|
long FileSize,
|
||||||
DateTime CreationUtc,
|
DateTimeOffset CreationUtc,
|
||||||
DateTime LastWriteUtc,
|
DateTimeOffset LastWriteUtc,
|
||||||
DateTime EmbeddedAtUtc,
|
DateTimeOffset EmbeddedAtUtc,
|
||||||
int ChunkCount,
|
int ChunkCount,
|
||||||
string ComplianceLevel,
|
string ComplianceLevel,
|
||||||
int ComplianceLevelRank);
|
int ComplianceLevelRank);
|
||||||
|
|||||||
@ -22,7 +22,7 @@ internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbC
|
|||||||
|
|
||||||
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
protected override void OnModelCreating(ModelBuilder modelBuilder)
|
||||||
{
|
{
|
||||||
var utcDateTimeConverter = new EmbeddingStateDateTimeConverter();
|
var utcDateTimeOffsetConverter = new EmbeddingStateDateTimeOffsetConverter();
|
||||||
|
|
||||||
modelBuilder.Entity<EmbeddingStateDataSourceEntity>(entity =>
|
modelBuilder.Entity<EmbeddingStateDataSourceEntity>(entity =>
|
||||||
{
|
{
|
||||||
@ -36,7 +36,7 @@ internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbC
|
|||||||
entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired();
|
entity.Property(dataSource => dataSource.EmbeddingSignature).HasColumnName("embedding_signature").IsRequired();
|
||||||
entity.Property(dataSource => dataSource.SourceHash).HasColumnName("source_hash").IsRequired().HasDefaultValue(string.Empty);
|
entity.Property(dataSource => dataSource.SourceHash).HasColumnName("source_hash").IsRequired().HasDefaultValue(string.Empty);
|
||||||
entity.Property(dataSource => dataSource.VectorSize).HasColumnName("vector_size").HasDefaultValue(0);
|
entity.Property(dataSource => dataSource.VectorSize).HasColumnName("vector_size").HasDefaultValue(0);
|
||||||
entity.Property(dataSource => dataSource.UpdatedAtUtc).HasColumnName("updated_at_utc").HasConversion(utcDateTimeConverter).IsRequired();
|
entity.Property(dataSource => dataSource.UpdatedAtUtc).HasColumnName("updated_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
|
||||||
|
|
||||||
entity
|
entity
|
||||||
.HasMany(dataSource => dataSource.Files)
|
.HasMany(dataSource => dataSource.Files)
|
||||||
@ -58,9 +58,9 @@ internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbC
|
|||||||
entity.Property(file => file.FileType).HasColumnName("file_type").IsRequired();
|
entity.Property(file => file.FileType).HasColumnName("file_type").IsRequired();
|
||||||
entity.Property(file => file.Fingerprint).HasColumnName("fingerprint").IsRequired();
|
entity.Property(file => file.Fingerprint).HasColumnName("fingerprint").IsRequired();
|
||||||
entity.Property(file => file.FileSize).HasColumnName("file_size");
|
entity.Property(file => file.FileSize).HasColumnName("file_size");
|
||||||
entity.Property(file => file.CreationUtc).HasColumnName("creation_utc").HasConversion(utcDateTimeConverter).IsRequired();
|
entity.Property(file => file.CreationUtc).HasColumnName("creation_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
|
||||||
entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeConverter).IsRequired();
|
entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
|
||||||
entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeConverter).IsRequired();
|
entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
|
||||||
entity.Property(file => file.ChunkCount).HasColumnName("chunk_count");
|
entity.Property(file => file.ChunkCount).HasColumnName("chunk_count");
|
||||||
entity.Property(file => file.ComplianceLevel).HasColumnName("compliance_level").IsRequired();
|
entity.Property(file => file.ComplianceLevel).HasColumnName("compliance_level").IsRequired();
|
||||||
entity.Property(file => file.ComplianceLevelRank).HasColumnName("compliance_level_rank");
|
entity.Property(file => file.ComplianceLevelRank).HasColumnName("compliance_level_rank");
|
||||||
@ -89,7 +89,7 @@ internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbC
|
|||||||
entity.Property(chunk => chunk.PageNumber).HasColumnName("page_number");
|
entity.Property(chunk => chunk.PageNumber).HasColumnName("page_number");
|
||||||
entity.Property(chunk => chunk.ChunkIndex).HasColumnName("chunk_index");
|
entity.Property(chunk => chunk.ChunkIndex).HasColumnName("chunk_index");
|
||||||
entity.Property(chunk => chunk.ChunkText).HasColumnName("chunk_text").IsRequired();
|
entity.Property(chunk => chunk.ChunkText).HasColumnName("chunk_text").IsRequired();
|
||||||
entity.Property(chunk => chunk.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeConverter).IsRequired();
|
entity.Property(chunk => chunk.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
|
||||||
|
|
||||||
entity.HasIndex(chunk => chunk.ChunkId).HasDatabaseName("idx_embedding_chunks_chunk_id").IsUnique();
|
entity.HasIndex(chunk => chunk.ChunkId).HasDatabaseName("idx_embedding_chunks_chunk_id").IsUnique();
|
||||||
entity.HasIndex(chunk => chunk.ParentFileId).HasDatabaseName("idx_embedding_chunks_parent_file");
|
entity.HasIndex(chunk => chunk.ParentFileId).HasDatabaseName("idx_embedding_chunks_parent_file");
|
||||||
@ -102,9 +102,9 @@ internal sealed class EmbeddingStateDbContext(DbContextOptions<EmbeddingStateDbC
|
|||||||
entity.HasNoKey();
|
entity.HasNoKey();
|
||||||
entity.ToView("embedding_chunk_search_results");
|
entity.ToView("embedding_chunk_search_results");
|
||||||
|
|
||||||
entity.Property(result => result.CreationUtc).HasConversion(utcDateTimeConverter);
|
entity.Property(result => result.CreationUtc).HasConversion(utcDateTimeOffsetConverter);
|
||||||
entity.Property(result => result.LastWriteUtc).HasConversion(utcDateTimeConverter);
|
entity.Property(result => result.LastWriteUtc).HasConversion(utcDateTimeOffsetConverter);
|
||||||
entity.Property(result => result.EmbeddedAtUtc).HasConversion(utcDateTimeConverter);
|
entity.Property(result => result.EmbeddedAtUtc).HasConversion(utcDateTimeOffsetConverter);
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -134,7 +134,7 @@ internal sealed class EmbeddingStateDataSourceEntity
|
|||||||
|
|
||||||
public int VectorSize { get; set; }
|
public int VectorSize { get; set; }
|
||||||
|
|
||||||
public DateTime UpdatedAtUtc { get; set; }
|
public DateTimeOffset UpdatedAtUtc { get; set; }
|
||||||
|
|
||||||
public List<EmbeddingStateFileEntity> Files { get; set; } = [];
|
public List<EmbeddingStateFileEntity> Files { get; set; } = [];
|
||||||
}
|
}
|
||||||
@ -157,11 +157,11 @@ internal sealed class EmbeddingStateFileEntity
|
|||||||
|
|
||||||
public long FileSize { get; set; }
|
public long FileSize { get; set; }
|
||||||
|
|
||||||
public DateTime CreationUtc { get; set; }
|
public DateTimeOffset CreationUtc { get; set; }
|
||||||
|
|
||||||
public DateTime LastWriteUtc { get; set; }
|
public DateTimeOffset LastWriteUtc { get; set; }
|
||||||
|
|
||||||
public DateTime EmbeddedAtUtc { get; set; }
|
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||||
|
|
||||||
public int ChunkCount { get; set; }
|
public int ChunkCount { get; set; }
|
||||||
|
|
||||||
@ -188,7 +188,7 @@ internal sealed class EmbeddingStateChunkEntity
|
|||||||
|
|
||||||
public string ChunkText { get; set; } = string.Empty;
|
public string ChunkText { get; set; } = string.Empty;
|
||||||
|
|
||||||
public DateTime EmbeddedAtUtc { get; set; }
|
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||||
|
|
||||||
public EmbeddingStateFileEntity? File { get; set; }
|
public EmbeddingStateFileEntity? File { get; set; }
|
||||||
}
|
}
|
||||||
@ -225,11 +225,11 @@ internal sealed class EmbeddingStateSearchResultEntity
|
|||||||
|
|
||||||
public long FileSize { get; set; }
|
public long FileSize { get; set; }
|
||||||
|
|
||||||
public DateTime CreationUtc { get; set; }
|
public DateTimeOffset CreationUtc { get; set; }
|
||||||
|
|
||||||
public DateTime LastWriteUtc { get; set; }
|
public DateTimeOffset LastWriteUtc { get; set; }
|
||||||
|
|
||||||
public DateTime EmbeddedAtUtc { get; set; }
|
public DateTimeOffset EmbeddedAtUtc { get; set; }
|
||||||
|
|
||||||
public int ChunkCount { get; set; }
|
public int ChunkCount { get; set; }
|
||||||
|
|
||||||
@ -238,22 +238,21 @@ internal sealed class EmbeddingStateSearchResultEntity
|
|||||||
public int ComplianceLevelRank { get; set; }
|
public int ComplianceLevelRank { get; set; }
|
||||||
}
|
}
|
||||||
|
|
||||||
internal sealed class EmbeddingStateDateTimeConverter() : ValueConverter<DateTime, string>(
|
internal sealed class EmbeddingStateDateTimeOffsetConverter() : ValueConverter<DateTimeOffset, string>(
|
||||||
value => EmbeddingStateDateTime.ToUtcText(value),
|
value => EmbeddingStateDateTimeOffset.ToUtcText(value),
|
||||||
value => EmbeddingStateDateTime.ParseUtc(value));
|
value => EmbeddingStateDateTimeOffset.ParseUtc(value));
|
||||||
|
|
||||||
internal static class EmbeddingStateDateTime
|
internal static class EmbeddingStateDateTimeOffset
|
||||||
{
|
{
|
||||||
public static string ToUtcText(DateTime dateTime)
|
public static string ToUtcText(DateTimeOffset dateTime)
|
||||||
{
|
{
|
||||||
var utc = dateTime.Kind is DateTimeKind.Utc ? dateTime : dateTime.ToUniversalTime();
|
return dateTime.ToUniversalTime().ToString("O", CultureInfo.InvariantCulture);
|
||||||
return utc.ToString("O", CultureInfo.InvariantCulture);
|
|
||||||
}
|
}
|
||||||
|
|
||||||
public static DateTime ParseUtc(string value)
|
public static DateTimeOffset ParseUtc(string value)
|
||||||
{
|
{
|
||||||
return DateTime.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind, out var dateTime)
|
return DateTimeOffset.TryParse(value, CultureInfo.InvariantCulture, DateTimeStyles.None, out var dateTime)
|
||||||
? dateTime.ToUniversalTime()
|
? dateTime.ToUniversalTime()
|
||||||
: DateTime.UnixEpoch;
|
: DateTimeOffset.UnixEpoch;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@ -12,7 +12,7 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
|||||||
{
|
{
|
||||||
#pragma warning disable 612, 618
|
#pragma warning disable 612, 618
|
||||||
modelBuilder.HasAnnotation("ProductVersion", "9.0.18");
|
modelBuilder.HasAnnotation("ProductVersion", "9.0.18");
|
||||||
var utcDateTimeConverter = new EmbeddingStateDateTimeConverter();
|
var utcDateTimeOffsetConverter = new EmbeddingStateDateTimeOffsetConverter();
|
||||||
|
|
||||||
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity =>
|
modelBuilder.Entity("AIStudio.Tools.Databases.EmbeddingState.EmbeddingStateDataSourceEntity", entity =>
|
||||||
{
|
{
|
||||||
@ -46,8 +46,8 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
|||||||
.HasColumnName("source_hash")
|
.HasColumnName("source_hash")
|
||||||
.HasDefaultValue(string.Empty);
|
.HasDefaultValue(string.Empty);
|
||||||
|
|
||||||
entity.Property<DateTime>("UpdatedAtUtc")
|
entity.Property<DateTimeOffset>("UpdatedAtUtc")
|
||||||
.HasConversion(utcDateTimeConverter)
|
.HasConversion(utcDateTimeOffsetConverter)
|
||||||
.HasColumnType("TEXT")
|
.HasColumnType("TEXT")
|
||||||
.HasColumnName("updated_at_utc");
|
.HasColumnName("updated_at_utc");
|
||||||
|
|
||||||
@ -86,8 +86,8 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
|||||||
.HasColumnType("INTEGER")
|
.HasColumnType("INTEGER")
|
||||||
.HasColumnName("compliance_level_rank");
|
.HasColumnName("compliance_level_rank");
|
||||||
|
|
||||||
entity.Property<DateTime>("CreationUtc")
|
entity.Property<DateTimeOffset>("CreationUtc")
|
||||||
.HasConversion(utcDateTimeConverter)
|
.HasConversion(utcDateTimeOffsetConverter)
|
||||||
.HasColumnType("TEXT")
|
.HasColumnType("TEXT")
|
||||||
.HasColumnName("creation_utc");
|
.HasColumnName("creation_utc");
|
||||||
|
|
||||||
@ -96,8 +96,8 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
|||||||
.HasColumnType("TEXT")
|
.HasColumnType("TEXT")
|
||||||
.HasColumnName("data_source_id");
|
.HasColumnName("data_source_id");
|
||||||
|
|
||||||
entity.Property<DateTime>("EmbeddedAtUtc")
|
entity.Property<DateTimeOffset>("EmbeddedAtUtc")
|
||||||
.HasConversion(utcDateTimeConverter)
|
.HasConversion(utcDateTimeOffsetConverter)
|
||||||
.HasColumnType("TEXT")
|
.HasColumnType("TEXT")
|
||||||
.HasColumnName("embedded_at_utc");
|
.HasColumnName("embedded_at_utc");
|
||||||
|
|
||||||
@ -120,8 +120,8 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
|||||||
.HasColumnType("TEXT")
|
.HasColumnType("TEXT")
|
||||||
.HasColumnName("fingerprint");
|
.HasColumnName("fingerprint");
|
||||||
|
|
||||||
entity.Property<DateTime>("LastWriteUtc")
|
entity.Property<DateTimeOffset>("LastWriteUtc")
|
||||||
.HasConversion(utcDateTimeConverter)
|
.HasConversion(utcDateTimeOffsetConverter)
|
||||||
.HasColumnType("TEXT")
|
.HasColumnType("TEXT")
|
||||||
.HasColumnName("last_write_utc");
|
.HasColumnName("last_write_utc");
|
||||||
|
|
||||||
@ -173,8 +173,8 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
|||||||
.HasColumnType("TEXT")
|
.HasColumnType("TEXT")
|
||||||
.HasColumnName("chunk_text");
|
.HasColumnName("chunk_text");
|
||||||
|
|
||||||
entity.Property<DateTime>("EmbeddedAtUtc")
|
entity.Property<DateTimeOffset>("EmbeddedAtUtc")
|
||||||
.HasConversion(utcDateTimeConverter)
|
.HasConversion(utcDateTimeOffsetConverter)
|
||||||
.HasColumnType("TEXT")
|
.HasColumnType("TEXT")
|
||||||
.HasColumnName("embedded_at_utc");
|
.HasColumnName("embedded_at_utc");
|
||||||
|
|
||||||
@ -233,8 +233,8 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
|||||||
entity.Property<int>("ComplianceLevelRank")
|
entity.Property<int>("ComplianceLevelRank")
|
||||||
.HasColumnType("INTEGER");
|
.HasColumnType("INTEGER");
|
||||||
|
|
||||||
entity.Property<DateTime>("CreationUtc")
|
entity.Property<DateTimeOffset>("CreationUtc")
|
||||||
.HasConversion(utcDateTimeConverter)
|
.HasConversion(utcDateTimeOffsetConverter)
|
||||||
.HasColumnType("TEXT");
|
.HasColumnType("TEXT");
|
||||||
|
|
||||||
entity.Property<string>("DataSourceId")
|
entity.Property<string>("DataSourceId")
|
||||||
@ -249,8 +249,8 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
|||||||
.IsRequired()
|
.IsRequired()
|
||||||
.HasColumnType("TEXT");
|
.HasColumnType("TEXT");
|
||||||
|
|
||||||
entity.Property<DateTime>("EmbeddedAtUtc")
|
entity.Property<DateTimeOffset>("EmbeddedAtUtc")
|
||||||
.HasConversion(utcDateTimeConverter)
|
.HasConversion(utcDateTimeOffsetConverter)
|
||||||
.HasColumnType("TEXT");
|
.HasColumnType("TEXT");
|
||||||
|
|
||||||
entity.Property<string>("FileName")
|
entity.Property<string>("FileName")
|
||||||
@ -268,8 +268,8 @@ partial class EmbeddingStateDbContextModelSnapshot : ModelSnapshot
|
|||||||
.IsRequired()
|
.IsRequired()
|
||||||
.HasColumnType("TEXT");
|
.HasColumnType("TEXT");
|
||||||
|
|
||||||
entity.Property<DateTime>("LastWriteUtc")
|
entity.Property<DateTimeOffset>("LastWriteUtc")
|
||||||
.HasConversion(utcDateTimeConverter)
|
.HasConversion(utcDateTimeOffsetConverter)
|
||||||
.HasColumnType("TEXT");
|
.HasColumnType("TEXT");
|
||||||
|
|
||||||
entity.Property<int?>("PageNumber")
|
entity.Property<int?>("PageNumber")
|
||||||
|
|||||||
@ -139,7 +139,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
|||||||
return;
|
return;
|
||||||
|
|
||||||
dataSource.VectorSize = vectorSize;
|
dataSource.VectorSize = vectorSize;
|
||||||
dataSource.UpdatedAtUtc = DateTime.UtcNow;
|
dataSource.UpdatedAtUtc = DateTimeOffset.UtcNow;
|
||||||
await context.SaveChangesAsync(token);
|
await context.SaveChangesAsync(token);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -151,7 +151,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
|||||||
return;
|
return;
|
||||||
|
|
||||||
dataSource.SourceHash = sourceHash;
|
dataSource.SourceHash = sourceHash;
|
||||||
dataSource.UpdatedAtUtc = DateTime.UtcNow;
|
dataSource.UpdatedAtUtc = DateTimeOffset.UtcNow;
|
||||||
await context.SaveChangesAsync(token);
|
await context.SaveChangesAsync(token);
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -343,7 +343,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
|
|||||||
dataSource.EmbeddingSignature = embeddingSignature;
|
dataSource.EmbeddingSignature = embeddingSignature;
|
||||||
dataSource.SourceHash = sourceHash;
|
dataSource.SourceHash = sourceHash;
|
||||||
dataSource.VectorSize = vectorSize;
|
dataSource.VectorSize = vectorSize;
|
||||||
dataSource.UpdatedAtUtc = DateTime.UtcNow;
|
dataSource.UpdatedAtUtc = DateTimeOffset.UtcNow;
|
||||||
}
|
}
|
||||||
|
|
||||||
private static void ApplyFile(EmbeddingStateFileEntity fileEntity, string dataSourceId, EmbeddingStateFile file)
|
private static void ApplyFile(EmbeddingStateFileEntity fileEntity, string dataSourceId, EmbeddingStateFile file)
|
||||||
|
|||||||
@ -17,8 +17,8 @@ public sealed record VectorStoragePoint(
|
|||||||
int ChunkIndex,
|
int ChunkIndex,
|
||||||
string Text,
|
string Text,
|
||||||
string Fingerprint,
|
string Fingerprint,
|
||||||
DateTime CreationUtc,
|
DateTimeOffset CreationUtc,
|
||||||
DateTime LastWriteUtc,
|
DateTimeOffset LastWriteUtc,
|
||||||
DateTime EmbeddedAtUtc,
|
DateTimeOffset EmbeddedAtUtc,
|
||||||
string ComplianceLevel,
|
string ComplianceLevel,
|
||||||
int ComplianceLevelRank);
|
int ComplianceLevelRank);
|
||||||
|
|||||||
@ -92,6 +92,6 @@ public sealed class DataSourceEmbeddingManifest
|
|||||||
public sealed record EmbeddedFileRecord(
|
public sealed record EmbeddedFileRecord(
|
||||||
string Fingerprint,
|
string Fingerprint,
|
||||||
long FileSize,
|
long FileSize,
|
||||||
DateTime LastWriteUtc,
|
DateTimeOffset LastWriteUtc,
|
||||||
DateTime EmbeddedAtUtc,
|
DateTimeOffset EmbeddedAtUtc,
|
||||||
int ChunkCount);
|
int ChunkCount);
|
||||||
|
|||||||
@ -1105,7 +1105,7 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
return Convert.ToHexString(bytes);
|
return Convert.ToHexString(bytes);
|
||||||
}
|
}
|
||||||
|
|
||||||
private EmbeddingStateFile CreateEmbeddingStateFile(IDataSource dataSource, FileInfo file, string fingerprint, int chunkCount, DateTime embeddedAtUtc)
|
private EmbeddingStateFile CreateEmbeddingStateFile(IDataSource dataSource, FileInfo file, string fingerprint, int chunkCount, DateTimeOffset embeddedAtUtc)
|
||||||
{
|
{
|
||||||
file.Refresh();
|
file.Refresh();
|
||||||
var absolutePath = Path.GetFullPath(file.FullName);
|
var absolutePath = Path.GetFullPath(file.FullName);
|
||||||
@ -1118,15 +1118,15 @@ public sealed partial class DataSourceEmbeddingService
|
|||||||
GetFileType(file),
|
GetFileType(file),
|
||||||
fingerprint,
|
fingerprint,
|
||||||
file.Exists ? file.Length : 0,
|
file.Exists ? file.Length : 0,
|
||||||
file.Exists ? file.CreationTimeUtc : DateTime.UnixEpoch,
|
file.Exists ? new DateTimeOffset(file.CreationTimeUtc) : DateTimeOffset.UnixEpoch,
|
||||||
file.Exists ? file.LastWriteTimeUtc : DateTime.UnixEpoch,
|
file.Exists ? new DateTimeOffset(file.LastWriteTimeUtc) : DateTimeOffset.UnixEpoch,
|
||||||
embeddedAtUtc,
|
embeddedAtUtc,
|
||||||
chunkCount,
|
chunkCount,
|
||||||
complianceLevel.ToString(),
|
complianceLevel.ToString(),
|
||||||
(int)complianceLevel);
|
(int)complianceLevel);
|
||||||
}
|
}
|
||||||
|
|
||||||
private IReadOnlyList<EmbeddingStateChunk> CreateEmbeddingStateChunks(EmbeddingStateFile parentFile, IReadOnlyList<EmbeddingChunkDraft> batch, DateTime embeddedAtUtc)
|
private IReadOnlyList<EmbeddingStateChunk> CreateEmbeddingStateChunks(EmbeddingStateFile parentFile, IReadOnlyList<EmbeddingChunkDraft> batch, DateTimeOffset embeddedAtUtc)
|
||||||
{
|
{
|
||||||
return batch
|
return batch
|
||||||
.Select(chunk => new EmbeddingStateChunk(
|
.Select(chunk => new EmbeddingStateChunk(
|
||||||
|
|||||||
@ -567,18 +567,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
ShortHash(fingerprint),
|
ShortHash(fingerprint),
|
||||||
skippedFiles + completedFiles + 1,
|
skippedFiles + completedFiles + 1,
|
||||||
totalFiles);
|
totalFiles);
|
||||||
var startedAtUtc = DateTime.UtcNow;
|
var startedAtUtc = DateTimeOffset.UtcNow;
|
||||||
var chunkCount = await this.IndexOneFileAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token);
|
var chunkCount = await this.IndexOneFileAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, optimizationTracker, token);
|
||||||
token.ThrowIfCancellationRequested();
|
token.ThrowIfCancellationRequested();
|
||||||
var fingerprintAfterEmbedding = BuildFileMetadataHash(file);
|
var fingerprintAfterEmbedding = BuildFileMetadataHash(file);
|
||||||
if (!string.Equals(fingerprint, fingerprintAfterEmbedding, StringComparison.Ordinal))
|
if (!string.Equals(fingerprint, fingerprintAfterEmbedding, StringComparison.Ordinal))
|
||||||
throw new IOException($"The file '{file.FullName}' changed while it was being embedded. Its partial embeddings will be discarded and the file will be retried on the next refresh.");
|
throw new IOException($"The file '{file.FullName}' changed while it was being embedded. Its partial embeddings will be discarded and the file will be retried on the next refresh.");
|
||||||
|
|
||||||
var embeddedAtUtc = DateTime.UtcNow;
|
var embeddedAtUtc = DateTimeOffset.UtcNow;
|
||||||
var record = new EmbeddedFileRecord(
|
var record = new EmbeddedFileRecord(
|
||||||
fingerprint,
|
fingerprint,
|
||||||
file.Length,
|
file.Length,
|
||||||
file.LastWriteTimeUtc,
|
new DateTimeOffset(file.LastWriteTimeUtc),
|
||||||
embeddedAtUtc,
|
embeddedAtUtc,
|
||||||
chunkCount);
|
chunkCount);
|
||||||
await embeddingState.UpsertFileAsync(
|
await embeddingState.UpsertFileAsync(
|
||||||
@ -598,7 +598,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
dataSource.Name,
|
dataSource.Name,
|
||||||
dataSource.Id,
|
dataSource.Id,
|
||||||
chunkCount,
|
chunkCount,
|
||||||
(DateTime.UtcNow - startedAtUtc).TotalMilliseconds);
|
(DateTimeOffset.UtcNow - startedAtUtc).TotalMilliseconds);
|
||||||
}
|
}
|
||||||
catch (OperationCanceledException) when (token.IsCancellationRequested)
|
catch (OperationCanceledException) when (token.IsCancellationRequested)
|
||||||
{
|
{
|
||||||
@ -668,7 +668,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
optimizationTracker.MarkChanged();
|
optimizationTracker.MarkChanged();
|
||||||
await embeddingState.DeleteFileAsync(dataSource.Id, file.FullName, token);
|
await embeddingState.DeleteFileAsync(dataSource.Id, file.FullName, token);
|
||||||
|
|
||||||
var parentFile = this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTime.UtcNow);
|
var parentFile = this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow);
|
||||||
await embeddingState.UpsertFileAsync(dataSource.Id, parentFile, token);
|
await embeddingState.UpsertFileAsync(dataSource.Id, parentFile, token);
|
||||||
|
|
||||||
var embeddingBatchSize = Math.Max(1, embeddingProvider.EffectiveEmbeddingBatchSize);
|
var embeddingBatchSize = Math.Max(1, embeddingProvider.EffectiveEmbeddingBatchSize);
|
||||||
@ -769,7 +769,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
}
|
}
|
||||||
|
|
||||||
token.ThrowIfCancellationRequested();
|
token.ThrowIfCancellationRequested();
|
||||||
var embeddedAtUtc = DateTime.UtcNow;
|
var embeddedAtUtc = DateTimeOffset.UtcNow;
|
||||||
await this.UpsertPointsAsync(
|
await this.UpsertPointsAsync(
|
||||||
vectorStore,
|
vectorStore,
|
||||||
collectionName,
|
collectionName,
|
||||||
@ -820,7 +820,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
EmbeddingStateFile parentFile,
|
EmbeddingStateFile parentFile,
|
||||||
IReadOnlyList<EmbeddingChunkDraft> batch,
|
IReadOnlyList<EmbeddingChunkDraft> batch,
|
||||||
IReadOnlyList<IReadOnlyList<float>> vectors,
|
IReadOnlyList<IReadOnlyList<float>> vectors,
|
||||||
DateTime embeddedAtUtc,
|
DateTimeOffset embeddedAtUtc,
|
||||||
CancellationToken token)
|
CancellationToken token)
|
||||||
{
|
{
|
||||||
var points = batch.Select((item, index) => new VectorStoragePoint(
|
var points = batch.Select((item, index) => new VectorStoragePoint(
|
||||||
@ -1150,7 +1150,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
|||||||
if (existingRecord.FileSize != file.Length)
|
if (existingRecord.FileSize != file.Length)
|
||||||
reasons.Add($"file size changed from {existingRecord.FileSize} to {file.Length} bytes");
|
reasons.Add($"file size changed from {existingRecord.FileSize} to {file.Length} bytes");
|
||||||
|
|
||||||
if (existingRecord.LastWriteUtc != file.LastWriteTimeUtc)
|
if (existingRecord.LastWriteUtc != new DateTimeOffset(file.LastWriteTimeUtc))
|
||||||
reasons.Add($"last modified time changed from {existingRecord.LastWriteUtc:O} to {file.LastWriteTimeUtc:O}");
|
reasons.Add($"last modified time changed from {existingRecord.LastWriteUtc:O} to {file.LastWriteTimeUtc:O}");
|
||||||
|
|
||||||
return reasons.Count == 0
|
return reasons.Count == 0
|
||||||
|
|||||||
Loading…
Reference in New Issue
Block a user