Allowed moving local data sources without embedding them again (#1091)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Paul KoudelkaandThorsten Sommer authored and GitHub committed 2026-10-20 18:50:09 +02:00
1 parent f6e48e95f3
commit 5d6e48ef93
49 files changed
+2280 -297

No files matched your search

@@ -29,6 +29,11 @@ public abstract partial class IndexStoreClient(string name, string path) : Datab
public abstract Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token);
/// <summary>
/// Updates the location and fingerprint of a document, preserving its chunks and their ids.
/// </summary>
public abstract Task RelocateFileAsync(string dataSourceId, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token);
public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token);
public abstract Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token);
@@ -52,6 +52,9 @@ public sealed partial class NoIndexStoreClient(string name, string? unavailableR
public override Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token) => Task.CompletedTask;
public override Task RelocateFileAsync(string dataSourceId, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token) =>
Task.FromException(new InvalidOperationException(unavailableReason ?? "The index of the data sources is not available."));
public override Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask;
public override Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token) => Task.CompletedTask;
@@ -149,7 +149,9 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
file.LastWriteUtc,
file.EmbeddedAtUtc,
file.ChunkCount,
file.TranscriptKey);
file.TranscriptKey,
file.RelativePath,
file.ParentFileId);
}
var permanentFailures = await context.PermanentIndexingFailures
@@ -233,6 +235,18 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
await context.SaveChangesAsync(token);
}
public override async Task RelocateFileAsync(string dataSourceId, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token)
{
await using var context = this.CreateContext();
var file = await context.EmbeddedFiles.SingleAsync(file => file.DataSourceId == dataSourceId && file.ParentFileId == parentFileId, token);
// Keep the parent and chunk ids: moving a document does not change its stored vectors.
file.AbsolutePath = absolutePath;
file.RelativePath = relativePath;
file.FileName = Path.GetFileName(absolutePath);
file.Fingerprint = fingerprint;
await context.SaveChangesAsync(token);
}
public override async Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token)
{
await using var context = this.CreateContext();
@@ -762,4 +776,4 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
client.SetLogger(databaseClientLogger);
return client;
}
}
}