Allowed moving local data sources without embedding them again (#1091)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Paul KoudelkaandThorsten Sommer authored and GitHub committed 2026-10-20 18:50:09 +02:00
1 parent f6e48e95f3
commit 5d6e48ef93
49 files changed
+2280 -297

No files matched your search

@@ -0,0 +1,752 @@
using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using AIStudio.Components;
using AIStudio.Dialogs;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tests.Tools.Databases;
using AIStudio.Tools;
using AIStudio.Tools.Databases;
using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.DataBackup;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services;
using AIStudio.Tools.Services.Indexing;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using MudBlazor;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Runs the actual edit, queue, cancellation, manifest preparation, and file indexing code against
/// a migrated SQLite store. Only the vector store and dialog rendering are replaced.
/// </summary>
[NonParallelizable]
public sealed class DataSourceRelocationWorkflowTests
{
private const string EMBEDDING_ID = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01";
// Frozen from the format before relocation support. Do not generate this with the current
// signature builder: doing that would hide an accidental rebuild of every existing index.
private const string LEGACY_SIGNATURE = "2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100";
private static readonly TimeSpan TEST_TIMEOUT = TimeSpan.FromSeconds(10);
private string testRoot = string.Empty;
private TemporaryIndexStore store = null!;
private SettingsManager settings = null!;
private DatabaseClientProvider databases = null!;
private DataSourceEmbeddingService service = null!;
private RelocationVectorStore vectors = null!;
private IInternalDataSource source = null!;
private readonly List<string> chunkIds = [];
private readonly List<Task> runningTasks = [];
private CancellationTokenSource lifetime = null!;
private string OriginalFolder => Path.Combine(this.testRoot, "original");
[SetUp]
public async Task SetUp()
{
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-relocation-workflow-{Guid.NewGuid():N}");
Directory.CreateDirectory(this.testRoot);
this.store = await TemporaryIndexStore.CreateAsync();
this.settings = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
this.settings.ConfigurationData.App.EnabledPreviewFeatures.Add(PreviewFeatures.PRE_RAG_2024);
this.settings.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh = false;
this.settings.ConfigurationData.EmbeddingProviders.Add(new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small")));
this.vectors = new RelocationVectorStore();
this.databases = new DatabaseClientProvider(null!, NullLoggerFactory.Instance);
var clients = Field<Dictionary<DatabaseRole, DatabaseClient>>(this.databases, "clients");
clients.Add(DatabaseRole.INDEX_STORE, this.store.Client);
clients.Add(DatabaseRole.VECTOR_STORE, this.vectors);
var guard = new PromptInjectionGuardService(null!, this.settings, NullLogger<PromptInjectionGuardService>.Instance, NullLoggerFactory.Instance);
this.service = new DataSourceEmbeddingService(this.settings, null!, this.databases, guard, null!, NullLogger<DataSourceEmbeddingService>.Instance);
this.lifetime = new CancellationTokenSource();
this.chunkIds.Clear();
this.runningTasks.Clear();
}
[TearDown]
public async Task TearDown()
{
this.lifetime.Cancel();
try
{
await Task.WhenAll(this.runningTasks).WaitAsync(TEST_TIMEOUT);
}
catch (OperationCanceledException) when (this.lifetime.IsCancellationRequested)
{
// A failed test may leave its deliberately blocked run for teardown to stop.
}
finally
{
this.service.Dispose();
this.databases.Dispose();
await this.store.DisposeAsync();
this.lifetime.Dispose();
Directory.Delete(this.testRoot, true);
}
}
[TestCase(false)]
[TestCase(true)]
public async Task AnExistingIndexSurvivesTheUpgradeWithoutReembedding(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.RelocatedPaths, Is.Empty, "An upgrade is not a relocation.");
Assert.That(this.vectors.DeletedStores, Is.Zero);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
});
}
[TestCase(false)]
[TestCase(true)]
public async Task EditingAndCancelingDoNothingAndSavingOnlyRelocatesUnchangedFiles(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
var newPath = this.MoveSource("moved");
var original = this.source;
var edit = this.OpenEdit(newPath);
Assert.Multiple(() =>
{
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
Assert.That(this.Queue.TryDequeue(out _), Is.False);
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
Assert.That(edit.Warning.ShowCount, Is.Zero, "Typing a path must not open the save confirmation.");
});
edit.Dialog.Instance.Cancel();
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
Assert.That(this.Queue.TryDequeue(out _), Is.False);
await this.SaveEditAsync(newPath);
Assert.That(this.vectors.RelocatedPaths, Is.Empty, "Saving queues the operation; the dialog must not write to either index.");
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
Assert.That(this.vectors.DeletedStores, Is.Zero);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
});
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
}
[TestCase(false)]
[TestCase(true)]
public async Task SavingAnUnchangedPathDoesNotWarnOrRelocate(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
var currentPath = this.source is DataSourceLocalDirectory folder ? folder.Path : ((DataSourceLocalFile)this.source).FilePath;
var edit = this.OpenEdit(currentPath);
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.Zero);
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
Assert.That(this.vectors.DeletedStores, Is.Zero);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
});
}
[TestCase(false)]
[TestCase(true)]
public async Task SavingWaitsForThePathWarningBeforeChangingConfigurationOrQueueingWork(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
var original = this.source;
var edit = this.OpenEdit(this.MoveSource("moved"));
var save = InvokeTask(edit.Component, "StoreValidatedDataSourceAsync");
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
Assert.That(save.IsCompleted, Is.False);
Assert.That(edit.ParentTask.IsCompleted, Is.False);
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
Assert.That(this.Queue.TryDequeue(out _), Is.False);
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
});
edit.Warning.Complete(DialogResult.Ok(true));
await save.WaitAsync(TEST_TIMEOUT);
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
}
[TestCase(false)]
[TestCase(true)]
public async Task CancelingThePathWarningKeepsTheConfiguredPathAndQueuesNothing(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
var original = this.source;
var edit = this.OpenEdit(this.MoveSource("moved"));
edit.Warning.Complete(DialogResult.Cancel());
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync");
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
Assert.That(edit.ParentTask.IsCompleted, Is.False, "Rejecting the warning leaves the edit dialog open.");
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
Assert.That(this.Queue.TryDequeue(out _), Is.False);
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
});
edit.Dialog.Instance.Cancel();
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
}
[TestCase(false)]
[TestCase(true)]
public async Task ANewLocationOfADataSourceWithoutAnIndexIsSavedWithoutAQuestion(bool singleFile)
{
await this.AddSourceAsync(singleFile);
var newPath = this.MoveSource("moved");
var edit = this.OpenEdit(newPath);
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
var saved = this.settings.ConfigurationData.DataSources.Single();
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.Zero, "Nothing was prepared for the data source, so nothing can be lost.");
Assert.That(saved is DataSourceLocalDirectory folder ? folder.Path : ((DataSourceLocalFile)saved).FilePath, Is.EqualTo(newPath));
});
}
[TestCase(false)]
[TestCase(true)]
public async Task ANewLocationTogetherWithANewEmbeddingAsksOnlyOnce(bool singleFile)
{
const string OTHER_EMBEDDING_ID = "c1b5d5e3-2a4f-4b1b-9d0e-6b7c8d9e0f12";
this.settings.ConfigurationData.EmbeddingProviders.Add(new EmbeddingProvider(2, OTHER_EMBEDDING_ID, "Other embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-large", "text-embedding-3-large")));
await this.SeedLegacyIndexAsync(singleFile);
var edit = this.OpenEdit(this.MoveSource("moved"));
SetField(edit.Component, "dataEmbeddingId", OTHER_EMBEDDING_ID);
edit.Warning.Complete(DialogResult.Ok(true));
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1), "Everything is prepared again anyway, so the location needs no question of its own.");
Assert.That(edit.Warning.Title, Is.EqualTo("Documents Will Be Prepared Again"));
});
}
[TestCase(false)]
[TestCase(true)]
public async Task ASecondSavedPathCancelsTheFirstRelocationAndRunsWithTheLatestPath(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
await this.SaveEditAsync(this.MoveSource("first"));
this.vectors.BlockNextRelocation = true;
var firstRun = this.RunNextAsync();
await this.vectors.RelocationStarted.Task.WaitAsync(TEST_TIMEOUT);
var secondPath = this.MoveSource("second");
await this.SaveEditAsync(secondPath);
await firstRun.WaitAsync(TEST_TIMEOUT);
Assert.That(this.vectors.CanceledRelocations, Is.EqualTo(1), "The first run must be canceled without finishing its remaining documents.");
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(1));
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.RelocatedPaths.Skip(1).ToArray(), Has.Length.EqualTo(this.chunkIds.Count));
Assert.That(this.vectors.Locations.Values.All(path => singleFile ? path == secondPath : path.StartsWith(secondPath + Path.DirectorySeparatorChar, StringComparison.Ordinal)), Is.True);
Assert.That(this.vectors.DeletedStores, Is.Zero);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.Queue.TryDequeue(out _), Is.False, "The obsolete destination must not leave another run behind.");
});
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
}
[TestCase(false)]
[TestCase(true)]
public async Task DeletingTheSourceCancelsRelocationBeforeRemovingTheStoredIndex(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
await this.SaveEditAsync(this.MoveSource("moved"));
this.vectors.BlockNextRelocation = true;
var run = this.RunNextAsync();
await this.vectors.RelocationStarted.Task.WaitAsync(TEST_TIMEOUT);
// A second save also leaves a pending run; deletion has to discard that one as well.
await this.service.QueueDataSourceAsync(this.source);
this.settings.ConfigurationData.DataSources.Clear();
await this.service.RemoveDataSourceAsync(this.source).WaitAsync(TEST_TIMEOUT);
await run.WaitAsync(TEST_TIMEOUT);
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.vectors.CanceledRelocations, Is.EqualTo(1));
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(1));
Assert.That(this.vectors.DeletedStores, Is.EqualTo(1));
Assert.That(this.vectors.Locations, Is.Empty);
Assert.That(manifest.Files, Is.Empty);
Assert.That(this.service.GetStatuses(), Is.Empty);
Assert.That(this.Queue.TryDequeue(out _), Is.False);
});
}
private DataSourceRunQueue Queue => Field<DataSourceRunQueue>(this.service, "runQueue");
[TestCase(@"E:\Documents", '\\', false)]
[TestCase(@"\\server\share\Documents", '\\', false)]
[TestCase("/home/alice/Documents", '/', false)]
[TestCase(@"E:\Documents", '\\', true)]
[TestCase(@"\\server\share\Documents", '\\', true)]
[TestCase("/Volumes/External drive/Documents", '/', true)]
public async Task ABackupCanReconnectFilesFromAnotherOSWithoutReembedding(string oldRoot, char separator, bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
var originalFolder = Path.Combine(this.testRoot, "original");
await this.StoreForeignPathsAsync(oldRoot, separator);
var restored = await this.BackupAndRestoreIndexAsync();
// Opening the restored index while the old drive or network share is unavailable must not
// prune anything. No real external volume or network connection is needed by these tests.
if (!RelocatedFilePaths.IsLocalAbsolute(oldRoot))
{
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
await this.AssertOriginalChunksAsync();
}
// User documents are not in an AI Studio backup. Copy them separately, preserving their
// modification times, as when reconnecting an external drive after restoring a computer.
var destination = Path.Combine(this.testRoot, "mounted volume", "Übertragene Dokumente");
foreach (var original in Directory.EnumerateFiles(originalFolder, "*", SearchOption.AllDirectories))
{
var localPath = Path.Combine(destination, Path.GetRelativePath(originalFolder, original));
Directory.CreateDirectory(Path.GetDirectoryName(localPath)!);
File.Copy(original, localPath);
File.SetLastWriteTimeUtc(localPath, File.GetLastWriteTimeUtc(original));
}
var newPath = singleFile ? Path.Combine(destination, "notes.md") : destination;
await this.SaveEditAsync(newPath);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
var manifest = await restored.GetManifestAsync(this.source.Id, CancellationToken.None);
var hits = await restored.SearchChunksAsync(this.source.Id, "indexed", 10, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
Assert.That(this.vectors.DeletedStores, Is.Zero);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
Assert.That(hits.Select(hit => hit.AbsolutePath), Is.EquivalentTo(this.vectors.Locations.Values));
Assert.That(hits.All(hit => PathTools.IsInsideDirectory(destination, hit.AbsolutePath)), Is.True);
});
// A restart/hash check after relocation must not relocate or prepare everything a second time.
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
await this.AssertOriginalChunksAsync();
// A disconnected external drive is not an empty folder. Reconnecting it must also keep
// the embeddings, rather than paying to prepare the same documents again.
var disconnected = destination + "-disconnected";
Directory.Move(destination, disconnected);
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
await this.AssertOriginalChunksAsync();
Directory.Move(disconnected, destination);
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
});
}
private async Task StoreForeignPathsAsync(string oldRoot, char separator)
{
await using var context = this.store.CreateContext();
foreach (var file in await context.EmbeddedFiles.ToListAsync())
{
var relative = file.RelativePath.Replace(Path.DirectorySeparatorChar, separator);
var oldPath = oldRoot + separator + relative;
file.AbsolutePath = oldPath;
file.RelativePath = relative;
file.Fingerprint = LegacyFingerprint(oldPath, file.FileName, file.LastWriteUtc.UtcDateTime.Ticks, file.FileSize);
this.vectors.Locations[file.ParentFileId] = oldPath;
}
await context.SaveChangesAsync();
this.source = this.source switch
{
DataSourceLocalDirectory folder => folder with { Path = oldRoot },
DataSourceLocalFile file => file with { FilePath = oldRoot + separator + "notes.md" },
_ => throw new InvalidOperationException(),
};
this.settings.ConfigurationData.DataSources[0] = this.source;
}
private async Task<SqliteIndexStoreClientImplementation> BackupAndRestoreIndexAsync()
{
var dataDirectory = Path.Combine(this.testRoot, "source-data");
var configDirectory = Path.Combine(this.testRoot, "source-config");
var copies = Path.Combine(this.testRoot, "index-copies");
Directory.CreateDirectory(dataDirectory);
Directory.CreateDirectory(configDirectory);
await this.store.Client.BackupToDirectoryAsync(Path.Combine(copies, "databases", "sqlite"), CancellationToken.None);
// This fixture's vector client stores metadata only. The runtime tests separately check
// backing up, reopening, and relocating actual Qdrant vectors without changing their values.
var vectorDirectory = Path.Combine(copies, "databases", "vector_database", "stores", "store_test");
Directory.CreateDirectory(vectorDirectory);
await File.WriteAllTextAsync(Path.Combine(vectorDirectory, "locations.json"), JsonSerializer.Serialize(this.vectors.Locations));
var archivePath = Path.Combine(this.testRoot, "backup.zip");
var settingsJson = JsonSerializer.Serialize(this.settings.ConfigurationData, SettingsManager.JSON_OPTIONS);
await using (var output = File.Create(archivePath))
await DataBackupWriter.WriteAsync(output, new DataBackupSources(dataDirectory, configDirectory, settingsJson, copies, []),
new DataBackupManifest(DataBackupFormat.FORMAT_VERSION, "26.10.1", "V6", DateTimeOffset.UtcNow, dataDirectory, configDirectory, true, []), null, CancellationToken.None);
var target = new DataRestoreTarget(Path.Combine(this.testRoot, "restored-data"), Path.Combine(this.testRoot, "restored-config"), "26.10.1", AIStudio.Settings.Version.V6, []);
var result = await DataRestorePreparer.PrepareAsync(archivePath, target, null, CancellationToken.None);
Assert.That(result.Success, Is.True);
// The runtime applies this prepared directory at restart. Open the exact restored bytes
// here, so this test exercises the real backup writer and restore preparer, not a rebuilt DB.
var pendingData = Path.Combine(target.DataDirectory, DataRestoreFormat.PENDING_DIRECTORY);
var restoredPath = Path.Combine(pendingData, "databases", "sqlite", "rag-index.sqlite3");
var restored = new SqliteIndexStoreClientImplementation("Restored SQLite", restoredPath, pendingData, string.Empty);
Field<Dictionary<DatabaseRole, DatabaseClient>>(this.databases, "clients")[DatabaseRole.INDEX_STORE] = restored;
this.vectors.Locations.Clear();
foreach (var (id, path) in JsonSerializer.Deserialize<Dictionary<string, string>>(await File.ReadAllTextAsync(Path.Combine(pendingData, "databases", "vector_database", "stores", "store_test", "locations.json")))!)
this.vectors.Locations.Add(id, path);
var restoredSettings = JsonSerializer.Deserialize<Data>(await File.ReadAllTextAsync(Path.Combine(target.ConfigDirectory, DataRestoreFormat.PENDING_DIRECTORY, "settings.json")), SettingsManager.JSON_OPTIONS)!;
this.settings.ConfigurationData.DataSources.Clear();
this.settings.ConfigurationData.DataSources.AddRange(restoredSettings.DataSources);
this.source = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
return restored;
}
[TestCase(false)]
[TestCase(true)]
public async Task AnEmptyMountDirectoryKeepsTheIndexAndReconnectingReusesIt(bool relocated)
{
await this.SeedLegacyIndexAsync(false);
var original = ((DataSourceLocalDirectory)this.source).Path;
var disconnected = Path.Combine(this.testRoot, "disconnected-drive");
Directory.Move(original, disconnected);
var mount = relocated ? Path.Combine(this.testRoot, "new-mount") : original;
Directory.CreateDirectory(mount);
var storedHash = (await this.store.Client.GetDataSourceStateAsync(this.source.Id, CancellationToken.None))!.SourceHash;
if (relocated)
await this.SaveEditAsync(mount);
else
await this.service.QueueDataSourceAsync(this.source);
// A second refresh must not accept the empty mount as a new baseline and prune the index.
for (var attempt = 0; attempt < 2; attempt++)
{
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
var state = await this.store.Client.GetDataSourceStateAsync(this.source.Id, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
Assert.That(this.service.GetStatuses().Single().LastError, Does.Contain("appears empty"));
Assert.That(state!.SourceHash, Is.EqualTo(storedHash));
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
});
if (attempt == 0)
await this.service.QueueDataSourceAsync(this.source);
}
Directory.Delete(mount);
Directory.Move(disconnected, mount);
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(relocated ? this.chunkIds.Count : 0));
});
}
[Test]
public async Task RemovingSomeFilesStillCleansUpOnlyTheirIndexEntries()
{
await this.SeedLegacyIndexAsync(false);
var removed = Path.Combine(((DataSourceLocalDirectory)this.source).Path, "notes.md");
var removedChunkId = IndexedDocumentIds.CreateChunkId(this.source.Id, LegacyFingerprint(new FileInfo(removed)), 0);
File.Delete(removed);
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
var chunks = await this.store.Client.SearchChunksAsync(this.source.Id, "indexed", 10, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(chunks.Select(chunk => chunk.ChunkId), Is.EquivalentTo(this.chunkIds.Except([removedChunkId])));
Assert.That(this.vectors.DeletedDocuments, Is.EqualTo(1));
Assert.That(this.vectors.InsertedBatches, Is.Zero);
});
}
[Test]
public async Task ExplicitlyExcludingEveryFileStillRemovesTheirIndexEntries()
{
await this.SeedLegacyIndexAsync(false);
this.source = ((DataSourceLocalDirectory)this.source) with { FilePatterns = "*.pdf" };
this.settings.ConfigurationData.DataSources[0] = this.source;
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(manifest.Files, Is.Empty);
Assert.That(this.vectors.Locations, Is.Empty);
Assert.That(this.vectors.DeletedDocuments, Is.EqualTo(2));
});
}
[Test]
public async Task AFolderWhoseNameChangesOnlyInCaseKeepsItsEmbeddings()
{
await this.SeedLegacyIndexAsync(false);
var renamed = Path.Combine(this.testRoot, "Original");
// Two moves, since a volume which ignores case may refuse to rename a folder to its own name:
var detour = Path.Combine(this.testRoot, "renaming");
Directory.Move(((DataSourceLocalDirectory)this.source).Path, detour);
Directory.Move(detour, renamed);
await this.SaveEditAsync(renamed);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
Assert.That(manifest.Files.Keys.All(path => path.StartsWith(renamed + Path.DirectorySeparatorChar, StringComparison.Ordinal)), Is.True, "The index must store the new spelling of the folder.");
});
}
private async Task SeedLegacyIndexAsync(bool singleFile)
{
var paths = await this.AddSourceAsync(singleFile);
var id = this.source.Id;
await this.store.Client.UpsertDataSourceAsync(id, this.source.Type.ToString(), EMBEDDING_ID, LEGACY_SIGNATURE, "legacy-source-hash", 3, CancellationToken.None);
foreach (var relative in paths)
{
var path = Path.Combine(this.OriginalFolder, relative);
var file = new FileInfo(path);
var fingerprint = LegacyFingerprint(file);
var parentId = IndexedDocumentIds.CreateParentId(id, path);
var chunkId = IndexedDocumentIds.CreateChunkId(id, fingerprint, 0);
var embeddedAt = DateTimeOffset.UtcNow.AddDays(-10);
await this.store.Client.UpsertFileAsync(id, new EmbeddingStateFile(parentId, path, file.Name, relative, "md", fingerprint, file.Length,
new DateTimeOffset(file.CreationTimeUtc), new DateTimeOffset(file.LastWriteTimeUtc), embeddedAt, 1), CancellationToken.None);
await this.store.Client.UpsertChunksAsync(id, [new EmbeddingStateChunk(chunkId, parentId, 1, 0, "Already indexed report", embeddedAt)], CancellationToken.None);
this.vectors.Locations.Add(parentId, path);
this.chunkIds.Add(chunkId);
}
}
/// <summary>
/// Writes the documents of a data source and configures it, without anything in the index.
/// </summary>
/// <returns>The paths of the documents, relative to the folder.</returns>
private async Task<string[]> AddSourceAsync(bool singleFile)
{
var root = this.OriginalFolder;
Directory.CreateDirectory(Path.Combine(root, "nested"));
var id = Guid.NewGuid().ToString();
var folder = new DataSourceLocalDirectory
{
Id = id, Name = "Existing documents", Type = DataSourceType.LOCAL_DIRECTORY, Path = root,
EmbeddingId = EMBEDDING_ID, ConfidenceLevel = ConfidenceLevel.NONE, MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 100,
FilePatterns = "*.md", ExcludePatterns = string.Empty,
};
this.source = singleFile ? new DataSourceLocalFile
{
Id = id, Name = folder.Name, Type = DataSourceType.LOCAL_FILE, FilePath = Path.Combine(root, "notes.md"),
EmbeddingId = EMBEDDING_ID, ConfidenceLevel = ConfidenceLevel.NONE, MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 100,
} : folder;
this.settings.ConfigurationData.DataSources.Add(this.source);
var paths = singleFile ? new[] { "notes.md" } : new[] { "notes.md", Path.Combine("nested", "report.md") };
foreach (var relative in paths)
await File.WriteAllTextAsync(Path.Combine(root, relative), "Already indexed report");
return paths;
}
private string MoveSource(string destinationName)
{
var current = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
var destination = Path.Combine(this.testRoot, destinationName);
if (current is DataSourceLocalDirectory folder)
Directory.Move(folder.Path, destination);
else
{
destination += ".md";
File.Move(((DataSourceLocalFile)current).FilePath, destination);
}
return destination;
}
private async Task SaveEditAsync(string newPath)
{
var edit = this.OpenEdit(newPath);
edit.Warning.Complete(DialogResult.Ok(true));
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
Assert.That(edit.Warning.ComponentType, Is.EqualTo(typeof(ConfirmDialog)));
Assert.That(edit.Warning.Title, Is.EqualTo("New Location for This Data Source"));
Assert.That(edit.Warning.Parameters!.Get<string>(nameof(ConfirmDialog.MarkdownBody)), Does.Contain("unchanged at the new location").And.Contain("removed from the index").And.Contain("files on disk stay as they are").And.Contain("costs money"));
});
this.source = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
}
private OpenedEdit OpenEdit(string newPath)
{
var storedSource = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
var dialog = new TestDataSourceDialog();
var warning = new TestDataSourceDialog();
var parent = new DataSourceManagement();
Property(parent, "SettingsManager", this.settings);
Property(parent, "MessageBus", MessageBus.INSTANCE);
Property(parent, "DataSourceEmbeddingService", this.service);
Property(parent, "DialogService", dialog.Service);
var parentTask = InvokeTask(parent, "EditDataSource", storedSource);
object component = storedSource is DataSourceLocalDirectory ? new DataSourceLocalDirectoryDialog() : new DataSourceLocalFileDialog();
Property(component, "SettingsManager", this.settings);
Property(component, "DialogService", warning.Service);
Property(component, "DataSourceEmbeddingService", this.service);
Property(component, "MudDialog", dialog.Instance);
Property(component, "IsEditing", true);
Property(component, "DataSource", storedSource);
SetField(component, "dataId", storedSource.Id);
SetField(component, "dataName", storedSource.Name);
SetField(component, "dataEmbeddingId", storedSource.EmbeddingId);
SetField(component, "dataConfidenceLevel", storedSource.ConfidenceLevel);
SetField(component, "dataMaxChunkTokenLength", storedSource.MaxChunkTokenLength);
SetField(component, "dataChunkOverlapTokenLength", storedSource.ChunkOverlapTokenLength);
SetField(component, storedSource is DataSourceLocalDirectory ? "dataPath" : "dataFilePath", newPath);
if (storedSource is DataSourceLocalDirectory folder)
{
SetField(component, "dataFilePatterns", folder.FilePatterns);
SetField(component, "dataExcludePatterns", folder.ExcludePatterns);
}
return new(component, dialog, warning, parentTask);
}
private Task RunNextAsync()
{
Assert.That(this.Queue.TryDequeue(out var request), Is.True, "Saving must queue a run.");
var task = InvokeTask(this.service, "RunDataSourceAsync", request!.DataSourceId, request.RefreshMode, this.lifetime.Token);
this.runningTasks.Add(task);
return task;
}
private async Task AssertOriginalChunksAsync()
{
var index = await this.databases.GetIndexStoreAsync();
var chunks = await index.SearchChunksAsync(this.source.Id, "indexed", 100, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(chunks.Select(chunk => chunk.ChunkId), Is.EquivalentTo(this.chunkIds));
Assert.That(chunks.All(chunk => chunk.ChunkText == "Already indexed report"), Is.True);
Assert.That(chunks.All(chunk => chunk.EmbeddedAtUtc < DateTimeOffset.UtcNow.AddDays(-9)), Is.True, "Updating the app or its paths must not recreate existing chunks.");
});
}
private static string LegacyFingerprint(FileInfo file)
{
return LegacyFingerprint(file.FullName, file.Name, file.LastWriteTimeUtc.Ticks, file.Length);
}
private static string LegacyFingerprint(string path, string name, long writeTicks, long size)
{
var parts = new[] { "file", path, name, writeTicks.ToString(), size.ToString() };
var serialized = string.Concat(parts.Select(part => $"{part.Length}:{part}|"));
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(serialized)));
}
// Keep reflection at the fixture boundary: exercise the production queue worker without
// starting the unrelated application startup loop, and install the isolated database clients.
private static T Field<T>(object instance, string name) => (T)instance.GetType().GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(instance)!;
private static void SetField(object instance, string name, object value) => instance.GetType().GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(instance, value);
private static void Property(object instance, string name, object value)
{
for (var type = instance.GetType(); type is not null; type = type.BaseType)
if (type.GetProperty(name, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly) is { } property)
{
property.SetValue(instance, value);
return;
}
throw new MissingMemberException(name);
}
private static Task InvokeTask(object instance, string name, params object[] arguments) =>
(Task)instance.GetType().GetMethod(name, BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(instance, arguments)!;
private sealed record OpenedEdit(object Component, TestDataSourceDialog Dialog, TestDataSourceDialog Warning, Task ParentTask);
}
@@ -0,0 +1,47 @@
using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.Services.Indexing;
namespace AIStudio.Tests.Tools.Databases;
public sealed class IndexStoreRelocationTests
{
[Test]
public async Task RelocationPreservesChunksAndTranscriptAndUpdatesSearchLocations()
{
await using var store = await TemporaryIndexStore.CreateAsync();
var sourceId = Guid.NewGuid().ToString();
var oldPath = Path.Combine(Path.GetTempPath(), "original", "notes.md");
var newPath = Path.Combine(Path.GetTempPath(), "moved", "renamed.md");
var parentId = IndexedDocumentIds.CreateParentId(sourceId, oldPath);
var chunkId = IndexedDocumentIds.CreateChunkId(sourceId, "old-fingerprint", 0);
var now = DateTimeOffset.UtcNow;
var token = CancellationToken.None;
await store.Client.UpsertDataSourceAsync(sourceId, "LOCAL_DIRECTORY", "provider", "signature", "old-hash", 3, token);
await store.Client.UpsertFileAsync(sourceId, new EmbeddingStateFile(parentId, oldPath, "notes.md", "notes.md", "md", "old-fingerprint", 100, now, now, now, 1) { TranscriptKey = "transcript" }, token);
await store.Client.UpsertChunksAsync(sourceId, [new EmbeddingStateChunk(chunkId, parentId, 2, 0, "report about relocation", now)], token);
// Repeating the metadata write after an interrupted relocation must keep the same chunks.
await store.Client.RelocateFileAsync(sourceId, parentId, newPath, "renamed.md", "new-fingerprint", token);
await store.Client.RelocateFileAsync(sourceId, parentId, newPath, "renamed.md", "new-fingerprint", token);
var manifest = await store.Client.GetManifestAsync(sourceId, token);
var hits = await store.Client.SearchChunksAsync(sourceId, "relocation", 10, token);
Assert.Multiple(() =>
{
Assert.That(manifest.Files.Keys, Is.EquivalentTo(new[] { newPath }));
Assert.That(manifest.Files[newPath].Fingerprint, Is.EqualTo("new-fingerprint"));
Assert.That(manifest.Files[newPath].ParentFileId, Is.EqualTo(parentId));
Assert.That(manifest.Files[newPath].RelativePath, Is.EqualTo("renamed.md"));
Assert.That(manifest.Files[newPath].TranscriptKey, Is.EqualTo("transcript"));
Assert.That(hits, Has.Count.EqualTo(1));
Assert.That(hits[0].ChunkId, Is.EqualTo(chunkId));
Assert.That(hits[0].AbsolutePath, Is.EqualTo(newPath));
Assert.That(hits[0].FileName, Is.EqualTo("renamed.md"));
Assert.That(hits[0].PageNumber, Is.EqualTo(2));
Assert.That(hits[0].ChunkText, Is.EqualTo("report about relocation"));
});
await store.Client.DeleteFileAsync(sourceId, newPath, token);
Assert.That(await store.Client.SearchChunksAsync(sourceId, "relocation", 10, token), Is.Empty, "Ordinary removal must still find a relocated document's chunks.");
}
}
@@ -35,6 +35,15 @@ public sealed class FileSourceIndexerWatcherTests
Is.False);
}
[Test]
public void SingleFileWatcherIgnoresAnotherCaseDistinctFileOnUnix()
{
var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(Path.GetTempPath(), false, TargetFileName: "Report.md");
Assert.That(FileSourceIndexer.IsRelevantWatcherPath(configuration, Path.Combine(Path.GetTempPath(), "report.md"), WatcherChangeTypes.Changed),
Is.EqualTo(OperatingSystem.IsWindows()));
}
[Test]
public void SingleFileWatcherRecognizesARenameAwayFromTheConfiguredFile()
{
@@ -262,6 +271,15 @@ public sealed class FileSourceIndexerWatcherTests
Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.False);
}
[Test]
public void RelocatingToADirectoryWithDifferentCasingUsesTheLocalPathRules()
{
var before = DefaultConfiguration(Path.Combine(Path.GetTempPath(), "Documents"));
var after = before with { RootPath = Path.Combine(Path.GetTempPath(), "documents") };
Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.EqualTo(OperatingSystem.IsWindows()), "On Unix these can be different directories; their watchers must not be treated as interchangeable.");
}
[Test]
public async Task DirectoryWatcherIgnoresTheBatchTranscriptWhenTheMediaIsSelected()
{
@@ -0,0 +1,217 @@
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Services;
using AIStudio.Tools.Services.Indexing;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.Tools;
public sealed class FileSourceRelocationTests
{
private string testRoot = string.Empty;
[SetUp]
public void SetUp()
{
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-relocation-{Guid.NewGuid():N}");
Directory.CreateDirectory(this.testRoot);
}
[TearDown]
public void TearDown() => Directory.Delete(this.testRoot, true);
[Test]
public void AMovedFolderFindsAnUnchangedNestedDocument()
{
var oldRoot = Path.Combine(this.testRoot, "original");
var oldFile = Path.Combine(oldRoot, "nested", "notes.md");
Directory.CreateDirectory(Path.GetDirectoryName(oldFile)!);
File.WriteAllText(oldFile, "Some indexed text");
var record = Record(oldFile, Path.Combine("nested", "notes.md"));
var newRoot = Path.Combine(this.testRoot, "moved");
Directory.Move(oldRoot, newRoot);
var found = FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = newRoot }, oldFile, record);
Assert.That(found?.FullName, Is.EqualTo(Path.Combine(newRoot, "nested", "notes.md")));
}
[TestCase("size")]
[TestCase("modification time")]
[TestCase("missing file")]
public void AChangedOrMissingDocumentCannotReuseItsEmbeddings(string change)
{
var file = Path.Combine(this.testRoot, "notes.md");
File.WriteAllText(file, "Some indexed text");
var record = Record(file, "notes.md");
switch (change)
{
case "size":
File.AppendAllText(file, " with more text");
File.SetLastWriteTimeUtc(file, record.LastWriteUtc.UtcDateTime);
break;
case "modification time":
File.SetLastWriteTimeUtc(file, record.LastWriteUtc.UtcDateTime.AddMinutes(1));
break;
case "missing file":
File.Delete(file);
break;
}
// The old place lies outside the selected folder, so only the change can prevent the reuse:
var oldPath = Path.Combine(Path.GetDirectoryName(this.testRoot)!, "moved-away", "notes.md");
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, oldPath, record), Is.Null);
}
[Test]
public void AStoredPathBelowTheSelectedFolderIsNotMappedToACopy()
{
// The folder grows to its parent, which holds a copy with the same size and time at the
// place the old relative path names. The original is still there and stays the document.
var original = Path.Combine(this.testRoot, "project", "notes.md");
Directory.CreateDirectory(Path.GetDirectoryName(original)!);
File.WriteAllText(original, "Some indexed text");
var copy = Path.Combine(this.testRoot, "notes.md");
File.Copy(original, copy);
File.SetLastWriteTimeUtc(copy, File.GetLastWriteTimeUtc(original));
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, original, Record(original, "notes.md")), Is.Null);
}
[Test]
public void ARenamedSingleFileCanReuseItsEmbeddings()
{
var oldFile = Path.Combine(this.testRoot, "notes.md");
File.WriteAllText(oldFile, "Some indexed text");
var record = Record(oldFile, "notes.md");
var newFile = Path.Combine(this.testRoot, "renamed.md");
File.Move(oldFile, newFile);
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalFile { FilePath = newFile }, oldFile, record)?.FullName, Is.EqualTo(newFile));
}
[Test]
public void ARelativePathCannotEscapeTheConfiguredFolder()
{
var outside = Path.Combine(this.testRoot, "notes.md");
File.WriteAllText(outside, "Some indexed text");
var record = Record(outside, Path.Combine("..", "notes.md"));
var folder = Path.Combine(this.testRoot, "selected");
Directory.CreateDirectory(folder);
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = folder }, Path.Combine(this.testRoot, "old", "notes.md"), record), Is.Null);
}
private static EmbeddedFileRecord Record(string path, string relativePath)
{
var file = new FileInfo(path);
return new("old-fingerprint", file.Length, new DateTimeOffset(file.LastWriteTimeUtc), DateTimeOffset.UtcNow, 1, RelativePath: relativePath, ParentFileId: Guid.NewGuid().ToString());
}
[TestCase(@"E:\Documents\nested\notes.md", @"nested\notes.md")]
[TestCase(@"\\server\share\Documents\nested\notes.md", @"nested\notes.md")]
[TestCase(@"\\?\E:\Documents\nested\notes.md", @"nested\notes.md")]
[TestCase("E:/Documents/nested/notes.md", "nested/notes.md")]
[TestCase("/home/alice/Documents/nested/notes.md", "nested/notes.md")]
[TestCase("/Volumes/External drive/Documents/nested/notes.md", "nested/notes.md")]
public void ABackupFromAnotherPathSyntaxFindsANestedFile(string oldPath, string relativePath)
{
var localFile = Path.Combine(this.testRoot, "nested", "notes.md");
Directory.CreateDirectory(Path.GetDirectoryName(localFile)!);
File.WriteAllText(localFile, "Some indexed text");
var found = FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, oldPath, Record(localFile, relativePath));
Assert.That(found?.FullName, Is.EqualTo(localFile));
}
[TestCase(@"C:\a.folder\README")]
[TestCase(@"\\server\a.folder\README")]
[TestCase("/media/a.folder/README")]
public void AForeignDirectoryNameDoesNotBecomeTheExtensionOfAnExtensionlessFile(string oldPath)
{
var localFile = Path.Combine(this.testRoot, "README");
File.WriteAllText(localFile, "Some indexed text");
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalFile { FilePath = localFile }, oldPath, Record(localFile, "README"))?.FullName, Is.EqualTo(localFile));
}
[TestCase(@"..\notes.md")]
[TestCase("../notes.md")]
[TestCase(@"nested\..\..\notes.md")]
[TestCase(@"C:\notes.md")]
[TestCase(@"C:notes.md")]
[TestCase(@"\\server\share\notes.md")]
[TestCase("/etc/notes.md")]
public void AStoredWindowsPathCannotEscapeTheNewRoot(string relativePath)
{
Assert.That(RelocatedFilePaths.Resolve(this.testRoot, @"E:\Documents\notes.md", relativePath), Is.Null);
}
[Test]
public void AnExternalDriveRootIsAValidRelocationDestination()
{
// No file needs to be written at the drive root: this checks resolving the destination.
var root = Path.GetPathRoot(this.testRoot)!;
Assert.That(RelocatedFilePaths.Resolve(root, @"E:\notes.md", "notes.md"), Is.EqualTo(Path.Combine(root, "notes.md")));
}
[Test]
public void AUnixBackslashInAFileNameIsNotReinterpretedAsADirectory()
{
if (OperatingSystem.IsWindows())
{
Assert.That(RelocatedFilePaths.Resolve(this.testRoot, @"/home/alice/notes\draft.md", @"notes\draft.md"), Is.Null, "A Windows filename cannot contain a backslash from a Unix filename.");
return;
}
var localFile = Path.Combine(this.testRoot, @"notes\draft.md");
File.WriteAllText(localFile, "Some indexed text");
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, @"/home/alice/notes\draft.md", Record(localFile, @"notes\draft.md"))?.FullName, Is.EqualTo(localFile));
}
[Test]
public void AnOldForeignRootIsNotResolvedRelativeToTheWorkingDirectory()
{
var foreignRoot = OperatingSystem.IsWindows() ? "/home/alice/Documents" : @"E:\Documents";
Assert.That(RelocatedFilePaths.Resolve(foreignRoot, @"E:\Documents\notes.md", "notes.md"), Is.Null);
}
[Test]
public void ReducedTimestampPrecisionOnAnotherDriveDoesNotSilentlyReuseOldContent()
{
var localFile = Path.Combine(this.testRoot, "notes.md");
File.WriteAllText(localFile, "Some indexed text");
var record = Record(localFile, "notes.md");
// The old filesystem stored a finer timestamp than the copied file now reports. Without
// a stored content hash, equal size alone is not enough to prove that it is unchanged.
record = record with { LastWriteUtc = record.LastWriteUtc.AddTicks(1) };
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, @"E:\Documents\notes.md", record), Is.Null);
}
[Test]
public void ANewFileAtAnOldLocationCannotOverwriteTheRelocatedDocument()
{
var source = new DataSourceLocalDirectory { Id = Guid.NewGuid().ToString(), Path = this.testRoot };
var originalPath = Path.Combine(this.testRoot, "nested", "notes.md");
var relocatedPath = Path.Combine(this.testRoot, "notes.md");
var originalId = IndexedDocumentIds.CreateParentId(source.Id, originalPath);
var manifest = new DataSourceEmbeddingManifest();
manifest.Files[relocatedPath] = new("fingerprint", 100, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 1, RelativePath: "notes.md", ParentFileId: originalId);
// Only identity assignment is exercised; none of the run's services are called.
var context = new IndexedRunContext(source, null!, null!, null!, null!, manifest,
new RunShare(RunShare.DEFAULT_DURATION), new IndexingPass(new DataSourceRunSignature("signature", "selection")), null!, null!, _ => { }, NullLogger.Instance);
var newId = context.GetFileParentId(originalPath);
Assert.Multiple(() =>
{
Assert.That(context.GetFileParentId(relocatedPath), Is.EqualTo(originalId));
Assert.That(newId, Is.Not.EqualTo(originalId));
Assert.That(context.GetFileParentId(originalPath), Is.EqualTo(newId), "A media file may ask for its identity once before and once after transcription.");
});
}
}
@@ -31,4 +31,21 @@ public sealed class IndexedDocumentIdsTests
{
Assert.That(IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, "/tmp/test-data/report.pdf"), Is.EqualTo("d06c4351-2e57-2941-aa4c-d7a814bfa418"));
}
[Test]
public void ADocumentStoredUnderTheIdOfItsKeyKeepsItsChunkIds()
{
const string KEY = "/tmp/test-data/report.pdf";
var source = IndexedDocumentIds.CreateChunkIdSource(DATA_SOURCE_ID, KEY, IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, KEY), "3F9A0C");
Assert.That(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, source, 0), Is.EqualTo("7c9d2822-f9d1-a94e-8d83-30cea4f0856f"), "Files and mails which never moved must keep the ids they are stored under.");
}
[Test]
public void TheChunkIdOfAMovedDocumentIsPinned()
{
var source = IndexedDocumentIds.CreateChunkIdSource(DATA_SOURCE_ID, "/tmp/moved/report.pdf", "0b5c2f7e-3d41-4a8e-9f60-2c7d1e8a4b93", "3F9A0C");
Assert.That(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, source, 0), Is.EqualTo("aa597d9b-ede8-5449-80d0-f209cde0ecc0"), "A file at the old place with the same fingerprint must not get the chunk ids of the moved one.");
}
}
+21 -2
View File
@@ -112,7 +112,7 @@ public sealed class OutdatedRunTests
}
[Test]
public void ASingleFileHasNoSelection()
public void RelocatingASingleFileStopsTheOldRunWithoutInvalidatingItsEmbeddings()
{
var dataSource = new DataSourceLocalFile
{
@@ -125,7 +125,26 @@ public sealed class OutdatedRunTests
FilePath = "/tmp/test-data/report.pdf",
};
Assert.That(DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM).SelectionSignature, Is.Empty, "The file is the one document of its data source, so there is nothing to choose among.");
var before = DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
var after = DataSourceEmbeddingService.GetRunSignature(dataSource with { FilePath = "/tmp/moved/report.pdf" }, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
Assert.Multiple(() =>
{
Assert.That(DataSourceEmbeddingService.IsRunOutdated(before, after), Is.True);
Assert.That(after.EmbeddingSignature, Is.EqualTo(before.EmbeddingSignature));
});
}
[Test]
public void RelocatingAFolderStopsTheOldRunWithoutInvalidatingItsEmbeddings()
{
var dataSource = DataSource(ConfidenceLevel.LOW);
var before = DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
var after = DataSourceEmbeddingService.GetRunSignature(dataSource with { Path = "/tmp/moved" }, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
Assert.Multiple(() =>
{
Assert.That(DataSourceEmbeddingService.IsRunOutdated(before, after), Is.True);
Assert.That(after.EmbeddingSignature, Is.EqualTo(before.EmbeddingSignature));
});
}
[Test]
+81
View File
@@ -0,0 +1,81 @@
using AIStudio.Tools;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Checks where a file dialog opens for a file or a folder which was moved or renamed.
/// </summary>
/// <remarks>
/// The native dialog cannot open a place which no longer exists and falls back to wherever it was
/// used last, which may be anywhere. The nearest folder which still exists is where the file or the
/// folder was moved from, so its new place is often close by.
/// </remarks>
[TestFixture]
public sealed class PathToolsTests
{
private string testRoot = string.Empty;
[SetUp]
public void CreateTestRoot()
{
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-path-tools-{Guid.NewGuid():N}");
Directory.CreateDirectory(this.testRoot);
}
[TearDown]
public void DeleteTestRoot()
{
if (Directory.Exists(this.testRoot))
Directory.Delete(this.testRoot, true);
}
[Test]
public void AnExistingFolderOpensAsItIs()
{
var folder = Directory.CreateDirectory(Path.Combine(this.testRoot, "documents")).FullName;
Assert.That(PathTools.FindNearestExistingPath(folder), Is.EqualTo(folder));
}
[Test]
public void AnExistingFileOpensAsItIs()
{
var file = Path.Combine(this.testRoot, "report.pdf");
File.WriteAllText(file, "report");
Assert.That(PathTools.FindNearestExistingPath(file), Is.EqualTo(file));
}
[Test]
public void AMovedFolderOpensAtItsNearestFolderWhichStillExists()
{
var parent = Directory.CreateDirectory(Path.Combine(this.testRoot, "projects")).FullName;
var movedFolder = Path.Combine(parent, "2026", "documents");
Assert.That(PathTools.FindNearestExistingPath(movedFolder), Is.EqualTo(parent), "Every parent folder counts, not only the immediate one.");
}
[Test]
public void AMovedFileOpensAtItsFolder()
{
var file = Path.Combine(this.testRoot, "report.pdf");
Assert.That(PathTools.FindNearestExistingPath(file), Is.EqualTo(this.testRoot));
}
[TestCase("")]
[TestCase(" ")]
[TestCase("documents/report.pdf")]
public void APathWithoutAPlaceOpensWhereverTheDialogWants(string path)
{
Assert.That(PathTools.FindNearestExistingPath(path), Is.Null);
}
[Test]
public void APathOfAnotherOperatingSystemOpensWhereverTheDialogWants()
{
var path = OperatingSystem.IsWindows() ? "/Users/somebody/documents" : @"C:\Users\somebody\documents";
Assert.That(PathTools.FindNearestExistingPath(path), Is.Null, "A backup from another computer must not make the dialog open at a folder which happens to exist here.");
}
}
+93
View File
@@ -0,0 +1,93 @@
using AIStudio.Tools.Databases.VectorStore;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Keeps the identities and locations of existing vectors. New embeddings are forbidden: these
/// tests reconnect previously indexed files and must never contact an embedding provider.
/// </summary>
internal sealed class RelocationVectorStore() : VectorStoreClient("Relocation test vectors", string.Empty)
{
public Dictionary<string, string> Locations { get; } = new(StringComparer.Ordinal);
public List<string> RelocatedPaths { get; } = [];
public TaskCompletionSource<object?> RelocationStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public bool BlockNextRelocation { get; set; }
public int CanceledRelocations { get; private set; }
public int DeletedStores { get; private set; }
public int DeletedDocuments { get; private set; }
public int InsertedBatches { get; private set; }
public override async Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token)
{
token.ThrowIfCancellationRequested();
Assert.That(this.Locations.ContainsKey(parentFileId), Is.True, "Relocation must address an existing vector identity.");
this.Locations[parentFileId] = absolutePath;
this.RelocatedPaths.Add(absolutePath);
if (!this.BlockNextRelocation)
return;
this.BlockNextRelocation = false;
this.RelocationStarted.TrySetResult(null);
try
{
await Task.Delay(Timeout.InfiniteTimeSpan, token);
}
catch (OperationCanceledException)
{
this.CanceledRelocations++;
throw;
}
}
public override Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) => Task.FromResult(new VectorStoreEnsureResult(false));
public override Task<bool> VectorStoreExists(string storeName, CancellationToken token) => Task.FromResult(true);
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token)
{
this.InsertedBatches++;
throw new InvalidOperationException("Existing unchanged files must not be embedded again.");
}
public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token)
{
this.DeletedDocuments++;
foreach (var id in this.Locations.Where(entry => entry.Key == parentFileId || entry.Value == filePath).Select(entry => entry.Key).ToArray())
this.Locations.Remove(id);
return Task.CompletedTask;
}
public override Task DeleteVectorStore(string storeName, CancellationToken token)
{
this.DeletedStores++;
this.Locations.Clear();
return Task.CompletedTask;
}
public override Task OptimizeVectorStore(string storeName, CancellationToken token) => Task.CompletedTask;
public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, CancellationToken token) => throw new NotSupportedException();
public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, CancellationToken token) => throw new NotSupportedException();
public override Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token) => throw new NotSupportedException();
public override Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) => throw new NotSupportedException();
public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo()
{
await Task.CompletedTask;
yield break;
}
public override void Dispose()
{
}
}
+76
View File
@@ -0,0 +1,76 @@
using System.Reflection;
using MudBlazor;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Controls a dialog result without rendering a Blazor circuit. The real dialog save handler and
/// the real parent edit handler still run, with the same parameters and result as in the UI.
/// </summary>
internal sealed class TestDataSourceDialog
{
private readonly TaskCompletionSource<DialogResult?> result = new(TaskCreationOptions.RunContinuationsAsynchronously);
public IDialogService Service { get; }
public IMudDialogInstance Instance { get; }
public DialogParameters? Parameters { get; private set; }
public Type? ComponentType { get; private set; }
public string? Title { get; private set; }
public int ShowCount { get; private set; }
public TestDataSourceDialog()
{
var reference = Create<IDialogReference>((method, _) => method.Name == "get_Result"
? this.result.Task
: throw new NotSupportedException(method.Name));
this.Service = Create<IDialogService>((method, arguments) =>
{
if (method.Name != "ShowAsync")
throw new NotSupportedException(method.Name);
ArgumentNullException.ThrowIfNull(arguments);
this.ShowCount++;
this.ComponentType = method.GetGenericArguments().Single();
this.Title = arguments.OfType<string>().FirstOrDefault();
this.Parameters = arguments.OfType<DialogParameters>().Single();
return Task.FromResult(reference);
});
this.Instance = Create<IMudDialogInstance>((method, arguments) =>
{
switch (method.Name)
{
case "Close":
this.Complete(arguments!.OfType<DialogResult>().Single());
return null;
case "Cancel":
this.Complete(DialogResult.Cancel());
return null;
default:
throw new NotSupportedException(method.Name);
}
});
}
public void Complete(DialogResult dialogResult) => this.result.TrySetResult(dialogResult);
private static T Create<T>(Func<MethodInfo, object?[]?, object?> invoke) where T : class
{
var proxy = DispatchProxy.Create<T, DialogCallProxy>();
((DialogCallProxy)(object)proxy).InvokeCall = invoke;
return proxy;
}
}
// DispatchProxy generates a subclass, so this type must be public and cannot be sealed.
public class DialogCallProxy : DispatchProxy
{
public Func<MethodInfo, object?[]?, object?> InvokeCall { get; set; } = null!;
protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) => this.InvokeCall(targetMethod!, args);
}
@@ -92,7 +92,7 @@ public sealed class SemanticSearchToolUnreachableTests
}
[Test]
public void TheResultNamesAnUnreachableDataSourceWithoutItsID()
public void TheResultNamesAnUnreachableDataSourceWithoutItsId()
{
var result = SemanticSearchTool.DescribeUnreachableResult(INTRANET);
+60 -3
View File
@@ -62,6 +62,49 @@ public sealed class UnreachableDataSourceTests
AssertTheIndexWasKept(outcome, statuses, manifest, 2, folder);
}
[Test]
public async Task AMissingRelocationTargetKeepsTheOriginalIndex()
{
var folder = Path.Combine(this.testRoot, "moved-but-unavailable");
var (outcome, statuses, manifest) = await RunAsync(Folder(folder), Path.Combine(this.testRoot, "original", "report.pdf"));
AssertTheIndexWasKept(outcome, statuses, manifest, 1, folder);
}
[TestCase(false)]
[TestCase(true)]
public async Task AReadableEmptyMountDirectoryKeepsItsIndex(bool emptySubfolder)
{
var folder = Path.Combine(this.testRoot, "mounted-drive");
Directory.CreateDirectory(emptySubfolder ? Path.Combine(folder, "documents") : folder);
var unfinishedFile = Path.Combine(folder, "unfinished.pdf");
var (outcome, statuses, manifest) = await RunAsync(Folder(folder), [Path.Combine(folder, "report.pdf")], [unfinishedFile]);
AssertTheIndexWasKept(outcome, statuses, manifest, 1, folder);
Assert.Multiple(() =>
{
Assert.That(manifest.UnfinishedFiles, Does.Contain(unfinishedFile), "An unmounted drive must not discard interrupted work either.");
Assert.That(statuses[^1].LastError, Does.Contain("appears empty").And.Contain("remove this data source"));
});
}
[Test]
public async Task ANewEmptyFolderCanFinishWithoutAnIndex()
{
var folder = Path.Combine(this.testRoot, "new-empty-folder");
Directory.CreateDirectory(folder);
var (outcome, statuses, manifest) = await RunAsync(Folder(folder));
Assert.Multiple(() =>
{
Assert.That(outcome, Is.EqualTo(IndexedRunOutcome.DONE));
Assert.That(statuses[^1].State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(manifest.Files, Is.Empty);
});
}
[Test]
public async Task AMissingFileKeepsItsIndex()
{
@@ -130,6 +173,15 @@ public sealed class UnreachableDataSourceTests
Assert.That(goneFiles, Is.EquivalentTo([deleted, besideSubfolder]), "A folder whose name only begins like the one which could not be read is no part of it.");
}
[Test]
public void AnUnreadableDriveRootKeepsItsIndexedFiles()
{
var root = Path.GetPathRoot(Path.GetFullPath(Path.GetTempPath()))!;
var file = Path.Combine(root, "documents", "notes.md");
Assert.That(FileSourceIndexer.GetGoneFiles([file], [], [root]), Is.Empty, "A drive root already ends in its separator; adding another would fail to protect its files.");
}
private static DataSourceLocalDirectory Folder(string path) => new()
{
Num = 1,
@@ -152,6 +204,7 @@ public sealed class UnreachableDataSourceTests
Assert.That(statuses[^1].State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
Assert.That(statuses[^1].IndexedDocuments, Is.EqualTo(indexedFiles), "The index still holds these files, and the page has to say so.");
Assert.That(statuses[^1].LastError, Does.Contain(unreachablePath));
Assert.That(statuses[^1].Attention, Is.EqualTo(DataSourceAttention.LOCATION_UNREACHABLE), "The embeddings page offers to edit the data source, e.g. to select the new location of a moved folder.");
});
}
@@ -181,12 +234,16 @@ public sealed class UnreachableDataSourceTests
var statuses = new List<DataSourceEmbeddingStatus>();
var manifest = new DataSourceEmbeddingManifest { SourceHash = STORED_SOURCE_HASH };
foreach (var file in indexedFiles)
manifest.Files[file] = new EmbeddedFileRecord("FINGERPRINT", 1024, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 4);
manifest.Files[file] = new EmbeddedFileRecord("FINGERPRINT", 1024, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 4,
RelativePath: Path.GetFileName(file), ParentFileId: IndexedDocumentIds.CreateParentId(dataSource.Id, file));
manifest.UnfinishedFiles.UnionWith(unfinishedFiles);
var scannedFile = Path.Combine(Path.GetDirectoryName(indexedFiles[0])!, "scan.pdf");
manifest.PermanentFailures[scannedFile] = new PermanentIndexingFailureRecord("FINGERPRINT", FileExtractionErrorCode.NO_TEXT_EXTRACTED, "No text could be read.", DateTimeOffset.UnixEpoch);
if (indexedFiles.Length > 0)
{
var scannedFile = Path.Combine(Path.GetDirectoryName(indexedFiles[0])!, "scan.pdf");
manifest.PermanentFailures[scannedFile] = new PermanentIndexingFailureRecord("FINGERPRINT", FileExtractionErrorCode.NO_TEXT_EXTRACTED, "No text could be read.", DateTimeOffset.UnixEpoch);
}
var embeddingProvider = new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"));
var settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);