mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-04 20:27:21 +00:00
Allowed moving local data sources without embedding them again (#1091)
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
1 parent
f6e48e95f3
commit
5d6e48ef93
49 files changed
+2280
-297
No files matched your search
@@ -0,0 +1,752 @@
|
||||
using System.Reflection;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Components;
|
||||
using AIStudio.Dialogs;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tests.Tools.Databases;
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.Databases;
|
||||
using AIStudio.Tools.Databases.IndexStore;
|
||||
using AIStudio.Tools.DataBackup;
|
||||
using AIStudio.Tools.Security;
|
||||
using AIStudio.Tools.Services;
|
||||
using AIStudio.Tools.Services.Indexing;
|
||||
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
using MudBlazor;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Runs the actual edit, queue, cancellation, manifest preparation, and file indexing code against
|
||||
/// a migrated SQLite store. Only the vector store and dialog rendering are replaced.
|
||||
/// </summary>
|
||||
[NonParallelizable]
|
||||
public sealed class DataSourceRelocationWorkflowTests
|
||||
{
|
||||
private const string EMBEDDING_ID = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01";
|
||||
|
||||
// Frozen from the format before relocation support. Do not generate this with the current
|
||||
// signature builder: doing that would hide an accidental rebuild of every existing index.
|
||||
private const string LEGACY_SIGNATURE = "2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100";
|
||||
|
||||
private static readonly TimeSpan TEST_TIMEOUT = TimeSpan.FromSeconds(10);
|
||||
|
||||
private string testRoot = string.Empty;
|
||||
private TemporaryIndexStore store = null!;
|
||||
private SettingsManager settings = null!;
|
||||
private DatabaseClientProvider databases = null!;
|
||||
private DataSourceEmbeddingService service = null!;
|
||||
private RelocationVectorStore vectors = null!;
|
||||
private IInternalDataSource source = null!;
|
||||
private readonly List<string> chunkIds = [];
|
||||
private readonly List<Task> runningTasks = [];
|
||||
private CancellationTokenSource lifetime = null!;
|
||||
|
||||
private string OriginalFolder => Path.Combine(this.testRoot, "original");
|
||||
|
||||
[SetUp]
|
||||
public async Task SetUp()
|
||||
{
|
||||
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-relocation-workflow-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(this.testRoot);
|
||||
this.store = await TemporaryIndexStore.CreateAsync();
|
||||
this.settings = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
|
||||
this.settings.ConfigurationData.App.EnabledPreviewFeatures.Add(PreviewFeatures.PRE_RAG_2024);
|
||||
this.settings.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh = false;
|
||||
this.settings.ConfigurationData.EmbeddingProviders.Add(new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small")));
|
||||
this.vectors = new RelocationVectorStore();
|
||||
this.databases = new DatabaseClientProvider(null!, NullLoggerFactory.Instance);
|
||||
var clients = Field<Dictionary<DatabaseRole, DatabaseClient>>(this.databases, "clients");
|
||||
clients.Add(DatabaseRole.INDEX_STORE, this.store.Client);
|
||||
clients.Add(DatabaseRole.VECTOR_STORE, this.vectors);
|
||||
var guard = new PromptInjectionGuardService(null!, this.settings, NullLogger<PromptInjectionGuardService>.Instance, NullLoggerFactory.Instance);
|
||||
this.service = new DataSourceEmbeddingService(this.settings, null!, this.databases, guard, null!, NullLogger<DataSourceEmbeddingService>.Instance);
|
||||
this.lifetime = new CancellationTokenSource();
|
||||
this.chunkIds.Clear();
|
||||
this.runningTasks.Clear();
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public async Task TearDown()
|
||||
{
|
||||
this.lifetime.Cancel();
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(this.runningTasks).WaitAsync(TEST_TIMEOUT);
|
||||
}
|
||||
catch (OperationCanceledException) when (this.lifetime.IsCancellationRequested)
|
||||
{
|
||||
// A failed test may leave its deliberately blocked run for teardown to stop.
|
||||
}
|
||||
finally
|
||||
{
|
||||
this.service.Dispose();
|
||||
this.databases.Dispose();
|
||||
await this.store.DisposeAsync();
|
||||
this.lifetime.Dispose();
|
||||
Directory.Delete(this.testRoot, true);
|
||||
}
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task AnExistingIndexSurvivesTheUpgradeWithoutReembedding(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty, "An upgrade is not a relocation.");
|
||||
Assert.That(this.vectors.DeletedStores, Is.Zero);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task EditingAndCancelingDoNothingAndSavingOnlyRelocatesUnchangedFiles(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var newPath = this.MoveSource("moved");
|
||||
var original = this.source;
|
||||
var edit = this.OpenEdit(newPath);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False);
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
|
||||
Assert.That(edit.Warning.ShowCount, Is.Zero, "Typing a path must not open the save confirmation.");
|
||||
});
|
||||
|
||||
edit.Dialog.Instance.Cancel();
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False);
|
||||
|
||||
await this.SaveEditAsync(newPath);
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty, "Saving queues the operation; the dialog must not write to either index.");
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
|
||||
Assert.That(this.vectors.DeletedStores, Is.Zero);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
});
|
||||
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task SavingAnUnchangedPathDoesNotWarnOrRelocate(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var currentPath = this.source is DataSourceLocalDirectory folder ? folder.Path : ((DataSourceLocalFile)this.source).FilePath;
|
||||
var edit = this.OpenEdit(currentPath);
|
||||
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.Zero);
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
|
||||
Assert.That(this.vectors.DeletedStores, Is.Zero);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task SavingWaitsForThePathWarningBeforeChangingConfigurationOrQueueingWork(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var original = this.source;
|
||||
var edit = this.OpenEdit(this.MoveSource("moved"));
|
||||
var save = InvokeTask(edit.Component, "StoreValidatedDataSourceAsync");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
|
||||
Assert.That(save.IsCompleted, Is.False);
|
||||
Assert.That(edit.ParentTask.IsCompleted, Is.False);
|
||||
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False);
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
|
||||
});
|
||||
|
||||
edit.Warning.Complete(DialogResult.Ok(true));
|
||||
await save.WaitAsync(TEST_TIMEOUT);
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task CancelingThePathWarningKeepsTheConfiguredPathAndQueuesNothing(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var original = this.source;
|
||||
var edit = this.OpenEdit(this.MoveSource("moved"));
|
||||
edit.Warning.Complete(DialogResult.Cancel());
|
||||
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
|
||||
Assert.That(edit.ParentTask.IsCompleted, Is.False, "Rejecting the warning leaves the edit dialog open.");
|
||||
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False);
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
|
||||
});
|
||||
edit.Dialog.Instance.Cancel();
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task ANewLocationOfADataSourceWithoutAnIndexIsSavedWithoutAQuestion(bool singleFile)
|
||||
{
|
||||
await this.AddSourceAsync(singleFile);
|
||||
var newPath = this.MoveSource("moved");
|
||||
var edit = this.OpenEdit(newPath);
|
||||
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
var saved = this.settings.ConfigurationData.DataSources.Single();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.Zero, "Nothing was prepared for the data source, so nothing can be lost.");
|
||||
Assert.That(saved is DataSourceLocalDirectory folder ? folder.Path : ((DataSourceLocalFile)saved).FilePath, Is.EqualTo(newPath));
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task ANewLocationTogetherWithANewEmbeddingAsksOnlyOnce(bool singleFile)
|
||||
{
|
||||
const string OTHER_EMBEDDING_ID = "c1b5d5e3-2a4f-4b1b-9d0e-6b7c8d9e0f12";
|
||||
this.settings.ConfigurationData.EmbeddingProviders.Add(new EmbeddingProvider(2, OTHER_EMBEDDING_ID, "Other embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-large", "text-embedding-3-large")));
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var edit = this.OpenEdit(this.MoveSource("moved"));
|
||||
SetField(edit.Component, "dataEmbeddingId", OTHER_EMBEDDING_ID);
|
||||
edit.Warning.Complete(DialogResult.Ok(true));
|
||||
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1), "Everything is prepared again anyway, so the location needs no question of its own.");
|
||||
Assert.That(edit.Warning.Title, Is.EqualTo("Documents Will Be Prepared Again"));
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task ASecondSavedPathCancelsTheFirstRelocationAndRunsWithTheLatestPath(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
await this.SaveEditAsync(this.MoveSource("first"));
|
||||
this.vectors.BlockNextRelocation = true;
|
||||
var firstRun = this.RunNextAsync();
|
||||
await this.vectors.RelocationStarted.Task.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
var secondPath = this.MoveSource("second");
|
||||
await this.SaveEditAsync(secondPath);
|
||||
await firstRun.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
Assert.That(this.vectors.CanceledRelocations, Is.EqualTo(1), "The first run must be canceled without finishing its remaining documents.");
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(1));
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.RelocatedPaths.Skip(1).ToArray(), Has.Length.EqualTo(this.chunkIds.Count));
|
||||
Assert.That(this.vectors.Locations.Values.All(path => singleFile ? path == secondPath : path.StartsWith(secondPath + Path.DirectorySeparatorChar, StringComparison.Ordinal)), Is.True);
|
||||
Assert.That(this.vectors.DeletedStores, Is.Zero);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False, "The obsolete destination must not leave another run behind.");
|
||||
});
|
||||
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task DeletingTheSourceCancelsRelocationBeforeRemovingTheStoredIndex(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
await this.SaveEditAsync(this.MoveSource("moved"));
|
||||
this.vectors.BlockNextRelocation = true;
|
||||
var run = this.RunNextAsync();
|
||||
await this.vectors.RelocationStarted.Task.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
// A second save also leaves a pending run; deletion has to discard that one as well.
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
this.settings.ConfigurationData.DataSources.Clear();
|
||||
await this.service.RemoveDataSourceAsync(this.source).WaitAsync(TEST_TIMEOUT);
|
||||
await run.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.vectors.CanceledRelocations, Is.EqualTo(1));
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(1));
|
||||
Assert.That(this.vectors.DeletedStores, Is.EqualTo(1));
|
||||
Assert.That(this.vectors.Locations, Is.Empty);
|
||||
Assert.That(manifest.Files, Is.Empty);
|
||||
Assert.That(this.service.GetStatuses(), Is.Empty);
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
private DataSourceRunQueue Queue => Field<DataSourceRunQueue>(this.service, "runQueue");
|
||||
|
||||
[TestCase(@"E:\Documents", '\\', false)]
|
||||
[TestCase(@"\\server\share\Documents", '\\', false)]
|
||||
[TestCase("/home/alice/Documents", '/', false)]
|
||||
[TestCase(@"E:\Documents", '\\', true)]
|
||||
[TestCase(@"\\server\share\Documents", '\\', true)]
|
||||
[TestCase("/Volumes/External drive/Documents", '/', true)]
|
||||
public async Task ABackupCanReconnectFilesFromAnotherOSWithoutReembedding(string oldRoot, char separator, bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var originalFolder = Path.Combine(this.testRoot, "original");
|
||||
await this.StoreForeignPathsAsync(oldRoot, separator);
|
||||
var restored = await this.BackupAndRestoreIndexAsync();
|
||||
|
||||
// Opening the restored index while the old drive or network share is unavailable must not
|
||||
// prune anything. No real external volume or network connection is needed by these tests.
|
||||
if (!RelocatedFilePaths.IsLocalAbsolute(oldRoot))
|
||||
{
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
await this.AssertOriginalChunksAsync();
|
||||
}
|
||||
|
||||
// User documents are not in an AI Studio backup. Copy them separately, preserving their
|
||||
// modification times, as when reconnecting an external drive after restoring a computer.
|
||||
var destination = Path.Combine(this.testRoot, "mounted volume", "Übertragene Dokumente");
|
||||
foreach (var original in Directory.EnumerateFiles(originalFolder, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var localPath = Path.Combine(destination, Path.GetRelativePath(originalFolder, original));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(localPath)!);
|
||||
File.Copy(original, localPath);
|
||||
File.SetLastWriteTimeUtc(localPath, File.GetLastWriteTimeUtc(original));
|
||||
}
|
||||
|
||||
var newPath = singleFile ? Path.Combine(destination, "notes.md") : destination;
|
||||
await this.SaveEditAsync(newPath);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
var manifest = await restored.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
var hits = await restored.SearchChunksAsync(this.source.Id, "indexed", 10, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
|
||||
Assert.That(this.vectors.DeletedStores, Is.Zero);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
|
||||
Assert.That(hits.Select(hit => hit.AbsolutePath), Is.EquivalentTo(this.vectors.Locations.Values));
|
||||
Assert.That(hits.All(hit => PathTools.IsInsideDirectory(destination, hit.AbsolutePath)), Is.True);
|
||||
});
|
||||
|
||||
// A restart/hash check after relocation must not relocate or prepare everything a second time.
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
|
||||
await this.AssertOriginalChunksAsync();
|
||||
|
||||
// A disconnected external drive is not an empty folder. Reconnecting it must also keep
|
||||
// the embeddings, rather than paying to prepare the same documents again.
|
||||
var disconnected = destination + "-disconnected";
|
||||
Directory.Move(destination, disconnected);
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Directory.Move(disconnected, destination);
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
|
||||
});
|
||||
}
|
||||
|
||||
private async Task StoreForeignPathsAsync(string oldRoot, char separator)
|
||||
{
|
||||
await using var context = this.store.CreateContext();
|
||||
foreach (var file in await context.EmbeddedFiles.ToListAsync())
|
||||
{
|
||||
var relative = file.RelativePath.Replace(Path.DirectorySeparatorChar, separator);
|
||||
var oldPath = oldRoot + separator + relative;
|
||||
file.AbsolutePath = oldPath;
|
||||
file.RelativePath = relative;
|
||||
file.Fingerprint = LegacyFingerprint(oldPath, file.FileName, file.LastWriteUtc.UtcDateTime.Ticks, file.FileSize);
|
||||
this.vectors.Locations[file.ParentFileId] = oldPath;
|
||||
}
|
||||
await context.SaveChangesAsync();
|
||||
this.source = this.source switch
|
||||
{
|
||||
DataSourceLocalDirectory folder => folder with { Path = oldRoot },
|
||||
DataSourceLocalFile file => file with { FilePath = oldRoot + separator + "notes.md" },
|
||||
_ => throw new InvalidOperationException(),
|
||||
};
|
||||
this.settings.ConfigurationData.DataSources[0] = this.source;
|
||||
}
|
||||
|
||||
private async Task<SqliteIndexStoreClientImplementation> BackupAndRestoreIndexAsync()
|
||||
{
|
||||
var dataDirectory = Path.Combine(this.testRoot, "source-data");
|
||||
var configDirectory = Path.Combine(this.testRoot, "source-config");
|
||||
var copies = Path.Combine(this.testRoot, "index-copies");
|
||||
Directory.CreateDirectory(dataDirectory);
|
||||
Directory.CreateDirectory(configDirectory);
|
||||
await this.store.Client.BackupToDirectoryAsync(Path.Combine(copies, "databases", "sqlite"), CancellationToken.None);
|
||||
|
||||
// This fixture's vector client stores metadata only. The runtime tests separately check
|
||||
// backing up, reopening, and relocating actual Qdrant vectors without changing their values.
|
||||
var vectorDirectory = Path.Combine(copies, "databases", "vector_database", "stores", "store_test");
|
||||
Directory.CreateDirectory(vectorDirectory);
|
||||
await File.WriteAllTextAsync(Path.Combine(vectorDirectory, "locations.json"), JsonSerializer.Serialize(this.vectors.Locations));
|
||||
var archivePath = Path.Combine(this.testRoot, "backup.zip");
|
||||
var settingsJson = JsonSerializer.Serialize(this.settings.ConfigurationData, SettingsManager.JSON_OPTIONS);
|
||||
await using (var output = File.Create(archivePath))
|
||||
await DataBackupWriter.WriteAsync(output, new DataBackupSources(dataDirectory, configDirectory, settingsJson, copies, []),
|
||||
new DataBackupManifest(DataBackupFormat.FORMAT_VERSION, "26.10.1", "V6", DateTimeOffset.UtcNow, dataDirectory, configDirectory, true, []), null, CancellationToken.None);
|
||||
|
||||
var target = new DataRestoreTarget(Path.Combine(this.testRoot, "restored-data"), Path.Combine(this.testRoot, "restored-config"), "26.10.1", AIStudio.Settings.Version.V6, []);
|
||||
var result = await DataRestorePreparer.PrepareAsync(archivePath, target, null, CancellationToken.None);
|
||||
Assert.That(result.Success, Is.True);
|
||||
|
||||
// The runtime applies this prepared directory at restart. Open the exact restored bytes
|
||||
// here, so this test exercises the real backup writer and restore preparer, not a rebuilt DB.
|
||||
var pendingData = Path.Combine(target.DataDirectory, DataRestoreFormat.PENDING_DIRECTORY);
|
||||
var restoredPath = Path.Combine(pendingData, "databases", "sqlite", "rag-index.sqlite3");
|
||||
var restored = new SqliteIndexStoreClientImplementation("Restored SQLite", restoredPath, pendingData, string.Empty);
|
||||
Field<Dictionary<DatabaseRole, DatabaseClient>>(this.databases, "clients")[DatabaseRole.INDEX_STORE] = restored;
|
||||
this.vectors.Locations.Clear();
|
||||
foreach (var (id, path) in JsonSerializer.Deserialize<Dictionary<string, string>>(await File.ReadAllTextAsync(Path.Combine(pendingData, "databases", "vector_database", "stores", "store_test", "locations.json")))!)
|
||||
this.vectors.Locations.Add(id, path);
|
||||
|
||||
var restoredSettings = JsonSerializer.Deserialize<Data>(await File.ReadAllTextAsync(Path.Combine(target.ConfigDirectory, DataRestoreFormat.PENDING_DIRECTORY, "settings.json")), SettingsManager.JSON_OPTIONS)!;
|
||||
this.settings.ConfigurationData.DataSources.Clear();
|
||||
this.settings.ConfigurationData.DataSources.AddRange(restoredSettings.DataSources);
|
||||
this.source = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
|
||||
return restored;
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task AnEmptyMountDirectoryKeepsTheIndexAndReconnectingReusesIt(bool relocated)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(false);
|
||||
var original = ((DataSourceLocalDirectory)this.source).Path;
|
||||
var disconnected = Path.Combine(this.testRoot, "disconnected-drive");
|
||||
Directory.Move(original, disconnected);
|
||||
var mount = relocated ? Path.Combine(this.testRoot, "new-mount") : original;
|
||||
Directory.CreateDirectory(mount);
|
||||
var storedHash = (await this.store.Client.GetDataSourceStateAsync(this.source.Id, CancellationToken.None))!.SourceHash;
|
||||
|
||||
if (relocated)
|
||||
await this.SaveEditAsync(mount);
|
||||
else
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
|
||||
// A second refresh must not accept the empty mount as a new baseline and prune the index.
|
||||
for (var attempt = 0; attempt < 2; attempt++)
|
||||
{
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
await this.AssertOriginalChunksAsync();
|
||||
var state = await this.store.Client.GetDataSourceStateAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
|
||||
Assert.That(this.service.GetStatuses().Single().LastError, Does.Contain("appears empty"));
|
||||
Assert.That(state!.SourceHash, Is.EqualTo(storedHash));
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
});
|
||||
if (attempt == 0)
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
}
|
||||
|
||||
Directory.Delete(mount);
|
||||
Directory.Move(disconnected, mount);
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(relocated ? this.chunkIds.Count : 0));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RemovingSomeFilesStillCleansUpOnlyTheirIndexEntries()
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(false);
|
||||
var removed = Path.Combine(((DataSourceLocalDirectory)this.source).Path, "notes.md");
|
||||
var removedChunkId = IndexedDocumentIds.CreateChunkId(this.source.Id, LegacyFingerprint(new FileInfo(removed)), 0);
|
||||
File.Delete(removed);
|
||||
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
var chunks = await this.store.Client.SearchChunksAsync(this.source.Id, "indexed", 10, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(chunks.Select(chunk => chunk.ChunkId), Is.EquivalentTo(this.chunkIds.Except([removedChunkId])));
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.EqualTo(1));
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ExplicitlyExcludingEveryFileStillRemovesTheirIndexEntries()
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(false);
|
||||
this.source = ((DataSourceLocalDirectory)this.source) with { FilePatterns = "*.pdf" };
|
||||
this.settings.ConfigurationData.DataSources[0] = this.source;
|
||||
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(manifest.Files, Is.Empty);
|
||||
Assert.That(this.vectors.Locations, Is.Empty);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.EqualTo(2));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFolderWhoseNameChangesOnlyInCaseKeepsItsEmbeddings()
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(false);
|
||||
var renamed = Path.Combine(this.testRoot, "Original");
|
||||
|
||||
// Two moves, since a volume which ignores case may refuse to rename a folder to its own name:
|
||||
var detour = Path.Combine(this.testRoot, "renaming");
|
||||
Directory.Move(((DataSourceLocalDirectory)this.source).Path, detour);
|
||||
Directory.Move(detour, renamed);
|
||||
|
||||
await this.SaveEditAsync(renamed);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
Assert.That(manifest.Files.Keys.All(path => path.StartsWith(renamed + Path.DirectorySeparatorChar, StringComparison.Ordinal)), Is.True, "The index must store the new spelling of the folder.");
|
||||
});
|
||||
}
|
||||
|
||||
private async Task SeedLegacyIndexAsync(bool singleFile)
|
||||
{
|
||||
var paths = await this.AddSourceAsync(singleFile);
|
||||
var id = this.source.Id;
|
||||
await this.store.Client.UpsertDataSourceAsync(id, this.source.Type.ToString(), EMBEDDING_ID, LEGACY_SIGNATURE, "legacy-source-hash", 3, CancellationToken.None);
|
||||
|
||||
foreach (var relative in paths)
|
||||
{
|
||||
var path = Path.Combine(this.OriginalFolder, relative);
|
||||
var file = new FileInfo(path);
|
||||
var fingerprint = LegacyFingerprint(file);
|
||||
var parentId = IndexedDocumentIds.CreateParentId(id, path);
|
||||
var chunkId = IndexedDocumentIds.CreateChunkId(id, fingerprint, 0);
|
||||
var embeddedAt = DateTimeOffset.UtcNow.AddDays(-10);
|
||||
await this.store.Client.UpsertFileAsync(id, new EmbeddingStateFile(parentId, path, file.Name, relative, "md", fingerprint, file.Length,
|
||||
new DateTimeOffset(file.CreationTimeUtc), new DateTimeOffset(file.LastWriteTimeUtc), embeddedAt, 1), CancellationToken.None);
|
||||
await this.store.Client.UpsertChunksAsync(id, [new EmbeddingStateChunk(chunkId, parentId, 1, 0, "Already indexed report", embeddedAt)], CancellationToken.None);
|
||||
this.vectors.Locations.Add(parentId, path);
|
||||
this.chunkIds.Add(chunkId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes the documents of a data source and configures it, without anything in the index.
|
||||
/// </summary>
|
||||
/// <returns>The paths of the documents, relative to the folder.</returns>
|
||||
private async Task<string[]> AddSourceAsync(bool singleFile)
|
||||
{
|
||||
var root = this.OriginalFolder;
|
||||
Directory.CreateDirectory(Path.Combine(root, "nested"));
|
||||
var id = Guid.NewGuid().ToString();
|
||||
var folder = new DataSourceLocalDirectory
|
||||
{
|
||||
Id = id, Name = "Existing documents", Type = DataSourceType.LOCAL_DIRECTORY, Path = root,
|
||||
EmbeddingId = EMBEDDING_ID, ConfidenceLevel = ConfidenceLevel.NONE, MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 100,
|
||||
FilePatterns = "*.md", ExcludePatterns = string.Empty,
|
||||
};
|
||||
this.source = singleFile ? new DataSourceLocalFile
|
||||
{
|
||||
Id = id, Name = folder.Name, Type = DataSourceType.LOCAL_FILE, FilePath = Path.Combine(root, "notes.md"),
|
||||
EmbeddingId = EMBEDDING_ID, ConfidenceLevel = ConfidenceLevel.NONE, MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 100,
|
||||
} : folder;
|
||||
this.settings.ConfigurationData.DataSources.Add(this.source);
|
||||
|
||||
var paths = singleFile ? new[] { "notes.md" } : new[] { "notes.md", Path.Combine("nested", "report.md") };
|
||||
foreach (var relative in paths)
|
||||
await File.WriteAllTextAsync(Path.Combine(root, relative), "Already indexed report");
|
||||
|
||||
return paths;
|
||||
}
|
||||
|
||||
private string MoveSource(string destinationName)
|
||||
{
|
||||
var current = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
|
||||
var destination = Path.Combine(this.testRoot, destinationName);
|
||||
if (current is DataSourceLocalDirectory folder)
|
||||
Directory.Move(folder.Path, destination);
|
||||
else
|
||||
{
|
||||
destination += ".md";
|
||||
File.Move(((DataSourceLocalFile)current).FilePath, destination);
|
||||
}
|
||||
return destination;
|
||||
}
|
||||
|
||||
private async Task SaveEditAsync(string newPath)
|
||||
{
|
||||
var edit = this.OpenEdit(newPath);
|
||||
edit.Warning.Complete(DialogResult.Ok(true));
|
||||
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
|
||||
Assert.That(edit.Warning.ComponentType, Is.EqualTo(typeof(ConfirmDialog)));
|
||||
Assert.That(edit.Warning.Title, Is.EqualTo("New Location for This Data Source"));
|
||||
Assert.That(edit.Warning.Parameters!.Get<string>(nameof(ConfirmDialog.MarkdownBody)), Does.Contain("unchanged at the new location").And.Contain("removed from the index").And.Contain("files on disk stay as they are").And.Contain("costs money"));
|
||||
});
|
||||
this.source = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
|
||||
}
|
||||
|
||||
private OpenedEdit OpenEdit(string newPath)
|
||||
{
|
||||
var storedSource = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
|
||||
var dialog = new TestDataSourceDialog();
|
||||
var warning = new TestDataSourceDialog();
|
||||
var parent = new DataSourceManagement();
|
||||
Property(parent, "SettingsManager", this.settings);
|
||||
Property(parent, "MessageBus", MessageBus.INSTANCE);
|
||||
Property(parent, "DataSourceEmbeddingService", this.service);
|
||||
Property(parent, "DialogService", dialog.Service);
|
||||
var parentTask = InvokeTask(parent, "EditDataSource", storedSource);
|
||||
object component = storedSource is DataSourceLocalDirectory ? new DataSourceLocalDirectoryDialog() : new DataSourceLocalFileDialog();
|
||||
Property(component, "SettingsManager", this.settings);
|
||||
Property(component, "DialogService", warning.Service);
|
||||
Property(component, "DataSourceEmbeddingService", this.service);
|
||||
Property(component, "MudDialog", dialog.Instance);
|
||||
Property(component, "IsEditing", true);
|
||||
Property(component, "DataSource", storedSource);
|
||||
SetField(component, "dataId", storedSource.Id);
|
||||
SetField(component, "dataName", storedSource.Name);
|
||||
SetField(component, "dataEmbeddingId", storedSource.EmbeddingId);
|
||||
SetField(component, "dataConfidenceLevel", storedSource.ConfidenceLevel);
|
||||
SetField(component, "dataMaxChunkTokenLength", storedSource.MaxChunkTokenLength);
|
||||
SetField(component, "dataChunkOverlapTokenLength", storedSource.ChunkOverlapTokenLength);
|
||||
SetField(component, storedSource is DataSourceLocalDirectory ? "dataPath" : "dataFilePath", newPath);
|
||||
if (storedSource is DataSourceLocalDirectory folder)
|
||||
{
|
||||
SetField(component, "dataFilePatterns", folder.FilePatterns);
|
||||
SetField(component, "dataExcludePatterns", folder.ExcludePatterns);
|
||||
}
|
||||
return new(component, dialog, warning, parentTask);
|
||||
}
|
||||
|
||||
private Task RunNextAsync()
|
||||
{
|
||||
Assert.That(this.Queue.TryDequeue(out var request), Is.True, "Saving must queue a run.");
|
||||
var task = InvokeTask(this.service, "RunDataSourceAsync", request!.DataSourceId, request.RefreshMode, this.lifetime.Token);
|
||||
this.runningTasks.Add(task);
|
||||
return task;
|
||||
}
|
||||
|
||||
private async Task AssertOriginalChunksAsync()
|
||||
{
|
||||
var index = await this.databases.GetIndexStoreAsync();
|
||||
var chunks = await index.SearchChunksAsync(this.source.Id, "indexed", 100, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(chunks.Select(chunk => chunk.ChunkId), Is.EquivalentTo(this.chunkIds));
|
||||
Assert.That(chunks.All(chunk => chunk.ChunkText == "Already indexed report"), Is.True);
|
||||
Assert.That(chunks.All(chunk => chunk.EmbeddedAtUtc < DateTimeOffset.UtcNow.AddDays(-9)), Is.True, "Updating the app or its paths must not recreate existing chunks.");
|
||||
});
|
||||
}
|
||||
|
||||
private static string LegacyFingerprint(FileInfo file)
|
||||
{
|
||||
return LegacyFingerprint(file.FullName, file.Name, file.LastWriteTimeUtc.Ticks, file.Length);
|
||||
}
|
||||
|
||||
private static string LegacyFingerprint(string path, string name, long writeTicks, long size)
|
||||
{
|
||||
var parts = new[] { "file", path, name, writeTicks.ToString(), size.ToString() };
|
||||
var serialized = string.Concat(parts.Select(part => $"{part.Length}:{part}|"));
|
||||
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(serialized)));
|
||||
}
|
||||
|
||||
// Keep reflection at the fixture boundary: exercise the production queue worker without
|
||||
// starting the unrelated application startup loop, and install the isolated database clients.
|
||||
private static T Field<T>(object instance, string name) => (T)instance.GetType().GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(instance)!;
|
||||
|
||||
private static void SetField(object instance, string name, object value) => instance.GetType().GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(instance, value);
|
||||
|
||||
private static void Property(object instance, string name, object value)
|
||||
{
|
||||
for (var type = instance.GetType(); type is not null; type = type.BaseType)
|
||||
if (type.GetProperty(name, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly) is { } property)
|
||||
{
|
||||
property.SetValue(instance, value);
|
||||
return;
|
||||
}
|
||||
throw new MissingMemberException(name);
|
||||
}
|
||||
|
||||
private static Task InvokeTask(object instance, string name, params object[] arguments) =>
|
||||
(Task)instance.GetType().GetMethod(name, BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(instance, arguments)!;
|
||||
|
||||
private sealed record OpenedEdit(object Component, TestDataSourceDialog Dialog, TestDataSourceDialog Warning, Task ParentTask);
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using AIStudio.Tools.Databases.IndexStore;
|
||||
using AIStudio.Tools.Services.Indexing;
|
||||
|
||||
namespace AIStudio.Tests.Tools.Databases;
|
||||
|
||||
public sealed class IndexStoreRelocationTests
|
||||
{
|
||||
[Test]
|
||||
public async Task RelocationPreservesChunksAndTranscriptAndUpdatesSearchLocations()
|
||||
{
|
||||
await using var store = await TemporaryIndexStore.CreateAsync();
|
||||
var sourceId = Guid.NewGuid().ToString();
|
||||
var oldPath = Path.Combine(Path.GetTempPath(), "original", "notes.md");
|
||||
var newPath = Path.Combine(Path.GetTempPath(), "moved", "renamed.md");
|
||||
var parentId = IndexedDocumentIds.CreateParentId(sourceId, oldPath);
|
||||
var chunkId = IndexedDocumentIds.CreateChunkId(sourceId, "old-fingerprint", 0);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var token = CancellationToken.None;
|
||||
await store.Client.UpsertDataSourceAsync(sourceId, "LOCAL_DIRECTORY", "provider", "signature", "old-hash", 3, token);
|
||||
await store.Client.UpsertFileAsync(sourceId, new EmbeddingStateFile(parentId, oldPath, "notes.md", "notes.md", "md", "old-fingerprint", 100, now, now, now, 1) { TranscriptKey = "transcript" }, token);
|
||||
await store.Client.UpsertChunksAsync(sourceId, [new EmbeddingStateChunk(chunkId, parentId, 2, 0, "report about relocation", now)], token);
|
||||
|
||||
// Repeating the metadata write after an interrupted relocation must keep the same chunks.
|
||||
await store.Client.RelocateFileAsync(sourceId, parentId, newPath, "renamed.md", "new-fingerprint", token);
|
||||
await store.Client.RelocateFileAsync(sourceId, parentId, newPath, "renamed.md", "new-fingerprint", token);
|
||||
var manifest = await store.Client.GetManifestAsync(sourceId, token);
|
||||
var hits = await store.Client.SearchChunksAsync(sourceId, "relocation", 10, token);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(manifest.Files.Keys, Is.EquivalentTo(new[] { newPath }));
|
||||
Assert.That(manifest.Files[newPath].Fingerprint, Is.EqualTo("new-fingerprint"));
|
||||
Assert.That(manifest.Files[newPath].ParentFileId, Is.EqualTo(parentId));
|
||||
Assert.That(manifest.Files[newPath].RelativePath, Is.EqualTo("renamed.md"));
|
||||
Assert.That(manifest.Files[newPath].TranscriptKey, Is.EqualTo("transcript"));
|
||||
Assert.That(hits, Has.Count.EqualTo(1));
|
||||
Assert.That(hits[0].ChunkId, Is.EqualTo(chunkId));
|
||||
Assert.That(hits[0].AbsolutePath, Is.EqualTo(newPath));
|
||||
Assert.That(hits[0].FileName, Is.EqualTo("renamed.md"));
|
||||
Assert.That(hits[0].PageNumber, Is.EqualTo(2));
|
||||
Assert.That(hits[0].ChunkText, Is.EqualTo("report about relocation"));
|
||||
});
|
||||
|
||||
await store.Client.DeleteFileAsync(sourceId, newPath, token);
|
||||
Assert.That(await store.Client.SearchChunksAsync(sourceId, "relocation", 10, token), Is.Empty, "Ordinary removal must still find a relocated document's chunks.");
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,15 @@ public sealed class FileSourceIndexerWatcherTests
|
||||
Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SingleFileWatcherIgnoresAnotherCaseDistinctFileOnUnix()
|
||||
{
|
||||
var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(Path.GetTempPath(), false, TargetFileName: "Report.md");
|
||||
|
||||
Assert.That(FileSourceIndexer.IsRelevantWatcherPath(configuration, Path.Combine(Path.GetTempPath(), "report.md"), WatcherChangeTypes.Changed),
|
||||
Is.EqualTo(OperatingSystem.IsWindows()));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SingleFileWatcherRecognizesARenameAwayFromTheConfiguredFile()
|
||||
{
|
||||
@@ -262,6 +271,15 @@ public sealed class FileSourceIndexerWatcherTests
|
||||
Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RelocatingToADirectoryWithDifferentCasingUsesTheLocalPathRules()
|
||||
{
|
||||
var before = DefaultConfiguration(Path.Combine(Path.GetTempPath(), "Documents"));
|
||||
var after = before with { RootPath = Path.Combine(Path.GetTempPath(), "documents") };
|
||||
|
||||
Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.EqualTo(OperatingSystem.IsWindows()), "On Unix these can be different directories; their watchers must not be treated as interchangeable.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DirectoryWatcherIgnoresTheBatchTranscriptWhenTheMediaIsSelected()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.Services;
|
||||
using AIStudio.Tools.Services.Indexing;
|
||||
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
public sealed class FileSourceRelocationTests
|
||||
{
|
||||
private string testRoot = string.Empty;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-relocation-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(this.testRoot);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown() => Directory.Delete(this.testRoot, true);
|
||||
|
||||
[Test]
|
||||
public void AMovedFolderFindsAnUnchangedNestedDocument()
|
||||
{
|
||||
var oldRoot = Path.Combine(this.testRoot, "original");
|
||||
var oldFile = Path.Combine(oldRoot, "nested", "notes.md");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(oldFile)!);
|
||||
File.WriteAllText(oldFile, "Some indexed text");
|
||||
var record = Record(oldFile, Path.Combine("nested", "notes.md"));
|
||||
var newRoot = Path.Combine(this.testRoot, "moved");
|
||||
Directory.Move(oldRoot, newRoot);
|
||||
|
||||
var found = FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = newRoot }, oldFile, record);
|
||||
|
||||
Assert.That(found?.FullName, Is.EqualTo(Path.Combine(newRoot, "nested", "notes.md")));
|
||||
}
|
||||
|
||||
[TestCase("size")]
|
||||
[TestCase("modification time")]
|
||||
[TestCase("missing file")]
|
||||
public void AChangedOrMissingDocumentCannotReuseItsEmbeddings(string change)
|
||||
{
|
||||
var file = Path.Combine(this.testRoot, "notes.md");
|
||||
File.WriteAllText(file, "Some indexed text");
|
||||
var record = Record(file, "notes.md");
|
||||
switch (change)
|
||||
{
|
||||
case "size":
|
||||
File.AppendAllText(file, " with more text");
|
||||
File.SetLastWriteTimeUtc(file, record.LastWriteUtc.UtcDateTime);
|
||||
break;
|
||||
case "modification time":
|
||||
File.SetLastWriteTimeUtc(file, record.LastWriteUtc.UtcDateTime.AddMinutes(1));
|
||||
break;
|
||||
case "missing file":
|
||||
File.Delete(file);
|
||||
break;
|
||||
}
|
||||
|
||||
// The old place lies outside the selected folder, so only the change can prevent the reuse:
|
||||
var oldPath = Path.Combine(Path.GetDirectoryName(this.testRoot)!, "moved-away", "notes.md");
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, oldPath, record), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AStoredPathBelowTheSelectedFolderIsNotMappedToACopy()
|
||||
{
|
||||
// The folder grows to its parent, which holds a copy with the same size and time at the
|
||||
// place the old relative path names. The original is still there and stays the document.
|
||||
var original = Path.Combine(this.testRoot, "project", "notes.md");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(original)!);
|
||||
File.WriteAllText(original, "Some indexed text");
|
||||
var copy = Path.Combine(this.testRoot, "notes.md");
|
||||
File.Copy(original, copy);
|
||||
File.SetLastWriteTimeUtc(copy, File.GetLastWriteTimeUtc(original));
|
||||
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, original, Record(original, "notes.md")), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARenamedSingleFileCanReuseItsEmbeddings()
|
||||
{
|
||||
var oldFile = Path.Combine(this.testRoot, "notes.md");
|
||||
File.WriteAllText(oldFile, "Some indexed text");
|
||||
var record = Record(oldFile, "notes.md");
|
||||
var newFile = Path.Combine(this.testRoot, "renamed.md");
|
||||
File.Move(oldFile, newFile);
|
||||
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalFile { FilePath = newFile }, oldFile, record)?.FullName, Is.EqualTo(newFile));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARelativePathCannotEscapeTheConfiguredFolder()
|
||||
{
|
||||
var outside = Path.Combine(this.testRoot, "notes.md");
|
||||
File.WriteAllText(outside, "Some indexed text");
|
||||
var record = Record(outside, Path.Combine("..", "notes.md"));
|
||||
var folder = Path.Combine(this.testRoot, "selected");
|
||||
Directory.CreateDirectory(folder);
|
||||
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = folder }, Path.Combine(this.testRoot, "old", "notes.md"), record), Is.Null);
|
||||
}
|
||||
|
||||
private static EmbeddedFileRecord Record(string path, string relativePath)
|
||||
{
|
||||
var file = new FileInfo(path);
|
||||
return new("old-fingerprint", file.Length, new DateTimeOffset(file.LastWriteTimeUtc), DateTimeOffset.UtcNow, 1, RelativePath: relativePath, ParentFileId: Guid.NewGuid().ToString());
|
||||
}
|
||||
|
||||
[TestCase(@"E:\Documents\nested\notes.md", @"nested\notes.md")]
|
||||
[TestCase(@"\\server\share\Documents\nested\notes.md", @"nested\notes.md")]
|
||||
[TestCase(@"\\?\E:\Documents\nested\notes.md", @"nested\notes.md")]
|
||||
[TestCase("E:/Documents/nested/notes.md", "nested/notes.md")]
|
||||
[TestCase("/home/alice/Documents/nested/notes.md", "nested/notes.md")]
|
||||
[TestCase("/Volumes/External drive/Documents/nested/notes.md", "nested/notes.md")]
|
||||
public void ABackupFromAnotherPathSyntaxFindsANestedFile(string oldPath, string relativePath)
|
||||
{
|
||||
var localFile = Path.Combine(this.testRoot, "nested", "notes.md");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(localFile)!);
|
||||
File.WriteAllText(localFile, "Some indexed text");
|
||||
|
||||
var found = FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, oldPath, Record(localFile, relativePath));
|
||||
|
||||
Assert.That(found?.FullName, Is.EqualTo(localFile));
|
||||
}
|
||||
|
||||
[TestCase(@"C:\a.folder\README")]
|
||||
[TestCase(@"\\server\a.folder\README")]
|
||||
[TestCase("/media/a.folder/README")]
|
||||
public void AForeignDirectoryNameDoesNotBecomeTheExtensionOfAnExtensionlessFile(string oldPath)
|
||||
{
|
||||
var localFile = Path.Combine(this.testRoot, "README");
|
||||
File.WriteAllText(localFile, "Some indexed text");
|
||||
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalFile { FilePath = localFile }, oldPath, Record(localFile, "README"))?.FullName, Is.EqualTo(localFile));
|
||||
}
|
||||
|
||||
[TestCase(@"..\notes.md")]
|
||||
[TestCase("../notes.md")]
|
||||
[TestCase(@"nested\..\..\notes.md")]
|
||||
[TestCase(@"C:\notes.md")]
|
||||
[TestCase(@"C:notes.md")]
|
||||
[TestCase(@"\\server\share\notes.md")]
|
||||
[TestCase("/etc/notes.md")]
|
||||
public void AStoredWindowsPathCannotEscapeTheNewRoot(string relativePath)
|
||||
{
|
||||
Assert.That(RelocatedFilePaths.Resolve(this.testRoot, @"E:\Documents\notes.md", relativePath), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnExternalDriveRootIsAValidRelocationDestination()
|
||||
{
|
||||
// No file needs to be written at the drive root: this checks resolving the destination.
|
||||
var root = Path.GetPathRoot(this.testRoot)!;
|
||||
|
||||
Assert.That(RelocatedFilePaths.Resolve(root, @"E:\notes.md", "notes.md"), Is.EqualTo(Path.Combine(root, "notes.md")));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AUnixBackslashInAFileNameIsNotReinterpretedAsADirectory()
|
||||
{
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
Assert.That(RelocatedFilePaths.Resolve(this.testRoot, @"/home/alice/notes\draft.md", @"notes\draft.md"), Is.Null, "A Windows filename cannot contain a backslash from a Unix filename.");
|
||||
return;
|
||||
}
|
||||
|
||||
var localFile = Path.Combine(this.testRoot, @"notes\draft.md");
|
||||
File.WriteAllText(localFile, "Some indexed text");
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, @"/home/alice/notes\draft.md", Record(localFile, @"notes\draft.md"))?.FullName, Is.EqualTo(localFile));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnOldForeignRootIsNotResolvedRelativeToTheWorkingDirectory()
|
||||
{
|
||||
var foreignRoot = OperatingSystem.IsWindows() ? "/home/alice/Documents" : @"E:\Documents";
|
||||
|
||||
Assert.That(RelocatedFilePaths.Resolve(foreignRoot, @"E:\Documents\notes.md", "notes.md"), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReducedTimestampPrecisionOnAnotherDriveDoesNotSilentlyReuseOldContent()
|
||||
{
|
||||
var localFile = Path.Combine(this.testRoot, "notes.md");
|
||||
File.WriteAllText(localFile, "Some indexed text");
|
||||
var record = Record(localFile, "notes.md");
|
||||
// The old filesystem stored a finer timestamp than the copied file now reports. Without
|
||||
// a stored content hash, equal size alone is not enough to prove that it is unchanged.
|
||||
record = record with { LastWriteUtc = record.LastWriteUtc.AddTicks(1) };
|
||||
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, @"E:\Documents\notes.md", record), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ANewFileAtAnOldLocationCannotOverwriteTheRelocatedDocument()
|
||||
{
|
||||
var source = new DataSourceLocalDirectory { Id = Guid.NewGuid().ToString(), Path = this.testRoot };
|
||||
var originalPath = Path.Combine(this.testRoot, "nested", "notes.md");
|
||||
var relocatedPath = Path.Combine(this.testRoot, "notes.md");
|
||||
var originalId = IndexedDocumentIds.CreateParentId(source.Id, originalPath);
|
||||
var manifest = new DataSourceEmbeddingManifest();
|
||||
manifest.Files[relocatedPath] = new("fingerprint", 100, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 1, RelativePath: "notes.md", ParentFileId: originalId);
|
||||
|
||||
// Only identity assignment is exercised; none of the run's services are called.
|
||||
var context = new IndexedRunContext(source, null!, null!, null!, null!, manifest,
|
||||
new RunShare(RunShare.DEFAULT_DURATION), new IndexingPass(new DataSourceRunSignature("signature", "selection")), null!, null!, _ => { }, NullLogger.Instance);
|
||||
var newId = context.GetFileParentId(originalPath);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(context.GetFileParentId(relocatedPath), Is.EqualTo(originalId));
|
||||
Assert.That(newId, Is.Not.EqualTo(originalId));
|
||||
Assert.That(context.GetFileParentId(originalPath), Is.EqualTo(newId), "A media file may ask for its identity once before and once after transcription.");
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -31,4 +31,21 @@ public sealed class IndexedDocumentIdsTests
|
||||
{
|
||||
Assert.That(IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, "/tmp/test-data/report.pdf"), Is.EqualTo("d06c4351-2e57-2941-aa4c-d7a814bfa418"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ADocumentStoredUnderTheIdOfItsKeyKeepsItsChunkIds()
|
||||
{
|
||||
const string KEY = "/tmp/test-data/report.pdf";
|
||||
var source = IndexedDocumentIds.CreateChunkIdSource(DATA_SOURCE_ID, KEY, IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, KEY), "3F9A0C");
|
||||
|
||||
Assert.That(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, source, 0), Is.EqualTo("7c9d2822-f9d1-a94e-8d83-30cea4f0856f"), "Files and mails which never moved must keep the ids they are stored under.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheChunkIdOfAMovedDocumentIsPinned()
|
||||
{
|
||||
var source = IndexedDocumentIds.CreateChunkIdSource(DATA_SOURCE_ID, "/tmp/moved/report.pdf", "0b5c2f7e-3d41-4a8e-9f60-2c7d1e8a4b93", "3F9A0C");
|
||||
|
||||
Assert.That(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, source, 0), Is.EqualTo("aa597d9b-ede8-5449-80d0-f209cde0ecc0"), "A file at the old place with the same fingerprint must not get the chunk ids of the moved one.");
|
||||
}
|
||||
}
|
||||
@@ -112,7 +112,7 @@ public sealed class OutdatedRunTests
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ASingleFileHasNoSelection()
|
||||
public void RelocatingASingleFileStopsTheOldRunWithoutInvalidatingItsEmbeddings()
|
||||
{
|
||||
var dataSource = new DataSourceLocalFile
|
||||
{
|
||||
@@ -125,7 +125,26 @@ public sealed class OutdatedRunTests
|
||||
FilePath = "/tmp/test-data/report.pdf",
|
||||
};
|
||||
|
||||
Assert.That(DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM).SelectionSignature, Is.Empty, "The file is the one document of its data source, so there is nothing to choose among.");
|
||||
var before = DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
|
||||
var after = DataSourceEmbeddingService.GetRunSignature(dataSource with { FilePath = "/tmp/moved/report.pdf" }, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(DataSourceEmbeddingService.IsRunOutdated(before, after), Is.True);
|
||||
Assert.That(after.EmbeddingSignature, Is.EqualTo(before.EmbeddingSignature));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RelocatingAFolderStopsTheOldRunWithoutInvalidatingItsEmbeddings()
|
||||
{
|
||||
var dataSource = DataSource(ConfidenceLevel.LOW);
|
||||
var before = DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
|
||||
var after = DataSourceEmbeddingService.GetRunSignature(dataSource with { Path = "/tmp/moved" }, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(DataSourceEmbeddingService.IsRunOutdated(before, after), Is.True);
|
||||
Assert.That(after.EmbeddingSignature, Is.EqualTo(before.EmbeddingSignature));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
using AIStudio.Tools;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Checks where a file dialog opens for a file or a folder which was moved or renamed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The native dialog cannot open a place which no longer exists and falls back to wherever it was
|
||||
/// used last, which may be anywhere. The nearest folder which still exists is where the file or the
|
||||
/// folder was moved from, so its new place is often close by.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class PathToolsTests
|
||||
{
|
||||
private string testRoot = string.Empty;
|
||||
|
||||
[SetUp]
|
||||
public void CreateTestRoot()
|
||||
{
|
||||
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-path-tools-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(this.testRoot);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void DeleteTestRoot()
|
||||
{
|
||||
if (Directory.Exists(this.testRoot))
|
||||
Directory.Delete(this.testRoot, true);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnExistingFolderOpensAsItIs()
|
||||
{
|
||||
var folder = Directory.CreateDirectory(Path.Combine(this.testRoot, "documents")).FullName;
|
||||
|
||||
Assert.That(PathTools.FindNearestExistingPath(folder), Is.EqualTo(folder));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnExistingFileOpensAsItIs()
|
||||
{
|
||||
var file = Path.Combine(this.testRoot, "report.pdf");
|
||||
File.WriteAllText(file, "report");
|
||||
|
||||
Assert.That(PathTools.FindNearestExistingPath(file), Is.EqualTo(file));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AMovedFolderOpensAtItsNearestFolderWhichStillExists()
|
||||
{
|
||||
var parent = Directory.CreateDirectory(Path.Combine(this.testRoot, "projects")).FullName;
|
||||
var movedFolder = Path.Combine(parent, "2026", "documents");
|
||||
|
||||
Assert.That(PathTools.FindNearestExistingPath(movedFolder), Is.EqualTo(parent), "Every parent folder counts, not only the immediate one.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AMovedFileOpensAtItsFolder()
|
||||
{
|
||||
var file = Path.Combine(this.testRoot, "report.pdf");
|
||||
|
||||
Assert.That(PathTools.FindNearestExistingPath(file), Is.EqualTo(this.testRoot));
|
||||
}
|
||||
|
||||
[TestCase("")]
|
||||
[TestCase(" ")]
|
||||
[TestCase("documents/report.pdf")]
|
||||
public void APathWithoutAPlaceOpensWhereverTheDialogWants(string path)
|
||||
{
|
||||
Assert.That(PathTools.FindNearestExistingPath(path), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void APathOfAnotherOperatingSystemOpensWhereverTheDialogWants()
|
||||
{
|
||||
var path = OperatingSystem.IsWindows() ? "/Users/somebody/documents" : @"C:\Users\somebody\documents";
|
||||
|
||||
Assert.That(PathTools.FindNearestExistingPath(path), Is.Null, "A backup from another computer must not make the dialog open at a folder which happens to exist here.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
using AIStudio.Tools.Databases.VectorStore;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Keeps the identities and locations of existing vectors. New embeddings are forbidden: these
|
||||
/// tests reconnect previously indexed files and must never contact an embedding provider.
|
||||
/// </summary>
|
||||
internal sealed class RelocationVectorStore() : VectorStoreClient("Relocation test vectors", string.Empty)
|
||||
{
|
||||
public Dictionary<string, string> Locations { get; } = new(StringComparer.Ordinal);
|
||||
|
||||
public List<string> RelocatedPaths { get; } = [];
|
||||
|
||||
public TaskCompletionSource<object?> RelocationStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public bool BlockNextRelocation { get; set; }
|
||||
|
||||
public int CanceledRelocations { get; private set; }
|
||||
|
||||
public int DeletedStores { get; private set; }
|
||||
|
||||
public int DeletedDocuments { get; private set; }
|
||||
|
||||
public int InsertedBatches { get; private set; }
|
||||
|
||||
public override async Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
Assert.That(this.Locations.ContainsKey(parentFileId), Is.True, "Relocation must address an existing vector identity.");
|
||||
this.Locations[parentFileId] = absolutePath;
|
||||
this.RelocatedPaths.Add(absolutePath);
|
||||
if (!this.BlockNextRelocation)
|
||||
return;
|
||||
|
||||
this.BlockNextRelocation = false;
|
||||
this.RelocationStarted.TrySetResult(null);
|
||||
try
|
||||
{
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, token);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
this.CanceledRelocations++;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public override Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) => Task.FromResult(new VectorStoreEnsureResult(false));
|
||||
|
||||
public override Task<bool> VectorStoreExists(string storeName, CancellationToken token) => Task.FromResult(true);
|
||||
|
||||
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token)
|
||||
{
|
||||
this.InsertedBatches++;
|
||||
throw new InvalidOperationException("Existing unchanged files must not be embedded again.");
|
||||
}
|
||||
|
||||
public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token)
|
||||
{
|
||||
this.DeletedDocuments++;
|
||||
foreach (var id in this.Locations.Where(entry => entry.Key == parentFileId || entry.Value == filePath).Select(entry => entry.Key).ToArray())
|
||||
this.Locations.Remove(id);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public override Task DeleteVectorStore(string storeName, CancellationToken token)
|
||||
{
|
||||
this.DeletedStores++;
|
||||
this.Locations.Clear();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public override Task OptimizeVectorStore(string storeName, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, CancellationToken token) => throw new NotSupportedException();
|
||||
|
||||
public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, CancellationToken token) => throw new NotSupportedException();
|
||||
|
||||
public override Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token) => throw new NotSupportedException();
|
||||
|
||||
public override Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) => throw new NotSupportedException();
|
||||
|
||||
public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo()
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
yield break;
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
using System.Reflection;
|
||||
|
||||
using MudBlazor;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Controls a dialog result without rendering a Blazor circuit. The real dialog save handler and
|
||||
/// the real parent edit handler still run, with the same parameters and result as in the UI.
|
||||
/// </summary>
|
||||
internal sealed class TestDataSourceDialog
|
||||
{
|
||||
private readonly TaskCompletionSource<DialogResult?> result = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public IDialogService Service { get; }
|
||||
|
||||
public IMudDialogInstance Instance { get; }
|
||||
|
||||
public DialogParameters? Parameters { get; private set; }
|
||||
|
||||
public Type? ComponentType { get; private set; }
|
||||
|
||||
public string? Title { get; private set; }
|
||||
|
||||
public int ShowCount { get; private set; }
|
||||
|
||||
public TestDataSourceDialog()
|
||||
{
|
||||
var reference = Create<IDialogReference>((method, _) => method.Name == "get_Result"
|
||||
? this.result.Task
|
||||
: throw new NotSupportedException(method.Name));
|
||||
this.Service = Create<IDialogService>((method, arguments) =>
|
||||
{
|
||||
if (method.Name != "ShowAsync")
|
||||
throw new NotSupportedException(method.Name);
|
||||
|
||||
ArgumentNullException.ThrowIfNull(arguments);
|
||||
this.ShowCount++;
|
||||
this.ComponentType = method.GetGenericArguments().Single();
|
||||
this.Title = arguments.OfType<string>().FirstOrDefault();
|
||||
this.Parameters = arguments.OfType<DialogParameters>().Single();
|
||||
return Task.FromResult(reference);
|
||||
});
|
||||
this.Instance = Create<IMudDialogInstance>((method, arguments) =>
|
||||
{
|
||||
switch (method.Name)
|
||||
{
|
||||
case "Close":
|
||||
this.Complete(arguments!.OfType<DialogResult>().Single());
|
||||
return null;
|
||||
case "Cancel":
|
||||
this.Complete(DialogResult.Cancel());
|
||||
return null;
|
||||
default:
|
||||
throw new NotSupportedException(method.Name);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public void Complete(DialogResult dialogResult) => this.result.TrySetResult(dialogResult);
|
||||
|
||||
private static T Create<T>(Func<MethodInfo, object?[]?, object?> invoke) where T : class
|
||||
{
|
||||
var proxy = DispatchProxy.Create<T, DialogCallProxy>();
|
||||
((DialogCallProxy)(object)proxy).InvokeCall = invoke;
|
||||
return proxy;
|
||||
}
|
||||
}
|
||||
|
||||
// DispatchProxy generates a subclass, so this type must be public and cannot be sealed.
|
||||
public class DialogCallProxy : DispatchProxy
|
||||
{
|
||||
public Func<MethodInfo, object?[]?, object?> InvokeCall { get; set; } = null!;
|
||||
|
||||
protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) => this.InvokeCall(targetMethod!, args);
|
||||
}
|
||||
@@ -92,7 +92,7 @@ public sealed class SemanticSearchToolUnreachableTests
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheResultNamesAnUnreachableDataSourceWithoutItsID()
|
||||
public void TheResultNamesAnUnreachableDataSourceWithoutItsId()
|
||||
{
|
||||
var result = SemanticSearchTool.DescribeUnreachableResult(INTRANET);
|
||||
|
||||
|
||||
@@ -62,6 +62,49 @@ public sealed class UnreachableDataSourceTests
|
||||
AssertTheIndexWasKept(outcome, statuses, manifest, 2, folder);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AMissingRelocationTargetKeepsTheOriginalIndex()
|
||||
{
|
||||
var folder = Path.Combine(this.testRoot, "moved-but-unavailable");
|
||||
var (outcome, statuses, manifest) = await RunAsync(Folder(folder), Path.Combine(this.testRoot, "original", "report.pdf"));
|
||||
|
||||
AssertTheIndexWasKept(outcome, statuses, manifest, 1, folder);
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task AReadableEmptyMountDirectoryKeepsItsIndex(bool emptySubfolder)
|
||||
{
|
||||
var folder = Path.Combine(this.testRoot, "mounted-drive");
|
||||
Directory.CreateDirectory(emptySubfolder ? Path.Combine(folder, "documents") : folder);
|
||||
var unfinishedFile = Path.Combine(folder, "unfinished.pdf");
|
||||
|
||||
var (outcome, statuses, manifest) = await RunAsync(Folder(folder), [Path.Combine(folder, "report.pdf")], [unfinishedFile]);
|
||||
|
||||
AssertTheIndexWasKept(outcome, statuses, manifest, 1, folder);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(manifest.UnfinishedFiles, Does.Contain(unfinishedFile), "An unmounted drive must not discard interrupted work either.");
|
||||
Assert.That(statuses[^1].LastError, Does.Contain("appears empty").And.Contain("remove this data source"));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANewEmptyFolderCanFinishWithoutAnIndex()
|
||||
{
|
||||
var folder = Path.Combine(this.testRoot, "new-empty-folder");
|
||||
Directory.CreateDirectory(folder);
|
||||
|
||||
var (outcome, statuses, manifest) = await RunAsync(Folder(folder));
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outcome, Is.EqualTo(IndexedRunOutcome.DONE));
|
||||
Assert.That(statuses[^1].State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(manifest.Files, Is.Empty);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AMissingFileKeepsItsIndex()
|
||||
{
|
||||
@@ -130,6 +173,15 @@ public sealed class UnreachableDataSourceTests
|
||||
Assert.That(goneFiles, Is.EquivalentTo([deleted, besideSubfolder]), "A folder whose name only begins like the one which could not be read is no part of it.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnUnreadableDriveRootKeepsItsIndexedFiles()
|
||||
{
|
||||
var root = Path.GetPathRoot(Path.GetFullPath(Path.GetTempPath()))!;
|
||||
var file = Path.Combine(root, "documents", "notes.md");
|
||||
|
||||
Assert.That(FileSourceIndexer.GetGoneFiles([file], [], [root]), Is.Empty, "A drive root already ends in its separator; adding another would fail to protect its files.");
|
||||
}
|
||||
|
||||
private static DataSourceLocalDirectory Folder(string path) => new()
|
||||
{
|
||||
Num = 1,
|
||||
@@ -152,6 +204,7 @@ public sealed class UnreachableDataSourceTests
|
||||
Assert.That(statuses[^1].State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
|
||||
Assert.That(statuses[^1].IndexedDocuments, Is.EqualTo(indexedFiles), "The index still holds these files, and the page has to say so.");
|
||||
Assert.That(statuses[^1].LastError, Does.Contain(unreachablePath));
|
||||
Assert.That(statuses[^1].Attention, Is.EqualTo(DataSourceAttention.LOCATION_UNREACHABLE), "The embeddings page offers to edit the data source, e.g. to select the new location of a moved folder.");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -181,12 +234,16 @@ public sealed class UnreachableDataSourceTests
|
||||
var statuses = new List<DataSourceEmbeddingStatus>();
|
||||
var manifest = new DataSourceEmbeddingManifest { SourceHash = STORED_SOURCE_HASH };
|
||||
foreach (var file in indexedFiles)
|
||||
manifest.Files[file] = new EmbeddedFileRecord("FINGERPRINT", 1024, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 4);
|
||||
manifest.Files[file] = new EmbeddedFileRecord("FINGERPRINT", 1024, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 4,
|
||||
RelativePath: Path.GetFileName(file), ParentFileId: IndexedDocumentIds.CreateParentId(dataSource.Id, file));
|
||||
|
||||
manifest.UnfinishedFiles.UnionWith(unfinishedFiles);
|
||||
|
||||
var scannedFile = Path.Combine(Path.GetDirectoryName(indexedFiles[0])!, "scan.pdf");
|
||||
manifest.PermanentFailures[scannedFile] = new PermanentIndexingFailureRecord("FINGERPRINT", FileExtractionErrorCode.NO_TEXT_EXTRACTED, "No text could be read.", DateTimeOffset.UnixEpoch);
|
||||
if (indexedFiles.Length > 0)
|
||||
{
|
||||
var scannedFile = Path.Combine(Path.GetDirectoryName(indexedFiles[0])!, "scan.pdf");
|
||||
manifest.PermanentFailures[scannedFile] = new PermanentIndexingFailureRecord("FINGERPRINT", FileExtractionErrorCode.NO_TEXT_EXTRACTED, "No text could be read.", DateTimeOffset.UnixEpoch);
|
||||
}
|
||||
|
||||
var embeddingProvider = new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"));
|
||||
var settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
|
||||
|
||||
Reference in new issue
Block a user