using AIStudio.Tools;
using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.Mail;
using AIStudio.Tools.Services.Indexing;
namespace AIStudio.Tests.Tools.Databases;
///
/// Checks the lines the information page shows about the local RAG index.
///
///
/// Mails are documents of the index just like files, so they sit in the same table. A line about
/// files must not count them nonetheless.
///
[TestFixture]
public sealed class IndexStoreDisplayInfoTests
{
private const string FOLDER_ID = "0b6e1f7c-3d2a-4e5f-9a8b-1c2d3e4f5a6b";
private const string MAILBOX_ID = "7a8b9c0d-1e2f-4a3b-8c4d-5e6f7a8b9c0d";
private const string EMBEDDING_ID = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01";
private static readonly CancellationToken TOKEN = CancellationToken.None;
[Test]
public async Task FilesAndMailsAreCountedApart()
{
await using var store = await TemporaryIndexStore.CreateAsync();
await store.Client.UpsertDataSourceAsync(FOLDER_ID, "LOCAL_DIRECTORY", EMBEDDING_ID, "signature", "source-hash", 3, TOKEN);
await store.Client.UpsertDataSourceAsync(MAILBOX_ID, "MAILBOX", EMBEDDING_ID, "signature", "source-hash", 3, TOKEN);
await AddDocumentAsync(store, FOLDER_ID, "/tmp/test-data/minutes.md");
await AddDocumentAsync(store, FOLDER_ID, "/tmp/test-data/budget.md");
await AddDocumentAsync(store, MAILBOX_ID, MailKey('a'));
await AddFailureAsync(store, FOLDER_ID, "/tmp/test-data/scan.pdf");
await AddFailureAsync(store, MAILBOX_ID, MailKey('b'));
var lines = await ReadDisplayInfoAsync(store);
Assert.Multiple(() =>
{
Assert.That(lines["Indexed files"], Is.EqualTo("2"));
Assert.That(lines["Permanently skipped files"], Is.EqualTo("1"));
Assert.That(lines["Indexed mails"], Is.EqualTo("1"));
Assert.That(lines["Permanently skipped mails"], Is.EqualTo("1"));
});
}
[Test]
public async Task WithoutMailsTheLinesAboutMailsAreLeftOut()
{
await using var store = await TemporaryIndexStore.CreateAsync();
await store.Client.UpsertDataSourceAsync(FOLDER_ID, "LOCAL_DIRECTORY", EMBEDDING_ID, "signature", "source-hash", 3, TOKEN);
await AddDocumentAsync(store, FOLDER_ID, "/tmp/test-data/minutes.md");
var lines = await ReadDisplayInfoAsync(store);
Assert.Multiple(() =>
{
Assert.That(lines["Indexed files"], Is.EqualTo("1"));
Assert.That(lines, Does.Not.ContainKey("Indexed mails"));
Assert.That(lines, Does.Not.ContainKey("Permanently skipped mails"));
});
}
private static string MailKey(char filler) => MailContentKey.PREFIX + new string(filler, 64);
private static Task AddDocumentAsync(TemporaryIndexStore store, string dataSourceId, string key)
{
var now = DateTimeOffset.UtcNow;
return store.Client.UpsertFileAsync(dataSourceId, new EmbeddingStateFile(IndexedDocumentIds.CreateParentId(dataSourceId, key), key, "name", "name", "md", "fingerprint", 64, now, now, now, 1), TOKEN);
}
private static Task AddFailureAsync(TemporaryIndexStore store, string dataSourceId, string key) => store.Client.UpsertPermanentFailureAsync(
dataSourceId,
new PermanentIndexingFailure(IndexedDocumentIds.CreateParentId(dataSourceId, key), key, "fingerprint", FileExtractionErrorCode.NO_CONTENT, "No text could be read.", DateTimeOffset.UtcNow),
TOKEN);
private static async Task> ReadDisplayInfoAsync(TemporaryIndexStore store)
{
var lines = new Dictionary();
await foreach (var (label, value) in store.Client.GetDisplayInfo())
lines[label] = value;
return lines;
}
}