Added mailboxes as local data sources (#1022)
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2026-10-04 12:26:26 +02:00
1 parent 93be539e50
commit c4400c0ff6
343 files changed
+26051 -3122

No files matched your search

@@ -0,0 +1,73 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <remarks>
/// The mail tools search, count and read mails through these. Each one asks about one mailbox,
/// so a mail of another mailbox never turns up, whatever id a caller hands in.
/// </remarks>
public abstract partial class IndexStoreClient
{
/// <summary>
/// Lists the mails of a mailbox which meet the conditions, the most recently received first.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="filter">The conditions.</param>
/// <param name="offset">How many mails to skip, for the pages after the first one.</param>
/// <param name="limit">How many mails to list at most.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The ids of the mails. The order stays the same between calls, so pages neither repeat nor skip a mail.</returns>
public abstract Task<IReadOnlyList<string>> QueryMailsAsync(string dataSourceId, MailFilter filter, int offset, int limit, CancellationToken token);
/// <summary>
/// Reads the ids of every chunk of the mails which meet the conditions, to restrict a vector search to them.
/// </summary>
/// <remarks>
/// The list can be long: it holds every chunk of every matching mail, and a broad condition
/// matches most of a mailbox. A caller without any condition does better to search the whole
/// store and drop what GetMailSummariesAsync does not know.
/// </remarks>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="filter">The conditions.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The chunk ids, which are the ids of their points in the vector store.</returns>
public abstract Task<IReadOnlyList<string>> GetMailChunkIdsAsync(string dataSourceId, MailFilter filter, CancellationToken token);
/// <summary>
/// Searches the chunks of the mails which meet the conditions by their words, with BM25.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="query">The words to search for.</param>
/// <param name="filter">The conditions.</param>
/// <param name="maxMatches">How many chunks to return at most.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The chunks, best first; the id of their document is the id of their mail.</returns>
public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchMailChunksAsync(string dataSourceId, string query, MailFilter filter, int maxMatches, CancellationToken token);
/// <summary>
/// Counts the mails of a mailbox which meet the conditions.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="filter">The conditions.</param>
/// <param name="grouping">How to break the number down.</param>
/// <param name="maxGroups">How many groups to return at most, the largest ones.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The total and the largest groups.</returns>
public abstract Task<MailCountResult> CountMailsAsync(string dataSourceId, MailFilter filter, MailCountGrouping grouping, int maxGroups, CancellationToken token);
/// <summary>
/// Reads what a list of mails shows about each of the given ones.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="mailIds">The ids of the mails, in the order the list shows them.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The summaries in the given order. A mail this mailbox does not hold, or which lies nowhere any more, is left out.</returns>
public abstract Task<IReadOnlyList<MailSummary>> GetMailSummariesAsync(string dataSourceId, IReadOnlyList<string> mailIds, CancellationToken token);
/// <summary>
/// Finds a mail by its Message-ID, e.g., the one another mail replies to.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="messageId">The Message-ID, without angle brackets.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The id of the mail, the one received first when there are copies; null when the mailbox holds no such mail.</returns>
public abstract Task<string?> FindMailByMessageIdAsync(string dataSourceId, string messageId, CancellationToken token);
}
@@ -0,0 +1,207 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <remarks>
/// Every mail is a document like any other: its row in the files, its chunks, its vectors. What
/// follows is what only a mail has, and it rests on that document. So a mail is stored after its
/// last chunk, and deleting its document takes all of this along.
///
/// A mail is addressed by its id, which is the id of its document. Folders are addressed by their
/// path, and have to be stored before the mails which lie in them.
/// </remarks>
public abstract partial class IndexStoreClient
{
/// <summary>
/// Reads the folders of a mailbox and how far their sync got.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The folders, ordered by their path.</returns>
public abstract Task<IReadOnlyList<MailFolderRecord>> GetMailFoldersAsync(string dataSourceId, CancellationToken token);
/// <summary>
/// Stores a folder of a mailbox, or how far its sync got.
/// </summary>
/// <remarks>
/// A UIDVALIDITY other than the stored one voids every UID stored for the folder, so its
/// locations are dropped. Mails left without any location are orphaned, not deleted: the sync
/// finds them again under their new UIDs and links them anew, without embedding them again.
/// That is why a new UIDVALIDITY has to be stored before the first location under it, not
/// at the end of the pass: by then, it would drop the very locations the pass just linked.
/// </remarks>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="folder">The folder, identified by its path.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task UpsertMailFolderAsync(string dataSourceId, MailFolderRecord folder, CancellationToken token);
/// <summary>
/// Removes a folder which is gone from the server, or no longer part of the mailbox.
/// </summary>
/// <remarks>
/// Its mails are not deleted along with it, only orphaned when they lie nowhere else: they may
/// just have moved to a folder the sync gets to later.
/// </remarks>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="folderPath">The path of the folder.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task DeleteMailFolderAsync(string dataSourceId, string folderPath, CancellationToken token);
/// <summary>
/// Stores what the index keeps about a mail beyond its chunks, replacing what it kept before.
/// </summary>
/// <remarks>
/// Only ever after the last chunk of the mail: indexing a document deletes it first, and that
/// takes everything stored here along. When the mail was seen before, the earlier of both
/// first sightings stays. A UID names one mail only, so should the index still hold one of the
/// given locations for another mail, that location moves over, and the other mail is orphaned
/// when it lies nowhere else.
/// </remarks>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="mail">The mail, with at least one location in a stored folder.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task UpsertMailAsync(string dataSourceId, MailRecord mail, CancellationToken token);
/// <summary>
/// Reads everything the index keeps about a mail beyond its chunks.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="mailId">The id of the mail.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The mail, or null when the mailbox holds no such mail.</returns>
public abstract Task<MailRecord?> GetMailAsync(string dataSourceId, string mailId, CancellationToken token);
/// <summary>
/// Links a mail the index already holds to one more place on the server, without embedding it again.
/// </summary>
/// <remarks>
/// This is how a moved mail and a mail under a new UIDVALIDITY are taken care of. The mail stops
/// being an orphan, and a location held for another mail moves over as with UpsertMailAsync.
/// </remarks>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="mailId">The id of the mail.</param>
/// <param name="location">The place, in a stored folder.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>True when the mail was linked, false when the mailbox holds no such mail and it has to be indexed.</returns>
public abstract Task<bool> AddMailLocationAsync(string dataSourceId, string mailId, MailLocationRecord location, CancellationToken token);
/// <summary>
/// Forgets places on the server where mails no longer lie.
/// </summary>
/// <remarks>
/// Mails left without any location are orphaned, not deleted, cf. GetOrphanedMailsAsync.
/// </remarks>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="folderPath">The path of the folder.</param>
/// <param name="uids">The UIDs which are gone from that folder.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task RemoveMailLocationsAsync(string dataSourceId, string folderPath, IReadOnlyCollection<long> uids, CancellationToken token);
/// <summary>
/// Reads which UIDs of a folder the index holds, and their flags.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="folderPath">The path of the folder.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The flags by UID, empty when the folder is not stored.</returns>
public abstract Task<IReadOnlyDictionary<long, MailFlags>> GetMailLocationsAsync(string dataSourceId, string folderPath, CancellationToken token);
/// <summary>
/// Stores the flags of mails in one folder. Flags of UIDs the index does not hold are ignored.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="folderPath">The path of the folder.</param>
/// <param name="flagsByUid">The flags by UID.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task UpdateMailFlagsAsync(string dataSourceId, string folderPath, IReadOnlyDictionary<long, MailFlags> flagsByUid, CancellationToken token);
/// <summary>
/// Reads the mails which lost their last location before the given point in time.
/// </summary>
/// <remarks>
/// A moved mail disappears from one folder before it shows up in the other one, maybe only in
/// the next run. Asking with the start of the current run therefore leaves out every mail
/// which became an orphan during that run, and those get one more run to be found again.
/// </remarks>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="orphanedBefore">Only mails orphaned before this point in time.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The keys of their documents, to delete them like any other document.</returns>
public abstract Task<IReadOnlyList<string>> GetOrphanedMailsAsync(string dataSourceId, DateTimeOffset orphanedBefore, CancellationToken token);
/// <summary>
/// Reads how the sync of a mailbox as a whole stands.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The state; every value is null for a mailbox which was never synced.</returns>
public abstract Task<MailboxSyncState> GetMailboxSyncStateAsync(string dataSourceId, CancellationToken token);
/// <summary>
/// Holds back a sync which would remove many mails from the index at once, until the user decides.
/// </summary>
/// <remarks>
/// A count other than the one held back so far voids an earlier approval: the user agreed to
/// a number, not to whatever a later sync comes up with.
/// </remarks>
/// <param name="dataSourceId">The mailbox, which has to be stored as a data source.</param>
/// <param name="removalCount">How many mails the sync would remove from the index.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task HoldBackMailRemovalAsync(string dataSourceId, int removalCount, CancellationToken token);
/// <summary>
/// Records that the user agreed to the held back removal.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="removalCount">The count the user was shown and agreed to.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>True when exactly that removal was held back and is now approved. False when nothing or another count is held back by now, and the user has to be asked again.</returns>
public abstract Task<bool> ApprovePendingMailRemovalAsync(string dataSourceId, int removalCount, CancellationToken token);
/// <summary>
/// Records a complete sync of a mailbox. A removal held back before has been dealt with by then.
/// </summary>
/// <param name="dataSourceId">The mailbox, which has to be stored as a data source.</param>
/// <param name="completedUtc">When the sync ended.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task CompleteMailboxSyncAsync(string dataSourceId, DateTimeOffset completedUtc, CancellationToken token);
/// <summary>
/// Reads whether the server of a mailbox refused a sign-in which nobody has dealt with yet.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The refused sign-in, or null when AI Studio may sign in.</returns>
public abstract Task<MailboxAuthFailure?> GetMailboxAuthFailureAsync(string dataSourceId, CancellationToken token);
/// <summary>
/// Records a sign-in the server refused. From then on, AI Studio does not sign in on its own,
/// not even after a restart, until the failure is cleared.
/// </summary>
/// <remarks>
/// Kept apart from everything else about the mailbox, so it survives a rebuild of the index,
/// and stored for a mailbox which was never indexed as well: the very first sign-in can fail.
/// </remarks>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="failure">The refused sign-in.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task UpsertMailboxAuthFailureAsync(string dataSourceId, MailboxAuthFailure failure, CancellationToken token);
/// <summary>
/// Clears a refused sign-in, when the user saved a new password, asked for another try, or
/// deleted the mailbox.
/// </summary>
/// <param name="dataSourceId">The mailbox.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task ClearMailboxAuthFailureAsync(string dataSourceId, CancellationToken token);
/// <summary>
/// Lists every mailbox the index keeps something of: an index, a refused sign-in, or both.
/// </summary>
/// <remarks>
/// The refused sign-ins count on their own, since they outlive the index on purpose. What is
/// listed here for a mailbox which is no longer configured is left over, cf.
/// DataSourceEmbeddingService.DeleteOrphanedMailboxIndexesAsync.
/// </remarks>
/// <param name="token">The cancellation token.</param>
/// <returns>The ids of the mailboxes, each once.</returns>
public abstract Task<IReadOnlyCollection<string>> GetStoredMailboxIdsAsync(CancellationToken token);
}
@@ -2,7 +2,7 @@ using AIStudio.Tools.Services;
namespace AIStudio.Tools.Databases.IndexStore;
public abstract class IndexStoreClient(string name, string path) : DatabaseClient(name, path)
public abstract partial class IndexStoreClient(string name, string path) : DatabaseClient(name, path)
{
public abstract Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token);
@@ -19,6 +19,20 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
public DbSet<IndexStoreSearchResultEntity> SearchResults => this.Set<IndexStoreSearchResultEntity>();
public DbSet<MailMessageEntity> MailMessages => this.Set<MailMessageEntity>();
public DbSet<MailAddressEntity> MailAddresses => this.Set<MailAddressEntity>();
public DbSet<MailPartEntity> MailParts => this.Set<MailPartEntity>();
public DbSet<MailFolderEntity> MailFolders => this.Set<MailFolderEntity>();
public DbSet<MailLocationEntity> MailLocations => this.Set<MailLocationEntity>();
public DbSet<MailboxSyncStateEntity> MailboxSyncStates => this.Set<MailboxSyncStateEntity>();
public DbSet<MailboxAuthStateEntity> MailboxAuthStates => this.Set<MailboxAuthStateEntity>();
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
var utcDateTimeOffsetConverter = new IndexStoreDateTimeOffsetConverter();
@@ -124,6 +138,171 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
entity.Property(result => result.LastWriteUtc).HasConversion(utcDateTimeOffsetConverter);
entity.Property(result => result.EmbeddedAtUtc).HasConversion(utcDateTimeOffsetConverter);
});
modelBuilder.Entity<MailMessageEntity>(entity =>
{
entity.ToTable("mail_messages");
entity.HasKey(mail => mail.ParentFileId);
entity.Property(mail => mail.ParentFileId).HasColumnName("parent_file_id");
entity.Property(mail => mail.DataSourceId).HasColumnName("data_source_id").IsRequired();
entity.Property(mail => mail.MessageId).HasColumnName("message_id").IsRequired();
entity.Property(mail => mail.InReplyTo).HasColumnName("in_reply_to").IsRequired();
entity.Property(mail => mail.ReferenceMessageIds).HasColumnName("reference_message_ids").IsRequired();
entity.Property(mail => mail.SentAtUtc).HasColumnName("sent_at_utc").HasConversion(utcDateTimeOffsetConverter);
entity.Property(mail => mail.ReceivedAtUtc).HasColumnName("received_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
entity.Property(mail => mail.Importance).HasColumnName("importance").IsRequired();
entity.Property(mail => mail.EncryptionKind).HasColumnName("encryption_kind").IsRequired();
entity.Property(mail => mail.MailHash).HasColumnName("mail_hash").IsRequired();
entity.Property(mail => mail.FirstSeenUtc).HasColumnName("first_seen_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
entity.Property(mail => mail.OrphanedAtUtc).HasColumnName("orphaned_at_utc").HasConversion(utcDateTimeOffsetConverter);
entity.HasIndex(mail => new { mail.DataSourceId, mail.ReceivedAtUtc }).HasDatabaseName("idx_mail_messages_data_source_received");
entity.HasIndex(mail => new { mail.DataSourceId, mail.MessageId }).HasDatabaseName("idx_mail_messages_data_source_message_id");
entity
.HasOne<EmbeddingStateFileEntity>()
.WithOne()
.HasForeignKey<MailMessageEntity>(mail => mail.ParentFileId)
.OnDelete(DeleteBehavior.Cascade);
entity
.HasOne<EmbeddingStateDataSourceEntity>()
.WithMany()
.HasForeignKey(mail => mail.DataSourceId)
.OnDelete(DeleteBehavior.Cascade);
entity
.HasMany(mail => mail.Addresses)
.WithOne(address => address.Message)
.HasForeignKey(address => address.ParentFileId)
.OnDelete(DeleteBehavior.Cascade);
entity
.HasMany(mail => mail.Parts)
.WithOne(part => part.Message)
.HasForeignKey(part => part.ParentFileId)
.OnDelete(DeleteBehavior.Cascade);
entity
.HasMany(mail => mail.Locations)
.WithOne(location => location.Message)
.HasForeignKey(location => location.ParentFileId)
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity<MailAddressEntity>(entity =>
{
entity.ToTable("mail_addresses");
entity.HasKey(address => address.Id);
entity.Property(address => address.Id).HasColumnName("id").ValueGeneratedOnAdd();
entity.Property(address => address.ParentFileId).HasColumnName("parent_file_id").IsRequired();
entity.Property(address => address.Role).HasColumnName("role").IsRequired();
entity.Property(address => address.Position).HasColumnName("position");
entity.Property(address => address.Address).HasColumnName("address").UseCollation("NOCASE").IsRequired();
entity.Property(address => address.DisplayName).HasColumnName("display_name").IsRequired();
entity.HasIndex(address => new { address.ParentFileId, address.Role, address.Position }).HasDatabaseName("idx_mail_addresses_parent_file_role_position").IsUnique();
entity.HasIndex(address => address.Address).HasDatabaseName("idx_mail_addresses_address");
});
modelBuilder.Entity<MailPartEntity>(entity =>
{
entity.ToTable("mail_parts");
entity.HasKey(part => part.Id);
entity.Property(part => part.Id).HasColumnName("id").ValueGeneratedOnAdd();
entity.Property(part => part.ParentFileId).HasColumnName("parent_file_id").IsRequired();
entity.Property(part => part.Kind).HasColumnName("kind").IsRequired();
entity.Property(part => part.Position).HasColumnName("position");
entity.Property(part => part.Name).HasColumnName("name").IsRequired();
entity.Property(part => part.ContentType).HasColumnName("content_type").IsRequired();
entity.Property(part => part.PartSize).HasColumnName("part_size");
entity.Property(part => part.Text).HasColumnName("text");
entity.Property(part => part.TextState).HasColumnName("text_state").IsRequired();
entity.HasIndex(part => new { part.ParentFileId, part.Kind, part.Position }).HasDatabaseName("idx_mail_parts_parent_file_kind_position").IsUnique();
});
modelBuilder.Entity<MailFolderEntity>(entity =>
{
entity.ToTable("mail_folders");
entity.HasKey(folder => folder.Id);
entity.Property(folder => folder.Id).HasColumnName("id").ValueGeneratedOnAdd();
entity.Property(folder => folder.DataSourceId).HasColumnName("data_source_id").IsRequired();
entity.Property(folder => folder.Path).HasColumnName("path").IsRequired();
entity.Property(folder => folder.SpecialUse).HasColumnName("special_use").IsRequired();
entity.Property(folder => folder.UidValidity).HasColumnName("uid_validity");
entity.Property(folder => folder.UidNext).HasColumnName("uid_next");
entity.Property(folder => folder.HighestModSeq).HasColumnName("highest_mod_seq");
entity.Property(folder => folder.ServerMessageCount).HasColumnName("server_message_count");
entity.Property(folder => folder.ServerUnseenCount).HasColumnName("server_unseen_count");
entity.Property(folder => folder.InitialSyncCompletedUtc).HasColumnName("initial_sync_completed_utc").HasConversion(utcDateTimeOffsetConverter);
entity.HasIndex(folder => new { folder.DataSourceId, folder.Path }).HasDatabaseName("idx_mail_folders_data_source_path").IsUnique();
entity
.HasOne<EmbeddingStateDataSourceEntity>()
.WithMany()
.HasForeignKey(folder => folder.DataSourceId)
.OnDelete(DeleteBehavior.Cascade);
entity
.HasMany(folder => folder.Locations)
.WithOne(location => location.Folder)
.HasForeignKey(location => location.FolderId)
.OnDelete(DeleteBehavior.Cascade);
});
modelBuilder.Entity<MailLocationEntity>(entity =>
{
entity.ToTable("mail_locations");
entity.HasKey(location => location.Id);
entity.Property(location => location.Id).HasColumnName("id").ValueGeneratedOnAdd();
entity.Property(location => location.ParentFileId).HasColumnName("parent_file_id").IsRequired();
entity.Property(location => location.FolderId).HasColumnName("folder_id");
entity.Property(location => location.Uid).HasColumnName("uid");
entity.Property(location => location.IsSeen).HasColumnName("is_seen");
entity.Property(location => location.IsFlagged).HasColumnName("is_flagged");
entity.Property(location => location.IsAnswered).HasColumnName("is_answered");
entity.HasIndex(location => new { location.FolderId, location.Uid }).HasDatabaseName("idx_mail_locations_folder_uid").IsUnique();
entity.HasIndex(location => location.ParentFileId).HasDatabaseName("idx_mail_locations_parent_file");
});
modelBuilder.Entity<MailboxSyncStateEntity>(entity =>
{
entity.ToTable("mailbox_sync_state");
entity.HasKey(state => state.DataSourceId);
entity.Property(state => state.DataSourceId).HasColumnName("data_source_id");
entity.Property(state => state.LastSyncCompletedUtc).HasColumnName("last_sync_completed_utc").HasConversion(utcDateTimeOffsetConverter);
entity.Property(state => state.PendingRemovalCount).HasColumnName("pending_removal_count");
entity.Property(state => state.PendingRemovalApprovedUtc).HasColumnName("pending_removal_approved_utc").HasConversion(utcDateTimeOffsetConverter);
entity
.HasOne<EmbeddingStateDataSourceEntity>()
.WithOne()
.HasForeignKey<MailboxSyncStateEntity>(state => state.DataSourceId)
.OnDelete(DeleteBehavior.Cascade);
});
//
// Without any relationship on purpose, cf. MailboxAuthStateEntity: a refused sign-in has
// to outlive a rebuild of the index.
//
modelBuilder.Entity<MailboxAuthStateEntity>(entity =>
{
entity.ToTable("mailbox_auth_state");
entity.HasKey(state => state.DataSourceId);
entity.Property(state => state.DataSourceId).HasColumnName("data_source_id");
entity.Property(state => state.FailedAtUtc).HasColumnName("failed_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
entity.Property(state => state.FailureMessage).HasColumnName("failure_message").IsRequired();
});
}
private static string BuildConnectionString(string databasePath) => new SqliteConnectionStringBuilder
@@ -10,6 +10,7 @@ internal static class IndexStoreSchemaMigrator
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.PermanentIndexingFailures))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropDataSourceName))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.MailboxIndex))]
public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
{
await context.Database.MigrateAsync(token);
@@ -0,0 +1,30 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// One address from the header of a mail.
/// </summary>
internal sealed class MailAddressEntity
{
public int Id { get; set; }
public string ParentFileId { get; set; } = string.Empty;
/// <summary>
/// The header the address comes from: FROM, SENDER, REPLY_TO, TO, CC or BCC, stored by name.
/// </summary>
public string Role { get; set; } = string.Empty;
/// <summary>
/// The place of the address within its header, starting at zero.
/// </summary>
public int Position { get; set; }
public string Address { get; set; } = string.Empty;
/// <summary>
/// The name shown next to the address, empty when the header gives none.
/// </summary>
public string DisplayName { get; set; } = string.Empty;
public MailMessageEntity? Message { get; set; }
}
@@ -0,0 +1,11 @@
using AIStudio.Tools.Mail;
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// One address from the header of a mail.
/// </summary>
/// <param name="Role">The header the address comes from.</param>
/// <param name="Address">The address itself.</param>
/// <param name="DisplayName">The name shown next to it, empty when the header gives none.</param>
public sealed record MailAddressRecord(MailAddressRole Role, string Address, string DisplayName);
@@ -0,0 +1,9 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// The number of mails in one group, e.g., one folder or one sender.
/// </summary>
/// <param name="Key">The folder path or the sender address.</param>
/// <param name="DisplayName">A name the sender uses, empty for folders and for senders without one.</param>
/// <param name="Count">How many mails belong to the group.</param>
public sealed record MailCountGroup(string Key, string DisplayName, long Count);
@@ -0,0 +1,22 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// How to break down the number of mails in a mailbox.
/// </summary>
public enum MailCountGrouping
{
/// <summary>
/// Only the total.
/// </summary>
NONE,
/// <summary>
/// By the folders the mails lie in. A mail in two folders counts in both.
/// </summary>
FOLDER,
/// <summary>
/// By the address in the From header.
/// </summary>
SENDER,
}
@@ -0,0 +1,8 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// How many mails of a mailbox meet some conditions.
/// </summary>
/// <param name="TotalCount">How many mails meet them, each mail counted once.</param>
/// <param name="Groups">The largest groups, largest first; empty without a grouping.</param>
public sealed record MailCountResult(long TotalCount, IReadOnlyList<MailCountGroup> Groups);
@@ -0,0 +1,69 @@
using AIStudio.Tools.Mail;
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// Conditions a mail has to meet. A condition left out holds for every mail.
/// </summary>
/// <remarks>
/// A mail counts as read, flagged or answered when it is so in any of its folders: whoever read
/// the copy in the inbox read the mail, even when another copy in an archive still says unread.
/// Mails which lost their last place on the server never match, since they are gone or about to
/// show up somewhere else.
/// </remarks>
public sealed record MailFilter
{
/// <summary>
/// A part of the address or the name of the sender, as the From or Sender header gives it.
/// </summary>
public string? From { get; init; }
/// <summary>
/// A part of the address or the name of a recipient, as the To, Cc or Bcc header gives it.
/// </summary>
public string? To { get; init; }
/// <summary>
/// Only mails which arrived at the server at this point in time or later.
/// </summary>
public DateTimeOffset? ReceivedSinceUtc { get; init; }
/// <summary>
/// Only mails which arrived at the server before this point in time.
/// </summary>
public DateTimeOffset? ReceivedBeforeUtc { get; init; }
public bool? IsUnread { get; init; }
public bool? IsFlagged { get; init; }
public bool? IsEncrypted { get; init; }
public MailImportance? Importance { get; init; }
public bool? HasAttachments { get; init; }
/// <summary>
/// Only mails which lie in one of these folders, given by their full paths.
/// </summary>
/// <remarks>
/// An empty collection matches no mail at all. It never turns into "any folder": a caller
/// whose folder names matched no folder must not get the whole mailbox back.
/// </remarks>
public IReadOnlyCollection<string>? FolderPaths { get; init; }
/// <summary>
/// Whether any condition is set at all.
/// </summary>
public bool HasConditions =>
!string.IsNullOrWhiteSpace(this.From)
|| !string.IsNullOrWhiteSpace(this.To)
|| this.ReceivedSinceUtc is not null
|| this.ReceivedBeforeUtc is not null
|| this.IsUnread is not null
|| this.IsFlagged is not null
|| this.IsEncrypted is not null
|| this.Importance is not null
|| this.HasAttachments is not null
|| this.FolderPaths is not null;
}
@@ -0,0 +1,9 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// The flags of a mail at one of its locations.
/// </summary>
/// <param name="IsSeen">Whether the mail was read (\Seen).</param>
/// <param name="IsFlagged">Whether the mail is flagged (\Flagged).</param>
/// <param name="IsAnswered">Whether the mail was answered (\Answered).</param>
public readonly record struct MailFlags(bool IsSeen, bool IsFlagged, bool IsAnswered);
@@ -0,0 +1,58 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// One folder of a mailbox and how far its sync got.
/// </summary>
internal sealed class MailFolderEntity
{
public int Id { get; set; }
public string DataSourceId { get; set; } = string.Empty;
/// <summary>
/// The full path of the folder, as the server names it.
/// </summary>
public string Path { get; set; } = string.Empty;
/// <summary>
/// The special use the server announces for the folder, e.g., SENT, or NONE, stored by name.
/// </summary>
public string SpecialUse { get; set; } = string.Empty;
/// <summary>
/// The UIDVALIDITY the stored locations of this folder belong to.
/// </summary>
/// <remarks>
/// When the server reports another one, every UID stored for this folder is void.
/// </remarks>
public long UidValidity { get; set; }
/// <summary>
/// The UIDNEXT at the end of the last complete pass over the folder, or null until one completed.
/// </summary>
public long? UidNext { get; set; }
/// <summary>
/// The HIGHESTMODSEQ at the end of the last complete pass over the folder, or null until one
/// completed, or when the server does not support CONDSTORE.
/// </summary>
public long? HighestModSeq { get; set; }
/// <summary>
/// How many mails the folder holds on the server (STATUS MESSAGES), or null when not read yet.
/// </summary>
public long? ServerMessageCount { get; set; }
/// <summary>
/// How many of them are unread (STATUS UNSEEN), or null when not read yet.
/// </summary>
public long? ServerUnseenCount { get; set; }
/// <summary>
/// When every mail of the folder which belongs into the index was in it for the first time, or
/// null while that first pass is still running.
/// </summary>
public DateTimeOffset? InitialSyncCompletedUtc { get; set; }
public List<MailLocationEntity> Locations { get; set; } = [];
}
@@ -0,0 +1,24 @@
using AIStudio.Tools.Mail;
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// One folder of a mailbox and how far its sync got.
/// </summary>
/// <param name="Path">The full path of the folder, as the server names it.</param>
/// <param name="SpecialUse">What the server says the folder is for.</param>
/// <param name="UidValidity">The UIDVALIDITY the stored locations of this folder belong to.</param>
/// <param name="UidNext">The UIDNEXT at the end of the last complete pass, or null until one completed.</param>
/// <param name="HighestModSeq">The HIGHESTMODSEQ at the end of the last complete pass, or null until one completed or without CONDSTORE.</param>
/// <param name="ServerMessageCount">How many mails the folder holds on the server, or null when not read yet.</param>
/// <param name="ServerUnseenCount">How many of them are unread, or null when not read yet.</param>
/// <param name="InitialSyncCompletedUtc">When the first pass over the folder completed, or null while it runs.</param>
public sealed record MailFolderRecord(
string Path,
MailFolderSpecialUse SpecialUse,
long UidValidity,
long? UidNext,
long? HighestModSeq,
long? ServerMessageCount,
long? ServerUnseenCount,
DateTimeOffset? InitialSyncCompletedUtc);
@@ -0,0 +1,33 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// One place on the server where a mail lies.
/// </summary>
/// <remarks>
/// A mail can lie in more than one folder, and moving it changes only its locations, never its
/// embedding. The flags are kept here, not with the mail, because IMAP keeps them per folder: a
/// copy in another folder has flags of its own.
/// </remarks>
internal sealed class MailLocationEntity
{
public int Id { get; set; }
public string ParentFileId { get; set; } = string.Empty;
public int FolderId { get; set; }
/// <summary>
/// The UID of the mail within its folder, valid for the UIDVALIDITY of that folder.
/// </summary>
public long Uid { get; set; }
public bool IsSeen { get; set; }
public bool IsFlagged { get; set; }
public bool IsAnswered { get; set; }
public MailMessageEntity? Message { get; set; }
public MailFolderEntity? Folder { get; set; }
}
@@ -0,0 +1,9 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// One place on the server where a mail lies.
/// </summary>
/// <param name="FolderPath">The full path of the folder, as the server names it.</param>
/// <param name="Uid">The UID of the mail within that folder.</param>
/// <param name="Flags">The flags of the mail there.</param>
public sealed record MailLocationRecord(string FolderPath, long Uid, MailFlags Flags);
@@ -0,0 +1,87 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// What the index knows about one mail beyond its text, one row per mail and mailbox.
/// </summary>
/// <remarks>
/// The mail itself is a row in embedded_files, whose primary key this row shares. Indexing a mail
/// again deletes that row first, and the cascade takes this one with it. So this row is written
/// after the last chunk of a mail, never before.
/// </remarks>
internal sealed class MailMessageEntity
{
public string ParentFileId { get; set; } = string.Empty;
public string DataSourceId { get; set; } = string.Empty;
/// <summary>
/// The Message-ID header, empty when the mail has none.
/// </summary>
public string MessageId { get; set; } = string.Empty;
/// <summary>
/// The Message-ID the In-Reply-To header names, empty when there is none.
/// </summary>
public string InReplyTo { get; set; } = string.Empty;
/// <summary>
/// The Message-IDs of the References header, oldest first and separated by spaces, the way the
/// header itself lists them. A Message-ID cannot contain a space, so nothing gets lost.
/// </summary>
public string ReferenceMessageIds { get; set; } = string.Empty;
/// <summary>
/// When the sender says the mail was written (Date header), or null when it does not say so readably.
/// </summary>
public DateTimeOffset? SentAtUtc { get; set; }
/// <summary>
/// When the mail arrived at the server (INTERNALDATE).
/// </summary>
/// <remarks>
/// Unlike the Date header, the sender has no say in this one. It is also the date IMAP compares
/// against when the time range of a mailbox gets searched.
/// </remarks>
public DateTimeOffset ReceivedAtUtc { get; set; }
/// <summary>
/// LOW, NORMAL or HIGH, stored by name.
/// </summary>
public string Importance { get; set; } = string.Empty;
/// <summary>
/// How the content of the mail is encrypted, NONE when it is not, stored by name.
/// </summary>
/// <remarks>
/// There is no separate flag for whether a mail is encrypted: it would only be a second copy of
/// this column, free to disagree with it. A kind written by a newer version still is not NONE,
/// so such a mail still counts as encrypted.
/// </remarks>
public string EncryptionKind { get; set; } = string.Empty;
/// <summary>
/// A hash over the mail as the server delivered it, independent of how AI Studio turns it into text.
/// </summary>
public string MailHash { get; set; } = string.Empty;
/// <summary>
/// When AI Studio found this mail for the first time.
/// </summary>
public DateTimeOffset FirstSeenUtc { get; set; }
/// <summary>
/// When the mail lost its last location, or null while it has one.
/// </summary>
/// <remarks>
/// A mail which is moved disappears from one folder before it shows up in the other one, maybe
/// only in the next run. Removing it at once would embed it again when it reappears, so it stays
/// for one more run.
/// </remarks>
public DateTimeOffset? OrphanedAtUtc { get; set; }
public List<MailAddressEntity> Addresses { get; set; } = [];
public List<MailPartEntity> Parts { get; set; } = [];
public List<MailLocationEntity> Locations { get; set; } = [];
}
@@ -0,0 +1,49 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// One part of a mail as AI Studio read it: the header block, the body or an attachment.
/// </summary>
/// <remarks>
/// The chunks hold the same text, but cut into overlapping pieces. Reading a mail as a whole, or
/// checking its header block, needs it in one piece, and without asking the server again.
/// </remarks>
internal sealed class MailPartEntity
{
public int Id { get; set; }
public string ParentFileId { get; set; } = string.Empty;
/// <summary>
/// HEADERS, BODY or ATTACHMENT, stored by name.
/// </summary>
public string Kind { get; set; } = string.Empty;
/// <summary>
/// The place of the part among the parts of its kind, starting at zero.
/// </summary>
public int Position { get; set; }
/// <summary>
/// The file name of an attachment, empty for the other parts.
/// </summary>
public string Name { get; set; } = string.Empty;
public string ContentType { get; set; } = string.Empty;
/// <summary>
/// The size of the part on the server, in bytes.
/// </summary>
public long PartSize { get; set; }
/// <summary>
/// The text of the part, or null when there is none. TextState says why.
/// </summary>
public string? Text { get; set; }
/// <summary>
/// Whether the text could be read and, when not, why, stored by name.
/// </summary>
public string TextState { get; set; } = string.Empty;
public MailMessageEntity? Message { get; set; }
}
@@ -0,0 +1,14 @@
using AIStudio.Tools.Mail;
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// One part of a mail as AI Studio read it.
/// </summary>
/// <param name="Kind">The header block, the body or an attachment.</param>
/// <param name="Name">The file name of an attachment, empty for the other parts.</param>
/// <param name="ContentType">The content type of the part.</param>
/// <param name="PartSize">The size of the part on the server, in bytes.</param>
/// <param name="Text">The text of the part, or null when there is none.</param>
/// <param name="TextState">Whether the text could be read and, when not, why.</param>
public sealed record MailPartRecord(MailPartKind Kind, string Name, string ContentType, long PartSize, string? Text, MailPartTextState TextState);
@@ -0,0 +1,38 @@
using AIStudio.Tools.Mail;
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// What the index keeps about one mail beyond its chunks.
/// </summary>
/// <remarks>
/// The addresses come back grouped by their role and the parts by their kind, each group in the
/// order it was stored in. The locations come back ordered by folder and UID.
/// </remarks>
/// <param name="MailId">The id of the mail, which is the id of its document.</param>
/// <param name="MessageId">The Message-ID header, empty when the mail has none.</param>
/// <param name="InReplyTo">The Message-ID the In-Reply-To header names, empty when there is none.</param>
/// <param name="ReferenceMessageIds">The Message-IDs of the References header, oldest first.</param>
/// <param name="SentAtUtc">When the sender says the mail was written, or null when it does not say so readably.</param>
/// <param name="ReceivedAtUtc">When the mail arrived at the server.</param>
/// <param name="Importance">How important the sender marked the mail.</param>
/// <param name="EncryptionKind">How the content of the mail is encrypted, NONE when it is not.</param>
/// <param name="MailHash">A hash over the mail as the server delivered it.</param>
/// <param name="FirstSeenUtc">When AI Studio found the mail for the first time.</param>
/// <param name="Addresses">The addresses from the header of the mail.</param>
/// <param name="Parts">The parts of the mail as AI Studio read them.</param>
/// <param name="Locations">Where the mail lies on the server, at least one place.</param>
public sealed record MailRecord(
string MailId,
string MessageId,
string InReplyTo,
IReadOnlyList<string> ReferenceMessageIds,
DateTimeOffset? SentAtUtc,
DateTimeOffset ReceivedAtUtc,
MailImportance Importance,
MailEncryptionKind EncryptionKind,
string MailHash,
DateTimeOffset FirstSeenUtc,
IReadOnlyList<MailAddressRecord> Addresses,
IReadOnlyList<MailPartRecord> Parts,
IReadOnlyList<MailLocationRecord> Locations);
@@ -0,0 +1,32 @@
using AIStudio.Tools.Mail;
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// What a list of mails shows about one of them, without its text.
/// </summary>
/// <param name="MailId">The id of the mail.</param>
/// <param name="Subject">The subject of the mail.</param>
/// <param name="ReceivedAtUtc">When the mail arrived at the server.</param>
/// <param name="SentAtUtc">When the sender says the mail was written, or null when it does not say so readably.</param>
/// <param name="MessageId">The Message-ID header, empty when the mail has none.</param>
/// <param name="InReplyTo">The Message-ID the In-Reply-To header names, empty when there is none.</param>
/// <param name="Addresses">The addresses from the header, grouped by their role.</param>
/// <param name="FolderPaths">The folders the mail lies in, ordered by path.</param>
/// <param name="Flags">The flags of the mail; each one is set when it is set in any folder.</param>
/// <param name="Importance">How important the sender marked the mail.</param>
/// <param name="EncryptionKind">How the content of the mail is encrypted, NONE when it is not.</param>
/// <param name="AttachmentNames">The file names of the attachments, in their order.</param>
public sealed record MailSummary(
string MailId,
string Subject,
DateTimeOffset ReceivedAtUtc,
DateTimeOffset? SentAtUtc,
string MessageId,
string InReplyTo,
IReadOnlyList<MailAddressRecord> Addresses,
IReadOnlyList<string> FolderPaths,
MailFlags Flags,
MailImportance Importance,
MailEncryptionKind EncryptionKind,
IReadOnlyList<string> AttachmentNames);
@@ -0,0 +1,29 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// A sign-in the server of a mailbox refused.
/// </summary>
/// <param name="FailedAtUtc">When the server refused the sign-in.</param>
/// <param name="FailureMessage">What the server answered, empty when it gave no reason.</param>
public sealed record MailboxAuthFailure(DateTimeOffset FailedAtUtc, string FailureMessage)
{
/// <summary>
/// How much of the answer of the server is kept. The reason is in the first sentence, if at all.
/// </summary>
public const int MAX_FAILURE_MESSAGE_LENGTH = 500;
/// <summary>
/// Records a sign-in the server refused just now, together with what it answered.
/// </summary>
/// <remarks>
/// The answer often says why, e.g., that the provider requires an app password. It may name
/// the user as well, which is why it is kept for the settings of the mailbox and never logged.
/// </remarks>
/// <param name="serverAnswer">What the server answered.</param>
/// <returns>The refused sign-in.</returns>
public static MailboxAuthFailure FromServerAnswer(string serverAnswer)
{
var trimmedAnswer = serverAnswer.Trim();
return new(DateTimeOffset.UtcNow, trimmedAnswer.Length > MAX_FAILURE_MESSAGE_LENGTH ? trimmedAnswer[..MAX_FAILURE_MESSAGE_LENGTH] : trimmedAnswer);
}
}
@@ -0,0 +1,21 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// A sign-in the server refused. While the row exists, AI Studio does not try to sign in again.
/// </summary>
/// <remarks>
/// The row has no foreign key on purpose. Rebuilding the index deletes the data source row and
/// everything which cascades from it, and a refused sign-in has to survive that: otherwise every
/// rebuild would try the refused password again, which is how a directory account gets locked.
/// </remarks>
internal sealed class MailboxAuthStateEntity
{
public string DataSourceId { get; set; } = string.Empty;
public DateTimeOffset FailedAtUtc { get; set; }
/// <summary>
/// What the server answered, empty when it gave no reason.
/// </summary>
public string FailureMessage { get; set; } = string.Empty;
}
@@ -0,0 +1,9 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// How the sync of a mailbox as a whole stands.
/// </summary>
/// <param name="LastSyncCompletedUtc">When the last complete sync ended, or null until one did.</param>
/// <param name="PendingRemovalCount">How many mails a sync would remove from the index at once, held back until the user decides, or null when nothing is held back.</param>
/// <param name="PendingRemovalApprovedUtc">When the user agreed to exactly that removal, or null while they did not.</param>
public sealed record MailboxSyncState(DateTimeOffset? LastSyncCompletedUtc, int? PendingRemovalCount, DateTimeOffset? PendingRemovalApprovedUtc);
@@ -0,0 +1,29 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// How the sync of a mailbox as a whole stands.
/// </summary>
internal sealed class MailboxSyncStateEntity
{
public string DataSourceId { get; set; } = string.Empty;
/// <summary>
/// When the last complete sync of the mailbox ended, or null until one did.
/// </summary>
public DateTimeOffset? LastSyncCompletedUtc { get; set; }
/// <summary>
/// How many mails a sync would remove from the index at once, held back until the user decides,
/// or null when nothing is held back.
/// </summary>
/// <remarks>
/// Nothing on the server is touched either way. But mails removed from the index have to be
/// embedded again should they come back, which costs time and, with a cloud provider, money.
/// </remarks>
public int? PendingRemovalCount { get; set; }
/// <summary>
/// When the user agreed to the held back removal, or null while they did not.
/// </summary>
public DateTimeOffset? PendingRemovalApprovedUtc { get; set; }
}
@@ -0,0 +1,251 @@
#nullable disable
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
/// <summary>
/// Adds what the index keeps about mailboxes beyond the text of their mails.
/// </summary>
/// <remarks>
/// Every mail is a row in embedded_files like any document, so search, chunks and full-text index
/// stay the same for both. These tables hold what only a mail has: its addresses, its parts, the
/// folders and places it lies in, and how far the sync of each mailbox got. All of them go with
/// their data source, except mailbox_auth_state: a refused sign-in has to survive a rebuild.
/// </remarks>
[DbContext(typeof(IndexStoreDbContext))]
[Migration("20261001000000_MailboxIndex")]
public partial class MailboxIndex : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "mail_messages",
columns: table => new
{
parent_file_id = table.Column<string>(type: "TEXT", nullable: false),
data_source_id = table.Column<string>(type: "TEXT", nullable: false),
message_id = table.Column<string>(type: "TEXT", nullable: false),
in_reply_to = table.Column<string>(type: "TEXT", nullable: false),
reference_message_ids = table.Column<string>(type: "TEXT", nullable: false),
sent_at_utc = table.Column<string>(type: "TEXT", nullable: true),
received_at_utc = table.Column<string>(type: "TEXT", nullable: false),
importance = table.Column<string>(type: "TEXT", nullable: false),
encryption_kind = table.Column<string>(type: "TEXT", nullable: false),
mail_hash = table.Column<string>(type: "TEXT", nullable: false),
first_seen_utc = table.Column<string>(type: "TEXT", nullable: false),
orphaned_at_utc = table.Column<string>(type: "TEXT", nullable: true),
},
constraints: table =>
{
table.PrimaryKey("PK_mail_messages", mail => mail.parent_file_id);
table.ForeignKey(
name: "FK_mail_messages_data_sources_data_source_id",
column: mail => mail.data_source_id,
principalTable: "data_sources",
principalColumn: "data_source_id",
onDelete: ReferentialAction.Cascade);
table.ForeignKey(
name: "FK_mail_messages_embedded_files_parent_file_id",
column: mail => mail.parent_file_id,
principalTable: "embedded_files",
principalColumn: "parent_file_id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateTable(
name: "mail_folders",
columns: table => new
{
id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
data_source_id = table.Column<string>(type: "TEXT", nullable: false),
path = table.Column<string>(type: "TEXT", nullable: false),
special_use = table.Column<string>(type: "TEXT", nullable: false),
uid_validity = table.Column<long>(type: "INTEGER", nullable: false),
uid_next = table.Column<long>(type: "INTEGER", nullable: true),
highest_mod_seq = table.Column<long>(type: "INTEGER", nullable: true),
server_message_count = table.Column<long>(type: "INTEGER", nullable: true),
server_unseen_count = table.Column<long>(type: "INTEGER", nullable: true),
initial_sync_completed_utc = table.Column<string>(type: "TEXT", nullable: true),
},
constraints: table =>
{
table.PrimaryKey("PK_mail_folders", folder => folder.id);
table.ForeignKey(
name: "FK_mail_folders_data_sources_data_source_id",
column: folder => folder.data_source_id,
principalTable: "data_sources",
principalColumn: "data_source_id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateTable(
name: "mail_addresses",
columns: table => new
{
id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
parent_file_id = table.Column<string>(type: "TEXT", nullable: false),
role = table.Column<string>(type: "TEXT", nullable: false),
position = table.Column<int>(type: "INTEGER", nullable: false),
address = table.Column<string>(type: "TEXT", nullable: false, collation: "NOCASE"),
display_name = table.Column<string>(type: "TEXT", nullable: false),
},
constraints: table =>
{
table.PrimaryKey("PK_mail_addresses", address => address.id);
table.ForeignKey(
name: "FK_mail_addresses_mail_messages_parent_file_id",
column: address => address.parent_file_id,
principalTable: "mail_messages",
principalColumn: "parent_file_id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateTable(
name: "mail_parts",
columns: table => new
{
id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
parent_file_id = table.Column<string>(type: "TEXT", nullable: false),
kind = table.Column<string>(type: "TEXT", nullable: false),
position = table.Column<int>(type: "INTEGER", nullable: false),
name = table.Column<string>(type: "TEXT", nullable: false),
content_type = table.Column<string>(type: "TEXT", nullable: false),
part_size = table.Column<long>(type: "INTEGER", nullable: false),
text = table.Column<string>(type: "TEXT", nullable: true),
text_state = table.Column<string>(type: "TEXT", nullable: false),
},
constraints: table =>
{
table.PrimaryKey("PK_mail_parts", part => part.id);
table.ForeignKey(
name: "FK_mail_parts_mail_messages_parent_file_id",
column: part => part.parent_file_id,
principalTable: "mail_messages",
principalColumn: "parent_file_id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateTable(
name: "mail_locations",
columns: table => new
{
id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
parent_file_id = table.Column<string>(type: "TEXT", nullable: false),
folder_id = table.Column<int>(type: "INTEGER", nullable: false),
uid = table.Column<long>(type: "INTEGER", nullable: false),
is_seen = table.Column<bool>(type: "INTEGER", nullable: false),
is_flagged = table.Column<bool>(type: "INTEGER", nullable: false),
is_answered = table.Column<bool>(type: "INTEGER", nullable: false),
},
constraints: table =>
{
table.PrimaryKey("PK_mail_locations", location => location.id);
table.ForeignKey(
name: "FK_mail_locations_mail_folders_folder_id",
column: location => location.folder_id,
principalTable: "mail_folders",
principalColumn: "id",
onDelete: ReferentialAction.Cascade);
table.ForeignKey(
name: "FK_mail_locations_mail_messages_parent_file_id",
column: location => location.parent_file_id,
principalTable: "mail_messages",
principalColumn: "parent_file_id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateTable(
name: "mailbox_sync_state",
columns: table => new
{
data_source_id = table.Column<string>(type: "TEXT", nullable: false),
last_sync_completed_utc = table.Column<string>(type: "TEXT", nullable: true),
pending_removal_count = table.Column<int>(type: "INTEGER", nullable: true),
pending_removal_approved_utc = table.Column<string>(type: "TEXT", nullable: true),
},
constraints: table =>
{
table.PrimaryKey("PK_mailbox_sync_state", state => state.data_source_id);
table.ForeignKey(
name: "FK_mailbox_sync_state_data_sources_data_source_id",
column: state => state.data_source_id,
principalTable: "data_sources",
principalColumn: "data_source_id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateTable(
name: "mailbox_auth_state",
columns: table => new
{
data_source_id = table.Column<string>(type: "TEXT", nullable: false),
failed_at_utc = table.Column<string>(type: "TEXT", nullable: false),
failure_message = table.Column<string>(type: "TEXT", nullable: false),
},
constraints: table =>
{
table.PrimaryKey("PK_mailbox_auth_state", state => state.data_source_id);
});
migrationBuilder.CreateIndex(
name: "idx_mail_messages_data_source_message_id",
table: "mail_messages",
columns: ["data_source_id", "message_id"]);
migrationBuilder.CreateIndex(
name: "idx_mail_messages_data_source_received",
table: "mail_messages",
columns: ["data_source_id", "received_at_utc"]);
migrationBuilder.CreateIndex(
name: "idx_mail_folders_data_source_path",
table: "mail_folders",
columns: ["data_source_id", "path"],
unique: true);
migrationBuilder.CreateIndex(
name: "idx_mail_addresses_address",
table: "mail_addresses",
column: "address");
migrationBuilder.CreateIndex(
name: "idx_mail_addresses_parent_file_role_position",
table: "mail_addresses",
columns: ["parent_file_id", "role", "position"],
unique: true);
migrationBuilder.CreateIndex(
name: "idx_mail_parts_parent_file_kind_position",
table: "mail_parts",
columns: ["parent_file_id", "kind", "position"],
unique: true);
migrationBuilder.CreateIndex(
name: "idx_mail_locations_folder_uid",
table: "mail_locations",
columns: ["folder_id", "uid"],
unique: true);
migrationBuilder.CreateIndex(
name: "idx_mail_locations_parent_file",
table: "mail_locations",
column: "parent_file_id");
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(name: "mailbox_auth_state");
migrationBuilder.DropTable(name: "mailbox_sync_state");
migrationBuilder.DropTable(name: "mail_locations");
migrationBuilder.DropTable(name: "mail_parts");
migrationBuilder.DropTable(name: "mail_addresses");
migrationBuilder.DropTable(name: "mail_folders");
migrationBuilder.DropTable(name: "mail_messages");
}
}
@@ -315,6 +315,325 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
entity.ToView("embedding_chunk_search_results");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailAddressEntity", entity =>
{
entity.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id")
.HasAnnotation("Sqlite:Autoincrement", true);
entity.Property<string>("Address")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("address")
.UseCollation("NOCASE");
entity.Property<string>("DisplayName")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("display_name");
entity.Property<string>("ParentFileId")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("parent_file_id");
entity.Property<int>("Position")
.HasColumnType("INTEGER")
.HasColumnName("position");
entity.Property<string>("Role")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("role");
entity.HasKey("Id");
entity.HasIndex("Address")
.HasDatabaseName("idx_mail_addresses_address");
entity.HasIndex("ParentFileId", "Role", "Position")
.IsUnique()
.HasDatabaseName("idx_mail_addresses_parent_file_role_position");
entity.ToTable("mail_addresses");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailFolderEntity", entity =>
{
entity.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id")
.HasAnnotation("Sqlite:Autoincrement", true);
entity.Property<string>("DataSourceId")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("data_source_id");
entity.Property<long?>("HighestModSeq")
.HasColumnType("INTEGER")
.HasColumnName("highest_mod_seq");
entity.Property<DateTimeOffset?>("InitialSyncCompletedUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
.HasColumnName("initial_sync_completed_utc");
entity.Property<string>("Path")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("path");
entity.Property<long?>("ServerMessageCount")
.HasColumnType("INTEGER")
.HasColumnName("server_message_count");
entity.Property<long?>("ServerUnseenCount")
.HasColumnType("INTEGER")
.HasColumnName("server_unseen_count");
entity.Property<string>("SpecialUse")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("special_use");
entity.Property<long?>("UidNext")
.HasColumnType("INTEGER")
.HasColumnName("uid_next");
entity.Property<long>("UidValidity")
.HasColumnType("INTEGER")
.HasColumnName("uid_validity");
entity.HasKey("Id");
entity.HasIndex("DataSourceId", "Path")
.IsUnique()
.HasDatabaseName("idx_mail_folders_data_source_path");
entity.ToTable("mail_folders");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailLocationEntity", entity =>
{
entity.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id")
.HasAnnotation("Sqlite:Autoincrement", true);
entity.Property<int>("FolderId")
.HasColumnType("INTEGER")
.HasColumnName("folder_id");
entity.Property<bool>("IsAnswered")
.HasColumnType("INTEGER")
.HasColumnName("is_answered");
entity.Property<bool>("IsFlagged")
.HasColumnType("INTEGER")
.HasColumnName("is_flagged");
entity.Property<bool>("IsSeen")
.HasColumnType("INTEGER")
.HasColumnName("is_seen");
entity.Property<string>("ParentFileId")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("parent_file_id");
entity.Property<long>("Uid")
.HasColumnType("INTEGER")
.HasColumnName("uid");
entity.HasKey("Id");
entity.HasIndex("ParentFileId")
.HasDatabaseName("idx_mail_locations_parent_file");
entity.HasIndex("FolderId", "Uid")
.IsUnique()
.HasDatabaseName("idx_mail_locations_folder_uid");
entity.ToTable("mail_locations");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailMessageEntity", entity =>
{
entity.Property<string>("ParentFileId")
.HasColumnType("TEXT")
.HasColumnName("parent_file_id");
entity.Property<string>("DataSourceId")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("data_source_id");
entity.Property<string>("EncryptionKind")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("encryption_kind");
entity.Property<DateTimeOffset>("FirstSeenUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
.HasColumnName("first_seen_utc");
entity.Property<string>("Importance")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("importance");
entity.Property<string>("InReplyTo")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("in_reply_to");
entity.Property<string>("MailHash")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("mail_hash");
entity.Property<string>("MessageId")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("message_id");
entity.Property<DateTimeOffset?>("OrphanedAtUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
.HasColumnName("orphaned_at_utc");
entity.Property<DateTimeOffset>("ReceivedAtUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
.HasColumnName("received_at_utc");
entity.Property<string>("ReferenceMessageIds")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("reference_message_ids");
entity.Property<DateTimeOffset?>("SentAtUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
.HasColumnName("sent_at_utc");
entity.HasKey("ParentFileId");
entity.HasIndex("DataSourceId", "MessageId")
.HasDatabaseName("idx_mail_messages_data_source_message_id");
entity.HasIndex("DataSourceId", "ReceivedAtUtc")
.HasDatabaseName("idx_mail_messages_data_source_received");
entity.ToTable("mail_messages");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailPartEntity", entity =>
{
entity.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasColumnName("id")
.HasAnnotation("Sqlite:Autoincrement", true);
entity.Property<string>("ContentType")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("content_type");
entity.Property<string>("Kind")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("kind");
entity.Property<string>("Name")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("name");
entity.Property<string>("ParentFileId")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("parent_file_id");
entity.Property<long>("PartSize")
.HasColumnType("INTEGER")
.HasColumnName("part_size");
entity.Property<int>("Position")
.HasColumnType("INTEGER")
.HasColumnName("position");
entity.Property<string>("Text")
.HasColumnType("TEXT")
.HasColumnName("text");
entity.Property<string>("TextState")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("text_state");
entity.HasKey("Id");
entity.HasIndex("ParentFileId", "Kind", "Position")
.IsUnique()
.HasDatabaseName("idx_mail_parts_parent_file_kind_position");
entity.ToTable("mail_parts");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailboxAuthStateEntity", entity =>
{
entity.Property<string>("DataSourceId")
.HasColumnType("TEXT")
.HasColumnName("data_source_id");
entity.Property<DateTimeOffset>("FailedAtUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
.HasColumnName("failed_at_utc");
entity.Property<string>("FailureMessage")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("failure_message");
entity.HasKey("DataSourceId");
entity.ToTable("mailbox_auth_state");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailboxSyncStateEntity", entity =>
{
entity.Property<string>("DataSourceId")
.HasColumnType("TEXT")
.HasColumnName("data_source_id");
entity.Property<DateTimeOffset?>("LastSyncCompletedUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
.HasColumnName("last_sync_completed_utc");
entity.Property<DateTimeOffset?>("PendingRemovalApprovedUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
.HasColumnName("pending_removal_approved_utc");
entity.Property<int?>("PendingRemovalCount")
.HasColumnType("INTEGER")
.HasColumnName("pending_removal_count");
entity.HasKey("DataSourceId");
entity.ToTable("mailbox_sync_state");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", entity =>
{
entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", "DataSource")
@@ -348,6 +667,80 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
entity.Navigation("DataSource");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailAddressEntity", entity =>
{
entity.HasOne("AIStudio.Tools.Databases.IndexStore.MailMessageEntity", "Message")
.WithMany("Addresses")
.HasForeignKey("ParentFileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
entity.Navigation("Message");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailFolderEntity", entity =>
{
entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", null)
.WithMany()
.HasForeignKey("DataSourceId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailLocationEntity", entity =>
{
entity.HasOne("AIStudio.Tools.Databases.IndexStore.MailFolderEntity", "Folder")
.WithMany("Locations")
.HasForeignKey("FolderId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
entity.HasOne("AIStudio.Tools.Databases.IndexStore.MailMessageEntity", "Message")
.WithMany("Locations")
.HasForeignKey("ParentFileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
entity.Navigation("Folder");
entity.Navigation("Message");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailMessageEntity", entity =>
{
entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", null)
.WithMany()
.HasForeignKey("DataSourceId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateFileEntity", null)
.WithOne()
.HasForeignKey("AIStudio.Tools.Databases.IndexStore.MailMessageEntity", "ParentFileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailPartEntity", entity =>
{
entity.HasOne("AIStudio.Tools.Databases.IndexStore.MailMessageEntity", "Message")
.WithMany("Parts")
.HasForeignKey("ParentFileId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
entity.Navigation("Message");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailboxSyncStateEntity", entity =>
{
entity.HasOne("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", null)
.WithOne()
.HasForeignKey("AIStudio.Tools.Databases.IndexStore.MailboxSyncStateEntity", "DataSourceId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.EmbeddingStateDataSourceEntity", entity =>
{
entity.Navigation("Files");
@@ -359,6 +752,20 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
{
entity.Navigation("Chunks");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailFolderEntity", entity =>
{
entity.Navigation("Locations");
});
modelBuilder.Entity("AIStudio.Tools.Databases.IndexStore.MailMessageEntity", entity =>
{
entity.Navigation("Addresses");
entity.Navigation("Locations");
entity.Navigation("Parts");
});
#pragma warning restore 612, 618
}
}
@@ -0,0 +1,17 @@
namespace AIStudio.Tools.Databases.IndexStore;
public sealed partial class NoIndexStoreClient
{
public override Task<IReadOnlyList<string>> QueryMailsAsync(string dataSourceId, MailFilter filter, int offset, int limit, CancellationToken token) => Task.FromResult<IReadOnlyList<string>>([]);
public override Task<IReadOnlyList<string>> GetMailChunkIdsAsync(string dataSourceId, MailFilter filter, CancellationToken token) => Task.FromResult<IReadOnlyList<string>>([]);
public override Task<IReadOnlyList<IndexStoreSearchResult>> SearchMailChunksAsync(string dataSourceId, string query, MailFilter filter, int maxMatches, CancellationToken token) =>
Task.FromResult<IReadOnlyList<IndexStoreSearchResult>>([]);
public override Task<MailCountResult> CountMailsAsync(string dataSourceId, MailFilter filter, MailCountGrouping grouping, int maxGroups, CancellationToken token) => Task.FromResult(new MailCountResult(0, []));
public override Task<IReadOnlyList<MailSummary>> GetMailSummariesAsync(string dataSourceId, IReadOnlyList<string> mailIds, CancellationToken token) => Task.FromResult<IReadOnlyList<MailSummary>>([]);
public override Task<string?> FindMailByMessageIdAsync(string dataSourceId, string messageId, CancellationToken token) => Task.FromResult<string?>(null);
}
@@ -0,0 +1,41 @@
namespace AIStudio.Tools.Databases.IndexStore;
public sealed partial class NoIndexStoreClient
{
public override Task<IReadOnlyList<MailFolderRecord>> GetMailFoldersAsync(string dataSourceId, CancellationToken token) => Task.FromResult<IReadOnlyList<MailFolderRecord>>([]);
public override Task UpsertMailFolderAsync(string dataSourceId, MailFolderRecord folder, CancellationToken token) => Task.CompletedTask;
public override Task DeleteMailFolderAsync(string dataSourceId, string folderPath, CancellationToken token) => Task.CompletedTask;
public override Task UpsertMailAsync(string dataSourceId, MailRecord mail, CancellationToken token) => Task.CompletedTask;
public override Task<MailRecord?> GetMailAsync(string dataSourceId, string mailId, CancellationToken token) => Task.FromResult<MailRecord?>(null);
public override Task<bool> AddMailLocationAsync(string dataSourceId, string mailId, MailLocationRecord location, CancellationToken token) => Task.FromResult(false);
public override Task RemoveMailLocationsAsync(string dataSourceId, string folderPath, IReadOnlyCollection<long> uids, CancellationToken token) => Task.CompletedTask;
public override Task<IReadOnlyDictionary<long, MailFlags>> GetMailLocationsAsync(string dataSourceId, string folderPath, CancellationToken token) =>
Task.FromResult<IReadOnlyDictionary<long, MailFlags>>(new Dictionary<long, MailFlags>());
public override Task UpdateMailFlagsAsync(string dataSourceId, string folderPath, IReadOnlyDictionary<long, MailFlags> flagsByUid, CancellationToken token) => Task.CompletedTask;
public override Task<IReadOnlyList<string>> GetOrphanedMailsAsync(string dataSourceId, DateTimeOffset orphanedBefore, CancellationToken token) => Task.FromResult<IReadOnlyList<string>>([]);
public override Task<MailboxSyncState> GetMailboxSyncStateAsync(string dataSourceId, CancellationToken token) => Task.FromResult(new MailboxSyncState(null, null, null));
public override Task HoldBackMailRemovalAsync(string dataSourceId, int removalCount, CancellationToken token) => Task.CompletedTask;
public override Task<bool> ApprovePendingMailRemovalAsync(string dataSourceId, int removalCount, CancellationToken token) => Task.FromResult(false);
public override Task CompleteMailboxSyncAsync(string dataSourceId, DateTimeOffset completedUtc, CancellationToken token) => Task.CompletedTask;
public override Task<MailboxAuthFailure?> GetMailboxAuthFailureAsync(string dataSourceId, CancellationToken token) => Task.FromResult<MailboxAuthFailure?>(null);
public override Task UpsertMailboxAuthFailureAsync(string dataSourceId, MailboxAuthFailure failure, CancellationToken token) => Task.CompletedTask;
public override Task ClearMailboxAuthFailureAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask;
public override Task<IReadOnlyCollection<string>> GetStoredMailboxIdsAsync(CancellationToken token) => Task.FromResult<IReadOnlyCollection<string>>([]);
}
@@ -3,7 +3,7 @@ using AIStudio.Tools.Services;
namespace AIStudio.Tools.Databases.IndexStore;
public sealed class NoIndexStoreClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : IndexStoreClient(name, string.Empty)
public sealed partial class NoIndexStoreClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : IndexStoreClient(name, string.Empty)
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoIndexStoreClient).Namespace, nameof(NoIndexStoreClient));
@@ -0,0 +1,254 @@
using AIStudio.Tools.Mail;
using Microsoft.EntityFrameworkCore;
namespace AIStudio.Tools.Databases.IndexStore;
public sealed partial class SqliteIndexStoreClientImplementation
{
private const string LIKE_ESCAPE = "\\";
private const string FROM_ROLE = nameof(MailAddressRole.FROM);
private const string ATTACHMENT_KIND = nameof(MailPartKind.ATTACHMENT);
private const string NOT_ENCRYPTED = nameof(MailEncryptionKind.NONE);
private static readonly string[] SENDER_ROLES = [nameof(MailAddressRole.FROM), nameof(MailAddressRole.SENDER)];
private static readonly string[] RECIPIENT_ROLES = [nameof(MailAddressRole.TO), nameof(MailAddressRole.CC), nameof(MailAddressRole.BCC)];
public override async Task<IReadOnlyList<string>> QueryMailsAsync(string dataSourceId, MailFilter filter, int offset, int limit, CancellationToken token)
{
ArgumentOutOfRangeException.ThrowIfNegative(offset);
if (limit <= 0)
return [];
await using var context = this.CreateContext();
return await FilterMails(context, dataSourceId, filter)
.OrderByDescending(mail => mail.ReceivedAtUtc)
.ThenBy(mail => mail.ParentFileId)
.Skip(offset)
.Take(limit)
.Select(mail => mail.ParentFileId)
.ToListAsync(token);
}
public override async Task<IReadOnlyList<string>> GetMailChunkIdsAsync(string dataSourceId, MailFilter filter, CancellationToken token)
{
await using var context = this.CreateContext();
var mailIds = FilterMails(context, dataSourceId, filter).Select(mail => mail.ParentFileId);
return await context.EmbeddingChunks
.AsNoTracking()
.Where(chunk => mailIds.Contains(chunk.ParentFileId))
.Select(chunk => chunk.ChunkId)
.ToListAsync(token);
}
public override async Task<IReadOnlyList<IndexStoreSearchResult>> SearchMailChunksAsync(string dataSourceId, string query, MailFilter filter, int maxMatches, CancellationToken token)
{
if (maxMatches <= 0)
return [];
var ftsQuery = BuildFtsQuery(query);
if (string.IsNullOrWhiteSpace(ftsQuery))
return [];
await using var context = this.CreateContext();
var mailIds = FilterMails(context, dataSourceId, filter).Select(mail => mail.ParentFileId);
var results = await InSearchOrder(MatchChunks(context, dataSourceId, ftsQuery).Where(result => mailIds.Contains(result.ParentFileId)))
.Take(maxMatches)
.ToListAsync(token);
return results.Select(ToSearchResult).ToList();
}
public override async Task<MailCountResult> CountMailsAsync(string dataSourceId, MailFilter filter, MailCountGrouping grouping, int maxGroups, CancellationToken token)
{
await using var context = this.CreateContext();
var mails = FilterMails(context, dataSourceId, filter);
var totalCount = await mails.LongCountAsync(token);
if (totalCount == 0 || maxGroups <= 0 || grouping is MailCountGrouping.NONE)
return new MailCountResult(totalCount, []);
//
// A mail can lie twice in one folder, as two copies under two UIDs, and it can name the same
// sender in From twice. Either way it is one mail, so the groups count distinct mails:
//
var mailIds = mails.Select(mail => mail.ParentFileId);
var groupQuery = grouping switch
{
MailCountGrouping.FOLDER => context.MailLocations
.Where(location => mailIds.Contains(location.ParentFileId))
.GroupBy(location => location.Folder!.Path)
.Select(group => new { group.Key, DisplayName = (string?)string.Empty, Count = group.Select(location => location.ParentFileId).Distinct().LongCount() }),
MailCountGrouping.SENDER => context.MailAddresses
.Where(address => address.Role == FROM_ROLE && mailIds.Contains(address.ParentFileId))
.GroupBy(address => address.Address)
.Select(group => new { group.Key, DisplayName = group.Max(address => address.DisplayName), Count = group.Select(address => address.ParentFileId).Distinct().LongCount() }),
_ => throw new ArgumentOutOfRangeException(nameof(grouping), grouping, "There is no such grouping."),
};
var groups = await groupQuery
.OrderByDescending(group => group.Count)
.ThenBy(group => group.Key)
.Take(maxGroups)
.ToListAsync(token);
return new MailCountResult(totalCount, groups.Select(group => new MailCountGroup(group.Key, group.DisplayName ?? string.Empty, group.Count)).ToList());
}
public override async Task<IReadOnlyList<MailSummary>> GetMailSummariesAsync(string dataSourceId, IReadOnlyList<string> mailIds, CancellationToken token)
{
if (mailIds.Count == 0)
return [];
var wantedIds = mailIds.Distinct(StringComparer.Ordinal).ToArray();
await using var context = this.CreateContext();
var mails = await context.MailMessages
.AsNoTracking()
.Where(mail => mail.DataSourceId == dataSourceId && mail.OrphanedAtUtc == null && wantedIds.Contains(mail.ParentFileId))
.ToDictionaryAsync(mail => mail.ParentFileId, StringComparer.Ordinal, token);
if (mails.Count == 0)
return [];
var foundIds = mails.Keys.ToArray();
var subjects = await context.EmbeddedFiles
.AsNoTracking()
.Where(file => foundIds.Contains(file.ParentFileId))
.ToDictionaryAsync(file => file.ParentFileId, file => file.FileName, StringComparer.Ordinal, token);
var addresses = (await context.MailAddresses
.AsNoTracking()
.Where(address => foundIds.Contains(address.ParentFileId))
.ToListAsync(token))
.ToLookup(address => address.ParentFileId, StringComparer.Ordinal);
var locations = (await context.MailLocations
.AsNoTracking()
.Where(location => foundIds.Contains(location.ParentFileId))
.Select(location => new { location.ParentFileId, location.Folder!.Path, location.IsSeen, location.IsFlagged, location.IsAnswered })
.ToListAsync(token))
.ToLookup(location => location.ParentFileId, StringComparer.Ordinal);
var attachmentNames = (await context.MailParts
.AsNoTracking()
.Where(part => foundIds.Contains(part.ParentFileId) && part.Kind == ATTACHMENT_KIND)
.OrderBy(part => part.Position)
.Select(part => new { part.ParentFileId, part.Name })
.ToListAsync(token))
.ToLookup(part => part.ParentFileId, part => part.Name, StringComparer.Ordinal);
return wantedIds
.Where(mails.ContainsKey)
.Select(mailId =>
{
var mail = mails[mailId];
var mailLocations = locations[mailId].ToList();
return new MailSummary(
mailId,
subjects.GetValueOrDefault(mailId, string.Empty),
mail.ReceivedAtUtc,
mail.SentAtUtc,
mail.MessageId,
mail.InReplyTo,
ToMailAddressRecords(addresses[mailId]),
mailLocations.Select(location => location.Path).Distinct(StringComparer.Ordinal).Order(StringComparer.Ordinal).ToList(),
new MailFlags(mailLocations.Any(location => location.IsSeen), mailLocations.Any(location => location.IsFlagged), mailLocations.Any(location => location.IsAnswered)),
ParseStoredName(mail.Importance, MailImportance.NORMAL),
ParseStoredName(mail.EncryptionKind, MailEncryptionKind.UNKNOWN),
attachmentNames[mailId].ToList());
})
.ToList();
}
public override async Task<string?> FindMailByMessageIdAsync(string dataSourceId, string messageId, CancellationToken token)
{
if (string.IsNullOrWhiteSpace(messageId))
return null;
await using var context = this.CreateContext();
return await context.MailMessages
.AsNoTracking()
.Where(mail => mail.DataSourceId == dataSourceId && mail.OrphanedAtUtc == null && mail.MessageId == messageId)
.OrderBy(mail => mail.ReceivedAtUtc)
.ThenBy(mail => mail.ParentFileId)
.Select(mail => mail.ParentFileId)
.FirstOrDefaultAsync(token);
}
/// <summary>
/// The mails of a mailbox which meet the conditions, as a query to build on.
/// </summary>
private static IQueryable<MailMessageEntity> FilterMails(IndexStoreDbContext context, string dataSourceId, MailFilter filter)
{
var mails = context.MailMessages
.AsNoTracking()
.Where(mail => mail.DataSourceId == dataSourceId && mail.OrphanedAtUtc == null);
if (!string.IsNullOrWhiteSpace(filter.From))
{
var pattern = ToContainsPattern(filter.From);
mails = mails.Where(mail => mail.Addresses.Any(address => SENDER_ROLES.Contains(address.Role) && (EF.Functions.Like(address.Address, pattern, LIKE_ESCAPE) || EF.Functions.Like(address.DisplayName, pattern, LIKE_ESCAPE))));
}
if (!string.IsNullOrWhiteSpace(filter.To))
{
var pattern = ToContainsPattern(filter.To);
mails = mails.Where(mail => mail.Addresses.Any(address => RECIPIENT_ROLES.Contains(address.Role) && (EF.Functions.Like(address.Address, pattern, LIKE_ESCAPE) || EF.Functions.Like(address.DisplayName, pattern, LIKE_ESCAPE))));
}
if (filter.ReceivedSinceUtc is { } receivedSince)
mails = mails.Where(mail => mail.ReceivedAtUtc >= receivedSince);
if (filter.ReceivedBeforeUtc is { } receivedBefore)
mails = mails.Where(mail => mail.ReceivedAtUtc < receivedBefore);
if (filter.IsUnread is { } isUnread)
mails = isUnread
? mails.Where(mail => !mail.Locations.Any(location => location.IsSeen))
: mails.Where(mail => mail.Locations.Any(location => location.IsSeen));
if (filter.IsFlagged is { } isFlagged)
mails = isFlagged
? mails.Where(mail => mail.Locations.Any(location => location.IsFlagged))
: mails.Where(mail => !mail.Locations.Any(location => location.IsFlagged));
if (filter.IsEncrypted is { } isEncrypted)
mails = isEncrypted
? mails.Where(mail => mail.EncryptionKind != NOT_ENCRYPTED)
: mails.Where(mail => mail.EncryptionKind == NOT_ENCRYPTED);
if (filter.Importance is { } importance)
{
var importanceName = importance.ToString();
mails = mails.Where(mail => mail.Importance == importanceName);
}
if (filter.HasAttachments is { } hasAttachments)
mails = hasAttachments
? mails.Where(mail => mail.Parts.Any(part => part.Kind == ATTACHMENT_KIND))
: mails.Where(mail => !mail.Parts.Any(part => part.Kind == ATTACHMENT_KIND));
if (filter.FolderPaths is { } folderPaths)
{
var paths = folderPaths.ToArray();
mails = mails.Where(mail => mail.Locations.Any(location => paths.Contains(location.Folder!.Path)));
}
return mails;
}
/// <summary>
/// A LIKE pattern which finds the text anywhere, with any % or _ in it taken literally.
/// </summary>
private static string ToContainsPattern(string text)
{
var escaped = text.Trim()
.Replace(LIKE_ESCAPE, LIKE_ESCAPE + LIKE_ESCAPE, StringComparison.Ordinal)
.Replace("%", LIKE_ESCAPE + "%", StringComparison.Ordinal)
.Replace("_", LIKE_ESCAPE + "_", StringComparison.Ordinal);
return $"%{escaped}%";
}
}
@@ -0,0 +1,532 @@
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Mail;
using Microsoft.EntityFrameworkCore;
namespace AIStudio.Tools.Databases.IndexStore;
public sealed partial class SqliteIndexStoreClientImplementation
{
/// <summary>
/// The type a mailbox is stored under in data_sources, as the embedding service writes it.
/// </summary>
private const string MAILBOX_DATA_SOURCE_TYPE = nameof(DataSourceType.MAILBOX);
public override async Task<IReadOnlyList<MailFolderRecord>> GetMailFoldersAsync(string dataSourceId, CancellationToken token)
{
await using var context = this.CreateContext();
var folders = await context.MailFolders
.AsNoTracking()
.Where(folder => folder.DataSourceId == dataSourceId)
.OrderBy(folder => folder.Path)
.ToListAsync(token);
return folders.Select(ToMailFolderRecord).ToList();
}
public override async Task UpsertMailFolderAsync(string dataSourceId, MailFolderRecord folder, CancellationToken token)
{
await using var context = this.CreateContext();
await using var transaction = await context.Database.BeginTransactionAsync(token);
var folderEntity = await context.MailFolders.FirstOrDefaultAsync(entity => entity.DataSourceId == dataSourceId && entity.Path == folder.Path, token);
if (folderEntity is null)
{
folderEntity = new MailFolderEntity
{
DataSourceId = dataSourceId,
Path = folder.Path,
};
context.MailFolders.Add(folderEntity);
}
else if (folderEntity.UidValidity != folder.UidValidity)
{
var folderId = folderEntity.Id;
await DropMailLocationsAsync(context, context.MailLocations.Where(location => location.FolderId == folderId), token);
}
ApplyMailFolder(folderEntity, folder);
await context.SaveChangesAsync(token);
await transaction.CommitAsync(token);
}
public override async Task DeleteMailFolderAsync(string dataSourceId, string folderPath, CancellationToken token)
{
await using var context = this.CreateContext();
await using var transaction = await context.Database.BeginTransactionAsync(token);
if (await FindMailFolderIdAsync(context, dataSourceId, folderPath, token) is not { } folderId)
return;
await DropMailLocationsAsync(context, context.MailLocations.Where(location => location.FolderId == folderId), token);
await context.MailFolders.Where(folder => folder.Id == folderId).ExecuteDeleteAsync(token);
await transaction.CommitAsync(token);
}
public override async Task UpsertMailAsync(string dataSourceId, MailRecord mail, CancellationToken token)
{
ValidateMail(mail);
await using var context = this.CreateContext();
await using var transaction = await context.Database.BeginTransactionAsync(token);
var folderIds = await GetMailFolderIdsAsync(context, dataSourceId, mail.Locations, token);
var mailEntity = await context.MailMessages.FirstOrDefaultAsync(entity => entity.ParentFileId == mail.MailId, token);
if (mailEntity is null)
{
mailEntity = new MailMessageEntity
{
ParentFileId = mail.MailId,
FirstSeenUtc = mail.FirstSeenUtc,
};
context.MailMessages.Add(mailEntity);
}
else
{
await context.MailAddresses.Where(address => address.ParentFileId == mail.MailId).ExecuteDeleteAsync(token);
await context.MailParts.Where(part => part.ParentFileId == mail.MailId).ExecuteDeleteAsync(token);
await context.MailLocations.Where(location => location.ParentFileId == mail.MailId).ExecuteDeleteAsync(token);
}
ApplyMail(mailEntity, dataSourceId, mail);
context.MailAddresses.AddRange(ToMailAddressEntities(mail));
context.MailParts.AddRange(ToMailPartEntities(mail));
var displacedMailIds = await ClaimMailLocationsAsync(context, mail.MailId, mail.Locations, folderIds, token);
await context.SaveChangesAsync(token);
await OrphanMailsWithoutLocationsAsync(context, displacedMailIds, token);
await transaction.CommitAsync(token);
}
public override async Task<MailRecord?> GetMailAsync(string dataSourceId, string mailId, CancellationToken token)
{
await using var context = this.CreateContext();
var mail = await context.MailMessages
.AsNoTracking()
.AsSplitQuery()
.Include(entity => entity.Addresses)
.Include(entity => entity.Parts)
.Include(entity => entity.Locations)
.ThenInclude(location => location.Folder)
.FirstOrDefaultAsync(entity => entity.DataSourceId == dataSourceId && entity.ParentFileId == mailId, token);
return mail is null ? null : ToMailRecord(mail);
}
public override async Task<bool> AddMailLocationAsync(string dataSourceId, string mailId, MailLocationRecord location, CancellationToken token)
{
await using var context = this.CreateContext();
await using var transaction = await context.Database.BeginTransactionAsync(token);
var mailEntity = await context.MailMessages.FirstOrDefaultAsync(entity => entity.DataSourceId == dataSourceId && entity.ParentFileId == mailId, token);
if (mailEntity is null)
return false;
var folderIds = await GetMailFolderIdsAsync(context, dataSourceId, [location], token);
var displacedMailIds = await ClaimMailLocationsAsync(context, mailId, [location], folderIds, token);
mailEntity.OrphanedAtUtc = null;
await context.SaveChangesAsync(token);
await OrphanMailsWithoutLocationsAsync(context, displacedMailIds, token);
await transaction.CommitAsync(token);
return true;
}
public override async Task RemoveMailLocationsAsync(string dataSourceId, string folderPath, IReadOnlyCollection<long> uids, CancellationToken token)
{
if (uids.Count == 0)
return;
await using var context = this.CreateContext();
await using var transaction = await context.Database.BeginTransactionAsync(token);
if (await FindMailFolderIdAsync(context, dataSourceId, folderPath, token) is not { } folderId)
return;
foreach (var uidBatch in uids.Chunk(CHUNK_UPSERT_BATCH_SIZE))
await DropMailLocationsAsync(context, context.MailLocations.Where(location => location.FolderId == folderId && uidBatch.Contains(location.Uid)), token);
await transaction.CommitAsync(token);
}
public override async Task<IReadOnlyDictionary<long, MailFlags>> GetMailLocationsAsync(string dataSourceId, string folderPath, CancellationToken token)
{
await using var context = this.CreateContext();
return await context.MailLocations
.AsNoTracking()
.Where(location => location.Folder!.DataSourceId == dataSourceId && location.Folder.Path == folderPath)
.ToDictionaryAsync(location => location.Uid, location => new MailFlags(location.IsSeen, location.IsFlagged, location.IsAnswered), token);
}
public override async Task UpdateMailFlagsAsync(string dataSourceId, string folderPath, IReadOnlyDictionary<long, MailFlags> flagsByUid, CancellationToken token)
{
if (flagsByUid.Count == 0)
return;
await using var context = this.CreateContext();
await using var transaction = await context.Database.BeginTransactionAsync(token);
if (await FindMailFolderIdAsync(context, dataSourceId, folderPath, token) is not { } folderId)
return;
foreach (var uidBatch in flagsByUid.Keys.Chunk(CHUNK_UPSERT_BATCH_SIZE))
{
token.ThrowIfCancellationRequested();
var locations = await context.MailLocations
.Where(location => location.FolderId == folderId && uidBatch.Contains(location.Uid))
.ToListAsync(token);
foreach (var location in locations)
{
var flags = flagsByUid[location.Uid];
location.IsSeen = flags.IsSeen;
location.IsFlagged = flags.IsFlagged;
location.IsAnswered = flags.IsAnswered;
}
await context.SaveChangesAsync(token);
context.ChangeTracker.Clear();
}
await transaction.CommitAsync(token);
}
public override async Task<IReadOnlyList<string>> GetOrphanedMailsAsync(string dataSourceId, DateTimeOffset orphanedBefore, CancellationToken token)
{
await using var context = this.CreateContext();
return await context.MailMessages
.AsNoTracking()
.Where(mail => mail.DataSourceId == dataSourceId && mail.OrphanedAtUtc != null && mail.OrphanedAtUtc < orphanedBefore)
.Join(context.EmbeddedFiles, mail => mail.ParentFileId, file => file.ParentFileId, (_, file) => file.AbsolutePath)
.ToListAsync(token);
}
public override async Task<MailboxSyncState> GetMailboxSyncStateAsync(string dataSourceId, CancellationToken token)
{
await using var context = this.CreateContext();
var state = await context.MailboxSyncStates
.AsNoTracking()
.FirstOrDefaultAsync(entity => entity.DataSourceId == dataSourceId, token);
return state is null
? new MailboxSyncState(null, null, null)
: new MailboxSyncState(state.LastSyncCompletedUtc, state.PendingRemovalCount, state.PendingRemovalApprovedUtc);
}
public override async Task HoldBackMailRemovalAsync(string dataSourceId, int removalCount, CancellationToken token)
{
ArgumentOutOfRangeException.ThrowIfNegativeOrZero(removalCount);
await using var context = this.CreateContext();
var state = await GetOrAddMailboxSyncStateAsync(context, dataSourceId, token);
if (state.PendingRemovalCount != removalCount)
{
state.PendingRemovalCount = removalCount;
state.PendingRemovalApprovedUtc = null;
}
await context.SaveChangesAsync(token);
}
public override async Task<bool> ApprovePendingMailRemovalAsync(string dataSourceId, int removalCount, CancellationToken token)
{
//
// One statement, so a sync cannot change the held back count between reading and writing
// it. Then the user would agree to a number they were never shown.
//
DateTimeOffset? approvedUtc = DateTimeOffset.UtcNow;
await using var context = this.CreateContext();
var approved = await context.MailboxSyncStates
.Where(state => state.DataSourceId == dataSourceId && state.PendingRemovalCount == removalCount)
.ExecuteUpdateAsync(setters => setters.SetProperty(state => state.PendingRemovalApprovedUtc, approvedUtc), token);
return approved > 0;
}
public override async Task CompleteMailboxSyncAsync(string dataSourceId, DateTimeOffset completedUtc, CancellationToken token)
{
await using var context = this.CreateContext();
var state = await GetOrAddMailboxSyncStateAsync(context, dataSourceId, token);
state.LastSyncCompletedUtc = completedUtc;
state.PendingRemovalCount = null;
state.PendingRemovalApprovedUtc = null;
await context.SaveChangesAsync(token);
}
public override async Task<MailboxAuthFailure?> GetMailboxAuthFailureAsync(string dataSourceId, CancellationToken token)
{
await using var context = this.CreateContext();
return await context.MailboxAuthStates
.AsNoTracking()
.Where(state => state.DataSourceId == dataSourceId)
.Select(state => new MailboxAuthFailure(state.FailedAtUtc, state.FailureMessage))
.FirstOrDefaultAsync(token);
}
public override async Task UpsertMailboxAuthFailureAsync(string dataSourceId, MailboxAuthFailure failure, CancellationToken token)
{
await using var context = this.CreateContext();
var state = await context.MailboxAuthStates.FirstOrDefaultAsync(entity => entity.DataSourceId == dataSourceId, token);
if (state is null)
{
state = new MailboxAuthStateEntity
{
DataSourceId = dataSourceId,
};
context.MailboxAuthStates.Add(state);
}
state.FailedAtUtc = failure.FailedAtUtc;
state.FailureMessage = failure.FailureMessage;
await context.SaveChangesAsync(token);
}
public override async Task ClearMailboxAuthFailureAsync(string dataSourceId, CancellationToken token)
{
await using var context = this.CreateContext();
await context.MailboxAuthStates
.Where(state => state.DataSourceId == dataSourceId)
.ExecuteDeleteAsync(token);
}
public override async Task<IReadOnlyCollection<string>> GetStoredMailboxIdsAsync(CancellationToken token)
{
await using var context = this.CreateContext();
var indexedIds = await context.DataSources
.AsNoTracking()
.Where(dataSource => dataSource.DataSourceType == MAILBOX_DATA_SOURCE_TYPE)
.Select(dataSource => dataSource.DataSourceId)
.ToListAsync(token);
var refusedIds = await context.MailboxAuthStates
.AsNoTracking()
.Select(state => state.DataSourceId)
.ToListAsync(token);
return indexedIds.Concat(refusedIds).ToHashSet(StringComparer.OrdinalIgnoreCase);
}
private static async Task<MailboxSyncStateEntity> GetOrAddMailboxSyncStateAsync(IndexStoreDbContext context, string dataSourceId, CancellationToken token)
{
var state = await context.MailboxSyncStates.FirstOrDefaultAsync(entity => entity.DataSourceId == dataSourceId, token);
if (state is not null)
return state;
state = new MailboxSyncStateEntity
{
DataSourceId = dataSourceId,
};
context.MailboxSyncStates.Add(state);
return state;
}
private static void ValidateMail(MailRecord mail)
{
if (mail.Locations.Count == 0)
throw new ArgumentException("A mail is only stored together with at least one place where it lies.", nameof(mail));
if (mail.Locations.DistinctBy(location => (location.FolderPath, location.Uid)).Count() != mail.Locations.Count)
throw new ArgumentException("A mail cannot lie at the same place twice.", nameof(mail));
//
// The Message-IDs are stored separated by spaces, which a Message-ID can never contain. One
// which does anyway would come back as two, so it is refused instead of being stored:
//
if (mail.ReferenceMessageIds.Any(messageId => messageId.Length == 0 || messageId.Any(char.IsWhiteSpace)))
throw new ArgumentException("A Message-ID can neither be empty nor contain white space.", nameof(mail));
}
private static async Task<int?> FindMailFolderIdAsync(IndexStoreDbContext context, string dataSourceId, string folderPath, CancellationToken token) =>
await context.MailFolders
.Where(folder => folder.DataSourceId == dataSourceId && folder.Path == folderPath)
.Select(folder => (int?)folder.Id)
.FirstOrDefaultAsync(token);
private static async Task<IReadOnlyDictionary<string, int>> GetMailFolderIdsAsync(IndexStoreDbContext context, string dataSourceId, IEnumerable<MailLocationRecord> locations, CancellationToken token)
{
var folderPaths = locations.Select(location => location.FolderPath).Distinct(StringComparer.Ordinal).ToArray();
var folderIds = await context.MailFolders
.Where(folder => folder.DataSourceId == dataSourceId && folderPaths.Contains(folder.Path))
.ToDictionaryAsync(folder => folder.Path, folder => folder.Id, StringComparer.Ordinal, token);
//
// The message names neither the folder nor the mailbox: folder names can tell a lot about
// a person, and this message may well end up in the log.
//
if (folderPaths.Any(folderPath => !folderIds.ContainsKey(folderPath)))
throw new InvalidOperationException("A mail lies in a folder the index does not hold for its mailbox. A folder has to be stored before the mails in it.");
return folderIds;
}
/// <summary>
/// Gives a mail the given places, taking them away from any other mail which still holds one.
/// </summary>
/// <returns>The ids of the mails which lost a place, to orphan those left without any.</returns>
private static async Task<IReadOnlyList<string>> ClaimMailLocationsAsync(IndexStoreDbContext context, string mailId, IEnumerable<MailLocationRecord> locations, IReadOnlyDictionary<string, int> folderIds, CancellationToken token)
{
var displacedMailIds = new List<string>();
foreach (var location in locations)
{
var folderId = folderIds[location.FolderPath];
var uid = location.Uid;
var heldLocations = context.MailLocations.Where(existing => existing.FolderId == folderId && existing.Uid == uid);
displacedMailIds.AddRange(await heldLocations.Select(existing => existing.ParentFileId).ToListAsync(token));
await heldLocations.ExecuteDeleteAsync(token);
context.MailLocations.Add(new MailLocationEntity
{
ParentFileId = mailId,
FolderId = folderId,
Uid = uid,
IsSeen = location.Flags.IsSeen,
IsFlagged = location.Flags.IsFlagged,
IsAnswered = location.Flags.IsAnswered,
});
}
return displacedMailIds;
}
/// <summary>
/// Removes the given places and orphans the mails which are left without any.
/// </summary>
private static async Task DropMailLocationsAsync(IndexStoreDbContext context, IQueryable<MailLocationEntity> locations, CancellationToken token)
{
var affectedMailIds = await locations.Select(location => location.ParentFileId).Distinct().ToListAsync(token);
await locations.ExecuteDeleteAsync(token);
await OrphanMailsWithoutLocationsAsync(context, affectedMailIds, token);
}
private static async Task OrphanMailsWithoutLocationsAsync(IndexStoreDbContext context, IReadOnlyCollection<string> mailIds, CancellationToken token)
{
if (mailIds.Count == 0)
return;
DateTimeOffset? orphanedAtUtc = DateTimeOffset.UtcNow;
foreach (var mailIdBatch in mailIds.Distinct(StringComparer.Ordinal).Chunk(CHUNK_UPSERT_BATCH_SIZE))
await context.MailMessages
.Where(mail => mailIdBatch.Contains(mail.ParentFileId) && mail.OrphanedAtUtc == null && !mail.Locations.Any())
.ExecuteUpdateAsync(setters => setters.SetProperty(mail => mail.OrphanedAtUtc, orphanedAtUtc), token);
}
private static void ApplyMailFolder(MailFolderEntity folderEntity, MailFolderRecord folder)
{
folderEntity.SpecialUse = folder.SpecialUse.ToString();
folderEntity.UidValidity = folder.UidValidity;
folderEntity.UidNext = folder.UidNext;
folderEntity.HighestModSeq = folder.HighestModSeq;
folderEntity.ServerMessageCount = folder.ServerMessageCount;
folderEntity.ServerUnseenCount = folder.ServerUnseenCount;
folderEntity.InitialSyncCompletedUtc = folder.InitialSyncCompletedUtc;
}
private static void ApplyMail(MailMessageEntity mailEntity, string dataSourceId, MailRecord mail)
{
mailEntity.DataSourceId = dataSourceId;
mailEntity.MessageId = mail.MessageId;
mailEntity.InReplyTo = mail.InReplyTo;
mailEntity.ReferenceMessageIds = string.Join(' ', mail.ReferenceMessageIds);
mailEntity.SentAtUtc = mail.SentAtUtc;
mailEntity.ReceivedAtUtc = mail.ReceivedAtUtc;
mailEntity.Importance = mail.Importance.ToString();
mailEntity.EncryptionKind = mail.EncryptionKind.ToString();
mailEntity.MailHash = mail.MailHash;
mailEntity.OrphanedAtUtc = null;
if (mail.FirstSeenUtc < mailEntity.FirstSeenUtc)
mailEntity.FirstSeenUtc = mail.FirstSeenUtc;
}
private static IEnumerable<MailAddressEntity> ToMailAddressEntities(MailRecord mail) =>
NumberWithinGroups(mail.Addresses, address => address.Role).Select(numbered => new MailAddressEntity
{
ParentFileId = mail.MailId,
Role = numbered.Item.Role.ToString(),
Position = numbered.Position,
Address = numbered.Item.Address,
DisplayName = numbered.Item.DisplayName,
});
private static IEnumerable<MailPartEntity> ToMailPartEntities(MailRecord mail) =>
NumberWithinGroups(mail.Parts, part => part.Kind).Select(numbered => new MailPartEntity
{
ParentFileId = mail.MailId,
Kind = numbered.Item.Kind.ToString(),
Position = numbered.Position,
Name = numbered.Item.Name,
ContentType = numbered.Item.ContentType,
PartSize = numbered.Item.PartSize,
Text = numbered.Item.Text,
TextState = numbered.Item.TextState.ToString(),
});
/// <summary>
/// Numbers the items within their group, in the order they come in, starting at zero for each group.
/// </summary>
private static IEnumerable<(T Item, int Position)> NumberWithinGroups<T, TGroup>(IEnumerable<T> items, Func<T, TGroup> groupOf) where TGroup : notnull
{
var nextPositions = new Dictionary<TGroup, int>();
foreach (var item in items)
{
var group = groupOf(item);
var position = nextPositions.GetValueOrDefault(group);
nextPositions[group] = position + 1;
yield return (item, position);
}
}
private static MailFolderRecord ToMailFolderRecord(MailFolderEntity folder) => new(
folder.Path,
ParseStoredName(folder.SpecialUse, MailFolderSpecialUse.UNKNOWN),
folder.UidValidity,
folder.UidNext,
folder.HighestModSeq,
folder.ServerMessageCount,
folder.ServerUnseenCount,
folder.InitialSyncCompletedUtc);
private static MailRecord ToMailRecord(MailMessageEntity mail) => new(
mail.ParentFileId,
mail.MessageId,
mail.InReplyTo,
mail.ReferenceMessageIds.Split(' ', StringSplitOptions.RemoveEmptyEntries),
mail.SentAtUtc,
mail.ReceivedAtUtc,
ParseStoredName(mail.Importance, MailImportance.NORMAL),
ParseStoredName(mail.EncryptionKind, MailEncryptionKind.UNKNOWN),
mail.MailHash,
mail.FirstSeenUtc,
ToMailAddressRecords(mail.Addresses),
mail.Parts
.Select(part => (Kind: ParseStoredName(part.Kind, MailPartKind.UNKNOWN), Part: part))
.OrderBy(entry => entry.Kind)
.ThenBy(entry => entry.Part.Position)
.Select(entry => new MailPartRecord(entry.Kind, entry.Part.Name, entry.Part.ContentType, entry.Part.PartSize, entry.Part.Text, ParseStoredName(entry.Part.TextState, MailPartTextState.UNKNOWN)))
.ToList(),
mail.Locations
.OrderBy(location => location.Folder!.Path, StringComparer.Ordinal)
.ThenBy(location => location.Uid)
.Select(location => new MailLocationRecord(location.Folder!.Path, location.Uid, new MailFlags(location.IsSeen, location.IsFlagged, location.IsAnswered)))
.ToList());
/// <summary>
/// The addresses of a mail, grouped by their role and in their order within each header.
/// </summary>
private static IReadOnlyList<MailAddressRecord> ToMailAddressRecords(IEnumerable<MailAddressEntity> addresses) => addresses
.Select(address => (Role: ParseStoredName(address.Role, MailAddressRole.UNKNOWN), Address: address))
.OrderBy(entry => entry.Role)
.ThenBy(entry => entry.Address.Position)
.Select(entry => new MailAddressRecord(entry.Role, entry.Address.Address, entry.Address.DisplayName))
.ToList();
/// <remarks>
/// A row written by a newer version may name a value this one does not know. The row still
/// says everything else it says, so only that one value falls back.
/// </remarks>
private static TEnum ParseStoredName<TEnum>(string name, TEnum fallback) where TEnum : struct, Enum =>
Enum.TryParse<TEnum>(name, ignoreCase: true, out var value) && Enum.IsDefined(value) ? value : fallback;
}
@@ -3,6 +3,7 @@ using System.Globalization;
using System.Text.RegularExpressions;
using AIStudio.Settings;
using AIStudio.Tools.Mail;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Services;
@@ -10,7 +11,7 @@ using Microsoft.EntityFrameworkCore;
namespace AIStudio.Tools.Databases.IndexStore;
public sealed class SqliteIndexStoreClientImplementation(string name, string databasePath, string basePath, string version) : IndexStoreClient(name, basePath)
public sealed partial class SqliteIndexStoreClientImplementation(string name, string databasePath, string basePath, string version) : IndexStoreClient(name, basePath)
{
private const string DATABASE_NAME = "SQLite";
private const string DATABASE_FILENAME = "rag-index.sqlite3";
@@ -88,6 +89,13 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
yield return (TB("Indexed data sources"), OrUnknown(snapshot.DataSourceCount));
yield return (TB("Indexed files"), OrUnknown(snapshot.FileCount));
yield return (TB("Permanently skipped files"), OrUnknown(snapshot.FailureCount));
// Only once a mailbox brought any mails. Without one, the lines would say nothing but zero:
if (snapshot.MailCount > 0 || snapshot.MailFailureCount > 0)
{
yield return (TB("Indexed mails"), OrUnknown(snapshot.MailCount));
yield return (TB("Permanently skipped mails"), OrUnknown(snapshot.MailFailureCount));
}
}
public override async Task<DataSourceIndexState?> GetDataSourceStateAsync(string dataSourceId, CancellationToken token)
@@ -310,48 +318,61 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
if (string.IsNullOrWhiteSpace(ftsQuery))
return [];
//
// Chunks of the same score keep the order of their rows. The results are cut into pages by
// asking for more of them each time, cf. RetrievalPaging. If ties could fall differently
// with every limit, a page might show a chunk again or skip one.
//
await using var context = this.CreateContext();
var results = await context.SearchResults
.FromSqlInterpolated($"""
SELECT
c.chunk_id AS ChunkId,
c.parent_file_id AS ParentFileId,
ds.data_source_id AS DataSourceId,
ds.data_source_type AS DataSourceType,
f.absolute_path AS AbsolutePath,
f.file_name AS FileName,
f.relative_path AS RelativePath,
f.file_type AS FileType,
c.page_number AS PageNumber,
c.chunk_index AS ChunkIndex,
c.chunk_text AS ChunkText,
bm25(embedding_chunks_fts) AS Score,
f.fingerprint AS Fingerprint,
f.file_size AS FileSize,
f.creation_utc AS CreationUtc,
f.last_write_utc AS LastWriteUtc,
c.embedded_at_utc AS EmbeddedAtUtc,
f.chunk_count AS ChunkCount
FROM embedding_chunks_fts
JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid
JOIN embedded_files f ON f.parent_file_id = c.parent_file_id
JOIN data_sources ds ON ds.data_source_id = f.data_source_id
WHERE ds.data_source_id = {dataSourceId}
AND embedding_chunks_fts MATCH {ftsQuery}
ORDER BY Score, c.id
LIMIT {maxMatches}
""")
.AsNoTracking()
var results = await InSearchOrder(MatchChunks(context, dataSourceId, ftsQuery))
.Take(maxMatches)
.ToListAsync(token);
return results.Select(ToSearchResult).ToList();
}
/// <summary>
/// The chunks of a data source which match a full-text query, unordered, as a query to build on.
/// </summary>
/// <remarks>
/// Kept free of ORDER BY and LIMIT, so EF Core can wrap it and filter it further before either
/// of them applies, cf. InSearchOrder.
/// </remarks>
private static IQueryable<IndexStoreSearchResultEntity> MatchChunks(IndexStoreDbContext context, string dataSourceId, string ftsQuery) => context.SearchResults
.FromSqlInterpolated($"""
SELECT
c.chunk_id AS ChunkId,
c.parent_file_id AS ParentFileId,
ds.data_source_id AS DataSourceId,
ds.data_source_type AS DataSourceType,
f.absolute_path AS AbsolutePath,
f.file_name AS FileName,
f.relative_path AS RelativePath,
f.file_type AS FileType,
c.page_number AS PageNumber,
c.chunk_index AS ChunkIndex,
c.chunk_text AS ChunkText,
bm25(embedding_chunks_fts) AS Score,
f.fingerprint AS Fingerprint,
f.file_size AS FileSize,
f.creation_utc AS CreationUtc,
f.last_write_utc AS LastWriteUtc,
c.embedded_at_utc AS EmbeddedAtUtc,
f.chunk_count AS ChunkCount
FROM embedding_chunks_fts
JOIN embedding_chunks c ON c.id = embedding_chunks_fts.rowid
JOIN embedded_files f ON f.parent_file_id = c.parent_file_id
JOIN data_sources ds ON ds.data_source_id = f.data_source_id
WHERE ds.data_source_id = {dataSourceId}
AND embedding_chunks_fts MATCH {ftsQuery}
""")
.AsNoTracking();
/// <remarks>
/// Chunks of the same score keep a fixed order, by their document and their place in it. The
/// results are cut into pages by asking for more of them each time, cf. RetrievalPaging. If
/// ties could fall differently with every limit, a page might show a chunk again or skip one.
/// </remarks>
private static IQueryable<IndexStoreSearchResultEntity> InSearchOrder(IQueryable<IndexStoreSearchResultEntity> results) => results
.OrderBy(result => result.Score)
.ThenBy(result => result.ParentFileId)
.ThenBy(result => result.ChunkIndex);
public override async Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token)
{
await using var context = this.CreateContext();
@@ -403,7 +424,8 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
/// Everything the display info reads out of the database in one go.
/// </summary>
/// <remarks>
/// Every property is empty when its probe could not answer. The caller turns that into "unknown".
/// Every property is empty or null when its probe could not answer. The caller turns that into "unknown".
/// The documents of mailboxes are counted apart from files, told apart by their keys, cf. MailContentKey.
/// </remarks>
private sealed record DisplaySnapshot
{
@@ -415,15 +437,21 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
public string TableCount { get; init; } = string.Empty;
public string DataSourceCount { get; init; } = string.Empty;
public int? DataSourceCount { get; init; }
public string FileCount { get; init; } = string.Empty;
public int? FileCount { get; init; }
public string FailureCount { get; init; } = string.Empty;
public int? FailureCount { get; init; }
public int? MailCount { get; init; }
public int? MailFailureCount { get; init; }
}
private static string OrUnknown(string value) => string.IsNullOrWhiteSpace(value) ? TB("unknown") : value;
private static string OrUnknown(int? count) => count?.CompactCount() ?? TB("unknown");
private async Task<DisplaySnapshot> ReadDisplaySnapshotAsync()
{
var token = CancellationToken.None;
@@ -436,9 +464,11 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
JournalMode = (await QueryScalarTextAsync(context, "PRAGMA journal_mode;", token)).ToUpperInvariant(),
SchemaVersion = await GetSchemaVersionAsync(context, token),
TableCount = await GetTableCountAsync(context, token),
DataSourceCount = await FormatCountAsync(context.DataSources, token),
FileCount = await FormatCountAsync(context.EmbeddedFiles, token),
FailureCount = await FormatCountAsync(context.PermanentIndexingFailures, token),
DataSourceCount = await TryCountAsync(context.DataSources, token),
FileCount = await TryCountAsync(context.EmbeddedFiles.Where(file => !file.AbsolutePath.StartsWith(MailContentKey.PREFIX)), token),
FailureCount = await TryCountAsync(context.PermanentIndexingFailures.Where(failure => !failure.AbsolutePath.StartsWith(MailContentKey.PREFIX)), token),
MailCount = await TryCountAsync(context.EmbeddedFiles.Where(file => file.AbsolutePath.StartsWith(MailContentKey.PREFIX)), token),
MailFailureCount = await TryCountAsync(context.PermanentIndexingFailures.Where(failure => failure.AbsolutePath.StartsWith(MailContentKey.PREFIX)), token),
};
}
catch (Exception exception)
@@ -523,15 +553,15 @@ public sealed class SqliteIndexStoreClientImplementation(string name, string dat
}
}
private static async Task<string> FormatCountAsync<T>(IQueryable<T> query, CancellationToken token) where T : class
private static async Task<int?> TryCountAsync<T>(IQueryable<T> query, CancellationToken token) where T : class
{
try
{
return (await query.CountAsync(token)).CompactCount();
return await query.CountAsync(token);
}
catch
{
return string.Empty;
return null;
}
}