Added mailboxes as local data sources (#1022)

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs

@@ -1,3 +1,4 @@
+using AIStudio.Settings.DataModel;
 using AIStudio.Tools.Databases.IndexStore;
 using AIStudio.Tools.Databases.VectorStore;
 
@@ -22,7 +23,7 @@ public sealed partial class DataSourceEmbeddingService
     /// <param name="dataSourceId">The data source to build anew.</param>
     public async Task RepairDataSourceAsync(string dataSourceId)
     {
-        if (!this.TryGetConfiguredDataSource(dataSourceId, out var dataSource) || !this.IsSupportedInternalDataSource(dataSource))
+        if (!this.TryGetConfiguredIndexedSource(dataSourceId, out var dataSource))
             return;
 
         logger.LogWarning(
@@ -37,11 +38,99 @@ public sealed partial class DataSourceEmbeddingService
 
         await this.ResetPersistedStateAsync(dataSourceId, null, null, CancellationToken.None);
         this.statuses.TryRemove(dataSourceId, out _);
-        this.PublishStatusChanged();
+        PublishStatusChanged();
 
         await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.MANUAL_RETRY);
     }
 
+    /// <summary>
+    /// Records that the user agreed to the removal a mailbox held back, and syncs the mailbox to carry it out.
+    /// </summary>
+    /// <remarks>
+    /// The count is the one the status showed the user, cf. DataSourceEmbeddingStatus.PendingRemovalCount.
+    /// The agreement holds for exactly that number: should a sync have come up with another one in the
+    /// meantime, nothing is agreed to, and the user is asked anew. The sync which carries the removal
+    /// out signs in like any other, so a recorded sign-in failure still keeps it from the server.
+    /// </remarks>
+    /// <param name="dataSourceId">The mailbox.</param>
+    /// <param name="removalCount">How many mails the user agreed to remove from the index.</param>
+    /// <returns>True when the agreement was recorded. False when the number changed in the meantime, or the index cannot be reached.</returns>
+    public async Task<bool> ApprovePendingMailRemovalAsync(string dataSourceId, int removalCount)
+    {
+        if (!this.TryGetConfiguredIndexedSource(dataSourceId, out var dataSource) || dataSource is not DataSourceMailbox)
+            return false;
+
+        var indexStore = await databaseClientProvider.GetIndexStoreAsync(CancellationToken.None);
+        if (!await indexStore.ApprovePendingMailRemovalAsync(dataSourceId, removalCount, CancellationToken.None))
+        {
+            logger.LogInformation("The removal held back for mailbox '{DataSourceId}' was not approved, since another number is held back by now or the index store is unavailable.", dataSourceId);
+            return false;
+        }
+
+        logger.LogInformation("The user approved removing {RemovalCount} mails from the index of mailbox '{DataSourceId}'.", removalCount, dataSourceId);
+        await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.HASH_CHECK);
+        return true;
+    }
+
+    /// <summary>
+    /// Deletes what the index keeps of mailboxes which are no longer configured.
+    /// </summary>
+    /// <remarks>
+    /// Such leftovers come about in two ways. An older version of AI Studio does not know mailboxes
+    /// and saves the settings without them. Or the stores could not be reached while the user
+    /// deleted a mailbox. Either way, nothing would ever delete the index then, nor a refused
+    /// sign-in kept next to it.
+    ///
+    /// Only with settings which were loaded and may be written. Settings which could not be read
+    /// look like settings without a single mailbox, and every index would be deleted for that. Both
+    /// stores have to be there as well: the row in the index store is all that leads to the
+    /// collection in the vector store, so deleting the one without the other would leave the other
+    /// behind for good.
+    ///
+    /// Runs before the first run, so none of the mailboxes is in use. The password stays in the OS
+    /// keyring: its entry carries the name of the mailbox, which nothing here knows anymore.
+    /// </remarks>
+    /// <param name="token">The cancellation token.</param>
+    private async Task DeleteOrphanedMailboxIndexesAsync(CancellationToken token)
+    {
+        if (!settingsManager.HasCompletedInitialSettingsLoad || settingsManager.SettingsWriteBlocked)
+        {
+            logger.LogInformation("Leaving the indexes of mailboxes alone, since the settings could not be loaded completely. SettingsWriteBlockReason={SettingsWriteBlockReason}.", settingsManager.SettingsWriteBlockReason);
+            return;
+        }
+
+        try
+        {
+            var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token);
+            var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
+            if (!vectorStore.IsAvailable || !indexStore.IsAvailable)
+                return;
+
+            var configuredIds = settingsManager.ConfigurationData.Mailboxes
+                .Select(mailbox => mailbox.Id)
+                .ToHashSet(StringComparer.OrdinalIgnoreCase);
+
+            foreach (var mailboxId in await indexStore.GetStoredMailboxIdsAsync(token))
+            {
+                if (configuredIds.Contains(mailboxId))
+                    continue;
+
+                await this.ResetPersistedStateAsync(mailboxId, vectorStore, indexStore, token);
+                await indexStore.ClearMailboxAuthFailureAsync(mailboxId, token);
+                logger.LogInformation("Deleted the index of mailbox '{DataSourceId}', which is no longer configured.", mailboxId);
+            }
+        }
+        catch (OperationCanceledException) when (token.IsCancellationRequested)
+        {
+            throw;
+        }
+        catch (Exception exception)
+        {
+            // Nothing depends on it: whatever is left over is tried again on the next start.
+            logger.LogWarning(exception, "Could not delete the indexes of mailboxes which are no longer configured.");
+        }
+    }
+
     private async Task ResetPersistedStateAsync(
         string dataSourceId,
         VectorStoreClient? vectorStore,