diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Configuration.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Configuration.cs new file mode 100644 index 00000000..7514a6ab --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Configuration.cs @@ -0,0 +1,127 @@ +using AIStudio.Tools.Databases.IndexStore; + +using Microsoft.AspNetCore.Components; + +using static AIStudio.Tools.Services.Indexing.IndexingLogFormat; + +namespace AIStudio.Tools.Services; + +/// +/// Changes to the embedding settings which do not pass through the queue: the configuration of an +/// organization, which may change an embedding provider whenever its plugin is applied, and the +/// confidence level of a provider, which is set for all of its uses at once. +/// +public sealed partial class DataSourceEmbeddingService +{ + #region Implementation of IMessageBusReceiver + + public Task ProcessMessage(ComponentBase? sendingComponent, Event triggeredEvent, T? data) + { + if (triggeredEvent is not (Event.PLUGINS_RELOADED or Event.CONFIGURATION_CHANGED)) + return Task.CompletedTask; + + // + // Not awaited: the plugins start, and the settings save, while this may wait for the index + // database. + // + var queueIdleDataSources = triggeredEvent is Event.PLUGINS_RELOADED; + _ = Task.Run(async () => + { + try + { + await this.StopOutdatedRunsAsync(); + if (queueIdleDataSources) + await this.QueueOutdatedIdleDataSourcesAsync(); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Failed to check the data sources against the changed configuration."); + } + }); + + return Task.CompletedTask; + } + + public Task> ProcessMessageWithResult(ComponentBase? sendingComponent, Event triggeredEvent, TPayload? data) => Task.FromResult(MessageBusAnswer.NONE); + + #endregion + + /// + /// Stops every active run which works for embedding settings which no longer apply. + /// + /// + /// Reads no database, because it runs on every change of a setting anywhere in the app. A run + /// which has not recorded its signature yet checks for itself once it does. + /// + private async Task StopOutdatedRunsAsync() + { + foreach (var (dataSourceId, activeRun) in this.activeRuns) + { + // Stopping already, e.g. because the change was saved through the queue: + if (activeRun.TokenSource.IsCancellationRequested) + continue; + + DataSourceRunSignature? runSignature; + lock (this.runSignatureLock) + runSignature = activeRun.RunSignature; + + if (IsRunOutdated(runSignature, this.GetCurrentRunSignature(dataSourceId))) + await this.StopOutdatedDataSourceRunAsync(dataSourceId); + } + } + + /// + /// Queues every data source at rest whose stored index belongs to embedding settings which no longer apply. + /// + /// + /// Such a data source stays out of the selection until its index is built anew, cf. + /// IsAwaitingReindexAsync, which so far happened only once one of its documents changed or the + /// app started again. The rebuild is the one the next run would do anyway, decided by the same + /// signature. It only comes sooner, and only while the data sources refresh on their own. + /// + /// A data source with a run, or with one queued, is left alone: StopOutdatedRunsAsync takes care + /// of the run, and a queued run reads the current settings when it starts. The same holds for the + /// data sources the startup check has yet to reach, which wait as queued as well. + /// + private async Task QueueOutdatedIdleDataSourcesAsync() + { + if (!settingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh) + return; + + // The startup check has yet to come, and it reads the current settings for every data source: + if (Volatile.Read(ref this.startupHashCheckStarted) == 0) + return; + + IndexStoreClient? indexStore = null; + foreach (var dataSource in this.GetConfiguredIndexedSources()) + { + if (this.activeRuns.ContainsKey(dataSource.Id)) + continue; + + if (this.statuses.TryGetValue(dataSource.Id, out var status) && status.State is DataSourceEmbeddingState.QUEUED or DataSourceEmbeddingState.RUNNING) + continue; + + // No run would start, and its status says why already: + var currentRunSignature = this.GetCurrentRunSignature(dataSource.Id); + if (currentRunSignature == DataSourceRunSignature.NONE) + continue; + + indexStore ??= await databaseClientProvider.GetIndexStoreAsync(CancellationToken.None); + if (!indexStore.IsAvailable) + return; + + var indexState = await indexStore.GetDataSourceStateAsync(dataSource.Id, CancellationToken.None); + if (string.Equals(indexState?.EmbeddingSignature, currentRunSignature.EmbeddingSignature, StringComparison.Ordinal)) + continue; + + logger.LogInformation( + "Queueing data source '{DataSourceName}' ({DataSourceId}) because the changed configuration changed its embedding settings. StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.", + dataSource.Name, + dataSource.Id, + ShortHash(indexState?.EmbeddingSignature ?? string.Empty), + ShortHash(currentRunSignature.EmbeddingSignature)); + + await this.QueueDataSourceAsync(dataSource); + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Signature.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Signature.cs index dda783c3..06d3c1f0 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Signature.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Signature.cs @@ -1,5 +1,6 @@ using System.Globalization; +using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.Mail; @@ -140,4 +141,40 @@ public sealed partial class DataSourceEmbeddingService /// The signature of this pairing. internal static string BuildEmbeddingSignature(IDataSourceBase dataSource, EmbeddingProvider embeddingProvider) => BuildEmbeddingSignature(dataSource, embeddingProvider, GetChunkingOptions(dataSource, embeddingProvider)); + + /// + /// Describes the indexing run a data source would get with a given embedding provider. + /// + /// + /// The embedding signature, as long as the provider may read the data source at all. Comparing + /// what a run started with against what it would start with now tells whether the run is + /// outdated: either it embeds for an index the next run throws away, or it sends documents to a + /// provider which may no longer read them. + /// + /// The data source to index. + /// The embedding provider the data source names. + /// The confidence level of that embedding provider. + /// The signature of the run, or NONE when the provider may not read the data source and no run would start. + internal static DataSourceRunSignature GetRunSignature(IIndexedDataSource dataSource, EmbeddingProvider embeddingProvider, ConfidenceLevel embeddingProviderConfidence) => + AllowsEmbedding(dataSource, embeddingProviderConfidence) + ? new(BuildEmbeddingSignature(dataSource, embeddingProvider)) + : DataSourceRunSignature.NONE; + + /// + /// Whether an indexing run works for embedding settings which no longer apply. + /// + /// The signature the run started with, or null while the run has not resolved its embedding provider yet. + /// The signature a run would start with now, or NONE when no run would start anymore. + /// True when the run should stop. + internal static bool IsRunOutdated(DataSourceRunSignature? runSignature, DataSourceRunSignature currentRunSignature) + { + // + // Still preparing, so nothing was embedded or sent anywhere yet. The run checks for itself + // once it records its signature. + // + if (runSignature is null) + return false; + + return currentRunSignature == DataSourceRunSignature.NONE || runSignature != currentRunSignature; + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 6ea6be21..22e464f6 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -17,7 +17,7 @@ using static AIStudio.Tools.Services.Indexing.IndexingLogFormat; namespace AIStudio.Tools.Services; public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, - PromptInjectionGuardService guardService, ILogger logger) : BackgroundService + PromptInjectionGuardService guardService, ILogger logger) : BackgroundService, IMessageBusReceiver { /// /// How long the re-index check waits for the index database before it gives up. @@ -40,6 +40,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM private readonly ConcurrentDictionary activeRuns = new(StringComparer.OrdinalIgnoreCase); private readonly ConcurrentDictionary statuses = new(StringComparer.OrdinalIgnoreCase); private readonly object queueStateLock = new(); + + /// + /// Guards the run signature of every active run, cf. DataSourceRunControl.RunSignature. + /// + private readonly Lock runSignatureLock = new(); + private int startupHashCheckStarted; private int startupHashCheckCompleted; @@ -55,7 +61,17 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM private sealed record DataSourceEmbeddingQueueItem(string DataSourceId, DataSourceEmbeddingRefreshMode RefreshMode); - private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource Completion); + private sealed record DataSourceRunControl(CancellationTokenSource TokenSource, TaskCompletionSource Completion) + { + /// + /// The embedding settings the run works for, as GetRunSignature describes them. + /// + /// + /// Null until the run has resolved its embedding provider. Read and written under + /// runSignatureLock only. + /// + public DataSourceRunSignature? RunSignature { get; set; } + } /// /// Creates one indexer per kind of data source, all of them cutting their text with the same chunker. @@ -443,6 +459,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM logger.LogDebug("Refreshed watcher state for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); var queueRequestResult = this.TryReserveDataSourceQueueSlot(dataSource.Id, queueAfterCurrentRun, refreshMode); + + // + // Every change of the embedding settings ends up here, whichever dialog made it. A run + // started before would carry on with the old settings, possibly for hours, only for the + // next run to throw its work away. It stops after the reservation, so the run which + // starts over with the new settings is certain before the old one ends. + // + this.CancelOutdatedDataSourceRun(dataSource.Id); + switch (queueRequestResult) { case DataSourceQueueRequestResult.ALREADY_QUEUED: @@ -505,6 +530,10 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM protected override async Task ExecuteAsync(CancellationToken stoppingToken) { + // Changes to the embedding settings which do not pass through the queue, cf. ProcessMessage: + MessageBus.INSTANCE.RegisterComponent(this); + this.ApplyFilters([], [Event.PLUGINS_RELOADED, Event.CONFIGURATION_CHANGED]); + await this.WaitForInitialSettingsAndBootstrapAsync(stoppingToken); while (!stoppingToken.IsCancellationRequested) @@ -559,6 +588,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } } + public override async Task StopAsync(CancellationToken cancellationToken) + { + MessageBus.INSTANCE.Unregister(this); + await base.StopAsync(cancellationToken); + } + public override void Dispose() { this.DisposeWatchers(); @@ -595,15 +630,34 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM try { - await this.ProcessDataSourceAsync(dataSource, refreshMode, runTokenSource.Token); + await this.ProcessDataSourceAsync(dataSource, refreshMode, runControl, runTokenSource.Token); } catch (OperationCanceledException) when (!parentToken.IsCancellationRequested && runTokenSource.IsCancellationRequested) { logger.LogInformation( - "Stopped background embeddings for data source '{DataSourceName}' ({DataSourceId}) because the data source was removed or canceled. RefreshMode={RefreshMode}.", + "Stopped background embeddings for data source '{DataSourceName}' ({DataSourceId}) because the data source was removed or canceled, or its embedding settings changed. RefreshMode={RefreshMode}.", dataSource.Name, dataSource.Id, refreshMode); + + // + // A data source which is still there has its next run queued already. Its row says so + // instead of showing this run as running: a run of the startup check would otherwise + // keep that row until the check has worked through every other data source. + // + if (this.TryGetConfiguredIndexedSource(dataSource.Id, out var configuredDataSource)) + { + this.statuses.TryGetValue(configuredDataSource.Id, out var currentStatus); + this.UpsertStatus(this.CreateStatus( + configuredDataSource, + DataSourceEmbeddingState.QUEUED, + currentStatus?.TotalDocuments ?? 0, + currentStatus?.IndexedDocuments ?? 0, + currentStatus?.FailedDocuments ?? 0, + lastError: currentStatus?.LastError ?? string.Empty, + failures: currentStatus?.Failures ?? [], + lastSyncUtc: currentStatus?.LastSyncUtc)); + } } finally { @@ -624,9 +678,10 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM /// /// The data source to index. /// Why the run was started, for the log. + /// The run, which records the embedding settings it works for. /// The cancellation token. /// The context of the run, or null when there is no run. - private async Task PrepareIndexedRunAsync(IIndexedDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken token) + private async Task PrepareIndexedRunAsync(IIndexedDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, DataSourceRunControl runControl, CancellationToken token) { logger.LogInformation( "Starting background embedding hash check for data source '{DataSourceName}' ({DataSourceId}). RefreshMode={RefreshMode}.", @@ -712,13 +767,36 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Name, dataSource.Id); - var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, collectionName, vectorStore, indexStore, token); + // + // From here on, a change of the embedding settings stops this run, cf. CancelOutdatedDataSourceRun. + // The provider passed the confidence check above, so this is the run signature as well. + // + var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider); + var runSignature = new DataSourceRunSignature(embeddingSignature); + lock (this.runSignatureLock) + runControl.RunSignature = runSignature; + + // + // A change which arrived while this run was still waiting for the databases found no + // signature to compare against, so nothing stopped it. Yet this run may have started from + // the data source as it was before the change. Recording above and the check in + // CancelOutdatedDataSourceRun take the same lock, so either that check saw the signature + // and stopped the run, or this check sees the change. Checked before the manifest, which + // would otherwise throw the stored index away for settings which no longer apply. + // + if (IsRunOutdated(runSignature, this.GetCurrentRunSignature(dataSource.Id))) + { + await this.StopOutdatedDataSourceRunAsync(dataSource.Id); + token.ThrowIfCancellationRequested(); + } + + var manifest = await this.EnsureCompatibleManifestAsync(dataSource, embeddingProvider, embeddingSignature, collectionName, vectorStore, indexStore, token); token.ThrowIfCancellationRequested(); return new IndexedRunContext(dataSource, embeddingProvider, embeddingProvider.CreateProvider(), vectorStore, indexStore, manifest, settingsManager, this.UpsertStatus, logger); } - private async Task ProcessDataSourceAsync(IIndexedDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken token) + private async Task ProcessDataSourceAsync(IIndexedDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, DataSourceRunControl runControl, CancellationToken token) { if (!this.TryGetIndexer(dataSource, out var indexer)) { @@ -729,7 +807,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM return; } - var context = await this.PrepareIndexedRunAsync(dataSource, refreshMode, token); + var context = await this.PrepareIndexedRunAsync(dataSource, refreshMode, runControl, token); if (context is null) return; @@ -952,13 +1030,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM private async Task EnsureCompatibleManifestAsync( IIndexedDataSource dataSource, EmbeddingProvider embeddingProvider, + string embeddingSignature, string collectionName, VectorStoreClient vectorStore, IndexStoreClient indexStore, CancellationToken token) { - var chunkingOptions = GetChunkingOptions(dataSource, embeddingProvider); - var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); var manifest = await indexStore.GetManifestAsync(dataSource.Id, token); logger.LogInformation( @@ -1143,16 +1220,94 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM "Canceling active embedding run for deleted data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); + + return TryCancelRun(activeRun) ? activeRun : null; + } + + /// + /// Stops the outdated run of a data source, and queues the run which starts over with the current settings. + /// + /// + /// For a run already found to be outdated. The queue checks again before it stops anything, so + /// a run whose settings were changed back in the meantime carries on. + /// + /// The id of the data source. + private Task StopOutdatedDataSourceRunAsync(string dataSourceId) + { + // Through the queue, which stops the run once the next one is certain: + if (this.TryGetConfiguredIndexedSource(dataSourceId, out var dataSource)) + return this.QueueDataSourceAsync(dataSource); + + // The data source is gone, so there is nothing to start over: + this.CancelOutdatedDataSourceRun(dataSourceId); + return Task.CompletedTask; + } + + /// + /// Stops the active run of a data source when it works for embedding settings which no longer apply. + /// + /// + /// Only what the run signature covers counts, cf. GetRunSignature: another embedding model, other + /// chunk settings, a deleted embedding provider, or one which may no longer read the data source. + /// A changed name or a changed document leaves the run alone. Whoever calls this has to make sure + /// a run follows which starts over with the current settings. + /// + /// The id of the data source. + private void CancelOutdatedDataSourceRun(string dataSourceId) + { + if (!this.activeRuns.TryGetValue(dataSourceId, out var activeRun)) + return; + + var currentRunSignature = this.GetCurrentRunSignature(dataSourceId); + DataSourceRunSignature? runSignature; + lock (this.runSignatureLock) + { + runSignature = activeRun.RunSignature; + if (!IsRunOutdated(runSignature, currentRunSignature)) + return; + } + + logger.LogInformation( + "Canceling the active embedding run for data source '{DataSourceId}' because its embedding settings changed or its embedding provider may no longer read it. RunSignaturePrefix={RunSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.", + dataSourceId, + ShortHash(runSignature?.EmbeddingSignature ?? string.Empty), + currentRunSignature == DataSourceRunSignature.NONE ? "" : ShortHash(currentRunSignature.EmbeddingSignature)); + + TryCancelRun(activeRun); + } + + /// + /// The embedding settings a run of a data source would start with now, as GetRunSignature describes them. + /// + /// The id of the data source. + /// The run signature, or NONE when no run would start: the data source is gone, its embedding provider is, or that provider may not read it. + private DataSourceRunSignature GetCurrentRunSignature(string dataSourceId) + { + if (!this.TryGetConfiguredIndexedSource(dataSourceId, out var dataSource)) + return DataSourceRunSignature.NONE; + + if (!this.TryResolveEmbeddingProvider(dataSource, out var embeddingProvider)) + return DataSourceRunSignature.NONE; + + return GetRunSignature(dataSource, embeddingProvider, embeddingProvider.GetConfidenceLevel(settingsManager)); + } + + /// + /// Asks a run to stop. + /// + /// The run. + /// False when the run has ended already. + private static bool TryCancelRun(DataSourceRunControl run) + { try { - activeRun.TokenSource.Cancel(); + run.TokenSource.Cancel(); + return true; } catch (ObjectDisposedException) { - return null; + return false; } - - return activeRun; } private async Task QueuePendingDataSourceRunAsync(string dataSourceId, CancellationToken token) diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceRunSignature.cs b/app/MindWork AI Studio/Tools/Services/DataSourceRunSignature.cs new file mode 100644 index 00000000..0f1d8a4c --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/DataSourceRunSignature.cs @@ -0,0 +1,18 @@ +namespace AIStudio.Tools.Services; + +/// +/// The embedding settings an indexing run of a data source works for. +/// +/// +/// Deliberately nothing but the embedding signature, which decides when a stored index is thrown +/// away. A run counts as outdated exactly when the next run would throw its work away, and a list of +/// fields of its own would have to be kept in step with that signature by hand. +/// +/// The embedding signature of the run, cf. DataSourceEmbeddingService.BuildEmbeddingSignature. +internal sealed record DataSourceRunSignature(string EmbeddingSignature) +{ + /// + /// No run would start: the data source or its embedding provider is gone, or that provider may not read it. + /// + public static readonly DataSourceRunSignature NONE = new(string.Empty); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md index 202bb4d0..3024f9aa 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md @@ -27,6 +27,9 @@ - Fixed answers breaking off with an error message from self-hosted AI servers such as vLLM when the AI made up the name of a tool or sent broken details for one. AI Studio now tells the AI that its tool call was invalid, and the AI can finish its answer. - Fixed OpenAI models using about 4,300 more tokens than necessary with every request. AI Studio offered them OpenAI's own web search in the background, even when you had not selected a web search. To let a model search the web, select the Web Search tool below the message field. - Fixed AI Studio sometimes rebuilding the index of a data source from scratch when the configuration of your organization was applied while the data source was being updated. +- Fixed AI Studio is continuing to update the index of a data source with the old settings after you, or the configuration of your organization, changed its embedding provider or the expert settings of the data source, which could take hours for nothing. AI Studio now stops right away and starts over with the new settings. +- Fixed AI Studio is continuing to send the content of a data source to an embedding provider after you deleted that provider, or after a confidence level changed so that the provider may no longer read the data source. AI Studio now stops right away. +- Fixed data sources are staying unavailable after the configuration of your organization changed their embedding provider, until one of their documents changed, or you restarted AI Studio. When AI Studio updates your data sources automatically, it now rebuilds their index right away. - Fixed justified texts being hyphenated by the rules of English even when AI Studio shows another language, such as German. Screen readers now also know which language AI Studio uses. - Fixed the log file of AI Studio showing the wrong time for some entries when AI Studio was busy. Each entry now shows when it actually happened. - Fixed AI Studio pausing for a moment while it opened the local index of a data source, such as a mailbox. Other work, for example applying the configuration of your organization, had to wait until then. diff --git a/app/Tests/Tools/OutdatedRunTests.cs b/app/Tests/Tools/OutdatedRunTests.cs new file mode 100644 index 00000000..45c03030 --- /dev/null +++ b/app/Tests/Tools/OutdatedRunTests.cs @@ -0,0 +1,110 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.Services; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks when an indexing run stops because the embedding settings of its data source changed. +/// +/// +/// A run resolves its data source and its embedding provider once, when it starts, and may then run +/// for hours. A run which carries on after a change of those settings embeds for an index the next +/// run throws away, at a cloud embedding provider for real money. Worse, it keeps sending documents +/// to a provider which was deleted or may no longer read them. Stopping a run for nothing is not free +/// either: everything it did so far is done again. +/// +[TestFixture] +public sealed class OutdatedRunTests +{ + private static readonly DataSourceRunSignature RUN_SIGNATURE = new("2|b0a4c4d2|OPEN_AI|text-embedding-3-small|512|100"); + + private static readonly DataSourceRunSignature OTHER_SIGNATURE = new("2|b0a4c4d2|OPEN_AI|text-embedding-3-large|512|100"); + + [Test] + public void ARunWhichIsStillPreparingCarriesOn() + { + Assert.Multiple(() => + { + Assert.That(DataSourceEmbeddingService.IsRunOutdated(null, OTHER_SIGNATURE), Is.False, "The run has not resolved its embedding provider yet, so it has embedded nothing so far."); + Assert.That(DataSourceEmbeddingService.IsRunOutdated(null, DataSourceRunSignature.NONE), Is.False, "The run has not resolved its embedding provider yet, so it has sent nothing anywhere so far."); + }); + } + + [Test] + public void ARunForTheCurrentSettingsCarriesOn() + { + var sameSignature = new DataSourceRunSignature(RUN_SIGNATURE.EmbeddingSignature); + Assert.That(DataSourceEmbeddingService.IsRunOutdated(RUN_SIGNATURE, sameSignature), Is.False, "Nothing the vectors depend on changed, so stopping the run would only repeat its work."); + } + + [Test] + public void ARunForOtherSettingsStops() + { + Assert.That(DataSourceEmbeddingService.IsRunOutdated(RUN_SIGNATURE, OTHER_SIGNATURE), Is.True, "The next run throws away whatever this run embeds."); + } + + [Test] + public void ARunWhichWouldNotStartAnymoreStops() + { + Assert.That(DataSourceEmbeddingService.IsRunOutdated(RUN_SIGNATURE, DataSourceRunSignature.NONE), Is.True, "The embedding provider is gone or may no longer read the data source, so the run must not send it anything else."); + } + + [Test] + public void AnAllowedProviderGetsARunWithTheEmbeddingSignature() + { + var dataSource = DataSource(ConfidenceLevel.LOW); + var embeddingProvider = TestEmbeddingProvider(); + + Assert.That( + DataSourceEmbeddingService.GetRunSignature(dataSource, embeddingProvider, ConfidenceLevel.MEDIUM).EmbeddingSignature, + Is.EqualTo(DataSourceEmbeddingService.BuildEmbeddingSignature(dataSource, embeddingProvider)), + "A run counts as outdated exactly when the stored index would be thrown away."); + } + + [Test] + public void ChangingTheConfidenceOfAnAllowedProviderKeepsTheRun() + { + var dataSource = DataSource(ConfidenceLevel.LOW); + var embeddingProvider = TestEmbeddingProvider(); + + Assert.That( + DataSourceEmbeddingService.GetRunSignature(dataSource, embeddingProvider, ConfidenceLevel.HIGH), + Is.EqualTo(DataSourceEmbeddingService.GetRunSignature(dataSource, embeddingProvider, ConfidenceLevel.MEDIUM)), + "The provider may still read the data source, and the confidence level changes no vector."); + } + + [Test] + public void AProviderBelowTheLevelOfTheDataSourceGetsNoRun() + { + var dataSource = DataSource(ConfidenceLevel.HIGH); + + Assert.That(DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.LOW), Is.EqualTo(DataSourceRunSignature.NONE), "A run must not send the documents of the data source to a provider which may not read them."); + } + + [Test] + public void NoProviderGetsARunForAMailboxWithoutAValidLevel() + { + var mailbox = new DataSourceMailbox { ConfidenceLevel = ConfidenceLevel.NONE }; + + Assert.That(DataSourceEmbeddingService.GetRunSignature(mailbox, TestEmbeddingProvider(), ConfidenceLevel.HIGH), Is.EqualTo(DataSourceRunSignature.NONE), "The run asked the rule of the other data sources, which lets every provider read a source without a level."); + } + + private static DataSourceLocalDirectory DataSource(ConfidenceLevel confidenceLevel) => new() + { + Num = 1, + Id = "6f1d6a4e-6a5e-4c62-9a4f-0f2d2c8b7a11", + Name = "Test data", + Description = "Documents used by the tests.", + Type = DataSourceType.LOCAL_DIRECTORY, + EmbeddingId = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", + MaxChunkTokenLength = 512, + ChunkOverlapTokenLength = 100, + ConfidenceLevel = confidenceLevel, + Path = "/tmp/test-data", + }; + + private static EmbeddingProvider TestEmbeddingProvider() => + new(1, "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small")); +} \ No newline at end of file