From dcba62b45c0997fa724600d5a1ddeb34523823e0 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Tue, 27 Oct 2026 08:05:52 +0100 Subject: [PATCH] Added a pause between indexing steps for shared embedding servers (#1114) --- .../Assistants/I18N/allTexts.lua | 21 +++ .../Components/DataSourceManagement.razor | 5 + .../Components/DataSourceManagement.razor.cs | 22 +++ app/MindWork AI Studio/Pages/Embeddings.razor | 1 + .../Pages/Embeddings.razor.cs | 18 +++ .../Plugins/configuration/plugin.lua | 16 ++ .../plugin.lua | 21 +++ .../plugin.lua | 21 +++ .../Settings/DataModel/DataApp.cs | 2 +- .../DataModel/DataDataSourceIndexing.cs | 34 ++++- .../Tools/PluginSystem/PluginConfiguration.cs | 15 ++ .../Services/DataSourceEmbeddingService.cs | 70 ++++++++- .../Tools/Services/Indexing/AGENTS.md | 11 ++ .../Services/Indexing/PauseBeforeStep.cs | 14 ++ .../Services/Indexing/PauseBetweenSteps.cs | 75 +++++++++ .../Tools/Services/Indexing/PauseRange.cs | 68 +++++++++ .../wwwroot/changelog/v26.10.1.md | 3 +- .../DataSourceRelocationWorkflowTests.cs | 2 +- app/Tests/Tools/PauseBetweenStepsTests.cs | 143 ++++++++++++++++++ app/Tests/Tools/PauseRangeTests.cs | 72 +++++++++ .../Tools/ToolCalling/MailDraftSyncTests.cs | 2 +- 21 files changed, 627 insertions(+), 9 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Services/Indexing/PauseBeforeStep.cs create mode 100644 app/MindWork AI Studio/Tools/Services/Indexing/PauseBetweenSteps.cs create mode 100644 app/MindWork AI Studio/Tools/Services/Indexing/PauseRange.cs create mode 100644 app/Tests/Tools/PauseBetweenStepsTests.cs create mode 100644 app/Tests/Tools/PauseRangeTests.cs diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 2c498a36d..35c4cdec8 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -4405,6 +4405,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "Yo -- Automatic local data source refresh UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1208397349"] = "Automatic local data source refresh" +-- Shortest pause between indexing steps +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1308136668"] = "Shortest pause between indexing steps" + -- Refresh UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T135637716"] = "Refresh" @@ -4423,6 +4426,12 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1592527757"] = "Ca -- External (ERI) UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1652430727"] = "External (ERI)" +-- Longest pause between indexing steps +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1654263778"] = "Longest pause between indexing steps" + +-- seconds +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1723256298"] = "seconds" + -- Delete Data Source UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1849107431"] = "Delete Data Source" @@ -4432,6 +4441,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2067030738"] = "Ar -- Local Directory Data Source Information UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2146756020"] = "Local Directory Data Source Information" +-- AI Studio indexes large data sources in steps of a few minutes. Between two steps, it waits for a random time between the shortest and the longest pause. This spares an embedding server which many people share, for example in your organization. Leave both at 0 seconds for no pause. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2352821039"] = "AI Studio indexes large data sources in steps of a few minutes. Between two steps, it waits for a random time between the shortest and the longest pause. This spares an embedding server which many people share, for example in your organization. Leave both at 0 seconds for no pause." + -- Local data sources refresh when files change, mailboxes every 16 minutes. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2511157475"] = "Local data sources refresh when files change, mailboxes every 16 minutes." @@ -4474,6 +4486,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3267849393"] = "Ed -- Are you sure you want to delete the data source '{0}' of type '{1}'? UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3337072977"] = "Are you sure you want to delete the data source '{0}' of type '{1}'?" +-- The upper end of the random pause between two indexing steps. AI Studio allows up to 300 seconds (5 minutes). +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3401303590"] = "The upper end of the random pause between two indexing steps. AI Studio allows up to 300 seconds (5 minutes)." + -- Unknown UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3424652889"] = "Unknown" @@ -10813,6 +10828,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1485378199"] = "These files match -- Outside the selection UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1512669069"] = "Outside the selection" +-- Between two steps, AI Studio pauses for {0} seconds, to spare an embedding server which many people share. +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1513645664"] = "Between two steps, AI Studio pauses for {0} seconds, to spare an embedding server which many people share." + -- Failed: {0} UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1531389696"] = "Failed: {0}" @@ -10921,6 +10939,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3865031940"] = "Actions" -- These file types and folders were found in the folder of the data source, but its file patterns or its list of excluded files and folders leave them out. They are intentionally excluded and do not affect the indexing status. Excluded folders are not searched at all. UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T4007318423"] = "These file types and folders were found in the folder of the data source, but its file patterns or its list of excluded files and folders leave them out. They are intentionally excluded and do not affect the indexing status. Excluded folders are not searched at all." +-- Between two steps, AI Studio pauses for {0} to {1} seconds, to spare an embedding server which many people share. +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T4167794090"] = "Between two steps, AI Studio pauses for {0} to {1} seconds, to spare an embedding server which many people share." + -- Remove them from the index UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T419343388"] = "Remove them from the index" diff --git a/app/MindWork AI Studio/Components/DataSourceManagement.razor b/app/MindWork AI Studio/Components/DataSourceManagement.razor index 23b3ad424..40d7a7e3c 100644 --- a/app/MindWork AI Studio/Components/DataSourceManagement.razor +++ b/app/MindWork AI Studio/Components/DataSourceManagement.razor @@ -22,6 +22,11 @@ + + + + + @{ var embeddingStatuses = this.DataSourceEmbeddingService.GetStatuses().ToDictionary(status => status.DataSourceId, StringComparer.OrdinalIgnoreCase); } diff --git a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs index 7e138a182..d9a03460c 100644 --- a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs +++ b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs @@ -5,6 +5,7 @@ using AIStudio.Tools.Databases; using AIStudio.Tools.ERIClient.DataModel; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Services; +using AIStudio.Tools.Services.Indexing; using Microsoft.AspNetCore.Components; @@ -39,6 +40,27 @@ public partial class DataSourceManagement : MSGComponentBase private bool AreMailboxesEnabled => PreviewFeatures.PRE_MAILBOXES_2026.IsEnabled(this.SettingsManager); + private static bool IsMinimumPauseLocked => ManagedConfiguration.TryGet(x => x.App.DataSourceIndexing, x => x.MinimumPauseBetweenStepsSeconds, out var meta) && meta.IsLocked; + + private static bool IsMaximumPauseLocked => ManagedConfiguration.TryGet(x => x.App.DataSourceIndexing, x => x.MaximumPauseBetweenStepsSeconds, out var meta) && meta.IsLocked; + + /// + /// A slider moves a value which falls out of its range into it. A locked slider therefore keeps + /// the whole range, since its value belongs to the organization. A free one ends at the locked + /// upper bound, and the free value adjusts to it, cf. PauseRange.GetEffective. + /// + private int MinimumPauseSliderMax => IsMaximumPauseLocked && !IsMinimumPauseLocked + ? PauseRange.FromSettings(this.SettingsManager.ConfigurationData.App.DataSourceIndexing).MaximumSeconds + : DataDataSourceIndexing.MAX_PAUSE_BETWEEN_STEPS_SECONDS; + + /// + /// The upper bound starts where the lower bound is, so raising the lower one above it raises the + /// upper one along with it. A locked slider keeps the whole range, cf. MinimumPauseSliderMax. + /// + private int MaximumPauseSliderMin => IsMaximumPauseLocked + ? 0 + : PauseRange.FromSettings(this.SettingsManager.ConfigurationData.App.DataSourceIndexing).MinimumSeconds; + #region Overrides of ComponentBase protected override async Task OnInitializedAsync() diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor b/app/MindWork AI Studio/Pages/Embeddings.razor index 93a2c7c6f..9bdcf34b6 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor +++ b/app/MindWork AI Studio/Pages/Embeddings.razor @@ -16,6 +16,7 @@ @T("AI Studio indexes large data sources in steps of a few minutes, so that your other data sources get their turn in between.") + @this.PauseBetweenStepsText @if (this.AreMailboxesEnabled) { diff --git a/app/MindWork AI Studio/Pages/Embeddings.razor.cs b/app/MindWork AI Studio/Pages/Embeddings.razor.cs index 6599cffb7..0c3b9fd71 100644 --- a/app/MindWork AI Studio/Pages/Embeddings.razor.cs +++ b/app/MindWork AI Studio/Pages/Embeddings.razor.cs @@ -6,6 +6,7 @@ using AIStudio.Provider; using AIStudio.Settings.DataModel; using AIStudio.Tools.Rust; using AIStudio.Tools.Services; +using AIStudio.Tools.Services.Indexing; using Microsoft.AspNetCore.Components; @@ -58,6 +59,23 @@ public partial class Embeddings : MSGComponentBase private bool AreMailboxesEnabled => PreviewFeatures.PRE_MAILBOXES_2026.IsEnabled(this.SettingsManager); + /// + /// Says nothing while there is no pause between the steps, which is the default. + /// + private string PauseBetweenStepsText + { + get + { + var pause = PauseRange.FromSettings(this.SettingsManager.ConfigurationData.App.DataSourceIndexing); + if (pause.IsNone) + return string.Empty; + + return pause.MinimumSeconds == pause.MaximumSeconds + ? string.Format(T("Between two steps, AI Studio pauses for {0} seconds, to spare an embedding server which many people share."), pause.MaximumSeconds) + : string.Format(T("Between two steps, AI Studio pauses for {0} to {1} seconds, to spare an embedding server which many people share."), pause.MinimumSeconds, pause.MaximumSeconds); + } + } + /// /// The chips above count documents, which says nothing about how far the list of data sources /// itself has come. While several of them wait their turn, this is the one line saying so. With a diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index 12f5ce913..a61b4efc8 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -1411,6 +1411,22 @@ CONFIG["SETTINGS"] = {} -- in CONFIG["MAILBOX_PROVIDERS"], this blocks every mailbox. -- CONFIG["SETTINGS"]["DataMailboxes.AllowOnlyOrganizationMailServers"] = true +-- Configure a pause between two indexing steps of local data sources, i.e., files, folders, and +-- mailboxes. AI Studio indexes large data sources in steps of a few minutes. Between two steps, it +-- waits for a random number of seconds between the lower and the upper bound, both included. This +-- spares an embedding server which many users share: a single user with large folders or +-- mailboxes no longer keeps it busy for hours. The pause applies after every step which worked on +-- documents, also when the same data source carries on. Time which passes anyway counts toward +-- it. Both values must be between 0 and 300 seconds. The default is 0 for both, i.e., no pause. +-- You may set one bound or both. A bound you set wins, and AI Studio adjusts the other one to it. +-- Should the upper bound be below the lower bound, the lower bound wins. +-- CONFIG["SETTINGS"]["DataDataSourceIndexing.MinimumPauseBetweenStepsSeconds"] = 30 +-- CONFIG["SETTINGS"]["DataDataSourceIndexing.MaximumPauseBetweenStepsSeconds"] = 90 +-- +-- Allow users to change the bounds, so that your values are only defaults: +-- CONFIG["SETTINGS"]["DataDataSourceIndexing.MinimumPauseBetweenStepsSeconds.AllowUserOverride"] = true +-- CONFIG["SETTINGS"]["DataDataSourceIndexing.MaximumPauseBetweenStepsSeconds.AllowUserOverride"] = true + -- Configure the data source selection agent. -- This agent is used when chat data source options enable AI-based data source selection. -- The provider must be one of the provider IDs defined in CONFIG["LLM_PROVIDERS"]. diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index ddfaf62ef..8a6a8e010 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -4407,6 +4407,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "Si -- Automatic local data source refresh UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1208397349"] = "Automatische Aktualisierung lokaler Datenquellen" +-- Shortest pause between indexing steps +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1308136668"] = "Kürzeste Pause zwischen den Etappen der Indexierung" + -- Refresh UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T135637716"] = "Aktualisieren" @@ -4425,6 +4428,12 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1592527757"] = "Di -- External (ERI) UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1652430727"] = "Extern (ERI)" +-- Longest pause between indexing steps +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1654263778"] = "Längste Pause zwischen den Etappen der Indexierung" + +-- seconds +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1723256298"] = "Sekunden" + -- Delete Data Source UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1849107431"] = "Datenquelle löschen" @@ -4434,6 +4443,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2067030738"] = "M -- Local Directory Data Source Information UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2146756020"] = "Informationen zur lokalen Ordner-Datenquelle" +-- AI Studio indexes large data sources in steps of a few minutes. Between two steps, it waits for a random time between the shortest and the longest pause. This spares an embedding server which many people share, for example in your organization. Leave both at 0 seconds for no pause. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2352821039"] = "AI Studio indexiert große Datenquellen in Etappen von wenigen Minuten. Zwischen zwei Etappen wartet AI Studio eine zufällige Zeit, die zwischen der kürzesten und der längsten Pause liegt. Das entlastet einen Einbettungsserver, den sich viele Nutzer teilen, zum Beispiel in Ihrer Organisation. Lassen Sie beide Werte auf 0 Sekunden, wenn keine Pause eingelegt werden soll." + -- Local data sources refresh when files change, mailboxes every 16 minutes. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2511157475"] = "Lokale Datenquellen werden aktualisiert, wenn sich Dateien ändern, Postfächer alle 16 Minuten." @@ -4476,6 +4488,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3267849393"] = "Be -- Are you sure you want to delete the data source '{0}' of type '{1}'? UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3337072977"] = "Möchten Sie die Datenquelle „{0}“ vom Typ „{1}“ wirklich löschen?" +-- The upper end of the random pause between two indexing steps. AI Studio allows up to 300 seconds (5 minutes). +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3401303590"] = "Das obere Ende der zufälligen Pause zwischen zwei Etappen der Indexierung. AI Studio erlaubt bis zu 300 Sekunden (5 Minuten)." + -- Unknown UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3424652889"] = "Unbekannt" @@ -10815,6 +10830,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1485378199"] = "Diese Dateien ent -- Outside the selection UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1512669069"] = "Außerhalb der Auswahl" +-- Between two steps, AI Studio pauses for {0} seconds, to spare an embedding server which many people share. +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1513645664"] = "Zwischen zwei Etappen pausiert AI Studio {0} Sekunden, um einen Einbettungsserver zu entlasten, den sich viele Nutzer teilen." + -- Failed: {0} UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1531389696"] = "Fehlgeschlagen: {0}" @@ -10923,6 +10941,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3865031940"] = "Aktionen" -- These file types and folders were found in the folder of the data source, but its file patterns or its list of excluded files and folders leave them out. They are intentionally excluded and do not affect the indexing status. Excluded folders are not searched at all. UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T4007318423"] = "Diese Dateitypen und Ordner wurden im Ordner der Datenquelle gefunden, aber ihre Dateimuster oder ihre Liste ausgeschlossener Dateien und Ordner lassen sie aus. Sie fehlen absichtlich und wirken sich nicht auf den Indexierungsstatus aus. Ausgeschlossene Ordner werden gar nicht durchsucht." +-- Between two steps, AI Studio pauses for {0} to {1} seconds, to spare an embedding server which many people share. +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T4167794090"] = "Zwischen zwei Etappen pausiert AI Studio {0} bis {1} Sekunden, um einen Einbettungsserver zu entlasten, den sich viele Nutzer teilen." + -- Remove them from the index UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T419343388"] = "Aus dem Index entfernen" diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 34b413577..eb1c14844 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -4407,6 +4407,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "Yo -- Automatic local data source refresh UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1208397349"] = "Automatic local data source refresh" +-- Shortest pause between indexing steps +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1308136668"] = "Shortest pause between indexing steps" + -- Refresh UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T135637716"] = "Refresh" @@ -4425,6 +4428,12 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1592527757"] = "Ca -- External (ERI) UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1652430727"] = "External (ERI)" +-- Longest pause between indexing steps +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1654263778"] = "Longest pause between indexing steps" + +-- seconds +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1723256298"] = "seconds" + -- Delete Data Source UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1849107431"] = "Delete Data Source" @@ -4434,6 +4443,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2067030738"] = "Ar -- Local Directory Data Source Information UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2146756020"] = "Local Directory Data Source Information" +-- AI Studio indexes large data sources in steps of a few minutes. Between two steps, it waits for a random time between the shortest and the longest pause. This spares an embedding server which many people share, for example in your organization. Leave both at 0 seconds for no pause. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2352821039"] = "AI Studio indexes large data sources in steps of a few minutes. Between two steps, it waits for a random time between the shortest and the longest pause. This spares an embedding server which many people share, for example in your organization. Leave both at 0 seconds for no pause." + -- Local data sources refresh when files change, mailboxes every 16 minutes. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2511157475"] = "Local data sources refresh when files change, mailboxes every 16 minutes." @@ -4476,6 +4488,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3267849393"] = "Ed -- Are you sure you want to delete the data source '{0}' of type '{1}'? UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3337072977"] = "Are you sure you want to delete the data source '{0}' of type '{1}'?" +-- The upper end of the random pause between two indexing steps. AI Studio allows up to 300 seconds (5 minutes). +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3401303590"] = "The upper end of the random pause between two indexing steps. AI Studio allows up to 300 seconds (5 minutes)." + -- Unknown UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T3424652889"] = "Unknown" @@ -10815,6 +10830,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1485378199"] = "These files match -- Outside the selection UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1512669069"] = "Outside the selection" +-- Between two steps, AI Studio pauses for {0} seconds, to spare an embedding server which many people share. +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1513645664"] = "Between two steps, AI Studio pauses for {0} seconds, to spare an embedding server which many people share." + -- Failed: {0} UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1531389696"] = "Failed: {0}" @@ -10923,6 +10941,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3865031940"] = "Actions" -- These file types and folders were found in the folder of the data source, but its file patterns or its list of excluded files and folders leave them out. They are intentionally excluded and do not affect the indexing status. Excluded folders are not searched at all. UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T4007318423"] = "These file types and folders were found in the folder of the data source, but its file patterns or its list of excluded files and folders leave them out. They are intentionally excluded and do not affect the indexing status. Excluded folders are not searched at all." +-- Between two steps, AI Studio pauses for {0} to {1} seconds, to spare an embedding server which many people share. +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T4167794090"] = "Between two steps, AI Studio pauses for {0} to {1} seconds, to spare an embedding server which many people share." + -- Remove them from the index UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T419343388"] = "Remove them from the index" diff --git a/app/MindWork AI Studio/Settings/DataModel/DataApp.cs b/app/MindWork AI Studio/Settings/DataModel/DataApp.cs index d009c36f7..a711b6180 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataApp.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataApp.cs @@ -355,7 +355,7 @@ public sealed class DataApp(Expression>? configSelection = n /// /// Settings for indexing local data sources. /// - public DataDataSourceIndexing DataSourceIndexing { get; init; } = new(); + public DataDataSourceIndexing DataSourceIndexing { get; init; } = new(x => x.App.DataSourceIndexing); /// /// List of assistants that should be hidden from the UI. diff --git a/app/MindWork AI Studio/Settings/DataModel/DataDataSourceIndexing.cs b/app/MindWork AI Studio/Settings/DataModel/DataDataSourceIndexing.cs index fdcb89974..85b16b3dd 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataDataSourceIndexing.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataDataSourceIndexing.cs @@ -1,9 +1,41 @@ +using System.Linq.Expressions; + namespace AIStudio.Settings.DataModel; -public sealed class DataDataSourceIndexing +/// +/// Settings for indexing local data sources. +/// +/// The managed-configuration selector. +public sealed class DataDataSourceIndexing(Expression>? configSelection = null) { + /// + /// The longest pause between two indexing steps either bound may ask for. + /// + public const int MAX_PAUSE_BETWEEN_STEPS_SECONDS = 300; + + /// + /// Initializes an unmanaged instance, e.g., for the JSON deserializer. + /// + public DataDataSourceIndexing() : this(null) + { + } + /// /// Whether local data source embeddings should refresh automatically when files change. /// public bool AutomaticRefresh { get; set; } = true; + + /// + /// The lower bound of the random pause between two indexing steps, in seconds. + /// + /// + /// The pause spares an embedding server which many people share, e.g., in an organization. Zero + /// together with a zero upper bound means no pause. + /// + public int MinimumPauseBetweenStepsSeconds { get; set; } = ManagedConfiguration.Register(configSelection, n => n.MinimumPauseBetweenStepsSeconds, 0); + + /// + /// The upper bound of the random pause between two indexing steps, in seconds. + /// + public int MaximumPauseBetweenStepsSeconds { get; set; } = ManagedConfiguration.Register(configSelection, n => n.MaximumPauseBetweenStepsSeconds, 0); } diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs index d1b748a22..9380cadca 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfiguration.cs @@ -382,6 +382,21 @@ public sealed class PluginConfiguration(bool isInternal, LuaState state, PluginT // Config: mailboxes only on the mail servers of the organization? ManagedConfiguration.TryProcessConfiguration(x => x.MailboxSettings, x => x.AllowOnlyOrganizationMailServers, this.Id, settingsTable, dryRun); + // Config: the random pause between two indexing steps of local data sources + var minimumPauseIsValid = ManagedConfiguration.TryProcessConfiguration(x => x.App.DataSourceIndexing, x => x.MinimumPauseBetweenStepsSeconds, this.Id, settingsTable, dryRun, validator: value => value is >= 0 and <= DataDataSourceIndexing.MAX_PAUSE_BETWEEN_STEPS_SECONDS); + if (!minimumPauseIsValid && settingsTable.TryGetValue("DataDataSourceIndexing.MinimumPauseBetweenStepsSeconds", out _)) + LOG.LogWarning("The lower bound of the pause between indexing steps configured by plugin {ConfigPluginId} must be between 0 and {MaximumPauseSeconds} seconds.", this.Id, DataDataSourceIndexing.MAX_PAUSE_BETWEEN_STEPS_SECONDS); + + var maximumPauseIsValid = ManagedConfiguration.TryProcessConfiguration(x => x.App.DataSourceIndexing, x => x.MaximumPauseBetweenStepsSeconds, this.Id, settingsTable, dryRun, validator: value => value is >= 0 and <= DataDataSourceIndexing.MAX_PAUSE_BETWEEN_STEPS_SECONDS); + if (!maximumPauseIsValid && settingsTable.TryGetValue("DataDataSourceIndexing.MaximumPauseBetweenStepsSeconds", out _)) + LOG.LogWarning("The upper bound of the pause between indexing steps configured by plugin {ConfigPluginId} must be between 0 and {MaximumPauseSeconds} seconds.", this.Id, DataDataSourceIndexing.MAX_PAUSE_BETWEEN_STEPS_SECONDS); + + if (minimumPauseIsValid && maximumPauseIsValid + && settingsTable.TryGetValue("DataDataSourceIndexing.MinimumPauseBetweenStepsSeconds", out var minimumPauseValue) && minimumPauseValue.TryRead(out var minimumPauseSeconds) + && settingsTable.TryGetValue("DataDataSourceIndexing.MaximumPauseBetweenStepsSeconds", out var maximumPauseValue) && maximumPauseValue.TryRead(out var maximumPauseSeconds) + && maximumPauseSeconds < minimumPauseSeconds) + LOG.LogWarning("The upper bound of the pause between indexing steps configured by plugin {ConfigPluginId} is below its lower bound. The lower bound of {MinimumPauseSeconds} seconds wins.", this.Id, minimumPauseSeconds); + // Config: data source selection agent settings ManagedConfiguration.TryProcessConfiguration(x => x.AgentDataSourceSelection, x => x.PreselectAgentOptions, this.Id, settingsTable, dryRun); ManagedConfiguration.TryProcessConfiguration(x => x.AgentDataSourceSelection, x => x.PreselectedAgentProvider, Guid.Empty, this.Id, settingsTable, dryRun); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 3d3c0b21b..cda9d6442 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -17,7 +17,7 @@ using static AIStudio.Tools.Services.Indexing.IndexingLogFormat; namespace AIStudio.Tools.Services; public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, - PromptInjectionGuardService guardService, MediaTranscriptionService mediaTranscriptionService, ILogger logger) : BackgroundService, IMessageBusReceiver + PromptInjectionGuardService guardService, MediaTranscriptionService mediaTranscriptionService, ThreadSafeRandom rng, ILogger logger) : BackgroundService, IMessageBusReceiver { /// /// How long the re-index check waits for the index database before it gives up. @@ -73,6 +73,11 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM /// private readonly IndexingPauses pauses = new(); + /// + /// The pause between two steps, cf. WaitForPauseBetweenStepsAsync. + /// + private readonly PauseBetweenSteps pauseBetweenSteps = new(); + /// /// An active run. /// @@ -680,13 +685,31 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM // queue once the automatic refresh is switched on while the app runs. The wait counts as // part of the run, so deleting or repairing the data source ends the wait as well, // instead of waiting for the run before it to finish first. A paused indexing holds - // every run back in front of the gate, cf. PauseIndexingAsync: + // every run back in front of the gate, cf. PauseIndexingAsync. Behind the gate, the run + // keeps its distance to the step before, cf. WaitForPauseBetweenStepsAsync: // await this.pauses.WaitWhilePausedAsync(runTokenSource.Token); await this.runGate.WaitAsync(runTokenSource.Token); try { - await this.ProcessDataSourceAsync(dataSource, refreshMode, runControl, runTokenSource.Token); + await this.WaitForPauseBetweenStepsAsync(dataSource, refreshMode, runTokenSource.Token); + + // + // Waiting may have taken minutes, while the user could change the data source. The + // run has resolved nothing yet, so it starts with the data source as it is now, + // instead of finding itself outdated once it records its signature: + // + if (!this.TryGetConfiguredIndexedSource(dataSource.Id, out var currentDataSource)) + { + logger.LogDebug( + "Skipping embedding run for data source '{DataSourceName}' ({DataSourceId}) because it is no longer configured. RefreshMode={RefreshMode}.", + dataSource.Name, + dataSource.Id, + refreshMode); + return; + } + + await this.ProcessDataSourceAsync(currentDataSource, refreshMode, runControl, runTokenSource.Token); } finally { @@ -718,6 +741,37 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } } + /// + /// Waits for what is left of the pause between the last step and the run which is about to start, cf. PauseBetweenSteps. + /// + /// + /// The pause spares an embedding server which many people share, so it holds whichever data + /// source comes next, the same one included. It is waited for behind the run gate: only one run + /// waits, and its token ends the wait, when the app shuts down, when the data source is deleted + /// or repaired, or when a backup pauses the indexing. Meanwhile, the data source shows as queued, + /// since nothing published a status for this run yet, and every other request is queued as usual. + /// + /// The data source of the run, for the log. + /// Why the run was asked for, for the log. + /// The token of the run. + private async Task WaitForPauseBetweenStepsAsync(IIndexedDataSource dataSource, DataSourceEmbeddingRefreshMode refreshMode, CancellationToken token) + { + var range = PauseRange.FromSettings(settingsManager.ConfigurationData.App.DataSourceIndexing); + var pause = this.pauseBetweenSteps.GetPauseBeforeNextStep(DateTimeOffset.UtcNow, range, rng); + if (pause.Remaining <= TimeSpan.Zero) + return; + + logger.LogInformation( + "Pausing {PauseSeconds:F0} seconds between two indexing steps. Data source '{DataSourceName}' ({DataSourceId}) starts in {RemainingSeconds:F0} seconds. RefreshMode={RefreshMode}.", + pause.Pause.TotalSeconds, + dataSource.Name, + dataSource.Id, + pause.Remaining.TotalSeconds, + refreshMode); + + await Task.Delay(pause.Remaining, token); + } + /// /// Works out everything an indexing run needs, before anything is read from the data source. /// @@ -891,6 +945,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } var outcome = IndexedRunOutcome.DONE; + IndexedRunContext? context = null; try { // @@ -899,7 +954,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM // removes something in forty of them is one thing that happened to the user, not forty. // await using var promptInjectionScope = guardService.BeginAction(); - var context = await this.PrepareIndexedRunAsync(dataSource, refreshMode, indexer.Share, runControl, promptInjectionScope, token); + context = await this.PrepareIndexedRunAsync(dataSource, refreshMode, indexer.Share, runControl, promptInjectionScope, token); if (context is null) return; @@ -932,6 +987,13 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } finally { + // + // Only a run which worked on documents put load on the embedding provider, so only such + // a step makes the next one wait, even when it failed or stopped halfway: + // + if (context?.ShareStartedUtc is not null) + this.pauseBetweenSteps.RecordStepEnded(DateTimeOffset.UtcNow); + // // A pass lives only while the last step left more to do. Whatever else ended this run, // the next one begins anew, and tries every document which failed so far once more: diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md b/app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md index ce067b543..79bcb9250 100644 --- a/app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md +++ b/app/MindWork AI Studio/Tools/Services/Indexing/AGENTS.md @@ -40,6 +40,10 @@ The parts: a mailbox at most 500 mails. A large data source is indexed in steps, one run each. A run which returns `MORE_TO_DO` gets its next step in the back lane of `DataSourceRunQueue`, behind every data source which begins a pass, so a mailbox waits for one step at most. +- **`PauseBetweenSteps`** - a gap after every step which worked on documents, so that many people can share one + embedding server. The next run, of whichever data source, waits for a random pause between the bounds of the + settings (`PauseRange`: 0 to 300 seconds, none by default), less the time which passed since the step ended. + Not to be confused with the pause of the indexing as a whole (`IndexingPauses`), which a backup takes. - **`IndexingPass`** - the steps of one data source, from its first step until one of them is done. It remembers which documents failed during an earlier step, so the next steps do not read them again, and which failures were reported to the user already. @@ -98,6 +102,13 @@ Rules which are easy to break: holds two stores which do not fit together. Discarding the stored index of a data source, when it is deleted or repaired, waits for the end of a pause as well (`WaitWhilePausedAsync`); the user interface asks `IsIndexingPaused` first and tells the user instead of waiting. +- **The pause between steps lies behind the run gate.** `ProcessDataSourceRunAsync` waits for it right after + `runGate.WaitAsync`, before the run resolves anything: only one run waits, its token ends the wait, and its + data source still shows as queued. Never wait for it in an indexer, where the run would hold its signature and + its stores, and show as running. Afterward, the run looks up its data source anew, since the user may have + changed it meanwhile. A step opens a gap only once it asked `HasDoneItsShare` (`IndexedRunContext.ShareStartedUtc`), + so a run which found nothing to read leaves none, and a startup check over unchanged data sources does not + slow down. - **A pass lives only while the last step left more to do.** The service drops the pass of a data source after every other outcome and every exception, and when the stored index is reset. A retry the user asks for, and a step with other embedding settings or another selection of documents, begin a new pass diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/PauseBeforeStep.cs b/app/MindWork AI Studio/Tools/Services/Indexing/PauseBeforeStep.cs new file mode 100644 index 000000000..81eaade35 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/Indexing/PauseBeforeStep.cs @@ -0,0 +1,14 @@ +namespace AIStudio.Tools.Services.Indexing; + +/// +/// How long the indexing step which is about to start waits, cf. PauseBetweenSteps.GetPauseBeforeNextStep. +/// +/// The pause drawn for the gap since the last step. +/// What is left of the pause now. More than zero, unless this is NONE. +internal sealed record PauseBeforeStep(TimeSpan Pause, TimeSpan Remaining) +{ + /// + /// The step starts right away. + /// + public static readonly PauseBeforeStep NONE = new(TimeSpan.Zero, TimeSpan.Zero); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/PauseBetweenSteps.cs b/app/MindWork AI Studio/Tools/Services/Indexing/PauseBetweenSteps.cs new file mode 100644 index 000000000..c52c6d10a --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/Indexing/PauseBetweenSteps.cs @@ -0,0 +1,75 @@ +namespace AIStudio.Tools.Services.Indexing; + +/// +/// The pause between two indexing steps, which spares an embedding server that many people share. +/// +/// +/// Not to be confused with the pauses of the indexing as a whole, e.g., while a backup copies the +/// indexes, cf. IndexingPauses.

+/// +/// The pause is a gap between the end of one step and the start of the next, whichever data source +/// either of them belongs to: it is about the load on the embedding server, not about the order of +/// the data sources. Only a step which worked on documents opens a gap. Its pause is drawn once the +/// next step is about to start, from the bounds which apply then, and the time which passed in the +/// meantime counts toward it. When no step comes, nothing is drawn. +/// +internal sealed class PauseBetweenSteps +{ + private readonly Lock stateLock = new(); + + /// + /// When the last step which worked on documents ended. Null before the first one. + /// + private DateTimeOffset? lastStepEndedUtc; + + /// + /// The pause drawn for the gap since the last step. Null until a step is about to start. + /// + private TimeSpan? drawnPause; + + /// + /// Notes that a step which worked on documents ended, which opens a new gap. + /// + /// The current time, passed in so the pause can be tested. + public void RecordStepEnded(DateTimeOffset nowUtc) + { + lock (this.stateLock) + { + this.lastStepEndedUtc = nowUtc; + this.drawnPause = null; + } + } + + /// + /// Works out how long the step which is about to start waits. + /// + /// + /// The first step which asks after a gap opened draws its pause, with both bounds included. A step + /// which stops while it waits, e.g., because its data source was deleted, leaves the same pause + /// to the step after it, which waits only for the rest. + /// + /// The current time, passed in so the pause can be tested. + /// The bounds which apply now. + /// Draws the pause. + /// The pause and what is left of it, or NONE when the step starts right away. + public PauseBeforeStep GetPauseBeforeNextStep(DateTimeOffset nowUtc, PauseRange range, Random random) + { + lock (this.stateLock) + { + if (this.lastStepEndedUtc is not { } stepEndedUtc) + return PauseBeforeStep.NONE; + + if (this.drawnPause is null) + { + // Nothing is drawn, so bounds which are set before the next step still apply to this gap: + if (range.IsNone) + return PauseBeforeStep.NONE; + + this.drawnPause = TimeSpan.FromSeconds(random.Next(range.MinimumSeconds, range.MaximumSeconds + 1)); + } + + var remaining = stepEndedUtc + this.drawnPause.Value - nowUtc; + return remaining > TimeSpan.Zero ? new(this.drawnPause.Value, remaining) : PauseBeforeStep.NONE; + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/PauseRange.cs b/app/MindWork AI Studio/Tools/Services/Indexing/PauseRange.cs new file mode 100644 index 000000000..7b1adb856 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/Indexing/PauseRange.cs @@ -0,0 +1,68 @@ +using AIStudio.Settings; +using AIStudio.Settings.DataModel; + +namespace AIStudio.Tools.Services.Indexing; + +/// +/// The bounds of the random pause between two indexing steps, cf. PauseBetweenSteps. +/// +/// The shortest pause. Between zero and MaximumSeconds. +/// The longest pause, which may be drawn as well. At most DataDataSourceIndexing.MAX_PAUSE_BETWEEN_STEPS_SECONDS. +internal sealed record PauseRange(int MinimumSeconds, int MaximumSeconds) +{ + /// + /// No pause at all, which is the default. + /// + public static readonly PauseRange NONE = new(0, 0); + + /// + /// Whether the next step starts right away. + /// + public bool IsNone => this.MaximumSeconds is 0; + + /// + /// The bounds which apply now, as the user set them and as configuration plugins lock them. + /// + /// The settings of the indexing. + /// The bounds which apply. + public static PauseRange FromSettings(DataDataSourceIndexing settings) + { + var minimumSeconds = settings.MinimumPauseBetweenStepsSeconds; + var maximumSeconds = settings.MaximumPauseBetweenStepsSeconds; + + // + // Which bound a configuration plugin locks matters only while the bounds are the wrong way + // round. Asking reads the managed state of the whole app, which bounds in order do without: + // + if (minimumSeconds <= maximumSeconds) + return GetEffective(minimumSeconds, false, maximumSeconds, false); + + var isMinimumLocked = ManagedConfiguration.TryGet(x => x.App.DataSourceIndexing, x => x.MinimumPauseBetweenStepsSeconds, out var minimumMeta) && minimumMeta.IsLocked; + var isMaximumLocked = ManagedConfiguration.TryGet(x => x.App.DataSourceIndexing, x => x.MaximumPauseBetweenStepsSeconds, out var maximumMeta) && maximumMeta.IsLocked; + return GetEffective(minimumSeconds, isMinimumLocked, maximumSeconds, isMaximumLocked); + } + + /// + /// Works out the bounds which apply, so that the lower one never exceeds the upper one. + /// + /// + /// A bound which a configuration plugin locks wins, and the other one adjusts to it. A bound the + /// plugin only presets counts like one the user set. Otherwise, the lower bound wins, i.e., the + /// longer pause: it spares the embedding server, which is what the pause is for. That holds as + /// well when a plugin locks both bounds the wrong way round. + /// + /// The lower bound as set. + /// Whether a configuration plugin locks the lower bound. + /// The upper bound as set. + /// Whether a configuration plugin locks the upper bound. + /// The bounds which apply. + public static PauseRange GetEffective(int minimumSeconds, bool isMinimumLocked, int maximumSeconds, bool isMaximumLocked) + { + var minimum = Math.Clamp(minimumSeconds, 0, DataDataSourceIndexing.MAX_PAUSE_BETWEEN_STEPS_SECONDS); + var maximum = Math.Clamp(maximumSeconds, 0, DataDataSourceIndexing.MAX_PAUSE_BETWEEN_STEPS_SECONDS); + if (isMaximumLocked && !isMinimumLocked) + return new(Math.Min(minimum, maximum), maximum); + + return new(minimum, Math.Max(minimum, maximum)); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md index 88b09de35..a6a7ba0e6 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md @@ -33,6 +33,7 @@ - Added a way for IT departments to provide the PowerPoint template of the Slide Assistant, its layouts for kinds of slides, which slides of the template a presentation keeps, the maximum number of bullet points per slide, whether slides keep empty picture placeholders, and a picture library, either fixed or as a default which users may change. The template and the picture library may come with the configuration of your organization. - Added a way to read what the AI received for an answer. In the list of sources below an answer, the eye button next to a source shows the passages the AI got from it, for example, from your data sources, a web page, a mail, or your Confluence wiki, as they were after AI Studio removed suspicious instructions. They stay readable when you open the chat again later. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution. - Added organization-wide management for whether AI Studio reads a file attached to a chat again after it has changed. IT departments can set this for everybody, either fixed or as a default which users may change. +- Added a way for IT departments to slow down the indexing of local data sources, so that a few people with large folders or mailboxes no longer keep a shared embedding server busy for hours. IT departments set the shortest and the longest pause between the indexing steps, either fixed or as a default which users may change. - Improved the Read Web Page tool. When you have not allowed the AI to choose web addresses freely, the tool now refuses every address that does not appear in your chat or in what the tools returned, instead of only asking the AI not to read it. - Improved the installation on macOS: AI Studio is now signed and checked by Apple, so it opens like any other app. You no longer need the Terminal. - Improved the installation on Windows: AI Studio is now signed, so Windows shows who published it. For a while, Windows may still ask you to confirm the installation until it knows the app better. @@ -41,7 +42,7 @@ - Improved the button in the chat for moving the chat to a workspace. You can now use it whenever the workspace list lets you move the chat as well, for example, right after you attached an audio or video file and before you sent your first message. - Improved how AI Studio keeps the message you have not sent yet. When AI Studio stores your chats automatically, your message and its attachments now stay with the chat, and they are back when you open the chat again, for example, after you switched to another chat. - Improved how AI Studio keeps the message of a new chat. When AI Studio stores your chats automatically and you leave a new chat before you send your message, AI Studio now stores the chat with your message. You find it among your disappearing chats, or in its workspace when you started it there. -- Improved how AI Studio indexes large local data sources. It now works through them in steps of a few minutes, so your other data sources, such as your mailboxes, stay up to date in between. +- Improved how AI Studio indexes large local data sources. It now works through them in steps of a few minutes, so your other data sources, such as your mailboxes, stay up to date in between, and you can set a pause between the steps to spare an embedding server you share with others. - Improved the plugins page with the details of each plugin. The info button next to the name of a plugin shows, among other things, its version, its authors, and where to get support. - Improved the details of a plugin on the plugins page, which now show when the plugin was last changed, in your time zone. Plugin authors can state this time, and AI Studio fills it in by itself when it writes a plugin for you, for example, with the Assistant Builder. - Improved local data sources for folders. You can now choose which files AI Studio indexes, for example, only PDF files, and whether it includes subfolders as well as hidden files and folders. By default, AI Studio now also indexes text files it did not recognize before, such as program code of any kind or a LICENSE file, so a Git repository makes a useful data source. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution. diff --git a/app/Tests/Tools/DataSourceRelocationWorkflowTests.cs b/app/Tests/Tools/DataSourceRelocationWorkflowTests.cs index 906331263..64fff8bbe 100644 --- a/app/Tests/Tools/DataSourceRelocationWorkflowTests.cs +++ b/app/Tests/Tools/DataSourceRelocationWorkflowTests.cs @@ -68,7 +68,7 @@ public sealed class DataSourceRelocationWorkflowTests clients.Add(DatabaseRole.INDEX_STORE, this.store.Client); clients.Add(DatabaseRole.VECTOR_STORE, this.vectors); var guard = new PromptInjectionGuardService(null!, this.settings, NullLogger.Instance, NullLoggerFactory.Instance); - this.service = new DataSourceEmbeddingService(this.settings, null!, this.databases, guard, null!, NullLogger.Instance); + this.service = new DataSourceEmbeddingService(this.settings, null!, this.databases, guard, null!, new ThreadSafeRandom(), NullLogger.Instance); this.lifetime = new CancellationTokenSource(); this.chunkIds.Clear(); this.runningTasks.Clear(); diff --git a/app/Tests/Tools/PauseBetweenStepsTests.cs b/app/Tests/Tools/PauseBetweenStepsTests.cs new file mode 100644 index 000000000..73cd36dda --- /dev/null +++ b/app/Tests/Tools/PauseBetweenStepsTests.cs @@ -0,0 +1,143 @@ +using AIStudio.Tools.Services.Indexing; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks the pause between two indexing steps, which spares an embedding server that many people share. +/// +/// +/// The pause is a gap after every step which worked on documents. It is drawn only once a step is +/// about to start, and the time which passed by then counts toward it. +/// +[TestFixture] +public sealed class PauseBetweenStepsTests +{ + private static readonly DateTimeOffset STEP_ENDED = new(2026, 10, 27, 14, 0, 0, TimeSpan.Zero); + + private static readonly PauseRange THIRTY_TO_SIXTY_SECONDS = new(30, 60); + + [Test] + public void BeforeTheFirstStepNothingWaits() + { + var random = new LargestRandom(); + var pause = new PauseBetweenSteps().GetPauseBeforeNextStep(STEP_ENDED, THIRTY_TO_SIXTY_SECONDS, random); + + Assert.Multiple(() => + { + Assert.That(pause, Is.EqualTo(PauseBeforeStep.NONE)); + Assert.That(random.Draws, Is.Zero); + }); + } + + [Test] + public void TheNextStepWaitsForTheWholePauseRightAfterAStep() + { + var pauses = new PauseBetweenSteps(); + pauses.RecordStepEnded(STEP_ENDED); + + var pause = pauses.GetPauseBeforeNextStep(STEP_ENDED, THIRTY_TO_SIXTY_SECONDS, new LargestRandom()); + + Assert.That(pause, Is.EqualTo(new PauseBeforeStep(TimeSpan.FromSeconds(60), TimeSpan.FromSeconds(60)))); + } + + [Test] + public void TheUpperBoundCanBeDrawn() + { + var pauses = new PauseBetweenSteps(); + pauses.RecordStepEnded(STEP_ENDED); + + var pause = pauses.GetPauseBeforeNextStep(STEP_ENDED, new PauseRange(45, 45), new LargestRandom()); + + Assert.That(pause.Pause, Is.EqualTo(TimeSpan.FromSeconds(45))); + } + + [Test] + public void TimeWhichPassedSinceTheStepCountsTowardThePause() + { + var pauses = new PauseBetweenSteps(); + pauses.RecordStepEnded(STEP_ENDED); + + var pause = pauses.GetPauseBeforeNextStep(STEP_ENDED + TimeSpan.FromSeconds(25), THIRTY_TO_SIXTY_SECONDS, new LargestRandom()); + + Assert.That(pause, Is.EqualTo(new PauseBeforeStep(TimeSpan.FromSeconds(60), TimeSpan.FromSeconds(35)))); + } + + [Test] + public void AStepWhichStartsLaterAnywayDoesNotWait() + { + var pauses = new PauseBetweenSteps(); + pauses.RecordStepEnded(STEP_ENDED); + + var pause = pauses.GetPauseBeforeNextStep(STEP_ENDED + TimeSpan.FromSeconds(60), THIRTY_TO_SIXTY_SECONDS, new LargestRandom()); + + Assert.That(pause, Is.EqualTo(PauseBeforeStep.NONE)); + } + + [Test] + public void NothingIsDrawnWhileThereIsNoPause() + { + var pauses = new PauseBetweenSteps(); + pauses.RecordStepEnded(STEP_ENDED); + var random = new LargestRandom(); + + var withoutPause = pauses.GetPauseBeforeNextStep(STEP_ENDED, PauseRange.NONE, random); + var withPause = pauses.GetPauseBeforeNextStep(STEP_ENDED + TimeSpan.FromSeconds(10), THIRTY_TO_SIXTY_SECONDS, random); + + Assert.Multiple(() => + { + Assert.That(withoutPause, Is.EqualTo(PauseBeforeStep.NONE)); + Assert.That(withPause, Is.EqualTo(new PauseBeforeStep(TimeSpan.FromSeconds(60), TimeSpan.FromSeconds(50)))); + Assert.That(random.Draws, Is.EqualTo(1)); + }); + } + + [Test] + public void AStepWhichStoppedWhileWaitingLeavesTheRestOfThePauseToTheNextOne() + { + var pauses = new PauseBetweenSteps(); + pauses.RecordStepEnded(STEP_ENDED); + var random = new LargestRandom(); + + pauses.GetPauseBeforeNextStep(STEP_ENDED, THIRTY_TO_SIXTY_SECONDS, random); + var nextPause = pauses.GetPauseBeforeNextStep(STEP_ENDED + TimeSpan.FromSeconds(20), new PauseRange(1, 1), random); + + Assert.Multiple(() => + { + Assert.That(nextPause, Is.EqualTo(new PauseBeforeStep(TimeSpan.FromSeconds(60), TimeSpan.FromSeconds(40)))); + Assert.That(random.Draws, Is.EqualTo(1)); + }); + } + + [Test] + public void ANewStepOpensANewGap() + { + var pauses = new PauseBetweenSteps(); + pauses.RecordStepEnded(STEP_ENDED); + var random = new LargestRandom(); + pauses.GetPauseBeforeNextStep(STEP_ENDED, THIRTY_TO_SIXTY_SECONDS, random); + + var nextStepEnded = STEP_ENDED + TimeSpan.FromMinutes(6); + pauses.RecordStepEnded(nextStepEnded); + var pause = pauses.GetPauseBeforeNextStep(nextStepEnded + TimeSpan.FromSeconds(5), new PauseRange(10, 10), random); + + Assert.Multiple(() => + { + Assert.That(pause, Is.EqualTo(new PauseBeforeStep(TimeSpan.FromSeconds(10), TimeSpan.FromSeconds(5)))); + Assert.That(random.Draws, Is.EqualTo(2)); + }); + } + + /// + /// Always draws the largest value it may, and counts how often it was asked. + /// + private sealed class LargestRandom : Random + { + public int Draws { get; private set; } + + public override int Next(int minValue, int maxValue) + { + this.Draws++; + return maxValue - 1; + } + } +} \ No newline at end of file diff --git a/app/Tests/Tools/PauseRangeTests.cs b/app/Tests/Tools/PauseRangeTests.cs new file mode 100644 index 000000000..035b4e4c1 --- /dev/null +++ b/app/Tests/Tools/PauseRangeTests.cs @@ -0,0 +1,72 @@ +using AIStudio.Settings.DataModel; +using AIStudio.Tools.Services.Indexing; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks which bounds of the pause between two indexing steps apply, as the user set them and as configuration plugins lock them. +/// +[TestFixture] +public sealed class PauseRangeTests +{ + [Test] + public void BoundsInOrderStayAsTheyAre() + { + Assert.That(PauseRange.GetEffective(10, false, 20, false), Is.EqualTo(new PauseRange(10, 20))); + } + + [Test] + public void AnUpperBoundBelowTheLowerOneIsRaisedToIt() + { + Assert.That(PauseRange.GetEffective(60, false, 20, false), Is.EqualTo(new PauseRange(60, 60))); + } + + [Test] + public void ALockedLowerBoundRaisesTheUpperOne() + { + Assert.That(PauseRange.GetEffective(60, true, 20, false), Is.EqualTo(new PauseRange(60, 60))); + } + + [Test] + public void ALockedUpperBoundLowersTheLowerOne() + { + Assert.That(PauseRange.GetEffective(60, false, 20, true), Is.EqualTo(new PauseRange(20, 20))); + } + + [Test] + public void BothBoundsLockedTheWrongWayRoundLetTheLongerPauseWin() + { + Assert.That(PauseRange.GetEffective(60, true, 20, true), Is.EqualTo(new PauseRange(60, 60))); + } + + [Test] + public void BoundsOutsideTheAllowedRangeAreClamped() + { + Assert.That(PauseRange.GetEffective(-5, false, 1_000, false), Is.EqualTo(new PauseRange(0, DataDataSourceIndexing.MAX_PAUSE_BETWEEN_STEPS_SECONDS))); + } + + [Test] + public void BoundsInOrderComeFromTheSettingsWithoutAskingForLocks() + { + var settings = new DataDataSourceIndexing + { + MinimumPauseBetweenStepsSeconds = 10, + MaximumPauseBetweenStepsSeconds = 20 + }; + + Assert.That(PauseRange.FromSettings(settings), Is.EqualTo(new PauseRange(10, 20))); + } + + [Test] + public void OnlyZeroForBothBoundsMeansNoPause() + { + Assert.Multiple(() => + { + Assert.That(PauseRange.GetEffective(0, false, 0, false).IsNone, Is.True); + Assert.That(PauseRange.GetEffective(0, false, 5, false).IsNone, Is.False); + Assert.That(PauseRange.GetEffective(30, false, 0, false), Is.EqualTo(new PauseRange(30, 30))); + Assert.That(new DataDataSourceIndexing().MinimumPauseBetweenStepsSeconds, Is.Zero); + Assert.That(new DataDataSourceIndexing().MaximumPauseBetweenStepsSeconds, Is.Zero); + }); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/ToolCalling/MailDraftSyncTests.cs b/app/Tests/Tools/ToolCalling/MailDraftSyncTests.cs index 9782c409b..771707550 100644 --- a/app/Tests/Tools/ToolCalling/MailDraftSyncTests.cs +++ b/app/Tests/Tools/ToolCalling/MailDraftSyncTests.cs @@ -117,5 +117,5 @@ public sealed class MailDraftSyncTests : ToolRegistryTestBase } // Queueing a run needs none of the services which carry it out: - private DataSourceEmbeddingService CreateService() => new(this.SettingsManager, null!, null!, null!, null!, NullLogger.Instance); + private DataSourceEmbeddingService CreateService() => new(this.SettingsManager, null!, null!, null!, null!, null!, NullLogger.Instance); } \ No newline at end of file