From b2f46a5611b0ea11962f8b6e22a64cf1f4878367 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 17 Sep 2026 09:48:00 +0200 Subject: [PATCH] Show what happens to the data sources while they are re-indexed (#979) --- .../Assistants/I18N/allTexts.lua | 18 ++ .../Components/DataSourceSelection.razor | 61 ++---- .../Components/DataSourceSelection.razor.cs | 34 ++++ .../Components/DataSourceSelectionRow.razor | 24 +++ .../DataSourceSelectionRow.razor.cs | 35 ++++ app/MindWork AI Studio/Pages/Embeddings.razor | 8 +- .../Pages/Embeddings.razor.cs | 74 ++++++- .../plugin.lua | 188 ++++++++++-------- .../plugin.lua | 24 ++- .../Tools/AllowedSelectedDataSources.cs | 11 +- .../IndexStore/DataSourceIndexState.cs | 19 ++ .../Databases/IndexStore/IndexStoreClient.cs | 8 + .../IndexStore/NoIndexStoreClient.cs | 5 +- .../SqliteIndexStoreClientImplementation.cs | 10 + .../Services/DataSourceEmbeddingService.cs | 173 +++++++++++++++- .../Services/DataSourceEmbeddingStatus.cs | 9 +- .../DataSourceLocalRetrievalService.cs | 19 +- .../Tools/Services/DataSourceService.cs | 64 +++++- app/Tests/Tools/ReindexPendingTests.cs | 89 +++++++++ 19 files changed, 709 insertions(+), 164 deletions(-) create mode 100644 app/MindWork AI Studio/Components/DataSourceSelectionRow.razor create mode 100644 app/MindWork AI Studio/Components/DataSourceSelectionRow.razor.cs create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/DataSourceIndexState.cs create mode 100644 app/Tests/Tools/ReindexPendingTests.cs diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index df20a810..b288b48f 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -3952,6 +3952,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T700666808"] = "Mana -- Available Data Sources UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T86053874"] = "Available Data Sources" +-- This data source is waiting to be indexed again. Until that is finished, it cannot be searched. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T1692539409"] = "This data source is waiting to be indexed again. Until that is finished, it cannot be searched." + -- Tools (Optional) UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERFORM::T1019749907"] = "Tools (Optional)" @@ -9097,12 +9100,18 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1143368054"] = "Other cause" -- Current file: {0} UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1166856644"] = "Current file: {0}" +-- File {0} of {1} is being indexed: block {2}, page {3}. +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1298290372"] = "File {0} of {1} is being indexed: block {2}, page {3}." + -- Could not open the file location: {0} UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1455637941"] = "Could not open the file location: {0}" -- Open the settings UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1582896271"] = "Open the settings" +-- File {0} of {1} is being indexed. +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1616414701"] = "File {0} of {1} is being indexed." + -- Tried again during the next run UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1946414905"] = "Tried again during the next run" @@ -9145,6 +9154,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T309404893"] = "Failed files: {0}" -- Show this file in the file browser of your system UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3273105305"] = "Show this file in the file browser of your system" +-- Data source {0} of {1} is being worked on. The others are waiting their turn. +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3389674086"] = "Data source {0} of {1} is being worked on. The others are waiting their turn." + -- Unknown error UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3461425987"] = "Unknown error" @@ -9160,6 +9172,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3865031940"] = "Actions" -- Skipped until the file changes UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T542386347"] = "Skipped until the file changes" +-- File {0} of {1} is being indexed: block {2}. +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T615458954"] = "File {0} of {1} is being indexed: block {2}." + -- File UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T723007075"] = "File" @@ -11983,6 +11998,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T29 -- The data source '{0}' was left out of the answer: its embedding provider '{1}' did not return a vector for your message. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T3469074321"] = "The data source '{0}' was left out of the answer: its embedding provider '{1}' did not return a vector for your message." +-- The data source '{0}' was left out of the answer: it is being indexed again and cannot be searched until that is finished. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4022014739"] = "The data source '{0}' was left out of the answer: it is being indexed again and cannot be searched until that is finished." + -- Page {0} UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Page {0}" diff --git a/app/MindWork AI Studio/Components/DataSourceSelection.razor b/app/MindWork AI Studio/Components/DataSourceSelection.razor index de7ae518..4a145120 100644 --- a/app/MindWork AI Studio/Components/DataSourceSelection.razor +++ b/app/MindWork AI Studio/Components/DataSourceSelection.razor @@ -69,7 +69,7 @@ @switch (this.aiBasedSourceSelection) { - case true when this.availableDataSources.Count == 0: + case true when this.GetListedDataSources().Count == 0: @T("Your data sources cannot be used with the selected providers due to data privacy or confidence-level requirements, or they are currently unavailable.") @@ -81,7 +81,7 @@ break; - case false when this.availableDataSources.Count == 0: + case false when this.GetListedDataSources().Count == 0: @T("Your data sources cannot be used with the selected providers due to data privacy or confidence-level requirements, or they are currently unavailable.") @@ -90,22 +90,9 @@ case false: - @foreach (var source in this.availableDataSources) + @foreach (var source in this.GetListedDataSources()) { - - - - @source.Name - - @if (source is IInternalDataSource internalSource) - { - - - - - } - - + } @@ -115,22 +102,9 @@ - @foreach (var source in this.availableDataSources) + @foreach (var source in this.GetListedDataSources()) { - - - - @source.Name - - @if (source is IInternalDataSource internalSource) - { - - - - - } - - + } @@ -166,13 +140,13 @@ break; } - @if (!this.aiBasedSourceSelection && this.GetUnavailablePreselectedDataSources().Count > 0) + @if (!this.aiBasedSourceSelection && this.GetUnavailablePreselectedDataSourcesToList().Count > 0) { @T("These data sources are preselected, but cannot be used right now, either due to data privacy or confidence-level requirements, or because they are unavailable:")