Merge branch 'main' into feature/configurable-transcription-bitrate

app/MindWork AI Studio/Assistants/I18N/allTexts.lua

@@ -3808,6 +3808,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2675917723"] = "No
 -- No valid embedding
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2698203405"] = "No valid embedding"
 
+-- Repair this data source by indexing it anew
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2771708618"] = "Repair this data source by indexing it anew"
+
 -- Embedding
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2838542994"] = "Embedding"
 
@@ -3955,6 +3958,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T86053874"] = "Avail
 -- This data source is waiting to be indexed again. Until that is finished, it cannot be searched.
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T1692539409"] = "This data source is waiting to be indexed again. Until that is finished, it cannot be searched."
 
+-- The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there.
+UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T4047623216"] = "The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there."
+
 -- Tools (Optional)
 UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERFORM::T1019749907"] = "Tools (Optional)"
 
@@ -9145,6 +9151,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2525374657"] = "{0} of {1} files
 -- Background embeddings
 UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2547971789"] = "Background embeddings"
 
+-- Repair this data source by indexing it anew
+UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2771708618"] = "Repair this data source by indexing it anew"
+
 -- Refresh this data source
 UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2901874229"] = "Refresh this data source"
 
@@ -10834,6 +10843,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
 -- Qdrant Edge is not available.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available."
 
+-- Repair Data Source
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Repair Data Source"
+
+-- The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T857336889"] = "The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?"
+
 -- The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::ERICLIENT::DATAMODEL::PROVIDERTYPEEXTENSIONS::T1555790630"] = "The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment."
 
@@ -12028,6 +12043,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T4515612
 -- The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T667058890"] = "The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider."
 
+-- The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T831900720"] = "The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."
+
 -- The folder '{0}' does not exist.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T871336081"] = "The folder '{0}' does not exist."
 
@@ -12076,6 +12094,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T40
 -- Page {0}
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Page {0}"
 
+-- The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings.
+UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T59210871"] = "The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings."
+
 -- The data source '{0}' was left out of the answer because searching it failed.
 UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T934856625"] = "The data source '{0}' was left out of the answer because searching it failed."
 

app/MindWork AI Studio/Components/DataSourceBlockReason.cs

@@ -0,0 +1,29 @@
+namespace AIStudio.Components;
+
+/// <summary>
+/// Why a data source is listed in the selection, but cannot be picked.
+/// </summary>
+/// <remarks>
+/// A reason rather than a yes or no, because the row has to say something different for each of
+/// them: one asks the user to wait, the other one asks them to act. Asking somebody to wait for
+/// something which will never happen on its own is the worse of the two mistakes.
+/// </remarks>
+public enum DataSourceBlockReason
+{
+    /// <summary>
+    /// Nothing is in the way, the data source can be picked.
+    /// </summary>
+    NONE,
+
+    /// <summary>
+    /// The index has to be built anew before this data source can answer a search. This passes by
+    /// itself, as soon as the background indexing has worked through the data source.
+    /// </summary>
+    AWAITING_REINDEX,
+
+    /// <summary>
+    /// The index cannot be read anymore. This does not pass by itself: only the user can start the
+    /// rebuild, because it sends every document to the embedding provider once more.
+    /// </summary>
+    NEEDS_REPAIR,
+}

app/MindWork AI Studio/Components/DataSourceManagement.razor

@@ -59,6 +59,18 @@
                 <MudTooltip Text="@T("Information")">
                     <MudIconButton Color="Color.Info" Icon="@Icons.Material.Outlined.Info" OnClick="@(() => this.ShowInformation(context))"/>
                 </MudTooltip>
+                @*
+                    Outside the two branches below on purpose: an index which cannot be read is a
+                    matter of this machine, not of the configuration. Hiding the repair for a data
+                    source the organization manages would leave it locked out of every chat with no
+                    way back, and the selection points here for it.
+                *@
+                @if (this.CanRepairDataSource(context))
+                {
+                    <MudTooltip Text="@T("Repair this data source by indexing it anew")">
+                        <MudIconButton Color="Color.Warning" Icon="@Icons.Material.Filled.Build" OnClick="@(() => this.RepairDataSource(context))"/>
+                    </MudTooltip>
+                }
                 @if (context.IsEnterpriseConfiguration)
                 {
                     <MudTooltip Text="@T("This data source is managed by your organization.")">

app/MindWork AI Studio/Components/DataSourceManagement.razor.cs

@@ -128,6 +128,25 @@ public partial class DataSourceManagement : MSGComponentBase
         return this.SettingsManager.ConfigurationData.DataSources.Any(this.CanRefreshDataSource);
     }
 
+    /// <remarks>
+    /// Shown only while the index of this data source cannot be read. The refresh button next to it
+    /// stays as it is: it would open the same store and fail the same way, but it is offered for
+    /// every internal data source regardless of state, and singling this one out would say more
+    /// about the state than that button ever has.
+    /// </remarks>
+    private bool CanRepairDataSource(IDataSource dataSource)
+    {
+        return this.DataSourceEmbeddingService.NeedsIndexRepair(dataSource);
+    }
+
+    private async Task RepairDataSource(IDataSource dataSource)
+    {
+        if (!this.CanRepairDataSource(dataSource))
+            return;
+
+        await DataSourceRepair.ConfirmAndRepairAsync(this.DialogService, this.DataSourceEmbeddingService, dataSource.Id, dataSource.Name);
+    }
+
     private async Task AutomaticRefreshChanged(bool enabled)
     {
         this.SettingsManager.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh = enabled;

app/MindWork AI Studio/Components/DataSourceSelection.razor

@@ -92,7 +92,7 @@
                                         <MudList T="IDataSource" Dense="@true" Class="data-source-rows" SelectionMode="@this.GetListSelectionMode()" @bind-SelectedValues:get="@this.selectedDataSources" @bind-SelectedValues:set="@this.SelectionChanged" Style="max-height: 14em;">
                                             @foreach (var source in this.GetListedDataSources())
                                             {
-                                                <DataSourceSelectionRow DataSource="@source" IsAwaitingReindex="@this.IsAwaitingReindex(source)"/>
+                                                <DataSourceSelectionRow DataSource="@source" BlockReason="@this.GetBlockReason(source)"/>
                                             }
                                         </MudList>
                                     </MudField>
@@ -104,7 +104,7 @@
                                             <MudList T="IDataSource" Dense="@true" Class="data-source-rows" SelectionMode="MudBlazor.SelectionMode.SingleSelection" SelectedValues="@this.selectedDataSources" Style="max-height: 14em;">
                                                 @foreach (var source in this.GetListedDataSources())
                                                 {
-                                                    <DataSourceSelectionRow DataSource="@source" IsAwaitingReindex="@this.IsAwaitingReindex(source)"/>
+                                                    <DataSourceSelectionRow DataSource="@source" BlockReason="@this.GetBlockReason(source)"/>
                                                 }
                                             </MudList>
                                         </ExpansionPanel>

app/MindWork AI Studio/Components/DataSourceSelection.razor.cs

@@ -51,6 +51,8 @@ public partial class DataSourceSelection : MSGComponentBase
     private IReadOnlyList<IDataSource> availableDataSources = [];
     private IReadOnlyList<IDataSource> dataSourcesAwaitingReindex = [];
     private HashSet<string> dataSourceIdsAwaitingReindex = new(StringComparer.Ordinal);
+    private IReadOnlyList<IDataSource> dataSourcesNeedingRepair = [];
+    private HashSet<string> dataSourceIdsNeedingRepair = new(StringComparer.Ordinal);
     private IReadOnlyCollection<IDataSource> selectedDataSources = [];
     private bool aiBasedSourceSelection;
     private bool aiBasedValidation;
@@ -230,15 +232,33 @@ public partial class DataSourceSelection : MSGComponentBase
         this.availableDataSources = sources.AllowedDataSources;
         this.dataSourcesAwaitingReindex = sources.DataSourcesAwaitingReindex;
         this.dataSourceIdsAwaitingReindex = sources.DataSourcesAwaitingReindex.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
+        this.dataSourcesNeedingRepair = sources.DataSourcesNeedingRepair;
+        this.dataSourceIdsNeedingRepair = sources.DataSourcesNeedingRepair.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
         this.selectedDataSources = sources.SelectedDataSources;
         this.waitingForDataSources = false;
         this.StateHasChanged();
     }
 
-    private bool IsAwaitingReindex(IDataSource dataSource) => this.dataSourceIdsAwaitingReindex.Contains(dataSource.Id);
+    /// <summary>
+    /// Why a data source is listed but cannot be picked, if it cannot.
+    /// </summary>
+    /// <remarks>
+    /// The repair is asked about first. The service hands a data source to one of the two lists
+    /// only, but should that ever change, the reason the user can act on is the one worth showing.
+    /// </remarks>
+    private DataSourceBlockReason GetBlockReason(IDataSource dataSource)
+    {
+        if (this.dataSourceIdsNeedingRepair.Contains(dataSource.Id))
+            return DataSourceBlockReason.NEEDS_REPAIR;
+
+        if (this.dataSourceIdsAwaitingReindex.Contains(dataSource.Id))
+            return DataSourceBlockReason.AWAITING_REINDEX;
+
+        return DataSourceBlockReason.NONE;
+    }
 
     /// <summary>
-    /// The data sources the list shows: the usable ones, plus the ones waiting for their index.
+    /// The data sources the list shows: the usable ones, plus the ones which cannot be searched.
     /// </summary>
     /// <remarks>
     /// Kept in the order the data sources were configured in, rather than usable ones first. A row
@@ -247,11 +267,12 @@ public partial class DataSourceSelection : MSGComponentBase
     /// </remarks>
     private IReadOnlyList<IDataSource> GetListedDataSources()
     {
-        if (this.dataSourcesAwaitingReindex.Count == 0)
+        if (this.dataSourcesAwaitingReindex.Count == 0 && this.dataSourcesNeedingRepair.Count == 0)
             return this.availableDataSources;
 
         var listedIds = this.availableDataSources.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
         listedIds.UnionWith(this.dataSourceIdsAwaitingReindex);
+        listedIds.UnionWith(this.dataSourceIdsNeedingRepair);
         return this.GetConfiguredDataSourcesSnapshot().Where(source => listedIds.Contains(source.Id)).ToList();
     }
 
@@ -259,11 +280,11 @@ public partial class DataSourceSelection : MSGComponentBase
     /// The preselected but unusable data sources the warning box lists.
     /// </summary>
     /// <remarks>
-    /// The ones waiting for their index are left out: they have a row of their own in the list
+    /// The ones which are only blocked are left out: they have a row of their own in the list
     /// above, which says the same thing in the place the user is already looking.
     /// </remarks>
     private IReadOnlyList<IDataSource> GetUnavailablePreselectedDataSourcesToList() =>
-        this.GetUnavailablePreselectedDataSources().Where(source => !this.IsAwaitingReindex(source)).ToList();
+        this.GetUnavailablePreselectedDataSources().Where(source => this.GetBlockReason(source) is DataSourceBlockReason.NONE).ToList();
     
     private async Task EnabledChanged(bool state)
     {

app/MindWork AI Studio/Components/DataSourceSelectionRow.razor

@@ -2,8 +2,8 @@
 @using AIStudio.Provider
 @inherits MSGComponentBase
 
-<MudTooltip Text="@T("This data source is waiting to be indexed again. Until that is finished, it cannot be searched.")" Disabled="@(!this.IsAwaitingReindex)" RootStyle="display: block;" Placement="Placement.Top">
+<MudTooltip Text="@this.GetBlockedTooltip()" Disabled="@(!this.IsBlocked)" RootStyle="display: block;" Placement="Placement.Top">
-    <MudListItem Value="@this.DataSource" Disabled="@this.IsAwaitingReindex">
+    <MudListItem Value="@this.DataSource" Disabled="@this.IsBlocked">
         <MudStack Row="true" AlignItems="AlignItems.Center" Spacing="1" Style="min-width: 0; width: 100%;">
             <MudText Typo="Typo.body2" Style="min-width: 0; white-space: normal; overflow-wrap: anywhere;">
                 @this.DataSource.Name
@@ -11,9 +11,9 @@
             @if (this.DataSource is IInternalDataSource internalSource)
             {
                 <MudSpacer/>
-                @if (this.IsAwaitingReindex)
+                @if (this.IsBlocked)
                 {
-                    <MudIcon Icon="@Icons.Material.Filled.HourglassTop" Size="Size.Small" Color="Color.Warning" Style="flex-shrink: 0;"/>
+                    <MudIcon Icon="@this.GetBlockedIcon()" Size="Size.Small" Color="@this.GetBlockedIconColor()" Style="flex-shrink: 0;"/>
                 }
                 <MudTooltip Text="@internalSource.ConfidenceLevel.GetName()">
                     <MudIcon Icon="@Icons.Material.Filled.Security" Size="Size.Small" Class="confidence-icon" Style="@this.GetConfidenceIconStyle(internalSource)"/>

app/MindWork AI Studio/Components/DataSourceSelectionRow.razor.cs

@@ -10,9 +10,13 @@ namespace AIStudio.Components;
 /// can be used right now.
 /// </summary>
 /// <remarks>
-/// A data source waiting for its index stays in the list instead of disappearing from it, but
+/// A data source which cannot be used stays in the list instead of disappearing from it, but cannot
-/// cannot be picked, and the tooltip says why. The tool selection next to it in the chat answers
+/// be picked, and the tooltip says why. The tool selection next to it in the chat answers the same
-/// the same question the same way.
+/// question the same way.
+///
+/// Why it cannot be used decides what the row says and which icon it wears: an index being built
+/// anew is a matter of waiting, an index which cannot be read is a matter of acting. Both are the
+/// same row otherwise, which is why this is one component with a reason rather than two components.
 ///
 /// The tooltip sits around the list item rather than inside it: a disabled item has its pointer
 /// events switched off and would swallow the hover.
@@ -26,10 +30,31 @@ public partial class DataSourceSelectionRow : MSGComponentBase
     public required IDataSource DataSource { get; set; }
 
     /// <summary>
-    /// Whether this data source has to be indexed anew before it can answer a search.
+    /// Why this data source cannot be picked right now, if it cannot.
     /// </summary>
     [Parameter]
-    public bool IsAwaitingReindex { get; set; }
+    public DataSourceBlockReason BlockReason { get; set; } = DataSourceBlockReason.NONE;
+
+    private bool IsBlocked => this.BlockReason is not DataSourceBlockReason.NONE;
+
+    private string GetBlockedTooltip() => this.BlockReason switch
+    {
+        DataSourceBlockReason.AWAITING_REINDEX => T("This data source is waiting to be indexed again. Until that is finished, it cannot be searched."),
+        DataSourceBlockReason.NEEDS_REPAIR => T("The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there."),
+        _ => string.Empty,
+    };
+
+    private string GetBlockedIcon() => this.BlockReason switch
+    {
+        DataSourceBlockReason.NEEDS_REPAIR => Icons.Material.Filled.ReportProblem,
+        _ => Icons.Material.Filled.HourglassTop,
+    };
+
+    private Color GetBlockedIconColor() => this.BlockReason switch
+    {
+        DataSourceBlockReason.NEEDS_REPAIR => Color.Error,
+        _ => Color.Warning,
+    };
 
     private string GetConfidenceIconStyle(IInternalDataSource dataSource) => $"{dataSource.ConfidenceLevel.SetColorStyle(this.SettingsManager)} flex-shrink: 0;";
 }

app/MindWork AI Studio/Pages/Embeddings.razor

@@ -61,6 +61,12 @@
                                     <MudIconButton Icon="@Icons.Material.Filled.Sync" Color="Color.Info" Size="Size.Small" OnClick="@(async () => await this.RefreshDataSource(status))" />
                                 </MudTooltip>
                             }
+                            @if (this.CanRepair(status))
+                            {
+                                <MudTooltip Text="@T("Repair this data source by indexing it anew")">
+                                    <MudIconButton Icon="@Icons.Material.Filled.Build" Color="Color.Warning" Size="Size.Small" OnClick="@(async () => await this.RepairDataSource(status))" />
+                                </MudTooltip>
+                            }
                         </div>
                     </TitleContent>
                     <ChildContent>

app/MindWork AI Studio/Pages/Embeddings.razor.cs

@@ -318,11 +318,26 @@ public partial class Embeddings : MSGComponentBase
         await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, string.Format(T("Could not open the file location: {0}"), issue)));
     }
 
+    /// <remarks>
+    /// An unreadable index is left to the repair button below: another attempt would open the same
+    /// store and fail the same way, so offering both would be offering one that does nothing.
+    /// </remarks>
     private bool CanRefresh(DataSourceEmbeddingStatus status)
     {
         return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) &&
-            status.State is not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED &&
+               status is { VectorStoreUnreadable: false, State: not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED } &&
-            (status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0);
+               (status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0);
+    }
+
+    /// <remarks>
+    /// Offered for the one failure which no further attempt gets past. It is a button of its own
+    /// and not the refresh one, because what it does is not what the user expects of a refresh:
+    /// everything indexed so far is thrown away and paid for again.
+    /// </remarks>
+    private bool CanRepair(DataSourceEmbeddingStatus status)
+    {
+        return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) &&
+            status is { State: DataSourceEmbeddingState.FAILED, VectorStoreUnreadable: true };
     }
 
     /// <summary>
@@ -340,4 +355,13 @@ public partial class Embeddings : MSGComponentBase
         this.ReloadStatuses();
         await this.InvokeAsync(this.StateHasChanged);
     }
+
+    private async Task RepairDataSource(DataSourceEmbeddingStatus status)
+    {
+        if (!await DataSourceRepair.ConfirmAndRepairAsync(this.DialogService, this.DataSourceEmbeddingService, status.DataSourceId, status.DataSourceName))
+            return;
+
+        this.ReloadStatuses();
+        await this.InvokeAsync(this.StateHasChanged);
+    }
 }

app/MindWork AI Studio/Tools/AllowedSelectedDataSources.cs

@@ -3,7 +3,7 @@ using AIStudio.Settings;
 namespace AIStudio.Tools;
 
 /// <summary>
-/// Contains the allowed and selected data sources, plus the ones waiting for their index.
+/// Contains the allowed and selected data sources, plus the ones which cannot be searched right now.
 /// </summary>
 /// <remarks>
 /// The selected data sources are a subset of the allowed data sources.
@@ -13,8 +13,13 @@ namespace AIStudio.Tools;
 /// -- takes it to mean "may be used to answer with", and a source whose index is being rebuilt
 /// cannot answer anything. It is listed separately so the user interface can still show it and say
 /// why it is greyed out, instead of letting it vanish without a word.
+///
+/// The same holds for the ones waiting for a repair, and they are a list of their own because the
+/// two reasons call for different words: one passes by itself, the other one waits for the user.
+/// A data source is in at most one of the two lists.
 /// </remarks>
 /// <param name="AllowedDataSources">The allowed data sources.</param>
 /// <param name="SelectedDataSources">The selected data sources, which are a subset of the allowed data sources.</param>
 /// <param name="DataSourcesAwaitingReindex">The data sources which passed every check but cannot be searched until their index has been rebuilt.</param>
-public readonly record struct AllowedSelectedDataSources(IReadOnlyList<IDataSource> AllowedDataSources, IReadOnlyList<IDataSource> SelectedDataSources, IReadOnlyList<IDataSource> DataSourcesAwaitingReindex);
+/// <param name="DataSourcesNeedingRepair">The data sources which passed every check but whose index cannot be read anymore, so that only the user can get them back.</param>
+public readonly record struct AllowedSelectedDataSources(IReadOnlyList<IDataSource> AllowedDataSources, IReadOnlyList<IDataSource> SelectedDataSources, IReadOnlyList<IDataSource> DataSourcesAwaitingReindex, IReadOnlyList<IDataSource> DataSourcesNeedingRepair);

app/MindWork AI Studio/Tools/DataSourceRepair.cs

@@ -0,0 +1,42 @@
+using AIStudio.Dialogs;
+using AIStudio.Tools.PluginSystem;
+using AIStudio.Tools.Services;
+
+namespace AIStudio.Tools;
+
+/// <summary>
+/// Asks whether a data source should be indexed anew, and starts the rebuild when the user agrees.
+/// </summary>
+/// <remarks>
+/// Kept here rather than in the two places which offer the repair -- the background embeddings page
+/// and the data source table -- so the sentence naming what a rebuild costs cannot drift apart
+/// between them. Naming both costs is the whole reason for asking at all.
+/// </remarks>
+public static class DataSourceRepair
+{
+    private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceRepair).Namespace, nameof(DataSourceRepair));
+
+    /// <summary>
+    /// Asks the user, and rebuilds the index of the data source when they agree.
+    /// </summary>
+    /// <param name="dialogService">The dialog service to ask with.</param>
+    /// <param name="embeddingService">The service which does the rebuild.</param>
+    /// <param name="dataSourceId">The data source to repair.</param>
+    /// <param name="dataSourceName">The name of that data source, as the question names it.</param>
+    /// <returns>True when the rebuild was started.</returns>
+    public static async Task<bool> ConfirmAndRepairAsync(IDialogService dialogService, DataSourceEmbeddingService embeddingService, string dataSourceId, string dataSourceName)
+    {
+        var dialogParameters = new DialogParameters<ConfirmDialog>
+        {
+            { x => x.Message, string.Format(TB("The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?"), dataSourceName) },
+        };
+
+        var dialogReference = await dialogService.ShowAsync<ConfirmDialog>(TB("Repair Data Source"), dialogParameters, Dialogs.DialogOptions.FULLSCREEN);
+        var dialogResult = await dialogReference.Result;
+        if (dialogResult is null || dialogResult.Canceled)
+            return false;
+
+        await embeddingService.RepairDataSourceAsync(dataSourceId);
+        return true;
+    }
+}

app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoreUnreadableException.cs

@@ -0,0 +1,13 @@
+namespace AIStudio.Tools.Databases.VectorStore;
+
+/// <summary>
+/// Thrown when a vector store is there on disk, but cannot be opened.
+/// </summary>
+/// <remarks>
+/// Separate from every other database failure, because it is the one which no retry heals and which
+/// the app must not heal on its own: building the index anew sends every document to the embedding
+/// provider once more, which costs real money and, for a large data source, hours. So this failure
+/// travels as its own type up to the places which can say so and offer the rebuild, and the decision
+/// stays with the user.
+/// </remarks>
+public sealed class VectorStoreUnreadableException(string message) : Exception(message);

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs

@@ -5,6 +5,43 @@ namespace AIStudio.Tools.Services;
 
 public sealed partial class DataSourceEmbeddingService
 {
+    /// <summary>
+    /// Throws away everything stored for one data source and starts a fresh indexing run.
+    /// </summary>
+    /// <remarks>
+    /// The one way out of an index which cannot be read, and nothing in the app takes it by itself:
+    /// a rebuild sends every document of the data source to the embedding provider once more, which
+    /// costs money with a cloud provider and hours with a large data source. It happens because the
+    /// user asked for it, after being told both.
+    ///
+    /// An active run is stopped first, the same way deleting a data source does it. The repair is
+    /// offered for a failed data source only, so there should be none -- but a file watcher may
+    /// well have queued one between the click and this call, and discarding the index next to a
+    /// live run would leave it half thrown away.
+    /// </remarks>
+    /// <param name="dataSourceId">The data source to build anew.</param>
+    public async Task RepairDataSourceAsync(string dataSourceId)
+    {
+        if (!this.TryGetConfiguredDataSource(dataSourceId, out var dataSource) || !this.IsSupportedInternalDataSource(dataSource))
+            return;
+
+        logger.LogWarning(
+            "Repairing data source '{DataSourceName}' ({DataSourceId}) on the user's request: the stored index is discarded and built anew.",
+            dataSource.Name,
+            dataSource.Id);
+
+        var activeRun = this.CancelActiveDataSourceRun(dataSource);
+        this.ClearQueuedDataSourceState(dataSourceId);
+        if (activeRun is not null)
+            await activeRun.Completion.Task;
+
+        await this.ResetPersistedStateAsync(dataSourceId, null, null, CancellationToken.None);
+        this.statuses.TryRemove(dataSourceId, out _);
+        this.PublishStatusChanged();
+
+        await this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.MANUAL_RETRY);
+    }
+
     private async Task ResetPersistedStateAsync(
         string dataSourceId,
         VectorStoreClient? vectorStore,

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs

@@ -317,6 +317,22 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         return runState is not DataSourceEmbeddingState.FAILED;
     }
 
+    /// <summary>
+    /// Whether a data source cannot be searched because its vector store cannot be read anymore.
+    /// </summary>
+    /// <remarks>
+    /// Unlike the re-index check above, this reads no database at all: the state comes from the run
+    /// or the search which ran into the unreadable store, and is kept in memory only. That it does
+    /// not survive a restart is deliberate. The very same store may well open on the next start,
+    /// and a mark written to disk would then be wrong with nobody noticing. Until something touches
+    /// the store again, the data source counts as usable, and a failing search says so on its own.
+    /// </remarks>
+    /// <param name="dataSource">The data source to ask about.</param>
+    /// <returns>True when the data source waits for the user to have its index rebuilt.</returns>
+    public bool NeedsIndexRepair(IDataSource dataSource) =>
+        this.statuses.TryGetValue(dataSource.Id, out var status) &&
+        status is { State: DataSourceEmbeddingState.FAILED, VectorStoreUnreadable: true };
+
     public Task QueueDataSourceAsync(IDataSource dataSource)
     {
         return this.QueueDataSourceAsync(dataSource, true, DataSourceEmbeddingRefreshMode.HASH_CHECK);
@@ -430,6 +446,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             {
                 break;
             }
+            catch (VectorStoreUnreadableException exception) when (dataSource is not null)
+            {
+                //
+                // Nothing is deleted and nothing is rebuilt here. The data source says what is
+                // wrong with it, stays out of the selection while it says so, and waits for the
+                // user to ask for the repair.
+                //
+                logger.LogError(
+                    exception,
+                    "The vector store of data source '{DataSourceName}' ({DataSourceId}) cannot be read. The data source is waiting for a repair.",
+                    dataSource.Name,
+                    dataSource.Id);
+                this.UpsertStatus(this.GetUnreadableVectorStoreStatus(dataSource));
+            }
             catch (Exception exception)
             {
                 if (dataSource is null)
@@ -873,6 +903,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
                     ShortHash(fingerprint));
                 this.UpsertStatus(this.CreateStatus(dataSource, DataSourceEmbeddingState.RUNNING, totalFiles, skippedFiles + completedFiles, failedFiles, file.Name, lastError, failureDetails, permanentlySkippedFiles));
             }
+            catch (VectorStoreUnreadableException)
+            {
+                //
+                // Not about this one file: the store of the whole data source cannot be opened, so
+                // every remaining file would fail the same way. Carrying on would fill the list
+                // with one entry per file and hide the single cause behind them.
+                //
+                throw;
+            }
             catch (Exception exception)
             {
                 //
@@ -1338,6 +1377,15 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             {
                 throw;
             }
+            catch (VectorStoreUnreadableException exception)
+            {
+                logger.LogError(
+                    exception,
+                    "The vector store of data source '{DataSourceName}' ({DataSourceId}) cannot be read. The data source is waiting for a repair.",
+                    dataSource.Name,
+                    dataSource.Id);
+                this.UpsertStatus(this.GetUnreadableVectorStoreStatus(dataSource));
+            }
             catch (Exception exception)
             {
                 logger.LogError(exception, "Initial embedding hash check failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);
@@ -1583,7 +1631,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
         IReadOnlyList<DataSourceEmbeddingFailure>? failures = null,
         int permanentlySkippedFiles = 0,
         int? currentFileBlock = null,
-        int? currentFilePage = null)
+        int? currentFilePage = null,
+        bool vectorStoreUnreadable = false)
     {
         return new DataSourceEmbeddingStatus(
             dataSource.Id,
@@ -1598,7 +1647,8 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             failures?.ToList() ?? [],
             permanentlySkippedFiles,
             currentFileBlock,
-            currentFilePage);
+            currentFilePage,
+            vectorStoreUnreadable);
     }
 
     /// <remarks>
@@ -1635,6 +1685,25 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
             failures: [new DataSourceEmbeddingFailure(dataSource.Name, errorMessage, DateTimeOffset.UtcNow)]);
     }
 
+    /// <remarks>
+    /// Deliberately not the message which came from the runtime: that one names a store name and a
+    /// path, is written in English for the log file, and says nothing about what happens next. What
+    /// the user needs to read is what this means for their chats and where the way out is.
+    /// </remarks>
+    private DataSourceEmbeddingStatus GetUnreadableVectorStoreStatus(IDataSource dataSource)
+    {
+        var errorMessage = string.Format(TB("The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that."), dataSource.Name);
+        return this.CreateStatus(
+            dataSource,
+            DataSourceEmbeddingState.FAILED,
+            0,
+            0,
+            1,
+            lastError: errorMessage,
+            failures: [new DataSourceEmbeddingFailure(dataSource.Name, errorMessage, DateTimeOffset.UtcNow)],
+            vectorStoreUnreadable: true);
+    }
+
     private DataSourceQueueRequestResult TryReserveDataSourceQueueSlot(string dataSourceId, bool queueAfterCurrentRun)
     {
         lock (this.queueStateLock)

app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs

@@ -7,6 +7,10 @@ namespace AIStudio.Tools.Services;
 /// CurrentFileBlock and CurrentFilePage are null rather than zero while nothing is known about
 /// them: a file which is only about to start has no first block, and not every kind of document
 /// has pages to count. Block numbers start at one, the way the page states them.
+///
+/// VectorStoreUnreadable says why a data source failed, not only that it did. The UI needs that
+/// difference to offer the repair for this one case, and it is carried as its own flag so nothing
+/// has to read it back out of the message in LastError.
 /// </remarks>
 public sealed record DataSourceEmbeddingStatus(
     string DataSourceId,
@@ -21,7 +25,8 @@ public sealed record DataSourceEmbeddingStatus(
     IReadOnlyList<DataSourceEmbeddingFailure> Failures,
     int PermanentlySkippedFiles = 0,
     int? CurrentFileBlock = null,
-    int? CurrentFilePage = null)
+    int? CurrentFilePage = null,
+    bool VectorStoreUnreadable = false)
 {
     private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEmbeddingStatus).Namespace, nameof(DataSourceEmbeddingStatus));
 

app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs

@@ -180,6 +180,17 @@ public sealed class DataSourceLocalRetrievalService(
             await this.ReportRetrievalGapAsync(dataSource, $"provider-{exception.FailureReason}", string.Format(TB("The data source '{0}' was left out of the answer. {1}"), dataSource.Name, exception.UserMessage));
             return [];
         }
+        catch (VectorStoreUnreadableException exception)
+        {
+            //
+            // Its own gap key, because this is not a search which went wrong but an index which has
+            // to be built anew. Saying that once per session is what turns a silently shortened
+            // answer into one the user can do something about.
+            //
+            logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}) because its vector store cannot be read.", dataSource.Name, dataSource.Id);
+            await this.ReportRetrievalGapAsync(dataSource, "vector-store-unreadable", string.Format(TB("The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings."), dataSource.Name));
+            return [];
+        }
         catch (Exception exception)
         {
             logger.LogWarning(exception, "Vector retrieval failed for data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id);

app/MindWork AI Studio/Tools/Services/DataSourceService.cs

@@ -51,7 +51,7 @@ public sealed class DataSourceService
         if (selectedLLMProvider == Settings.Provider.NONE)
         {
             this.logger.LogWarning("The selected LLM provider is not set. We cannot filter the data sources by any means.");
-            return new([], [], []);
+            return new([], [], [], []);
         }
         
         var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
@@ -83,12 +83,13 @@ public sealed class DataSourceService
         var allowedDataSources = await this.GetAllowedDataSources(usingTrustedProvider, participatingProviders, requestedDataSources);
 
         //
-        // Whoever asks this way has no list to show, so a data source waiting for its index is
+        // Whoever asks this way has no list to show, so a data source which cannot be searched is
         // dropped rather than marked. Handing it back would start a chat with a data source which
         // finds nothing -- the very thing being greyed out elsewhere is meant to prevent.
         //
-        var awaitingReindexIds = (await this.GetDataSourcesAwaitingReindex(allowedDataSources)).Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
+        var unsearchableIds = (await this.GetDataSourcesAwaitingReindex(allowedDataSources)).Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
-        return allowedDataSources.Where(source => !awaitingReindexIds.Contains(source.Id)).ToList();
+        unsearchableIds.UnionWith(this.GetDataSourcesNeedingRepair(allowedDataSources).Select(source => source.Id));
+        return allowedDataSources.Where(source => !unsearchableIds.Contains(source.Id)).ToList();
     }
     
     /// <summary>
@@ -109,7 +110,7 @@ public sealed class DataSourceService
         if (selectedLLMProvider is NoProvider)
         {
             this.logger.LogWarning("The selected LLM provider is the default provider. We cannot filter the data sources by any means.");
-            return new([], [], []);
+            return new([], [], [], []);
         }
         
         var usingTrustedProvider = selectedLLMProvider.IsTrustedForDataSourceSecurityChecks(this.settingsManager);
@@ -159,12 +160,21 @@ public sealed class DataSourceService
         // being rebuilt is usable again in a while, and saying so on its own row beats letting it
         // disappear from the selection without a word.
         //
-        var awaitingReindex = await this.GetDataSourcesAwaitingReindex(filteredDataSources);
+        // A source whose index cannot be read is asked about first and then kept out of the other
-        var awaitingReindexIds = awaitingReindex.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
+        // list: both reasons can be true at once, and of the two it is the only one the user can do
-        var usableDataSources = filteredDataSources.Where(source => !awaitingReindexIds.Contains(source.Id)).ToList();
+        // anything about. Telling them to wait instead would be telling them to wait forever.
+        //
+        var needingRepair = this.GetDataSourcesNeedingRepair(filteredDataSources);
+        var needingRepairIds = needingRepair.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
+        var awaitingReindex = (await this.GetDataSourcesAwaitingReindex(filteredDataSources)).Where(source => !needingRepairIds.Contains(source.Id)).ToList();
+
+        var blockedIds = awaitingReindex.Select(source => source.Id).ToHashSet(StringComparer.Ordinal);
+        blockedIds.UnionWith(needingRepairIds);
+
+        var usableDataSources = filteredDataSources.Where(source => !blockedIds.Contains(source.Id)).ToList();
         var filteredSelectedDataSources = usableDataSources.Where(source => previousSelectedDataSourceIds.Contains(source.Id)).ToList();
 
-        return new(usableDataSources, filteredSelectedDataSources, awaitingReindex);
+        return new(usableDataSources, filteredSelectedDataSources, awaitingReindex, needingRepair);
     }
 
     /// <summary>
@@ -195,6 +205,30 @@ public sealed class DataSourceService
         return awaitingReindex;
     }
 
+    /// <summary>
+    /// Picks out the data sources whose index cannot be read anymore, so that they wait for a repair.
+    /// </summary>
+    /// <remarks>
+    /// Reads nothing from a database, unlike the re-index check above: the state is held in memory
+    /// by the embedding service, which is why this one needs no parallelism and no timeout.
+    /// </remarks>
+    /// <param name="dataSources">The data sources which passed every other check.</param>
+    /// <returns>Those of them which wait for a repair, in the order they came in.</returns>
+    private IReadOnlyList<IDataSource> GetDataSourcesNeedingRepair(IReadOnlyList<IDataSource> dataSources)
+    {
+        var needingRepair = new List<IDataSource>();
+        foreach (var dataSource in dataSources)
+        {
+            if (!this.embeddingService.NeedsIndexRepair(dataSource))
+                continue;
+
+            this.logger.LogInformation("The index of data source '{DataSourceName}' ({DataSourceId}) cannot be read. It is shown, but cannot be selected until it was repaired.", dataSource.Name, dataSource.Id);
+            needingRepair.Add(dataSource);
+        }
+
+        return needingRepair;
+    }
+
     private async Task<IReadOnlyList<IDataSource>> GetAllowedDataSources(bool usingTrustedProvider, IReadOnlyList<ParticipatingProvider> participatingProviders, IReadOnlyCollection<IDataSource> requestedDataSources)
     {
         var filteredDataSources = new List<IDataSource>(requestedDataSources.Count);

app/MindWork AI Studio/Tools/Services/RustService.Databases.cs

@@ -1,7 +1,18 @@
+using AIStudio.Tools.Databases.VectorStore;
+
 namespace AIStudio.Tools.Services;
 
 public sealed partial class RustService
 {
+    /// <summary>
+    /// The issue code the Rust runtime sends when a vector store is there, but cannot be opened.
+    /// </summary>
+    /// <remarks>
+    /// Mirrors ISSUE_CODE_STORE_UNREADABLE in runtime/src/qdrant_edge_database.rs. Reading the code
+    /// rather than the message is what keeps a reworded message on the Rust side harmless here.
+    /// </remarks>
+    private const string ISSUE_CODE_STORE_UNREADABLE = "store-unreadable";
+
     public async Task<TDatabaseInfo> GetDatabaseInfo<TDatabaseInfo>(
         string databaseName,
         string infoPath,
@@ -46,7 +57,7 @@ public sealed partial class RustService
 
         var operation = await response.Content.ReadFromJsonAsync<DatabaseOperationResponse>(this.jsonRustSerializerOptions, cts.Token);
         if (operation is not { Success: true })
-            throw new InvalidOperationException(operation?.Issue ?? $"The {databaseName} operation failed.");
+            throw CreateDatabaseException(operation?.Issue, operation?.IssueCode, $"The {databaseName} operation failed.");
     }
 
     public async Task<TResult?> ExecuteDatabaseQuery<TRequest, TResult>(string databaseName, string path, TRequest request, CancellationToken cancellationToken = default)
@@ -59,12 +70,31 @@ public sealed partial class RustService
 
         var operation = await response.Content.ReadFromJsonAsync<DatabaseQueryResponse<TResult>>(this.jsonRustSerializerOptions, cts.Token);
         if (operation is not { Success: true })
-            throw new InvalidOperationException(operation?.Issue ?? $"The {databaseName} query failed.");
+            throw CreateDatabaseException(operation?.Issue, operation?.IssueCode, $"The {databaseName} query failed.");
 
         return operation.Data;
     }
 
-    private sealed record DatabaseOperationResponse(bool Success, string Issue);
+    /// <summary>
+    /// Turns a failed database response into the exception which fits its issue code.
+    /// </summary>
+    /// <remarks>
+    /// Almost every failure says all it has to say in its message. A store which cannot be opened is
+    /// the exception: the only way out of it is a rebuild which costs the user money and time, so it
+    /// gets a type of its own and reaches the places which can offer that rebuild instead of
+    /// starting it unasked.
+    /// </remarks>
+    private static Exception CreateDatabaseException(string? issue, string? issueCode, string fallbackMessage)
+    {
+        var message = string.IsNullOrWhiteSpace(issue) ? fallbackMessage : issue;
+        return issueCode switch
+        {
+            ISSUE_CODE_STORE_UNREADABLE => new VectorStoreUnreadableException(message),
+            _ => new InvalidOperationException(message),
+        };
+    }
 
-    private sealed record DatabaseQueryResponse<TResult>(bool Success, string Issue, TResult? Data);
+    private sealed record DatabaseOperationResponse(bool Success, string Issue, string IssueCode);
+
+    private sealed record DatabaseQueryResponse<TResult>(bool Success, string Issue, string IssueCode, TResult? Data);
 }

app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md

@@ -22,6 +22,8 @@
 - Added a way to show a source of your own documents (RAG) in your file manager.
 - Added the page of a passage to what the AI is told when it answers from your own documents (RAG), so it can name the page an answer rests on.
 - Added the details of the two databases behind local RAG to the information page: which versions they run, how much space they use on your disk, and how much they hold.
+- Added a repair for your local data sources. Should the index of a data source ever become unreadable, AI Studio now says so instead of quietly finding nothing and leaves that source out of your chats until it works again.
+- Added the repair itself as a button next to each of your data sources. Rebuilding an index sends your documents to your embedding provider once more, so AI Studio asks you first and never starts it on its own.
 - Added support for several drop areas on the same page. More complex assistants can now receive files or folders by drag and drop at more than one place.
 - Added drag and drop to the input and output folder of the Batch Processing assistant: drop a folder onto either field to choose it.
 - Added ways to load text from a file and drop zones for them, throughout the assistants and dialogs. We went through them one by one, so many fields that used to accept typed text only now take the content of a file as well.
@@ -59,3 +61,4 @@
 - Fixed AI Studio asking such a server for its models with an empty key attached when you had stored none at all. Servers behind a login turn those requests down.
 - Fixed a key that could not be saved going unmentioned for the servers you host yourself. You are now told what went wrong, instead of the settings simply staying open.
 - Upgraded the Visual Briefing Assistant (in preview) from the prototype to the beta state. The assistant is now completely implemented and is undergoing a deeper testing phase in preparation for release. To try it, open the app settings, allow preview features down to beta, and then enable the Visual Briefing Assistant there.
+- Upgraded the vector database behind local RAG (Qdrant Edge) to version 0.8.0.

metadata.txt

@@ -9,4 +9,4 @@
 3c18a7bfdb3, release
 osx-arm64
 148.0.7763.0
-0.7.2
+0.8.0

runtime/Cargo.lock

@@ -113,6 +113,12 @@ version = "0.2.21"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
 
+[[package]]
+name = "allocator-api2"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c880a97d28a3681c0267bd29cff89621202715b065127cd445fa0f0fe0aa2880"
+
 [[package]]
 name = "android_system_properties"
 version = "0.1.5"
@@ -884,6 +890,15 @@ dependencies = [
  "constant_time_eq 0.1.5",
 ]
 
+[[package]]
+name = "blink-alloc"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ce4c15bad517bc0fb4a44523adf470e2c3eb3a365769327acdba849948ea3705"
+dependencies = [
+ "allocator-api2 0.4.0",
+]
+
 [[package]]
 name = "block-buffer"
 version = "0.10.4"
@@ -952,14 +967,6 @@ dependencies = [
  "piper",
 ]
 
-[[package]]
-name = "bm25"
-version = "0.1.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "murmur3_32",
-]
-
 [[package]]
 name = "brotli"
 version = "8.0.2"
@@ -1366,53 +1373,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "common"
-version = "0.0.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "ahash",
- "aligned-vec",
- "atomicwrites",
- "bincode 1.3.3",
- "bitvec",
- "bytemuck",
- "chrono",
- "fs-err",
- "fs4",
- "fs_extra",
- "io-uring",
- "itertools",
- "log",
- "memmap2",
- "nix 0.31.3",
- "num-traits",
- "num_cpus",
- "ordered-float 5.3.0",
- "parking_lot",
- "ph",
- "procfs",
- "quick_cache",
- "rand 0.10.2",
- "roaring",
- "schemars 0.8.22",
- "self_cell",
- "semver",
- "serde",
- "serde_json",
- "slab",
- "strum",
- "tap",
- "tar",
- "tempfile",
- "thiserror 2.0.18",
- "thread-priority",
- "tokio",
- "validator",
- "walkdir",
- "zerocopy",
-]
-
 [[package]]
 name = "compact_str"
 version = "0.9.1"
@@ -1550,6 +1510,17 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "core_affinity"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342"
+dependencies = [
+ "libc",
+ "num_cpus",
+ "winapi",
+]
+
 [[package]]
 name = "cpubits"
 version = "0.1.1"
@@ -1810,20 +1781,6 @@ version = "2.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "a4ae5f15dda3c708c0ade84bfee31ccab44a3da4f88015ed22f63732abe300c8"
 
-[[package]]
-name = "dataset"
-version = "0.0.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "anyhow",
- "flate2",
- "fs-err",
- "indicatif",
- "reqwest",
- "serde",
- "serde_json",
-]
-
 [[package]]
 name = "dbus"
 version = "0.9.7"
@@ -2882,10 +2839,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c4567c8db10ae91089c99af84c68c38da3ec2f087c3f82960bcdbf3656b6f4d7"
 dependencies = [
  "cfg-if",
- "js-sys",
  "libc",
  "wasi 0.11.1+wasi-snapshot-preview1",
- "wasm-bindgen",
 ]
 
 [[package]]
@@ -3046,42 +3001,6 @@ dependencies = [
  "system-deps",
 ]
 
-[[package]]
-name = "gpu"
-version = "0.1.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "log",
- "parking_lot",
- "zerocopy",
-]
-
-[[package]]
-name = "gridstore"
-version = "0.1.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "ahash",
- "bitvec",
- "bytemuck",
- "common",
- "dataset",
- "ecow",
- "fs-err",
- "itertools",
- "log",
- "lz4_flex",
- "parking_lot",
- "rand 0.10.2",
- "serde",
- "serde_cbor",
- "serde_json",
- "smallvec",
- "tempfile",
- "thiserror 2.0.18",
- "zerocopy",
-]
-
 [[package]]
 name = "gtk"
 version = "0.18.2"
@@ -3195,7 +3114,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1"
 dependencies = [
  "ahash",
- "allocator-api2",
+ "allocator-api2 0.2.21",
 ]
 
 [[package]]
@@ -3204,7 +3123,7 @@ version = "0.15.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289"
 dependencies = [
- "allocator-api2",
+ "allocator-api2 0.2.21",
  "equivalent",
  "foldhash 0.1.5",
 ]
@@ -3215,7 +3134,7 @@ version = "0.16.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100"
 dependencies = [
- "allocator-api2",
+ "allocator-api2 0.2.21",
  "equivalent",
  "foldhash 0.2.0",
 ]
@@ -3343,6 +3262,12 @@ version = "1.0.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
 
+[[package]]
+name = "humantime"
+version = "2.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "15cdd26707701c53297e2fa6afb323d55fbc1d0810c3aec078ae3ef0424c3c15"
+
 [[package]]
 name = "hybrid-array"
 version = "0.4.12"
@@ -3761,7 +3686,6 @@ checksum = "25470f23803092da7d239834776d653104d551bc4d7eacaf31e6837854b8e9eb"
 dependencies = [
  "console",
  "portable-atomic",
- "rayon",
  "unicode-width",
  "unit-prefix",
  "web-time",
@@ -3885,6 +3809,15 @@ dependencies = [
  "either",
 ]
 
+[[package]]
+name = "itertools"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b4baf93f58d4425749ca49a51c50ebab072c5df6994d08fed93541c331481dc"
+dependencies = [
+ "either",
+]
+
 [[package]]
 name = "itoa"
 version = "1.0.11"
@@ -4221,17 +4154,11 @@ dependencies = [
  "imgref",
 ]
 
-[[package]]
-name = "lru-slab"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "112b39cec0b298b6c1999fee3e31427f74f676e4cb9879ed1a121b43661a4154"
-
 [[package]]
 name = "lz4_flex"
-version = "0.13.1"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7ef0d4ed8669f8f8826eb00dc878084aa8f253506c4fd5e8f58f5bce72ddb97e"
+checksum = "ecbdfe44b1bd960b68170b417450a628c43f7cf56bb3c5317e61cb230ee7f226"
 
 [[package]]
 name = "lzma-rust2"
@@ -4258,16 +4185,6 @@ version = "0.2.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30"
 
-[[package]]
-name = "macros"
-version = "0.1.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "proc-macro2",
- "quote",
- "syn 2.0.117",
-]
-
 [[package]]
 name = "markup5ever"
 version = "0.38.0"
@@ -4382,7 +4299,7 @@ dependencies = [
  "strum_macros",
  "symphonia",
  "sys-locale",
- "sysinfo 0.39.6",
+ "sysinfo",
  "tauri",
  "tauri-build",
  "tauri-plugin-dialog",
@@ -5291,7 +5208,7 @@ dependencies = [
  "console_error_panic_hook",
  "console_log",
  "image",
- "itertools",
+ "itertools 0.14.0",
  "js-sys",
  "libloading 0.8.6",
  "log",
@@ -5523,16 +5440,6 @@ dependencies = [
  "portable-atomic",
 ]
 
-[[package]]
-name = "posting_list"
-version = "0.0.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "bitpacking",
- "common",
- "zerocopy",
-]
-
 [[package]]
 name = "powerfmt"
 version = "0.2.0"
@@ -5757,25 +5664,97 @@ checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea"
 
 [[package]]
 name = "qdrant-edge"
-version = "0.7.2"
+version = "0.8.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0b8072302c87506a34bffec9bc16dbdcd36df8ab1321406b6e141530348c7e54"
 dependencies = [
  "ahash",
- "bm25",
+ "aligned-vec",
- "common",
+ "arrayvec 0.7.6",
+ "atomic_refcell",
+ "atomicwrites",
+ "bincode 1.3.3",
+ "bitpacking",
+ "bitvec",
+ "blink-alloc",
+ "bytemuck",
+ "byteorder",
+ "cc",
+ "cgroups-rs",
+ "charabia",
+ "chrono",
+ "core_affinity",
+ "crc32c",
+ "data-encoding",
+ "docopt",
+ "duplicate",
+ "ecow",
+ "env_logger",
+ "fnv",
  "fs-err",
- "itertools",
+ "fs4",
+ "fs_extra",
+ "geo",
+ "geohash",
+ "half 2.7.1",
+ "humantime",
+ "indexmap 2.14.0",
+ "integer-encoding",
+ "io-uring",
+ "itertools 0.15.0",
  "log",
+ "lz4_flex",
+ "macro_rules_attribute",
+ "memmap2",
+ "murmur3_32",
+ "nix 0.31.3",
+ "nom 8.0.0",
+ "num-cmp",
+ "num-derive",
+ "num-traits",
+ "num_cpus",
+ "once_cell",
  "ordered-float 5.3.0",
  "parking_lot",
+ "permutation_iterator",
+ "ph",
+ "procfs",
+ "qdrant-rust-stemmers",
+ "quick_cache",
  "rand 0.10.2",
- "segment",
+ "rand_distr",
+ "rayon",
+ "rmp-serde",
+ "roaring",
+ "rustix 1.1.4",
+ "schemars 0.8.22",
+ "self_cell",
+ "semver",
  "serde",
+ "serde-untagged",
+ "serde-value",
+ "serde_cbor",
  "serde_json",
- "shard",
+ "serde_variant",
- "sparse",
+ "sha2 0.11.0",
+ "siphasher",
+ "slab",
+ "smallvec",
+ "strum",
+ "sysinfo",
+ "tap",
+ "tar",
+ "tempfile",
+ "thiserror 2.0.18",
+ "thread-priority",
+ "tinyvec",
+ "tokio",
+ "tonic",
  "uuid",
- "wal",
+ "validator",
+ "vaporetto",
+ "walkdir",
+ "zerocopy",
 ]
 
 [[package]]
@@ -5797,27 +5776,6 @@ dependencies = [
  "bytemuck",
 ]
 
-[[package]]
-name = "quantization"
-version = "0.1.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "arrayvec 0.7.6",
- "bytemuck",
- "cc",
- "common",
- "fs-err",
- "num-traits",
- "ordered-float 5.3.0",
- "parking_lot",
- "permutation_iterator",
- "rand 0.10.2",
- "rayon",
- "serde",
- "serde_json",
- "strum",
-]
-
 [[package]]
 name = "quick-error"
 version = "2.0.1"
@@ -5854,73 +5812,16 @@ dependencies = [
 
 [[package]]
 name = "quick_cache"
-version = "0.6.22"
+version = "0.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1c821816e9b928e20e92ed59bb3ac4aab321d16ca2316871c9fe7ca739cd477"
+checksum = "403c1a912fec895cafb223201e368234842acb9220aaf08ab042ae89ba5f135c"
 dependencies = [
- "ahash",
  "equivalent",
- "hashbrown 0.16.1",
+ "foldhash 0.2.0",
+ "hashbrown 0.17.0",
  "parking_lot",
 ]
 
-[[package]]
-name = "quinn"
-version = "0.11.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20"
-dependencies = [
- "bytes",
- "cfg_aliases",
- "pin-project-lite",
- "quinn-proto",
- "quinn-udp",
- "rustc-hash",
- "rustls",
- "socket2",
- "thiserror 2.0.18",
- "tokio",
- "tracing",
- "web-time",
-]
-
-[[package]]
-name = "quinn-proto"
-version = "0.11.16"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2f4bfc015262b9df63c8845072ce59068853ff5872180c2ce2f13038b970e560"
-dependencies = [
- "aws-lc-rs",
- "bytes",
- "getrandom 0.4.2",
- "lru-slab",
- "rand 0.10.2",
- "rand_pcg",
- "ring",
- "rustc-hash",
- "rustls",
- "rustls-pki-types",
- "slab",
- "thiserror 2.0.18",
- "tinyvec",
- "tracing",
- "web-time",
-]
-
-[[package]]
-name = "quinn-udp"
-version = "0.5.14"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd"
-dependencies = [
- "cfg_aliases",
- "libc",
- "once_cell",
- "socket2",
- "tracing",
- "windows-sys 0.60.2",
-]
-
 [[package]]
 name = "quote"
 version = "1.0.45"
@@ -6065,7 +5966,7 @@ dependencies = [
  "built",
  "cfg-if",
  "interpolate_name",
- "itertools",
+ "itertools 0.14.0",
  "libc",
  "libfuzzer-sys",
  "log",
@@ -6131,7 +6032,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f"
 dependencies = [
  "either",
- "itertools",
+ "itertools 0.14.0",
  "rayon",
 ]
 
@@ -6266,10 +6167,8 @@ checksum = "219c5811de6525e5416c7d5d53bb656d3afdbc6c5af816e0802bcfa42dbdc1c3"
 dependencies = [
  "base64 0.22.1",
  "bytes",
- "futures-channel",
  "futures-core",
  "futures-util",
- "h2",
  "http",
  "http-body",
  "http-body-util",
@@ -6280,7 +6179,6 @@ dependencies = [
  "log",
  "percent-encoding",
  "pin-project-lite",
- "quinn",
  "rustls",
  "rustls-pki-types",
  "rustls-platform-verifier",
@@ -6526,7 +6424,6 @@ version = "1.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "be040f8b0a225e40375822a563fa9524378b9d63112f53e19ffff34df5d33fdd"
 dependencies = [
- "web-time",
  "zeroize",
 ]
 
@@ -6700,78 +6597,6 @@ dependencies = [
  "xxhash-rust",
 ]
 
-[[package]]
-name = "segment"
-version = "0.6.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "ahash",
- "atomic_refcell",
- "atomicwrites",
- "bincode 1.3.3",
- "bitvec",
- "bytemuck",
- "byteorder",
- "cc",
- "cgroups-rs",
- "charabia",
- "chrono",
- "common",
- "data-encoding",
- "duplicate",
- "ecow",
- "fnv",
- "fs-err",
- "fs_extra",
- "geo",
- "geohash",
- "gpu",
- "gridstore",
- "half 2.7.1",
- "indexmap 2.14.0",
- "integer-encoding",
- "io-uring",
- "itertools",
- "log",
- "macro_rules_attribute",
- "macros",
- "memmap2",
- "nom 8.0.0",
- "num-cmp",
- "num-derive",
- "num-traits",
- "ordered-float 5.3.0",
- "parking_lot",
- "posting_list",
- "procfs",
- "qdrant-rust-stemmers",
- "quantization",
- "rand 0.10.2",
- "rayon",
- "roaring",
- "schemars 0.8.22",
- "self_cell",
- "serde",
- "serde-untagged",
- "serde-value",
- "serde_cbor",
- "serde_json",
- "serde_variant",
- "sha2 0.11.0",
- "smallvec",
- "sparse",
- "strum",
- "sysinfo 0.38.4",
- "tap",
- "tempfile",
- "thiserror 2.0.18",
- "tinyvec",
- "uuid",
- "validator",
- "vaporetto",
- "zerocopy",
-]
-
 [[package]]
 name = "selectors"
 version = "0.36.1"
@@ -7050,39 +6875,6 @@ dependencies = [
  "digest 0.11.3",
 ]
 
-[[package]]
-name = "shard"
-version = "0.1.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "ahash",
- "chrono",
- "common",
- "fs-err",
- "fs4",
- "indexmap 2.14.0",
- "itertools",
- "log",
- "ordered-float 5.3.0",
- "parking_lot",
- "rand 0.10.2",
- "rmp-serde",
- "schemars 0.8.22",
- "segment",
- "serde",
- "serde_cbor",
- "serde_json",
- "smallvec",
- "sparse",
- "strum",
- "tempfile",
- "thiserror 2.0.18",
- "tonic",
- "uuid",
- "validator",
- "wal",
-]
-
 [[package]]
 name = "shared_child"
 version = "1.0.0"
@@ -7223,32 +7015,6 @@ dependencies = [
  "smallvec",
 ]
 
-[[package]]
-name = "sparse"
-version = "0.1.0"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "bincode 1.3.3",
- "bitpacking",
- "common",
- "fs-err",
- "gridstore",
- "half 2.7.1",
- "itertools",
- "log",
- "memmap2",
- "ordered-float 5.3.0",
- "parking_lot",
- "rand 0.10.2",
- "schemars 0.8.22",
- "serde",
- "serde_json",
- "tempfile",
- "typed-arena",
- "validator",
- "zerocopy",
-]
-
 [[package]]
 name = "spm_precompiled"
 version = "0.1.4"
@@ -7590,20 +7356,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "sysinfo"
-version = "0.38.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f"
-dependencies = [
- "libc",
- "memchr",
- "ntapi",
- "objc2-core-foundation",
- "objc2-io-kit",
- "windows 0.62.2",
-]
-
 [[package]]
 name = "sysinfo"
 version = "0.39.6"
@@ -8080,7 +7832,7 @@ dependencies = [
  "serde_with",
  "swift-rs",
  "thiserror 2.0.18",
- "toml 0.9.12+spec-1.1.0",
+ "toml 1.1.4+spec-1.1.0",
  "url",
  "urlpattern",
  "uuid",
@@ -8260,7 +8012,7 @@ dependencies = [
  "esaxx-rs",
  "getrandom 0.3.1",
  "indicatif",
- "itertools",
+ "itertools 0.14.0",
  "log",
  "macro_rules_attribute",
  "monostate",
@@ -8626,12 +8378,6 @@ version = "0.2.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
 
-[[package]]
-name = "typed-arena"
-version = "2.0.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6af6ae20167a9ece4bcb41af5b80f8a1f1df981f6391189ce00fd257af04126a"
-
 [[package]]
 name = "typed-path"
 version = "0.12.2"
@@ -8948,25 +8694,6 @@ dependencies = [
  "libc",
 ]
 
-[[package]]
-name = "wal"
-version = "0.1.4"
-source = "git+https://github.com/SommerEngineering/qdrant.git?rev=462c84d82ced126e4a2b7914544bfde16a509eb1#462c84d82ced126e4a2b7914544bfde16a509eb1"
-dependencies = [
- "byteorder",
- "crc32c",
- "docopt",
- "env_logger",
- "fs-err",
- "fs4",
- "log",
- "memmap2",
- "rand 0.10.2",
- "rand_distr",
- "rustix 1.1.4",
- "serde",
-]
-
 [[package]]
 name = "walkdir"
 version = "2.5.0"

runtime/Cargo.toml

@@ -67,7 +67,7 @@ tempfile = "3.27.0"
 strum_macros = "0.28.0"
 sysinfo = "0.39.6"
 bytes = "1.12.1"
-qdrant-edge = "0.7.2"
+qdrant-edge = "0.8.0"
 
 # Prompt-injection detection. `regex` gives us linear-time matching without backtracking, so
 # a hostile document cannot make a scan blow up, and `aho-corasick` matches the ~1600 fixed
@@ -79,12 +79,9 @@ image = { version = "0.25.10", default-features = false, features = ["jpeg", "pn
 tokenizers = "0.23.1"
 
 [patch.crates-io]
-# Issue: It was not possible to build qdrant-edge for macOS. See PR 9312: https://github.com/qdrant/qdrant/pull/9312
-# State: The PR was merged, but not yet released. We use the git version for now.
-qdrant-edge = { git = "https://github.com/SommerEngineering/qdrant.git", rev = "462c84d82ced126e4a2b7914544bfde16a509eb1" }
-
 # Issue: This repo was not updated since 2020. The rand crate was outdated. We patched it to use a newer version of rand.
-# State: There is a PR for a long time, but it was not merged. We use the git version for now.
+# State: There is a PR for a long time, but it was not merged. We use the git version for now. Qdrant Edge still depends
+#        on this crate in version 0.1.2, so the patch stays relevant: https://github.com/asimihsan/permutation-iterator-rs/pull/14
 permutation_iterator = { git = "https://github.com/SommerEngineering/permutation-iterator-rs.git", rev = "76836ed316d18dfef530ba908f58481c343e80d7" }
 
 [target.'cfg(target_os = "windows")'.dependencies]

runtime/patches/README.md

@@ -2,81 +2,35 @@
 
 This directory documents temporary patches for third-party Rust dependencies.
 
-## Qdrant Edge
+## permutation_iterator
 
-AI Studio temporarily uses a pinned commit from `SommerEngineering/qdrant` for `qdrant-edge`.
+`qdrant-edge` depends on `permutation_iterator 0.1.2`, and that crate has seen no release since
-The fork commit exposes Qdrant's internal `lib/edge` crate as `qdrant-edge` and applies the
+2019. Its published version pulls in an outdated `rand` line, which drags a second copy of the
-trait-solver fix from Qdrant PR #9312.
+whole `rand` family into our dependency tree: `rand 0.7.3`, `rand_core 0.5.1`, `rand_chacha 0.2`,
+`rand_hc`, `getrandom 0.1.16`, `wasi 0.9` and `cfg-if 0.1`, next to the current ones everything
+else uses.
 
-When updating to a newer Qdrant Edge version, replace the placeholder values first:
+The fork `SommerEngineering/permutation-iterator-rs` is the published 0.1.2 with `rand` raised to
+0.8, so it still satisfies what `qdrant-edge` asks for. AI Studio pins it in `runtime/Cargo.toml`:
 
-```bash
+```toml
-export QDRANT_EDGE_VERSION="0.7.2"
+[patch.crates-io]
-export QDRANT_BRANCH="ai-studio-qdrant-edge-${QDRANT_EDGE_VERSION}"
+permutation_iterator = { git = "https://github.com/SommerEngineering/permutation-iterator-rs.git", rev = "..." }
-export AISTUDIO_REPO="xxx/mindwork-ai-studio"
-export QDRANT_REPO="xxx/qdrant"
 ```
 
-1. Sync the Qdrant fork with upstream:
+The same change was offered upstream in
+[asimihsan/permutation-iterator-rs#14](https://github.com/asimihsan/permutation-iterator-rs/pull/14),
+where it has been waiting since 2021. This is tree hygiene, not a build failure: without the patch
+the runtime still builds, it just carries the old `rand` family along.
 
-```bash
+### When this patch can go
-cd "$QDRANT_REPO"
-git remote add upstream https://github.com/qdrant/qdrant.git 2>/dev/null || true
-git fetch upstream
-git fetch origin
-git switch master
-git merge --ff-only upstream/master
-git push origin master
-```
 
-2. Create a fresh AI Studio branch in the Qdrant fork:
+Either of these is enough, and both are worth a look whenever `qdrant-edge` is updated:
 
-```bash
+- crates.io carries a `permutation_iterator` newer than 0.1.2 which uses a current `rand`. Then the
-cd "$QDRANT_REPO"
+  `[patch.crates-io]` entry goes, and `qdrant-edge`'s own requirement decides the version.
-git switch -c "$QDRANT_BRANCH" master
+- `qdrant-edge` stops depending on `permutation_iterator` at all. Check with
-```
+  `cargo tree -i permutation_iterator` after the update.
 
-3. Apply the AI Studio patch if upstream has not released the fix yet:
+Afterward, `grep 'name = "rand"' -A 2 runtime/Cargo.lock` must not show a 0.7 version anymore.
-
+`SommerEngineering/permutation-iterator-rs` can then be deleted.
-```bash
-cd "$QDRANT_REPO"
-git apply "$AISTUDIO_REPO/runtime/patches/qdrant-edge-ai-studio.patch"
-```
-
-4. Update the exposed `qdrant-edge` version in the fork:
-
-```bash
-cd "$QDRANT_REPO"
-perl -0pi -e "s/name = \"qdrant-edge\"\\nversion = \"[^\"]+\"/name = \"qdrant-edge\"\\nversion = \"$ENV{QDRANT_EDGE_VERSION}\"/" lib/edge/Cargo.toml
-```
-
-5. Commit and push the fork branch:
-
-```bash
-cd "$QDRANT_REPO"
-git diff
-git add lib/edge/Cargo.toml lib/segment/src/common/anonymize.rs
-git commit -m "Expose qdrant-edge ${QDRANT_EDGE_VERSION} package for AI Studio"
-git push origin "$QDRANT_BRANCH"
-export QDRANT_EDGE_COMMIT="$(git rev-parse HEAD)"
-echo "$QDRANT_EDGE_COMMIT"
-```
-
-6. Update AI Studio to use the new Qdrant Edge version and fork commit:
-
-```bash
-cd "$AISTUDIO_REPO"
-perl -0pi -e "s/qdrant-edge = \"[^\"]+\"/qdrant-edge = \"$ENV{QDRANT_EDGE_VERSION}\"/" runtime/Cargo.toml
-perl -0pi -e "s/rev = \"[0-9a-f]+\"/rev = \"$ENV{QDRANT_EDGE_COMMIT}\"/" runtime/Cargo.toml
-```
-
-7. Refresh the AI Studio lock file and verify the Rust runtime:
-
-```bash
-cd "$AISTUDIO_REPO/runtime"
-cargo update -p qdrant-edge
-cargo check
-```
-
-Remove the patch and the `[patch.crates-io]` override once Qdrant publishes a fixed `qdrant-edge`
-release on crates.io.

runtime/patches/qdrant-edge-ai-studio.patch

@@ -1,26 +0,0 @@
-diff --git a/lib/edge/Cargo.toml b/lib/edge/Cargo.toml
-index 7c2cf6037..d21e3c053 100644
---- a/lib/edge/Cargo.toml
-+++ b/lib/edge/Cargo.toml
-@@ -1,6 +1,6 @@
- [package]
--name = "edge"
--version = "0.1.0"
-+name = "qdrant-edge"
-+version = "0.7.2"
- authors = ["Qdrant Team <info@qdrant.tech>"]
- license = "Apache-2.0"
- edition = "2024"
-diff --git a/lib/segment/src/common/anonymize.rs b/lib/segment/src/common/anonymize.rs
-index 6b5d19b12..c73d24433 100644
---- a/lib/segment/src/common/anonymize.rs
-+++ b/lib/segment/src/common/anonymize.rs
-@@ -105,7 +105,7 @@ where
- {
-     collection_opt
-         .as_ref()
--        .map(|c| anonymize_collection_values(c))
-+        .map(|c| anonymize_collection_values::<C, K, V>(c))
- }
- 
- impl Anonymize for String {

runtime/src/qdrant_edge_database.rs

@@ -1,4 +1,4 @@
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
 use std::fs;
 use std::path::{Path, PathBuf};
 use std::sync::Mutex;
@@ -11,8 +11,9 @@ use qdrant_edge::external::uuid::Uuid;
 use qdrant_edge::{
     Condition, Distance, EdgeConfig, EdgeOptimizersConfig, EdgeShard, EdgeVectorParams,
     FieldCondition, Filter, HnswIndexConfig, Match, MatchValue, NamedQuery, Payload, PointId,
-    PointInsertOperations, PointOperations, PointStruct, QueryEnum, ScoredPoint, SearchRequest,
+    PointInsertOperations, PointOperations, PointStruct, QueryEnum, QueryRequest, ScoredPoint,
-    UpdateOperation, ValueVariants, VectorInternal, Vectors, WithPayloadInterface, WithVector,
+    ScoringQuery, UpdateOperation, ValueVariants, VectorInternal, Vectors, WithPayloadInterface,
+    WithVector,
 };
 use serde::{Deserialize, Serialize};
 use tauri::Manager;
@@ -32,6 +33,10 @@ const STORE_INITIALIZATION_MARKER_TEMP: &str = "store_name.tmp";
 const STORE_DISPLAY_NAME_MARKER: &str = "data_source_name.txt";
 const STORE_DISPLAY_NAME_MARKER_TEMP: &str = "data_source_name.tmp";
 
+/// Marks a response whose store exists on disk but cannot be opened. The .NET side keys its repair
+/// offer off this value instead of parsing `issue`, so rewording the message stays harmless.
+const ISSUE_CODE_STORE_UNREADABLE: &str = "store-unreadable";
+
 type QdrantEdgeResult<T> = Result<T, Box<dyn std::error::Error + Send + Sync>>;
 
 static QDRANT_EDGE_DATABASE: Lazy<Mutex<Option<QdrantEdgeDatabase>>> =
@@ -127,9 +132,39 @@ pub struct DeleteQdrantEdgeStoreRequest {
 pub struct QdrantEdgeResponse<T> {
     pub success: bool,
     pub issue: String,
+    pub issue_code: &'static str,
     pub data: Option<T>,
 }
 
+/// A vector store which is initialized on disk but which Qdrant Edge refuses to open.
+///
+/// This is deliberately its own error type rather than one more formatted string: a broken store
+/// is the one failure the user can act on, and the request layer has to recognize it to label the
+/// response. Nothing here deletes the store -- rebuilding the embeddings costs the user time and,
+/// with a cloud embedding provider, money, so that stays their decision.
+#[derive(Debug)]
+struct StoreUnreadableError {
+    store_name: String,
+    message: String,
+}
+
+impl StoreUnreadableError {
+    fn new(store_name: &str, path: &Path, source: impl std::fmt::Display) -> Self {
+        Self {
+            store_name: store_name.to_string(),
+            message: format!("Failed to load vector store '{store_name}' from '{}': {source}", path.display()),
+        }
+    }
+}
+
+impl std::fmt::Display for StoreUnreadableError {
+    fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        formatter.write_str(&self.message)
+    }
+}
+
+impl std::error::Error for StoreUnreadableError {}
+
 #[derive(Serialize)]
 pub struct QdrantEdgeEnsureStoreResult {
     pub created: bool,
@@ -168,6 +203,10 @@ pub struct QdrantEdgeInfo {
 pub struct QdrantEdgeDatabase {
     base_path: PathBuf,
     shards: HashMap<String, EdgeShard>,
+
+    /// Stores whose unreadability has already been logged. A broken store is hit by every single
+    /// request against it, and one log line per request would bury everything else.
+    reported_unreadable_stores: HashSet<String>,
 }
 
 impl QdrantEdgeDatabase {
@@ -175,9 +214,16 @@ impl QdrantEdgeDatabase {
         Self {
             base_path,
             shards: HashMap::new(),
+            reported_unreadable_stores: HashSet::new(),
         }
     }
 
+    /// Whether this store's defect still has to be written to the log. True exactly once per store,
+    /// until the store loads again.
+    fn report_unreadable_store(&mut self, store_name: &str) -> bool {
+        self.reported_unreadable_stores.insert(store_name.to_string())
+    }
+
     fn store_path(&self, store_name: &str) -> QdrantEdgeResult<PathBuf> {
         validate_store_name(store_name)?;
         Ok(self.base_path.join("stores").join(store_directory_name(store_name)))
@@ -191,9 +237,10 @@ impl QdrantEdgeDatabase {
         }
 
         let shard = if is_initialized {
-            EdgeShard::load(&path, None).map_err(|error| {
+            match EdgeShard::load(&path, None) {
-                format!("Failed to load vector store '{store_name}' from '{}': {error}", path.display())
+                Ok(shard) => shard,
-            })?
+                Err(error) => return Err(StoreUnreadableError::new(store_name, &path, error).into()),
+            }
         } else {
             fs::create_dir_all(&path).map_err(|error| {
                 format!("Failed to create directory for vector store '{store_name}' at '{}': {error}", path.display())
@@ -215,6 +262,7 @@ impl QdrantEdgeDatabase {
             shard
         };
 
+        self.reported_unreadable_stores.remove(store_name);
         self.shards.insert(store_name.to_string(), shard);
         Ok((self.shards.get(store_name).unwrap(), !is_initialized))
     }
@@ -230,9 +278,12 @@ impl QdrantEdgeDatabase {
             return Ok(None);
         }
 
-        let shard = EdgeShard::load(&path, None).map_err(|error| {
+        let shard = match EdgeShard::load(&path, None) {
-            format!("Failed to load vector store '{store_name}' from '{}': {error}", path.display())
+            Ok(shard) => shard,
-        })?;
+            Err(error) => return Err(StoreUnreadableError::new(store_name, &path, error).into()),
+        };
+
+        self.reported_unreadable_stores.remove(store_name);
         self.shards.insert(store_name.to_string(), shard);
         Ok(self.shards.get(store_name))
     }
@@ -306,7 +357,7 @@ impl QdrantEdgeDatabase {
         shard.update(UpdateOperation::PointOperation(
             PointOperations::UpsertPoints(PointInsertOperations::PointsList(points)),
         ))?;
-        shard.flush();
+        shard.flush()?;
         Ok(())
     }
 
@@ -320,18 +371,19 @@ impl QdrantEdgeDatabase {
             return Ok(vec![]);
         };
 
-        let search_results = shard.search(SearchRequest {
+        let search_results = shard.query(QueryRequest {
-            query: QueryEnum::Nearest(NamedQuery::new(
+            prefetches: Vec::new(),
+            query: Some(ScoringQuery::Vector(QueryEnum::Nearest(NamedQuery::new(
                 VectorInternal::Dense(vector),
                 VECTOR_NAME,
-            )),
+            )))),
             filter: None,
-            params: None,
+            score_threshold: None,
             limit: max_matches,
             offset: 0,
-            with_payload: Some(WithPayloadInterface::Bool(true)),
+            params: None,
-            with_vector: Some(WithVector::Bool(false)),
+            with_vector: WithVector::Bool(false),
-            score_threshold: None,
+            with_payload: WithPayloadInterface::Bool(true),
         })?;
 
         Ok(search_results
@@ -348,7 +400,7 @@ impl QdrantEdgeDatabase {
         shard.update(UpdateOperation::PointOperation(
             PointOperations::DeletePointsByFilter(match_keyword_filter("file_path", file_path)?),
         ))?;
-        shard.flush();
+        shard.flush()?;
         Ok(())
     }
 
@@ -361,7 +413,7 @@ impl QdrantEdgeDatabase {
         if optimized {
             info!(Source = "Qdrant Edge"; "Optimized vector store '{}'.", store_name);
         }
-        shard.flush();
+        shard.flush()?;
         Ok(())
     }
 
@@ -503,6 +555,7 @@ where
         return Json(QdrantEdgeResponse {
             success: false,
             issue: "Qdrant Edge is not available.".to_string(),
+            issue_code: "",
             data: None,
         });
     };
@@ -511,14 +564,36 @@ where
         Ok(data) => Json(QdrantEdgeResponse {
             success: true,
             issue: String::new(),
+            issue_code: "",
             data: Some(data),
         }),
         Err(e) => {
             let issue = e.to_string();
-            error!(Source = "Qdrant Edge"; "Qdrant Edge request failed: {issue}");
+
+            //
+            // An unreadable store keeps failing for as long as the user leaves it alone, so it is
+            // logged once and then only answered. Every other failure is logged as it happens,
+            // because those are one-offs worth seeing each time.
+            //
+            let issue_code = match e.downcast_ref::<StoreUnreadableError>() {
+                Some(unreadable) => {
+                    if database.report_unreadable_store(&unreadable.store_name) {
+                        error!(Source = "Qdrant Edge"; "Qdrant Edge request failed: {issue}");
+                    }
+
+                    ISSUE_CODE_STORE_UNREADABLE
+                },
+
+                None => {
+                    error!(Source = "Qdrant Edge"; "Qdrant Edge request failed: {issue}");
+                    ""
+                },
+            };
+
             Json(QdrantEdgeResponse {
                 success: false,
                 issue,
+                issue_code,
                 data: None,
             })
         },
@@ -595,7 +670,7 @@ fn remove_obsolete_qdrant_path(path: &Path) {
 
 fn edge_config(vector_size: usize) -> EdgeConfig {
     EdgeConfig {
-        on_disk_payload: true,
+        on_disk_payload: Some(true),
         vectors: HashMap::from([(
             VECTOR_NAME.to_string(),
             EdgeVectorParams {
@@ -609,13 +684,20 @@ fn edge_config(vector_size: usize) -> EdgeConfig {
             },
         )]),
         sparse_vectors: HashMap::new(),
-        hnsw_config: hnsw_config(),
+        hnsw_config: Some(hnsw_config()),
         quantization_config: None,
-        optimizers: edge_optimizers_config(),
+        optimizers: Some(edge_optimizers_config()),
         wal_options: None,
+        max_search_threads: None,
+        search_pool_core: None,
     }
 }
 
+// `on_disk` is deprecated in favor of `memory`, but Qdrant Edge does not re-export the `Memory`
+// type, so the new field cannot be named from here. Leaving both unset is not an option either:
+// the effective placement would fall back to cached instead of on-disk, which is a real change
+// and would have the optimizers rebuild the HNSW graph.
+#[allow(deprecated)]
 fn hnsw_config() -> HnswIndexConfig {
     HnswIndexConfig {
         m: HNSW_M,
@@ -623,6 +705,7 @@ fn hnsw_config() -> HnswIndexConfig {
         full_scan_threshold: HNSW_FULL_SCAN_THRESHOLD_KB,
         max_indexing_threads: HNSW_MAX_INDEXING_THREADS,
         on_disk: Some(true),
+        memory: None,
         payload_m: None,
         inline_storage: None,
     }
@@ -902,6 +985,44 @@ mod tests {
         fs::remove_dir_all(test_directory).unwrap();
     }
 
+    #[test]
+    fn an_unreadable_store_is_reported_but_never_deleted() {
+        let test_directory = std::env::temp_dir().join(format!(
+            "ai-studio-qdrant-unreadable-{}-{}",
+            std::process::id(),
+            std::time::SystemTime::now()
+                .duration_since(std::time::UNIX_EPOCH)
+                .unwrap()
+                .as_nanos()
+        ));
+        let store_name = "rag_6cc665a82b1e4d42bc748015b7b391ec";
+
+        let mut database = QdrantEdgeDatabase::new(test_directory.clone());
+        assert!(database.ensure_store_exists(store_name, "Some source", 3).unwrap().created);
+        let store_path = database.store_path(store_name).unwrap();
+
+        // Release the shard before breaking it, so the files are not held open any more.
+        drop(database);
+        fs::write(store_path.join("edge_config.json"), "this is not a config").unwrap();
+
+        let mut database = QdrantEdgeDatabase::new(test_directory.clone());
+        let error = database.get_existing_store(store_name).unwrap_err();
+        assert!(
+            error.downcast_ref::<StoreUnreadableError>().is_some(),
+            "a store which cannot be opened has to be recognizable as such, not just a message"
+        );
+
+        // The whole point: the user's embeddings survive a defect until they ask for a rebuild.
+        assert!(store_path.join("segments").is_dir());
+        assert!(store_path.join(STORE_INITIALIZATION_MARKER).is_file());
+
+        // And the defect is logged once, not once per request.
+        assert!(database.report_unreadable_store(store_name));
+        assert!(!database.report_unreadable_store(store_name));
+
+        fs::remove_dir_all(test_directory).unwrap();
+    }
+
     #[test]
     fn point_ids_must_be_valid_uuids() {
         assert!(to_point_id("6cc665a8-2b1e-4d42-bc74-8015b7b391ec").is_ok());