From e42277beba9e6b11d06edec0c8564c1f3531ac48 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Sat, 19 Sep 2026 10:12:06 +0200 Subject: [PATCH 1/2] Ask before local data sources are indexed again (#983) --- AGENTS.md | 9 + .../Assistants/I18N/allTexts.lua | 40 ++- .../DataSourceCloudEmbeddingWarning.razor | 4 +- .../Components/DataSourceManagement.razor.cs | 8 +- .../Settings/SettingsPanelEmbeddings.razor.cs | 18 +- .../DataSourceLocalDirectoryDialog.razor | 53 ++-- .../DataSourceLocalDirectoryDialog.razor.cs | 49 ++-- .../Dialogs/DataSourceLocalFileDialog.razor | 53 ++-- .../DataSourceLocalFileDialog.razor.cs | 49 ++-- .../Dialogs/EmbeddingProviderDialog.razor.cs | 46 +++- .../plugin.lua | 61 ++++- .../plugin.lua | 61 ++++- .../Settings/EmbeddingProvider.cs | 5 +- .../Tools/DataSourceReindexWarning.cs | 194 ++++++++++++++ .../PluginSystem/PluginConfigurationObject.cs | 12 +- .../DataSourceEmbeddingService.Files.cs | 42 ++- .../Services/DataSourceEmbeddingService.cs | 89 ++++++- .../Tools/Services/EmbeddingChangeImpact.cs | 47 ++++ .../Tools/TokenizerFingerprint.cs | 46 ++++ .../wwwroot/changelog/v26.9.1.md | 6 +- app/Tests/Tools/EmbeddingChangeImpactTests.cs | 247 ++++++++++++++++++ app/Tests/Tools/EmbeddingSignatureTests.cs | 65 ++++- metadata.txt | 2 +- 23 files changed, 1045 insertions(+), 161 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/DataSourceReindexWarning.cs create mode 100644 app/MindWork AI Studio/Tools/Services/EmbeddingChangeImpact.cs create mode 100644 app/MindWork AI Studio/Tools/TokenizerFingerprint.cs create mode 100644 app/Tests/Tools/EmbeddingChangeImpactTests.cs diff --git a/AGENTS.md b/AGENTS.md index 689a3f92..a143c86d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -344,3 +344,12 @@ following words: - Upgraded The entire changelog is sorted by these categories in the order shown above. The language used for the changelog is US English. + +**Every entry has to stand on its own.** Never refer back to another entry, neither by wording such +as "the same question", "that dialog", or "as described above", nor by relying on one read just +before it. Readers pick out the entries which concern them; an entry which only makes sense after +reading its neighbors turns the changelog into something nobody reads at all. Name the context +inside the entry instead, even when that repeats a few words from another one. + +**Split a topic into several short entries** rather than growing a single long one, and address the +reader with "you". diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 59749c43..b49906f9 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -6088,9 +6088,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265" -- How many matches do you want at most per query? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611"] = "How many matches do you want at most per query?" --- This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1845247453"] = "This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model." - -- Update UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Update" @@ -6127,9 +6124,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900" -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings" --- Unknown -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3424652889"] = "Unknown" - -- Select the base directory UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory" @@ -6139,6 +6133,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T787118522"] -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name" +-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T870152265"] = "The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed." + -- Cancel UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T900713019"] = "Cancel" @@ -6286,15 +6283,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = " -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings" --- Unknown -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3424652889"] = "Unknown" +-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3731767732"] = "The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed." -- Select the file UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file" --- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model." - -- The data source token limit must not be larger than the embedding provider token limit ({0}). UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "The data source token limit must not be larger than the embedding provider token limit ({0})." @@ -10822,6 +10816,30 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM -- Qdrant Edge is not available. UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available." +-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one." + +-- and {0} more. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "and {0} more." + +-- This change makes the prepared documents of the following data sources unusable ({0}): +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "This change makes the prepared documents of the following data sources unusable ({0}):" + +-- Do you want to apply this change anyway? +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Do you want to apply this change anyway?" + +-- Documents Will Be Prepared Again +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Documents Will Be Prepared Again" + +-- Your embedding provider runs in the cloud, so preparing everything again costs money. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T774305382"] = "Your embedding provider runs in the cloud, so preparing everything again costs money." + +-- These data sources are set up with this embedding provider ({0}): +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T858000918"] = "These data sources are set up with this embedding provider ({0}):" + +-- Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T874850580"] = "Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while." + -- Repair Data Source UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Repair Data Source" diff --git a/app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor b/app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor index 2ad8c465..84b7ec42 100644 --- a/app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor +++ b/app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor @@ -1,9 +1,9 @@ @inherits MSGComponentBase - + @this.WarningText - + diff --git a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs index 47461d8f..2aaacbb7 100644 --- a/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs +++ b/app/MindWork AI Studio/Components/DataSourceManagement.razor.cs @@ -335,8 +335,8 @@ public partial class DataSourceManagement : MSGComponentBase return; IDataSource? editedDataSource = null; - var lockDataSourceIdentity = dataSource is IInternalDataSource - && await this.DataSourceEmbeddingService.ShouldLockDataSourceIdentityAsync(dataSource.Id); + var lockDataSourceOrigin = dataSource is IInternalDataSource + && await this.DataSourceEmbeddingService.ShouldLockDataSourceOriginAsync(dataSource.Id); switch (dataSource) { case DataSourceLocalFile localFile: @@ -344,7 +344,7 @@ public partial class DataSourceManagement : MSGComponentBase { { x => x.IsEditing, true }, { x => x.DataSource, localFile }, - { x => x.LockSourceAndEmbedding, lockDataSourceIdentity }, + { x => x.LockSource, lockDataSourceOrigin }, { x => x.AvailableEmbeddings, this.availableEmbeddingProviders } }; @@ -361,7 +361,7 @@ public partial class DataSourceManagement : MSGComponentBase { { x => x.IsEditing, true }, { x => x.DataSource, localDirectory }, - { x => x.LockSourceAndEmbedding, lockDataSourceIdentity }, + { x => x.LockSource, lockDataSourceOrigin }, { x => x.AvailableEmbeddings, this.availableEmbeddingProviders } }; diff --git a/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs b/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs index 38999b40..a48b3f7d 100644 --- a/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs +++ b/app/MindWork AI Studio/Components/Settings/SettingsPanelEmbeddings.razor.cs @@ -94,6 +94,7 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase { x => x.IsEditing, true }, { x => x.DataHost, embeddingProvider.Host }, { x => x.DataTokenizerPath, embeddingProvider.TokenizerPath }, + { x => x.DataTokenizerFingerprint, embeddingProvider.TokenizerFingerprint }, { x => x.DataTokenLimit, embeddingProvider.EffectiveTokenLimit }, { x => x.DataEmbeddingBatchSize, embeddingProvider.EffectiveEmbeddingBatchSize }, { x => x.HFInferenceProviderId, embeddingProvider.HFInferenceProvider }, @@ -132,11 +133,18 @@ public partial class SettingsPanelEmbeddings : SettingsPanelProviderBase private async Task DeleteEmbeddingProvider(EmbeddingProvider provider) { - var dialogParameters = new DialogParameters - { - { x => x.Message, string.Format(T("Are you sure you want to delete the embedding provider '{0}'?"), provider.Name) }, - }; - + var question = string.Format(T("Are you sure you want to delete the embedding provider '{0}'?"), provider.Name); + var affectedDataSources = DataSourceReindexWarning.DescribeDataSourcesLosingTheirProvider(this.SettingsManager, provider); + + // + // The names arrive as a Markdown list, so the question travels as Markdown as well as soon + // as there is something to name. With no data source behind the provider, the plain message + // stays what it always was: + // + var dialogParameters = string.IsNullOrEmpty(affectedDataSources) + ? new DialogParameters { { x => x.Message, question } } + : new DialogParameters { { x => x.MarkdownBody, $"{affectedDataSources}{Environment.NewLine}{question}" } }; + var dialogReference = await this.DialogService.ShowAsync(T("Delete Embedding Provider"), dialogParameters, DialogOptions.FULLSCREEN); var dialogResult = await dialogReference.Result; if (dialogResult is null || dialogResult.Canceled) diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index ad4f7c0f..600fd819 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -45,13 +45,13 @@ @T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.") - @if (!this.CanChangeSourceAndEmbedding) + @if (!this.CanChangeSource) { - @T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.") + @T("The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed.") } - @if (this.CanChangeSourceAndEmbedding) + @if (this.CanChangeSource) { } @@ -74,38 +74,21 @@ @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.") - @if (this.CanChangeSourceAndEmbedding) - { - - @foreach (var embedding in this.AvailableEmbeddings) - { - - @if (this.GetEmbeddingProvider(embedding.Value) is { } provider) - { - - } - else - { - @embedding.Name - } - - } - - } - else - { - - } + + @foreach (var embedding in this.AvailableEmbeddings) + { + + @if (this.GetEmbeddingProvider(embedding.Value) is { } provider) + { + + } + else + { + @embedding.Name + } + + } + @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId)) { diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs index 996bf28c..ebd4a374 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs @@ -20,12 +20,25 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase [Parameter] public DataSourceLocalDirectory DataSource { get; set; } + /// + /// Whether the folder this data source reads must stay as it is. + /// + /// + /// Set once the index holds something for this data source. The embedding is not locked along + /// with it: it can be changed, and DataSourceReindexWarning asks what that costs. + /// [Parameter] - public bool LockSourceAndEmbedding { get; set; } + public bool LockSource { get; set; } [Parameter] public IReadOnlyList> AvailableEmbeddings { get; set; } = []; - + + [Inject] + private IDialogService DialogService { get; init; } = null!; + + [Inject] + private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!; + private static readonly Dictionary SPELLCHECK_ATTRIBUTES = new(); private readonly DataSourceValidation dataSourceValidation; @@ -120,19 +133,10 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false }; - private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding; + private bool CanChangeSource => !this.IsEditing || !this.LockSource; private IEnumerable> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData(); - private string SelectedEmbeddingNameText - { - get - { - var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId); - return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name; - } - } - private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null ? T("No embedding selected") : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath) @@ -156,8 +160,11 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase Name = this.dataName, Description = this.dataDescription, Type = DataSourceType.LOCAL_DIRECTORY, - EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId, - Path = this.CanChangeSourceAndEmbedding ? this.dataPath : this.DataSource.Path, + EmbeddingId = this.dataEmbeddingId, + + // Kept out of reach of the form while the source is locked, so a stale field cannot point an + // indexed data source somewhere else: + Path = this.CanChangeSource ? this.dataPath : this.DataSource.Path, MaxChunkTokenLength = this.dataMaxChunkTokenLength, ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, ConfidenceLevel = this.dataConfidenceLevel, @@ -171,8 +178,20 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase // When the data is not valid, we don't store it: if (!this.dataIsValid) return; - + var addedDataSource = this.CreateDataSource(); + + // + // Ask while the dialog is still open, so a token limit which would have cost the prepared + // documents can be corrected right away. Asking in DataSourceManagement instead would have + // to be written once per data source kind, and by then the numbers are out of reach. + // + // Only when editing: while adding, DataSource is still default -- both local data sources + // are record structs -- and nothing has been prepared for a source which does not exist yet. + // + if (this.IsEditing && !await DataSourceReindexWarning.ConfirmDataSourceChangeAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, this.DataSource, addedDataSource)) + return; + this.MudDialog.Close(DialogResult.Ok(addedDataSource)); } diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor index 3a94c100..6d26a3d4 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor @@ -45,13 +45,13 @@ @T("Select a file for this data source. The content of this file will be processed for the data source.") - @if (!this.CanChangeSourceAndEmbedding) + @if (!this.CanChangeSource) { - @T("This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.") + @T("The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed.") } - @if (this.CanChangeSourceAndEmbedding) + @if (this.CanChangeSource) { } @@ -74,38 +74,21 @@ @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.") - @if (this.CanChangeSourceAndEmbedding) - { - - @foreach (var embedding in this.AvailableEmbeddings) - { - - @if (this.GetEmbeddingProvider(embedding.Value) is { } provider) - { - - } - else - { - @embedding.Name - } - - } - - } - else - { - - } + + @foreach (var embedding in this.AvailableEmbeddings) + { + + @if (this.GetEmbeddingProvider(embedding.Value) is { } provider) + { + + } + else + { + @embedding.Name + } + + } + @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId)) { diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs index 698652dc..cdbe26ab 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs @@ -20,12 +20,25 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase [Parameter] public DataSourceLocalFile DataSource { get; set; } + /// + /// Whether the file this data source reads must stay as it is. + /// + /// + /// Set once the index holds something for this data source. The embedding is not locked along + /// with it: it can be changed, and DataSourceReindexWarning asks what that costs. + /// [Parameter] - public bool LockSourceAndEmbedding { get; set; } + public bool LockSource { get; set; } [Parameter] public IReadOnlyList> AvailableEmbeddings { get; set; } = []; - + + [Inject] + private IDialogService DialogService { get; init; } = null!; + + [Inject] + private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!; + private static readonly Dictionary SPELLCHECK_ATTRIBUTES = new(); private readonly DataSourceValidation dataSourceValidation; @@ -120,19 +133,10 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false }; - private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding; + private bool CanChangeSource => !this.IsEditing || !this.LockSource; private IEnumerable> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData(); - private string SelectedEmbeddingNameText - { - get - { - var selectedEmbedding = this.AvailableEmbeddings.FirstOrDefault(x => x.Value == this.dataEmbeddingId); - return string.IsNullOrWhiteSpace(selectedEmbedding.Name) ? T("Unknown") : selectedEmbedding.Name; - } - } - private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null ? T("No embedding selected") : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath) @@ -156,8 +160,11 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase Name = this.dataName, Description = this.dataDescription, Type = DataSourceType.LOCAL_FILE, - EmbeddingId = this.CanChangeSourceAndEmbedding ? this.dataEmbeddingId : this.DataSource.EmbeddingId, - FilePath = this.CanChangeSourceAndEmbedding ? this.dataFilePath : this.DataSource.FilePath, + EmbeddingId = this.dataEmbeddingId, + + // Kept out of reach of the form while the source is locked, so a stale field cannot point an + // indexed data source somewhere else: + FilePath = this.CanChangeSource ? this.dataFilePath : this.DataSource.FilePath, MaxChunkTokenLength = this.dataMaxChunkTokenLength, ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, ConfidenceLevel = this.dataConfidenceLevel, @@ -171,8 +178,20 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase // When the data is not valid, we don't store it: if (!this.dataIsValid) return; - + var addedDataSource = this.CreateDataSource(); + + // + // Ask while the dialog is still open, so a token limit which would have cost the prepared + // documents can be corrected right away. Asking in DataSourceManagement instead would have + // to be written once per data source kind, and by then the numbers are out of reach. + // + // Only when editing: while adding, DataSource is still default -- both local data sources + // are record structs -- and nothing has been prepared for a source which does not exist yet. + // + if (this.IsEditing && !await DataSourceReindexWarning.ConfirmDataSourceChangeAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, this.DataSource, addedDataSource)) + return; + this.MudDialog.Close(DialogResult.Ok(addedDataSource)); } diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs index 1ed37e94..03479185 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs @@ -86,6 +86,17 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId [Parameter] public string DataTokenizerPath { get; set; } = string.Empty; + /// + /// The fingerprint of the tokenizer this provider was stored with. + /// + /// + /// Carried through the dialog untouched as long as the user leaves the tokenizer alone. Rebuilding + /// it from the path on every open would read a file for nothing, and an unreadable one would look + /// like another tokenizer and cost every data source of this provider its index. + /// + [Parameter] + public string DataTokenizerFingerprint { get; set; } = string.Empty; + [Parameter] public int DataTokenLimit { get; set; } = EmbeddingProvider.DEFAULT_TOKEN_LIMIT; @@ -105,6 +116,12 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId [Inject] private ILogger Logger { get; init; } = null!; + [Inject] + private IDialogService DialogService { get; init; } = null!; + + [Inject] + private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!; + private static readonly Dictionary SPELLCHECK_ATTRIBUTES = new(); /// @@ -121,6 +138,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId private string dataEditingPreviousInstanceName = string.Empty; private string dataLoadingModelsIssue = string.Empty; private string dataFilePath = string.Empty; + private string dataTokenizerFingerprint = string.Empty; private string dataCustomTokenizerValidationIssue = string.Empty; private Task dataTokenizerValidationTask = Task.CompletedTask; private bool dataStoreWasAttempted; @@ -176,6 +194,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId IsEnterpriseConfiguration = this.IsEnterpriseConfiguration, EnterpriseConfigurationPluginId = Guid.Empty, TokenizerPath = this.dataFilePath, + TokenizerFingerprint = this.dataTokenizerFingerprint, EmbeddingBatchSize = this.DataEmbeddingBatchSize, TokenLimit = this.DataTokenLimit, CustomIconDataUrl = this.DataCustomIconDataUrl, @@ -201,6 +220,7 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId { this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant(); this.dataFilePath = this.DataTokenizerPath; + this.dataTokenizerFingerprint = this.DataTokenizerFingerprint; this.showExpertSettings = !string.IsNullOrWhiteSpace(this.DataTokenizerPath) || this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT || this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE; @@ -279,7 +299,22 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId // When the data is not valid, we don't store it: if (!this.dataIsValid) return; - + + // + // Ask before anything is written. Storing a tokenizer deletes the previous one before it + // copies, and the API key goes into the OS keyring right after, so asking any later would + // leave those changes behind even when the user says no. Saying no also keeps this dialog + // open, which is the point: the value which would have cost the index can be corrected + // right away. + // + // Enterprise-managed providers are left out. Every field which reaches the embedding + // signature is locked for them, and their data sources are not queued for indexing either. + // + if (this.IsEditing && !this.IsEnterpriseConfiguration && !await DataSourceReindexWarning.ConfirmEmbeddingProviderChangeAsync( + this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, + this.SettingsManager.GetEmbeddingProviderById(this.DataId), this.CreateEmbeddingProviderSettings())) + return; + var response = await this.StoreOrDeleteTokenizerAsync(); if (!response.Success) { @@ -413,9 +448,18 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId this.dataTokenizerValidationTask = this.ValidateCustomTokenizer(filePath, validationRevision); await this.dataTokenizerValidationTask; + // + // The embedding signature carries the tokenizer's content, so it has to be read while we have + // the file the user just picked. Reading it here rather than while storing also keeps a large + // file off that path, where it would stall the circuit. + // + var tokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(filePath); + if (validationRevision != this.dataTokenizerValidationRevision) return; + this.dataTokenizerFingerprint = tokenizerFingerprint; + if (this.dataStoreWasAttempted) await this.form.Validate(); else diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 5dc241fe..ed18cde1 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -3810,6 +3810,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2675917723"] = "Ni -- No valid embedding UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2698203405"] = "Keine gültige Einbettung" +-- Repair this data source by indexing it anew +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2771708618"] = "Diese Datenquelle durch erneutes Indexieren reparieren" + -- Embedding UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2838542994"] = "Einbettung" @@ -3957,6 +3960,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T86053874"] = "Verf -- This data source is waiting to be indexed again. Until that is finished, it cannot be searched. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T1692539409"] = "Diese Datenquelle wartet darauf, erneut indexiert zu werden. Bis dies abgeschlossen ist, kann sie nicht durchsucht werden." +-- The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T4047623216"] = "Der Index dieser Datenquelle kann nicht mehr gelesen werden. Öffnen Sie die Einstellungen Ihrer Datenquelle über das Zahnradsymbol oben und führen Sie dort die Reparaturaktion aus." + -- Tools (Optional) UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERFORM::T1019749907"] = "Werkzeuge (optional)" @@ -6084,9 +6090,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265" -- How many matches do you want at most per query? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611"] = "Wie viele Treffer möchten Sie maximal pro Abfrage erhalten?" --- This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1845247453"] = "Diese Datenquelle verfügt bereits über indexierte Einbettungen. Löschen und erstellen Sie sie neu, um den Ordnerpfad oder das Einbettungsmodell zu ändern." - -- Update UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Aktualisieren" @@ -6123,9 +6126,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900" -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Experten-Einstellungen anzeigen" --- Unknown -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3424652889"] = "Unbekannt" - -- Select the base directory UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Wählen Sie das Stammverzeichnis aus" @@ -6135,6 +6135,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T787118522"] -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Name der Datenquelle" +-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T870152265"] = "Die Dokumente dieser Datenquelle sind bereits vorbereitet, daher kann ihr Ordner nicht geändert werden. Ein anderer Ordner enthält andere Dokumente und ist damit eine andere Datenquelle: Bitte fügen Sie dafür eine neue hinzu. Die Einbettungsmethode darunter können Sie ändern." + -- Cancel UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T900713019"] = "Abbrechen" @@ -6282,15 +6285,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = " -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Experten-Einstellungen anzeigen" --- Unknown -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3424652889"] = "Unbekannt" +-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3731767732"] = "Die Dokumente dieser Datenquelle sind bereits vorbereitet, daher kann ihre Datei nicht geändert werden. Eine andere Datei enthält andere Inhalte und ist damit eine andere Datenquelle: Bitte fügen Sie dafür eine neue hinzu. Die Einbettungsmethode darunter können Sie ändern." -- Select the file UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Datei auswählen" --- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "Diese Datenquelle verfügt bereits über indexierte Einbettungen. Löschen und erstellen Sie sie neu, um den Dateipfad oder das Einbettungsmodell zu ändern." - -- The data source token limit must not be larger than the embedding provider token limit ({0}). UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "Das Token-Limit der Datenquelle darf nicht größer sein als das Token-Limit des Einbettungsanbieters ({0})." @@ -9141,6 +9141,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2525374657"] = "{0} von {1} Datei -- Background embeddings UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2547971789"] = "Einbettungen im Hintergrund" +-- Repair this data source by indexing it anew +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2771708618"] = "Diese Datenquelle durch erneutes Indexieren reparieren" + -- Refresh this data source UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2901874229"] = "Diese Datenquelle aktualisieren" @@ -10815,6 +10818,36 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM -- Qdrant Edge is not available. UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge ist nicht verfügbar." +-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "Sie beantworten weiterhin Stichwortsuchen, aber die Suche nach Bedeutung funktioniert nicht mehr, und es können keine weiteren Dokumente für sie vorbereitet werden. Bereits vorbereitete Datenquellen bleiben zudem an diesen Anbieter gebunden, sodass Sie sie nicht einfach auf einen anderen umstellen können." + +-- and {0} more. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "und {0} weitere." + +-- This change makes the prepared documents of the following data sources unusable ({0}): +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "Durch diese Änderung werden die vorbereiteten Dokumente der folgenden Datenquellen unbrauchbar ({0}):" + +-- Do you want to apply this change anyway? +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Möchten Sie diese Änderung trotzdem übernehmen?" + +-- Documents Will Be Prepared Again +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Dokumente werden erneut vorbereitet" + +-- Your embedding provider runs in the cloud, so preparing everything again costs money. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T774305382"] = "Ihr Einbettungsanbieter läuft in der Cloud, daher kostet es Geld, alles erneut vorzubereiten." + +-- These data sources are set up with this embedding provider ({0}): +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T858000918"] = "Diese Datenquellen sind mit diesem Einbettungsanbieter eingerichtet ({0}):" + +-- Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T874850580"] = "Alles, was für sie vorbereitet wurde, wird verworfen, und jedes ihrer Dokumente wird erneut an Ihren Einbettungsanbieter gesendet. Bei einer großen Datenquelle dauert dies eine Weile." + +-- Repair Data Source +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Datenquelle reparieren" + +-- The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now? +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T857336889"] = "Der Index der Datenquelle „{0}“ kann nicht mehr gelesen werden. Bei einer Reparatur wird der Index vollständig neu erstellt: Alle bisher indexierten Daten werden verworfen und jedes Dokument dieser Datenquelle erneut an Ihren Einbettungsanbieter gesendet. Bei einem Cloud-Anbieter entstehen dadurch Kosten, und bei einer großen Datenquelle kann dies einige Zeit dauern. Möchten Sie diese Datenquelle jetzt reparieren?" + -- The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment. UI_TEXT_CONTENT["AISTUDIO::TOOLS::ERICLIENT::DATAMODEL::PROVIDERTYPEEXTENSIONS::T1555790630"] = "Die zugehörigen Daten dürfen an keinen LLM-Anbieter gesendet werden. Das bedeutet, dass diese Datenquelle momentan nicht verwendet werden kann." @@ -12009,6 +12042,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T4515612 -- The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T667058890"] = "Der Einbettungsanbieter hat für {1} Teile der Datei „{2}“ {0} Vektoren zurückgegeben. Bitte wählen Sie ein anderes Einbettungsmodell oder einen anderen Anbieter aus." +-- The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T831900720"] = "Der Index der Datenquelle „{0}“ kann nicht mehr gelesen werden. Die Datenquelle wird in Ihren Chats nicht verwendet, bis ihr Index neu erstellt wurde. Verwenden Sie die Aktion „Reparieren“, um dies zu starten." + -- The folder '{0}' does not exist. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T871336081"] = "Der Ordner „{0}“ existiert nicht." @@ -12057,6 +12093,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T40 -- Page {0} UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Seite {0}" +-- The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T59210871"] = "Die Datenquelle „{0}“ wurde aus der Antwort weggelassen, weil ihr Index nicht mehr gelesen werden kann. Sie können ihn in den Einstellungen der Datenquelle reparieren." + -- The data source '{0}' was left out of the answer because searching it failed. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T934856625"] = "Die Datenquelle „{0}“ wurde aus der Antwort weggelassen, weil die Suche darin fehlgeschlagen ist." diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index ba4c56cd..8853aa1d 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -3810,6 +3810,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2675917723"] = "No -- No valid embedding UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2698203405"] = "No valid embedding" +-- Repair this data source by indexing it anew +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2771708618"] = "Repair this data source by indexing it anew" + -- Embedding UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2838542994"] = "Embedding" @@ -3957,6 +3960,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTION::T86053874"] = "Avail -- This data source is waiting to be indexed again. Until that is finished, it cannot be searched. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T1692539409"] = "This data source is waiting to be indexed again. Until that is finished, it cannot be searched." +-- The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCESELECTIONROW::T4047623216"] = "The index of this data source cannot be read anymore. Open your data source settings with the gear icon above, then use the repair action there." + -- Tools (Optional) UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERFORM::T1019749907"] = "Tools (Optional)" @@ -6084,9 +6090,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265" -- How many matches do you want at most per query? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611"] = "How many matches do you want at most per query?" --- This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1845247453"] = "This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model." - -- Update UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Update" @@ -6123,9 +6126,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900" -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings" --- Unknown -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3424652889"] = "Unknown" - -- Select the base directory UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory" @@ -6135,6 +6135,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T787118522"] -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name" +-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T870152265"] = "The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding method below can be changed." + -- Cancel UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T900713019"] = "Cancel" @@ -6282,15 +6285,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = " -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings" --- Unknown -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3424652889"] = "Unknown" +-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3731767732"] = "The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding method below can be changed." -- Select the file UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file" --- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model." - -- The data source token limit must not be larger than the embedding provider token limit ({0}). UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "The data source token limit must not be larger than the embedding provider token limit ({0})." @@ -9141,6 +9141,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2525374657"] = "{0} of {1} files -- Background embeddings UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2547971789"] = "Background embeddings" +-- Repair this data source by indexing it anew +UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2771708618"] = "Repair this data source by indexing it anew" + -- Refresh this data source UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T2901874229"] = "Refresh this data source" @@ -10815,6 +10818,36 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM -- Qdrant Edge is not available. UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available." +-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one." + +-- and {0} more. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "and {0} more." + +-- This change makes the prepared documents of the following data sources unusable ({0}): +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "This change makes the prepared documents of the following data sources unusable ({0}):" + +-- Do you want to apply this change anyway? +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Do you want to apply this change anyway?" + +-- Documents Will Be Prepared Again +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Documents Will Be Prepared Again" + +-- Your embedding provider runs in the cloud, so preparing everything again costs money. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T774305382"] = "Your embedding provider runs in the cloud, so preparing everything again costs money." + +-- These data sources are set up with this embedding provider ({0}): +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T858000918"] = "These data sources are set up with this embedding provider ({0}):" + +-- Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T874850580"] = "Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while." + +-- Repair Data Source +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T4175865785"] = "Repair Data Source" + +-- The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now? +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREPAIR::T857336889"] = "The index of the data source '{0}' cannot be read anymore. Repairing it means building the index from scratch: everything indexed so far is thrown away, and every document of this data source is sent to your embedding provider once more. With a cloud provider, this costs money, and with a large data source it takes a while. Do you want to repair this data source now?" + -- The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment. UI_TEXT_CONTENT["AISTUDIO::TOOLS::ERICLIENT::DATAMODEL::PROVIDERTYPEEXTENSIONS::T1555790630"] = "The related data is not allowed to be sent to any LLM provider. This means that this data source cannot be used at the moment." @@ -12009,6 +12042,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T4515612 -- The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T667058890"] = "The embedding provider answered with {0} vectors for {1} parts of the file '{2}'. Please select another embedding model or provider." +-- The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T831900720"] = "The index of the data source '{0}' cannot be read anymore. The data source stays out of your chats until its index was built anew. Use the repair action to start that." + -- The folder '{0}' does not exist. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSERVICE::T871336081"] = "The folder '{0}' does not exist." @@ -12057,6 +12093,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T40 -- Page {0} UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Page {0}" +-- The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T59210871"] = "The data source '{0}' was left out of the answer: its index cannot be read anymore. You can repair it in your data source settings." + -- The data source '{0}' was left out of the answer because searching it failed. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T934856625"] = "The data source '{0}' was left out of the answer because searching it failed." diff --git a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs index 46402e1c..8aa411b1 100644 --- a/app/MindWork AI Studio/Settings/EmbeddingProvider.cs +++ b/app/MindWork AI Studio/Settings/EmbeddingProvider.cs @@ -23,6 +23,7 @@ public sealed record EmbeddingProvider( string Hostname = "http://localhost:1234", Host Host = Host.NONE, string TokenizerPath = "", + string TokenizerFingerprint = "", int EmbeddingBatchSize = 0, int TokenLimit = 0, bool AllowUserProvidedAPIKey = false, @@ -113,14 +114,14 @@ public sealed record EmbeddingProvider( } var tokenLimit = DEFAULT_TOKEN_LIMIT; - if (table.TryGetValue("TokenLimit", out var tokenLimitValue) && (!tokenLimitValue.TryRead(out tokenLimit) || tokenLimit < 1)) + if (table.TryGetValue("TokenLimit", out var tokenLimitValue) && (!tokenLimitValue.TryRead(out tokenLimit) || tokenLimit < 1)) { LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid token limit. Falling back to {DEFAULT_TOKEN_LIMIT}. (Plugin ID: {configPluginId})"); tokenLimit = DEFAULT_TOKEN_LIMIT; } var embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE; - if (table.TryGetValue("EmbeddingBatchSize", out var embeddingBatchSizeValue) && (!embeddingBatchSizeValue.TryRead(out embeddingBatchSize) || embeddingBatchSize < 1)) + if (table.TryGetValue("EmbeddingBatchSize", out var embeddingBatchSizeValue) && (!embeddingBatchSizeValue.TryRead(out embeddingBatchSize) || embeddingBatchSize < 1)) { LOGGER.LogWarning($"The configured embedding provider {idx} does not contain a valid embedding batch size. Falling back to {DEFAULT_EMBEDDING_BATCH_SIZE}. (Plugin ID: {configPluginId})"); embeddingBatchSize = DEFAULT_EMBEDDING_BATCH_SIZE; diff --git a/app/MindWork AI Studio/Tools/DataSourceReindexWarning.cs b/app/MindWork AI Studio/Tools/DataSourceReindexWarning.cs new file mode 100644 index 00000000..9f896de3 --- /dev/null +++ b/app/MindWork AI Studio/Tools/DataSourceReindexWarning.cs @@ -0,0 +1,194 @@ +using System.Text; + +using AIStudio.Dialogs; +using AIStudio.Settings; +using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.Services; + +namespace AIStudio.Tools; + +/// +/// Asks before an edit makes the prepared documents of data sources useless, and names the data +/// sources which depend on an embedding provider somebody is about to delete. +/// +/// +/// Kept here rather than in the dialogs which ask -- the embedding provider dialog and the two data +/// source dialogs -- so the sentence naming what a rebuild costs cannot drift apart between them. +/// That is the same reason DataSourceRepair sits next to it, and both name the same two costs. +/// +/// Nothing is asked when nothing is lost. A data source only reaches the question when the edit +/// really changes its embedding signature and when the index already holds something for it, so +/// renaming an embedding provider or editing a data source nobody has indexed yet stays silent. +/// +public static class DataSourceReindexWarning +{ + /// + /// How many data sources are named before the rest is only counted. + /// + private const int MAX_NAMED_DATA_SOURCES = 10; + + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceReindexWarning).Namespace, nameof(DataSourceReindexWarning)); + + /// + /// Asks before an edited embedding provider is saved. + /// + /// The dialog service to ask with. + /// The settings, read for the data sources behind the provider. + /// The service which knows what the index holds. + /// The embedding provider as it is stored. + /// The embedding provider as it would be stored. + /// The cancellation token. + /// True when the edit may be saved. + public static async Task ConfirmEmbeddingProviderChangeAsync(IDialogService dialogService, SettingsManager settingsManager, DataSourceEmbeddingService embeddingService, + EmbeddingProvider before, EmbeddingProvider after, CancellationToken token = default) + { + // Nothing was stored under this id, so no data source can point at it: + if (before == EmbeddingProvider.NONE) + return true; + + var candidates = GetDataSourcesUsing(settingsManager, before.Id) + .Where(dataSource => EmbeddingChangeImpact.AffectsStoredIndex(dataSource, before, after)) + .Cast() + .ToList(); + + if (candidates.Count == 0) + return true; + + var affected = await embeddingService.GetDataSourcesWithStoredIndexAsync(candidates, token); + return await ConfirmAsync(dialogService, affected, !after.IsSelfHosted); + } + + /// + /// Asks before an edited data source is saved. + /// + /// The dialog service to ask with. + /// The settings, read for the embedding provider of the data source. + /// The service which knows what the index holds. + /// The data source as it is stored. + /// The data source as it would be stored. + /// The cancellation token. + /// True when the edit may be saved. + public static async Task ConfirmDataSourceChangeAsync(IDialogService dialogService, SettingsManager settingsManager, DataSourceEmbeddingService embeddingService, + IInternalDataSource before, IInternalDataSource after, CancellationToken token = default) + { + // Without a provider nothing is embedded at all, so nothing can be lost: + if (!DataSourceEmbeddingProviders.TryResolve(settingsManager, after, out var afterProvider)) + return true; + + // + // The provider a data source points at today may be gone -- somebody deleted it, and this + // edit is how the source is put back to work. Standing in for it with NONE gives a signature + // of its own, so that edit is asked about as well, which is right: what is stored was made by + // a provider nobody can reach any more. + // + DataSourceEmbeddingProviders.TryResolve(settingsManager, before, out var resolvedBeforeProvider); + var beforeProvider = resolvedBeforeProvider ?? EmbeddingProvider.NONE; + + if (!EmbeddingChangeImpact.AffectsStoredIndex(before, beforeProvider, after, afterProvider)) + return true; + + var affected = await embeddingService.GetDataSourcesWithStoredIndexAsync([after], token); + return await ConfirmAsync(dialogService, affected, !afterProvider.IsSelfHosted); + } + + /// + /// Names the data sources which would lose their embedding provider, for the deletion question. + /// + /// + /// Deleting is the one case where nothing prepared is thrown away: the documents stay where they + /// are, but nothing can reach them by meaning any more, and nothing new can be prepared either. + /// The names come from the same place as the ones in the questions above so that both lists read + /// alike, which is also why this returns the text instead of asking on its own -- the deletion + /// question has more to say than this. + /// + /// Every data source pointing at the provider is named, prepared or not. A source which was never + /// indexed loses just as much: it can no longer be prepared at all. + /// + /// The settings holding the data sources. + /// The embedding provider which is about to be deleted. + /// The Markdown text, or an empty string when no data source uses that provider. + public static string DescribeDataSourcesLosingTheirProvider(SettingsManager settingsManager, EmbeddingProvider embeddingProvider) + { + if (embeddingProvider == EmbeddingProvider.NONE) + return string.Empty; + + var affected = GetDataSourcesUsing(settingsManager, embeddingProvider.Id).Cast().ToList(); + if (affected.Count == 0) + return string.Empty; + + var body = new StringBuilder(); + + // Counted rather than put into a plural form: the I18N has no mechanism for one. + body.AppendLine(string.Format(TB("These data sources are set up with this embedding provider ({0}):"), affected.Count.CompactCount())); + body.AppendLine(); + body.AppendLine(FormatDataSourceNames(affected)); + body.AppendLine(); + body.AppendLine(TB("They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one.")); + + return body.ToString(); + } + + /// + /// The data sources which are indexed with a given embedding provider. + /// + /// The settings holding the data sources. + /// The id of the embedding provider. + /// The data sources pointing at that embedding provider. + private static IReadOnlyList GetDataSourcesUsing(SettingsManager settingsManager, string embeddingProviderId) => + settingsManager.ConfigurationData.DataSources + .OfType() + .Where(dataSource => embeddingProviderId.Equals(dataSource.EmbeddingId, StringComparison.OrdinalIgnoreCase)) + .ToList(); + + /// + /// Names data sources as a Markdown list, counting the rest when there are too many to name. + /// + /// The data sources to name. + /// The Markdown list. + private static string FormatDataSourceNames(IReadOnlyList dataSources) + { + var names = dataSources + .Select(dataSource => dataSource.Name) + .OrderBy(name => name, StringComparer.OrdinalIgnoreCase) + .ToList(); + + var lines = names.Take(MAX_NAMED_DATA_SOURCES).Select(name => $"- {name}").ToList(); + if (names.Count > MAX_NAMED_DATA_SOURCES) + lines.Add($"- {string.Format(TB("and {0} more."), (names.Count - MAX_NAMED_DATA_SOURCES).CompactCount())}"); + + return string.Join(Environment.NewLine, lines); + } + + private static async Task ConfirmAsync(IDialogService dialogService, IReadOnlyList affected, bool usesCloudEmbedding) + { + if (affected.Count == 0) + return true; + + var body = new StringBuilder(); + + // Counted rather than put into a plural form: the I18N has no mechanism for one. + body.AppendLine(string.Format(TB("This change makes the prepared documents of the following data sources unusable ({0}):"), affected.Count.CompactCount())); + body.AppendLine(); + body.AppendLine(FormatDataSourceNames(affected)); + body.AppendLine(); + body.AppendLine(TB("Everything prepared for them is thrown away, and every one of their documents goes to your embedding provider once more. With a large data source, this takes a while.")); + + if (usesCloudEmbedding) + { + body.AppendLine(); + body.AppendLine(TB("Your embedding provider runs in the cloud, so preparing everything again costs money.")); + } + + body.AppendLine(); + body.AppendLine(TB("Do you want to apply this change anyway?")); + + var dialogParameters = new DialogParameters + { + { x => x.MarkdownBody, body.ToString() }, + }; + + var dialogReference = await dialogService.ShowAsync(TB("Documents Will Be Prepared Again"), dialogParameters, Dialogs.DialogOptions.FULLSCREEN); + var dialogResult = await dialogReference.Result; + return dialogResult is not null && !dialogResult.Canceled; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs index b5e6516e..7b2f045e 100644 --- a/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs +++ b/app/MindWork AI Studio/Tools/PluginSystem/PluginConfigurationObject.cs @@ -497,7 +497,17 @@ public sealed record PluginConfigurationObject TokenizerModelId.ForEmbeddingProvider(provider), $"embedding provider '{provider.Name}'"); - return provider with { TokenizerPath = syncedTokenizerPath }; + // + // The embedding signature is built from the tokenizer's content, so the fingerprint travels + // with the provider. An unreadable file yields nothing, and writing that would look like + // another tokenizer and cost every data source of this provider its index -- so in that case + // the previous fingerprint is kept rather than cleared. + // + var syncedTokenizerFingerprint = await TokenizerFingerprint.ForFileAsync(syncedTokenizerPath); + if (string.IsNullOrEmpty(syncedTokenizerFingerprint) && !string.IsNullOrWhiteSpace(syncedTokenizerPath)) + syncedTokenizerFingerprint = provider.TokenizerFingerprint; + + return provider with { TokenizerPath = syncedTokenizerPath, TokenizerFingerprint = syncedTokenizerFingerprint }; } private static async Task SyncTokenizerAsync(string configuredTokenizerPath, string pluginPath, string modelId, string logName) diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 0348c558..e6dc8450 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -67,7 +67,7 @@ public sealed partial class DataSourceEmbeddingService private async IAsyncEnumerable StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) { - var options = this.GetChunkingOptions(dataSource, embeddingProvider); + var options = GetChunkingOptions(dataSource, embeddingProvider); var strategy = this.GetChunkingStrategy(filePath); var content = await this.ReadExtractedFileContentAsync(filePath, embeddingProvider, token); @@ -582,7 +582,18 @@ public sealed partial class DataSourceEmbeddingService throw new InvalidOperationException(string.Format(TB("The tokens of the text could not be counted for the embedding provider '{0}'. {1}"), embeddingProvider.Name, message)); } - private ChunkingOptions GetChunkingOptions(IDataSource dataSource, EmbeddingProvider embeddingProvider) + /// + /// Works out how the text of a data source is cut for a given embedding provider. + /// + /// + /// Static, because the answer follows from its two arguments alone. That lets the embedding + /// signature be built for a configuration which is not stored yet, which is what the dialogs ask + /// before they save a change. + /// + /// The data source whose own chunk settings apply. + /// The embedding provider whose token limit caps them. + /// The chunk size and overlap which are actually used. + internal static ChunkingOptions GetChunkingOptions(IDataSource dataSource, EmbeddingProvider embeddingProvider) { var providerMaxChunkTokenLength = Math.Max(1, embeddingProvider.EffectiveTokenLimit); var dataSourceMaxChunkTokenLength = dataSource is IInternalDataSource { MaxChunkTokenLength: > 0 } internalDataSource @@ -994,6 +1005,19 @@ public sealed partial class DataSourceEmbeddingService /// where it runs, how the text was cut for it, and the chunk metadata version — the things a /// vector actually depends on. /// + /// Two of them are less obvious than they look. The Hugging Face inference provider belongs to + /// where the model runs: the same model name served by another backend is another vector source. + /// And a custom tokenizer enters through its content, not through its path, because a tokenizer + /// is stored under the name it came with — almost always tokenizer.json — so swapping one for + /// another lands on the identical path, while moving the data directory changes every path + /// without changing a single tokenizer. + /// + /// The chunk settings enter only as what they amount to, never as what somebody typed. A data + /// source storing 0 means "follow the embedding provider", and writing that provider's own limit + /// into the field changes nothing about how the text is cut. Carrying the typed numbers as well + /// made that a different signature, so opening the expert settings of a data source — which + /// fills an empty limit with the provider's — threw the whole index away for nothing. + /// /// The confidence level a data source asks of a provider is deliberately not among them. It /// changes no vector, and it is enforced live on every request anyway: DataSourceService checks /// it against the participating chat providers and against the embedding provider, and this @@ -1010,14 +1034,22 @@ public sealed partial class DataSourceEmbeddingService embeddingProvider.Model.Id, embeddingProvider.Host, embeddingProvider.Hostname, - embeddingProvider.TokenizerPath, + embeddingProvider.HFInferenceProvider, + embeddingProvider.TokenizerFingerprint, embeddingProvider.EffectiveTokenLimit, - dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0, - dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH, chunkingOptions.MaxChunkTokenLength, chunkingOptions.OverlapTokenLength); } + /// + /// Describes how the vectors of a data source were made, working the chunking out along the way. + /// + /// The data source the vectors belong to. + /// The embedding provider which makes them. + /// The signature of this pairing. + internal static string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider) => + BuildEmbeddingSignature(dataSource, embeddingProvider, GetChunkingOptions(dataSource, embeddingProvider)); + private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList indexedFiles) { var fileHashes = indexedFiles diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 01ea7b7c..642ef9a1 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -199,16 +199,44 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM this.CanRefreshDataSource(dataSource); } - public async Task ShouldLockDataSourceIdentityAsync(string dataSourceId, CancellationToken token = default) + /// + /// Whether the file or folder a data source reads must stay as it is. + /// + /// + /// Locked as soon as the index holds anything, because where a data source reads from is what it + /// is: another folder is another data source, and the path reaches no signature, so swapping it + /// would leave the stored index describing documents nobody points at any more. + /// + /// The embedding provider used to be locked along with it and no longer is. It does reach the + /// signature, so changing it rebuilds the index cleanly -- and DataSourceReindexWarning asks + /// before it does. Locking it as well left a data source whose provider was deleted stuck on + /// keyword search for good, with no way back. + /// + /// Unclear counts as locked: an unavailable index database says nothing about what is stored. + /// + /// The data source to ask about. + /// The cancellation token. + /// True when the source must not be changed. + public async Task ShouldLockDataSourceOriginAsync(string dataSourceId, CancellationToken token = default) { var indexStore = await databaseClientProvider.GetIndexStoreAsync(token); if (!indexStore.IsAvailable) { - logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name); + logger.LogWarning("Locking the source of data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, indexStore.Name); return true; } var manifest = await indexStore.GetManifestAsync(dataSourceId, token); + return HasStoredIndexState(manifest); + } + + /// + /// Whether the index holds anything at all about a data source. + /// + /// What the index store returned for it. + /// True when there is stored index state. + private static bool HasStoredIndexState(DataSourceEmbeddingManifest manifest) + { return !string.IsNullOrWhiteSpace(manifest.EmbeddingProviderId) || !string.IsNullOrWhiteSpace(manifest.EmbeddingSignature) || !string.IsNullOrWhiteSpace(manifest.SourceHash) @@ -220,6 +248,59 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM || manifest.PermanentFailures.Count > 0; } + /// + /// Picks the data sources which already hold something in the index. + /// + /// + /// Asked before a setting is saved which would throw those indexes away, so the question can be + /// put to the user with the names in it. Anything unclear counts as holding something — the + /// opposite of IsAwaitingReindexAsync, and for the opposite reason: there, a wrongly greyed-out + /// row would stay wrong for good, while a question asked once too often costs a click, and one + /// skipped costs whatever a cloud provider charges for embedding everything again. + /// + /// The data sources to ask about. + /// The cancellation token. + /// Those of them which have stored index state. + public async Task> GetDataSourcesWithStoredIndexAsync(IReadOnlyCollection dataSources, CancellationToken token = default) + { + // + // Filtering first also keeps the index database from being created while local RAG is off: + // asking for the store runs its migrations on the first call, which must not happen because + // somebody opened a dialog. + // + var candidates = dataSources.Where(this.IsSupportedInternalDataSource).ToList(); + if (candidates.Count == 0) + return []; + + try + { + using var timeout = CancellationTokenSource.CreateLinkedTokenSource(token); + timeout.CancelAfter(REINDEX_CHECK_TIMEOUT); + + var indexStore = await databaseClientProvider.GetIndexStoreAsync(timeout.Token); + if (!indexStore.IsAvailable) + { + logger.LogWarning("Could not tell which data sources hold a stored index because the local RAG index database '{DatabaseName}' is unavailable. Treating all {DataSourceCount} of them as affected.", indexStore.Name, candidates.Count); + return candidates; + } + + var affected = new List(candidates.Count); + foreach (var dataSource in candidates) + { + var manifest = await indexStore.GetManifestAsync(dataSource.Id, timeout.Token); + if (HasStoredIndexState(manifest)) + affected.Add(dataSource); + } + + return affected; + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not tell which of {DataSourceCount} data source(s) hold a stored index. Treating all of them as affected.", candidates.Count); + return candidates; + } + } + /// /// Whether a data source cannot answer a search right now because its index has to be built anew. /// @@ -269,7 +350,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM return false; var indexState = await indexStore.GetDataSourceStateAsync(dataSource.Id, timeout.Token); - var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider); + var chunkingOptions = GetChunkingOptions(dataSource, embeddingProvider); var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); var runState = this.statuses.TryGetValue(dataSource.Id, out var status) ? status.State : (DataSourceEmbeddingState?)null; @@ -1444,7 +1525,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM IndexStoreClient indexStore, CancellationToken token) { - var chunkingOptions = this.GetChunkingOptions(dataSource, embeddingProvider); + var chunkingOptions = GetChunkingOptions(dataSource, embeddingProvider); var embeddingSignature = BuildEmbeddingSignature(dataSource, embeddingProvider, chunkingOptions); var manifest = await indexStore.GetManifestAsync(dataSource.Id, token); diff --git a/app/MindWork AI Studio/Tools/Services/EmbeddingChangeImpact.cs b/app/MindWork AI Studio/Tools/Services/EmbeddingChangeImpact.cs new file mode 100644 index 00000000..4e2b925f --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/EmbeddingChangeImpact.cs @@ -0,0 +1,47 @@ +using AIStudio.Settings; + +namespace AIStudio.Tools.Services; + +/// +/// Answers whether an edit throws the stored index of a data source away. +/// +/// +/// Nothing here knows which settings matter. Both questions are answered by building the embedding +/// signature twice and comparing the two, so the single place which decides stays +/// BuildEmbeddingSignature and this cannot drift away from what an indexing run then does. +/// +internal static class EmbeddingChangeImpact +{ + /// + /// Whether an edited embedding provider invalidates what is stored for one of its data sources. + /// + /// The data source, which the edit leaves alone. + /// The embedding provider as it is stored. + /// The embedding provider as it would be stored. + /// True when the stored index would be discarded. + public static bool AffectsStoredIndex(IDataSource dataSource, EmbeddingProvider before, EmbeddingProvider after) => + !string.Equals( + DataSourceEmbeddingService.BuildEmbeddingSignature(dataSource, before), + DataSourceEmbeddingService.BuildEmbeddingSignature(dataSource, after), + StringComparison.Ordinal); + + /// + /// Whether an edited data source invalidates what is stored for it. + /// + /// + /// Each side is asked with the embedding provider it points at, never both with the same one. A + /// data source carries only the id of its provider, while the signature carries what that provider + /// is, so comparing both sides against one of them would report a changed embedding as no change + /// at all -- and the next indexing run would then rebuild everything unannounced. + /// + /// The data source as it is stored. + /// The embedding provider it points at today. + /// The data source as it would be stored. + /// The embedding provider it would point at. + /// True when the stored index would be discarded. + public static bool AffectsStoredIndex(IDataSource before, EmbeddingProvider beforeProvider, IDataSource after, EmbeddingProvider afterProvider) => + !string.Equals( + DataSourceEmbeddingService.BuildEmbeddingSignature(before, beforeProvider), + DataSourceEmbeddingService.BuildEmbeddingSignature(after, afterProvider), + StringComparison.Ordinal); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs b/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs new file mode 100644 index 00000000..6bbdc1e4 --- /dev/null +++ b/app/MindWork AI Studio/Tools/TokenizerFingerprint.cs @@ -0,0 +1,46 @@ +using System.Security.Cryptography; + +namespace AIStudio.Tools; + +/// +/// Identifies a tokenizer by what is inside its file, not by where the file lies. +/// +/// +/// The embedding signature asks this to decide whether stored vectors still belong to the current +/// configuration, and the path cannot answer it. A tokenizer is stored below the data directory under +/// the model it belongs to, keeping the name it came with -- and the usual name for one is +/// tokenizer.json. Picking a different tokenizer with that name lands on the identical path, so the +/// index would be kept although the chunk boundaries moved. The other way round, moving the data +/// directory changes every path without changing a single tokenizer. +/// +public static class TokenizerFingerprint +{ + /// + /// Reads a tokenizer file and returns a fingerprint of its content. + /// + /// The tokenizer file to read. May be empty when no tokenizer is set. + /// The cancellation token. + /// The fingerprint, or an empty string when there is no readable file. + public static async Task ForFileAsync(string tokenizerPath, CancellationToken token = default) + { + if (string.IsNullOrWhiteSpace(tokenizerPath)) + return string.Empty; + + try + { + await using var stream = File.OpenRead(tokenizerPath); + return Convert.ToHexString(await SHA256.HashDataAsync(stream, token)); + } + catch + { + // + // An unreadable tokenizer is not this method's problem to report: the dialog validates the + // file before it ever gets here, and an indexing run says so again when it cannot tokenize + // anything. Whoever stores a provider has to decide what an empty answer means for them, + // because writing it into the settings would look like another tokenizer and throw the + // stored vectors away. + // + return string.Empty; + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md index ad8e2f9b..4cef055b 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md @@ -20,10 +20,10 @@ - Added a way to open the sources of your own documents: click a source below an answer, and the document opens in the program your system uses for it. - Added a jump to the right page for the sources of your own documents (RAG), so a PDF opens directly where the passage was found, wherever your system and its program support it. - Added a way to show a source of your own documents (RAG) in your file manager. -- Added the page of a passage to what the AI is told when it answers from your own documents (RAG), so it can name the page an answer rests on. - Added the details of the two databases behind local RAG to the information page: which versions they run, how much space they use on your disk, and how much they hold. - Added a repair for your local data sources. Should the index of a data source ever become unreadable, AI Studio now says so instead of quietly finding nothing and leaves that source out of your chats until it works again. - Added the repair itself as a button next to each of your data sources. Rebuilding an index sends your documents to your embedding provider once more, so AI Studio asks you first and never starts it on its own. +- Added a check that recognizes your own tokenizer by its content rather than by its file name. Tokenizers are almost always named alike, so swapping one for another is now noticed, and you are asked about it. - Added support for several drop areas on the same page. More complex assistants can now receive files or folders by drag and drop at more than one place. - Added drag and drop to the input and output folder of the Batch Processing assistant: drop a folder onto either field to choose it. - Added ways to load text from a file and drop zones for them, throughout the assistants and dialogs. We went through them one by one, so many fields that used to accept typed text only now take the content of a file as well. @@ -34,6 +34,10 @@ - Improved the list of your attached files: every file now appears under the folder it came from, and each folder is named only once, no matter in which order you attached your files. - Improved organization-wide provider management: IT departments can now separately prevent users from adding chat, transcription, or embedding providers. The existing master setting still overrides all three provider-specific settings. - Improved the provider selection throughout the assistants: when there is nothing to choose from, it now says why. Either you have not set up a provider yet, or none of yours is trusted enough for what you are doing. Before, the list was simply empty. +- Improved the settings of your embedding providers and data sources. Some of them decide how your documents are read, so changing one means preparing every document all over again. AI Studio now asks before that happens, names the data sources it would affect, and says when a cloud provider charges you for it. +- Improved the dialogs of your embedding providers and data sources: when a change would mean preparing all your documents again, and you decide against it, nothing is saved and the dialog stays open with your change in front of you, ready to be corrected. +- Improved the question AI Studio asks before you delete an embedding provider. It now names the data sources depending on that provider, together with what they can still do without it. +- Improved what the AI is told when it answers from your own documents (RAG): it now learns which page a passage came from, so it can name the page an answer rests on. - Changed how provider trust and provider confidence work together. Marking a provider as trustworthy in a configuration no longer also satisfies a required confidence level: one says who runs the provider, the other how confidential it is. Organizations raise a provider's level in their own confidence scheme instead. This applies beyond local data sources, for example, when a model reads a page from your intranet. - Fixed the abilities AI Studio assumed for many models. We checked the families against their documentation: some models gained image input, reasoning, or tool calling, others lost an ability they never had. - Fixed the Document Analysis assistant freezing while you edited a policy. It needed a change of yours to be saved in the background just as you were making the next one — picking a provider, for instance — which is why it hit some of you again and again and others never at all. diff --git a/app/Tests/Tools/EmbeddingChangeImpactTests.cs b/app/Tests/Tools/EmbeddingChangeImpactTests.cs new file mode 100644 index 00000000..dcea6626 --- /dev/null +++ b/app/Tests/Tools/EmbeddingChangeImpactTests.cs @@ -0,0 +1,247 @@ +using AIStudio.Provider; +using AIStudio.Provider.HuggingFace; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools.Services; + +using Host = AIStudio.Provider.SelfHosted.Host; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks which edits have to be asked about before they are saved. +/// +/// +/// An edit which changes the embedding signature throws away everything indexed for the data sources +/// behind it, and sends every one of their documents to the embedding provider again. Asking about an +/// edit which costs nothing trains people to click the question away; not asking about one which does +/// costs them money at a cloud provider. So both directions are pinned down here. +/// +[TestFixture] +public sealed class EmbeddingChangeImpactTests +{ + [Test] + public void HarmlessEmbeddingProviderEditsKeepTheStoredIndex() + { + var dataSource = StoredDataSource(); + var stored = StoredEmbeddingProvider(); + + Assert.Multiple(() => + { + Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Name = "Another name" }), Is.False, "The name of an embedding provider reaches no vector."); + Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Num = 42 }), Is.False, "The number is there to sort the list with."); + Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { EmbeddingBatchSize = 16 }), Is.False, "How many chunks travel in one request says nothing about the vectors which come back."); + Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { CustomIconDataUrl = "data:image/png;base64,AAAA" }), Is.False, "An icon is an icon."); + }); + } + + [Test] + public void ChangingTheModelDropsTheStoredIndex() + { + var dataSource = StoredDataSource(); + var stored = StoredEmbeddingProvider(); + + Assert.That( + EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Model = new("text-embedding-3-large", "text-embedding-3-large") }), + Is.True, + "Another model means another vector space."); + } + + [Test] + public void ChangingTheTokenLimitDropsTheStoredIndex() + { + var dataSource = StoredDataSource(); + var stored = StoredEmbeddingProvider(); + + Assert.That( + EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { TokenLimit = 4096 }), + Is.True, + "The token limit decides where the text is cut, and other chunks are other vectors."); + } + + [Test] + public void ChangingWhereTheProviderRunsDropsTheStoredIndex() + { + var dataSource = StoredDataSource(); + var stored = StoredEmbeddingProvider(); + + Assert.Multiple(() => + { + Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Hostname = "http://localhost:9999" }), Is.True, "Another server can serve another model under the same name."); + Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Host = Host.LM_STUDIO }), Is.True, "Another kind of host speaks another API."); + Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { HFInferenceProvider = HFInferenceProvider.GROQ }), Is.True, "The same model name served by another backend is another vector source."); + }); + } + + [Test] + public void TheTokenizerIsComparedByItsContentNotItsPath() + { + var dataSource = StoredDataSource(); + var stored = StoredEmbeddingProvider() with { TokenizerPath = "/data/tokenizers/embeddings/tokenizer.json", TokenizerFingerprint = "AAAA" }; + + Assert.Multiple(() => + { + Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { TokenizerFingerprint = "BBBB" }), Is.True, "Another tokenizer counts tokens differently, so the text is cut elsewhere."); + Assert.That(EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { TokenizerPath = "/somewhere/else/tokenizer.json" }), Is.False, "It is the same tokenizer under another path."); + }); + } + + [Test] + public void ChangingTheChunkSettingsOfADataSourceDropsItsStoredIndex() + { + var embeddingProvider = StoredEmbeddingProvider(); + var stored = StoredDataSource(); + + Assert.Multiple(() => + { + Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { MaxChunkTokenLength = 256 }), Is.True, "Other chunk boundaries mean other vectors."); + Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { ChunkOverlapTokenLength = 50 }), Is.True, "Another overlap changes what every chunk starts with."); + }); + } + + /// + /// Writing out what a data source already follows must not cost it its index. + /// + /// + /// A token limit of 0 means "follow the embedding provider", and opening the expert settings of a + /// data source fills that empty field with exactly the provider's limit. Both are the same cut, + /// so nobody may be asked about it -- and above all, nothing may be re-embedded for it. Somebody + /// who only wanted to change how many matches an answer may use paid for a full rebuild. + /// + [Test] + public void SpellingOutWhatTheProviderAlreadyDictatesKeepsTheStoredIndex() + { + var embeddingProvider = StoredEmbeddingProvider() with { TokenLimit = 8192 }; + var followingTheProvider = StoredDataSource() with { MaxChunkTokenLength = 0 }; + + Assert.Multiple(() => + { + Assert.That( + EditKeepingTheProvider(embeddingProvider, followingTheProvider, followingTheProvider with { MaxChunkTokenLength = 8192 }), + Is.False, + "The provider limit typed into the field is the cut the data source already had."); + + Assert.That( + EditKeepingTheProvider(embeddingProvider, followingTheProvider, followingTheProvider with { MaxChunkTokenLength = 4096 }), + Is.True, + "Anything below the provider limit really does cut the text elsewhere."); + }); + } + + /// + /// An overlap larger than the chunk is capped, so several of them are the same cut. + /// + /// + /// The same reasoning as for the token limit: what counts is where the text is cut, not what + /// somebody typed into the field. + /// + [Test] + public void AnOverlapWhichIsCappedAnywayKeepsTheStoredIndex() + { + var embeddingProvider = StoredEmbeddingProvider(); + var stored = StoredDataSource() with { MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 600 }; + + Assert.That( + EditKeepingTheProvider(embeddingProvider, stored, stored with { ChunkOverlapTokenLength = 700 }), + Is.False, + "Both overlaps are capped to the chunk size, so the text is cut identically."); + } + + [Test] + public void HarmlessDataSourceEditsKeepTheStoredIndex() + { + var embeddingProvider = StoredEmbeddingProvider(); + var stored = StoredDataSource(); + + Assert.Multiple(() => + { + Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { Name = "Another name" }), Is.False, "The name is how the data source is offered, not how it was read."); + Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { Description = "Another description" }), Is.False, "The description is there for the agent which picks data sources."); + Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { MaxMatches = 42 }), Is.False, "How many matches an answer may use is decided per query."); + Assert.That(EditKeepingTheProvider(embeddingProvider, stored, stored with { ConfidenceLevel = ConfidenceLevel.HIGH }), Is.False, "The confidence level is enforced live on every request and changes no vector."); + }); + } + + /// + /// Checks what changing the embedding of a data source costs. + /// + /// + /// This is why each side has to be asked with its own provider: a data source carries only the id + /// of its embedding provider, and that id is nowhere in the signature. Asking both sides with the + /// same provider would call this edit harmless, while the next indexing run throws everything away. + /// + [Test] + public void ChangingTheEmbeddingOfADataSourceDropsItsStoredIndex() + { + var storedProvider = StoredEmbeddingProvider(); + var anotherProvider = AnotherEmbeddingProvider(); + var stored = StoredDataSource(); + var moved = stored with { EmbeddingId = anotherProvider.Id }; + + Assert.That( + EmbeddingChangeImpact.AffectsStoredIndex(stored, storedProvider, moved, anotherProvider), + Is.True, + "Another embedding provider means another vector space, so nothing stored survives it."); + } + + /// + /// Putting a data source back to work after its provider was deleted is a rebuild as well. + /// + /// + /// The provider a data source points at can be gone. What is stored was made by it, so pointing + /// the source at any provider at all discards that -- and nobody may be surprised by it. + /// + [Test] + public void RepointingADataSourceWhoseProviderIsGoneDropsItsStoredIndex() + { + var stored = StoredDataSource(); + var anotherProvider = AnotherEmbeddingProvider(); + + Assert.That( + EmbeddingChangeImpact.AffectsStoredIndex(stored, EmbeddingProvider.NONE, stored with { EmbeddingId = anotherProvider.Id }, anotherProvider), + Is.True, + "A provider which cannot be resolved stands in as NONE, which is a signature of its own."); + } + + [Test] + public void KeepingTheEmbeddingKeepsTheStoredIndex() + { + var storedProvider = StoredEmbeddingProvider(); + var stored = StoredDataSource(); + + Assert.That( + EmbeddingChangeImpact.AffectsStoredIndex(stored, storedProvider, stored with { Name = "Another name" }, storedProvider with { Name = "Renamed provider" }), + Is.False, + "Neither name reaches a vector, and the data source still points at the same provider."); + } + + /// + /// Asks the question for an edit which leaves the embedding provider of the data source alone. + /// + /// The provider both sides point at. + /// The data source as it is stored. + /// The data source as it would be stored. + /// True when the stored index would be discarded. + private static bool EditKeepingTheProvider(EmbeddingProvider embeddingProvider, IDataSource before, IDataSource after) => + EmbeddingChangeImpact.AffectsStoredIndex(before, embeddingProvider, after, embeddingProvider); + + private static DataSourceLocalDirectory StoredDataSource() => new() + { + Num = 1, + Id = "6f1d6a4e-6a5e-4c62-9a4f-0f2d2c8b7a11", + Name = "Test data", + Description = "Documents used by the tests.", + Type = DataSourceType.LOCAL_DIRECTORY, + EmbeddingId = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", + MaxChunkTokenLength = 512, + ChunkOverlapTokenLength = 100, + ConfidenceLevel = ConfidenceLevel.LOW, + Path = "/tmp/test-data", + }; + + private static EmbeddingProvider StoredEmbeddingProvider() => + new(1, "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small")); + + private static EmbeddingProvider AnotherEmbeddingProvider() => + new(2, "c1b5d5e3-2a4f-4b1b-9dae-6b7c8d9e0f12", "Other embeddings", LLMProviders.MISTRAL, new("mistral-embed", "mistral-embed")); +} \ No newline at end of file diff --git a/app/Tests/Tools/EmbeddingSignatureTests.cs b/app/Tests/Tools/EmbeddingSignatureTests.cs index 4a0c6eac..94513cfa 100644 --- a/app/Tests/Tools/EmbeddingSignatureTests.cs +++ b/app/Tests/Tools/EmbeddingSignatureTests.cs @@ -1,4 +1,5 @@ using AIStudio.Provider; +using AIStudio.Provider.HuggingFace; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.Services; @@ -47,11 +48,68 @@ public sealed class EmbeddingSignatureTests "Another model means another vector space, so nothing stored may be kept."); } + [Test] + public void ChangingTheTokenizerContentDropsTheStoredEmbeddings() + { + var dataSource = DataSource(ConfidenceLevel.LOW); + var oneTokenizer = TokenizerAt("/data/tokenizers/embeddings/tokenizer.json", "AAAA"); + var anotherTokenizer = oneTokenizer with { TokenizerFingerprint = "BBBB" }; + + Assert.That( + Signature(dataSource, anotherTokenizer), + Is.Not.EqualTo(Signature(dataSource, oneTokenizer)), + "Another tokenizer cuts the text at other places. A tokenizer is stored under the name it came with, almost always tokenizer.json, so the path alone would not notice the swap."); + } + + [Test] + public void MovingTheTokenizerFileKeepsTheStoredEmbeddings() + { + var dataSource = DataSource(ConfidenceLevel.LOW); + var here = TokenizerAt("/data/tokenizers/embeddings/tokenizer.json", "AAAA"); + var there = here with { TokenizerPath = "/somewhere/else/tokenizers/embeddings/tokenizer.json" }; + + Assert.That( + Signature(dataSource, there), + Is.EqualTo(Signature(dataSource, here)), + "It is the same tokenizer and only the data directory moved, so embedding everything again would buy nothing."); + } + + [Test] + public void ChangingTheHuggingFaceInferenceProviderDropsTheStoredEmbeddings() + { + var dataSource = DataSource(ConfidenceLevel.LOW); + var oneBackend = EmbeddingProviderFor("text-embedding-3-small") with { HFInferenceProvider = HFInferenceProvider.GROQ }; + var anotherBackend = oneBackend with { HFInferenceProvider = HFInferenceProvider.CEREBRAS }; + + Assert.That( + Signature(dataSource, anotherBackend), + Is.Not.EqualTo(Signature(dataSource, oneBackend)), + "The same model name served by another backend is another vector source."); + } + + [Test] + public void TheSignatureOfAKnownConfigurationIsPinned() + { + Assert.That( + Signature(DataSource(ConfidenceLevel.LOW)), + Is.EqualTo("2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100"), + "Reordering or extending the signature throws away every index anybody has. This test makes that a decision somebody takes rather than something which happens on the way past."); + } + + /// + /// Builds the signature the way an indexing run does, working the chunking out along the way. + /// + /// + /// Handing in fixed chunking options instead would hide exactly what these tests are here for: + /// the signature would then no longer notice a data source being cut differently. + /// + /// The data source to build the signature for. + /// The embedding provider, or the test default. + /// The signature of that pairing. private static string Signature(DataSourceLocalDirectory dataSource, EmbeddingProvider? embeddingProvider = null) => DataSourceEmbeddingService.BuildEmbeddingSignature( dataSource, - embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small"), - new(512, 100)); + embeddingProvider ?? EmbeddingProviderFor("text-embedding-3-small")); private static DataSourceLocalDirectory DataSource(ConfidenceLevel confidenceLevel) => new() { @@ -67,6 +125,9 @@ public sealed class EmbeddingSignatureTests Path = "/tmp/test-data", }; + private static EmbeddingProvider TokenizerAt(string tokenizerPath, string tokenizerFingerprint) => + EmbeddingProviderFor("text-embedding-3-small") with { TokenizerPath = tokenizerPath, TokenizerFingerprint = tokenizerFingerprint }; + private static EmbeddingProvider EmbeddingProviderFor(string modelId) => new(1, "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "Test embeddings", LLMProviders.OPEN_AI, new(modelId, modelId)); } \ No newline at end of file diff --git a/metadata.txt b/metadata.txt index f39e0e50..d39310fb 100644 --- a/metadata.txt +++ b/metadata.txt @@ -9,4 +9,4 @@ 3c18a7bfdb3, release osx-arm64 148.0.7763.0 -0.8.0 +0.8.0 \ No newline at end of file From f9c6075c500a1622155146f8fff6a86b30d229a4 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Sat, 19 Sep 2026 17:54:10 +0200 Subject: [PATCH 2/2] Sorted every model list by what a model is made for (#984) --- .../Assistants/I18N/allTexts.lua | 18 +- .../Dialogs/EmbeddingProviderDialog.razor | 66 +++--- .../Dialogs/EmbeddingProviderDialog.razor.cs | 84 ++++--- .../Dialogs/TranscriptionProviderDialog.razor | 55 ++--- .../TranscriptionProviderDialog.razor.cs | 42 +--- .../Alibaba/ModelStudioEmbeddingFamily.cs | 9 +- .../Models/Google/AqaFamily.cs | 32 +++ .../Models/Google/GoogleAgentFamily.cs | 48 ++++ .../Models/Google/LyriaFamily.cs | 35 +++ .../Models/Kinds/EmbeddingModelsFamily.cs | 21 +- .../Kinds/ImageGenerationModelsFamily.cs | 6 +- .../Models/Kinds/RealtimeModelsFamily.cs | 15 +- .../Kinds/TextCompletionModelsFamily.cs | 10 +- .../Models/Kinds/TranscriptionModelsFamily.cs | 22 +- .../Models/Mistral/CodestralFamily.cs | 23 +- .../plugin.lua | 18 +- .../plugin.lua | 18 +- .../Plugins/models/plugin.lua | 9 +- .../AlibabaCloud/ProviderAlibabaCloud.cs | 66 ++---- .../Provider/Anthropic/ProviderAnthropic.cs | 9 +- .../Provider/Fireworks/ProviderFireworks.cs | 10 + .../Provider/GWDG/ProviderGWDG.cs | 32 ++- .../Provider/Google/ProviderGoogle.cs | 12 +- .../Provider/LLMProvidersExtensions.cs | 11 - .../Provider/Mistral/ProviderMistral.cs | 30 +-- app/MindWork AI Studio/Provider/ModelKind.cs | 31 +++ .../Provider/OpenRouter/ProviderOpenRouter.cs | 7 +- .../Provider/SelfHosted/ProviderSelfHosted.cs | 80 +++++-- .../Provider/X/ProviderX.cs | 25 +- .../wwwroot/changelog/v26.9.1.md | 7 +- .../Models/Corpus/CapabilitySnapshot.txt | 7 +- app/Tests/Models/Corpus/CorpusOrigin.cs | 4 +- app/Tests/Models/Corpus/ModelCorpus.cs | 33 +-- app/Tests/Models/Corpus/ModelKindCorpus.cs | 215 +++++++++++++++++- .../Provider/SelfHostedModelListTests.cs | 135 +++++++++++ app/Tests/Tools/EmbeddingChangeImpactTests.cs | 35 +++ 36 files changed, 947 insertions(+), 333 deletions(-) create mode 100644 app/MindWork AI Studio/Models/Google/AqaFamily.cs create mode 100644 app/MindWork AI Studio/Models/Google/GoogleAgentFamily.cs create mode 100644 app/MindWork AI Studio/Models/Google/LyriaFamily.cs create mode 100644 app/Tests/Provider/SelfHostedModelListTests.cs diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index b49906f9..234fee68 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -6547,9 +6547,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1356621346"] = "Cr -- Failed to validate the selected tokenizer. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1384494471"] = "Failed to validate the selected tokenizer. Please try again." --- Please enter an embedding model name. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1661085403"] = "Please enter an embedding model name." - -- Hostname UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1727440780"] = "Hostname" @@ -6604,9 +6601,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2810182573"] = "No -- Instance Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "Instance Name" --- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually." - -- Token limit UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "Token limit" @@ -6616,6 +6610,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Pl -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3361153305"] = "Show Expert Settings" +-- This server does not offer the selected model right now. It stays selected, so the documents you already prepared keep working. Choosing another model means every document of the data sources behind this provider is prepared again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3571276758"] = "This server does not offer the selected model right now. It stays selected, so the documents you already prepared keep working. Choosing another model means every document of the data sources behind this provider is prepared again." + -- How many chunks are sent to the embedding provider at once. The default is 1. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3780233303"] = "How many chunks are sent to the embedding provider at once. The default is 1." @@ -6637,6 +6634,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T808120719"] = "Hos -- Please enter an embedding batch size greater than 0. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T840259907"] = "Please enter an embedding batch size greater than 0." +-- Your server answered, but none of the models it serves is one we know to create embeddings. Either there is none installed, or it runs under a name we do not recognize. In the latter case, your organization can describe the model in a model plugin, and it will show up here. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T859645108"] = "Your server answered, but none of the models it serves is one we know to create embeddings. Either there is none installed, or it runs under a name we do not recognize. In the latter case, your organization can describe the model in a model plugin, and it will show up here." + -- Provider UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T900237532"] = "Provider" @@ -8716,9 +8716,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1324664716"] = -- Create account UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1356621346"] = "Create account" --- Currently, we cannot query the transcription models for the selected provider and/or host. Therefore, please enter the model name manually. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1381635232"] = "Currently, we cannot query the transcription models for the selected provider and/or host. Therefore, please enter the model name manually." - -- Hostname UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1727440780"] = "Hostname" @@ -8752,9 +8749,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T2842060373"] = -- Hugging Face transcribes audio through a few of its inference providers only, which is why this list is shorter than the one for chatting. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T3397943774"] = "Hugging Face transcribes audio through a few of its inference providers only, which is why this list is shorter than the one for chatting." --- Please enter a transcription model name. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T3703662664"] = "Please enter a transcription model name." - -- This host uses the model configured at the provider level. No model selection is available. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T3783329915"] = "This host uses the model configured at the provider level. No model selection is available." diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor index 26ceeb33..566e0640 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor @@ -85,48 +85,42 @@ - @if (this.DataLLMProvider.IsEmbeddingModelProvidedManually(this.DataHost)) + + @T("Load") + + @if (this.availableModels.Count is 0) { - + + @T("No models loaded or available.") + } else { - - @T("Load") - - @if (this.availableModels.Count is 0) - { - - @T("No models loaded or available.") - - } - else - { - - @foreach (var model in this.availableModels) - { - - @model - - } - - } + + @foreach (var model in this.availableModels) + { + + @model + + } + } + @if (this.dataConfiguredModelIsNotOffered) + { + + @T("This server does not offer the selected model right now. It stays selected, so the documents you already prepared keep working. Choosing another model means every document of the data sources behind this provider is prepared again.") + + } + @if (this.ServerNamedNoEmbeddingModel) + { + + @T("Your server answered, but none of the models it serves is one we know to create embeddings. Either there is none installed, or it runs under a name we do not recognize. In the latter case, your organization can describe the model in a model plugin, and it will show up here.") + + } @if (!string.IsNullOrWhiteSpace(this.dataLoadingModelsIssue)) { diff --git a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs index 03479185..765e7aa5 100644 --- a/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/EmbeddingProviderDialog.razor.cs @@ -133,10 +133,11 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId private string[] dataIssues = []; private string dataAPIKey = string.Empty; private bool dataHadStoredAPIKeyOnLoad; - private string dataManuallyModel = string.Empty; private string dataAPIKeyStorageIssue = string.Empty; private string dataEditingPreviousInstanceName = string.Empty; private string dataLoadingModelsIssue = string.Empty; + private bool dataConfiguredModelIsNotOffered; + private bool dataServerWasAskedForItsModels; private string dataFilePath = string.Empty; private string dataTokenizerFingerprint = string.Empty; private string dataCustomTokenizerValidationIssue = string.Empty; @@ -162,7 +163,6 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId GetPreviousInstanceName = () => this.dataEditingPreviousInstanceName, GetUsedInstanceNames = () => this.UsedInstanceNames, GetHost = () => this.DataHost, - IsModelProvidedManually = () => this.DataLLMProvider.IsEmbeddingModelProvidedManually(this.DataHost), GetCustomTokenizerValidationIssue = () => this.dataCustomTokenizerValidationIssue, }; } @@ -170,24 +170,13 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId private EmbeddingProvider CreateEmbeddingProviderSettings() { var cleanedHostname = this.DataHostname.Trim(); - Model model = default; - if(this.DataLLMProvider is LLMProviders.SELF_HOSTED) - { - if (this.DataLLMProvider.IsEmbeddingModelProvidedManually(this.DataHost)) - model = new Model(this.dataManuallyModel, null); - else if (this.DataHost is Host.LM_STUDIO) - model = this.DataModel; - } - else - model = this.DataModel; - return new() { Num = this.DataNum, Id = this.DataId, Name = this.DataName, UsedLLMProvider = this.DataLLMProvider, - Model = model, + Model = this.DataModel, IsSelfHosted = this.DataLLMProvider is LLMProviders.SELF_HOSTED, Hostname = cleanedHostname.EndsWith('/') ? cleanedHostname[..^1] : cleanedHostname, Host = this.DataHost, @@ -225,10 +214,6 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId || this.DataTokenLimit != EmbeddingProvider.DEFAULT_TOKEN_LIMIT || this.DataEmbeddingBatchSize != EmbeddingProvider.DEFAULT_EMBEDDING_BATCH_SIZE; - // When using self-hosted embedding, we must copy the model name: - if (this.DataLLMProvider is LLMProviders.SELF_HOSTED) - this.dataManuallyModel = this.DataModel.Id; - // Load the API key. A self-hosted server may well need one: LM Studio can ask for a // token of its own, and any of these servers can sit behind an authenticating proxy. // So we try for every host and treat a missing key as the normal case (isTrying). @@ -358,14 +343,6 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId this.MudDialog.Close(DialogResult.Ok(addedProviderSettings)); } - private string? ValidateManuallyModel(string manuallyModel) - { - if (this.DataLLMProvider is LLMProviders.SELF_HOSTED && string.IsNullOrWhiteSpace(manuallyModel)) - return T("Please enter an embedding model name."); - - return null; - } - private string? ValidateTokenLimit(int tokenLimit) { if (tokenLimit < 1) @@ -516,9 +493,9 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId // When the host changes, reset the model selection state: this.DataHost = selectedHost; this.DataModel = default; - this.dataManuallyModel = string.Empty; this.availableModels.Clear(); this.dataLoadingModelsIssue = string.Empty; + this.dataConfiguredModelIsNotOffered = false; } /// @@ -535,11 +512,13 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId this.DataModel = default; this.availableModels.Clear(); this.dataLoadingModelsIssue = string.Empty; + this.dataConfiguredModelIsNotOffered = false; } private async Task ReloadModels() { this.dataLoadingModelsIssue = string.Empty; + this.dataServerWasAskedForItsModels = true; var currentEmbeddingProviderSettings = this.CreateEmbeddingProviderSettings(); var provider = currentEmbeddingProviderSettings.CreateProvider(); if (provider is NoProvider) @@ -562,8 +541,57 @@ public partial class EmbeddingProviderDialog : MSGComponentBase, ISecretId this.Logger.LogError($"Failed to load models from provider '{this.DataLLMProvider}' (host={this.DataHost}, hostname='{this.DataHostname}'): {e.Message}"); this.dataLoadingModelsIssue = T("We are currently unable to communicate with the provider to load models. Please try again later."); } + + // Whatever the server answered, and whether it answered at all, the model this provider was + // configured with stays on the list: + this.PinConfiguredModel(); } - + + /// + /// Keeps the configured model selectable, also when the server does not offer it right now. + /// + /// + /// This is deliberately the opposite of what the chat provider dialog does, which replaces the + /// configured model with the one the server reported. An embedding provider carries indexed data + /// sources, and its model ID is part of the embedding signature: changing it -- even only in its + /// spelling -- means every prepared document is prepared again. So the stored model is added to + /// the list here rather than the list being applied to the stored model. A model nobody serves + /// any more stays visible and stays chosen, and changing it stays the user's decision, which + /// storing then asks about. + /// + /// Comparing is what Model does, which is by ID and ordinal. Matching a differing spelling would + /// mean writing that other spelling into the settings, and that is the very change this avoids. + /// + private void PinConfiguredModel() + { + if (string.IsNullOrWhiteSpace(this.DataModel.Id)) + { + this.dataConfiguredModelIsNotOffered = false; + return; + } + + this.dataConfiguredModelIsNotOffered = !this.availableModels.Contains(this.DataModel); + if (this.dataConfiguredModelIsNotOffered) + this.availableModels.Insert(0, this.DataModel); + } + + /// + /// Whether the server answered without naming a single embedding model. + /// + /// + /// Two situations end up here, and the user is the only one who can tell them apart: a server + /// running no embedding model at all, and one running an embedding model under a name no rule + /// covers. Saying so beats the bare "No models loaded or available.", which reads like a + /// failure and leaves nobody anywhere to go -- the field for typing a name is gone, on purpose. + /// Describing such a model in a model plugin is the way out, and naming it here is what turns + /// a dead end into one. + /// + private bool ServerNamedNoEmbeddingModel => + this.DataLLMProvider is LLMProviders.SELF_HOSTED && + this.dataServerWasAskedForItsModels && + string.IsNullOrWhiteSpace(this.dataLoadingModelsIssue) && + this.availableModels.Count is 0; + private string APIKeyText => this.DataLLMProvider switch { LLMProviders.SELF_HOSTED => T("(Optional) API Key"), diff --git a/app/MindWork AI Studio/Dialogs/TranscriptionProviderDialog.razor b/app/MindWork AI Studio/Dialogs/TranscriptionProviderDialog.razor index 826a2959..003129b2 100644 --- a/app/MindWork AI Studio/Dialogs/TranscriptionProviderDialog.razor +++ b/app/MindWork AI Studio/Dialogs/TranscriptionProviderDialog.razor @@ -87,47 +87,28 @@ { - @if (this.DataLLMProvider.IsTranscriptionModelProvidedManually(this.DataHost)) + + @T("Load") + + @if(this.availableModels.Count is 0) { - + + @T("No models loaded or available.") + } else { - - @T("Load") - - @if(this.availableModels.Count is 0) - { - - @T("No models loaded or available.") - - } - else - { - - @foreach (var model in this.availableModels) - { - - @model - - } - - } + + @foreach (var model in this.availableModels) + { + + @model + + } + } @if (!string.IsNullOrWhiteSpace(this.dataLoadingModelsIssue)) diff --git a/app/MindWork AI Studio/Dialogs/TranscriptionProviderDialog.razor.cs b/app/MindWork AI Studio/Dialogs/TranscriptionProviderDialog.razor.cs index 1f3bc0de..dd463a06 100644 --- a/app/MindWork AI Studio/Dialogs/TranscriptionProviderDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/TranscriptionProviderDialog.razor.cs @@ -106,7 +106,6 @@ public partial class TranscriptionProviderDialog : MSGComponentBase, ISecretId private string[] dataIssues = []; private string dataAPIKey = string.Empty; private bool dataHadStoredAPIKeyOnLoad; - private string dataManuallyModel = string.Empty; private string dataAPIKeyStorageIssue = string.Empty; private string dataEditingPreviousInstanceName = string.Empty; private string dataLoadingModelsIssue = string.Empty; @@ -127,7 +126,6 @@ public partial class TranscriptionProviderDialog : MSGComponentBase, ISecretId GetPreviousInstanceName = () => this.dataEditingPreviousInstanceName, GetUsedInstanceNames = () => this.UsedInstanceNames, GetHost = () => this.DataHost, - IsModelProvidedManually = () => this.DataLLMProvider.IsTranscriptionModelProvidedManually(this.DataHost), }; } @@ -135,30 +133,9 @@ public partial class TranscriptionProviderDialog : MSGComponentBase, ISecretId { var cleanedHostname = this.DataHostname.Trim(); - // Determine the model based on the provider and host configuration: - Model model; - if (this.DataLLMProvider.IsTranscriptionModelSelectionHidden(this.DataHost)) - { - // Use system model placeholder for hosts that don't support model selection (e.g., whisper.cpp): - model = Model.SYSTEM_MODEL; - } - else if (this.DataLLMProvider is LLMProviders.SELF_HOSTED) - { - switch (this.DataHost) - { - case Host.OLLAMA: - model = new Model(this.dataManuallyModel, null); - break; - - case Host.VLLM: - case Host.LM_STUDIO: - default: - model = this.DataModel; - break; - } - } - else - model = this.DataModel; + // whisper.cpp serves whatever it was started with and names no models, so the placeholder + // stands in for the one model there is. Everywhere else the user picked one from the list: + var model = this.DataLLMProvider.IsTranscriptionModelSelectionHidden(this.DataHost) ? Model.SYSTEM_MODEL : this.DataModel; return new() { @@ -194,10 +171,6 @@ public partial class TranscriptionProviderDialog : MSGComponentBase, ISecretId if(this.IsEditing) { this.dataEditingPreviousInstanceName = this.DataName.ToLowerInvariant(); - - // When using self-hosted models, we must copy the model name: - if (this.DataLLMProvider is LLMProviders.SELF_HOSTED) - this.dataManuallyModel = this.DataModel.Id; // Load the API key. A self-hosted server may well need one: LM Studio can ask for a // token of its own, and any of these servers can sit behind an authenticating proxy. @@ -302,14 +275,6 @@ public partial class TranscriptionProviderDialog : MSGComponentBase, ISecretId this.MudDialog.Close(DialogResult.Ok(addedProviderSettings)); } - private string? ValidateManuallyModel(string manuallyModel) - { - if (this.DataLLMProvider is LLMProviders.SELF_HOSTED && string.IsNullOrWhiteSpace(manuallyModel)) - return T("Please enter a transcription model name."); - - return null; - } - private void Cancel() => this.MudDialog.Cancel(); private async Task OnAPIKeyChanged(string apiKey) @@ -327,7 +292,6 @@ public partial class TranscriptionProviderDialog : MSGComponentBase, ISecretId // When the host changes, reset the model selection state: this.DataHost = selectedHost; this.DataModel = default; - this.dataManuallyModel = string.Empty; this.availableModels.Clear(); this.dataLoadingModelsIssue = string.Empty; } diff --git a/app/MindWork AI Studio/Models/Alibaba/ModelStudioEmbeddingFamily.cs b/app/MindWork AI Studio/Models/Alibaba/ModelStudioEmbeddingFamily.cs index 392c2026..fd0c4147 100644 --- a/app/MindWork AI Studio/Models/Alibaba/ModelStudioEmbeddingFamily.cs +++ b/app/MindWork AI Studio/Models/Alibaba/ModelStudioEmbeddingFamily.cs @@ -9,9 +9,10 @@ namespace AIStudio.Models.Alibaba; /// /// /// The previous rules answered for these with the Model Studio default and told them they call -/// functions. The prefix is Alibaba's own: the app filters the catalog by "text-embedding-" to find -/// them, which is also why the rule may be written that broadly -- bound to this provider, it can -/// only ever meet the models Alibaba names that way. +/// functions. The prefix is Alibaba's own, and the rule may be written that broadly because it is +/// bound to this provider: it can only ever meet the models Alibaba names that way. The provider +/// carried the same prefix as a filter of its own until it started asking here, so this is now the +/// only place which says what those names mean. /// public sealed class ModelStudioEmbeddingFamily : ModelFamily { @@ -19,7 +20,7 @@ public sealed class ModelStudioEmbeddingFamily : ModelFamily public override ModelVendor Vendor => ModelVendor.ALIBABA; /// - public override ModelSource Source => new("https://www.alibabacloud.com/help/en/model-studio/embedding", new DateOnly(2026, 9, 11), "Provider/AlibabaCloud/ProviderAlibabaCloud.cs adds these in GetEmbeddingModels and filters the catalog by the prefix \"text-embedding-\"."); + public override ModelSource Source => new("https://www.alibabacloud.com/help/en/model-studio/embedding", new DateOnly(2026, 9, 11), "Provider/AlibabaCloud/ProviderAlibabaCloud.cs used to add these in GetEmbeddingModels and to filter the catalog by the prefix \"text-embedding-\"; it asks this rule instead."); /// protected override void Declare(ModelFamilyBuilder builder) => diff --git a/app/MindWork AI Studio/Models/Google/AqaFamily.cs b/app/MindWork AI Studio/Models/Google/AqaFamily.cs new file mode 100644 index 00000000..f9f6c0d5 --- /dev/null +++ b/app/MindWork AI Studio/Models/Google/AqaFamily.cs @@ -0,0 +1,32 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Google; + +/// +/// AQA, which answers a question out of the passages it was handed. +/// +/// +/// Attributed Question Answering, and the one entry in Google's catalog whose whole name is three +/// letters. It answers on generateAnswer rather than on generateContent, together with the semantic +/// retriever, and what comes back is the answer, the passages it rests on, and an estimate of +/// whether the question could be answered from them at all. +/// +/// Bound to Google, and it has to be: three letters are three letters, and a rule that short has no +/// business meeting a name from somewhere else. The catalog holds exactly one model it can match. +/// +public sealed class AqaFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.GOOGLE; + + /// + public override ModelSource Source => new("https://ai.google.dev/gemini-api/docs/semantic_retrieval", new DateOnly(2026, 9, 19), "Reads 7,168 tokens and writes 1,024, which is a size for an answer rather than for a conversation. The route it answers on is generateAnswer, so nothing the app sends over the chat completion API reaches it."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("aqa").AsExact().OnlyOn(LLMProviders.GOOGLE) + .Capabilities(TEXT_INPUT | TEXT_OUTPUT) + .Kind(ModelKind.GROUNDED_ANSWERING); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Google/GoogleAgentFamily.cs b/app/MindWork AI Studio/Models/Google/GoogleAgentFamily.cs new file mode 100644 index 00000000..58b370d5 --- /dev/null +++ b/app/MindWork AI Studio/Models/Google/GoogleAgentFamily.cs @@ -0,0 +1,48 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Google; + +/// +/// The Google models which are handed a job rather than a message. +/// +/// +/// Both of these answer on the Interactions API alone, never on generateContent: one request starts +/// an autonomous loop which plans, runs code, manages files and searches the web, and a research +/// run takes minutes rather than seconds. A chat request does not time out against them -- it never +/// arrives. +/// +/// Deep Research is the reason these rules are bound to Google instead of standing among the kinds. +/// Perplexity and OpenAI both sell something under that name, and both of those answer over the +/// chat completion API like any other model: sonar-deep-research states it in its own family, and +/// o3-deep-research is held in the corpus. The same two words, three different things, and only the +/// provider tells them apart. +/// +/// Written as a prefix on top of that, because Google puts the words at the front of the name while +/// the other two hang them onto a model they already had. Either guard alone would do; together +/// they also cover whatever Google names this way next. +/// +/// Antigravity needs no such guard -- nobody else names a model that -- but it is a statement about +/// Google's catalog all the same, so it stands where the other one stands. +/// +public sealed class GoogleAgentFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.GOOGLE; + + /// + public override ModelSource Source => new("https://ai.google.dev/gemini-api/docs/deep-research", new DateOnly(2026, 9, 19), "The page states it for the two 04-2026 models: Deep Research runs only through the Interactions API, never through generateContent, and only in the background, because a single run takes five to twenty minutes. The catalog also serves deep-research-pro-preview-12-2025, which the page no longer lists; that it works the same way is read off the naming line rather than off a source. The Antigravity agent is documented at https://ai.google.dev/gemini-api/docs/antigravity-agent and runs on a sandbox Google hosts."); + + /// + protected override void Declare(ModelFamilyBuilder builder) + { + builder.Rule("deep-research").AsPrefix().OnlyOn(LLMProviders.GOOGLE) + .Capabilities(TEXT_INPUT) + .Kind(ModelKind.AGENT); + + builder.Rule("antigravity").AsSegment().OnlyOn(LLMProviders.GOOGLE) + .Capabilities(TEXT_INPUT) + .Kind(ModelKind.AGENT); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Google/LyriaFamily.cs b/app/MindWork AI Studio/Models/Google/LyriaFamily.cs new file mode 100644 index 00000000..f53697d4 --- /dev/null +++ b/app/MindWork AI Studio/Models/Google/LyriaFamily.cs @@ -0,0 +1,35 @@ +using AIStudio.Provider; + +using static AIStudio.Provider.Capability; + +namespace AIStudio.Models.Google; + +/// +/// Lyria, which writes music from a description. +/// +/// +/// Nothing knew the name, so the catalog answered for it the way it answers for everything nobody +/// wrote a rule for: a chat model which reads text, writes text and calls tools. What comes back is +/// a stereo recording with instruments and, from 3.5 on, sung lyrics. +/// +/// Nobody ran into it because the Google provider showed only names beginning with "gemini", which +/// kept four Lyria models out of sight along with the two Gemma models somebody actually wants. The +/// prefix is gone now, so the rule has to carry what the prefix carried by accident. +/// +/// Whole name parts rather than a substring, for the reason Imagen gives next door. The rule is not +/// bound to Google, unlike the agent ones: wherever a model called Lyria turns up, it is this. +/// +public sealed class LyriaFamily : ModelFamily +{ + /// + public override ModelVendor Vendor => ModelVendor.GOOGLE; + + /// + public override ModelSource Source => new("https://ai.google.dev/gemini-api/docs/music-generation", new DateOnly(2026, 9, 19), "A description goes in and 44.1 kHz stereo music comes out, with vocals and timed lyrics from Lyria 3.5 on. The realtime variant holds a connection open instead, which the realtime rule states and outranks this with."); + + /// + protected override void Declare(ModelFamilyBuilder builder) => + builder.Rule("lyria").AsSegment() + .Capabilities(TEXT_INPUT) + .Kind(ModelKind.MUSIC_GENERATION); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/EmbeddingModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/EmbeddingModelsFamily.cs index 0fb1988a..3d65d109 100644 --- a/app/MindWork AI Studio/Models/Kinds/EmbeddingModelsFamily.cs +++ b/app/MindWork AI Studio/Models/Kinds/EmbeddingModelsFamily.cs @@ -27,7 +27,7 @@ public sealed class EmbeddingModelsFamily : ModelFamily public override ModelVendor Vendor => ModelVendor.UNKNOWN; /// - public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=feature-extraction", new DateOnly(2026, 9, 12), "Ported from the embedding markers of Provider/ModelKindExtensions.cs. The e5 line says it in its own family, so it is not repeated here."); + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=feature-extraction", new DateOnly(2026, 9, 19), "Ported from the embedding markers of Provider/ModelKindExtensions.cs. The e5 line says it in its own family, so it is not repeated here. The four names at the end came later, from going through the widely used embedding models whose name carries none of the words above."); /// protected override void Declare(ModelFamilyBuilder builder) @@ -55,5 +55,24 @@ public sealed class EmbeddingModelsFamily : ModelFamily // General Text Embeddings, from Alibaba. Written as a name part rather than as a substring, // because three letters appear inside far too many unrelated words: builder.Modifier("gte").AsSegment().Inherits(); + + // + // Four that say nothing about embedding in their name, and are embedding models all the + // same. Every one of them is widely used, so leaving them out does not cost an exotic case: + // it puts them among the chat models, where somebody picks one and waits for an answer it + // cannot give. All four are written as name parts rather than as substrings, for the reason + // gte above gives -- short words which appear inside unrelated names. + // + // Note that "instructor" is a different word from the "instruct" which half the chat models + // carry, and a name part never matches half of one. + // + builder.Modifier("stella").AsSegment().Inherits(); + + builder.Modifier("labse").AsSegment().Inherits(); + + builder.Modifier("instructor").AsSegment().Inherits(); + + // Generalizable T5 Retrieval, from Google: + builder.Modifier("gtr").AsSegment().Inherits(); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/ImageGenerationModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/ImageGenerationModelsFamily.cs index edc516ce..34572ff0 100644 --- a/app/MindWork AI Studio/Models/Kinds/ImageGenerationModelsFamily.cs +++ b/app/MindWork AI Studio/Models/Kinds/ImageGenerationModelsFamily.cs @@ -19,7 +19,7 @@ public sealed class ImageGenerationModelsFamily : ModelFamily public override ModelVendor Vendor => ModelVendor.UNKNOWN; /// - public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-to-image", new DateOnly(2026, 9, 12), "Ported from the image generation markers of Provider/ModelKindExtensions.cs. Imagen and the Gemini image models state it in their own families as well, where the capabilities stand next to it."); + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=text-to-image", new DateOnly(2026, 9, 12), "Ported from the image generation markers of Provider/ModelKindExtensions.cs. Imagen and the Gemini image models state it in their own families as well, where the capabilities stand next to it. Nano Banana came later, off Google's own catalog at https://generativelanguage.googleapis.com/v1beta/openai/models, read on 2026-09-19."); /// protected override void Declare(ModelFamilyBuilder builder) @@ -38,5 +38,9 @@ public sealed class ImageGenerationModelsFamily : ModelFamily // The other half of Grok Imagine, which the video rule steps aside for: builder.Modifier("grok-imagine").AsSegment().NotContains("video").Inherits(); + + // Google's codename for the image model it serves next to Gemini, and the one name in its + // catalog which says nothing about drawing: nano-banana-pro-preview. + builder.Modifier("nano-banana").AsSegment().Inherits(); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/RealtimeModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/RealtimeModelsFamily.cs index 6997f516..65327771 100644 --- a/app/MindWork AI Studio/Models/Kinds/RealtimeModelsFamily.cs +++ b/app/MindWork AI Studio/Models/Kinds/RealtimeModelsFamily.cs @@ -23,7 +23,7 @@ public sealed class RealtimeModelsFamily : ModelFamily public override ModelVendor Vendor => ModelVendor.UNKNOWN; /// - public override ModelSource Source => new("https://developers.openai.com/api/docs/models/gpt-live-1", new DateOnly(2026, 9, 12), "Ported from the realtime marker of Provider/ModelKindExtensions.cs, where the same precedence was written as the order of two if statements. GPT-Live was added after it turned up in the chat list while testing."); + public override ModelSource Source => new("https://developers.openai.com/api/docs/models/gpt-live-1", new DateOnly(2026, 9, 12), "Ported from the realtime marker of Provider/ModelKindExtensions.cs, where the same precedence was written as the order of two if statements. The live rule was added after GPT-Live turned up in the chat list while testing, and widened when Google's catalog turned out to name its whole two-way line that way: https://ai.google.dev/gemini-api/docs/live"); /// protected override void Declare(ModelFamilyBuilder builder) @@ -33,10 +33,15 @@ public sealed class RealtimeModelsFamily : ModelFamily .Kind(ModelKind.REALTIME); // - // The line which dropped the word. GPT-Live listens and speaks at the same time and leaves - // the thinking to a text model behind it, so there is even less of a conversation in it than - // in the realtime models it succeeds -- and nothing in the name says so any more. + // The other word for the same connection. OpenAI's GPT-Live listens and speaks at once and + // leaves the thinking to a text model behind it, and Google names its whole two-way line + // this way: gemini-3.8-live, gemini-3.1-flash-live-preview, gemini-3.5-live-translate-preview. + // None of them can be talked to the way a chat model can, and all of them stood in the chat + // list until this rule was written. // - builder.Modifier("gpt-live").AsSegment().Inherits(); + // A name part rather than a substring, because four letters sit inside "delivery", + // "olive" and plenty of words which promise no connection at all. + // + builder.Modifier("live").AsSegment().Inherits(); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/TextCompletionModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/TextCompletionModelsFamily.cs index 9bd06ee5..33c8f6ed 100644 --- a/app/MindWork AI Studio/Models/Kinds/TextCompletionModelsFamily.cs +++ b/app/MindWork AI Studio/Models/Kinds/TextCompletionModelsFamily.cs @@ -19,7 +19,7 @@ public sealed class TextCompletionModelsFamily : ModelFamily public override ModelVendor Vendor => ModelVendor.UNKNOWN; /// - public override ModelSource Source => new("https://platform.openai.com/docs/api-reference/completions", new DateOnly(2026, 9, 12), "Ported unchanged from the text completion markers of Provider/ModelKindExtensions.cs."); + public override ModelSource Source => new("https://platform.openai.com/docs/api-reference/completions", new DateOnly(2026, 9, 12), "Ported unchanged from the text completion markers of Provider/ModelKindExtensions.cs. Fill-in-the-middle came later, off Mistral's own catalog at https://api.mistral.ai/v1/models, read on 2026-09-19."); /// protected override void Declare(ModelFamilyBuilder builder) @@ -32,5 +32,13 @@ public sealed class TextCompletionModelsFamily : ModelFamily // The one model of the 3.5 line which never learned to chat, next to the ones which did: builder.Modifier("gpt-3.5-turbo-instruct").AsSegment().Inherits(); + + // + // Fill in the middle: a model handed the code on either side of a gap rather than a + // conversation. Mistral writes it into the name of the one it serves for that, + // mistral-code-fim-latest, which stood in the chat list because nothing looked for the + // word. Codestral does the same job and says so through its family instead. + // + builder.Modifier("fim").AsSegment().Inherits(); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Kinds/TranscriptionModelsFamily.cs b/app/MindWork AI Studio/Models/Kinds/TranscriptionModelsFamily.cs index ec8d3e30..bba1e1e9 100644 --- a/app/MindWork AI Studio/Models/Kinds/TranscriptionModelsFamily.cs +++ b/app/MindWork AI Studio/Models/Kinds/TranscriptionModelsFamily.cs @@ -16,7 +16,7 @@ public sealed class TranscriptionModelsFamily : ModelFamily public override ModelVendor Vendor => ModelVendor.UNKNOWN; /// - public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=automatic-speech-recognition", new DateOnly(2026, 9, 12), "Ported from the transcription markers of Provider/ModelKindExtensions.cs, minus the two which their own families now state."); + public override ModelSource Source => new("https://huggingface.co/models?pipeline_tag=automatic-speech-recognition", new DateOnly(2026, 9, 19), "Ported from the transcription markers of Provider/ModelKindExtensions.cs, minus the two which their own families now state. Canary and asr came later: the markers named neither, so both reached the answer meant for everything nobody wrote a rule for. Alibaba's speech line is documented at https://www.alibabacloud.com/help/en/model-studio/qwen-asr-api-reference."); /// protected override void Declare(ModelFamilyBuilder builder) @@ -27,5 +27,25 @@ public sealed class TranscriptionModelsFamily : ModelFamily builder.Modifier("wav2vec").AsSubstring().Inherits(); builder.Modifier("parakeet").AsSubstring().Inherits(); + + // NVIDIA's other line of speech models, written plain: canary-1b, canary-1b-flash, + // canary-180m-flash. A segment rather than a substring, because canary is an ordinary + // English word which would otherwise reach into names it has nothing to do with -- the + // same reason the embedding family gives for gte. + builder.Modifier("canary").AsSegment().Inherits(); + + // + // The abbreviation the whole field goes by, and the one Alibaba names its speech line + // after: qwen3-asr-flash, qwen3-asr-1.7b, fun-asr-realtime. Three letters, so a name part + // and never a substring -- "laser" and "eraser" carry them without meaning any of this. + // + builder.Modifier("asr").AsSegment().Inherits(); + + // + // Alibaba also builds the line into its audio models, and "audio" is the longer word, so + // without this the speech synthesis rule would answer for a model which only listens. A + // name carrying both words is a transcription model whatever else it is called. + // + builder.Modifier("audio").AsSegment().AlsoContains("asr").Inherits(); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Models/Mistral/CodestralFamily.cs b/app/MindWork AI Studio/Models/Mistral/CodestralFamily.cs index c56f1660..90f704aa 100644 --- a/app/MindWork AI Studio/Models/Mistral/CodestralFamily.cs +++ b/app/MindWork AI Studio/Models/Mistral/CodestralFamily.cs @@ -1,3 +1,5 @@ +using AIStudio.Provider; + using static AIStudio.Provider.Capability; namespace AIStudio.Models.Mistral; @@ -10,6 +12,12 @@ namespace AIStudio.Models.Mistral; /// words the Mistral block looked for, so it walked past every rule and reached the answer meant /// for everything nobody had written one for. That answer happened to describe it correctly, which /// is why nothing looked wrong -- and is exactly the situation this rebuild is meant to end. +/// +/// That Mistral serves it to fill in the middle of a file rather than to talk to was the one thing +/// the rebuild left behind: it stayed in the Mistral provider, as a name check which dropped every +/// model whose ID begins with "code". It says something about a model, so it belongs to the model, +/// and it is bound to the provider because it is only true there. Somebody's own server and the +/// gateways serve the same weights to chat with, which is what the unbound rule above keeps saying. /// public sealed class CodestralFamily : ModelFamily { @@ -17,11 +25,22 @@ public sealed class CodestralFamily : ModelFamily public override ModelVendor Vendor => ModelVendor.MISTRAL_AI; /// - public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 11), "The answer the previous rules gave it through their fallback: text in, text out, tool calling."); + public override ModelSource Source => new("https://docs.mistral.ai/getting-started/models/models_overview/", new DateOnly(2026, 9, 19), "The answer the previous rules gave it through their fallback: text in, text out, tool calling. What Mistral's own catalog makes of it was read from the same page, where Codestral is the model behind the FIM endpoint."); /// - protected override void Declare(ModelFamilyBuilder builder) => + protected override void Declare(ModelFamilyBuilder builder) + { builder.Rule("codestral").AsSegment() .Capabilities(TEXT_INPUT | TEXT_OUTPUT | FUNCTION_CALLING) .Apis(CHAT_COMPLETION_API); + + // + // Bound to Mistral, which makes it the more specific of the two and lets it win there + // without anybody writing an order. It continues a text instead of answering in a + // conversation, so it must not stand among the models somebody picks for a chat. + // + builder.Rule("codestral").AsSegment().OnlyOn(LLMProviders.MISTRAL) + .Inherits() + .Kind(ModelKind.TEXT_COMPLETION); + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index ed18cde1..6cc59cf0 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -6549,9 +6549,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1356621346"] = "Ko -- Failed to validate the selected tokenizer. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1384494471"] = "Die Überprüfung des ausgewählten Tokenizers ist fehlgeschlagen. Bitte versuchen Sie es erneut." --- Please enter an embedding model name. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1661085403"] = "Bitte geben Sie einen Modellnamen für die Einbettung ein." - -- Hostname UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1727440780"] = "Hostname" @@ -6606,9 +6603,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2810182573"] = "Ke -- Instance Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "Instanzname" --- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Derzeit können wir die Einbettungs-Modelle für den ausgewählten Anbieter und/oder Host nicht abfragen. Bitte geben Sie daher den Modellnamen manuell ein." - -- Token limit UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "Token-Limit" @@ -6618,6 +6612,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Bi -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3361153305"] = "Experten-Einstellungen anzeigen" +-- This server does not offer the selected model right now. It stays selected, so the documents you already prepared keep working. Choosing another model means every document of the data sources behind this provider is prepared again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3571276758"] = "Dieser Server bietet das ausgewählte Modell derzeit nicht an. Das Modell bleibt ausgewählt, damit die bereits vorbereiteten Dokumente weiterhin funktionieren. Wenn Sie ein anderes Modell wählen, werden alle Dokumente der Datenquellen dieses Anbieters erneut vorbereitet." + -- How many chunks are sent to the embedding provider at once. The default is 1. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3780233303"] = "Wie viele Blöcke gleichzeitig an den Einbettungsanbieter gesendet werden. Der Standardwert ist 1." @@ -6639,6 +6636,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T808120719"] = "Hos -- Please enter an embedding batch size greater than 0. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T840259907"] = "Bitte geben Sie eine Batch-Größe für Einbettungen größer als 0 ein." +-- Your server answered, but none of the models it serves is one we know to create embeddings. Either there is none installed, or it runs under a name we do not recognize. In the latter case, your organization can describe the model in a model plugin, and it will show up here. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T859645108"] = "Ihr Server hat geantwortet, aber keines der bereitgestellten Modelle ist uns als Modell zur Erstellung von Einbettungen bekannt. Entweder ist kein solches Modell installiert oder es läuft unter einem Namen, den wir nicht erkennen. Im letzteren Fall kann Ihre Organisation das Modell in einem Modell-Plugin beschreiben. Dann wird es hier angezeigt." + -- Provider UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T900237532"] = "Anbieter" @@ -8718,9 +8718,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1324664716"] = -- Create account UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1356621346"] = "Konto erstellen" --- Currently, we cannot query the transcription models for the selected provider and/or host. Therefore, please enter the model name manually. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1381635232"] = "Derzeit können wir die Modelle für Transkriptionen für den ausgewählten Anbieter und/oder Host nicht abfragen. Bitte geben Sie daher den Modellnamen manuell ein." - -- Hostname UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1727440780"] = "Hostname" @@ -8754,9 +8751,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T2842060373"] = -- Hugging Face transcribes audio through a few of its inference providers only, which is why this list is shorter than the one for chatting. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T3397943774"] = "Hugging Face transkribiert Audio nur über einige seiner Inferenzanbieter. Deshalb ist diese Liste kürzer als die für den Chat." --- Please enter a transcription model name. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T3703662664"] = "Bitte geben Sie den Namen eines Transkriptionsmodells ein." - -- This host uses the model configured at the provider level. No model selection is available. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T3783329915"] = "Dieser Host verwendet das auf Anbieterebene konfigurierte Modell. Eine Modellauswahl ist nicht verfügbar." diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 8853aa1d..1b7808b6 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -6549,9 +6549,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1356621346"] = "Cr -- Failed to validate the selected tokenizer. Please try again. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1384494471"] = "Failed to validate the selected tokenizer. Please try again." --- Please enter an embedding model name. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1661085403"] = "Please enter an embedding model name." - -- Hostname UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T1727440780"] = "Hostname" @@ -6606,9 +6603,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2810182573"] = "No -- Instance Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "Instance Name" --- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually." - -- Token limit UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "Token limit" @@ -6618,6 +6612,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Pl -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3361153305"] = "Show Expert Settings" +-- This server does not offer the selected model right now. It stays selected, so the documents you already prepared keep working. Choosing another model means every document of the data sources behind this provider is prepared again. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3571276758"] = "This server does not offer the selected model right now. It stays selected, so the documents you already prepared keep working. Choosing another model means every document of the data sources behind this provider is prepared again." + -- How many chunks are sent to the embedding provider at once. The default is 1. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3780233303"] = "How many chunks are sent to the embedding provider at once. The default is 1." @@ -6639,6 +6636,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T808120719"] = "Hos -- Please enter an embedding batch size greater than 0. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T840259907"] = "Please enter an embedding batch size greater than 0." +-- Your server answered, but none of the models it serves is one we know to create embeddings. Either there is none installed, or it runs under a name we do not recognize. In the latter case, your organization can describe the model in a model plugin, and it will show up here. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T859645108"] = "Your server answered, but none of the models it serves is one we know to create embeddings. Either there is none installed, or it runs under a name we do not recognize. In the latter case, your organization can describe the model in a model plugin, and it will show up here." + -- Provider UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T900237532"] = "Provider" @@ -8718,9 +8718,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1324664716"] = -- Create account UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1356621346"] = "Create account" --- Currently, we cannot query the transcription models for the selected provider and/or host. Therefore, please enter the model name manually. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1381635232"] = "Currently, we cannot query the transcription models for the selected provider and/or host. Therefore, please enter the model name manually." - -- Hostname UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1727440780"] = "Hostname" @@ -8754,9 +8751,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T2842060373"] = -- Hugging Face transcribes audio through a few of its inference providers only, which is why this list is shorter than the one for chatting. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T3397943774"] = "Hugging Face transcribes audio through a few of its inference providers only, which is why this list is shorter than the one for chatting." --- Please enter a transcription model name. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T3703662664"] = "Please enter a transcription model name." - -- This host uses the model configured at the provider level. No model selection is available. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T3783329915"] = "This host uses the model configured at the provider level. No model selection is available." diff --git a/app/MindWork AI Studio/Plugins/models/plugin.lua b/app/MindWork AI Studio/Plugins/models/plugin.lua index 82f691ae..d68f2b3a 100644 --- a/app/MindWork AI Studio/Plugins/models/plugin.lua +++ b/app/MindWork AI Studio/Plugins/models/plugin.lua @@ -146,10 +146,13 @@ MODELS = {} -- -- -- What the model is made for. Optional, defaults to CHAT. -- -- Allowed values are: CHAT, TEXT_COMPLETION, EMBEDDING, RERANKING, --- -- IMAGE_GENERATION, VIDEO_GENERATION, TRANSCRIPTION, SPEECH_SYNTHESIS, --- -- REALTIME, COMPUTER_USE, OCR, MODERATION, OTHER +-- -- IMAGE_GENERATION, VIDEO_GENERATION, MUSIC_GENERATION, TRANSCRIPTION, +-- -- SPEECH_SYNTHESIS, REALTIME, COMPUTER_USE, AGENT, GROUNDED_ANSWERING, +-- -- OCR, MODERATION, OTHER -- -- This decides which lists the model appears in. Use OTHER for entries --- -- which are no models at all. +-- -- which are no models at all. Use AGENT for a model which is handed a +-- -- job and works on it by itself, and GROUNDED_ANSWERING for one which +-- -- answers out of passages it is given and cites them. -- ["KIND"] = "CHAT", -- -- -- Optional: how many tokens the model reads and writes in one diff --git a/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs b/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs index 8a735ebe..af95d336 100644 --- a/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs +++ b/app/MindWork AI Studio/Provider/AlibabaCloud/ProviderAlibabaCloud.cs @@ -76,40 +76,8 @@ public sealed class ProviderAlibabaCloud() : BaseProvider(LLMProviders.ALIBABA_C /// public override async Task GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) { - var additionalModels = new[] - { - new Model("qwq-plus", "QwQ plus"), // reasoning model - new Model("qwen-max-latest", "Qwen-Max (Latest)"), - new Model("qwen-plus-latest", "Qwen-Plus (Latest)"), - new Model("qwen-turbo-latest", "Qwen-Turbo (Latest)"), - new Model("qvq-max", "QVQ Max"), // visual reasoning model - new Model("qvq-max-latest", "QVQ Max (Latest)"), // visual reasoning model - new Model("qwen-vl-max", "Qwen-VL Max"), // text generation model that can understand and process images - new Model("qwen-vl-plus", "Qwen-VL Plus"), // text generation model that can understand and process images - new Model("qwen-mt-plus", "Qwen-MT Plus"), // machine translation - new Model("qwen-mt-turbo", "Qwen-MT Turbo"), // machine translation - - //Open source - new Model("qwen2.5-14b-instruct-1m", "Qwen2.5 14b 1m context"), - new Model("qwen2.5-7b-instruct-1m", "Qwen2.5 7b 1m context"), - new Model("qwen2.5-72b-instruct", "Qwen2.5 72b"), - new Model("qwen2.5-32b-instruct", "Qwen2.5 32b"), - new Model("qwen2.5-14b-instruct", "Qwen2.5 14b"), - new Model("qwen2.5-7b-instruct", "Qwen2.5 7b"), - new Model("qwen2.5-omni-7b", "Qwen2.5-Omni 7b"), // omni-modal understanding and generation model - new Model("qwen2.5-vl-72b-instruct", "Qwen2.5-VL 72b"), - new Model("qwen2.5-vl-32b-instruct", "Qwen2.5-VL 32b"), - new Model("qwen2.5-vl-7b-instruct", "Qwen2.5-VL 7b"), - new Model("qwen2.5-vl-3b-instruct", "Qwen2.5-VL 3b"), - }; - - var result = await this.LoadModels(["q"], SecretStoreType.LLM_PROVIDER, apiKeyProvisional, token); - return result with - { - // The API is the authority: when it reports a model we also keep as a fallback above, - // its entry comes first and the fallback is dropped. - Models = [..result.Models.Concat(additionalModels).DistinctBy(x => x.Id).OrderBy(x => x.Id)] - }; + var result = await this.LoadModels(SecretStoreType.LLM_PROVIDER, apiKeyProvisional, token); + return result with { Models = [..result.Models.Where(model => model.IsChatModel(this.Provider)).OrderBy(x => x.Id)] }; } /// @@ -121,19 +89,8 @@ public sealed class ProviderAlibabaCloud() : BaseProvider(LLMProviders.ALIBABA_C /// public override async Task GetEmbeddingModels(string? apiKeyProvisional = null, CancellationToken token = default) { - - var additionalModels = new[] - { - new Model("text-embedding-v3", "text-embedding-v3"), - }; - - var result = await this.LoadModels(["text-embedding-"], SecretStoreType.EMBEDDING_PROVIDER, apiKeyProvisional, token); - return result with - { - // The API is the authority: when it reports a model we also keep as a fallback above, - // its entry comes first and the fallback is dropped. - Models = [..result.Models.Concat(additionalModels).DistinctBy(x => x.Id).OrderBy(x => x.Id)] - }; + var result = await this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, apiKeyProvisional, token); + return result with { Models = [..result.Models.Where(model => model.IsEmbeddingModel(this.Provider)).OrderBy(x => x.Id)] }; } #region Overrides of BaseProvider @@ -148,12 +105,23 @@ public sealed class ProviderAlibabaCloud() : BaseProvider(LLMProviders.ALIBABA_C #endregion - private Task LoadModels(string[] prefixes, SecretStoreType storeType, string? apiKeyProvisional, CancellationToken token) + /// + /// Reads Model Studio's catalog, whole. + /// + /// + /// It used to be read through a prefix per list -- a single "q" for the models to talk to, and + /// "text-embedding-" for the ones which answer in vectors. Neither survived what Model Studio + /// became: the letter also brings qwen-image, qwen-tts, qwen3-asr and qwen-vl-ocr into the chat + /// list, while it locks out DeepSeek, Kimi, GLM and MiniMax, which Alibaba serves through this + /// very endpoint. A name has never been a statement about what a model is for; the callers ask + /// the registry instead. + /// + private Task LoadModels(SecretStoreType storeType, string? apiKeyProvisional, CancellationToken token) { return this.LoadModelsResponse( storeType, "models", - modelResponse => modelResponse.Data.Where(model => prefixes.Any(prefix => model.Id.StartsWith(prefix, StringComparison.InvariantCulture))), + modelResponse => modelResponse.Data, apiKeyProvisional, token: token); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs index 71138df9..17ad8309 100644 --- a/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs +++ b/app/MindWork AI Studio/Provider/Anthropic/ProviderAnthropic.cs @@ -213,9 +213,14 @@ public sealed class ProviderAnthropic() : BaseProvider(LLMProviders.ANTHROPIC, n var result = await this.LoadModels(SecretStoreType.LLM_PROVIDER, apiKeyProvisional, token); return result with { + // // The API is the authority: when it reports a model we also keep as a fallback above, - // its entry comes first and the fallback is dropped. - Models = [..result.Models.Concat(additionalModels).DistinctBy(x => x.Id).OrderBy(x => x.Id)] + // its entry comes first and the fallback is dropped. What it reports is asked about + // first, though -- the route says nothing about what a model is made for, and Claude + // has not always been only something to talk to. The six above skip that question + // because they are not a catalog: every one of them was picked by hand. + // + Models = [..result.Models.Where(model => model.IsChatModel(this.Provider)).Concat(additionalModels).DistinctBy(x => x.Id).OrderBy(x => x.Id)] }; } diff --git a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs index 2a63180b..410efac4 100644 --- a/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs +++ b/app/MindWork AI Studio/Provider/Fireworks/ProviderFireworks.cs @@ -93,6 +93,16 @@ public class ProviderFireworks() : BaseProvider(LLMProviders.FIREWORKS, new Uri( } /// + /// + /// The one transcription list which stays a plain list, where GWDG and Mistral ask their + /// endpoint first. There is nothing to ask here: HasModelLoadingCapability is false and every + /// other method above answers with nothing, which is why a chat model at Fireworks has to be + /// typed in by hand. A list is all there is. + /// + /// The commented-out entry is no oversight either. The documentation names Whisper v3 Turbo, + /// and trying it does not work -- which is worth keeping written down, so that nobody adds it + /// back and finds out the same way again. + /// public override Task GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default) { // Source: https://docs.fireworks.ai/api-reference/audio-transcriptions#param-model diff --git a/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs b/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs index 54b32e57..3b77344a 100644 --- a/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs +++ b/app/MindWork AI Studio/Provider/GWDG/ProviderGWDG.cs @@ -18,6 +18,12 @@ public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, new Uri("ht new("qwen3-embedding-4b", "Qwen3 Embedding 4B"), ]; + // Source: https://docs.hpc.gwdg.de/services/saia/index.html#voice-to-text + private static readonly Model[] KNOWN_TRANSCRIPTION_MODELS = + [ + new("whisper-large-v2", "Whisper v2 Large"), + ]; + #region Implementation of IProvider /// @@ -123,13 +129,27 @@ public sealed class ProviderGWDG() : BaseProvider(LLMProviders.GWDG, new Uri("ht } /// - public override Task GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default) + /// + /// Built the same way as the embedding models above, and for the same reason: SAIA answers the + /// models endpoint with its chat models only, so this comes back empty and the documented list + /// stands in. Asking first costs nothing and means a speech model appearing in that answer one + /// day shows up on its own, rather than waiting for somebody to notice and edit this file. A + /// failed request is passed on unchanged, so a wrong API key stays visible as such. + /// + public override async Task GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default) { - // Source: https://docs.hpc.gwdg.de/services/saia/index.html#voice-to-text - return Task.FromResult(ModelLoadResult.FromModels( - [ - new Model("whisper-large-v2", "Whisper v2 Large"), - ])); + var result = await this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, apiKeyProvisional, token); + if (!result.Success) + return result; + + var transcriptionModels = result.Models.Where(model => model.IsTranscriptionModel(this.Provider)).ToList(); + if (transcriptionModels.Count is 0) + return ModelLoadResult.FromModels(KNOWN_TRANSCRIPTION_MODELS); + + return result with + { + Models = [..transcriptionModels] + }; } #endregion diff --git a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs index af4486f3..0d143ecd 100644 --- a/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs +++ b/app/MindWork AI Studio/Provider/Google/ProviderGoogle.cs @@ -172,11 +172,15 @@ public class ProviderGoogle() : BaseProvider(LLMProviders.GOOGLE, new Uri("https // Asking what a model is made for, rather than only ruling out the embedding ones. // Google names everything after the chat model it grew out of, so the catalog is // full of names which look like something to talk to and are not: the image models, - // and the computer use model whose API refuses a request without its tool. + // the computer use model whose API refuses a request without its tool, and the live + // line which wants a connection held open in both directions. // - ..result.Models.Where(model => - model.Id.StartsWith("gemini-", StringComparison.OrdinalIgnoreCase) && - model.IsChatModel(this.Provider)) + // The question used to be asked of names beginning with "gemini" alone, and that + // cost the two Gemma models Google serves on this very route. What the prefix kept + // out besides them -- Lyria, Imagen, Veo, the research and coding agents, AQA -- + // is kept out by a rule now, where the reason is written down. + // + ..result.Models.Where(model => model.IsChatModel(this.Provider)) .Select(this.WithDisplayNameFallback) ] }; diff --git a/app/MindWork AI Studio/Provider/LLMProvidersExtensions.cs b/app/MindWork AI Studio/Provider/LLMProvidersExtensions.cs index a7992cb2..3a10e5d0 100644 --- a/app/MindWork AI Studio/Provider/LLMProvidersExtensions.cs +++ b/app/MindWork AI Studio/Provider/LLMProvidersExtensions.cs @@ -421,17 +421,6 @@ public static class LLMProvidersExtensions _ => false, }; - public static bool IsEmbeddingModelProvidedManually(this LLMProviders provider, Host host) => provider switch - { - LLMProviders.SELF_HOSTED => host is not Host.LM_STUDIO, - _ => false, - }; - - public static bool IsTranscriptionModelProvidedManually(this LLMProviders provider, Host host) => provider switch - { - _ => false, - }; - /// /// Determines if the model selection should be completely hidden for LLM providers. /// This is the case when the host does not support model selection. diff --git a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs index 15299569..17e65b98 100644 --- a/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs +++ b/app/MindWork AI Studio/Provider/Mistral/ProviderMistral.cs @@ -93,12 +93,13 @@ public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, new U { Models = [ - // Codestral is a fill-in-the-middle model, which we cannot use for chats. That is - // specific to Mistral's catalog, which is why it is not part of the shared model - // kind detection: - ..modelResponse.Models.Where(n => - !n.Id.StartsWith("code", StringComparison.OrdinalIgnoreCase) && - n.IsChatModel(this.Provider)) + // + // Codestral is a fill-in-the-middle model, which we cannot use for chats. Its own + // family says so now, bound to this provider, so the word "code" no longer has to + // be tested for here -- and testing for it never reached mistral-code-fim-latest, + // which does the same job under a name that begins differently. + // + ..modelResponse.Models.Where(n => n.IsChatModel(this.Provider)) ] }; } @@ -123,13 +124,16 @@ public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, new U } /// - public override Task GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default) + public override async Task GetTranscriptionModels(string? apiKeyProvisional = null, CancellationToken token = default) { - // Source: https://docs.mistral.ai/capabilities/audio_transcription - return Task.FromResult(ModelLoadResult.FromModels( - [ - new Provider.Model("voxtral-mini-latest", "Voxtral Mini Latest"), - ])); + var modelResponse = await this.LoadModelList(SecretStoreType.TRANSCRIPTION_PROVIDER, apiKeyProvisional, token); + if (!modelResponse.Success) + return modelResponse; + + return modelResponse with + { + Models = [..modelResponse.Models.Where(n => n.IsTranscriptionModel(this.Provider))] + }; } #endregion @@ -144,4 +148,4 @@ public sealed class ProviderMistral() : BaseProvider(LLMProviders.MISTRAL, new U listingFactory: modelResponse => modelResponse.Data.Select(n => ModelListing.For(n.Id, n.ContextWindowTokens)), token: token); } -} +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/ModelKind.cs b/app/MindWork AI Studio/Provider/ModelKind.cs index 75a7e49f..f6baa0b3 100644 --- a/app/MindWork AI Studio/Provider/ModelKind.cs +++ b/app/MindWork AI Studio/Provider/ModelKind.cs @@ -52,6 +52,16 @@ public enum ModelKind /// VIDEO_GENERATION, + /// + /// The model composes music. + /// + /// + /// Audio comes out of it, but not speech: instruments, arrangement, and in Lyria's case singing + /// with lyrics. Neither the speech synthesis list nor any other one fits, and a chat request to + /// such a model gets nothing back that reads like an answer. + /// + MUSIC_GENERATION, + /// /// The model transcribes audio into text. /// @@ -86,6 +96,27 @@ public enum ModelKind /// COMPUTER_USE, + /// + /// The model runs an errand of its own instead of answering. + /// + /// + /// One request starts a loop which plans, calls tools, runs code and reads the web, and it can + /// take minutes. Google serves its research and coding agents this way, through an API of their + /// own which a chat request never reaches. Note that the name alone decides nothing here: what + /// Perplexity calls deep research is an ordinary chat model with web search. + /// + AGENT, + + /// + /// The model answers a question out of sources handed to it, and says where the answer came from. + /// + /// + /// Built for retrieval rather than for conversation: it is given passages along with the + /// question, and returns the answer, the citations, and an estimate of whether the question + /// could be answered from them at all. Reached through a route of its own. + /// + GROUNDED_ANSWERING, + /// /// The model extracts text from images or scanned documents. /// diff --git a/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs b/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs index 0ff35fb0..98bbbc17 100644 --- a/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs +++ b/app/MindWork AI Studio/Provider/OpenRouter/ProviderOpenRouter.cs @@ -137,6 +137,11 @@ public sealed class ProviderOpenRouter() : BaseProvider(LLMProviders.OPEN_ROUTER /// Nothing is reported from here: this route answers with the embedding models alone, and what /// is reported replaces everything an instance said before. The windows of the chat models /// would go missing the moment somebody opens the embedding settings. + /// + /// Nothing is filtered either, for the same reason. The route is the statement: OpenRouter + /// serves these to embed with, which is more than a name can say. Asking the registry on top + /// could only drop a model whose name we do not recognize -- and where the two disagree, the + /// answer is a rule in Models/, not a model missing from this list. /// /// An API key which is not stored yet. /// The cancellation token to use. @@ -156,4 +161,4 @@ public sealed class ProviderOpenRouter() : BaseProvider(LLMProviders.OPEN_ROUTER }, token: token); } -} +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs index c24e7238..999e0b44 100644 --- a/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs +++ b/app/MindWork AI Studio/Provider/SelfHosted/ProviderSelfHosted.cs @@ -97,12 +97,16 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide switch (host) { case Host.LLAMA_CPP: - return await this.LoadLlamaCppTextModels(["embed"], [], apiKeyProvisional, token); - + return await this.LoadLlamaCppTextModels(apiKeyProvisional, token); + case Host.LM_STUDIO: case Host.OLLAMA: case Host.VLLM: - return await this.LoadModels( SecretStoreType.LLM_PROVIDER, ["embed"], [], apiKeyProvisional, token); + var result = await this.LoadModels(SecretStoreType.LLM_PROVIDER, apiKeyProvisional, token); + return result with + { + Models = [..result.Models.Where(model => model.IsChatModel(this.Provider))] + }; } return ModelLoadResult.FromModels([]); @@ -129,14 +133,18 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide case Host.LM_STUDIO: case Host.OLLAMA: case Host.VLLM: - return await this.LoadModels( SecretStoreType.EMBEDDING_PROVIDER, [], ["embed"], apiKeyProvisional, token); + var result = await this.LoadModels(SecretStoreType.EMBEDDING_PROVIDER, apiKeyProvisional, token); + return result with + { + Models = [..result.Models.Where(model => model.IsEmbeddingModel(this.Provider))] + }; } return ModelLoadResult.FromModels([]); } catch(Exception e) { - LOGGER.LogError($"Failed to load text models from self-hosted provider: {e.Message}"); + LOGGER.LogError($"Failed to load embedding models from self-hosted provider: {e.Message}"); return ModelLoadResult.Failure(ModelLoadFailureReason.UNKNOWN, e.Message); } } @@ -154,10 +162,22 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide new Provider.Model("loaded-model", TB("Model as configured by whisper.cpp")), ]); + // + // These two answer the models endpoint with everything they serve, and nothing in + // that answer says which of them listens. Asking what each model is made for is the + // only thing standing between this list and every chat and embedding model of the + // installation, which is what it used to hold. An engine running no speech model at + // all therefore offers nothing here, and says so, rather than offering models which + // would fail the moment audio reaches them. + // case Host.OLLAMA: case Host.VLLM: - return await this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, [], [], apiKeyProvisional, token); - + var result = await this.LoadModels(SecretStoreType.TRANSCRIPTION_PROVIDER, apiKeyProvisional, token); + return result with + { + Models = [..result.Models.Where(model => model.IsTranscriptionModel(this.Provider))] + }; + default: return ModelLoadResult.FromModels([]); } @@ -171,7 +191,21 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide #endregion - private async Task LoadModels(SecretStoreType storeType, string[] ignorePhrases, string[] filterPhrases, string? apiKeyProvisional, CancellationToken token) + /// + /// Everything the engine lists, in the order it listed it. + /// + /// + /// What kind of model each of these is stays unanswered here. It used to be answered right in + /// this method, by looking for the word "embed" in the name: the text models were the ones + /// without it, the embedding models the ones with it. That reading lost bge-m3 and all-minilm, + /// which say what they are through another word, and handed them to the chat list instead. The + /// callers ask the shared model kind detection now, the way every other provider does. + /// + /// Which key to send along. + /// A key from a dialog which has not stored it yet. + /// The cancellation token. + /// The models the engine named, unsorted and unfiltered. + private async Task LoadModels(SecretStoreType storeType, string? apiKeyProvisional, CancellationToken token) { var secretKey = await this.GetModelLoadingSecretKey(storeType, apiKeyProvisional, isTryingSecret: true); @@ -211,10 +245,8 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide // ListedModels.Shared.Report(this.ConfiguredProviderId, ListingsOf(models)); - return SuccessfulModelLoadResult(models. - Where(model => !string.IsNullOrWhiteSpace(model.Id) && - !ignorePhrases.Any(ignorePhrase => model.Id.Contains(ignorePhrase, StringComparison.InvariantCulture)) && - filterPhrases.All( filter => model.Id.Contains(filter, StringComparison.InvariantCulture))) + return SuccessfulModelLoadResult(models + .Where(model => !string.IsNullOrWhiteSpace(model.Id)) .Select(n => new Provider.Model(n.Id, null))); } catch (Exception e) when (this.IsTimeoutException(e, token)) @@ -229,7 +261,7 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide if (host is not Host.LLAMA_CPP || !chatModel.IsSystemModel) return chatModel; - var modelLoadResult = await this.LoadLlamaCppTextModels(["embed"], [], null, token); + var modelLoadResult = await this.LoadLlamaCppTextModels(null, token); if (!modelLoadResult.Success) return chatModel; @@ -266,7 +298,7 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide return chatModel; } - private async Task LoadLlamaCppTextModels(string[] ignorePhrases, string[] filterPhrases, string? apiKeyProvisional, CancellationToken token) + private async Task LoadLlamaCppTextModels(string? apiKeyProvisional, CancellationToken token) { var secretKey = await this.GetModelLoadingSecretKey(SecretStoreType.LLM_PROVIDER, apiKeyProvisional, true); @@ -298,7 +330,7 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide return LlamaCppLegacyModelResult(); var models = responseModels - .Where(model => IsMatchingLlamaCppTextModel(model, ignorePhrases, filterPhrases)) + .Where(this.IsMatchingLlamaCppTextModel) .Select(model => new Provider.Model(model.Id, null)) .ToList(); @@ -330,15 +362,23 @@ public sealed class ProviderSelfHosted(Host host, string hostname) : BaseProvide /// One listing per model, which says nothing for the models the engine was silent about. private static IEnumerable ListingsOf(IEnumerable models) => models.Select(model => ModelListing.For(model.Id, model.ContextWindowTokens)); - private static bool IsMatchingLlamaCppTextModel(Model model, string[] ignorePhrases, string[] filterPhrases) + /// + /// Whether this is a model somebody can chat with, as far as llama.cpp and the rules say. + /// + /// + /// Two sources, and both have to agree. What a model is made for comes from the shared rules, + /// the same answer the other engines get. What the running build of it puts out comes from + /// llama.cpp itself, which states the modalities on this route: an engine serving a model that + /// answers in something other than text knows that before any rule about the name could. + /// + /// The model as llama.cpp listed it. + /// True when both agree that it answers a chat in text. + private bool IsMatchingLlamaCppTextModel(Model model) { if (string.IsNullOrWhiteSpace(model.Id)) return false; - if (ignorePhrases.Any(ignorePhrase => model.Id.Contains(ignorePhrase, StringComparison.InvariantCultureIgnoreCase))) - return false; - - if (!filterPhrases.All(filter => model.Id.Contains(filter, StringComparison.InvariantCultureIgnoreCase))) + if (!new Provider.Model(model.Id, null).IsChatModel(this.Provider)) return false; var outputModalities = model.Architecture?.OutputModalities; diff --git a/app/MindWork AI Studio/Provider/X/ProviderX.cs b/app/MindWork AI Studio/Provider/X/ProviderX.cs index 46d96be8..b10f1ef2 100644 --- a/app/MindWork AI Studio/Provider/X/ProviderX.cs +++ b/app/MindWork AI Studio/Provider/X/ProviderX.cs @@ -76,7 +76,7 @@ public sealed class ProviderX() : BaseProvider(LLMProviders.X, new Uri("https:// /// public override async Task GetTextModels(string? apiKeyProvisional = null, CancellationToken token = default) { - var result = await this.LoadModels(SecretStoreType.LLM_PROVIDER, ["grok-"], apiKeyProvisional, token); + var result = await this.LoadModels(SecretStoreType.LLM_PROVIDER, apiKeyProvisional, token); return result with { // @@ -108,19 +108,24 @@ public sealed class ProviderX() : BaseProvider(LLMProviders.X, new Uri("https:// #endregion - private Task LoadModels(SecretStoreType storeType, string[] prefixes, string? apiKeyProvisional, CancellationToken token) + /// + /// Reads the xAI catalog, whole. + /// + /// + /// Every name in it begins with "grok", which is why the prefix this used to filter by never + /// took anything away -- and why it said nothing either. What it did carry was Grok 2, appended + /// to every answer whether xAI still served it or not. It does not: the catalog has moved on to + /// Grok 4, and an entry nobody can talk to is worse than one missing from the list. + /// + /// What the catalog does hold besides the chat models is five names which draw or film. The + /// caller asks the registry about those. + /// + private Task LoadModels(SecretStoreType storeType, string? apiKeyProvisional, CancellationToken token) { return this.LoadModelsResponse( storeType, "models", - modelResponse => modelResponse.Data.Where(model => prefixes.Any(prefix => model.Id.StartsWith(prefix, StringComparison.InvariantCulture))) - .Concat([ - new Model - { - Id = "grok-2-latest", - DisplayName = "Grok 2.0 (latest)", - } - ]), + modelResponse => modelResponse.Data, apiKeyProvisional, token: token); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md index 4cef055b..24798cd6 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.9.1.md @@ -36,6 +36,7 @@ - Improved the provider selection throughout the assistants: when there is nothing to choose from, it now says why. Either you have not set up a provider yet, or none of yours is trusted enough for what you are doing. Before, the list was simply empty. - Improved the settings of your embedding providers and data sources. Some of them decide how your documents are read, so changing one means preparing every document all over again. AI Studio now asks before that happens, names the data sources it would affect, and says when a cloud provider charges you for it. - Improved the dialogs of your embedding providers and data sources: when a change would mean preparing all your documents again, and you decide against it, nothing is saved and the dialog stays open with your change in front of you, ready to be corrected. +- Improved what happens when you open an embedding provider whose server is unreachable or no longer offers the model you chose. That model stays selected, and AI Studio tells you the server does not have it right now. The documents you already prepared keep working, and you are not asked to prepare them again over a change you never made. - Improved the question AI Studio asks before you delete an embedding provider. It now names the data sources depending on that provider, together with what they can still do without it. - Improved what the AI is told when it answers from your own documents (RAG): it now learns which page a passage came from, so it can name the page an answer rests on. - Changed how provider trust and provider confidence work together. Marking a provider as trustworthy in a configuration no longer also satisfies a required confidence level: one says who runs the provider, the other how confidential it is. Organizations raise a provider's level in their own confidence scheme instead. This applies beyond local data sources, for example, when a model reads a page from your intranet. @@ -44,7 +45,11 @@ - Fixed a renamed policy losing its new name in the Document Analysis assistant. The name was kept only when you happened to change something else afterward. - Fixed model names that a provider writes in its own way not being recognized at all, such as the colon Ollama puts before the variant. Those models were treated as plain text models and lost every other ability. - Fixed a model resold under a plain name not getting the abilities it really has. -- Fixed image and video generation models showing up among the chat models. +- Fixed models showing up among the chat models, although nobody can chat with them, such as the ones that draw, film, compose music, transcribe speech, read scanned documents, or work through a task on their own. The same goes for models that need a live connection AI Studio cannot open. +- Fixed the model lists of your providers showing the wrong models. Some your provider offers were missing, among them Google's Gemma models, while others that no longer exist were still on offer, among them an older Grok version. The lists now follow what your provider reports. +- Fixed having to type the name of an embedding model by hand when you set up a provider on your own Ollama or vLLM server. AI Studio now asks your server which models it has and offers them in a list, the same way it has always done for LM Studio. +- Fixed the model list for transcription on your own server offering everything the server has, including models that can only chat or create embeddings. You are now offered the models that can actually transcribe, and nothing else. +- Fixed the model of a transcription provider on your own Ollama server not being saved. An empty model was stored instead, so transcribing with it could not work, and picking a different model changed nothing. - Fixed a dropped file being processed several times, e.g., after the computer woke up from sleep. - Fixed nothing happening when you dropped a file onto the list of your attached files. You can now add files to that list while it is open. - Fixed the preview of an attached file ignoring dropped files. Drop another file onto the preview, and it is attached and shown right away. diff --git a/app/Tests/Models/Corpus/CapabilitySnapshot.txt b/app/Tests/Models/Corpus/CapabilitySnapshot.txt index 0e45f251..f684fa3e 100644 --- a/app/Tests/Models/Corpus/CapabilitySnapshot.txt +++ b/app/Tests/Models/Corpus/CapabilitySnapshot.txt @@ -20,10 +20,7 @@ # what each of them must answer, but it states capabilities alone. # ALIBABA_CLOUD | qvq-max | ALWAYS_REASONING, CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) -ALIBABA_CLOUD | qwen-max-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) ALIBABA_CLOUD | qwen-mt-plus | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) -ALIBABA_CLOUD | qwen-plus-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) -ALIBABA_CLOUD | qwen-turbo-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) ALIBABA_CLOUD | qwen-vl-max | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) ALIBABA_CLOUD | qwen2.5-14b-instruct-1m | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) ALIBABA_CLOUD | qwen2.5-72b-instruct | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) @@ -67,7 +64,7 @@ FIREWORKS | accounts/fireworks/models/qwen3-235b-a22b | CHAT_COMPLETION_API, FUN FIREWORKS | whisper-v3 | SPEECH_INPUT, TEXT_OUTPUT | TRANSCRIPTION | (unknown) | (unknown) | (unknown) GOOGLE | gemini-1.0-pro-vision | CHAT_COMPLETION_API, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) GOOGLE | gemini-2.0-flash | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | (unknown) | 3600 per request | PROVIDER_API countTokens -GOOGLE | gemini-2.0-flash-live-001 | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, SPEECH_INPUT, SPEECH_OUTPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | (unknown) | (unknown) | (unknown) +GOOGLE | gemini-2.0-flash-live-001 | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, SPEECH_INPUT, SPEECH_OUTPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | REALTIME | (unknown) | (unknown) | (unknown) GOOGLE | gemini-2.5-flash | ALWAYS_REASONING, AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | 1048576 | 3600 per request | PROVIDER_API countTokens GOOGLE | gemini-2.5-flash-image | CHAT_COMPLETION_API, IMAGE_OUTPUT, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | IMAGE_GENERATION | (unknown) | (unknown) | (unknown) GOOGLE | gemini-2.5-flash-lite | AUDIO_INPUT, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, OPTIONAL_REASONING, SPEECH_INPUT, TEXT_INPUT, TEXT_OUTPUT, VIDEO_INPUT | CHAT | 1048576 | 3600 per request | PROVIDER_API countTokens @@ -115,7 +112,7 @@ LITE_LLM | anthropic/claude-sonnet-5 | CHAT_COMPLETION_API, FUNCTION_CALLING, MU LITE_LLM | azure/gpt-5.6 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, REASONING_BY_DEFAULT, TEXT_INPUT, TEXT_OUTPUT, WEB_SEARCH | CHAT | 1050000 | (unknown) | TIKTOKEN o200k_base LITE_LLM | bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) LITE_LLM | the-fast-one | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) -MISTRAL | codestral-2508 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) +MISTRAL | codestral-2508 | CHAT_COMPLETION_API, FUNCTION_CALLING, TEXT_INPUT, TEXT_OUTPUT | TEXT_COMPLETION | (unknown) | (unknown) | (unknown) MISTRAL | magistral-medium-2506 | ALWAYS_REASONING, CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) MISTRAL | ministral-14b-2512 | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) MISTRAL | ministral-3b-latest | CHAT_COMPLETION_API, FUNCTION_CALLING, MULTIPLE_IMAGE_INPUT, TEXT_INPUT, TEXT_OUTPUT | CHAT | (unknown) | (unknown) | (unknown) diff --git a/app/Tests/Models/Corpus/CorpusOrigin.cs b/app/Tests/Models/Corpus/CorpusOrigin.cs index 7d72fdb0..6544b0d4 100644 --- a/app/Tests/Models/Corpus/CorpusOrigin.cs +++ b/app/Tests/Models/Corpus/CorpusOrigin.cs @@ -17,8 +17,8 @@ public enum CorpusOrigin NAMED_BY_A_RULE, /// - /// The app carries this model in a built-in list, such as the one Alibaba Cloud models are - /// picked from when the provider serves no catalog. + /// The app carries this model in a built-in list, such as the aliases Anthropic answers to but + /// does not list, or the transcription model GWDG serves without naming it. /// BUILT_INTO_THE_APP, diff --git a/app/Tests/Models/Corpus/ModelCorpus.cs b/app/Tests/Models/Corpus/ModelCorpus.cs index 9be1e9c9..1cfcdbf5 100644 --- a/app/Tests/Models/Corpus/ModelCorpus.cs +++ b/app/Tests/Models/Corpus/ModelCorpus.cs @@ -134,23 +134,28 @@ public static class ModelCorpus ]; /// - /// Alibaba Cloud. Everything below the two dozen models the app carries is one Qwen tier per - /// entry, because each tier answers differently about thinking and vision. + /// Alibaba Cloud. One Qwen tier per entry, because each tier answers differently about thinking + /// and vision. /// + /// + /// The app carried two dozen of these names in a list of its own until the catalog became the + /// only source. Three went with it and are not replaced: qwen-max-latest, qwen-plus-latest and + /// qwen-turbo-latest are a naming convention Alibaba has left behind -- its rolling names carry + /// no suffix now, and the tier once called turbo is called flash. The ones which stayed are + /// here for the other reason: no rule spells any of them out, so they say what becomes of a + /// name the rules were not written for. + /// private static readonly CorpusEntry[] ALIBABA_ENTRIES = [ - new(ALIBABA_CLOUD, "qwq-plus", BUILT_INTO_THE_APP), - new(ALIBABA_CLOUD, "qwen-max-latest", BUILT_INTO_THE_APP), - new(ALIBABA_CLOUD, "qwen-plus-latest", BUILT_INTO_THE_APP), - new(ALIBABA_CLOUD, "qwen-turbo-latest", BUILT_INTO_THE_APP), - new(ALIBABA_CLOUD, "qvq-max", BUILT_INTO_THE_APP), - new(ALIBABA_CLOUD, "qwen-vl-max", BUILT_INTO_THE_APP), - new(ALIBABA_CLOUD, "qwen-mt-plus", BUILT_INTO_THE_APP), - new(ALIBABA_CLOUD, "qwen2.5-72b-instruct", BUILT_INTO_THE_APP), - new(ALIBABA_CLOUD, "qwen2.5-14b-instruct-1m", BUILT_INTO_THE_APP), - new(ALIBABA_CLOUD, "qwen2.5-omni-7b", BUILT_INTO_THE_APP), - new(ALIBABA_CLOUD, "qwen2.5-vl-72b-instruct", BUILT_INTO_THE_APP), - new(ALIBABA_CLOUD, "text-embedding-v3", BUILT_INTO_THE_APP), + new(ALIBABA_CLOUD, "qwq-plus", NAMED_BY_NO_RULE), + new(ALIBABA_CLOUD, "qvq-max", NAMED_BY_NO_RULE), + new(ALIBABA_CLOUD, "qwen-vl-max", NAMED_BY_NO_RULE), + new(ALIBABA_CLOUD, "qwen-mt-plus", NAMED_BY_NO_RULE), + new(ALIBABA_CLOUD, "qwen2.5-72b-instruct", NAMED_BY_NO_RULE), + new(ALIBABA_CLOUD, "qwen2.5-14b-instruct-1m", NAMED_BY_NO_RULE), + new(ALIBABA_CLOUD, "qwen2.5-omni-7b", NAMED_BY_NO_RULE), + new(ALIBABA_CLOUD, "qwen2.5-vl-72b-instruct", NAMED_BY_NO_RULE), + new(ALIBABA_CLOUD, "text-embedding-v3", NAMED_BY_NO_RULE), new(ALIBABA_CLOUD, "qwen3-omni-flash", NAMED_BY_A_RULE), new(ALIBABA_CLOUD, "qwen3-vl-plus", NAMED_BY_A_RULE), new(ALIBABA_CLOUD, "qwen3-235b-a22b", NAMED_BY_A_RULE), diff --git a/app/Tests/Models/Corpus/ModelKindCorpus.cs b/app/Tests/Models/Corpus/ModelKindCorpus.cs index 4b30075f..e91d27bf 100644 --- a/app/Tests/Models/Corpus/ModelKindCorpus.cs +++ b/app/Tests/Models/Corpus/ModelKindCorpus.cs @@ -33,6 +33,39 @@ public static class ModelKindCorpus // The one name whose only marker used to be the organization it was published under: new(SELF_HOSTED, "sentence-transformers/all-MiniLM-L6-v2", EMBEDDING), + + // Mistral's embedding checkpoint for code. It carries the name of a family which is a text + // completion model on this very provider, and stays an embedding model regardless: what a + // model is for is said by the word which says it, not by the family it was built from. + new(MISTRAL, "codestral-embed", EMBEDDING), + + // Alibaba names its own by the prefix the provider used to filter the catalog by. The rule + // says it now, so the prefix is free to go: + new(ALIBABA_CLOUD, "text-embedding-v3", EMBEDDING), + new(ALIBABA_CLOUD, "text-embedding-v4", EMBEDDING), + + // + // What a local Ollama installation serves, taken off its models endpoint rather than + // written from memory. The last two are the ones worth having: neither name carries the + // word "embed", so both were lost by the phrase the self-hosted provider used to filter + // with, and turned up among the chat models instead. Here they are answered by "bge" and + // "minilm", which is what those words are written for. + // + new(SELF_HOSTED, "qwen3-embedding:0.6b", EMBEDDING), + new(SELF_HOSTED, "qwen3-embedding:latest", EMBEDDING), + new(SELF_HOSTED, "nomic-embed-text:latest", EMBEDDING), + new(SELF_HOSTED, "bge-m3:latest", EMBEDDING), + new(SELF_HOSTED, "all-minilm:latest", EMBEDDING), + + // + // The four whose names say nothing about embedding at all. They are here because the list + // above answers them through a word they happen to carry, and these carry none: without a + // rule of their own they would count as chat models, which is where they stood. + // + new(SELF_HOSTED, "stella_en_400M_v5", EMBEDDING), + new(SELF_HOSTED, "LaBSE", EMBEDDING), + new(SELF_HOSTED, "instructor-xl", EMBEDDING), + new(SELF_HOSTED, "gtr-t5-large", EMBEDDING), ]; /// @@ -57,6 +90,12 @@ public static class ModelKindCorpus new(GOOGLE, "gemini-3-pro-image", IMAGE_GENERATION), new(GOOGLE, "imagen-4.0-generate-001", IMAGE_GENERATION, AnsweredTodayAs: CHAT, Reason: "The markers never knew the name; the family ported in the Google step states it. Nobody noticed because the Google provider shows only names beginning with gemini."), + + new(ALIBABA_CLOUD, "qwen-image-edit", IMAGE_GENERATION), + + // Google ships one image model under a codename instead of a description. Nothing in it + // says drawing, and the provider only ever saw it because its catalog was read whole. + new(GOOGLE, "nano-banana-pro-preview", IMAGE_GENERATION), ]; /// @@ -82,6 +121,29 @@ public static class ModelKindCorpus new(SELF_HOSTED, "parakeet-tdt-0.6b-v2", TRANSCRIPTION), new(SELF_HOSTED, "wav2vec2-large-xlsr-53", TRANSCRIPTION), new(MISTRAL, "voxtral-mini-latest", TRANSCRIPTION), + + // The rest of what Mistral actually serves, off its own catalog. The app offered the one + // name above alone, because that is the one the documentation names; these three were there + // the whole time. Both sizes come as a rolling name and as a dated snapshot. + new(MISTRAL, "voxtral-small-latest", TRANSCRIPTION), + new(MISTRAL, "voxtral-mini-2602", TRANSCRIPTION), + new(MISTRAL, "voxtral-small-2507", TRANSCRIPTION), + + // NVIDIA's other speech line, once plain and once as the hub names it. The second one is + // what makes the rule a segment worth keeping: the organization comes off before any rule + // sees the name, so what is left has to carry the word on its own. + new(SELF_HOSTED, "canary-1b-flash", TRANSCRIPTION), + new(SELF_HOSTED, "nvidia/canary-180m-flash", TRANSCRIPTION), + + // + // Alibaba's speech line. Every one of these begins with the letter the Alibaba Cloud + // provider kept its whole chat list by, so all of them stood among the models somebody + // talks to. The last one carries two words at once, and the one which decides is not the + // longer one. + // + new(ALIBABA_CLOUD, "qwen3-asr-flash", TRANSCRIPTION), + new(ALIBABA_CLOUD, "qwen3-asr-1.7b", TRANSCRIPTION), + new(ALIBABA_CLOUD, "qwen-audio-3.0-asr-flash-streaming", TRANSCRIPTION), ]; /// @@ -97,6 +159,13 @@ public static class ModelKindCorpus // The one name which glues the word to something else, and the reason the three words are // not loosened into substrings: new(SELF_HOSTED, "xtts-v2", SPEECH_SYNTHESIS), + + new(ALIBABA_CLOUD, "qwen-tts", SPEECH_SYNTHESIS), + + // The Voxtral which speaks instead of listening. It stands here to hold the other half of + // Mistral's transcription list: the catalog is asked now, so what is not a transcription + // model has to be kept out by what it is, not by the list having been short. + new(MISTRAL, "voxtral-mini-tts-latest", SPEECH_SYNTHESIS), ]; /// @@ -111,6 +180,75 @@ public static class ModelKindCorpus new(OPEN_AI, "gpt-realtime-whisper", REALTIME), new(OPEN_AI, "gpt-live-1", REALTIME, AnsweredTodayAs: CHAT, Reason: "Found in the chat list while testing. The line which succeeds the realtime models dropped the word, and it is even less of a chat partner: it listens and speaks at once and leaves the thinking to a text model behind it."), + + // + // Alibaba builds the word into both of its speech lines, and both are here to hold the + // rank the realtime rule carries: a connection AI Studio cannot open stays out of every + // list, whether the model would otherwise have spoken or listened. + // + new(ALIBABA_CLOUD, "qwen-tts-realtime", REALTIME), + new(ALIBABA_CLOUD, "qwen3-asr-flash-realtime", REALTIME), + + // The other two of Mistral's Voxtral line. The second one carries "transcribe" and stays + // out of the transcription list all the same: whatever it does, it does over a connection + // the app cannot open, and that is what the rank on the realtime rule is for. + new(MISTRAL, "voxtral-mini-realtime-latest", REALTIME), + new(MISTRAL, "voxtral-mini-transcribe-realtime-2602", REALTIME), + + // + // Google's whole two-way line, taken off its catalog rather than written from memory. It + // uses the other word for the thing OpenAI calls realtime, so none of these was recognized + // and all four stood among the models to talk to -- which none of them can be. The last + // one translates between two people speaking, which is as far from a chat as it gets. + // + new(GOOGLE, "gemini-3.8-live", REALTIME), + new(GOOGLE, "gemini-3.8-live-extended-thinking", REALTIME), + new(GOOGLE, "gemini-3.1-flash-live-preview", REALTIME), + new(GOOGLE, "gemini-3.5-live-translate-preview", REALTIME), + + // Google's experimental music model, which is recognized already: it carries the other + // word, and this holds that the two rules do not fall out with each other. + new(GOOGLE, "lyria-realtime-exp", REALTIME), + ]; + + /// + /// The models which write music. + /// + /// + /// All four off Google's own catalog. They were never seen because the provider showed only + /// names beginning with "gemini" -- the same prefix which kept Gemma out, which is why it had + /// to go and why these needed a rule of their own before it could. + /// + private static readonly ModelKindExample[] MUSIC_ENTRIES = + [ + new(GOOGLE, "lyria-3.5", MUSIC_GENERATION), + new(GOOGLE, "lyria-3-pro-preview", MUSIC_GENERATION), + new(GOOGLE, "lyria-3-clip-preview", MUSIC_GENERATION), + ]; + + /// + /// The models which are handed a job instead of a message. + /// + /// + /// The last two are the point of binding these rules to Google. Perplexity sells deep research + /// as well, and what it sells is a chat model -- so the same two words have to mean different + /// things at different providers, which is exactly what a binding is for. + /// + private static readonly ModelKindExample[] AGENT_ENTRIES = + [ + new(GOOGLE, "deep-research-preview-04-2026", AGENT), + new(GOOGLE, "deep-research-max-preview-04-2026", AGENT), + new(GOOGLE, "deep-research-pro-preview-12-2025", AGENT), + new(GOOGLE, "antigravity-preview-05-2026", AGENT), + new(GOOGLE, "antigravity-preview-09-2026", AGENT), + ]; + + /// + /// The model which answers out of what it was handed, and says where the answer came from. + /// + private static readonly ModelKindExample[] GROUNDED_ANSWERING_ENTRIES = + [ + new(GOOGLE, "aqa", GROUNDED_ANSWERING), ]; /// @@ -122,13 +260,23 @@ public static class ModelKindCorpus ]; /// - /// The models from before chat completions existed. + /// The models which continue a text instead of answering in a conversation. /// private static readonly ModelKindExample[] TEXT_COMPLETION_ENTRIES = [ new(HELMHOLTZ, "text-davinci-003", TEXT_COMPLETION), + + // The model Mistral serves for filling a gap in a file. Its name does not begin with the + // word the provider used to sort its list by, so it stood among the chat models -- next to + // Codestral, which does the same job and was kept out by that very word. + new(MISTRAL, "mistral-code-fim-latest", TEXT_COMPLETION), new(OPEN_AI, "babbage-002", TEXT_COMPLETION), new(OPEN_AI, "gpt-3.5-turbo-instruct", TEXT_COMPLETION), + + // The one of these which is not old: Mistral serves Codestral to fill in the middle of a + // file. It says so only for Mistral's own catalog, which is why the open weights of the + // same name stay a chat model further down. + new(MISTRAL, "codestral-latest", TEXT_COMPLETION), ]; /// @@ -137,6 +285,7 @@ public static class ModelKindCorpus private static readonly ModelKindExample[] OCR_ENTRIES = [ new(MISTRAL, "mistral-ocr-latest", OCR), + new(ALIBABA_CLOUD, "qwen-vl-ocr", OCR), ]; /// @@ -173,6 +322,67 @@ public static class ModelKindCorpus new(SELF_HOSTED, "llama3.3:70b", CHAT), new(OPEN_AI, "gpt-5.1", CHAT), + // The open weights of the model Mistral itself serves to fill in the middle of a file. + // Whoever runs them runs them behind a chat completion API, so here the name means + // something to talk to -- which is what binding that other rule to Mistral protects. + new(SELF_HOSTED, "codestral-22b-v0.1", CHAT), + + // The other half of that same Ollama installation, and the pair which makes the point: + // qwen3.8 and qwen3-embedding are one family and two answers. A rule written to select + // rather than to modify would have to beat the family name to get there. + new(SELF_HOSTED, "qwen3.8:latest", CHAT), + new(SELF_HOSTED, "gpt-oss:latest", CHAT), + + // Half the chat models of the world carry this word, and one of the embedding names above + // is one letter longer than it. A name part is what keeps the two apart: + new(SELF_HOSTED, "mistral-7b-instruct", CHAT), + + // + // What somebody actually comes to Alibaba Cloud for, held here because the provider is + // about to stop keeping its chat list by the letter every one of these begins with. The + // last one translates rather than converses, and it does so through the chat completion + // API like the others, so this is where it belongs. + // + new(ALIBABA_CLOUD, "qwen3.8-max", CHAT), + new(ALIBABA_CLOUD, "qwq-plus", CHAT), + new(ALIBABA_CLOUD, "qvq-max", CHAT), + new(ALIBABA_CLOUD, "qwen-mt-turbo", CHAT), + + // + // Perplexity's whole catalog, which the app carries as a list because there is no route to + // ask. A list somebody picked by hand is not filtered at runtime -- a filter over it could + // only ever take a model away, never find one -- so it is held here instead: a rule which + // turns one of these into something other than a chat model fails the build rather than + // quietly emptying the dropdown. + // + new(PERPLEXITY, "sonar", CHAT), + new(PERPLEXITY, "sonar-pro", CHAT), + new(PERPLEXITY, "sonar-reasoning", CHAT), + new(PERPLEXITY, "sonar-reasoning-pro", CHAT), + new(PERPLEXITY, "sonar-deep-research", CHAT), + + // + // The other two deep research models of the world, and the reason Google's rule is bound + // to Google and written as a prefix. Perplexity and OpenAI both sell something under that + // name which answers over the API the app already speaks, so both stay chat models. Only + // Google's own line, whose names start with the words, is handed a job instead. + // + new(OPEN_AI, "o3-deep-research", CHAT), + new(OPEN_AI, "o4-mini-deep-research", CHAT), + + // + // The counter-sample to the three rules above, taken off the same three catalogs. Every + // one of these carries a word which now means something -- code, live, image -- without + // being what that word says, and every one of them has to stay a chat model. + // + new(MISTRAL, "mistral-code-latest", CHAT), + new(MISTRAL, "mistral-vibe-cli-latest", CHAT), + new(MISTRAL, "zai-glm-latest", CHAT), + new(GOOGLE, "gemma-4-31b-it", CHAT), + new(GOOGLE, "gemini-3.8-flash", CHAT), + new(X, "grok-4.6", CHAT), + new(X, "grok-build-0.1", CHAT), + // // Three which were questioned while testing and stay all the same. Grok Build is the coding // model behind the xAI CLI and answers like any other Grok. The Groq compound systems are @@ -199,6 +409,9 @@ public static class ModelKindCorpus ..TRANSCRIPTION_ENTRIES, ..SPEECH_ENTRIES, ..REALTIME_ENTRIES, + ..MUSIC_ENTRIES, + ..AGENT_ENTRIES, + ..GROUNDED_ANSWERING_ENTRIES, ..COMPUTER_USE_ENTRIES, ..TEXT_COMPLETION_ENTRIES, ..OCR_ENTRIES, diff --git a/app/Tests/Provider/SelfHostedModelListTests.cs b/app/Tests/Provider/SelfHostedModelListTests.cs new file mode 100644 index 00000000..21cd4989 --- /dev/null +++ b/app/Tests/Provider/SelfHostedModelListTests.cs @@ -0,0 +1,135 @@ +using System.Text.Json; + +using AIStudio.Provider; +using AIStudio.Settings; + +using SelfHostedModelsResponse = AIStudio.Provider.SelfHosted.ModelsResponse; + +namespace AIStudio.Tests.Provider; + +/// +/// Checks how the models of somebody's own server are sorted into the three lists they are offered in. +/// +/// +/// The engines answer one route with everything they serve, and that answer says nothing about what +/// any of it is made for: an ID, the word "model", and at Ollama a timestamp. Which list a model +/// ends up in is therefore decided afterwards, and for a long time it was decided by looking for +/// the word "embed" in the name -- the chat list was everything without it, the embedding list +/// everything with it, and the transcription list was not filtered at all. +/// +/// The body below is the real answer of a local Ollama, copied off the route rather than written +/// from memory, and it holds the two names which that reading got wrong. What it costs is visible +/// in the assertions: an embedding model in the chat list is one somebody picks and then waits for +/// an answer which never comes. +/// +[TestFixture] +public sealed class SelfHostedModelListTests +{ + private static readonly JsonSerializerOptions AS_THE_PROVIDERS_READ_IT = new() + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + }; + + /// + /// What "GET /v1/models" answers on a local Ollama, shortened to the fields it sends. + /// + private const string WHAT_A_LOCAL_OLLAMA_ANSWERS = + """ + { + "object": "list", + "data": [ + { "id": "all-minilm:latest", "object": "model", "created": 1789809739, "owned_by": "library" }, + { "id": "bge-m3:latest", "object": "model", "created": 1789809702, "owned_by": "library" }, + { "id": "qwen3-embedding:0.6b", "object": "model", "created": 1788699716, "owned_by": "library" }, + { "id": "qwen3-embedding:4b", "object": "model", "created": 1788699586, "owned_by": "library" }, + { "id": "qwen3-embedding:latest", "object": "model", "created": 1788699253, "owned_by": "library" }, + { "id": "qwen3.8:latest", "object": "model", "created": 1788698492, "owned_by": "library" }, + { "id": "gpt-oss:latest", "object": "model", "created": 1756645805, "owned_by": "library" } + ] + } + """; + + private static IReadOnlyList TheModelsTheEngineListed() + { + var response = JsonSerializer.Deserialize(WHAT_A_LOCAL_OLLAMA_ANSWERS, AS_THE_PROVIDERS_READ_IT); + Assert.That(response.Data, Is.Not.Null, "The answer has to be readable before anything can be sorted out of it."); + + return response.Data! + .Where(model => !string.IsNullOrWhiteSpace(model.Id)) + .Select(model => new Model(model.Id, null)) + .ToList(); + } + + [Test] + public void TheChatListHoldsWhatSomebodyCanTalkTo() + { + var chatModels = TheModelsTheEngineListed() + .Where(model => model.IsChatModel(LLMProviders.SELF_HOSTED)) + .Select(model => model.Id) + .ToList(); + + Assert.That(chatModels, Is.EquivalentTo(new[] { "qwen3.8:latest", "gpt-oss:latest" })); + } + + [Test] + public void TheEmbeddingListHoldsTheModelsWhichSayNothingAboutEmbedding() + { + var embeddingModels = TheModelsTheEngineListed() + .Where(model => model.IsEmbeddingModel(LLMProviders.SELF_HOSTED)) + .Select(model => model.Id) + .ToList(); + + Assert.Multiple(() => + { + Assert.That(embeddingModels, Does.Contain("bge-m3:latest"), "Named after the family which built it, with no word about what it does."); + Assert.That(embeddingModels, Does.Contain("all-minilm:latest"), "The same, and without the organization which used to be the only marker."); + Assert.That(embeddingModels, Has.Count.EqualTo(5), "The three Qwen embedding tags belong here as well, and nothing else does."); + }); + } + + [Test] + public void NothingIsInTwoListsAtOnce() + { + // + // The two lists were cut from one name with one word, so a model could only ever be in one + // of them. They are cut by two questions now, and two questions can both say yes. + // + var models = TheModelsTheEngineListed(); + var inBothLists = models + .Where(model => model.IsChatModel(LLMProviders.SELF_HOSTED) && model.IsEmbeddingModel(LLMProviders.SELF_HOSTED)) + .Select(model => model.Id) + .ToList(); + + Assert.That(inBothLists, Is.Empty); + } + + [Test] + public void AnEngineWithoutASpeechModelOffersNoneForTranscription() + { + // + // Ollama serves no speech-to-text model of its own, and the list said otherwise: it was + // handed through unfiltered, so all seven of these stood there to be picked. + // + var transcriptionModels = TheModelsTheEngineListed() + .Where(model => model.IsTranscriptionModel(LLMProviders.SELF_HOSTED)) + .ToList(); + + Assert.That(transcriptionModels, Is.Empty); + } + + [Test] + public void ASpeechModelOnSuchAServerIsOfferedForTranscription() + { + // + // The other half of the one above: the empty list has to come from there being no speech + // model, not from the question never saying yes on this provider. + // + var models = new[] { "whisper-large-v3", "faster-whisper-large-v3", "canary-1b-flash" } + .Select(id => new Model(id, null)) + .Where(model => model.IsTranscriptionModel(LLMProviders.SELF_HOSTED)) + .Select(model => model.Id) + .ToList(); + + Assert.That(models, Has.Count.EqualTo(3)); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/EmbeddingChangeImpactTests.cs b/app/Tests/Tools/EmbeddingChangeImpactTests.cs index dcea6626..53e91f80 100644 --- a/app/Tests/Tools/EmbeddingChangeImpactTests.cs +++ b/app/Tests/Tools/EmbeddingChangeImpactTests.cs @@ -47,6 +47,41 @@ public sealed class EmbeddingChangeImpactTests "Another model means another vector space."); } + /// + /// A model ID which differs only in how it is written is a different model here. + /// + /// + /// This is why the embedding provider dialog adds a configured model to the list it loaded + /// instead of matching it against that list. A server which writes the same model slightly + /// differently -- with a tag where the user typed none, or in another case -- would otherwise + /// have its spelling written into the settings on the next save, and every document of every + /// data source behind that provider would be prepared again for a change nobody made. + /// + [Test] + public void AModelIdWhichOnlyReadsDifferentlyDropsTheStoredIndexAsWell() + { + var dataSource = StoredDataSource(); + var stored = StoredEmbeddingProvider() with { Model = new("nomic-embed-text", null) }; + + Assert.Multiple(() => + { + Assert.That( + EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Model = new("nomic-embed-text:latest", null) }), + Is.True, + "The tag a server appends is part of the ID, and the ID is part of the signature."); + + Assert.That( + EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Model = new("NOMIC-EMBED-TEXT", null) }), + Is.True, + "Compared ordinally, so another case is another model rather than the same one written louder."); + + Assert.That( + EmbeddingChangeImpact.AffectsStoredIndex(dataSource, stored, stored with { Model = new("nomic-embed-text", "Nomic Embed Text") }), + Is.False, + "The display name is decoration and reaches no vector, so loading the list may fill it in."); + }); + } + [Test] public void ChangingTheTokenLimitDropsTheStoredIndex() {