MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-08-15 19:02:10 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

showed user default values in expert settings for datasources

Browse Source
This commit is contained in:
PaulKoudelka 2026-08-14 16:02:40 +02:00
parent 15afe1c80c
commit aafc58b31a
9 changed files with 118 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

60
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View File

@ -5008,12 +5008,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1312296210"
-- No embedding selected
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1359179968"] = "No embedding selected"
-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
-- Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1588814044"] = "Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens."
-- Tokenizer
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1696723386"] = "Tokenizer"
-- Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1720021383"] = "Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens."
-- Description
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1725856265"] = "Description"
@ -5035,24 +5038,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2101951526"
-- Required provider confidence level
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T236253137"] = "Required provider confidence level"
-- Please enter a token limit of at least 1.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2406580478"] = "Please enter a token limit of at least 1."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add"
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
-- The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2834874159"] = "The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."
-- Embedding
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"] = "Embedding"
-- Token limit
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2961294165"] = "Token limit"
-- Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3181127959"] = "Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap."
-- Please enter 0 or a positive overlap length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
@ -5065,12 +5065,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"
-- Unknown
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3424652889"] = "Unknown"
-- Please enter 0 or a positive token limit.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3659673500"] = "Please enter 0 or a positive token limit."
-- Select the base directory
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory"
-- The data source token limit must not be larger than the embedding provider token limit ({0}).
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T787118522"] = "The data source token limit must not be larger than the embedding provider token limit ({0})."
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name"
@ -5167,12 +5167,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1220918127"] = "
-- No embedding selected
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1359179968"] = "No embedding selected"
-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
-- Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1588814044"] = "Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens."
-- Tokenizer
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1696723386"] = "Tokenizer"
-- Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1720021383"] = "Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens."
-- Description
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1725856265"] = "Description"
@ -5191,15 +5194,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2101951526"] = "
-- Required provider confidence level
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T236253137"] = "Required provider confidence level"
-- Please enter a token limit of at least 1.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2406580478"] = "Please enter a token limit of at least 1."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add"
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
-- The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2834874159"] = "The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."
-- Embedding
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2838542994"] = "Embedding"
@ -5209,9 +5212,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = "
-- Token limit
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2961294165"] = "Token limit"
-- Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3181127959"] = "Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap."
-- Please enter 0 or a positive overlap length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
@ -5224,15 +5224,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "
-- Unknown
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3424652889"] = "Unknown"
-- Please enter 0 or a positive token limit.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3659673500"] = "Please enter 0 or a positive token limit."
-- Select the file
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file"
-- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model."
-- The data source token limit must not be larger than the embedding provider token limit ({0}).
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "The data source token limit must not be larger than the embedding provider token limit ({0})."
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name"
@ -9833,19 +9833,25 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGININSTALLSERVICE::T8163
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::ASSISTANTPLUGININSTALLSERVICE::T939708112"] = "The generated assistant plugin is invalid. Issue: {0}"
-- Running
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGMODELS::T1160324588"] = "Running"
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T1160324588"] = "Running"
-- Idle
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGMODELS::T1168775091"] = "Idle"
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T1168775091"] = "Idle"
-- Needs attention
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGMODELS::T1566837660"] = "Needs attention"
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T1566837660"] = "Needs attention"
-- Queued
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGMODELS::T2655222900"] = "Queued"
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T2655222900"] = "Queued"
-- Completed
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGMODELS::T3968379570"] = "Completed"
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T3968379570"] = "Completed"
-- Chunk {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T2544251224"] = "Chunk {0}"
-- Page {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Page {0}"
-- The voice recording shortcut currently works only while AI Studio is focused.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::GLOBALSHORTCUTSERVICE::T1204510649"] = "The voice recording shortcut currently works only while AI Studio is focused."

6
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor
View File

@ -150,10 +150,10 @@
@bind-Value="@this.dataMaxChunkTokenLength"
Label="@T("Token limit")"
Class="mb-3"
Min="0"
Min="1"
Immediate="@true"
Validation="@this.ValidateMaxChunkTokenLength"
HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")"
HelperText="@this.MaxChunkTokenLengthHelperText"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
@ -165,7 +165,7 @@
Min="0"
Immediate="@true"
Validation="@this.ValidateChunkOverlapTokenLength"
HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap.")"
HelperText="@this.ChunkOverlapTokenLengthHelperText"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.CompareArrows"

41
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs
View File

@ -2,6 +2,7 @@ using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
using Microsoft.AspNetCore.Components;
@ -46,7 +47,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
private string dataEmbeddingId = string.Empty;
private string dataPath = string.Empty;
private int dataMaxChunkTokenLength;
private int dataChunkOverlapTokenLength;
private int dataChunkOverlapTokenLength = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
private ushort dataMaxMatches = 10;
private bool showExpertSettings;
private ConfidenceLevel dataConfidenceLevel = ConfidenceLevel.UNKNOWN;
@ -91,7 +92,6 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
this.dataConfidenceLevel = this.DataSource.ConfidenceLevel;
this.dataMaxMatches = this.DataSource.MaxMatches;
this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
}
await base.OnInitializedAsync();
@ -133,6 +133,16 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
? T("Default tokenizer")
: System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
private int ProviderMaxChunkTokenLength => this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
private string MaxChunkTokenLengthHelperText => string.Format(
T("Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens."),
this.ProviderMaxChunkTokenLength);
private string ChunkOverlapTokenLengthHelperText => string.Format(
T("Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens."),
DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH);
private DataSourceLocalDirectory CreateDataSource() => new()
{
Id = this.dataId,
@ -164,31 +174,42 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
{
if (maxChunkTokenLength < 0)
return T("Please enter 0 or a positive token limit.");
if (!this.showExpertSettings)
return null;
var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength)
return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength);
if (maxChunkTokenLength < 1)
return T("Please enter a token limit of at least 1.");
var providerMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
if (maxChunkTokenLength > providerMaxChunkTokenLength)
return string.Format(T("The data source token limit must not be larger than the embedding provider token limit ({0})."), providerMaxChunkTokenLength);
return null;
}
private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
{
if (!this.showExpertSettings)
return null;
if (chunkOverlapTokenLength < 0)
return T("Please enter 0 or a positive overlap length.");
var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0
var effectiveMaxChunkTokenLength = this.showExpertSettings && this.dataMaxChunkTokenLength > 0
? this.dataMaxChunkTokenLength
: this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
: this.ProviderMaxChunkTokenLength;
if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
return T("The overlap must be smaller than the effective token limit.");
return null;
}
private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
private void ToggleExpertSettings()
{
this.showExpertSettings = !this.showExpertSettings;
if (this.showExpertSettings && this.dataMaxChunkTokenLength < 1)
this.dataMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
}
private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
}

6
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor
View File

@ -150,10 +150,10 @@
@bind-Value="@this.dataMaxChunkTokenLength"
Label="@T("Token limit")"
Class="mb-3"
Min="0"
Min="1"
Immediate="@true"
Validation="@this.ValidateMaxChunkTokenLength"
HelperText="@T("Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.")"
HelperText="@this.MaxChunkTokenLengthHelperText"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
@ -165,7 +165,7 @@
Min="0"
Immediate="@true"
Validation="@this.ValidateChunkOverlapTokenLength"
HelperText="@T("Number of tokens repeated at the start of the next chunk. Use 0 to use the default overlap.")"
HelperText="@this.ChunkOverlapTokenLengthHelperText"
Variant="Variant.Outlined"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.CompareArrows"

41
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs
View File

@ -2,6 +2,7 @@ using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
using Microsoft.AspNetCore.Components;
@ -46,7 +47,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
private string dataEmbeddingId = string.Empty;
private string dataFilePath = string.Empty;
private int dataMaxChunkTokenLength;
private int dataChunkOverlapTokenLength;
private int dataChunkOverlapTokenLength = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
private ushort dataMaxMatches = 10;
private bool showExpertSettings;
private ConfidenceLevel dataConfidenceLevel = ConfidenceLevel.UNKNOWN;
@ -91,7 +92,6 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
this.dataConfidenceLevel = this.DataSource.ConfidenceLevel;
this.dataMaxMatches = this.DataSource.MaxMatches;
this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
}
await base.OnInitializedAsync();
@ -133,6 +133,16 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
? T("Default tokenizer")
: System.IO.Path.GetFileName(this.SelectedEmbedding.TokenizerPath);
private int ProviderMaxChunkTokenLength => this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
private string MaxChunkTokenLengthHelperText => string.Format(
T("Maximum number of tokens per chunk for this data source. The embedding provider default is {0} tokens."),
this.ProviderMaxChunkTokenLength);
private string ChunkOverlapTokenLengthHelperText => string.Format(
T("Number of tokens repeated at the start of the next chunk. The default overlap is {0} tokens."),
DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH);
private DataSourceLocalFile CreateDataSource() => new()
{
Id = this.dataId,
@ -164,31 +174,42 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
private string? ValidateMaxChunkTokenLength(int maxChunkTokenLength)
{
if (maxChunkTokenLength < 0)
return T("Please enter 0 or a positive token limit.");
if (!this.showExpertSettings)
return null;
var providerMaxChunkTokenLength = this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
if (maxChunkTokenLength > 0 && maxChunkTokenLength >= providerMaxChunkTokenLength)
return string.Format(T("The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."), providerMaxChunkTokenLength);
if (maxChunkTokenLength < 1)
return T("Please enter a token limit of at least 1.");
var providerMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
if (maxChunkTokenLength > providerMaxChunkTokenLength)
return string.Format(T("The data source token limit must not be larger than the embedding provider token limit ({0})."), providerMaxChunkTokenLength);
return null;
}
private string? ValidateChunkOverlapTokenLength(int chunkOverlapTokenLength)
{
if (!this.showExpertSettings)
return null;
if (chunkOverlapTokenLength < 0)
return T("Please enter 0 or a positive overlap length.");
var effectiveMaxChunkTokenLength = this.dataMaxChunkTokenLength > 0
var effectiveMaxChunkTokenLength = this.showExpertSettings && this.dataMaxChunkTokenLength > 0
? this.dataMaxChunkTokenLength
: this.SelectedEmbedding?.EffectiveTokenLimit ?? EmbeddingProvider.DEFAULT_TOKEN_LIMIT;
: this.ProviderMaxChunkTokenLength;
if (chunkOverlapTokenLength >= effectiveMaxChunkTokenLength)
return T("The overlap must be smaller than the effective token limit.");
return null;
}
private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings;
private void ToggleExpertSettings()
{
this.showExpertSettings = !this.showExpertSettings;
if (this.showExpertSettings && this.dataMaxChunkTokenLength < 1)
this.dataMaxChunkTokenLength = this.ProviderMaxChunkTokenLength;
}
private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty;
}

2
app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs
View File

@ -39,7 +39,7 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
public int MaxChunkTokenLength { get; init; }
/// <inheritdoc />
public int ChunkOverlapTokenLength { get; init; }
public int ChunkOverlapTokenLength { get; init; } = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
/// <inheritdoc />
public ConfidenceLevel ConfidenceLevel { get; init; } = ConfidenceLevel.UNKNOWN;

2
app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs
View File

@ -39,7 +39,7 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
public int MaxChunkTokenLength { get; init; }
/// <inheritdoc />
public int ChunkOverlapTokenLength { get; init; }
public int ChunkOverlapTokenLength { get; init; } = DataSourceEmbeddingService.DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
/// <inheritdoc />
public ConfidenceLevel ConfidenceLevel { get; init; } = ConfidenceLevel.UNKNOWN;

9
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
View File

@ -14,7 +14,7 @@ namespace AIStudio.Tools.Services;
public sealed partial class DataSourceEmbeddingService
{
private const string OFFICE_LOCK_FILE_PREFIX = "~$";
private const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300;
internal const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300;
private const bool IMAGE_EMBEDDING_ENABLED = false;
private enum RagFileIndexingDecision
@ -542,11 +542,8 @@ public sealed partial class DataSourceEmbeddingService
var configuredOverlapTokenLength = dataSource is IInternalDataSource overlapDataSource
? overlapDataSource.ChunkOverlapTokenLength
: 0;
var requestedOverlapTokenLength = configuredOverlapTokenLength > 0
? configuredOverlapTokenLength
: DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH;
var overlapTokenLength = Math.Clamp(requestedOverlapTokenLength, 0, Math.Max(0, maxChunkTokenLength - 1));
var overlapTokenLength = Math.Clamp(configuredOverlapTokenLength, 0, Math.Max(0, maxChunkTokenLength - 1));
return new(maxChunkTokenLength, overlapTokenLength);
}
@ -948,7 +945,7 @@ public sealed partial class DataSourceEmbeddingService
embeddingProvider.EffectiveTokenLimit,
GetDataSourceConfidenceLevel(dataSource).ToString(),
dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : 0,
dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH,
chunkingOptions.MaxChunkTokenLength,
chunkingOptions.OverlapTokenLength);
}

18
app/MindWork AI Studio/Tools/Services/DataSourceLocalRetrievalService.cs
View File

@ -5,6 +5,7 @@ using AIStudio.Settings.DataModel;
using AIStudio.Tools.Databases;
using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.Databases.VectorStore;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG;
using AIStudio.Tools.Rust;
@ -14,6 +15,8 @@ public sealed class DataSourceLocalRetrievalService(
SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider,
ILogger<DataSourceLocalRetrievalService> logger)
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceLocalRetrievalService).Namespace, nameof(DataSourceLocalRetrievalService));
private enum RetrievalChannel
{
VECTOR,
@ -340,13 +343,16 @@ public sealed class DataSourceLocalRetrievalService(
private static string BuildReferenceTitle(LocalRetrievalHit hit)
{
var sourceName = FirstNonEmpty(hit.FileName, hit.DataSourceName);
return BuildChunkTitle(sourceName, hit.ChunkIndex, hit.PageNumber);
return BuildLocatedReferenceTitle(sourceName, hit.ChunkIndex, hit.PageNumber);
}
private static string BuildChunkTitle(string sourceName, int chunkIndex, int? pageNumber)
private static string BuildLocatedReferenceTitle(string sourceName, int chunkIndex, int? pageNumber)
{
var page = pageNumber is > 0 ? $", page {pageNumber}" : string.Empty;
return $"{sourceName} (chunk {chunkIndex + 1}{page})";
var location = pageNumber is > 0
? string.Format(TB("Page {0}"), pageNumber)
: string.Format(TB("Chunk {0}"), chunkIndex + 1);
return $"{sourceName} ({location})";
}
private static string BuildReferenceLink(string path, LocalRetrievalHit hit)
@ -412,7 +418,7 @@ public sealed class DataSourceLocalRetrievalService(
result.Result.ParentFileId,
result.Result.FileName,
FirstNonEmpty(result.Result.AbsolutePath, result.Result.FilePath),
BuildChunkTitle(FirstNonEmpty(result.Result.FileName, dataSource.Name), result.Result.ChunkIndex, result.Result.PageNumber));
BuildLocatedReferenceTitle(FirstNonEmpty(result.Result.FileName, dataSource.Name), result.Result.ChunkIndex, result.Result.PageNumber));
}
}
@ -436,7 +442,7 @@ public sealed class DataSourceLocalRetrievalService(
result.Result.ParentFileId,
result.Result.FileName,
result.Result.AbsolutePath,
BuildChunkTitle(FirstNonEmpty(result.Result.FileName, dataSource.Name), result.Result.ChunkIndex, result.Result.PageNumber));
BuildLocatedReferenceTitle(FirstNonEmpty(result.Result.FileName, dataSource.Name), result.Result.ChunkIndex, result.Result.PageNumber));
}
}
}