mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-09-27 12:13:37 +00:00
fe35630eff
Resolved 29 conflicting files. The notable decisions: Confidence: main's tool-calling gate (RequiredProviderConfidence) and this branch's local-RAG gate (DataConfidenceLevel) turned out to be the same rule on the same axis, so they are now one field. Both tool results and data sources raise it through RequireProviderConfidence(). The gate checks the level strictly and no longer exempts providers trusted by configuration: TrustedProviderIds is documented as applying to data-source security checks only, and organizations set confidence through DataConfidence .CustomConfidenceScheme instead. The security axis (DataSecurity, ERI, IsTrustedForDataSourceSecurityChecks) is unchanged. Provider creation: main's CreateProvider signature won (hfEndpointKind, capabilityOverrides, no model parameter); tokenizerPath was added to it and is set for every provider, including the new Hetzner, IONOS and LiteLLM. Provider and EmbeddingProvider combine the record parameters, Lua parsing and Lua serialization of both sides. File types: main's hierarchy (ODT leaf, WORD parent, PowerPoint without the legacy .ppt, TABULAR instead of DELIMITED_TABLE) plus this branch's SPREADSHEET parent with ODS and the xlsm/xlsb/xla/xlam extensions, which the runtime already reads. Both sides had added a conflicting HTML filter; the reading family keeps the name, and the export path uses a narrow HTML_DOCUMENT, following the existing LATEX/TEX split. Runtime: main's file_data.rs is the base, including the prompt-injection sanitizer and the extraction routes. Token counting and chunk segmentation moved into take_released, so they act on the text the filter has released rather than on text it is still holding. A failed count is logged and left out instead of ending the extraction, because the app counts such a segment itself. Data sources: the participating-provider checks of this branch are kept, and main's GetAllowedDataSources overload now builds on them. DirectChatService resolves the launched chat's data source options before the check, so filter and chat see the same options. .NET and Rust both build clean; I18N regenerated to 4060 keys.
211 lines
10 KiB
Plaintext
211 lines
10 KiB
Plaintext
@using AIStudio.Settings.DataModel
|
|
@using AIStudio.Tools.Validation
|
|
@using AIStudio.Provider
|
|
@inherits MSGComponentBase
|
|
|
|
<MudDialog>
|
|
<DialogContent>
|
|
<MudForm @ref="@this.form" @bind-IsValid="@this.dataIsValid" @bind-Errors="@this.dataIssues">
|
|
@* ReSharper disable once CSharpWarnings::CS8974 *@
|
|
<MudTextField
|
|
T="string"
|
|
@bind-Text="@this.dataName"
|
|
Label="@T("Data Source Name")"
|
|
Class="mb-6"
|
|
MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
|
|
Counter="@DataSourceValidation.MAX_NAME_LENGTH"
|
|
Immediate="@true"
|
|
Validation="@this.dataSourceValidation.ValidatingName"
|
|
Adornment="Adornment.Start"
|
|
AdornmentIcon="@Icons.Material.Filled.Lightbulb"
|
|
AdornmentColor="Color.Info"
|
|
UserAttributes="@SPELLCHECK_ATTRIBUTES"
|
|
Variant="Variant.Outlined"
|
|
/>
|
|
|
|
<MudTextField
|
|
T="string"
|
|
@bind-Text="@this.dataDescription"
|
|
Label="@T("Description")"
|
|
Class="mb-6"
|
|
MaxLength="200"
|
|
Counter="200"
|
|
Immediate="@true"
|
|
Adornment="Adornment.Start"
|
|
AdornmentIcon="@Icons.Material.Filled.Description"
|
|
AdornmentColor="Color.Info"
|
|
HelperText="@T("Describe what data this directory contains to help the AI select it.")"
|
|
UserAttributes="@SPELLCHECK_ATTRIBUTES"
|
|
Variant="Variant.Outlined"
|
|
Lines="3"
|
|
/>
|
|
|
|
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
|
|
@T("Select a root directory for this data source. All data in this directory and all its subdirectories will be processed for this data source.")
|
|
</MudJustifiedText>
|
|
@if (!this.CanChangeSourceAndEmbedding)
|
|
{
|
|
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
|
|
@T("This data source already has indexed embeddings. Delete and recreate it to change the folder path or embedding model.")
|
|
</MudAlert>
|
|
}
|
|
@if (this.CanChangeSourceAndEmbedding)
|
|
{
|
|
<SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" />
|
|
}
|
|
else
|
|
{
|
|
<MudTextField
|
|
T="string"
|
|
Text="@this.dataPath"
|
|
Label="@T("Selected base directory for this data source")"
|
|
Class="mb-3"
|
|
ReadOnly="@true"
|
|
Validation="@this.dataSourceValidation.ValidatePath"
|
|
Adornment="Adornment.Start"
|
|
AdornmentIcon="@Icons.Material.Filled.Folder"
|
|
UserAttributes="@SPELLCHECK_ATTRIBUTES"
|
|
Variant="Variant.Outlined"
|
|
/>
|
|
}
|
|
|
|
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
|
|
@T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.")
|
|
</MudJustifiedText>
|
|
@if (this.CanChangeSourceAndEmbedding)
|
|
{
|
|
<MudSelect @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
|
|
@foreach (var embedding in this.AvailableEmbeddings)
|
|
{
|
|
<MudSelectItem Value="@embedding.Value">
|
|
@if (this.GetEmbeddingProvider(embedding.Value) is { } provider)
|
|
{
|
|
<ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@embedding.Name" />
|
|
}
|
|
else
|
|
{
|
|
@embedding.Name
|
|
}
|
|
</MudSelectItem>
|
|
}
|
|
</MudSelect>
|
|
}
|
|
else
|
|
{
|
|
<MudTextField
|
|
T="string"
|
|
Text="@this.SelectedEmbeddingNameText"
|
|
Label="@T("Embedding")"
|
|
Class="mb-3"
|
|
ReadOnly="@true"
|
|
Adornment="Adornment.Start"
|
|
AdornmentIcon="@Icons.Material.Filled.DataObject"
|
|
AdornmentColor="Color.Info"
|
|
Variant="Variant.Outlined"
|
|
/>
|
|
}
|
|
|
|
@if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
|
|
{
|
|
if (this.SelectedCloudEmbedding)
|
|
{
|
|
<DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
|
|
}
|
|
else
|
|
{
|
|
<MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
|
|
@T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
|
|
</MudJustifiedText>
|
|
}
|
|
}
|
|
|
|
<ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
|
|
|
|
<MudSelect @bind-Value="@this.dataConfidenceLevel" Text="@this.dataConfidenceLevel.GetName()" Label="@T("Required provider confidence level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateDataSourceConfidenceLevel">
|
|
@foreach (var level in this.ConfidenceLevels)
|
|
{
|
|
<MudSelectItem Value="@level.Value">
|
|
@level.Name
|
|
</MudSelectItem>
|
|
}
|
|
</MudSelect>
|
|
|
|
<MudStack Class="mb-3">
|
|
<MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
|
|
@(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
|
|
</MudButton>
|
|
<MudDivider/>
|
|
<MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
|
|
@if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
|
|
{
|
|
<MudTextField
|
|
T="string"
|
|
Text="@this.SelectedEmbeddingTokenizerText"
|
|
Label="@T("Tokenizer")"
|
|
Class="mb-3"
|
|
ReadOnly="@true"
|
|
Variant="Variant.Outlined"
|
|
Adornment="Adornment.Start"
|
|
AdornmentIcon="@Icons.Material.Filled.TextFields"
|
|
AdornmentColor="Color.Info"/>
|
|
}
|
|
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
|
|
@T("Optional expert settings for how this data source is split before embedding.")
|
|
</MudJustifiedText>
|
|
<MudNumericField
|
|
T="int"
|
|
@bind-Value="@this.dataMaxChunkTokenLength"
|
|
Label="@T("Token limit")"
|
|
Class="mb-3"
|
|
Min="1"
|
|
Immediate="@true"
|
|
Validation="@this.ValidateMaxChunkTokenLength"
|
|
HelperText="@this.MaxChunkTokenLengthHelperText"
|
|
Variant="Variant.Outlined"
|
|
Adornment="Adornment.Start"
|
|
AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
|
|
AdornmentColor="Color.Info"/>
|
|
<MudNumericField
|
|
T="int"
|
|
@bind-Value="@this.dataChunkOverlapTokenLength"
|
|
Label="@T("Token overlap")"
|
|
Min="0"
|
|
Immediate="@true"
|
|
Validation="@this.ValidateChunkOverlapTokenLength"
|
|
HelperText="@this.ChunkOverlapTokenLengthHelperText"
|
|
Variant="Variant.Outlined"
|
|
Adornment="Adornment.Start"
|
|
AdornmentIcon="@Icons.Material.Filled.CompareArrows"
|
|
AdornmentColor="Color.Info"/>
|
|
<MudNumericField
|
|
T="ushort"
|
|
Min="10"
|
|
@bind-Value="@this.dataMaxMatches"
|
|
Label="@T("How many matches do you want at most per query?")"
|
|
Variant="Variant.Outlined"
|
|
Step="10"
|
|
Adornment="Adornment.Start"
|
|
AdornmentIcon="@Icons.Material.Filled.Search"
|
|
AdornmentColor="Color.Info"/>
|
|
</MudCollapse>
|
|
</MudStack>
|
|
</MudForm>
|
|
<Issues IssuesData="@this.dataIssues"/>
|
|
</DialogContent>
|
|
<DialogActions>
|
|
<MudButton OnClick="@this.Cancel" Variant="Variant.Filled">
|
|
@T("Cancel")
|
|
</MudButton>
|
|
<MudButton OnClick="@this.Store" Variant="Variant.Filled" Color="Color.Primary">
|
|
@if(this.IsEditing)
|
|
{
|
|
@T("Update")
|
|
}
|
|
else
|
|
{
|
|
@T("Add")
|
|
}
|
|
</MudButton>
|
|
</DialogActions>
|
|
</MudDialog>
|