AI-Studio/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@using AIStudio.Settings.DataModel
@using AIStudio.Tools.Validation
@using AIStudio.Provider
@inherits MSGComponentBase

<MudDialog>
    <DialogContent>
        @* A drop anywhere in this dialog belongs to the dialog, not to the page behind it: *@
        <PathDropZone IsArea="@true">
        <MudForm @ref="@this.form" @bind-IsValid="@this.dataIsValid" @bind-Errors="@this.dataIssues" FieldChanged="@this.RevalidateAfterFieldChange">
            <DataSourceMissingEmbeddingWarning IsInDataSourceDialog="@true" EmbeddingProviderAdded="@this.SelectAddedEmbeddingProvider"/>

            @* ReSharper disable once CSharpWarnings::CS8974 *@
            <MudTextField
                T="string"
                @bind-Text="@this.dataName"
                Label="@T("Data Source Name")"
                Class="mb-6"
                MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
                Counter="@DataSourceValidation.MAX_NAME_LENGTH"
                Immediate="@true"
                Validation="@this.dataSourceValidation.ValidatingName"
                Adornment="Adornment.Start"
                AdornmentIcon="@Icons.Material.Filled.Lightbulb"
                AdornmentColor="Color.Info"
                UserAttributes="@SPELLCHECK_ATTRIBUTES"
                Variant="Variant.Outlined"
            />

            <MudTextField
                T="string"
                @bind-Text="@this.dataDescription"
                Label="@T("Description")"
                Class="mb-6"
                MaxLength="200"
                Counter="200"
                Immediate="@true"
                Adornment="Adornment.Start"
                AdornmentIcon="@Icons.Material.Filled.Description"
                AdornmentColor="Color.Info"
                HelperText="@T("Describe what data this directory contains to help the AI select it.")"
                UserAttributes="@SPELLCHECK_ATTRIBUTES"
                Variant="Variant.Outlined"
                Lines="3"
            />

            <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                @T("Select a root directory for this data source.")
            </MudJustifiedText>
            @if (!this.CanChangeSource)
            {
                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
                    @T("The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed.")
                </MudAlert>
            }
            @if (this.CanChangeSource)
            {
                <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" EnableDragDrop="true" CatchAllDocuments="true" />
            }
            else
            {
                <MudTextField
                    T="string"
                    Text="@this.dataPath"
                    Label="@T("Selected base directory for this data source")"
                    Class="mb-3"
                    ReadOnly="@true"
                    Validation="@this.dataSourceValidation.ValidatePath"
                    Adornment="Adornment.Start"
                    AdornmentIcon="@Icons.Material.Filled.Folder"
                    UserAttributes="@SPELLCHECK_ATTRIBUTES"
                    Variant="Variant.Outlined"
                />
            }

            <MudTextSwitch
                Label="@T("Include subfolders?")"
                Value="@this.dataIncludeSubdirectories"
                ValueChanged="@(value => this.dataIncludeSubdirectories = value)"
                LabelOn="@T("Yes, index files in subfolders as well")"
                LabelOff="@T("No, only index files in the selected folder")"
            />

            <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                @T("Select the file patterns this data source should index.")
            </MudJustifiedText>

            <MudTextField
                T="string"
                @bind-Text="@this.dataFilePatterns"
                Label="@T("File patterns")"
                Class="mb-3"
                Immediate="@true"
                Validation="@this.dataSourceValidation.ValidateFilePatterns"
                HelperText="@T("Which files should be indexed? Separate multiple patterns with a semicolon, e.g., *.txt;*.pdf;*.docx;.gitignore. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file. Audio and video files, e.g., *.mp3 or *.mp4, are transcribed by your transcription provider.")"
                Adornment="Adornment.Start"
                AdornmentIcon="@Icons.Material.Filled.FilterAlt"
                AdornmentColor="Color.Info"
                UserAttributes="@SPELLCHECK_ATTRIBUTES"
                Variant="Variant.Outlined"
            />

            <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding provider.")
            </MudJustifiedText>
            <MudSelect @ref="@this.embeddingSelect" @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
                @foreach (var provider in this.SettingsManager.GetAllEmbeddingProviders())
                {
                    <MudSelectItem Value="@provider.Id">
                        <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@provider.Name" />
                    </MudSelectItem>
                }
            </MudSelect>
            
            @if (this.CurrentCloudProcessing is not CloudProcessing.NONE)
            {
                <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" SendsEmbeddingsToCloud="@this.SelectedCloudEmbedding" CloudTranscriptionProvider="@this.CloudTranscriptionProviderName" @bind-UserAcknowledged="@this.UserAcknowledgedCloudProcessing" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
            }
            else if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
            {
                <MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
                    @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
                </MudJustifiedText>
            }

            <ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>

            <MudSelect @ref="@this.confidenceLevelSelect" @bind-Value="@this.dataConfidenceLevel" Text="@this.dataConfidenceLevel.GetName()" Label="@T("Required provider confidence level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateDataSourceConfidenceLevel">
                @foreach (var level in this.ConfidenceLevels)
                {
                    <MudSelectItem Value="@level.Value">
                        @level.Name
                    </MudSelectItem>
                }
            </MudSelect>

            @if (this.MediaTranscriptionHint is { } mediaTranscriptionHint)
            {
                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Dense="@true" Class="mb-3">
                    @mediaTranscriptionHint
                </MudAlert>
            }

            <MudStack Class="mb-3">
                <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
                    @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
                </MudButton>
                <MudDivider/>
                <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
                    <MudTextSwitch
                        Label="@T("Include hidden files and folders?")"
                        Value="@this.dataIncludeHiddenFilesAndFolders"
                        ValueChanged="@(value => this.dataIncludeHiddenFilesAndFolders = value)"
                        LabelOn="@T("Yes, index hidden files and folders as well")"
                        LabelOff="@T("No, ignore hidden files and folders")"
                    />
                    <MudJustifiedText Typo="Typo.body2" Class="mb-3">
                        @T("Hidden are files and folders whose names start with a dot, such as .git or .env, and those hidden by the operating system. A file pattern which starts with a dot, such as .gitignore, selects such files anyway.")
                    </MudJustifiedText>
                    @* Without an icon: in a field of several lines, it floats in the middle of the text. *@
                    <MudTextField
                        T="string"
                        @bind-Text="@this.dataExcludePatterns"
                        Label="@T("Excluded files and folders")"
                        Lines="@FileExclusions.DEFAULT_LINE_COUNT"
                        AutoGrow="@true"
                        MaxLines="@FileExclusions.MAX_LINE_COUNT"
                        Immediate="@true"
                        Validation="@this.dataSourceValidation.ValidateExcludePatterns"
                        HelperText="@T("These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. Other entries name files, e.g., *.min.js.")"
                        UserAttributes="@SPELLCHECK_ATTRIBUTES"
                        Variant="Variant.Outlined"
                    />
                    <RestoreDefaultButton Text="@T("Restore default")" OnClick="@this.RestoreDefaultExcludePatterns" Class="mb-3"/>
                    <MudNumericField
                        T="int"
                        @bind-Value="@this.dataMaxFileSizeMegabytes"
                        Label="@T("Ignore files larger than")"
                        Class="mb-3"
                        Min="@FolderFileSelection.SMALLEST_MAX_FILE_SIZE_MEGABYTES"
                        Immediate="@true"
                        HelperText="@T("Selected files above this size are not indexed, e.g., images of virtual machines. The embedding page lists them, so you can raise the limit when needed.")"
                        Variant="Variant.Outlined"
                        Adornment="Adornment.End"
                        AdornmentText="MB"/>
                    @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
                    {
                        <MudTextField
                            T="string"
                            Text="@this.SelectedEmbeddingTokenizerText"
                            Label="@T("Tokenizer")"
                            Class="mb-3"
                            ReadOnly="@true"
                            Variant="Variant.Outlined"
                            Adornment="Adornment.Start"
                            AdornmentIcon="@Icons.Material.Filled.TextFields"
                            AdornmentColor="Color.Info"/>
                    }
                    <MudJustifiedText Typo="Typo.body1" Class="mb-3">
                        @T("Optional expert settings for how this data source is split before embedding.")
                    </MudJustifiedText>
                    <MudNumericField
                        @ref="@this.maxChunkTokenLengthField"
                        T="int"
                        @bind-Value="@this.dataMaxChunkTokenLength"
                        Label="@T("Token limit")"
                        Class="mb-3"
                        Min="1"
                        Immediate="@true"
                        Validation="@this.ValidateMaxChunkTokenLength"
                        HelperText="@this.MaxChunkTokenLengthHelperText"
                        Variant="Variant.Outlined"
                        Adornment="Adornment.Start"
                        AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
                        AdornmentColor="Color.Info"/>
                    <MudNumericField
                        @ref="@this.chunkOverlapTokenLengthField"
                        T="int"
                        @bind-Value="@this.dataChunkOverlapTokenLength"
                        Label="@T("Token overlap")"
                        Min="0"
                        Immediate="@true"
                        Validation="@this.ValidateChunkOverlapTokenLength"
                        HelperText="@this.ChunkOverlapTokenLengthHelperText"
                        Variant="Variant.Outlined"
                        Adornment="Adornment.Start"
                        AdornmentIcon="@Icons.Material.Filled.CompareArrows"
                        AdornmentColor="Color.Info"/>
                    <MudNumericField
                        T="ushort"
                        Min="10"
                        @bind-Value="@this.dataMaxMatches"
                        Label="@T("How many matches do you want at most per query?")"
                        Variant="Variant.Outlined"
                        Step="10"
                        Adornment="Adornment.Start"
                        AdornmentIcon="@Icons.Material.Filled.Search"
                        AdornmentColor="Color.Info"/>
                </MudCollapse>
            </MudStack>
        </MudForm>
        <Issues IssuesData="@this.dataIssues"/>
        </PathDropZone>
    </DialogContent>
    <DialogActions>
        <MudButton OnClick="@this.Cancel" Variant="Variant.Filled">
            @T("Cancel")
        </MudButton>
        <MudButton OnClick="@this.Store" Variant="Variant.Filled" Color="Color.Primary">
            @if(this.IsEditing)
            {
                @T("Update")
            }
            else
            {
                @T("Add")
            }
        </MudButton>
    </DialogActions>
</MudDialog>