AI-Studio/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@using AIStudio.Settings.DataModel
@using AIStudio.Tools.Validation
@using AIStudio.Provider
@inherits MSGComponentBase

<MudDialog>
    <DialogContent>
        @* A drop anywhere in this dialog belongs to the dialog, not to the page behind it: *@
        <PathDropZone IsArea="@true">
        <MudForm @ref="@this.form" @bind-IsValid="@this.dataIsValid" @bind-Errors="@this.dataIssues" FieldChanged="@this.RevalidateAfterFieldChange">
            <DataSourceMissingEmbeddingWarning IsInDataSourceDialog="@true" EmbeddingProviderAdded="@this.SelectAddedEmbeddingProvider"/>

            <MudText Typo="Typo.h6" Class="mb-3">
                @T("Data source")
            </MudText>
            @* ReSharper disable once CSharpWarnings::CS8974 *@
            <MudTextField
                T="string"
                @bind-Text="@this.dataName"
                Label="@T("Data Source Name")"
                Class="mb-3"
                MaxLength="@DataSourceValidation.MAX_NAME_LENGTH"
                Counter="@DataSourceValidation.MAX_NAME_LENGTH"
                Immediate="@true"
                Validation="@this.dataSourceValidation.ValidatingName"
                Adornment="Adornment.Start"
                AdornmentIcon="@Icons.Material.Filled.Lightbulb"
                AdornmentColor="Color.Info"
                UserAttributes="@SPELLCHECK_ATTRIBUTES"
                Variant="Variant.Outlined"
            />

            <MudTextField
                T="string"
                @bind-Text="@this.dataDescription"
                Label="@T("Description")"
                Class="mb-0"
                MaxLength="200"
                Counter="200"
                Immediate="@true"
                Adornment="Adornment.Start"
                AdornmentIcon="@Icons.Material.Filled.Description"
                AdornmentColor="Color.Info"
                HelperText="@T("Describe what data this directory contains to help the AI select it.")"
                UserAttributes="@SPELLCHECK_ATTRIBUTES"
                Variant="Variant.Outlined"
                Lines="3"
            />

            <MudText Typo="Typo.h6" Class="mb-3 mt-6">
                @T("Folder")
            </MudText>
            <MudJustifiedText Typo="Typo.body2" Class="mb-3">
                @T("Select a root directory for this data source.")
            </MudJustifiedText>
            <SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" EnableDragDrop="true" CatchAllDocuments="true" />

            <MudTextSwitch
                Label="@T("Include subfolders?")"
                Value="@this.dataIncludeSubdirectories"
                ValueChanged="@(value => this.dataIncludeSubdirectories = value)"
                LabelOn="@T("Yes, index files in subfolders as well")"
                LabelOff="@T("No, only index files in the selected folder")"
            />

            <MudText Typo="Typo.h6" Class="mb-3 mt-6">
                @T("Files to include")
            </MudText>
            <MudJustifiedText Typo="Typo.body2" Class="mb-3">
                @T("Select the file patterns this data source should index. Audio and video files, e.g., *.mp3 or *.mp4, are transcribed by your transcription provider.")
            </MudJustifiedText>

            <FilePatternSelection>
                <AllowedContent>
                    <MudTextField
                        T="string"
                        @bind-Text="@this.dataFilePatterns"
                        Label="@T("Files to include")"
                        Immediate="@true"
                        Validation="@this.dataSourceValidation.ValidateFilePatterns"
                        Adornment="Adornment.End"
                        AdornmentIcon="@Icons.Material.Filled.Restore"
                        OnAdornmentClick="@this.RestoreDefaultFilePatterns"
                        AdornmentAriaLabel="@T("Restore default patterns")"
                        UserAttributes="@SPELLCHECK_ATTRIBUTES"
                        Variant="Variant.Outlined"
                        Margin="Margin.Dense"
                        Lines="1"
                        AutoGrow="@true"
                        MaxLines="@FilePatternSelection.MAX_FIELD_LINES"
                        Class="pattern-text-field"
                    />
                </AllowedContent>
            </FilePatternSelection>

            <MudText Typo="Typo.h6" Class="mb-3 mt-6">
                @T("Embedding")
            </MudText>
            <MudText Typo="Typo.body2" Class="mb-3">
                @T("Choose an embedding provider to make this data source searchable.")
            </MudText>
            <MudSelect @ref="@this.embeddingSelect" @bind-Value="@this.dataEmbeddingId" Label="@T("Embedding")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateEmbeddingProviderAccess">
                @foreach (var provider in this.SettingsManager.GetAllEmbeddingProviders())
                {
                    <MudSelectItem Value="@provider.Id">
                        <ProviderLabel ProviderType="@provider.UsedLLMProvider" CustomIconDataUrl="@provider.CustomIconDataUrl" Text="@provider.Name" />
                    </MudSelectItem>
                }
            </MudSelect>
            
            @if (this.CurrentCloudProcessing is not CloudProcessing.NONE)
            {
                <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" SendsEmbeddingsToCloud="@this.SelectedCloudEmbedding" CloudTranscriptionProvider="@this.CloudTranscriptionProviderName" @bind-UserAcknowledged="@this.UserAcknowledgedCloudProcessing" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
            }
            else if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
            {
                <MudJustifiedText Typo="Typo.body2" Color="Color.Tertiary" Class="mb-3">
                    @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
                </MudJustifiedText>
            }

            <ManagePandocDependency IntroText="@T("MindWork AI Studio uses Pandoc to read HTML files.")"/>

            <MudText Typo="Typo.h6" Class="mb-3 mt-6">
                @T("Data sharing")
            </MudText>
            <MudSelect @ref="@this.confidenceLevelSelect" @bind-Value="@this.dataConfidenceLevel" Text="@this.dataConfidenceLevel.GetName()" Label="@T("Required provider confidence level")" HelperText="@T("Only providers with this confidence level or higher may receive and process data from this data source.")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateDataSourceConfidenceLevel">
                @foreach (var level in this.ConfidenceLevels)
                {
                    <MudSelectItem Value="@level.Value">
                        @level.Name
                    </MudSelectItem>
                }
            </MudSelect>

            <OutboundDataRestrictionSelection @bind-Value="@this.SelectedOutboundDataRestriction" Label="@T("Where a chat may send data after reading this data source")" Explanation="@T("This setting decides where a chat may still send data once the AI found something in this data source, so that its content does not end up in a web search, for example. Choose no restriction only for content anybody may read anyway.")" OrganizationMinimum="@this.OrganizationMinimumOutboundDataRestriction"/>

            @if (this.MediaTranscriptionHint is { } mediaTranscriptionHint)
            {
                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Dense="@true" Class="mb-3">
                    @mediaTranscriptionHint
                </MudAlert>
            }

            <MudStack Class="mb-3">
                <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
                    @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
                </MudButton>
                <MudDivider/>
                <MudCollapse Expanded="@this.showExpertSettings" Class="@this.GetExpertStyles">
                    <MudText Typo="Typo.subtitle2" Class="mb-2">@T("Hidden files and folders")</MudText>
                    <MudTextSwitch
                        Label="@T("Include hidden files and folders?")"
                        Value="@this.dataIncludeHiddenFilesAndFolders"
                        ValueChanged="@(value => this.dataIncludeHiddenFilesAndFolders = value)"
                        LabelOn="@T("Yes, index hidden files and folders as well")"
                        LabelOff="@T("No, ignore hidden files and folders")"
                    />
                    <MudJustifiedText Typo="Typo.body2" Class="mb-3">
                        @T("Hidden are files and folders whose names start with a dot, such as .git or .env, and those hidden by the operating system. A file pattern which starts with a dot, such as .gitignore, selects such files anyway.")
                    </MudJustifiedText>
                    <div class="mt-4">
                        <FileExclusionEditor @bind-Text="@this.dataExcludePatterns" Label="@T("Excluded files and folders")" ResetButtonText="@T("Restore default")"/>
                    </div>
                    <MudText Typo="Typo.subtitle2" Class="mb-2 mt-4">@T("File size limit")</MudText>
                    <MudNumericField
                        T="int"
                        @bind-Value="@this.dataMaxFileSizeMegabytes"
                        Label="@T("Ignore files larger than")"
                        Class="mb-3"
                        Min="@FolderFileSelection.SMALLEST_MAX_FILE_SIZE_MEGABYTES"
                        Immediate="@true"
                        HelperText="@T("Selected files above this size are not indexed, e.g., images of virtual machines. The embedding page lists them, so you can raise the limit when needed.")"
                        Variant="Variant.Outlined"
                        Adornment="Adornment.End"
                        AdornmentText="MB"/>
                    <MudText Typo="Typo.subtitle2" Class="mb-2 mt-4">@T("Text chunking")</MudText>
                    <MudJustifiedText Typo="Typo.body2" Class="mb-3">
                        @T("Optional expert settings for how this data source is split before embedding.")
                    </MudJustifiedText>
                    @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
                    {
                        <MudTextField
                            T="string"
                            Text="@this.SelectedEmbeddingTokenizerText"
                            Label="@T("Tokenizer")"
                            Class="mb-3"
                            ReadOnly="@true"
                            Variant="Variant.Outlined"
                            Adornment="Adornment.Start"
                            AdornmentIcon="@Icons.Material.Filled.TextFields"
                            AdornmentColor="Color.Info"/>
                    }
                    <MudNumericField
                        @ref="@this.maxChunkTokenLengthField"
                        T="int"
                        @bind-Value="@this.dataMaxChunkTokenLength"
                        Label="@T("Token limit")"
                        Class="mb-3"
                        Min="1"
                        Immediate="@true"
                        Validation="@this.ValidateMaxChunkTokenLength"
                        HelperText="@this.MaxChunkTokenLengthHelperText"
                        Variant="Variant.Outlined"
                        Adornment="Adornment.Start"
                        AdornmentIcon="@Icons.Material.Filled.FormatListNumbered"
                        AdornmentColor="Color.Info"/>
                    <MudNumericField
                        @ref="@this.chunkOverlapTokenLengthField"
                        T="int"
                        @bind-Value="@this.dataChunkOverlapTokenLength"
                        Label="@T("Token overlap")"
                        Min="0"
                        Immediate="@true"
                        Validation="@this.ValidateChunkOverlapTokenLength"
                        HelperText="@this.ChunkOverlapTokenLengthHelperText"
                        Variant="Variant.Outlined"
                        Adornment="Adornment.Start"
                        AdornmentIcon="@Icons.Material.Filled.CompareArrows"
                        AdornmentColor="Color.Info"/>
                    <MudText Typo="Typo.subtitle2" Class="mb-2 mt-4">@T("Search results")</MudText>
                    <MudNumericField
                        T="ushort"
                        Min="10"
                        @bind-Value="@this.dataMaxMatches"
                        Label="@T("How many matches do you want at most per query?")"
                        Variant="Variant.Outlined"
                        Step="10"
                        Adornment="Adornment.Start"
                        AdornmentIcon="@Icons.Material.Filled.Search"
                        AdornmentColor="Color.Info"/>
                </MudCollapse>
            </MudStack>
        </MudForm>
        <Issues IssuesData="@this.dataIssues"/>
        </PathDropZone>
    </DialogContent>
    <DialogActions>
        <MudButton OnClick="@this.Cancel" Variant="Variant.Filled">
            @T("Cancel")
        </MudButton>
        <MudButton OnClick="@this.Store" Variant="Variant.Filled" Color="Color.Primary">
            @if(this.IsEditing)
            {
                @T("Update")
            }
            else
            {
                @T("Add")
            }
        </MudButton>
    </DialogActions>
</MudDialog>