From f7e6ceb915854c7ad4f3952ade52182149dce136 Mon Sep 17 00:00:00 2001 From: Paul Koudelka <106623909+PaulKoudelka@users.noreply.github.com> Date: Tue, 27 Oct 2026 19:57:04 +0100 Subject: [PATCH] Improved the dialogs for local data sources and the Batch Processing assistant (#1096) Co-authored-by: Thorsten Sommer --- .../AssistantBatchProcessing.razor | 27 +- ...sistantBatchProcessing.razor.Validation.cs | 7 +- .../AssistantBatchProcessing.razor.cs | 4 +- .../Assistants/I18N/allTexts.lua | 200 +++++++--- .../ConfigurationFileExclusions.razor | 3 + .../ConfigurationFileExclusions.razor.cs | 17 + .../Components/ConfigurationText.razor | 17 +- .../Components/ConfigurationText.razor.cs | 77 +++- .../Components/FileExclusionEditor.razor | 25 ++ .../Components/FileExclusionEditor.razor.cs | 115 ++++++ .../Components/FileExclusionEditorState.cs | 97 +++++ .../Components/FilePatternHelp.razor | 23 ++ .../Components/FilePatternHelp.razor.cs | 22 + .../Components/FilePatternSelection.razor | 27 ++ .../Components/FilePatternSelection.razor.cs | 31 ++ .../Components/ManagePandocDependency.razor | 19 +- .../ManagePandocDependency.razor.cs | 76 +--- .../Components/TextInfoLines.razor | 4 +- .../Components/TextInfoLines.razor.cs | 6 + .../DataSourceLocalDirectoryDialog.razor | 101 +++-- .../DataSourceLocalDirectoryDialog.razor.cs | 4 +- .../DataSourceLocalDirectoryInfoDialog.razor | 7 +- ...ataSourceLocalDirectoryInfoDialog.razor.cs | 2 + .../Dialogs/DataSourceLocalFileDialog.razor | 38 +- .../SettingsDialogBatchProcessing.razor | 13 +- .../plugin.lua | 206 +++++++--- .../plugin.lua | 200 +++++++--- .../Tools/FileExclusionGroups.cs | 8 + .../Tools/FileExclusions.cs | 61 ++- .../Tools/FileNamePatterns.cs | 15 +- .../Tools/LongExtensions.cs | 25 +- .../Tools/Validation/DataSourceValidation.cs | 2 - app/MindWork AI Studio/wwwroot/app.css | 26 ++ .../wwwroot/changelog/v26.10.1.md | 4 +- .../ConfigurationFileExclusionsTests.cs | 201 ++++++++++ .../FileExclusionEditorStateTests.cs | 377 ++++++++++++++++++ .../Components/FileExclusionEditorTests.cs | 204 ++++++++++ .../Components/ManagePandocDependencyTests.cs | 31 ++ app/Tests/Tools/EmbeddingSignatureTests.cs | 14 +- app/Tests/Tools/FileExclusionsTests.cs | 82 +++- app/Tests/Tools/FilePatternExclusionTests.cs | 153 +++++++ .../Tools/FileSourceIndexerWatcherTests.cs | 26 ++ .../Tools/LocalDirectoryFilePatternsTests.cs | 61 ++- app/Tests/Tools/LongExtensionsTests.cs | 25 ++ app/Tests/Tools/MediaFileSelectionTests.cs | 192 +++++++++ 45 files changed, 2514 insertions(+), 361 deletions(-) create mode 100644 app/MindWork AI Studio/Components/ConfigurationFileExclusions.razor create mode 100644 app/MindWork AI Studio/Components/ConfigurationFileExclusions.razor.cs create mode 100644 app/MindWork AI Studio/Components/FileExclusionEditor.razor create mode 100644 app/MindWork AI Studio/Components/FileExclusionEditor.razor.cs create mode 100644 app/MindWork AI Studio/Components/FileExclusionEditorState.cs create mode 100644 app/MindWork AI Studio/Components/FilePatternHelp.razor create mode 100644 app/MindWork AI Studio/Components/FilePatternHelp.razor.cs create mode 100644 app/MindWork AI Studio/Components/FilePatternSelection.razor create mode 100644 app/MindWork AI Studio/Components/FilePatternSelection.razor.cs create mode 100644 app/MindWork AI Studio/Tools/FileExclusionGroups.cs create mode 100644 app/Tests/Components/ConfigurationFileExclusionsTests.cs create mode 100644 app/Tests/Components/FileExclusionEditorStateTests.cs create mode 100644 app/Tests/Components/FileExclusionEditorTests.cs create mode 100644 app/Tests/Components/ManagePandocDependencyTests.cs create mode 100644 app/Tests/Tools/FilePatternExclusionTests.cs create mode 100644 app/Tests/Tools/LongExtensionsTests.cs create mode 100644 app/Tests/Tools/MediaFileSelectionTests.cs diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor index e528c0e50..e2178b6b2 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor @@ -3,7 +3,7 @@ @using AIStudio.Settings.DataModel @using AIStudio.Tools.Rust - + @T("Input") @@ -18,8 +18,13 @@ } - - +
+ + + + + +
@T("Supported audio and video files are transcribed automatically without an additional dialog. Each transcript is stored next to its media file as '.transcript.md' and reused when an interrupted run is continued.") @@ -35,20 +40,22 @@ + @T("Hidden files and folders") - + @T("Instructions") @@ -134,7 +141,7 @@ else } } - + @T("Output") @@ -189,7 +196,7 @@ else @T("We always write a semicolon-separated log named log.csv, which lists every document with its processing time, the model, the status, and the details of any error. When you start another run on the same output folder, we ask you whether to continue that run: documents which failed or are missing in the log are then processed again. When no output folder is selected, everything is written to the subfolder 'ai-results' within the input folder.") - + @T("Processing pace") @@ -230,7 +237,7 @@ else @if (this.fileResults.Count > 0) { - + @T("Progress") diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs index dc8643d74..f6d26f0c7 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs @@ -18,8 +18,6 @@ public partial class AssistantBatchProcessing private string? ValidateFilePatterns(string patterns) => FileNamePatterns.Validate(patterns); - private string? ValidateExcludePatterns(string patterns) => FileExclusions.Validate(patterns); - private string? ValidateCsvFileName(string fileName) { if (string.IsNullOrWhiteSpace(fileName)) @@ -135,9 +133,10 @@ public partial class AssistantBatchProcessing // // The exclusion list sits in the expert settings, which are folded away by default. Unfold - // them, so the user sees the message next to the list: + // them, so the user sees the message next to the list. Only the editor knows about an invalid + // draft, since it passes on valid lists only: // - if (this.ValidateExcludePatterns(this.excludePatterns) is not null) + if (this.exclusionEditor is { IsValid: false }) this.showExpertSettings = true; var instructionIssue = this.ValidateInstructionSource(); diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs index 5610dacde..4e49c3ef5 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs @@ -1,3 +1,4 @@ +using AIStudio.Components; using AIStudio.Dialogs.Settings; using AIStudio.Provider; using AIStudio.Settings.DataModel; @@ -126,6 +127,7 @@ public partial class AssistantBatchProcessing : AssistantBaseCore fileResults = []; private readonly HashSet usedResultFileNames = new(StringComparer.OrdinalIgnoreCase); + private FileExclusionEditor? exclusionEditor; private bool showExpertSettings; private bool isProcessingBatch; private bool hasReportedWriteFailure; @@ -171,8 +173,6 @@ public partial class AssistantBatchProcessing : AssistantBaseCore this.filePatterns = DataBatchProcessing.DEFAULT_FILE_PATTERNS; - private void RestoreDefaultExcludePatterns() => this.excludePatterns = FileExclusions.DEFAULT_PATTERNS; - private void ToggleExpertSettings() => this.showExpertSettings = !this.showExpertSettings; private string GetExpertStyles => this.showExpertSettings ? "border-2 border-dashed rounded pa-2" : string.Empty; diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 7e6f669b2..83e518482 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -436,9 +436,15 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING -- Please provide a file name without a path, e.g., my-results.csv UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T189587595"] = "Please provide a file name without a path, e.g., my-results.csv" +-- Hidden files and folders +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1902969218"] = "Hidden files and folders" + -- Select the folder containing your documents UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1926838679"] = "Select the folder containing your documents" +-- Files to include +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2000156297"] = "Files to include" + -- Include subfolders? UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2086334687"] = "Include subfolders?" @@ -484,9 +490,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING -- Was not able to read the existing transcript: {0} UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2397111152"] = "Was not able to read the existing transcript: {0}" --- File patterns -UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2460883298"] = "File patterns" - -- Load prompt from file UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2474257795"] = "Load prompt from file" @@ -538,6 +541,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING -- The batch run finished, but one file could not be processed. See the progress table and log for details. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3201532790"] = "The batch run finished, but one file could not be processed. See the progress table and log for details." +-- File size limit +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3225743883"] = "File size limit" + -- The selected model cannot read images, so the {0} images among the selected files will fail. All other files are processed as usual. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T324865195"] = "The selected model cannot read images, so the {0} images among the selected files will fail. All other files are processed as usual." @@ -568,9 +574,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING -- Processing pace UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3428873429"] = "Processing pace" --- Which files should be processed? Separate multiple patterns with a semicolon, e.g., *.pdf;*.docx. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file. -UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3436936162"] = "Which files should be processed? Separate multiple patterns with a semicolon, e.g., *.pdf;*.docx. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file." - -- The file name of the CSV results table. We add the ending .csv when it is missing. When left empty, we use 'batch-results.csv'. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3439247329"] = "The file name of the CSV results table. We add the ending .csv when it is missing. When left empty, we use 'batch-results.csv'." @@ -607,9 +610,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING -- Was not able to write '{0}'. Please make sure that the file is not opened in another application. The results of this run are incomplete on disk. The message is: '{1}' UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3899869356"] = "Was not able to write '{0}'. Please make sure that the file is not opened in another application. The results of this run are incomplete on disk. The message is: '{1}'" --- These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. Other entries name files, e.g., *.min.js. -UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3934601817"] = "These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. Other entries name files, e.g., *.min.js." - -- Select the file with your instructions UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3943995624"] = "Select the file with your instructions" @@ -4771,9 +4771,57 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T3596972550"] = "Open this w -- Open the file in the program your system uses for it UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T4218155268"] = "Open the file in the program your system uses for it" +-- Exclusion help and examples +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXCLUSIONEDITOR::T1309128010"] = "Exclusion help and examples" + +-- Folders +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXCLUSIONEDITOR::T1412427146"] = "Folders" + +-- Excluded files and folders +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXCLUSIONEDITOR::T286958948"] = "Excluded files and folders" + +-- Files +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXCLUSIONEDITOR::T3877444400"] = "Files" + +-- These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXCLUSIONEDITOR::T499835724"] = "These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/." + +-- Restore default +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXCLUSIONEDITOR::T913168756"] = "Restore default" + -- Saving '{0}'... UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTRUNNINGMESSAGE::T1063135893"] = "Saving '{0}'..." +-- Exclude folders ending in build, such as build or app-build +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEPATTERNHELP::T1241240542"] = "Exclude folders ending in build, such as build or app-build" + +-- Include readable text files, even those without an extension +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEPATTERNHELP::T1482113736"] = "Include readable text files, even those without an extension" + +-- File names starting with draft- +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEPATTERNHELP::T1676890987"] = "File names starting with draft-" + +-- Pattern examples +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEPATTERNHELP::T2430822600"] = "Pattern examples" + +-- Include all files, even audio and video +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEPATTERNHELP::T2681459563"] = "Include all files, even audio and video" + +-- Separate patterns with semicolons. Exclusions take priority over included files. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEPATTERNHELP::T2728429336"] = "Separate patterns with semicolons. Exclusions take priority over included files." + +-- PDF and Word files +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEPATTERNHELP::T3467839719"] = "PDF and Word files" + +-- Patterns match file names regardless of letter case. Folder paths are not supported. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEPATTERNHELP::T3691425703"] = "Patterns match file names regardless of letter case. Folder paths are not supported." + +-- Include all documents and readable text files +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEPATTERNHELP::T967949655"] = "Include all documents and readable text files" + +-- Pattern help and examples +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEPATTERNSELECTION::T1369669520"] = "Pattern help and examples" + -- LLMs can make mistakes. Check important information. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::HALLUZINATIONREMINDER::T3528806904"] = "LLMs can make mistakes. Check important information." @@ -4810,12 +4858,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANAGEDTOOLSWARNING::T3008114108"] = "Too -- Some tools selected for this run are switched off in your tool settings and stay unused: {0}. Switch them on again in the app settings to use them. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANAGEDTOOLSWARNING::T3803950041"] = "Some tools selected for this run are switched off in your tool settings and stay unused: {0}. Switch them on again in the app settings to use them." --- Your Pandoc installation meets the requirements. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANAGEPANDOCDEPENDENCY::T1167365374"] = "Your Pandoc installation meets the requirements." - --- Please wait while we check the availability of Pandoc. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANAGEPANDOCDEPENDENCY::T1336364299"] = "Please wait while we check the availability of Pandoc." - -- Pandoc Installation UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANAGEPANDOCDEPENDENCY::T185447014"] = "Pandoc Installation" @@ -4828,9 +4870,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANAGEPANDOCDEPENDENCY::T3147551473"] = " -- Update Pandoc UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANAGEPANDOCDEPENDENCY::T3249965383"] = "Update Pandoc" --- Check your Pandoc installation -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANAGEPANDOCDEPENDENCY::T527187983"] = "Check your Pandoc installation" - -- Install Pandoc UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANAGEPANDOCDEPENDENCY::T986578435"] = "Install Pandoc" @@ -7231,6 +7270,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERIV1USERNAMEPASSWORDEXPORTDIALOG: -- Cancel UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERIV1USERNAMEPASSWORDEXPORTDIALOG::T900713019"] = "Cancel" +-- Folder +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T104168189"] = "Folder" + -- Hide Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1108876344"] = "Hide Expert Settings" @@ -7249,6 +7291,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1311820297" -- Selected base directory for this data source UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1312296210"] = "Selected base directory for this data source" +-- MindWork AI Studio uses Pandoc to read HTML files. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T133018120"] = "MindWork AI Studio uses Pandoc to read HTML files." + -- No embedding selected UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1359179968"] = "No embedding selected" @@ -7279,23 +7324,32 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1827669611" -- Update UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"] = "Update" +-- Hidden files and folders +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1902969218"] = "Hidden files and folders" + +-- Files to include +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2000156297"] = "Files to include" + +-- Data source +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2034620186"] = "Data source" + -- Include subfolders? UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2086334687"] = "Include subfolders?" -- The overlap must be smaller than the effective token limit. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2101951526"] = "The overlap must be smaller than the effective token limit." --- Select the file patterns this data source should index. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2244471011"] = "Select the file patterns this data source should index." - -- Required provider confidence level UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T236253137"] = "Required provider confidence level" -- Please enter a token limit of at least 1. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2406580478"] = "Please enter a token limit of at least 1." --- File patterns -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2460883298"] = "File patterns" +-- Search results +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2499087427"] = "Search results" + +-- Data sharing +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2623330465"] = "Data sharing" -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add" @@ -7306,6 +7360,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210" -- Embedding UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"] = "Embedding" +-- Choose an embedding provider to make this data source searchable. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2861534545"] = "Choose an embedding provider to make this data source searchable." + -- Excluded files and folders UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T286958948"] = "Excluded files and folders" @@ -7315,18 +7372,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2961294165" -- Audio and video files which this data source selects are skipped, because your transcription provider does not meet the confidence level this data source requires. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3017942214"] = "Audio and video files which this data source selects are skipped, because your transcription provider does not meet the confidence level this data source requires." +-- Text chunking +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3105524153"] = "Text chunking" + -- Audio and video files which this data source selects are skipped until you choose a transcription provider in the settings. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3119708878"] = "Audio and video files which this data source selects are skipped until you choose a transcription provider in the settings." +-- File size limit +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3225743883"] = "File size limit" + -- Please enter 0 or a positive overlap length. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length." -- Ignore files larger than UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3353866230"] = "Ignore files larger than" --- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc." - -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings" @@ -7339,21 +7399,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3584816527" -- This setting decides where a chat may still send data once the AI found something in this data source, so that its content does not end up in a web search, for example. Choose no restriction only for content anybody may read anyway. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3738459730"] = "This setting decides where a chat may still send data once the AI found something in this data source, so that its content does not end up in a web search, for example. Choose no restriction only for content anybody may read anyway." --- These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. Other entries name files, e.g., *.min.js. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3934601817"] = "These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. Other entries name files, e.g., *.min.js." +-- Only providers with this confidence level or higher may receive and process data from this data source. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3755939304"] = "Only providers with this confidence level or higher may receive and process data from this data source." --- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding provider. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T529871858"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding provider." +-- Select the file patterns this data source should index. Audio and video files, e.g., *.mp3 or *.mp4, are transcribed by your transcription provider. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4289641992"] = "Select the file patterns this data source should index. Audio and video files, e.g., *.mp3 or *.mp4, are transcribed by your transcription provider." -- Select the base directory UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory" --- Which files should be indexed? Separate multiple patterns with a semicolon, e.g., *.txt;*.pdf;*.docx;.gitignore. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file. Audio and video files, e.g., *.mp3 or *.mp4, are transcribed by your transcription provider. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T599406584"] = "Which files should be indexed? Separate multiple patterns with a semicolon, e.g., *.txt;*.pdf;*.docx;.gitignore. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file. Audio and video files, e.g., *.mp3 or *.mp4, are transcribed by your transcription provider." - -- Yes, index hidden files and folders as well UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T607153623"] = "Yes, index hidden files and folders as well" +-- Restore default patterns +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T7425959"] = "Restore default patterns" + -- The data source token limit must not be larger than the embedding provider token limit ({0}). UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T787118522"] = "The data source token limit must not be larger than the embedding provider token limit ({0})." @@ -7393,6 +7453,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T1447415 -- No UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T1642511898"] = "No" +-- the excluded folders +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T1647237555"] = "the excluded folders" + -- Include subfolders UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T1710166842"] = "Include subfolders" @@ -7405,6 +7468,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T1725856 -- The embedding runs locally or in your organization. Your data is not sent to the cloud. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T1950544032"] = "The embedding runs locally or in your organization. Your data is not sent to the cloud." +-- Files to include +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T2000156297"] = "Files to include" + -- the file patterns UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T2013714981"] = "the file patterns" @@ -7417,24 +7483,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T2333573 -- Required provider confidence level UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T236253137"] = "Required provider confidence level" --- File patterns -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T2460883298"] = "File patterns" - -- the maximum number of matches per query UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T2479753122"] = "the maximum number of matches per query" -- the total size of matching files UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T2515807708"] = "the total size of matching files" +-- Excluded files +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T2590452128"] = "Excluded files" + -- the description UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T2658359966"] = "the description" -- the data source name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T2717738728"] = "the data source name" --- Excluded files and folders -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T286958948"] = "Excluded files and folders" - -- The directory chosen for the data source does not exist anymore. Please edit the data source and correct the path. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T2875614207"] = "The directory chosen for the data source does not exist anymore. Please edit the data source and correct the path." @@ -7459,6 +7522,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3127438 -- Embedding name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3153201045"] = "Embedding name" +-- Excluded folders +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3345276250"] = "Excluded folders" + -- Ignore files larger than UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3353866230"] = "Ignore files larger than" @@ -7480,8 +7546,8 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3818527 -- Path UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3949388886"] = "Path" --- the excluded files and folders -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T4001527349"] = "the excluded files and folders" +-- the excluded files +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T4083345389"] = "the excluded files" -- The directory chosen for the data source exists. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T445858624"] = "The directory chosen for the data source exists." @@ -7498,6 +7564,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1133561850"] = " -- Default tokenizer UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1220918127"] = "Default tokenizer" +-- MindWork AI Studio uses Pandoc to read HTML files. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T133018120"] = "MindWork AI Studio uses Pandoc to read HTML files." + -- No embedding selected UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1359179968"] = "No embedding selected" @@ -7519,6 +7588,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1827669611"] = " -- Update UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1847791252"] = "Update" +-- Data source +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2034620186"] = "Data source" + -- The overlap must be smaller than the effective token limit. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2101951526"] = "The overlap must be smaller than the effective token limit." @@ -7531,6 +7603,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2406580478"] = " -- This audio or video file is skipped, because your transcription provider does not meet the confidence level this data source requires. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2468760910"] = "This audio or video file is skipped, because your transcription provider does not meet the confidence level this data source requires." +-- Search results +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2499087427"] = "Search results" + +-- Data sharing +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2623330465"] = "Data sharing" + -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add" @@ -7543,15 +7621,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2838542994"] = " -- Describe what data this file contains to help the AI select it. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = "Describe what data this file contains to help the AI select it." +-- Choose an embedding provider to make this data source searchable. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2861534545"] = "Choose an embedding provider to make this data source searchable." + -- Token limit UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2961294165"] = "Token limit" +-- Text chunking +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3105524153"] = "Text chunking" + -- Please enter 0 or a positive overlap length. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length." --- For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc." - -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings" @@ -7564,11 +7645,14 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3738459730"] = " -- Select the file UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file" +-- Only providers with this confidence level or higher may receive and process data from this data source. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3755939304"] = "Only providers with this confidence level or higher may receive and process data from this data source." + -- This audio or video file is skipped until you choose a transcription provider in the settings. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4026150982"] = "This audio or video file is skipped until you choose a transcription provider in the settings." --- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding provider. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T529871858"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding provider." +-- File +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T723007075"] = "File" -- The data source token limit must not be larger than the embedding provider token limit ({0}). UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "The data source token limit must not be larger than the embedding provider token limit ({0})." @@ -9151,9 +9235,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T17 -- Batch processing options are preselected UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1893713430"] = "Batch processing options are preselected" +-- Hidden files and folders +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1902969218"] = "Hidden files and folders" + -- Default custom column separator UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T19367494"] = "Default custom column separator" +-- Files to include +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2000156297"] = "Files to include" + -- Default document analysis policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2015391667"] = "Default document analysis policy" @@ -9202,9 +9292,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T28 -- Please enter exactly one punctuation or symbol character. Letters, numbers, spaces, quotation marks, and line breaks cannot be used as CSV separators. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2908365499"] = "Please enter exactly one punctuation or symbol character. Letters, numbers, spaces, quotation marks, and line breaks cannot be used as CSV separators." --- Default file patterns -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2909693903"] = "Default file patterns" - -- Only the selected folder is processed UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2958253681"] = "Only the selected folder is processed" @@ -9220,6 +9307,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T31 -- These instructions are applied to every document of a new batch run. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T3195548336"] = "These instructions are applied to every document of a new batch run." +-- File size limit +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T3225743883"] = "File size limit" + -- Ignore files larger than UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T3353866230"] = "Ignore files larger than" @@ -9250,9 +9340,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T37 -- Default results table name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T3816237687"] = "Default results table name" --- These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. Other entries name files, e.g., *.min.js. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T3934601817"] = "These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. Other entries name files, e.g., *.min.js." - -- Default Markdown instructions file UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T3967465682"] = "Default Markdown instructions file" @@ -13156,6 +13243,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::EXTERNALHTTPCLIENTTIMEOUT::T599774443"] = "The -- policy files UI_TEXT_CONTENT["AISTUDIO::TOOLS::EXTERNALHTTPCLIENTTIMEOUT::T632340680"] = "policy files" +-- Enter folder patterns ending with / in 'Folders'. File patterns must not end with /. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXCLUSIONS::T2019378403"] = "Enter folder patterns ending with / in 'Folders'. File patterns must not end with /." + +-- Folder patterns must end with /, e.g., build/. Enter file patterns such as *.pdf in 'Files'. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXCLUSIONS::T2695719308"] = "Folder patterns must end with /, e.g., build/. Enter file patterns such as *.pdf in 'Files'." + -- Selectors such as :text: cannot be excluded. Please use file name patterns such as *.min.js or folder names such as bin/. UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXCLUSIONS::T405437626"] = "Selectors such as :text: cannot be excluded. Please use file name patterns such as *.min.js or folder names such as bin/." @@ -13384,9 +13477,6 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILENAMEPATTERNS::T2207479763"] = "One of the -- Please provide at least one file pattern, e.g., :documents: or *.txt. Separate multiple patterns with a semicolon. UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILENAMEPATTERNS::T3371062811"] = "Please provide at least one file pattern, e.g., :documents: or *.txt. Separate multiple patterns with a semicolon." --- Please remove empty file patterns. Separate valid patterns with a single semicolon. -UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILENAMEPATTERNS::T368919579"] = "Please remove empty file patterns. Separate valid patterns with a single semicolon." - -- Please use only single asterisks as wildcards, e.g., *.txt or report-*.md. UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILENAMEPATTERNS::T495100949"] = "Please use only single asterisks as wildcards, e.g., *.txt or report-*.md." diff --git a/app/MindWork AI Studio/Components/ConfigurationFileExclusions.razor b/app/MindWork AI Studio/Components/ConfigurationFileExclusions.razor new file mode 100644 index 000000000..bbea9399c --- /dev/null +++ b/app/MindWork AI Studio/Components/ConfigurationFileExclusions.razor @@ -0,0 +1,3 @@ +@inherits ConfigurationText + + diff --git a/app/MindWork AI Studio/Components/ConfigurationFileExclusions.razor.cs b/app/MindWork AI Studio/Components/ConfigurationFileExclusions.razor.cs new file mode 100644 index 000000000..569439679 --- /dev/null +++ b/app/MindWork AI Studio/Components/ConfigurationFileExclusions.razor.cs @@ -0,0 +1,17 @@ +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +/// +/// Reuses ConfigurationText's persistence, validation, reset, and organization locks with the +/// shared exclusion editor. The editor supplies its own outlined fields and heading. +/// +public partial class ConfigurationFileExclusions : ConfigurationText +{ + [Parameter] + public bool ShowHelp { get; set; } = true; + + protected override Variant Variant => Variant.Text; + + protected override string Label => string.Empty; +} diff --git a/app/MindWork AI Studio/Components/ConfigurationText.razor b/app/MindWork AI Studio/Components/ConfigurationText.razor index 8561639f9..6ea70c8f2 100644 --- a/app/MindWork AI Studio/Components/ConfigurationText.razor +++ b/app/MindWork AI Studio/Components/ConfigurationText.razor @@ -4,7 +4,7 @@ { - + @if (!this.ResetButtonIconOnly) + { + + } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ConfigurationText.razor.cs b/app/MindWork AI Studio/Components/ConfigurationText.razor.cs index c2648fb41..301845c25 100644 --- a/app/MindWork AI Studio/Components/ConfigurationText.razor.cs +++ b/app/MindWork AI Studio/Components/ConfigurationText.razor.cs @@ -1,4 +1,5 @@ using Microsoft.AspNetCore.Components; +using Microsoft.AspNetCore.Components.Web; using Timer = System.Timers.Timer; @@ -24,6 +25,24 @@ public partial class ConfigurationText : ConfigurationBaseCore [Parameter] public string Icon { get; set; } = Icons.Material.Filled.Info; + /// + /// Removes the default bottom margin when the field is part of a compact group. + /// + [Parameter] + public bool Compact { get; set; } + + /// + /// Wraps and grows with its text, including while the field is not focused, up to the lines of a + /// field for file patterns. + /// + /// + /// Such a field shows the draft rather than the stored value. Otherwise, a reset which leaves the + /// stored value as it was would not reach the field, e.g., restoring the default while the field + /// shows a draft which the validation rejected. + /// + [Parameter] + public bool ExpandToContent { get; set; } + /// /// The color of the icon to use. /// @@ -54,6 +73,12 @@ public partial class ConfigurationText : ConfigurationBaseCore [Parameter] public string ResetButtonText { get; set; } = string.Empty; + /// + /// Uses an accessible reset icon inside the field. + /// + [Parameter] + public bool ResetButtonIconOnly { get; set; } + /// /// Validates the configured text before it is stored. /// @@ -61,10 +86,14 @@ public partial class ConfigurationText : ConfigurationBaseCore public Func? Validation { get; set; } private string internalText = string.Empty; + private string? lastSourceText; + private readonly Timer timer = new(TimeSpan.FromMilliseconds(500)) { AutoReset = false }; + + protected string EditingText => this.internalText; #region Overrides of ConfigurationBase @@ -72,6 +101,8 @@ public partial class ConfigurationText : ConfigurationBaseCore protected override bool Stretch => true; protected override Variant Variant => Variant.Outlined; + + protected override string MarginClass => this.Compact ? "mb-0" : base.MarginClass; protected override string Label => this.OptionDescription; @@ -83,15 +114,24 @@ public partial class ConfigurationText : ConfigurationBaseCore protected override async Task OnParametersSetAsync() { - this.internalText = this.Text(); + var sourceText = this.Text(); + // Re-rendering the parent must not replace a draft which is waiting for its debounce. + // A genuinely changed setting, e.g. from a configuration plugin, still replaces it. + if (this.IsDisabled || !string.Equals(sourceText, this.lastSourceText, StringComparison.Ordinal)) + { + this.internalText = sourceText; + this.lastSourceText = sourceText; + } await base.OnParametersSetAsync(); } #endregion - private bool AutoGrow => this.NumLines > 1; + private bool AutoGrow => this.ExpandToContent || this.NumLines > 1; - private int GetMaxLines => this.AutoGrow ? this.MaxLines : 1; + private int GetMaxLines => this.ExpandToContent ? FilePatternSelection.MAX_FIELD_LINES : this.AutoGrow ? this.MaxLines : 1; + + private string InputClass => this.ExpandToContent ? "pattern-text-field" : string.Empty; // // MudBlazor centers the icon vertically, so in a field of several lines it floats in the middle @@ -99,29 +139,50 @@ public partial class ConfigurationText : ConfigurationBaseCore // private Adornment IconAdornment => this.NumLines == 1 ? Adornment.Start : Adornment.None; - private void InternalUpdate(string text) + protected void InternalUpdate(string text) { + if (this.IsDisabled) + return; + this.timer.Stop(); this.internalText = text; this.timer.Start(); } - private async Task ResetTextAsync() + private EventCallback ResetAdornmentClick => this.ResetButtonIconOnly + ? EventCallback.Factory.Create(this, this.ResetTextAsync) + : default; + + protected async Task ResetTextAsync() { if (this.ResetValue is null || this.IsDisabled) return; + await this.ApplyTextAsync(this.ResetValue()); + } + + /// + /// Applies an explicit value immediately, including a reset of part of a compound setting. + /// + protected async Task ApplyTextAsync(string text) + { + if (this.IsDisabled) + return; + this.timer.Stop(); - this.internalText = this.ResetValue(); + this.internalText = text; await this.OptionChanged(this.internalText); } - private async Task OptionChanged(string updatedText) + protected async Task OptionChanged(string updatedText) { - if (this.Validation?.Invoke(updatedText) is not null) + if (this.IsDisabled || this.Validation?.Invoke(updatedText) is not null) return; this.TextUpdate(updatedText); + // A save may finish after another keystroke. Remember the value it applied before + // awaiting storage, so its configuration-change notification cannot erase that keystroke. + this.lastSourceText = this.Text(); await this.SettingsManager.StoreSettings(); await this.InformAboutChange(); } diff --git a/app/MindWork AI Studio/Components/FileExclusionEditor.razor b/app/MindWork AI Studio/Components/FileExclusionEditor.razor new file mode 100644 index 000000000..3e110c155 --- /dev/null +++ b/app/MindWork AI Studio/Components/FileExclusionEditor.razor @@ -0,0 +1,25 @@ +@inherits MSGComponentBase + +
+ + @this.EditorLabel + @if (this.ShowHelp) + { + + + + } + + + + + + + + @T("These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/.") + +
+ +
+ +
diff --git a/app/MindWork AI Studio/Components/FileExclusionEditor.razor.cs b/app/MindWork AI Studio/Components/FileExclusionEditor.razor.cs new file mode 100644 index 000000000..7d4b0b674 --- /dev/null +++ b/app/MindWork AI Studio/Components/FileExclusionEditor.razor.cs @@ -0,0 +1,115 @@ +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +/// +/// A compact, two-group editor for the exclusion string shared by folder-based features. +/// +public partial class FileExclusionEditor : MSGComponentBase +{ + private static readonly Dictionary INPUT_ATTRIBUTES = new() + { + ["spellcheck"] = false, + ["autocorrect"] = "off", + ["autocapitalize"] = "off", + }; + + [Parameter] + public string? Text { get; set; } = string.Empty; + + [Parameter] + public EventCallback TextChanged { get; set; } + + [Parameter] + public string Label { get; set; } = string.Empty; + + [Parameter] + public bool Disabled { get; set; } + + [Parameter] + public bool ShowReset { get; set; } = true; + + [Parameter] + public bool ShowHelp { get; set; } = true; + + [Parameter] + public string ResetValue { get; set; } = FileExclusions.DEFAULT_PATTERNS; + + [Parameter] + public string ResetButtonText { get; set; } = string.Empty; + + /// + /// Immediately persists the combined value after restoring just one of its groups. + /// Ordinary edits continue through TextChanged. + /// + [Parameter] + public EventCallback ResetRequested { get; set; } + + private readonly FileExclusionEditorState state = new(); + private readonly string folderInputId = $"exclusion-folders-{Guid.NewGuid():N}"; + private readonly string fileInputId = $"exclusion-files-{Guid.NewGuid():N}"; + private bool showHelp; + + /// + /// False while a field holds an invalid draft. The editor keeps such a draft to itself, so the + /// bound text is still the last valid one and cannot tell. + /// + public bool IsValid => this.state.IsValid; + + private string EditorLabel => string.IsNullOrWhiteSpace(this.Label) ? T("Excluded files and folders") : this.Label; + + private string RestoreLabel => string.IsNullOrWhiteSpace(this.ResetButtonText) ? T("Restore default") : this.ResetButtonText; + + private string FolderRestoreLabel => $"{this.RestoreLabel}: {T("Folders")}"; + + private string FileRestoreLabel => $"{this.RestoreLabel}: {T("Files")}"; + + protected override void OnParametersSet() + { + this.state.Receive(this.Text, discardDraft: this.Disabled); + base.OnParametersSet(); + } + + protected Task ChangeFoldersAsync(string text) => this.ChangePatternsAsync(text, folders: true); + + protected Task ChangeFilesAsync(string text) => this.ChangePatternsAsync(text, folders: false); + + private async Task ChangePatternsAsync(string text, bool folders) + { + if (this.Disabled) + return; + + var previousText = this.state.Text; + var wasValid = this.state.IsValid; + if (folders) + this.state.EditFolders(text); + else + this.state.EditFiles(text); + + // Moving a rule to the correct field can keep the combined text identical. It must still + // publish the now-valid draft, because the previous, invalid arrangement was not sent. + if (this.state.IsValid && (!wasValid || !string.Equals(previousText, this.state.Text, StringComparison.Ordinal))) + await this.TextChanged.InvokeAsync(this.state.Text); + } + + protected async Task RestoreDefaultsAsync(bool folders) + { + if (this.Disabled || !this.ShowReset) + return; + + if (folders) + this.state.RestoreFolders(this.ResetValue); + else + this.state.RestoreFiles(this.ResetValue); + + if (!this.state.IsValid) + return; + + if (this.ResetRequested.HasDelegate) + await this.ResetRequested.InvokeAsync(this.state.Text); + else + await this.TextChanged.InvokeAsync(this.state.Text); + } + + private void ToggleHelp() => this.showHelp = !this.showHelp; +} diff --git a/app/MindWork AI Studio/Components/FileExclusionEditorState.cs b/app/MindWork AI Studio/Components/FileExclusionEditorState.cs new file mode 100644 index 000000000..ccba3ebb5 --- /dev/null +++ b/app/MindWork AI Studio/Components/FileExclusionEditorState.cs @@ -0,0 +1,97 @@ +namespace AIStudio.Components; + +/// +/// Presents the existing exclusion string as two editable groups without losing the other group +/// or letting delayed parameter echoes overwrite newer edits. +/// +internal sealed class FileExclusionEditorState +{ + private string? receivedText; + private readonly HashSet pendingValues = new(StringComparer.Ordinal); + + public string Text { get; private set; } = string.Empty; + + public string FolderText { get; private set; } = string.Empty; + + public string FileText { get; private set; } = string.Empty; + + public bool IsValid => FileExclusions.IsValidFolders(this.FolderText) && FileExclusions.IsValidFiles(this.FileText); + + /// + /// Takes over the text of the parent, which renders again after every published edit and passes + /// that edit back, possibly only after the user typed on. Such an echo must not replace the newer + /// draft; any other text does, e.g., a change by a configuration plugin. + /// + /// The text the parent passes in. + /// Whether to take over the text in any case, e.g., while the editor is disabled. + /// Whether the text replaced the draft. + public bool Receive(string? text, bool discardDraft = false) + { + text ??= string.Empty; + if (discardDraft) + { + var changed = !string.Equals(text, this.Text, StringComparison.Ordinal); + this.receivedText = text; + this.Reset(text); + return changed; + } + + // The parent passes the same text with every render; only a different one counts: + if (string.Equals(text, this.receivedText, StringComparison.Ordinal)) + return false; + + this.receivedText = text; + + // The parent caught up with the draft. Its renders arrive in order, so no older echo follows: + if (string.Equals(text, this.Text, StringComparison.Ordinal)) + { + this.pendingValues.Clear(); + return false; + } + + // The echo of an earlier edit, which a later one replaced already: + if (this.pendingValues.Remove(text)) + return false; + + this.Reset(text); + return true; + } + + public void Reset(string text) + { + this.pendingValues.Clear(); + this.Text = text; + this.Regroup(); + } + + public void EditFolders(string? text) + { + this.FolderText = text ?? string.Empty; + this.UpdateText(); + } + + public void EditFiles(string? text) + { + this.FileText = text ?? string.Empty; + this.UpdateText(); + } + + public void RestoreFolders(string defaults) => this.EditFolders(FileExclusions.Group(defaults).Folders); + + public void RestoreFiles(string defaults) => this.EditFiles(FileExclusions.Group(defaults).Files); + + private void Regroup() + { + var groups = FileExclusions.Group(this.Text); + this.FolderText = groups.Folders; + this.FileText = groups.Files; + } + + private void UpdateText() + { + this.Text = string.Join(FileExclusions.ENTRY_SEPARATOR, FileExclusions.Parse(this.FolderText) + .Concat(FileExclusions.Parse(this.FileText)) + .Distinct(StringComparer.OrdinalIgnoreCase)); + this.pendingValues.Add(this.Text); + } +} diff --git a/app/MindWork AI Studio/Components/FilePatternHelp.razor b/app/MindWork AI Studio/Components/FilePatternHelp.razor new file mode 100644 index 000000000..c7bd6de28 --- /dev/null +++ b/app/MindWork AI Studio/Components/FilePatternHelp.razor @@ -0,0 +1,23 @@ +@inherits MSGComponentBase + + + @T("Pattern examples") + @T("Separate patterns with semicolons. Exclusions take priority over included files.") + + + *.pdf;*.docx@T("PDF and Word files") + @if (this.ShowInclusionExamples) + { + :documents:@T("Include all documents and readable text files") + *@T("Include all files, even audio and video") + :text:@T("Include readable text files, even those without an extension") + } + draft-*@T("File names starting with draft-") + @if (this.ShowExclusionExamples) + { + *build/@T("Exclude folders ending in build, such as build or app-build") + } + + + @T("Patterns match file names regardless of letter case. Folder paths are not supported.") + \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/FilePatternHelp.razor.cs b/app/MindWork AI Studio/Components/FilePatternHelp.razor.cs new file mode 100644 index 000000000..6272af0a9 --- /dev/null +++ b/app/MindWork AI Studio/Components/FilePatternHelp.razor.cs @@ -0,0 +1,22 @@ +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +/// +/// Examples of file patterns, shared by the fields for files to include and for exclusions, so both +/// explain the patterns in the same words. +/// +public partial class FilePatternHelp : MSGComponentBase +{ + /// + /// Shows the selectors and wildcards which only the files to include accept, such as :text:. + /// + [Parameter] + public bool ShowInclusionExamples { get; set; } = true; + + /// + /// Shows an entry which excludes folders. + /// + [Parameter] + public bool ShowExclusionExamples { get; set; } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/FilePatternSelection.razor b/app/MindWork AI Studio/Components/FilePatternSelection.razor new file mode 100644 index 000000000..688d8eb12 --- /dev/null +++ b/app/MindWork AI Studio/Components/FilePatternSelection.razor @@ -0,0 +1,27 @@ +@inherits MSGComponentBase + + +
+
+ @this.AllowedContent +
+ + + +
+ @if (this.HasExclusions) + { +
+
+ @this.ExcludedContent +
+
+ } + +
+
+ +
+
+ + \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/FilePatternSelection.razor.cs b/app/MindWork AI Studio/Components/FilePatternSelection.razor.cs new file mode 100644 index 000000000..79531fc3c --- /dev/null +++ b/app/MindWork AI Studio/Components/FilePatternSelection.razor.cs @@ -0,0 +1,31 @@ +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +/// +/// A file-pattern field with shared, collapsible help and optional, always-visible exclusions. +/// Callers supply their existing fields, preserving validation and managed-configuration locks. +/// +public partial class FilePatternSelection : MSGComponentBase +{ + /// + /// How many lines a field for file patterns or exclusions grows to. A longer list scrolls. + /// + public const int MAX_FIELD_LINES = 6; + + [Parameter, EditorRequired] + public RenderFragment AllowedContent { get; set; } = null!; + + /// + /// When supplied, shows the existing exclusion field below the inclusion field. + /// Both groups stay visible and use the same help control. + /// + [Parameter] + public RenderFragment? ExcludedContent { get; set; } + + private bool HasExclusions => this.ExcludedContent is not null; + + private bool showHelp; + + private void ToggleHelp() => this.showHelp = !this.showHelp; +} diff --git a/app/MindWork AI Studio/Components/ManagePandocDependency.razor b/app/MindWork AI Studio/Components/ManagePandocDependency.razor index 7208b1c45..9d505d188 100644 --- a/app/MindWork AI Studio/Components/ManagePandocDependency.razor +++ b/app/MindWork AI Studio/Components/ManagePandocDependency.razor @@ -1,10 +1,13 @@ @inherits MSGComponentBase - - - @this.DetermineIntroText() - - - @this.DetermineButtonText() - - \ No newline at end of file +@if (ShouldShowInstallationPrompt(this.pandocInstallation)) +{ + + + @this.PromptText + + + @this.ButtonText + + +} diff --git a/app/MindWork AI Studio/Components/ManagePandocDependency.razor.cs b/app/MindWork AI Studio/Components/ManagePandocDependency.razor.cs index 886b1ff4d..a4e506a6f 100644 --- a/app/MindWork AI Studio/Components/ManagePandocDependency.razor.cs +++ b/app/MindWork AI Studio/Components/ManagePandocDependency.razor.cs @@ -7,6 +7,11 @@ using DialogOptions = AIStudio.Dialogs.DialogOptions; namespace AIStudio.Components; +/// +/// Asks to install or update Pandoc where a feature needs it for some files only, such as the HTML files +/// of a local data source. It stays hidden while the check runs and once Pandoc is ready, so it does not +/// distract from the rest of the dialog. +/// public partial class ManagePandocDependency : MSGComponentBase { [Parameter] @@ -18,7 +23,10 @@ public partial class ManagePandocDependency : MSGComponentBase [Inject] private RustService RustService { get; init; } = null!; - private PandocInstallation pandocInstallation; + // Null means the silent availability check has not completed yet. + private PandocInstallation? pandocInstallation; + + internal static bool ShouldShowInstallationPrompt(PandocInstallation? installation) => installation is { IsAvailable: false }; #region Overrides of MSGComponentBase @@ -30,60 +38,20 @@ public partial class ManagePandocDependency : MSGComponentBase #endregion - private Color DetermineButtonColor() - { - if (this.pandocInstallation == default) - return Color.Default; - - switch (this.pandocInstallation) - { - case { CheckWasSuccessful: true, IsAvailable: true }: - return Color.Default; - - case { CheckWasSuccessful: true, IsAvailable: false }: - return Color.Warning; - - case { CheckWasSuccessful: false }: - return Color.Error; - } - } - - private string DetermineButtonText() - { - if(this.pandocInstallation == default) - return T("Please wait while we check the availability of Pandoc."); - - switch (this.pandocInstallation) - { - case { CheckWasSuccessful: true, IsAvailable: true }: - return T("Check your Pandoc installation"); - - case { CheckWasSuccessful: true, IsAvailable: false }: - return T("Update Pandoc"); - - case { CheckWasSuccessful: false }: - return T("Install Pandoc"); - } - } + // + // The prompt shows up only for an installation which is not available, so it is either outdated, + // when the check found one, or missing: + // + private bool IsOutdated => this.pandocInstallation is { CheckWasSuccessful: true }; + + private Color ButtonColor => this.IsOutdated ? Color.Warning : Color.Error; + + private string ButtonText => this.IsOutdated ? T("Update Pandoc") : T("Install Pandoc"); + + private string PromptText => this.IsOutdated + ? $"{this.IntroText} {T("Your Pandoc installation is outdated. Please update it to the latest version to ensure compatibility with all features.")}" + : $"{this.IntroText} {T("Pandoc is not installed or not available. Please install it to use the features that require Pandoc.")}"; - private string DetermineIntroText() - { - if (this.pandocInstallation == default) - return $"{this.IntroText} {T("Please wait while we check the availability of Pandoc.")}"; - - switch (this.pandocInstallation) - { - case { CheckWasSuccessful: true, IsAvailable: true }: - return $"{this.IntroText} {T("Your Pandoc installation meets the requirements.")}"; - - case { CheckWasSuccessful: true, IsAvailable: false }: - return $"{this.IntroText} {T("Your Pandoc installation is outdated. Please update it to the latest version to ensure compatibility with all features.")}"; - - case { CheckWasSuccessful: false }: - return $"{this.IntroText} {T("Pandoc is not installed or not available. Please install it to use the features that require Pandoc.")}"; - } - } - private async Task ShowPandocDialogAsync() { var dialogReference = await this.DialogService.ShowAsync(T("Pandoc Installation"), DialogOptions.FULLSCREEN); diff --git a/app/MindWork AI Studio/Components/TextInfoLines.razor b/app/MindWork AI Studio/Components/TextInfoLines.razor index a6a8e824c..b5ed3923f 100644 --- a/app/MindWork AI Studio/Components/TextInfoLines.razor +++ b/app/MindWork AI Studio/Components/TextInfoLines.razor @@ -7,9 +7,11 @@ Text="@this.Value" Variant="Variant.Outlined" Margin="Margin.Dense" - Lines="3" + Lines="@this.Lines" MaxLines="@this.MaxLines" AutoGrow="@true" + Adornment="@(string.IsNullOrEmpty(this.Icon) ? Adornment.None : Adornment.Start)" + AdornmentIcon="@this.Icon" Style="@this.GetColor()" UserAttributes="@USER_INPUT_ATTRIBUTES" /> diff --git a/app/MindWork AI Studio/Components/TextInfoLines.razor.cs b/app/MindWork AI Studio/Components/TextInfoLines.razor.cs index e33a78ee7..f143ddef5 100644 --- a/app/MindWork AI Studio/Components/TextInfoLines.razor.cs +++ b/app/MindWork AI Studio/Components/TextInfoLines.razor.cs @@ -9,6 +9,9 @@ public partial class TextInfoLines : MSGComponentBase [Parameter] public string Label { get; set; } = string.Empty; + [Parameter] + public string Icon { get; set; } = string.Empty; + [Parameter] public string Value { get; set; } = string.Empty; @@ -18,6 +21,9 @@ public partial class TextInfoLines : MSGComponentBase [Parameter] public int MaxLines { get; set; } = 30; + [Parameter] + public int Lines { get; set; } = 3; + [Parameter] public bool ShowingCopyButton { get; set; } = true; diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index 861f9d8be..638b31887 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -10,12 +10,15 @@ + + @T("Data source") + @* ReSharper disable once CSharpWarnings::CS8974 *@ - + + @T("Folder") + + @T("Select a root directory for this data source.") @@ -57,28 +63,42 @@ LabelOff="@T("No, only index files in the selected folder")" /> - - @T("Select the file patterns this data source should index.") + + @T("Files to include") + + + @T("Select the file patterns this data source should index. Audio and video files, e.g., *.mp3 or *.mp4, are transcribed by your transcription provider.") - + + + + + - - @T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding provider.") - + + @T("Embedding") + + + @T("Choose an embedding provider to make this data source searchable.") + @foreach (var provider in this.SettingsManager.GetAllEmbeddingProviders()) { @@ -94,14 +114,17 @@ } else if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId)) { - + @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.") } - + - + + @T("Data sharing") + + @foreach (var level in this.ConfidenceLevels) { @@ -125,6 +148,7 @@ + @T("Hidden files and folders") - @* Without an icon: in a field of several lines, it floats in the middle of the text. *@ - - +
+ +
+ @T("File size limit") + @T("Text chunking") + + @T("Optional expert settings for how this data source is split before embedding.") + @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId)) { } - - @T("Optional expert settings for how this data source is split before embedding.") - + @T("Search results") FileNamePatterns.Normalize(this.dataFilePatterns); - private void RestoreDefaultExcludePatterns() => this.dataExcludePatterns = FileExclusions.DEFAULT_PATTERNS; + private void RestoreDefaultFilePatterns() => this.dataFilePatterns = DataSourceLocalDirectory.DEFAULT_FILE_PATTERNS; /// /// Gives the fields which are checked against each other a fresh verdict: the embedding provider diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor index cc1079e5c..340edf6cb 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor @@ -12,8 +12,9 @@ public const char FOLDER_SUFFIX = '/'; + /// + /// What AI Studio joins entries with when it writes a list, e.g., after an edit in the exclusion editor. + /// + public const string ENTRY_SEPARATOR = "; "; + // // Folders of dependencies and build outputs which are not hidden. Hidden ones, such as .git, // .venv, or .gradle, are left out by the rule for hidden files and folders anyway. Names which @@ -65,7 +70,9 @@ internal static class FileExclusions private const string DISK_IMAGE_FILES = "*.vmdk; *.vdi; *.vhd; *.vhdx; *.qcow2; *.hdd; *.hds; *.iso; *.dmg; *.img"; /// - /// The list a data source or the batch processing assistant starts with, one group per line. + /// The list a data source or the batch processing assistant starts with. Its line breaks only + /// group the entries for the reader of this code: the exclusion editor shows folders and files in + /// fields of their own. /// public const string DEFAULT_PATTERNS = DEPENDENCY_AND_BUILD_FOLDERS + "\n" + @@ -77,17 +84,6 @@ internal static class FileExclusions DATABASE_FILES + "\n" + DISK_IMAGE_FILES; - /// - /// How many lines the default list has, so that a field shows all of it at once. - /// - public static readonly int DEFAULT_LINE_COUNT = DEFAULT_PATTERNS.Split('\n').Length; - - /// - /// How many lines a field for the list grows to before it scrolls: the default list, with room for - /// a few entries of the user. - /// - public const int MAX_LINE_COUNT = 12; - private const string OFFICE_LOCK_FILE_PATTERN = "~$*"; /// @@ -112,6 +108,8 @@ internal static class FileExclusions PATH, RECURSIVE_WILDCARD, INVALID_CHARACTER, + FILE_IN_FOLDER_LIST, + FOLDER_IN_FILE_LIST, } private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileExclusions).Namespace, nameof(FileExclusions)); @@ -124,20 +122,50 @@ internal static class FileExclusions .Distinct(StringComparer.OrdinalIgnoreCase) .ToArray(); + /// + /// Splits a list into the entries for folders and those for files, as the exclusion editor and the + /// information about a data source show them. + /// + public static FileExclusionGroups Group(string? patterns) + { + var entries = Parse(patterns); + return new( + string.Join(ENTRY_SEPARATOR, entries.Where(IsFolderEntry)), + string.Join(ENTRY_SEPARATOR, entries.Where(entry => !IsFolderEntry(entry)))); + } + public static bool IsValid(string? patterns) => GetValidationIssue(patterns) is ValidationIssue.NONE; + public static bool IsValidFolders(string? patterns) => GetValidationIssue(patterns, folders: true) is ValidationIssue.NONE; + + public static bool IsValidFiles(string? patterns) => GetValidationIssue(patterns, folders: false) is ValidationIssue.NONE; + /// /// Validates a list of excluded files and folders. An empty list is valid: it excludes nothing /// beyond what is built in. /// - public static string? Validate(string? patterns) => GetValidationIssue(patterns) switch + public static string? Validate(string? patterns) => DescribeValidationIssue(GetValidationIssue(patterns)); + + /// + /// Validates the folder field without guessing whether an entry without a slash was meant as a folder. + /// + public static string? ValidateFolders(string? patterns) => DescribeValidationIssue(GetValidationIssue(patterns, folders: true)); + + /// + /// Validates the file field, directing folder rules to their own field. + /// + public static string? ValidateFiles(string? patterns) => DescribeValidationIssue(GetValidationIssue(patterns, folders: false)); + + private static string? DescribeValidationIssue(ValidationIssue issue) => issue switch { ValidationIssue.NONE => null, ValidationIssue.SELECTOR => TB("Selectors such as :text: cannot be excluded. Please use file name patterns such as *.min.js or folder names such as bin/."), ValidationIssue.PATH => TB("Please write excluded folders as a name without a path, ending with a slash, e.g., node_modules/ or bin/. Such an entry excludes every folder of that name."), ValidationIssue.RECURSIVE_WILDCARD => TB("Please use only single asterisks as wildcards, e.g., *.min.js or cmake-build-*/."), ValidationIssue.INVALID_CHARACTER => TB("One of the excluded files or folders contains an invalid character."), - _ => throw new ArgumentOutOfRangeException(nameof(patterns)), + ValidationIssue.FILE_IN_FOLDER_LIST => TB("Folder patterns must end with /, e.g., build/. Enter file patterns such as *.pdf in 'Files'."), + ValidationIssue.FOLDER_IN_FILE_LIST => TB("Enter folder patterns ending with / in 'Folders'. File patterns must not end with /."), + _ => throw new ArgumentOutOfRangeException(nameof(issue)), }; /// @@ -174,7 +202,7 @@ internal static class FileExclusions private static bool IsFolderEntry(string entry) => entry.EndsWith(FOLDER_SUFFIX); - private static ValidationIssue GetValidationIssue(string? patterns) + private static ValidationIssue GetValidationIssue(string? patterns, bool? folders = null) { foreach (var entry in Parse(patterns)) { @@ -193,6 +221,9 @@ internal static class FileExclusions if (name.IndexOfAny(FileNamePatterns.INVALID_PATTERN_CHARACTERS) >= 0) return ValidationIssue.INVALID_CHARACTER; + + if (folders is not null && IsFolderEntry(entry) != folders.Value) + return folders.Value ? ValidationIssue.FILE_IN_FOLDER_LIST : ValidationIssue.FOLDER_IN_FILE_LIST; } return ValidationIssue.NONE; diff --git a/app/MindWork AI Studio/Tools/FileNamePatterns.cs b/app/MindWork AI Studio/Tools/FileNamePatterns.cs index 827f8d1d2..c57ca6c72 100644 --- a/app/MindWork AI Studio/Tools/FileNamePatterns.cs +++ b/app/MindWork AI Studio/Tools/FileNamePatterns.cs @@ -34,7 +34,6 @@ internal static class FileNamePatterns { NONE, MISSING, - EMPTY_ENTRY, UNKNOWN_SELECTOR, RECURSIVE_WILDCARD, PATH, @@ -50,6 +49,9 @@ internal static class FileNamePatterns private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileNamePatterns).Namespace, nameof(FileNamePatterns)); + /// + /// Splits the list, ignoring empty entries, including a trailing semicolon. + /// public static string[] Parse(string? patterns) => (patterns ?? string.Empty) .Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) .Distinct(StringComparer.OrdinalIgnoreCase) @@ -124,7 +126,6 @@ internal static class FileNamePatterns { ValidationIssue.NONE => null, ValidationIssue.MISSING => TB("Please provide at least one file pattern, e.g., :documents: or *.txt. Separate multiple patterns with a semicolon."), - ValidationIssue.EMPTY_ENTRY => TB("Please remove empty file patterns. Separate valid patterns with a single semicolon."), ValidationIssue.UNKNOWN_SELECTOR => TB("One of the file patterns is an unknown selector. Please use :documents: or :text:, or a file name pattern such as *.txt."), ValidationIssue.RECURSIVE_WILDCARD => TB("Please use only single asterisks as wildcards, e.g., *.txt or report-*.md."), ValidationIssue.PATH => TB("Please use file name patterns without folder paths, e.g., *.txt or report-*.md."), @@ -134,16 +135,12 @@ internal static class FileNamePatterns private static ValidationIssue GetValidationIssue(string? patterns) { - if (string.IsNullOrWhiteSpace(patterns)) + var individualPatterns = Parse(patterns); + if (individualPatterns.Length == 0) return ValidationIssue.MISSING; - var individualPatterns = patterns.Split(';'); - if (individualPatterns.Any(string.IsNullOrWhiteSpace)) - return ValidationIssue.EMPTY_ENTRY; - - foreach (var patternEntry in individualPatterns) + foreach (var pattern in individualPatterns) { - var pattern = patternEntry.Trim(); if (IsSelector(pattern)) continue; diff --git a/app/MindWork AI Studio/Tools/LongExtensions.cs b/app/MindWork AI Studio/Tools/LongExtensions.cs index 660593a69..a06b91bb2 100644 --- a/app/MindWork AI Studio/Tools/LongExtensions.cs +++ b/app/MindWork AI Studio/Tools/LongExtensions.cs @@ -6,6 +6,8 @@ public static class LongExtensions { private static readonly string[] COUNT_SUFFIXES = ["k", "M", "B", "T"]; + private static readonly string[] SIZE_UNITS = ["B", "kB", "MB", "GB", "TB"]; + /// /// Formats a count so that large numbers stay readable: 1456 becomes 1.46k, 4512900 becomes 4.51M. /// @@ -42,20 +44,31 @@ public static class LongExtensions public static string CompactCount(this int count) => ((long)count).CompactCount(); /// - /// Formats the file size in a human-readable format. + /// Formats the file size in a human-readable format, e.g., 1.9 MB. /// + /// + /// Like CompactCount, the decimal separator follows the culture of the active language plugin. + /// /// The size in bytes. /// The formatted file size. - public static string FileSize(this long sizeBytes) + public static string FileSize(this long sizeBytes) => sizeBytes.FileSize(I18N.I.Culture); + + /// + /// Formats the file size in a human-readable format, with the decimal separator of the given culture. + /// + /// The size in bytes. + /// The culture whose decimal separator to use. + /// The formatted file size. + internal static string FileSize(this long sizeBytes, IFormatProvider culture) { - string[] sizes = { "B", "kB", "MB", "GB", "TB" }; var order = 0; - while (sizeBytes >= 1024 && order < sizes.Length - 1) + double size = sizeBytes; + while (size >= 1024 && order < SIZE_UNITS.Length - 1) { order++; - sizeBytes /= 1024; + size /= 1024; } - return $"{sizeBytes:0.##} {sizes[order]}"; + return $"{size.ToString("0.##", culture)} {SIZE_UNITS[order]}"; } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs index 75a049f2a..ce88b2e02 100644 --- a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs +++ b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs @@ -168,8 +168,6 @@ public sealed class DataSourceValidation public string? ValidateFilePatterns(string patterns) => FileNamePatterns.Validate(patterns); - public string? ValidateExcludePatterns(string patterns) => FileExclusions.Validate(patterns); - public string? ValidateEmbeddingId(string embeddingId) { if(string.IsNullOrWhiteSpace(embeddingId)) diff --git a/app/MindWork AI Studio/wwwroot/app.css b/app/MindWork AI Studio/wwwroot/app.css index cadaf76af..f8a01c0d1 100644 --- a/app/MindWork AI Studio/wwwroot/app.css +++ b/app/MindWork AI Studio/wwwroot/app.css @@ -550,6 +550,32 @@ tr:has(> .provider-group-header) .mud-icon-button { padding-bottom: 0.5rem; } +/* A field for file patterns with its help button on the right, which keeps its width while the field grows. */ +.pattern-row { + display: grid; + grid-template-columns: minmax(0, 1fr) 48px; + column-gap: 8px; + align-items: center; +} + +/* The field takes the first column, and so does what follows below it, such as the help, which keeps them equally wide. */ +.pattern-row-main { + grid-column: 1; + min-width: 0; +} + +/* + * Pattern lists wrap and grow with their content, up to a limit beyond which they scroll. MudBlazor + * measures the height only on input, after a render with new text, and when the window changes its + * size. When the field gets narrower otherwise, e.g., in a dialog, more lines wrap than it is high, + * and MudBlazor hides the scrollbar below the limit. Allowing it always keeps every line reachable. + */ +.pattern-text-field textarea.mud-input-slot { + white-space: pre-wrap; + overflow-wrap: anywhere; + overflow-y: auto !important; +} + /* * Dialogs whose footer carries buttons on the left as well, such as Add and Import next to Close. * MudBlazor pads the content with 24px to the left and right, but the footer with only 8px, so the diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md index 822f9f331..8adeaf744 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md @@ -49,7 +49,7 @@ - Improved local data sources for folders. You can now choose which files AI Studio indexes, for example, only PDF files, and whether it includes subfolders as well as hidden files and folders. By default, AI Studio now also indexes text files it did not recognize before, such as program code of any kind or a LICENSE file, so a Git repository makes a useful data source. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution. - Improved local data sources for folders, which now leave out hidden files and folders as well as build output and downloaded dependencies, such as the folders `node_modules`, `bin`, and `obj`. You can change this in the expert settings of the data source. When an existing data source contains such files, AI Studio removes them from its index during the next update. - Improved the background embeddings page, which now lists the files of a local data source that AI Studio left out, for example, images or files that are too large. It also shows which file types and folders lie outside what the data source selects. -- Improved the Batch Processing assistant, which now also processes images in JPG, PNG, WebP, GIF, BMP, and TIFF format when the selected model can read images. If you used the assistant before, your saved file patterns stay as they are: click "Restore default patterns" or add patterns such as `*.jpg;*.png` yourself. +- Improved the Batch Processing assistant, which now also processes images in JPG, PNG, WebP, GIF, BMP, and TIFF format when the selected model can read images. If you used the assistant before, your saved file patterns stay as they are: click the restore button at the end of the field "Files to include" or add patterns such as `*.jpg;*.png` yourself. - Improved the Batch Processing assistant, which now leaves out hidden files and folders as well as build output and downloaded dependencies, such as the folder `node_modules`. You can change both in the expert settings of the assistant. - Improved local data sources for folders and the Batch Processing assistant, which now leave out files larger than 1 GB, such as images of virtual machines. You can change this limit. - Improved local data sources for files and folders. When you move or rename the file or folder, for example, to another drive or a new computer, you can now select its new location when you edit the data source. AI Studio keeps what it has already prepared for every unchanged file, so these files do not go to your embedding provider again. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution. @@ -78,6 +78,7 @@ - Improved the export menu of answers with many tables or code blocks. From six of them on, the menu groups them by kind, so the formats for the entire answer stay in view. When an answer holds code blocks in different languages, the menu now names each block by its language, such as Python or Rust, instead of numbering them. - Improved saving a single code block. When you point at a code block, whether in an answer or in one of your own messages, a save button now appears next to its copy button. You no longer have to scroll up to the export menu of a long answer, and you can save code you sent weeks ago, for example, to get back an earlier version of it. - Improved the search field of the workspace list. It now tells you whether it searches only the names of your chats or their contents as well. +- Improved the dialogs for local data sources and the Batch Processing assistant, whose settings are now grouped into sections. A help button next to the file patterns shows you examples, and you enter the folders and the files to leave out in a field each. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution. - Changed the banner on the start page to a fresh design that shows what AI Studio can do today. - Changed when a chat you add to a workspace in the workspace list appears there. Like any other new chat, it now appears once there is something to keep, for example, your first message, instead of right away. - Changed the name of the Slide Planner Assistant to Slide Assistant. With a PowerPoint template, it now creates the slides themselves, not just a plan for them. @@ -152,6 +153,7 @@ - Fixed exporting a briefing of the Visual Briefing assistant into a file which another program keeps open or which cannot be written. AI Studio now tells you that the export failed. - Fixed the German warning about tools which the chosen AI provider is not trusted enough to use. It named these tools as if they were the provider. - Fixed the configuration of your organization being able to preset a format for the result files of the Batch Processing assistant which cannot hold an entire answer, such as a table. AI Studio now ignores such a format and keeps the one you chose. +- Fixed file sizes being rounded down to whole numbers, e.g., in the information about a local data source, where 1.9 MB appeared as 1 MB. They now show their decimals with the separator of your language. - Updated the code contributions on the supporters page, which now thank everyone who has contributed code to AI Studio so far. - Upgraded several libraries to improve security. - Upgraded to Rust v1.99.1 diff --git a/app/Tests/Components/ConfigurationFileExclusionsTests.cs b/app/Tests/Components/ConfigurationFileExclusionsTests.cs new file mode 100644 index 000000000..395f4f4f1 --- /dev/null +++ b/app/Tests/Components/ConfigurationFileExclusionsTests.cs @@ -0,0 +1,201 @@ +using AIStudio.Components; + +namespace AIStudio.Tests.Components; + +public sealed class ConfigurationFileExclusionsTests +{ + [Test] + public async Task AParentRefreshDoesNotReplaceAnUncommittedDraft() + { + var component = new TestableConfigurationFileExclusions(); + try + { + await component.RefreshAsync(); + component.Edit("*.new"); + await component.RefreshAsync(); + + Assert.That(component.Draft, Is.EqualTo("*.new")); + } + finally + { + component.ReleaseResources(); + } + } + + [Test] + public async Task AnExternalSettingsChangeReplacesTheDraft() + { + var component = new TestableConfigurationFileExclusions(); + try + { + await component.RefreshAsync(); + component.Edit("*.new"); + component.StoredText = "*.external"; + await component.RefreshAsync(); + + Assert.That(component.Draft, Is.EqualTo("*.external")); + } + finally + { + component.ReleaseResources(); + } + } + + [TestCase(true, false)] + [TestCase(false, true)] + public async Task DisabledOrLockedFieldsRejectEdits(bool disabled, bool locked) + { + var component = new TestableConfigurationFileExclusions(); + try + { + await component.RefreshAsync(); + component.DisableField = disabled; + component.LockField = locked; + component.Edit("*.new"); + + Assert.That(component.Draft, Is.EqualTo("*.original")); + } + finally + { + component.ReleaseResources(); + } + } + + [TestCase(true, false)] + [TestCase(false, true)] + public async Task BecomingDisabledOrLockedBeforeAutosaveRejectsThePendingWrite(bool disabled, bool locked) + { + var component = new TestableConfigurationFileExclusions(); + try + { + await component.RefreshAsync(); + component.Edit("*.new"); + component.DisableField = disabled; + component.LockField = locked; + await component.FlushAsync(); + + Assert.That(component.StoredText, Is.EqualTo("*.original")); + } + finally + { + component.ReleaseResources(); + } + } + + [Test] + public async Task LockingAFieldReplacesItsDraftWithTheStoredSetting() + { + var component = new TestableConfigurationFileExclusions(); + try + { + await component.RefreshAsync(); + component.Edit("*.new"); + component.LockField = true; + await component.RefreshAsync(); + + Assert.That(component.Draft, Is.EqualTo("*.original")); + } + finally + { + component.ReleaseResources(); + } + } + + [Test] + public async Task InvalidRulesAreNotPersisted() + { + var component = new TestableConfigurationFileExclusions(); + try + { + await component.RefreshAsync(); + component.RejectEdits(); + component.Edit(":text:"); + await component.FlushAsync(); + + Assert.That(component.StoredText, Is.EqualTo("*.original")); + } + finally + { + component.ReleaseResources(); + } + } + + [TestCase(true, false)] + [TestCase(false, true)] + public async Task DisabledOrLockedFieldsRejectExplicitReplacement(bool disabled, bool locked) + { + var component = new TestableConfigurationFileExclusions(); + try + { + await component.RefreshAsync(); + component.DisableField = disabled; + component.LockField = locked; + await component.ReplaceAsync("bin/;*.tmp"); + + Assert.Multiple(() => + { + Assert.That(component.Draft, Is.EqualTo("*.original")); + Assert.That(component.StoredText, Is.EqualTo("*.original")); + }); + } + finally + { + component.ReleaseResources(); + } + } + + [Test] + public async Task ExplicitReplacementKeepsAnInvalidOtherGroupsDraftWithoutPersistingIt() + { + var component = new TestableConfigurationFileExclusions(); + try + { + await component.RefreshAsync(); + component.RejectEdits(); + component.Edit("old/;:text:"); + await component.ReplaceAsync("bin/;:text:"); + + Assert.Multiple(() => + { + Assert.That(component.Draft, Is.EqualTo("bin/;:text:")); + Assert.That(component.StoredText, Is.EqualTo("*.original")); + }); + } + finally + { + component.ReleaseResources(); + } + } + + private sealed class TestableConfigurationFileExclusions : ConfigurationFileExclusions + { + public string StoredText { get; set; } = "*.original"; + + public bool DisableField { get; set; } + + public bool LockField { get; set; } + + public string Draft => this.EditingText; + + public TestableConfigurationFileExclusions() + { + this.Text = () => this.StoredText; + this.TextUpdate = text => this.StoredText = text; + this.Disabled = () => this.DisableField; + this.IsLocked = () => this.LockField; + } + + public void Edit(string text) => this.InternalUpdate(text); + + public void RejectEdits() => this.Validation = _ => "Invalid rule"; + + public Task RefreshAsync() => this.OnParametersSetAsync(); + + public Task FlushAsync() => this.OptionChanged(this.EditingText); + + public Task ReplaceAsync(string text) => this.ApplyTextAsync(text); + + // These state-only tests never register with a message bus or render the component. + public void ReleaseResources() => this.DisposeResources(); + } +} diff --git a/app/Tests/Components/FileExclusionEditorStateTests.cs b/app/Tests/Components/FileExclusionEditorStateTests.cs new file mode 100644 index 000000000..cf0767d65 --- /dev/null +++ b/app/Tests/Components/FileExclusionEditorStateTests.cs @@ -0,0 +1,377 @@ +using AIStudio.Components; +using AIStudio.Tools; + +namespace AIStudio.Tests.Components; + +public sealed class FileExclusionEditorStateTests +{ + [TestCase("")] + [TestCase(FileExclusions.DEFAULT_PATTERNS)] + [TestCase(" node_modules/ ; bin/\n*.min.js;draft-* ")] + public void OpeningTheEditorDoesNotRewriteTheStoredText(string text) + { + var state = new FileExclusionEditorState(); + state.Receive(text); + + Assert.That(state.Text, Is.EqualTo(text)); + } + + [TestCase("node_modules/;bin/;*.min.js;draft-*")] + [TestCase("node_modules/\nbin/\n*.min.js\ndraft-*")] + [TestCase("node_modules/\r\nbin/\r\n*.min.js\r\ndraft-*")] + [TestCase("node_modules/; bin/\r\n*.min.js; draft-*")] + public void BothGroupsDisplaySemicolonSeparatedPatternsWithoutRewritingStoredLists(string text) + { + var state = Create(text); + + Assert.Multiple(() => + { + Assert.That(state.FolderText, Is.EqualTo("node_modules/; bin/")); + Assert.That(state.FileText, Is.EqualTo("*.min.js; draft-*")); + Assert.That(state.Text, Is.EqualTo(text)); + }); + } + + [Test] + public void EditingFoldersKeepsEveryFilePattern() + { + var state = Create("bin/;*.min.js;draft-*"); + state.EditFolders("obj/\nbuild/"); + + Assert.Multiple(() => + { + Assert.That(FileExclusions.Parse(state.Text), Is.EqualTo(new[] { "obj/", "build/", "*.min.js", "draft-*" })); + Assert.That(state.FileText, Is.EqualTo("*.min.js; draft-*")); + Assert.That(state.Text, Is.EqualTo("obj/; build/; *.min.js; draft-*")); + }); + } + + [Test] + public void EditingFilesKeepsEveryFolderPattern() + { + var state = Create("bin/;node_modules/;*.min.js"); + state.EditFiles("*.wav;*.md"); + + Assert.Multiple(() => + { + Assert.That(FileExclusions.Parse(state.Text), Is.EqualTo(new[] { "bin/", "node_modules/", "*.wav", "*.md" })); + Assert.That(state.FolderText, Is.EqualTo("bin/; node_modules/")); + Assert.That(state.Text, Is.EqualTo("bin/; node_modules/; *.wav; *.md")); + }); + } + + [TestCase("")] + [TestCase(" ")] + public void ClearingOneGroupKeepsTheOtherGroup(string emptyText) + { + var state = Create("bin/;*.min.js"); + state.EditFolders(emptyText); + Assert.That(FileExclusions.Parse(state.Text), Is.EqualTo(new[] { "*.min.js" })); + + state = Create("bin/;*.min.js"); + state.EditFiles(emptyText); + Assert.That(FileExclusions.Parse(state.Text), Is.EqualTo(new[] { "bin/" })); + } + + [Test] + public void ExplicitFolderRulesKeepTheirExistingMatchingRules() + { + var state = Create(string.Empty); + state.EditFolders("node_modules/;cmake-build-*/"); + var patterns = FileExclusions.Parse(state.Text); + + Assert.Multiple(() => + { + Assert.That(patterns, Is.EqualTo(new[] { "node_modules/", "cmake-build-*/" })); + Assert.That(FileExclusions.IsValid(state.Text), Is.True); + Assert.That(state.IsValid, Is.True); + Assert.That(FileExclusions.ExcludesFolder("node_modules", patterns), Is.True); + Assert.That(FileExclusions.ExcludesFolder("cmake-build-debug", patterns), Is.True); + Assert.That(FileExclusions.ExcludesFile("node_modules", patterns), Is.False); + }); + } + + [TestCase(":text:")] + [TestCase("src/bin/")] + [TestCase("/")] + [TestCase("**/*.md")] + public void EditingTheOtherGroupNeverDropsAnInvalidStoredRule(string invalidRule) + { + var state = Create(invalidRule); + if (invalidRule.EndsWith('/')) + state.EditFiles("*.tmp"); + else + state.EditFolders("bin/"); + + Assert.Multiple(() => + { + Assert.That(FileExclusions.Parse(state.Text), Does.Contain(invalidRule)); + Assert.That(FileExclusions.IsValid(state.Text), Is.False); + }); + } + + [TestCase(":text:")] + [TestCase(":documents:")] + public void FolderEditingDoesNotTurnAnInvalidSelectorIntoAValidLiteralFolder(string selector) + { + var state = Create(string.Empty); + state.EditFolders(selector); + + Assert.Multiple(() => + { + Assert.That(state.Text, Is.EqualTo(selector)); + Assert.That(FileExclusions.IsValid(state.Text), Is.False); + }); + } + + [Test] + public void ExistingFolderNamesWhichResembleSelectorsArePreservedExactly() + { + var state = Create(":text:/;*.tmp"); + state.EditFiles("*.bak"); + + Assert.That(FileExclusions.Parse(state.Text), Is.EqualTo(new[] { ":text:/", "*.bak" })); + } + + [Test] + public void RestoringOneGroupDeduplicatesRulesWithoutLosingOtherRules() + { + var state = Create("custom/;*.TMP;*.tmp;Caddyfile"); + state.RestoreFolders("node_modules/;NODE_MODULES/;*.default"); + + Assert.That(FileExclusions.Parse(state.Text), Is.EqualTo(new[] { "node_modules/", "*.TMP", "Caddyfile" })); + } + + [TestCase(true)] + [TestCase(false)] + public void RestoringOneFieldPreservesTheOtherFieldsDraft(bool folders) + { + var state = Create("custom/;*.custom"); + state.EditFolders("latest/; "); + state.EditFiles("*.latest; "); + + const string DEFAULTS = "bin/;node_modules/;*.tmp;*.bak"; + if (folders) + state.RestoreFolders(DEFAULTS); + else + state.RestoreFiles(DEFAULTS); + + Assert.Multiple(() => + { + Assert.That(state.FolderText, Is.EqualTo(folders ? "bin/; node_modules/" : "latest/; ")); + Assert.That(state.FileText, Is.EqualTo(folders ? "*.latest; " : "*.tmp; *.bak")); + Assert.That(state.Text, Is.EqualTo(folders ? "bin/; node_modules/; *.latest" : "latest/; *.tmp; *.bak")); + }); + } + + [TestCase(true)] + [TestCase(false)] + public void EmptyDefaultsClearOnlyTheRestoredField(bool folders) + { + var state = Create("custom/;*.custom"); + if (folders) + state.RestoreFolders(string.Empty); + else + state.RestoreFiles(string.Empty); + + Assert.That(state.Text, Is.EqualTo(folders ? "*.custom" : "custom/")); + } + + [TestCase(true, ":text:")] + [TestCase(false, "src/bin/")] + public void RestoringOneFieldPreservesInvalidRulesInTheOtherField(bool folders, string invalidRule) + { + var state = Create(invalidRule); + if (folders) + state.RestoreFolders(FileExclusions.DEFAULT_PATTERNS); + else + state.RestoreFiles(FileExclusions.DEFAULT_PATTERNS); + + Assert.Multiple(() => + { + Assert.That(FileExclusions.Parse(state.Text), Does.Contain(invalidRule)); + Assert.That(FileExclusions.IsValid(state.Text), Is.False); + }); + } + + [Test] + public void EditingTheFileFieldDoesNotSilentlyMoveFolderRules() + { + var state = Create("bin/;*.min.js"); + state.EditFiles("obj/;*.wav;draft-*"); + Assert.Multiple(() => + { + Assert.That(state.IsValid, Is.False); + Assert.That(state.Text, Is.EqualTo("bin/; obj/; *.wav; draft-*")); + Assert.That(state.FolderText, Is.EqualTo("bin/")); + Assert.That(state.FileText, Is.EqualTo("obj/;*.wav;draft-*")); + }); + } + + [TestCase(true)] + [TestCase(false)] + public void EditingBothVisibleFieldsPreservesBothDraftsAndAllRules(bool foldersFirst) + { + var state = Create("bin/;*.tmp"); + string earlierValue; + if (foldersFirst) + { + state.EditFolders("obj/; build/; "); + earlierValue = state.Text; + state.EditFiles("*.wav; draft-*; "); + } + else + { + state.EditFiles("*.wav; draft-*; "); + earlierValue = state.Text; + state.EditFolders("obj/; build/; "); + } + + state.Receive(earlierValue); + state.Receive(state.Text); + + Assert.Multiple(() => + { + Assert.That(state.FolderText, Is.EqualTo("obj/; build/; ")); + Assert.That(state.FileText, Is.EqualTo("*.wav; draft-*; ")); + Assert.That(state.Text, Is.EqualTo("obj/; build/; *.wav; draft-*")); + }); + } + + [Test] + public void ADebouncedEchoDoesNotOverwriteANewerDraft() + { + var state = Create("*.original"); + state.EditFiles("*.first"); + var earlierValue = state.Text; + state.EditFiles("*.latest"); + state.Receive(earlierValue); + + Assert.Multiple(() => + { + Assert.That(state.Text, Is.EqualTo("*.latest")); + Assert.That(state.FileText, Is.EqualTo("*.latest")); + }); + } + + [Test] + public void AnUnchangedParentValueDoesNotDiscardAnUncommittedEdit() + { + var state = Create("*.original"); + state.EditFiles("*.latest; "); + state.Receive("*.original"); + + Assert.That(state.FileText, Is.EqualTo("*.latest; ")); + } + + [Test] + public void ANewExternalValueReplacesBothGroups() + { + var state = Create("bin/;*.tmp"); + state.EditFiles("*.latest"); + state.Receive("policy/;*.secret"); + + Assert.Multiple(() => + { + Assert.That(state.FolderText, Is.EqualTo("policy/")); + Assert.That(state.FileText, Is.EqualTo("*.secret")); + }); + } + + [Test] + public void ADisabledEditorShowsTheActualSettingEvenWhenItMatchesAnOlderDraft() + { + var state = Create("*.original"); + state.EditFiles("*.earlier"); + state.EditFiles("*.latest"); + state.Receive("*.earlier", discardDraft: true); + + Assert.That(state.FileText, Is.EqualTo("*.earlier")); + } + + [Test] + public void ReformattingTheDefaultsDoesNotChangeTheSelectionSignature() + { + var state = Create(FileExclusions.DEFAULT_PATTERNS); + state.EditFiles(state.FileText); + var before = new FolderFileSelection( + ["*"], FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS), true, false, + FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES); + var after = new FolderFileSelection( + ["*"], FileExclusions.Parse(state.Text), true, false, + FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES); + + Assert.That(after.Describe(), Is.EqualTo(before.Describe())); + } + + [Test] + public void RestoringDefaultsReplacesBothGroupsAndKeepsTheExactDefaultString() + { + var state = Create("custom/;*.custom"); + state.EditFolders(":text:"); + state.Reset(FileExclusions.DEFAULT_PATTERNS); + + Assert.Multiple(() => + { + Assert.That(state.Text, Is.EqualTo(FileExclusions.DEFAULT_PATTERNS)); + Assert.That(FileExclusions.IsValid(state.Text), Is.True); + Assert.That(state.FolderText, Does.Contain("node_modules/")); + Assert.That(state.FileText, Does.Contain("*.exe")); + Assert.That(state.FolderText, Does.Not.Contain("\n").And.Not.Contain("\r")); + Assert.That(state.FileText, Does.Not.Contain("\n").And.Not.Contain("\r")); + }); + } + + [Test] + public void EditingADefaultGroupPreservesAllRulesInTheOtherGroup() + { + var state = Create(FileExclusions.DEFAULT_PATTERNS); + state.EditFiles(state.FileText + "\n*.scratch"); + + Assert.That( + FileExclusions.Parse(state.Text), + Is.EquivalentTo(FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS).Append("*.scratch"))); + } + + [TestCase("*.pdf")] + [TestCase("Dockerfile")] + [TestCase("node_modules")] + public void WrongFieldDraftsStayVisibleWithoutAddingAFolderSuffix(string pattern) + { + var state = Create("bin/;*.tmp"); + state.EditFolders(pattern); + + Assert.Multiple(() => + { + Assert.That(state.IsValid, Is.False); + Assert.That(state.FolderText, Is.EqualTo(pattern)); + Assert.That(FileExclusions.Parse(state.Text), Does.Contain(pattern)); + Assert.That(FileExclusions.Parse(state.Text), Does.Not.Contain(pattern + "/")); + Assert.That(state.FileText, Is.EqualTo("*.tmp")); + }); + } + + [Test] + public void FolderRulesInTheFileFieldStayVisibleAndInvalidAfterAParentEcho() + { + var state = Create("bin/;*.tmp"); + state.EditFiles("*.bak"); + var earlierValue = state.Text; + state.EditFiles("*.bak;obj/"); + state.Receive(earlierValue); + + Assert.Multiple(() => + { + Assert.That(state.IsValid, Is.False); + Assert.That(state.FileText, Is.EqualTo("*.bak;obj/")); + Assert.That(state.FolderText, Is.EqualTo("bin/")); + }); + } + + private static FileExclusionEditorState Create(string text) + { + var state = new FileExclusionEditorState(); + state.Receive(text); + return state; + } +} diff --git a/app/Tests/Components/FileExclusionEditorTests.cs b/app/Tests/Components/FileExclusionEditorTests.cs new file mode 100644 index 000000000..45160d739 --- /dev/null +++ b/app/Tests/Components/FileExclusionEditorTests.cs @@ -0,0 +1,204 @@ +using AIStudio.Components; + +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Tests.Components; + +public sealed class FileExclusionEditorTests +{ + [TestCase(true, "*.pdf")] + [TestCase(true, "Dockerfile")] + [TestCase(true, "node_modules")] + [TestCase(false, "bin/")] + [TestCase(false, "*.pdf;obj/")] + [TestCase(true, ":text:")] + [TestCase(false, ":documents:")] + [TestCase(true, "src/bin/")] + [TestCase(false, "**/*.tmp")] + public async Task WrongFieldAndInvalidRulesAreNotPublished(bool folders, string patterns) + { + var published = new List(); + var editor = Create(published); + + await editor.EditAsync(patterns, folders); + + Assert.That(published, Is.Empty); + } + + [Test] + public async Task EditingTheOtherFieldCannotPublishAnInvalidDraft() + { + var published = new List(); + var editor = Create(published); + + await editor.EditAsync("*.pdf", folders: true); + await editor.EditAsync("*.bak", folders: false); + Assert.That(published, Is.Empty); + + await editor.EditAsync("build/", folders: true); + Assert.That(published, Is.EqualTo(new[] { "build/; *.bak" })); + } + + [Test] + public async Task MovingARuleToItsCorrectFieldPublishesItEvenIfTheCombinedTextDoesNotChange() + { + var published = new List(); + var editor = Create(published, "bin/"); + + await editor.EditAsync("bin/;*.pdf", folders: true); + await editor.EditAsync("*.pdf", folders: false); + Assert.That(published, Is.Empty); + + await editor.EditAsync("bin/", folders: true); + Assert.That(published, Is.EqualTo(new[] { "bin/; *.pdf" })); + } + + [TestCase(true, true)] + [TestCase(true, false)] + [TestCase(false, true)] + [TestCase(false, false)] + public async Task ResettingOneFieldCannotPersistInvalidRulesInTheOtherField(bool folders, bool explicitReset) + { + var published = new List(); + var editor = Create(published); + var resets = new List(); + if (explicitReset) + editor.SetParameters(new() { [nameof(FileExclusionEditor.ResetRequested)] = Collect(resets) }); + + await editor.EditAsync(folders ? "*.keep;wrong/" : "*.wrong", folders: !folders); + await editor.ResetAsync(folders); + + Assert.Multiple(() => + { + Assert.That(published, Is.Empty); + Assert.That(resets, Is.Empty); + }); + + await editor.EditAsync(folders ? "*.keep" : "keep/", folders: !folders); + Assert.That(published, Is.EqualTo(new[] { folders ? "bin/; *.keep" : "keep/; *.tmp" })); + } + + [TestCase(true)] + [TestCase(false)] + public async Task ResettingTheInvalidFieldRestoresAValidValue(bool folders) + { + var published = new List(); + var editor = Create(published); + var resets = new List(); + editor.SetParameters(new() { [nameof(FileExclusionEditor.ResetRequested)] = Collect(resets) }); + + await editor.EditAsync(folders ? "*.wrong" : "wrong/", folders); + await editor.ResetAsync(folders); + + Assert.Multiple(() => + { + Assert.That(published, Is.Empty); + Assert.That(resets, Is.EqualTo(new[] { folders ? "bin/; *.custom" : "custom/; *.tmp" })); + }); + } + + [Test] + public async Task DisabledEditorRejectsEditsAndBothResetCallbacks() + { + var published = new List(); + var resets = new List(); + var editor = Create(published); + editor.SetParameters(new() + { + [nameof(FileExclusionEditor.Disabled)] = true, + [nameof(FileExclusionEditor.ResetRequested)] = Collect(resets), + }); + + await editor.EditAsync("changed/", folders: true); + await editor.EditAsync("*.changed", folders: false); + await editor.ResetAsync(folders: true); + await editor.ResetAsync(folders: false); + + Assert.Multiple(() => + { + Assert.That(published, Is.Empty); + Assert.That(resets, Is.Empty); + }); + + editor.SetParameters(new() { [nameof(FileExclusionEditor.ResetRequested)] = default(EventCallback) }); + await editor.ResetAsync(folders: true); + await editor.ResetAsync(folders: false); + Assert.That(published, Is.Empty); + + // Returning to editing must reveal the unchanged source, not either attempted edit: + editor.SetParameters(new() { [nameof(FileExclusionEditor.Disabled)] = false }); + await editor.EditAsync("*.other", folders: false); + Assert.That(published, Is.EqualTo(new[] { "custom/; *.other" })); + } + + [Test] + public async Task AnInvalidDraftMakesTheEditorInvalidWhileTheBoundTextStaysValid() + { + var published = new List(); + var editor = Create(published); + Assert.That(editor.IsValid, Is.True); + + await editor.EditAsync("*.wrong", folders: true); + + Assert.Multiple(() => + { + Assert.That(editor.IsValid, Is.False); + Assert.That(published, Is.Empty); + }); + + await editor.EditAsync("custom/", folders: true); + Assert.That(editor.IsValid, Is.True); + } + + [Test] + public void AnInvalidReceivedListMakesTheEditorInvalid() + { + var editor = Create(new List(), "**/*.tmp"); + + Assert.That(editor.IsValid, Is.False); + } + + [Test] + public async Task HiddenResetButtonsCannotChangeTheRules() + { + var published = new List(); + var editor = Create(published); + editor.SetParameters(new() { [nameof(FileExclusionEditor.ShowReset)] = false }); + + await editor.ResetAsync(folders: true); + await editor.ResetAsync(folders: false); + + Assert.That(published, Is.Empty); + } + + private static TestableFileExclusionEditor Create(List published, string initial = "custom/;*.custom") + { + var editor = new TestableFileExclusionEditor(); + editor.SetParameters(new() + { + [nameof(FileExclusionEditor.Text)] = initial, + [nameof(FileExclusionEditor.ResetValue)] = "bin/;*.tmp", + [nameof(FileExclusionEditor.TextChanged)] = Collect(published), + }); + return editor; + } + + private static EventCallback Collect(List values) => EventCallback.Factory.Create(new object(), values.Add); + + private sealed class TestableFileExclusionEditor : FileExclusionEditor + { + // + // Sets the parameters the way a renderer would, which this test has none of, and then runs + // the step of the lifecycle which reacts to them: + // + public void SetParameters(Dictionary parameters) + { + ParameterView.FromDictionary(parameters).SetParameterProperties(this); + this.OnParametersSet(); + } + + public Task EditAsync(string patterns, bool folders) => folders ? this.ChangeFoldersAsync(patterns) : this.ChangeFilesAsync(patterns); + + public Task ResetAsync(bool folders) => this.RestoreDefaultsAsync(folders); + } +} \ No newline at end of file diff --git a/app/Tests/Components/ManagePandocDependencyTests.cs b/app/Tests/Components/ManagePandocDependencyTests.cs new file mode 100644 index 000000000..cfb147e26 --- /dev/null +++ b/app/Tests/Components/ManagePandocDependencyTests.cs @@ -0,0 +1,31 @@ +using AIStudio.Components; +using AIStudio.Tools; + +namespace AIStudio.Tests.Components; + +public sealed class ManagePandocDependencyTests +{ + [Test] + public void TheInstallationPromptStaysHiddenWhileAvailabilityIsChecked() + { + Assert.That(ManagePandocDependency.ShouldShowInstallationPrompt(null), Is.False); + } + + [TestCase(true)] + [TestCase(false)] + public void AnAvailableInstallationNeedsNoPrompt(bool isLocal) + { + var installation = new PandocInstallation(true, string.Empty, true, "3.7.0.2", isLocal); + + Assert.That(ManagePandocDependency.ShouldShowInstallationPrompt(installation), Is.False); + } + + [TestCase(true)] + [TestCase(false)] + public void AnUnavailableOrOutdatedInstallationShowsThePrompt(bool checkWasSuccessful) + { + var installation = new PandocInstallation(checkWasSuccessful, "Unavailable", false, string.Empty, false); + + Assert.That(ManagePandocDependency.ShouldShowInstallationPrompt(installation), Is.True); + } +} diff --git a/app/Tests/Tools/EmbeddingSignatureTests.cs b/app/Tests/Tools/EmbeddingSignatureTests.cs index fbdab52c2..f169bad6f 100644 --- a/app/Tests/Tools/EmbeddingSignatureTests.cs +++ b/app/Tests/Tools/EmbeddingSignatureTests.cs @@ -40,6 +40,18 @@ public sealed class EmbeddingSignatureTests "Patterns change which files belong to the source, not how retained files are embedded; the indexing run removes and adds only the affected files."); } + [Test] + public void ChangingDirectoryExclusionsKeepsTheStoredEmbeddingsOfRetainedFiles() + { + var allFiles = DataSource(ConfidenceLevel.LOW); + var filteredFiles = allFiles with { ExcludePatterns = "*.mp3;draft-*" }; + + Assert.That( + Signature(filteredFiles), + Is.EqualTo(Signature(allFiles)), + "Exclusions change which files belong to the source, not how retained files are embedded."); + } + [Test] public void ChangingDirectoryRecursionKeepsTheStoredEmbeddingsOfRetainedFiles() { @@ -244,4 +256,4 @@ public sealed class EmbeddingSignatureTests public int ChunkOverlapTokenLength { get; init; } } -} \ No newline at end of file +} diff --git a/app/Tests/Tools/FileExclusionsTests.cs b/app/Tests/Tools/FileExclusionsTests.cs index 79ae22fde..127e00323 100644 --- a/app/Tests/Tools/FileExclusionsTests.cs +++ b/app/Tests/Tools/FileExclusionsTests.cs @@ -4,7 +4,6 @@ using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools; using AIStudio.Tools.Services; -using AIStudio.Tools.Validation; namespace AIStudio.Tests.Tools; @@ -22,7 +21,7 @@ public sealed class FileExclusionsTests [TestCase(FileExclusions.DEFAULT_PATTERNS)] public void ValidExclusionListsAreAccepted(string patterns) { - Assert.That(new DataSourceValidation().ValidateExcludePatterns(patterns), Is.Null); + Assert.That(FileExclusions.Validate(patterns), Is.Null); } [TestCase("a/b/")] @@ -33,7 +32,72 @@ public sealed class FileExclusionsTests [TestCase("**/bin/")] public void InvalidExclusionListsAreRejected(string patterns) { - Assert.That(new DataSourceValidation().ValidateExcludePatterns(patterns), Is.Not.Null); + Assert.That(FileExclusions.Validate(patterns), Is.Not.Null); + } + + [TestCase(null, true)] + [TestCase("", true)] + [TestCase(" ; \r\n", true)] + [TestCase("build/;cmake-build-*/", true)] + [TestCase(".env/;archive.v1/;*.vmwarevm/", true)] + [TestCase("*.pdf", false)] + [TestCase("Dockerfile", false)] + [TestCase("node_modules", false)] + [TestCase("build/;*.txt", false)] + public void FolderFieldRequiresExplicitFolderRules(string? patterns, bool valid) + { + Assert.Multiple(() => + { + Assert.That(FileExclusions.IsValidFolders(patterns), Is.EqualTo(valid)); + Assert.That(FileExclusions.ValidateFolders(patterns) is null, Is.EqualTo(valid)); + }); + } + + [TestCase(null, true)] + [TestCase("", true)] + [TestCase(" ; \r\n", true)] + [TestCase("*.pdf;draft-*;report-?.md", true)] + [TestCase("Dockerfile;Caddyfile;.gitignore", true)] + [TestCase("build", true)] + [TestCase("build/", false)] + [TestCase("*.tmp;obj/", false)] + public void FileFieldRejectsFolderRulesButAcceptsExtensionlessFiles(string? patterns, bool valid) + { + Assert.Multiple(() => + { + Assert.That(FileExclusions.IsValidFiles(patterns), Is.EqualTo(valid)); + Assert.That(FileExclusions.ValidateFiles(patterns) is null, Is.EqualTo(valid)); + }); + } + + [TestCase(":text:")] + [TestCase(":documents:")] + [TestCase("src/build/")] + [TestCase("**/*.tmp")] + [TestCase("/")] + [TestCase("./")] + public void BothFieldValidatorsReuseTheExistingSyntaxErrors(string patterns) + { + Assert.Multiple(() => + { + Assert.That(FileExclusions.IsValidFolders(patterns), Is.False); + Assert.That(FileExclusions.IsValidFiles(patterns), Is.False); + Assert.That(FileExclusions.ValidateFolders(patterns), Is.EqualTo(FileExclusions.Validate(patterns))); + Assert.That(FileExclusions.ValidateFiles(patterns), Is.EqualTo(FileExclusions.Validate(patterns))); + }); + } + + [Test] + public void StoredListsStillAcceptBothKindsOfRulesTogether() + { + const string PATTERNS = "bin/;*.pdf;Dockerfile;.git/"; + Assert.Multiple(() => + { + Assert.That(FileExclusions.IsValid(PATTERNS), Is.True); + Assert.That(FileExclusions.Validate(PATTERNS), Is.Null); + Assert.That(FileExclusions.ExcludesFolder("bin", FileExclusions.Parse(PATTERNS)), Is.True); + Assert.That(FileExclusions.ExcludesFile("report.pdf", FileExclusions.Parse(PATTERNS)), Is.True); + }); } [Test] @@ -42,6 +106,18 @@ public sealed class FileExclusionsTests Assert.That(FileExclusions.Parse(" bin/ ;obj/\r\n\r\n*.min.js\nBIN/ "), Is.EqualTo(new[] { "bin/", "obj/", "*.min.js" })); } + [Test] + public void GroupingSplitsFoldersFromFilesAndDropsDuplicates() + { + var groups = FileExclusions.Group(" bin/ ;*.min.js\r\nobj/;BIN/;;\n*.MIN.js"); + + Assert.Multiple(() => + { + Assert.That(groups.Folders, Is.EqualTo("bin/; obj/")); + Assert.That(groups.Files, Is.EqualTo("*.min.js")); + }); + } + [Test] public void SettingsFromBeforeTheListGetTheDefault() { diff --git a/app/Tests/Tools/FilePatternExclusionTests.cs b/app/Tests/Tools/FilePatternExclusionTests.cs new file mode 100644 index 000000000..7addbdacf --- /dev/null +++ b/app/Tests/Tools/FilePatternExclusionTests.cs @@ -0,0 +1,153 @@ +using AIStudio.Tools; + +namespace AIStudio.Tests.Tools; + +public sealed class FilePatternExclusionTests +{ + [TestCase("*", "*.mp3;*.webm", "recording.WEBM", false)] + [TestCase("*", "*.mp3;*.webm", "report.pdf", true)] + [TestCase("*.txt;*.pdf", "draft-*", "draft-report.pdf", false)] + [TestCase("*.txt;*.pdf", "draft-*", "report.pdf", true)] + [TestCase("*.txt", "*.mp3", "report.pdf", false)] + [TestCase("*", "Dockerfile;Caddyfile", "Dockerfile", false)] + [TestCase("*", ".env", ".env", false)] + [TestCase("*", "*", "report.pdf", false)] + [TestCase("*", "", "report.pdf", true)] + [TestCase("*", "*-draft-*", "report-draft-final.md", false)] + [TestCase("*", "draft-*", "draft-", false)] + [TestCase("*", "report-?.md", "report-a.md", false)] + [TestCase("*", "report-?.md", "report-ab.md", true)] + public void ExclusionsOverrideIncludesButNeverSelectOtherFiles(string include, string exclude, string fileName, bool expected) + { + var path = Path.Combine(Path.GetTempPath(), "nested", fileName); + var selection = new FolderFileSelection( + FileNamePatterns.Parse(include), + FileExclusions.Parse(exclude), + false, + false, + FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES); + + Assert.That(selection.Selects(path), Is.EqualTo(expected)); + } + + [TestCase("build", true)] + [TestCase("app-build", true)] + [TestCase("my-build", true)] + [TestCase("BUILD", true)] + [TestCase("builder", false)] + [TestCase("build-output", false)] + public void TheFolderExampleMatchesNamesEndingInBuildButNotFiles(string name, bool expected) + { + var patterns = FileExclusions.Parse("*build/"); + var path = Path.Combine(Path.GetTempPath(), "test-data", name); + + Assert.Multiple(() => + { + Assert.That(FileExclusions.ExcludesFolder(path, patterns), Is.EqualTo(expected)); + Assert.That(FileExclusions.ExcludesFile(path, patterns), Is.False); + }); + } + + [TestCase(false, "", true, false)] + [TestCase(true, "", true, true)] + [TestCase(false, ".gitignore", false, false)] + [TestCase(true, ".gitignore", false, false)] + [TestCase(false, ".hidden/", true, false)] + [TestCase(true, ".hidden/", true, false)] + public async Task ExplicitDotPatternsBypassOnlyTheHiddenFileRuleNotExclusionsOrHiddenFolders( + bool includeHidden, string exclusions, bool selectsRootFile, bool selectsNestedFile) + { + var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-dot-pattern-priority-{Guid.NewGuid():N}"); + var hiddenDirectory = Path.Combine(directoryPath, ".hidden"); + var rootFile = Path.Combine(directoryPath, ".gitignore"); + var nestedFile = Path.Combine(hiddenDirectory, ".gitignore"); + Directory.CreateDirectory(hiddenDirectory); + + try + { + await File.WriteAllTextAsync(rootFile, "bin/"); + await File.WriteAllTextAsync(nestedFile, "obj/"); + var selection = new FolderFileSelection( + [".gitignore"], FileExclusions.Parse(exclusions), true, includeHidden, + FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES); + var selectedFiles = selection.EnumerateCandidates(directoryPath).Where(selection.Selects).ToArray(); + + Assert.Multiple(() => + { + Assert.That(selectedFiles.Contains(rootFile), Is.EqualTo(selectsRootFile)); + Assert.That(selectedFiles.Contains(nestedFile), Is.EqualTo(selectsNestedFile)); + Assert.That(selection.LeavesOut(directoryPath, rootFile), Is.EqualTo(!selectsRootFile)); + Assert.That(selection.LeavesOut(directoryPath, nestedFile), Is.EqualTo(!selectsNestedFile)); + }); + } + finally + { + Directory.Delete(directoryPath, true); + } + } + + [Test] + public void EnvironmentDefaultsExcludeVisibleEnvironmentFoldersRatherThanDotEnvFiles() + { + var defaults = FileExclusions.Parse(FileExclusions.DEFAULT_PATTERNS); + + Assert.Multiple(() => + { + Assert.That(FileExclusions.ExcludesFolder("env", defaults), Is.True); + Assert.That(FileExclusions.ExcludesFolder("venv", defaults), Is.True); + Assert.That(FileExclusions.ExcludesFile(".env", defaults), Is.False); + Assert.That(FileExclusions.ExcludesFolder(".env", defaults), Is.False, + "Hidden folders are handled by the hidden-file switch, not by the default exclusion list."); + }); + } + + [TestCase(null)] + [TestCase("")] + [TestCase(" ")] + public void ExclusionsMayBeEmptyWhileIncludesAreRequired(string? patterns) + { + Assert.Multiple(() => + { + Assert.That(FileExclusions.Validate(patterns), Is.Null); + Assert.That(FileExclusions.IsValid(patterns), Is.True); + Assert.That(FileNamePatterns.IsValid(patterns), Is.False); + }); + } + + [TestCase("**/*.txt")] + [TestCase("folder/*.txt")] + public void OptionalExclusionsStillRejectMalformedRules(string patterns) + { + Assert.Multiple(() => + { + Assert.That(FileExclusions.Validate(patterns), Is.Not.Null); + Assert.That(FileExclusions.IsValid(patterns), Is.False); + }); + } + + [Test] + public async Task TextSelectorsCanBeCombinedWithFilenameExclusions() + { + var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-exclusions-{Guid.NewGuid():N}"); + Directory.CreateDirectory(directoryPath); + try + { + var terraform = Path.Combine(directoryPath, "main.tf"); + var caddy = Path.Combine(directoryPath, "Caddyfile"); + await File.WriteAllTextAsync(terraform, "resource example {}"); + await File.WriteAllTextAsync(caddy, "localhost { respond hello }"); + var selection = new FolderFileSelection( + [":text:"], ["*.tf"], false, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES); + + Assert.Multiple(() => + { + Assert.That(selection.Selects(terraform), Is.False); + Assert.That(selection.Selects(caddy), Is.True); + }); + } + finally + { + Directory.Delete(directoryPath, true); + } + } +} \ No newline at end of file diff --git a/app/Tests/Tools/FileSourceIndexerWatcherTests.cs b/app/Tests/Tools/FileSourceIndexerWatcherTests.cs index 6d9314426..acff93320 100644 --- a/app/Tests/Tools/FileSourceIndexerWatcherTests.cs +++ b/app/Tests/Tools/FileSourceIndexerWatcherTests.cs @@ -271,6 +271,32 @@ public sealed class FileSourceIndexerWatcherTests Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.False); } + [Test] + public void ReorderingEquivalentExclusionsDoesNotRecreateTheWatcher() + { + var before = DefaultConfiguration(Path.GetTempPath()) with + { + ExcludePatterns = FileExclusions.Parse("*.mp3;draft-*"), + }; + var after = before with { ExcludePatterns = FileExclusions.Parse("DRAFT-*;*.MP3;*.mp3") }; + + Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.True); + } + + [Test] + public void RenamingAnIncludedFileIntoAnExcludedNameStillRefreshesTheIndex() + { + var rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-excluded-rename-{Guid.NewGuid():N}"); + var configuration = DefaultConfiguration(rootPath) with + { + FilePatterns = FileNamePatterns.Parse("*.md"), + ExcludePatterns = FileExclusions.Parse("draft-*"), + }; + var renamed = new RenamedEventArgs(WatcherChangeTypes.Renamed, rootPath, "draft-report.md", "report.md"); + + Assert.That(FileSourceIndexer.IsRelevantWatcherEvent(configuration, renamed), Is.True); + } + [Test] public void RelocatingToADirectoryWithDifferentCasingUsesTheLocalPathRules() { diff --git a/app/Tests/Tools/LocalDirectoryFilePatternsTests.cs b/app/Tests/Tools/LocalDirectoryFilePatternsTests.cs index c03cf7e3f..463a10a52 100644 --- a/app/Tests/Tools/LocalDirectoryFilePatternsTests.cs +++ b/app/Tests/Tools/LocalDirectoryFilePatternsTests.cs @@ -39,7 +39,8 @@ public sealed class LocalDirectoryFilePatternsTests [TestCase(null)] [TestCase("")] [TestCase(" ")] - [TestCase("*.txt;")] + [TestCase(";")] + [TestCase(" ; ; ")] [TestCase("folder/*.txt")] public void InvalidStoredPatternsFallBackWithoutRemovingExistingDocuments(string? storedPatterns) { @@ -48,12 +49,60 @@ public sealed class LocalDirectoryFilePatternsTests Assert.That(dataSource.EffectiveFilePatterns, Is.EqualTo(DataSourceLocalDirectory.DEFAULT_FILE_PATTERNS)); } + [Test] + public void BatchExclusionsSurviveSettingsSerialization() + { + var written = new DataBatchProcessing { ExcludePatterns = "*.mp3;*.webm;draft-*" }; + var json = JsonSerializer.Serialize(written, SettingsManager.JSON_OPTIONS); + var read = JsonSerializer.Deserialize(json, SettingsManager.JSON_OPTIONS); + + Assert.That(read!.ExcludePatterns, Is.EqualTo(written.ExcludePatterns)); + } + [Test] public void PatternsAreSplitAndTrimmed() { Assert.That(FileNamePatterns.Parse(" *.txt ; *.tf ; .gitignore "), Is.EqualTo(new[] { "*.txt", "*.tf", ".gitignore" })); } + [TestCase("*.txt;")] + [TestCase(";*.txt")] + [TestCase(";; *.txt ; ; ")] + public void EmptyEntriesAreIgnoredWithoutBroadeningTheSelection(string patterns) + { + var parsed = FileNamePatterns.Parse(patterns); + var dataSource = new DataSourceLocalDirectory { FilePatterns = patterns }; + var selection = new FolderFileSelection(parsed, [], false, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES); + var canonical = new FolderFileSelection(["*.txt"], [], false, false, FolderFileSelection.DEFAULT_MAX_FILE_SIZE_MEGABYTES); + + Assert.Multiple(() => + { + Assert.That(FileNamePatterns.IsValid(patterns), Is.True); + Assert.That(FileNamePatterns.Validate(patterns), Is.Null); + Assert.That(parsed, Is.EqualTo(new[] { "*.txt" })); + Assert.That(dataSource.EffectiveFilePatterns, Is.EqualTo(patterns), "Empty entries must not trigger the default document selection."); + Assert.That(selection.Selects("report.txt"), Is.True); + Assert.That(selection.Selects("report.md"), Is.False); + Assert.That(selection.Selects("recording.wav"), Is.False); + Assert.That(selection.Describe(), Is.EqualTo(canonical.Describe()), "Empty entries must not change the indexing selection signature."); + }); + } + + [TestCase(null)] + [TestCase("")] + [TestCase(" ")] + [TestCase(";")] + [TestCase(" ; ;; ")] + public void IncludesStillRequireAtLeastOneNonEmptyPattern(string? patterns) + { + Assert.Multiple(() => + { + Assert.That(FileNamePatterns.Parse(patterns), Is.Empty); + Assert.That(FileNamePatterns.IsValid(patterns), Is.False); + Assert.That(FileNamePatterns.Validate(patterns), Is.Not.Null); + }); + } + [Test] public void NormalizingPatternsRemovesCaseInsensitiveDuplicates() { @@ -75,6 +124,7 @@ public sealed class LocalDirectoryFilePatternsTests { Type = DataSourceType.LOCAL_DIRECTORY, FilePatterns = "*.txt;*.tf;.gitignore", + ExcludePatterns = "secret-*;*.backup", IncludeSubdirectories = false, IncludeHiddenFilesAndFolders = true, }; @@ -85,8 +135,10 @@ public sealed class LocalDirectoryFilePatternsTests Assert.Multiple(() => { Assert.That(json, Does.Not.Contain(nameof(DataSourceLocalDirectory.EffectiveFilePatterns))); + Assert.That(json, Does.Not.Contain(nameof(DataSourceLocalDirectory.EffectiveExcludePatterns))); Assert.That(read, Is.TypeOf()); Assert.That(((DataSourceLocalDirectory)read!).FilePatterns, Is.EqualTo("*.txt;*.tf;.gitignore")); + Assert.That(((DataSourceLocalDirectory)read).ExcludePatterns, Is.EqualTo("secret-*;*.backup")); Assert.That(((DataSourceLocalDirectory)read).IncludeSubdirectories, Is.False); Assert.That(((DataSourceLocalDirectory)read).IncludeHiddenFilesAndFolders, Is.True); }); @@ -419,17 +471,22 @@ public sealed class LocalDirectoryFilePatternsTests [TestCase(":documents:")] [TestCase(":DOCUMENTS:;*.tf")] [TestCase("report-??.md")] + [TestCase("*.txt;")] + [TestCase(";*.txt;; *.tf ; ;")] + [TestCase(":documents:;;")] public void DataSourceValidationAcceptsSimplePatterns(string patterns) { Assert.That(new DataSourceValidation().ValidateFilePatterns(patterns), Is.Null); } [TestCase("")] - [TestCase("*.txt;")] + [TestCase(" ; ;; ")] [TestCase("**/*.txt")] [TestCase("folder/*.txt")] [TestCase(":document:")] [TestCase("*.txt;:docs:")] + [TestCase(";*.txt;; :docs: ;")] + [TestCase("*.txt;;folder/*.md;")] public void DataSourceValidationRejectsInvalidPatterns(string patterns) { Assert.That(new DataSourceValidation().ValidateFilePatterns(patterns), Is.Not.Null); diff --git a/app/Tests/Tools/LongExtensionsTests.cs b/app/Tests/Tools/LongExtensionsTests.cs new file mode 100644 index 000000000..3cd7c416d --- /dev/null +++ b/app/Tests/Tools/LongExtensionsTests.cs @@ -0,0 +1,25 @@ +using System.Globalization; + +using AIStudio.Tools; + +namespace AIStudio.Tests.Tools; + +public sealed class LongExtensionsTests +{ + [TestCase(0L, "0 B")] + [TestCase(1_023L, "1023 B")] + [TestCase(1_536L, "1.5 kB")] + [TestCase(1_992_294L, "1.9 MB")] + [TestCase(1_073_741_824L, "1 GB")] + [TestCase(1_572_864_000L, "1.46 GB")] + public void FileSizesKeepTheirFraction(long sizeBytes, string expected) + { + Assert.That(sizeBytes.FileSize(CultureInfo.InvariantCulture), Is.EqualTo(expected)); + } + + [Test] + public void FileSizesUseTheDecimalSeparatorOfTheCulture() + { + Assert.That(1_992_294L.FileSize(CultureInfo.GetCultureInfo("de-DE")), Is.EqualTo("1,9 MB")); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/MediaFileSelectionTests.cs b/app/Tests/Tools/MediaFileSelectionTests.cs new file mode 100644 index 000000000..f19edf911 --- /dev/null +++ b/app/Tests/Tools/MediaFileSelectionTests.cs @@ -0,0 +1,192 @@ +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools; +using AIStudio.Tools.Media; +using AIStudio.Tools.Services; +using AIStudio.Tools.Services.Indexing; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks combinations of media, transcripts, and ordinary Markdown against the current folder +/// selection rules. Transcript reuse exercises the shared implementation used by RAG and batch. +/// +public sealed class MediaFileSelectionTests +{ + private const string TRANSCRIPT_TEXT = "# Transcript\nSpoken content from the recording."; + + private string rootPath = string.Empty; + private string userFolder = string.Empty; + + [SetUp] + public void CreateDirectory() + { + this.rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-media-selection-{Guid.NewGuid():N}"); + this.userFolder = Path.Combine(this.rootPath, "user"); + Directory.CreateDirectory(this.userFolder); + } + + [TearDown] + public void DeleteDirectory() => Directory.Delete(this.rootPath, true); + + [TestCase("*.wav", "*.md")] + [TestCase("*.wav;*.md", "*.md")] + [TestCase("*", "*.md")] + public async Task ExcludingMarkdownStillSelectsTheMedia(string includes, string excludes) + { + var mediaPath = await this.CreateMediaAndDocumentsAsync(); + + Assert.That(this.Discover(includes, excludes), Is.EqualTo(new[] { mediaPath })); + } + + [TestCase("*.md")] + [TestCase("*.wav;*.md")] + [TestCase("*")] + public async Task ExcludingMediaKeepsOrdinaryMarkdownAndTheIndependentTranscript(string includes) + { + var mediaPath = await this.CreateMediaAndDocumentsAsync(); + + Assert.That(this.Discover(includes, "*.wav"), Is.EquivalentTo(new[] + { + MediaSidecarTranscripts.GetFilePath(mediaPath), + Path.Combine(this.userFolder, "notes.md"), + Path.Combine(this.userFolder, "readme.md"), + })); + } + + [Test] + public async Task SelectingMediaAndMarkdownDoesNotProcessTheTranscriptTwice() + { + var mediaPath = await this.CreateMediaAndDocumentsAsync(); + + Assert.That(this.Discover("*.wav;*.md", string.Empty), Is.EquivalentTo(new[] + { + mediaPath, + Path.Combine(this.userFolder, "notes.md"), + Path.Combine(this.userFolder, "readme.md"), + })); + } + + [TestCase(false)] + [TestCase(true)] + public async Task ExistingTranscriptsAreReusedWithoutAProviderOrARewrite(bool dataSourceTranscript) + { + var mediaPath = await this.CreateMediaAndDocumentsAsync(); + IMediaTranscriptLocation location = dataSourceTranscript + ? this.TranscriptStore() + : MediaSidecarTranscripts.INSTANCE; + var slot = await location.ResolveAsync(mediaPath, CancellationToken.None); + Directory.CreateDirectory(Path.GetDirectoryName(slot.Path)!); + await File.WriteAllTextAsync(slot.Path, TRANSCRIPT_TEXT); + var originalWriteTime = File.GetLastWriteTimeUtc(slot.Path); + var calls = 0; + Task Transcribe(string _, TranscriptionProvider __, CancellationToken ___) + { + calls++; + throw new InvalidOperationException("An existing transcript must not ask a provider."); + } + + for (var attempt = 0; attempt < 2; attempt++) + { + var result = await PersistentMediaTranscript.ReadOrCreateAsync( + location, mediaPath, null, Transcribe, NullLogger.Instance, CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY)); + Assert.That(result.Reused, Is.True); + Assert.That(File.ReadAllText(result.TranscriptPath), Is.EqualTo(TRANSCRIPT_TEXT)); + Assert.That(File.GetLastWriteTimeUtc(slot.Path), Is.EqualTo(originalWriteTime)); + Assert.That(calls, Is.Zero); + }); + } + } + + [Test] + public async Task ChangedMediaDoesNotReuseAnOldDataSourceTranscriptEvenWithTheSameSizeAndWriteTime() + { + var mediaPath = await this.CreateMediaAndDocumentsAsync(); + var store = this.TranscriptStore(); + var originalSlot = await store.ResolveAsync(mediaPath, CancellationToken.None); + Directory.CreateDirectory(Path.GetDirectoryName(originalSlot.Path)!); + await File.WriteAllTextAsync(originalSlot.Path, TRANSCRIPT_TEXT); + var originalWriteTime = File.GetLastWriteTimeUtc(mediaPath); + + await File.WriteAllBytesAsync(mediaPath, [4, 5, 6]); + File.SetLastWriteTimeUtc(mediaPath, originalWriteTime); + var changedSlot = await store.ResolveAsync(mediaPath, CancellationToken.None); + var result = await PersistentMediaTranscript.ReadOrCreateAsync( + store, mediaPath, null, + (_, _, _) => throw new InvalidOperationException("No provider was configured."), + NullLogger.Instance, CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(changedSlot.Key, Is.Not.EqualTo(originalSlot.Key)); + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.NO_PROVIDER)); + Assert.That(File.ReadAllText(originalSlot.Path), Is.EqualTo(TRANSCRIPT_TEXT)); + }); + } + + [TestCase("*.wav", "*.md", false, false)] + [TestCase("*.md", "*.wav", true, true)] + [TestCase("*.wav;*.md", "", true, false)] + public async Task WatcherUsesTheSelectionOfTheMediaWhenDecidingAboutItsSidecar( + string includes, string excludes, bool markdownChangesMatter, bool transcriptChangesMatter) + { + var mediaPath = await this.CreateMediaAndDocumentsAsync(); + var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration( + this.userFolder, + false, + FilePatterns: FileNamePatterns.Parse(includes), + ExcludePatterns: FileExclusions.Parse(excludes)); + + Assert.Multiple(() => + { + Assert.That( + FileSourceIndexer.IsRelevantWatcherPath(configuration, Path.Combine(this.userFolder, "notes.md"), WatcherChangeTypes.Changed), + Is.EqualTo(markdownChangesMatter)); + Assert.That( + FileSourceIndexer.IsRelevantWatcherPath(configuration, MediaSidecarTranscripts.GetFilePath(mediaPath), WatcherChangeTypes.Changed), + Is.EqualTo(transcriptChangesMatter)); + }); + } + + private async Task CreateMediaAndDocumentsAsync() + { + var mediaPath = Path.Combine(this.userFolder, "test123.wav"); + // These bytes cannot be decoded: reuse must find the transcript before asking a provider. + await File.WriteAllBytesAsync(mediaPath, [1, 2, 3]); + await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(mediaPath), TRANSCRIPT_TEXT); + await File.WriteAllTextAsync(Path.Combine(this.userFolder, "notes.md"), "# Ordinary notes"); + await File.WriteAllTextAsync(Path.Combine(this.userFolder, "readme.md"), "# Ordinary readme"); + return mediaPath; + } + + private IReadOnlyList Discover(string includes, string excludes) + { + using var indexer = new FileSourceIndexer(null!, null!, null!, null!, NullLogger.Instance); + var result = indexer.GetInputFiles(new DataSourceLocalDirectory + { + Type = DataSourceType.LOCAL_DIRECTORY, + Path = this.userFolder, + FilePatterns = includes, + ExcludePatterns = excludes, + IncludeSubdirectories = false, + }, new DataSourceEmbeddingManifest(), + MediaTranscriptionAvailability.Allowed(new TranscriptionProvider { Id = "test-transcription-provider" })); + + Assert.Multiple(() => + { + Assert.That(result.Failures, Is.Empty); + Assert.That(result.IgnoredFiles, Is.Empty); + }); + + return result.Files.Select(file => file.FullName).ToArray(); + } + + private DataSourceTranscriptStore TranscriptStore() => new( + Path.Combine(this.rootPath, "data"), "d401ce54-9a42-4e78-85de-2402e07f7f7e"); +}