Added file patterns and exclusions to folder data sources (#1057)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>

app/MindWork AI Studio/Components/ConfigurationNumber.razor

@@ -0,0 +1,4 @@
+@typeparam T
+@inherits ConfigurationBaseCore
+
+<MudNumericField T="@T" Value="@this.Value()" ValueChanged="@this.OptionChanged" Min="@this.Min" Max="@this.Max" Adornment="Adornment.End" AdornmentText="@this.Unit" Disabled="@this.IsDisabled" Variant="Variant.Text" Margin="Margin.Dense" Class="mb-1"/>

app/MindWork AI Studio/Components/ConfigurationNumber.razor.cs

@@ -0,0 +1,85 @@
+using System.Numerics;
+
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Components;
+
+/// <summary>
+/// A number typed into a field, for a range too wide for a slider, e.g., a size limit in megabytes.
+/// </summary>
+public partial class ConfigurationNumber<T> : ConfigurationBaseCore where T : struct, INumber<T>, IMinMaxValue<T>
+{
+    /// <summary>
+    /// The smallest allowed value.
+    /// </summary>
+    [Parameter]
+    public T Min { get; set; } = T.MinValue;
+
+    /// <summary>
+    /// The largest allowed value.
+    /// </summary>
+    [Parameter]
+    public T Max { get; set; } = T.MaxValue;
+
+    /// <summary>
+    /// The unit to display next to the value.
+    /// </summary>
+    [Parameter]
+    public string Unit { get; set; } = string.Empty;
+
+    /// <summary>
+    /// The value used for the field.
+    /// </summary>
+    [Parameter]
+    public Func<T> Value { get; set; } = () => T.Zero;
+
+    /// <summary>
+    /// An action which is called when the option is changed.
+    /// </summary>
+    [Parameter]
+    public Action<T> ValueUpdate { get; set; } = _ => { };
+
+    #region Overrides of ConfigurationBase
+
+    /// <inheritdoc />
+    protected override bool Stretch => true;
+
+    /// <inheritdoc />
+    protected override Variant Variant => Variant.Outlined;
+
+    protected override string Label => this.OptionDescription;
+
+    #endregion
+
+    #region Overrides of ComponentBase
+
+    protected override async Task OnInitializedAsync()
+    {
+        await this.EnsureMinMax();
+        await base.OnInitializedAsync();
+    }
+
+    protected override async Task OnParametersSetAsync()
+    {
+        await this.EnsureMinMax();
+        await base.OnParametersSetAsync();
+    }
+
+    #endregion
+
+    private async Task OptionChanged(T updatedValue)
+    {
+        this.ValueUpdate(updatedValue);
+        await this.SettingsManager.StoreSettings();
+        await this.InformAboutChange();
+    }
+
+    private async Task EnsureMinMax()
+    {
+        if (this.Value() < this.Min)
+            await this.OptionChanged(this.Min);
+
+        else if (this.Value() > this.Max)
+            await this.OptionChanged(this.Max);
+    }
+}

app/MindWork AI Studio/Components/ConfigurationText.razor

@@ -7,7 +7,7 @@
         Text="@this.Text()"
         TextChanged="@this.InternalUpdate"
         Disabled="@this.IsDisabled"
-        Adornment="Adornment.Start"
+        Adornment="@this.IconAdornment"
         AdornmentIcon="@this.Icon"
         AdornmentColor="@this.IconColor"
         UserAttributes="@SPELLCHECK_ATTRIBUTES"
@@ -21,13 +21,14 @@
 }
 else
 {
-    <MudStack Row="true" AlignItems="AlignItems.Center" Spacing="2">
+    @* One element, since a locked option places it next to its lock icon: *@
+    <div>
         <MudTextField
             T="string"
             Text="@this.Text()"
             TextChanged="@this.InternalUpdate"
             Disabled="@this.IsDisabled"
-            Adornment="Adornment.Start"
+            Adornment="@this.IconAdornment"
             AdornmentIcon="@this.Icon"
             AdornmentColor="@this.IconColor"
             UserAttributes="@SPELLCHECK_ATTRIBUTES"
@@ -37,10 +38,7 @@ else
             Immediate="@true"
             Validation="@this.Validation"
             Underline="false"
-            Class="flex-grow-1"
         />
-        <MudButton Variant="Variant.Outlined" StartIcon="@Icons.Material.Filled.Restore" Disabled="@this.IsDisabled" OnClick="@this.ResetTextAsync" Class="mb-1">
-            @this.ResetButtonText
-        </MudButton>
-    </MudStack>
+        <RestoreDefaultButton Text="@this.ResetButtonText" Disabled="@this.IsDisabled" OnClick="@this.ResetTextAsync"/>
+    </div>
 }

app/MindWork AI Studio/Components/ConfigurationText.razor.cs

@@ -19,7 +19,7 @@ public partial class ConfigurationText : ConfigurationBaseCore
     public Action<string> TextUpdate { get; set; } = _ => { };
 
     /// <summary>
-    /// The icon to display next to the textfield.
+    /// The icon to display next to the textfield, when it has a single line.
     /// </summary>
     [Parameter]
     public string Icon { get; set; } = Icons.Material.Filled.Info;
@@ -93,6 +93,12 @@ public partial class ConfigurationText : ConfigurationBaseCore
     
     private int GetMaxLines => this.AutoGrow ? this.MaxLines : 1;
 
+    //
+    // MudBlazor centers the icon vertically, so in a field of several lines it floats in the middle
+    // of the text. Such a field goes without:
+    //
+    private Adornment IconAdornment => this.NumLines == 1 ? Adornment.Start : Adornment.None;
+
     private void InternalUpdate(string text)
     {
         this.timer.Stop();

app/MindWork AI Studio/Components/DataSourceCloudEmbeddingWarning.razor.cs

@@ -32,12 +32,17 @@ public partial class DataSourceCloudEmbeddingWarning : MSGComponentBase
         }
     }
 
+    /// <remarks>
+    /// Which files of a folder are sent depends on its file patterns and switches. The sentence does
+    /// not repeat them: they are right next to it in the dialog, and a sentence per combination would
+    /// have to change with every new setting.
+    /// </remarks>
     private string GetSubjectText()
     {
         if (string.IsNullOrWhiteSpace(this.SourcePath))
             return this.DataSourceType switch
             {
-                DataSourceType.LOCAL_DIRECTORY => T("All files in this folder and its subfolders"),
+                DataSourceType.LOCAL_DIRECTORY => T("Every file this data source selects from this folder"),
                 DataSourceType.LOCAL_FILE => T("The selected file"),
                 DataSourceType.MAILBOX => T("Every mail of this mailbox in the selected folder and period, together with the text of its attachments,"),
                 _ => T("The selected data")
@@ -45,7 +50,7 @@ public partial class DataSourceCloudEmbeddingWarning : MSGComponentBase
 
         return this.DataSourceType switch
         {
-            DataSourceType.LOCAL_DIRECTORY => string.Format(T("All files in the folder '{0}' and its subfolders"), this.SourcePath),
+            DataSourceType.LOCAL_DIRECTORY => string.Format(T("Every file this data source selects from the folder '{0}'"), this.SourcePath),
             DataSourceType.LOCAL_FILE => string.Format(T("The file '{0}'"), this.SourcePath),
             DataSourceType.MAILBOX => string.Format(T("Every mail of the mailbox '{0}' in the selected folder and period, together with the text of its attachments,"), this.SourcePath),
             _ => string.Format(T("The data source '{0}'"), this.SourcePath)

app/MindWork AI Studio/Components/DataSourceManagement.razor

@@ -42,9 +42,18 @@
     <RowTemplate>
         @{ var embeddingStatus = embeddingStatuses.GetValueOrDefault(context.Id); }
         <MudTd>@context.Num</MudTd>
-        <MudTd Style="white-space: normal; overflow-wrap: anywhere;">@context.Name</MudTd>
-        <MudTd Style="white-space: nowrap;">@context.Type.GetDisplayName()</MudTd>
-        <MudTd Style="white-space: nowrap;">@this.GetEmbeddingName(context)</MudTd>
+        @*
+            The name gets the room the other columns leave, which in the dialog was next to nothing: it
+            may break anywhere, so a single character was enough for the browser. The minimum width
+            keeps it readable, and the type and the embedding wrap instead, so that the table still fits.
+            The minimum sits on a div, since browsers differ in whether they honor it on a table cell.
+            With "break-word" instead, a long name without spaces would push the table out of the dialog.
+        *@
+        <MudTd Style="white-space: normal; overflow-wrap: anywhere;">
+            <div style="min-width: 10em;">@context.Name</div>
+        </MudTd>
+        <MudTd>@context.Type.GetDisplayName()</MudTd>
+        <MudTd>@this.GetEmbeddingName(context)</MudTd>
         <MudTd>
             @if (context is IIndexedDataSource)
             {

app/MindWork AI Studio/Components/DataSourceManagement.razor.cs

@@ -83,17 +83,21 @@ public partial class DataSourceManagement : MSGComponentBase
     }
 
     /// <remarks>
-    /// Files which were skipped for good are none of the failed ones, so a data source made of
-    /// scanned documents stays green: there is nothing here for the user to fix.
+    /// Files which were skipped for good are none of the failed ones, so an otherwise usable data
+    /// source stays green. The shared health policy warns when none of the selected documents could
+    /// be indexed.
     /// </remarks>
     private static Color GetIndexingStatusColor(DataSourceEmbeddingStatus? status)
     {
         if (status is null || status.State is DataSourceEmbeddingState.IDLE or DataSourceEmbeddingState.QUEUED or DataSourceEmbeddingState.RUNNING)
             return Color.Warning;
 
-        return status.State is DataSourceEmbeddingState.FAILED || status.FailedDocuments > 0
-            ? Color.Error
-            : Color.Success;
+        return status.Health switch
+        {
+            DataSourceEmbeddingHealth.ERROR => Color.Error,
+            DataSourceEmbeddingHealth.WARNING => Color.Warning,
+            _ => Color.Success,
+        };
     }
 
     /// <summary>
@@ -109,11 +113,11 @@ public partial class DataSourceManagement : MSGComponentBase
         if (status is null)
             return T("Waiting for indexing status");
 
-        // Only files are skipped for want of readable text. A mail is read from the server, never from a file:
-        if (status.PermanentlySkippedDocuments == 0 || status.DataSourceType is DataSourceType.MAILBOX)
+        // Only files have enumeration exclusions. A mail is read from the server, never from a file:
+        if (status.SkippedDocuments == 0 || status.DataSourceType is DataSourceType.MAILBOX)
             return status.StateLabel;
 
-        return $"{status.StateLabel} — {string.Format(T("{0} files were skipped because they contain no readable text. AI Studio reads them again once they change."), status.PermanentlySkippedDocuments)}";
+        return $"{status.StateLabel} — {string.Format(T("Skipped files: {0}. Open Background embeddings to see why and what you can do."), status.SkippedDocuments)}";
     }
 
     private string GetEmbeddingName(IDataSourceBase dataSource)

app/MindWork AI Studio/Components/RestoreDefaultButton.razor

@@ -0,0 +1,5 @@
+<MudStack Row="@true" Justify="Justify.FlexEnd" Class="@this.Class">
+    <MudButton Variant="Variant.Text" Size="Size.Small" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Restore" Disabled="@this.Disabled" OnClick="@this.OnClick">
+        @this.Text
+    </MudButton>
+</MudStack>

app/MindWork AI Studio/Components/RestoreDefaultButton.razor.cs

@@ -0,0 +1,35 @@
+using Microsoft.AspNetCore.Components;
+
+namespace AIStudio.Components;
+
+/// <summary>
+/// A small button below a field, which puts the default value back into that field.
+/// </summary>
+/// <remarks>
+/// The button takes a row of its own, aligned to the right. Next to the field, it took away the width
+/// the field needs, wrapped or got cut off with a longer text, and every field of a form ended at
+/// another place, depending on the length of its button.
+/// </remarks>
+public partial class RestoreDefaultButton : ComponentBase
+{
+    /// <summary>
+    /// The text on the button, e.g., "Restore default".
+    /// </summary>
+    [Parameter]
+    public string Text { get; set; } = string.Empty;
+
+    /// <summary>
+    /// Puts the default value back into the field.
+    /// </summary>
+    [Parameter]
+    public EventCallback OnClick { get; set; }
+
+    [Parameter]
+    public bool Disabled { get; set; }
+
+    /// <summary>
+    /// The CSS classes of the row, e.g., for the space below it.
+    /// </summary>
+    [Parameter]
+    public string Class { get; set; } = string.Empty;
+}