Added transcription of audio and video files to local data sources (#1081)

Co-authored-by: Paul Koudelka <106623909+PaulKoudelka@users.noreply.github.com>

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor

@@ -90,7 +90,7 @@
                 Class="mb-3"
                 Immediate="@true"
                 Validation="@this.dataSourceValidation.ValidateFilePatterns"
-                HelperText="@T("Which files should be indexed? Separate multiple patterns with a semicolon, e.g., *.txt;*.pdf;*.docx;.gitignore. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file.")"
+                HelperText="@T("Which files should be indexed? Separate multiple patterns with a semicolon, e.g., *.txt;*.pdf;*.docx;.gitignore. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file. Audio and video files, e.g., *.mp3 or *.mp4, are transcribed by your transcription provider.")"
                 Adornment="Adornment.Start"
                 AdornmentIcon="@Icons.Material.Filled.FilterAlt"
                 AdornmentColor="Color.Info"
@@ -117,20 +117,17 @@
                 }
             </MudSelect>
             
-            @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
+            @if (this.CurrentCloudProcessing is not CloudProcessing.NONE)
             {
-                if (this.SelectedCloudEmbedding)
-                {
-                    <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
-                }
-                else
-                {
-                    <MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
-                        @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
-                    </MudJustifiedText>
-                }
+                <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_DIRECTORY" SourcePath="@this.dataPath" SendsEmbeddingsToCloud="@this.SelectedCloudEmbedding" CloudTranscriptionProvider="@this.CloudTranscriptionProviderName" @bind-UserAcknowledged="@this.UserAcknowledgedCloudProcessing" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
             }
-            
+            else if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
+            {
+                <MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
+                    @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
+                </MudJustifiedText>
+            }
+
             <ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
 
             <MudSelect @ref="@this.confidenceLevelSelect" @bind-Value="@this.dataConfidenceLevel" Text="@this.dataConfidenceLevel.GetName()" Label="@T("Required provider confidence level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start" Validation="@this.dataSourceValidation.ValidateDataSourceConfidenceLevel">
@@ -141,7 +138,14 @@
                     </MudSelectItem>
                 }
             </MudSelect>
-            
+
+            @if (this.MediaTranscriptionHint is { } mediaTranscriptionHint)
+            {
+                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Dense="@true" Class="mb-3">
+                    @mediaTranscriptionHint
+                </MudAlert>
+            }
+
             <MudStack Class="mb-3">
                 <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
                     @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))

app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs

@@ -42,6 +42,9 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     [Inject]
     private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
 
+    [Inject]
+    private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
+
     private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
     
     private readonly DataSourceValidation dataSourceValidation;
@@ -59,7 +62,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
     private string dataId = Guid.NewGuid().ToString();
     private string dataName = string.Empty;
     private string dataDescription = string.Empty;
-    private bool dataUserAcknowledgedCloudEmbedding;
+    private CloudProcessing acknowledgedCloudProcessing;
     private string dataEmbeddingId = string.Empty;
     private string dataPath = string.Empty;
     private string dataFilePatterns = DataSourceLocalDirectory.DEFAULT_FILE_PATTERNS;
@@ -88,6 +91,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
         this.dataSourceValidation = new()
         {
             GetSelectedCloudEmbedding = () => this.SelectedCloudEmbedding,
+            GetSelectedCloudTranscription = () => this.SelectedCloudTranscription,
             GetSelectedEmbeddingProvider = () => this.SelectedEmbedding,
             GetConfidenceLevel = () => this.dataConfidenceLevel,
             GetSettingsManager = () => this.SettingsManager,
@@ -162,6 +166,42 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
 
     private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
+    /// <summary>
+    /// Whether the file patterns may select audio or video files, which the data source transcribes.
+    /// </summary>
+    private bool SelectsMedia => FileNamePatterns.MaySelectMedia(FileNamePatterns.Parse(this.dataFilePatterns));
+
+    /// <summary>
+    /// The transcription provider which may hear the audio and video files of this data source, or null.
+    /// </summary>
+    private TranscriptionProvider? AllowedTranscriptionProvider => this.MediaTranscriptionService.GetUsableProvider(this.CreateDataSource());
+
+    private bool SelectedCloudTranscription => this.SelectsMedia && this.AllowedTranscriptionProvider is { IsSelfHosted: false };
+
+    private string CloudTranscriptionProviderName => this.SelectedCloudTranscription ? this.AllowedTranscriptionProvider?.Name ?? string.Empty : string.Empty;
+
+    private CloudProcessing CurrentCloudProcessing => (this.SelectedCloudEmbedding ? CloudProcessing.EMBEDDING : CloudProcessing.NONE) | (this.SelectedCloudTranscription ? CloudProcessing.TRANSCRIPTION : CloudProcessing.NONE);
+
+    /// <remarks>
+    /// The acknowledgment holds for the providers it was given for, cf. CloudProcessingExtensions.Covers.
+    /// </remarks>
+    private bool UserAcknowledgedCloudProcessing
+    {
+        get => this.acknowledgedCloudProcessing.Covers(this.CurrentCloudProcessing);
+        set => this.acknowledgedCloudProcessing = value ? this.CurrentCloudProcessing : CloudProcessing.NONE;
+    }
+
+    /// <summary>
+    /// Why audio and video files which the patterns select would not be indexed, or null when they would.
+    /// </summary>
+    private string? MediaTranscriptionHint => !this.SelectsMedia
+        ? null
+        : this.MediaTranscriptionService.GetUsableProvider() is null
+            ? T("Audio and video files which this data source selects are skipped until you choose a transcription provider in the settings.")
+            : this.AllowedTranscriptionProvider is null
+                ? T("Audio and video files which this data source selects are skipped, because your transcription provider does not meet the confidence level this data source requires.")
+                : null;
+
     private bool CanChangeSource => !this.IsEditing || !this.LockSource;
 
     private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor

@@ -43,7 +43,7 @@
             />
 
             <MudJustifiedText Typo="Typo.body1" Class="mb-3">
-                @T("Select a file for this data source. The content of this file will be processed for the data source.")
+                @T("Select a file for this data source. The content of this file will be processed for the data source. An audio or video file is transcribed by your transcription provider first.")
             </MudJustifiedText>
             @if (!this.CanChangeSource)
             {
@@ -90,18 +90,15 @@
                 }
             </MudSelect>
             
-            @if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
+            @if (this.CurrentCloudProcessing is not CloudProcessing.NONE)
             {
-                if (this.SelectedCloudEmbedding)
-                {
-                    <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_FILE" SourcePath="@this.dataFilePath" @bind-UserAcknowledged="@this.dataUserAcknowledgedCloudEmbedding" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
-                }
-                else
-                {
-                    <MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
-                        @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
-                    </MudJustifiedText>
-                }
+                <DataSourceCloudEmbeddingWarning DataSourceType="DataSourceType.LOCAL_FILE" SourcePath="@this.dataFilePath" SendsEmbeddingsToCloud="@this.SelectedCloudEmbedding" CloudTranscriptionProvider="@this.CloudTranscriptionProviderName" @bind-UserAcknowledged="@this.UserAcknowledgedCloudProcessing" Validation="@this.dataSourceValidation.ValidateUserAcknowledgedCloudEmbedding"/>
+            }
+            else if (!string.IsNullOrWhiteSpace(this.dataEmbeddingId))
+            {
+                <MudJustifiedText Typo="Typo.body1" Color="Color.Tertiary" Class="mb-3">
+                    @T("The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.")
+                </MudJustifiedText>
             }
             
             <ManagePandocDependency IntroText="@T("For some data types, such as Office files, MindWork AI Studio requires the open-source application Pandoc.")"/>
@@ -115,6 +112,13 @@
                 }
             </MudSelect>
 
+            @if (this.MediaTranscriptionHint is { } mediaTranscriptionHint)
+            {
+                <MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Dense="@true" Class="mb-3">
+                    @mediaTranscriptionHint
+                </MudAlert>
+            }
+
             <MudStack Class="mb-3">
                 <MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
                     @(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))

app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs

@@ -2,6 +2,7 @@ using AIStudio.Components;
 using AIStudio.Provider;
 using AIStudio.Settings;
 using AIStudio.Settings.DataModel;
+using AIStudio.Tools.Media;
 using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
 
@@ -42,6 +43,9 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     [Inject]
     private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
 
+    [Inject]
+    private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
+
     private static readonly Dictionary<string, object?> SPELLCHECK_ATTRIBUTES = new();
     
     private readonly DataSourceValidation dataSourceValidation;
@@ -59,7 +63,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
     private string dataId = Guid.NewGuid().ToString();
     private string dataName = string.Empty;
     private string dataDescription = string.Empty;
-    private bool dataUserAcknowledgedCloudEmbedding;
+    private CloudProcessing acknowledgedCloudProcessing;
     private string dataEmbeddingId = string.Empty;
     private string dataFilePath = string.Empty;
     private int dataMaxChunkTokenLength;
@@ -83,6 +87,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
         this.dataSourceValidation = new()
         {
             GetSelectedCloudEmbedding = () => this.SelectedCloudEmbedding,
+            GetSelectedCloudTranscription = () => this.SelectedCloudTranscription,
             GetSelectedEmbeddingProvider = () => this.SelectedEmbedding,
             GetConfidenceLevel = () => this.dataConfidenceLevel,
             GetSettingsManager = () => this.SettingsManager,
@@ -152,6 +157,42 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
 
     private bool SelectedCloudEmbedding => this.SelectedEmbedding is { IsSelfHosted: false };
 
+    /// <summary>
+    /// Whether the selected file is an audio or video file, which the data source transcribes.
+    /// </summary>
+    private bool SelectsMedia => PersistentMediaTranscript.IsMediaPath(this.dataFilePath);
+
+    /// <summary>
+    /// The transcription provider which may hear the audio and video files of this data source, or null.
+    /// </summary>
+    private TranscriptionProvider? AllowedTranscriptionProvider => this.MediaTranscriptionService.GetUsableProvider(this.CreateDataSource());
+
+    private bool SelectedCloudTranscription => this.SelectsMedia && this.AllowedTranscriptionProvider is { IsSelfHosted: false };
+
+    private string CloudTranscriptionProviderName => this.SelectedCloudTranscription ? this.AllowedTranscriptionProvider?.Name ?? string.Empty : string.Empty;
+
+    private CloudProcessing CurrentCloudProcessing => (this.SelectedCloudEmbedding ? CloudProcessing.EMBEDDING : CloudProcessing.NONE) | (this.SelectedCloudTranscription ? CloudProcessing.TRANSCRIPTION : CloudProcessing.NONE);
+
+    /// <remarks>
+    /// The acknowledgment holds for the providers it was given for, cf. CloudProcessingExtensions.Covers.
+    /// </remarks>
+    private bool UserAcknowledgedCloudProcessing
+    {
+        get => this.acknowledgedCloudProcessing.Covers(this.CurrentCloudProcessing);
+        set => this.acknowledgedCloudProcessing = value ? this.CurrentCloudProcessing : CloudProcessing.NONE;
+    }
+
+    /// <summary>
+    /// Why the selected audio or video file would not be indexed, or null when it would.
+    /// </summary>
+    private string? MediaTranscriptionHint => !this.SelectsMedia
+        ? null
+        : this.MediaTranscriptionService.GetUsableProvider() is null
+            ? T("This audio or video file is skipped until you choose a transcription provider in the settings.")
+            : this.AllowedTranscriptionProvider is null
+                ? T("This audio or video file is skipped, because your transcription provider does not meet the confidence level this data source requires.")
+                : null;
+
     private bool CanChangeSource => !this.IsEditing || !this.LockSource;
 
     private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();