Added audio and video transcription for chats and assistants (#856)

app/MindWork AI Studio/Components/ReadFileContent.razor.cs

@@ -1,3 +1,5 @@
+using AIStudio.Dialogs;
+using AIStudio.Tools.Media;
 using AIStudio.Tools.Rust;
 using AIStudio.Tools.Services;
 using AIStudio.Tools.Validation;
@@ -8,6 +10,22 @@ namespace AIStudio.Components;
 
 public partial class ReadFileContent : MSGComponentBase
 {
+    private readonly MediaImportOwner fallbackMediaImportOwner = new(MediaImportOwnerKind.ASSISTANT, $"read-file-content:{Guid.NewGuid():N}");
+
+    [CascadingParameter]
+    private MediaImportOwner? ImportOwner { get; set; }
+
+    private MediaImportOwner EffectiveImportOwner => this.ImportOwner ?? this.fallbackMediaImportOwner;
+
+    [Parameter]
+    public string MediaImportTargetId { get; set; } = string.Empty;
+
+    private string EffectiveMediaImportTargetId => string.IsNullOrWhiteSpace(this.MediaImportTargetId)
+        ? string.IsNullOrWhiteSpace(this.Text) ? "primary" : this.Text
+        : this.MediaImportTargetId;
+
+    private MediaImportTarget EffectiveMediaImportTarget => new(this.EffectiveImportOwner, this.EffectiveMediaImportTargetId);
+
     [Parameter]
     public string Text { get; set; } = string.Empty;
     
@@ -47,17 +65,24 @@ public partial class ReadFileContent : MSGComponentBase
     [Inject]
     private PandocAvailabilityService PandocAvailabilityService { get; init; } = null!;
 
+    [Inject]
+    private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
+
     private const string DEFAULT_DRAG_CLASS = "relative rounded-lg border-2 border-dashed pa-3 mb-3 mud-width-full";
 
     private string ButtonText => string.IsNullOrWhiteSpace(this.Text) ? T("Use file content as input") : this.Text;
     private string dragClass = DEFAULT_DRAG_CLASS;
     private uint numDropAreasAboveThis;
     private bool isComponentHovered;
+    private bool IsCurrentTargetBusy => this.MediaTranscriptionService.GetSnapshot(this.EffectiveImportOwner) is { IsBusy: true } snapshot
+                                        && snapshot.Target == this.EffectiveMediaImportTarget;
+    private bool IsUnavailable => this.Disabled || this.MediaTranscriptionService.IsBusy(this.EffectiveImportOwner);
 
     #region Overrides of MSGComponentBase
 
     protected override async Task OnInitializedAsync()
     {
+        this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged;
         if (this.EnableDragDrop)
         {
             this.ApplyFilters([], [ Event.TAURI_EVENT_RECEIVED, Event.REGISTER_FILE_DROP_AREA, Event.UNREGISTER_FILE_DROP_AREA ]);
@@ -65,6 +90,69 @@ public partial class ReadFileContent : MSGComponentBase
         }
 
         await base.OnInitializedAsync();
+        await this.SyncCompletedMediaTextAsync();
+    }
+
+    /// <summary>Refreshes disabled controls when the shared import lane changes.</summary>
+    private void OnMediaImportStateChanged(MediaImportOwner owner)
+    {
+        if (owner == this.EffectiveImportOwner)
+            _ = this.InvokeAsync(async () =>
+            {
+                await this.SyncCompletedMediaTextAsync();
+                await this.ConsumeStandaloneMediaOutcomeAsync();
+                this.StateHasChanged();
+            });
+    }
+
+    /// <summary>Consumes outcomes for dialog-local controls that have no assistant owner surface.</summary>
+    private async Task ConsumeStandaloneMediaOutcomeAsync()
+    {
+        if (this.ImportOwner is not null)
+            return;
+
+        var outcome = this.MediaTranscriptionService.TryConsumeOutcome(this.EffectiveImportOwner);
+        if (outcome is null)
+            return;
+
+        if (outcome.Failures.Count > 0)
+        {
+            var message = string.Join(Environment.NewLine, outcome.Failures.Select(failure => $"{failure.FileName}: {failure.UserMessage}"));
+            await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, message));
+        }
+        else if (outcome.Status is MediaImportStatus.FAILED)
+        {
+            await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed.")));
+        }
+
+        if (outcome.Warnings.Count > 0)
+        {
+            var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}"));
+            await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message));
+        }
+
+        if (outcome.Status is MediaImportStatus.CANCELLED)
+        {
+            await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled.")));
+        }
+    }
+
+    /// <summary>Applies a completed target transcript after progress or navigation.</summary>
+    private async Task SyncCompletedMediaTextAsync()
+    {
+        var delivery = this.MediaTranscriptionService.GetPendingDelivery(this.EffectiveMediaImportTarget);
+        if (delivery is null || delivery.Text is not { } text)
+            return;
+
+        await this.FileContentChanged.InvokeAsync(text);
+        this.MediaTranscriptionService.AcknowledgeDelivery(delivery);
+    }
+
+    /// <summary>Unsubscribes from the singleton media service.</summary>
+    protected override void DisposeResources()
+    {
+        this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged;
+        base.DisposeResources();
     }
 
     protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
@@ -72,7 +160,7 @@ public partial class ReadFileContent : MSGComponentBase
         if (!this.EnableDragDrop)
             return;
 
-        if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
+        if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
             return;
 
         switch (triggeredEvent)
@@ -126,10 +214,7 @@ public partial class ReadFileContent : MSGComponentBase
     
     private async Task SelectFile()
     {
-        if (this.Disabled)
-            return;
-
-        if (!await this.EnsurePandocAvailability())
+        if (this.IsUnavailable)
             return;
 
         var selectedFile = await this.RustService.SelectFile(T("Select file to read its content"));
@@ -161,9 +246,6 @@ public partial class ReadFileContent : MSGComponentBase
 
     private async Task LoadFirstValidFile(List<string> paths)
     {
-        if (!await this.EnsurePandocAvailability())
-            return;
-
         foreach (var path in paths)
         {
             if (await this.LoadFileIfValid(path))
@@ -179,6 +261,12 @@ public partial class ReadFileContent : MSGComponentBase
             return false;
         }
 
+        if (FileTypes.IsAllowedPath(filePath, FileTypes.AUDIO) || FileTypes.IsAllowedPath(filePath, FileTypes.VIDEO))
+            return await this.LoadMediaTranscriptAsync(filePath);
+
+        if (!await this.EnsurePandocAvailability())
+            return false;
+
         if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.DIRECTLY_LOADING_CONTENT, filePath))
         {
             this.Logger.LogWarning("User attempted to load unsupported file: {FilePath}", filePath);
@@ -200,6 +288,42 @@ public partial class ReadFileContent : MSGComponentBase
         }
     }
 
+    private async Task<bool> LoadMediaTranscriptAsync(string filePath)
+    {
+        if (string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider))
+        {
+            await this.MessageBus.SendWarning(new(
+                Icons.Material.Filled.VoiceChat,
+                this.T("Media files require a configured transcription provider. Configure one in the transcription settings.")));
+            return false;
+        }
+
+        var message = this.T("The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider.");
+        var dialogParameters = new DialogParameters<ConfirmDialog>
+        {
+            {
+                x => x.MarkdownBody,
+                $"""
+                 {message}
+
+                 - {Markdown.EscapeInlineText(Path.GetFileName(filePath))}
+                 """
+            },
+        };
+        var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(
+            this.T("Transcribe media file"),
+            dialogParameters,
+            Dialogs.DialogOptions.FULLSCREEN);
+
+        var dialogResult = await dialogReference.Result;
+        if (dialogResult is null || dialogResult.Canceled)
+            return false;
+
+        return this.MediaTranscriptionService.TryStartTextImport(
+            filePath,
+            this.EffectiveMediaImportTarget);
+    }
+
     private bool CanCatchDroppedFile() => this.numDropAreasAboveThis is 0 && (this.isComponentHovered || this.CatchAllDocuments);
 
     private void SetDragClass() => this.dragClass = $"{DEFAULT_DRAG_CLASS} mud-border-primary border-2";
@@ -208,7 +332,7 @@ public partial class ReadFileContent : MSGComponentBase
 
     private void OnMouseEnter(EventArgs _)
     {
-        if(this.Disabled || this.numDropAreasAboveThis > 0)
+        if(this.IsUnavailable || this.numDropAreasAboveThis > 0)
             return;
 
         this.Logger.LogDebug("Read file content component is hovered.");
@@ -219,7 +343,7 @@ public partial class ReadFileContent : MSGComponentBase
 
     private void OnMouseLeave(EventArgs _)
     {
-        if(this.Disabled)
+        if(this.IsUnavailable)
             return;
 
         this.Logger.LogDebug("Read file content component is no longer hovered.");