mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-14 07:33:48 +00:00
Added audio and video transcription for chats and assistants (#856)
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
This commit is contained in:
1 parent
5dc6282908
commit
aaf77b6882
83 files changed
+5985
-442
No files matched your search
@@ -1,6 +1,9 @@
|
||||
using AIStudio.Dialogs.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.AssistantSessions;
|
||||
using AIStudio.Tools.Media;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
using Microsoft.AspNetCore.Components;
|
||||
using DialogOptions = AIStudio.Dialogs.DialogOptions;
|
||||
|
||||
@@ -60,6 +63,9 @@ public partial class AssistantBlock<TSettings> : MSGComponentBase where TSetting
|
||||
|
||||
[Inject]
|
||||
private AssistantSessionService AssistantSessionService { get; init; } = null!;
|
||||
|
||||
[Inject]
|
||||
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
|
||||
|
||||
private async Task OpenSettingsDialog()
|
||||
{
|
||||
@@ -71,7 +77,7 @@ public partial class AssistantBlock<TSettings> : MSGComponentBase where TSetting
|
||||
await this.DialogService.ShowAsync<TSettings>(T("Open Settings"), dialogParameters, DialogOptions.FULLSCREEN);
|
||||
}
|
||||
|
||||
private string BorderColor => this.AssistantSessionSnapshot?.IsActive is true ? this.ColorTheme.GetActivityIndicatorColor(this.SettingsManager) : this.SettingsManager.IsDarkMode switch
|
||||
private string BorderColor => this.AssistantSessionSnapshot?.IsActive is true || this.MediaImportSnapshot?.IsBusy is true ? this.ColorTheme.GetActivityIndicatorColor(this.SettingsManager) : this.SettingsManager.IsDarkMode switch
|
||||
{
|
||||
true => this.ColorTheme.GetCurrentPalette(this.SettingsManager).GrayDefault,
|
||||
false => this.ColorTheme.GetCurrentPalette(this.SettingsManager).GrayDefault,
|
||||
@@ -92,10 +98,29 @@ public partial class AssistantBlock<TSettings> : MSGComponentBase where TSetting
|
||||
? this.AssistantSessionService.GetSnapshots().FirstOrDefault(snapshot => snapshot.Key.Component == this.Component)
|
||||
: this.AssistantSessionService.GetSnapshots().FirstOrDefault(snapshot => snapshot.Key.InstanceId == this.AssistantSessionInstanceId);
|
||||
|
||||
private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForAssistant(new AssistantSessionKey(this.Component, this.AssistantSessionInstanceId));
|
||||
|
||||
private MediaImportSnapshot? MediaImportSnapshot => string.IsNullOrWhiteSpace(this.AssistantSessionInstanceId)
|
||||
? this.MediaTranscriptionService.GetSnapshots().FirstOrDefault(snapshot =>
|
||||
snapshot.Owner.Kind is MediaImportOwnerKind.ASSISTANT
|
||||
&& snapshot.Owner.Id.StartsWith($"{this.Component}:", StringComparison.Ordinal))
|
||||
: this.MediaTranscriptionService.GetSnapshot(this.CurrentMediaImportOwner);
|
||||
|
||||
/// <summary>
|
||||
/// Gets the assistant session indicator shown on top of the assistant icon.
|
||||
/// </summary>
|
||||
private AssistantSessionIndicatorData? AssistantSessionIndicator => this.AssistantSessionSnapshot?.Status switch
|
||||
private AssistantSessionIndicatorData? AssistantSessionIndicator => this.MediaImportSnapshot?.Status switch
|
||||
{
|
||||
MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => new(Icons.Material.Filled.ChangeCircle, Color.Info, this.T("Media is still being prepared.")),
|
||||
MediaImportStatus.SUCCEEDED => new(Icons.Material.Filled.TaskAlt, Color.Success, this.T("The media transcript is ready.")),
|
||||
MediaImportStatus.WARNING => new(Icons.Material.Filled.WarningAmber, Color.Warning, this.T("Media transcription completed with a warning. Open the assistant to review it.")),
|
||||
MediaImportStatus.FAILED => new(Icons.Material.Filled.Error, Color.Error, this.T("Media transcription failed. Open the assistant to review it.")),
|
||||
MediaImportStatus.CANCELLED => new(Icons.Material.Filled.Cancel, Color.Warning, this.T("Media transcription was canceled. Open the assistant to review it.")),
|
||||
|
||||
_ => this.AssistantSessionIndicatorWithoutMedia,
|
||||
};
|
||||
|
||||
private AssistantSessionIndicatorData? AssistantSessionIndicatorWithoutMedia => this.AssistantSessionSnapshot?.Status switch
|
||||
{
|
||||
AssistantSessionStatus.RUNNING or AssistantSessionStatus.CANCELING => new(Icons.Material.Filled.ChangeCircle, Color.Info, this.T("Assistant is still running.")),
|
||||
AssistantSessionStatus.COMPLETED => new(Icons.Material.Filled.TaskAlt, Color.Success, this.T("The result is ready.")),
|
||||
@@ -104,6 +129,28 @@ public partial class AssistantBlock<TSettings> : MSGComponentBase where TSetting
|
||||
_ => null,
|
||||
};
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged;
|
||||
await base.OnInitializedAsync();
|
||||
}
|
||||
|
||||
private void OnMediaImportStateChanged(MediaImportOwner owner)
|
||||
{
|
||||
var matches = string.IsNullOrWhiteSpace(this.AssistantSessionInstanceId)
|
||||
? owner.Kind is MediaImportOwnerKind.ASSISTANT && owner.Id.StartsWith($"{this.Component}:", StringComparison.Ordinal)
|
||||
: owner == this.CurrentMediaImportOwner;
|
||||
|
||||
if (matches)
|
||||
_ = this.InvokeAsync(this.StateHasChanged);
|
||||
}
|
||||
|
||||
protected override void DisposeResources()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged;
|
||||
base.DisposeResources();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Refreshes the block when assistant session activity changes.
|
||||
/// </summary>
|
||||
|
||||
@@ -2,57 +2,66 @@
|
||||
|
||||
@if (this.UseSmallForm)
|
||||
{
|
||||
<div @onmouseenter="@this.OnMouseEnter" @onmouseleave="@this.OnMouseLeave">
|
||||
@if (this.isDraggingOver)
|
||||
{
|
||||
<MudBadge
|
||||
Content="@this.DocumentPaths.Count"
|
||||
Color="Color.Primary"
|
||||
Overlap="true"
|
||||
Class="cursor-pointer"
|
||||
OnClick="@this.OpenAttachmentsDialog">
|
||||
<MudLink OnClick="@this.AddFilesManually" Style="text-decoration: none;">
|
||||
<MudTextField T="string"
|
||||
Text="@DROP_FILES_HERE_TEXT"
|
||||
Adornment="Adornment.Start"
|
||||
AdornmentIcon="@Icons.Material.Filled.AttachFile"
|
||||
Typo="Typo.body2"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
/>
|
||||
</MudLink>
|
||||
</MudBadge>
|
||||
}
|
||||
else if (this.DocumentPaths.Any())
|
||||
{
|
||||
<MudTooltip Text="@T("Click the paperclip to attach files, or click the number to see your attached files.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
|
||||
<MudStack Spacing="0">
|
||||
<div @onmouseenter="@this.OnMouseEnter" @onmouseleave="@this.OnMouseLeave">
|
||||
@if (this.isDraggingOver)
|
||||
{
|
||||
<MudBadge
|
||||
Content="@this.DocumentPaths.Count"
|
||||
Color="Color.Primary"
|
||||
Overlap="true"
|
||||
Class="cursor-pointer"
|
||||
OnClick="@this.OpenAttachmentsDialog">
|
||||
<MudLink OnClick="@this.AddFilesManually" Style="text-decoration: none;">
|
||||
<MudTextField T="string"
|
||||
Text="@DROP_FILES_HERE_TEXT"
|
||||
Adornment="Adornment.Start"
|
||||
AdornmentIcon="@Icons.Material.Filled.AttachFile"
|
||||
Typo="Typo.body2"
|
||||
Variant="Variant.Outlined"
|
||||
ReadOnly="true"
|
||||
Disabled="@this.IsUnavailable"
|
||||
/>
|
||||
</MudLink>
|
||||
</MudBadge>
|
||||
}
|
||||
else if (this.DocumentPaths.Any())
|
||||
{
|
||||
<MudTooltip Text="@T("Click the paperclip to attach files, or click the number to see your attached files.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
|
||||
<MudBadge
|
||||
Content="@this.DocumentPaths.Count"
|
||||
Color="Color.Primary"
|
||||
Overlap="true"
|
||||
Class="cursor-pointer"
|
||||
OnClick="@this.OpenAttachmentsDialog">
|
||||
<MudIconButton
|
||||
Icon="@Icons.Material.Filled.AttachFile"
|
||||
Color="Color.Default"
|
||||
Disabled="@this.IsUnavailable"
|
||||
OnClick="@this.AddFilesManually"/>
|
||||
</MudBadge>
|
||||
</MudTooltip>
|
||||
}
|
||||
else
|
||||
{
|
||||
<MudTooltip Text="@T("Click here to attach files.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
|
||||
<MudIconButton
|
||||
Icon="@Icons.Material.Filled.AttachFile"
|
||||
Color="Color.Default"
|
||||
Disabled="@this.IsUnavailable"
|
||||
OnClick="@this.AddFilesManually"/>
|
||||
</MudBadge>
|
||||
</MudTooltip>
|
||||
}
|
||||
else
|
||||
</MudTooltip>
|
||||
}
|
||||
</div>
|
||||
@if (this.ShowMediaStatus)
|
||||
{
|
||||
<MudTooltip Text="@T("Click here to attach files.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
|
||||
<MudIconButton
|
||||
Icon="@Icons.Material.Filled.AttachFile"
|
||||
Color="Color.Default"
|
||||
OnClick="@this.AddFilesManually"/>
|
||||
</MudTooltip>
|
||||
<MediaTranscriptionStatus Owner="@this.EffectiveImportOwner" TargetId="@this.EffectiveMediaImportTarget.TargetId" Compact="true"/>
|
||||
}
|
||||
</div>
|
||||
</MudStack>
|
||||
}
|
||||
else
|
||||
{
|
||||
@if (!this.Disabled)
|
||||
@if (!this.IsUnavailable)
|
||||
{
|
||||
<MudStack Row="true" AlignItems="AlignItems.Center" StretchItems="StretchItems.None" Wrap="Wrap.Wrap">
|
||||
<MudText Typo="Typo.body1" Inline="true">
|
||||
@@ -69,11 +78,15 @@ else
|
||||
</MudButton>
|
||||
</MudStack>
|
||||
}
|
||||
@if (this.ShowMediaStatus)
|
||||
{
|
||||
<MediaTranscriptionStatus Owner="@this.EffectiveImportOwner" TargetId="@this.EffectiveMediaImportTarget.TargetId"/>
|
||||
}
|
||||
<div @onmouseenter="@this.OnMouseEnter" @onmouseleave="@this.OnMouseLeave">
|
||||
<MudPaper Height="20em" Outlined="true" Class="@this.dragClass" Style="overflow-y: auto;">
|
||||
@foreach (var fileAttachment in this.DocumentPaths)
|
||||
{
|
||||
@if (this.Disabled)
|
||||
@if (this.IsUnavailable)
|
||||
{
|
||||
<MudChip T="string" Color="Color.Dark" Text="@fileAttachment.FileName" tabindex="-1" Icon="@Icons.Material.Filled.Search" OnClick="@(() => this.InvestigateFile(fileAttachment))"/>
|
||||
}
|
||||
@@ -84,7 +97,7 @@ else
|
||||
}
|
||||
</MudPaper>
|
||||
</div>
|
||||
@if (!this.Disabled)
|
||||
@if (!this.IsUnavailable)
|
||||
{
|
||||
<MudButton OnClick="@(async () => await this.ClearAllFiles())" Variant="Variant.Filled" Color="Color.Info" Class="mt-2" StartIcon="@Icons.Material.Filled.Delete">
|
||||
@T("Clear file list")
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Dialogs;
|
||||
using AIStudio.Tools.Media;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.Rust;
|
||||
using AIStudio.Tools.Services;
|
||||
@@ -13,6 +14,11 @@ using DialogOptions = Dialogs.DialogOptions;
|
||||
|
||||
public partial class AttachDocuments : MSGComponentBase
|
||||
{
|
||||
private readonly MediaImportOwner fallbackMediaImportOwner = new(MediaImportOwnerKind.CHAT, $"attachments:{Guid.NewGuid():N}");
|
||||
|
||||
[CascadingParameter]
|
||||
private MediaImportOwner? ImportOwner { get; set; }
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(AttachDocuments).Namespace, nameof(AttachDocuments));
|
||||
|
||||
[Parameter]
|
||||
@@ -48,6 +54,10 @@ public partial class AttachDocuments : MSGComponentBase
|
||||
[Parameter]
|
||||
public bool UseSmallForm { get; set; }
|
||||
|
||||
/// <summary>Whether this control renders its own media status.</summary>
|
||||
[Parameter]
|
||||
public bool ShowMediaStatus { get; set; } = true;
|
||||
|
||||
[Parameter]
|
||||
public bool Disabled { get; set; }
|
||||
|
||||
@@ -63,6 +73,14 @@ public partial class AttachDocuments : MSGComponentBase
|
||||
[Parameter]
|
||||
public AIStudio.Settings.Provider? Provider { get; set; }
|
||||
|
||||
/// <summary>Optional persisted chat that can own transcript files immediately.</summary>
|
||||
[Parameter]
|
||||
public ChatThread? OwnerChat { get; set; }
|
||||
|
||||
/// <summary>Creates and persists a draft owner after media import confirmation.</summary>
|
||||
[Parameter]
|
||||
public Func<string, Task<ChatThread?>> EnsureOwnerChatAsync { get; set; } = _ => Task.FromResult<ChatThread?>(null);
|
||||
|
||||
[Inject]
|
||||
private ILogger<AttachDocuments> Logger { get; set; } = null!;
|
||||
|
||||
@@ -75,17 +93,28 @@ public partial class AttachDocuments : MSGComponentBase
|
||||
[Inject]
|
||||
private PandocAvailabilityService PandocAvailabilityService { get; init; } = null!;
|
||||
|
||||
[Inject]
|
||||
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
|
||||
|
||||
private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top;
|
||||
private static readonly string DROP_FILES_HERE_TEXT = TB("Drop files here to attach them.");
|
||||
|
||||
private uint numDropAreasAboveThis;
|
||||
private bool isComponentHovered;
|
||||
private bool isDraggingOver;
|
||||
private MediaImportOwner EffectiveImportOwner => this.OwnerChat is not null
|
||||
? MediaImportOwner.ForChat(this.OwnerChat.ChatId)
|
||||
: this.ImportOwner ?? this.fallbackMediaImportOwner;
|
||||
|
||||
private MediaImportTarget EffectiveMediaImportTarget => new(this.EffectiveImportOwner, string.IsNullOrWhiteSpace(this.Name) ? "attachments" : this.Name);
|
||||
|
||||
private bool IsUnavailable => this.Disabled || this.MediaTranscriptionService.IsBusy(this.EffectiveImportOwner);
|
||||
|
||||
#region Overrides of MSGComponentBase
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged;
|
||||
this.ApplyFilters([], [ Event.TAURI_EVENT_RECEIVED, Event.REGISTER_FILE_DROP_AREA, Event.UNREGISTER_FILE_DROP_AREA ]);
|
||||
|
||||
// Register this drop area:
|
||||
@@ -93,9 +122,101 @@ public partial class AttachDocuments : MSGComponentBase
|
||||
await base.OnInitializedAsync();
|
||||
}
|
||||
|
||||
/// <summary>Rehydrates results after the component is assigned another chat or target.</summary>
|
||||
protected override async Task OnParametersSetAsync()
|
||||
{
|
||||
await base.OnParametersSetAsync();
|
||||
await this.SyncCompletedMediaAttachmentsAsync();
|
||||
}
|
||||
|
||||
/// <summary>Refreshes disabled controls when the shared import lane changes.</summary>
|
||||
private void OnMediaImportStateChanged(MediaImportOwner owner)
|
||||
{
|
||||
if (owner == this.EffectiveImportOwner)
|
||||
_ = this.InvokeAsync(async () =>
|
||||
{
|
||||
await this.SyncCompletedMediaAttachmentsAsync();
|
||||
await this.ConsumeStandaloneMediaOutcomeAsync();
|
||||
this.StateHasChanged();
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Consumes outcomes for dialog-local controls that have no chat or assistant owner surface.</summary>
|
||||
private async Task ConsumeStandaloneMediaOutcomeAsync()
|
||||
{
|
||||
if (this.ImportOwner is not null || this.OwnerChat is not null)
|
||||
return;
|
||||
|
||||
var outcome = this.MediaTranscriptionService.TryConsumeOutcome(this.EffectiveImportOwner);
|
||||
if (outcome is null)
|
||||
return;
|
||||
|
||||
if (outcome.Failures.Count > 0)
|
||||
{
|
||||
var message = string.Join(Environment.NewLine, outcome.Failures.Select(failure => $"{failure.FileName}: {failure.UserMessage}"));
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, message));
|
||||
}
|
||||
else if (outcome.Status is MediaImportStatus.FAILED)
|
||||
{
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed.")));
|
||||
}
|
||||
|
||||
if (outcome.Warnings.Count > 0)
|
||||
{
|
||||
var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}"));
|
||||
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message));
|
||||
}
|
||||
|
||||
if (outcome.Status is MediaImportStatus.CANCELLED)
|
||||
{
|
||||
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled.")));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Reattaches completed owner results after progress updates or navigation.</summary>
|
||||
private async Task SyncCompletedMediaAttachmentsAsync()
|
||||
{
|
||||
var delivery = this.MediaTranscriptionService.GetPendingDelivery(this.EffectiveMediaImportTarget);
|
||||
var completed = delivery?.Attachments ?? [];
|
||||
var pending = this.OwnerChat?.PendingMediaTranscripts ?? [];
|
||||
var changed = false;
|
||||
var ownerPendingChanged = false;
|
||||
|
||||
foreach (var attachment in completed.Concat(pending))
|
||||
changed |= this.DocumentPaths.Add(attachment);
|
||||
|
||||
if (this.OwnerChat is not null)
|
||||
{
|
||||
foreach (var attachment in completed.OfType<ManagedTranscriptAttachment>())
|
||||
{
|
||||
if (this.OwnerChat.PendingMediaTranscripts.All(existing => existing.FilePath != attachment.FilePath))
|
||||
{
|
||||
this.OwnerChat.PendingMediaTranscripts.Add(attachment);
|
||||
ownerPendingChanged = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (changed || ownerPendingChanged)
|
||||
{
|
||||
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
|
||||
await this.OnChange(this.DocumentPaths);
|
||||
}
|
||||
|
||||
if (delivery is not null)
|
||||
this.MediaTranscriptionService.AcknowledgeDelivery(delivery);
|
||||
}
|
||||
|
||||
/// <summary>Unsubscribes from the singleton media service.</summary>
|
||||
protected override void DisposeResources()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged;
|
||||
base.DisposeResources();
|
||||
}
|
||||
|
||||
protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
|
||||
{
|
||||
if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
|
||||
if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
|
||||
return;
|
||||
|
||||
switch (triggeredEvent)
|
||||
@@ -168,29 +289,7 @@ public partial class AttachDocuments : MSGComponentBase
|
||||
return;
|
||||
}
|
||||
|
||||
// Ensure that Pandoc is installed and ready:
|
||||
var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync(
|
||||
showSuccessMessage: false,
|
||||
showDialog: true);
|
||||
|
||||
// If Pandoc is not available (user cancelled installation), abort file drop:
|
||||
if (!pandocState.IsAvailable)
|
||||
{
|
||||
this.Logger.LogWarning("The user cancelled the Pandoc installation or Pandoc is not available. Aborting file drop.");
|
||||
this.isDraggingOver = false;
|
||||
this.ClearDragClass();
|
||||
this.StateHasChanged();
|
||||
return;
|
||||
}
|
||||
|
||||
foreach (var path in paths)
|
||||
{
|
||||
if(!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, path, this.ValidateMediaFileTypes, this.Provider))
|
||||
continue;
|
||||
|
||||
this.DocumentPaths.Add(FileAttachment.FromPath(path));
|
||||
}
|
||||
|
||||
await this.AddFileBatchAsync(paths);
|
||||
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
|
||||
await this.OnChange(this.DocumentPaths);
|
||||
this.isDraggingOver = false;
|
||||
@@ -208,54 +307,41 @@ public partial class AttachDocuments : MSGComponentBase
|
||||
|
||||
private async Task AddFilesManually()
|
||||
{
|
||||
if (this.Disabled)
|
||||
if (this.IsUnavailable)
|
||||
return;
|
||||
|
||||
// Ensure that Pandoc is installed and ready:
|
||||
var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync(
|
||||
showSuccessMessage: false,
|
||||
showDialog: true);
|
||||
|
||||
// If Pandoc is not available (user cancelled installation), abort file selection:
|
||||
if (!pandocState.IsAvailable)
|
||||
{
|
||||
this.Logger.LogWarning("The user cancelled the Pandoc installation or Pandoc is not available. Aborting file selection.");
|
||||
return;
|
||||
}
|
||||
|
||||
var selectFiles = await this.RustService.SelectFiles(T("Select files to attach"));
|
||||
if (selectFiles.UserCancelled)
|
||||
return;
|
||||
|
||||
foreach (var selectedFilePath in selectFiles.SelectedFilePaths)
|
||||
{
|
||||
if (!File.Exists(selectedFilePath))
|
||||
continue;
|
||||
|
||||
if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, selectedFilePath, this.ValidateMediaFileTypes, this.Provider))
|
||||
continue;
|
||||
|
||||
this.DocumentPaths.Add(FileAttachment.FromPath(selectedFilePath));
|
||||
}
|
||||
|
||||
await this.AddFileBatchAsync(selectFiles.SelectedFilePaths);
|
||||
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
|
||||
await this.OnChange(this.DocumentPaths);
|
||||
}
|
||||
|
||||
private async Task OpenAttachmentsDialog()
|
||||
{
|
||||
if (this.Disabled)
|
||||
if (this.IsUnavailable)
|
||||
return;
|
||||
|
||||
var previousAttachments = this.DocumentPaths.ToHashSet();
|
||||
this.DocumentPaths = await ReviewAttachmentsDialog.OpenDialogAsync(this.DialogService, this.DocumentPaths);
|
||||
foreach (var removedAttachment in previousAttachments.Except(this.DocumentPaths))
|
||||
ManagedTranscriptAttachment.TryDeleteOwnedFile(removedAttachment);
|
||||
|
||||
this.ReconcileOwnerPendingTranscripts();
|
||||
}
|
||||
|
||||
private async Task ClearAllFiles()
|
||||
{
|
||||
if (this.Disabled)
|
||||
if (this.IsUnavailable)
|
||||
return;
|
||||
|
||||
foreach (var attachment in this.DocumentPaths)
|
||||
ManagedTranscriptAttachment.TryDeleteOwnedFile(attachment);
|
||||
|
||||
this.DocumentPaths.Clear();
|
||||
this.ReconcileOwnerPendingTranscripts();
|
||||
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
|
||||
await this.OnChange(this.DocumentPaths);
|
||||
}
|
||||
@@ -266,7 +352,7 @@ public partial class AttachDocuments : MSGComponentBase
|
||||
|
||||
private void OnMouseEnter(EventArgs _)
|
||||
{
|
||||
if(this.Disabled || this.PauseCatchingDrops)
|
||||
if(this.IsUnavailable || this.PauseCatchingDrops)
|
||||
return;
|
||||
|
||||
this.Logger.LogDebug("Attach documents component '{Name}' is hovered.", this.Name);
|
||||
@@ -277,7 +363,7 @@ public partial class AttachDocuments : MSGComponentBase
|
||||
|
||||
private void OnMouseLeave(EventArgs _)
|
||||
{
|
||||
if(this.Disabled || this.PauseCatchingDrops)
|
||||
if(this.IsUnavailable || this.PauseCatchingDrops)
|
||||
return;
|
||||
|
||||
this.Logger.LogDebug("Attach documents component '{Name}' is no longer hovered.", this.Name);
|
||||
@@ -288,15 +374,98 @@ public partial class AttachDocuments : MSGComponentBase
|
||||
|
||||
private async Task RemoveDocument(FileAttachment fileAttachment)
|
||||
{
|
||||
if (this.Disabled)
|
||||
if (this.IsUnavailable)
|
||||
return;
|
||||
|
||||
this.DocumentPaths.Remove(fileAttachment);
|
||||
ManagedTranscriptAttachment.TryDeleteOwnedFile(fileAttachment);
|
||||
this.ReconcileOwnerPendingTranscripts();
|
||||
|
||||
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
|
||||
await this.OnChange(this.DocumentPaths);
|
||||
}
|
||||
|
||||
/// <summary>Keeps persisted chat-draft transcript references aligned with the composer.</summary>
|
||||
private void ReconcileOwnerPendingTranscripts()
|
||||
{
|
||||
if (this.OwnerChat is null)
|
||||
return;
|
||||
|
||||
var retainedPaths = this.DocumentPaths.Select(attachment => attachment.FilePath).ToHashSet(StringComparer.Ordinal);
|
||||
this.OwnerChat.PendingMediaTranscripts.RemoveAll(attachment => !retainedPaths.Contains(attachment.FilePath));
|
||||
}
|
||||
|
||||
private async Task AddFileBatchAsync(IEnumerable<string> paths)
|
||||
{
|
||||
var existingPaths = paths.Where(File.Exists).ToList();
|
||||
var mediaPaths = existingPaths.Where(IsTranscribableMedia).ToList();
|
||||
var regularPaths = existingPaths.Except(mediaPaths).ToList();
|
||||
|
||||
var canAddRegularFiles = true;
|
||||
if (regularPaths.Count > 0)
|
||||
{
|
||||
var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync(
|
||||
showSuccessMessage: false,
|
||||
showDialog: true);
|
||||
canAddRegularFiles = pandocState.IsAvailable;
|
||||
}
|
||||
|
||||
foreach (var path in regularPaths)
|
||||
{
|
||||
if (!canAddRegularFiles)
|
||||
break;
|
||||
|
||||
if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(
|
||||
FileExtensionValidation.UseCase.ATTACHING_CONTENT,
|
||||
path,
|
||||
this.ValidateMediaFileTypes,
|
||||
this.Provider))
|
||||
continue;
|
||||
this.DocumentPaths.Add(FileAttachment.FromPath(path));
|
||||
}
|
||||
|
||||
if (mediaPaths.Count is 0)
|
||||
return;
|
||||
|
||||
if (string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider))
|
||||
{
|
||||
await this.MessageBus.SendWarning(new(
|
||||
Icons.Material.Filled.VoiceChat,
|
||||
this.T("Media files require a configured transcription provider. Configure one in the transcription settings.")));
|
||||
return;
|
||||
}
|
||||
|
||||
var names = string.Join('\n', mediaPaths.Select(path => $"- {Markdown.EscapeInlineText(Path.GetFileName(path))}"));
|
||||
var message = this.T("The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider.");
|
||||
var dialogParameters = new DialogParameters<ConfirmDialog>
|
||||
{
|
||||
{
|
||||
x => x.MarkdownBody,
|
||||
$"""
|
||||
{message}
|
||||
|
||||
{names}
|
||||
"""
|
||||
},
|
||||
};
|
||||
|
||||
var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(
|
||||
this.T("Transcribe media files"),
|
||||
dialogParameters,
|
||||
DialogOptions.FULLSCREEN);
|
||||
|
||||
var dialogResult = await dialogReference.Result;
|
||||
if (dialogResult is null || dialogResult.Canceled)
|
||||
return;
|
||||
|
||||
if (this.OwnerChat is null)
|
||||
this.OwnerChat = await this.EnsureOwnerChatAsync(mediaPaths[0]);
|
||||
|
||||
this.MediaTranscriptionService.TryStartAttachmentBatch(mediaPaths, this.EffectiveMediaImportTarget, this.OwnerChat);
|
||||
}
|
||||
|
||||
private static bool IsTranscribableMedia(string path) => FileTypes.IsAllowedPath(path, FileTypes.AUDIO) || FileTypes.IsAllowedPath(path, FileTypes.VIDEO);
|
||||
|
||||
/// <summary>
|
||||
/// The user might want to check what we actually extract from his file and therefore give the LLM as an input.
|
||||
/// </summary>
|
||||
|
||||
@@ -33,6 +33,7 @@
|
||||
}
|
||||
</ChildContent>
|
||||
<FooterContent>
|
||||
<MediaTranscriptionStatus Owner="@this.CurrentMediaImportOwner"/>
|
||||
<MudElement Style="flex: 0 0 auto;">
|
||||
<MudTextField
|
||||
T="string"
|
||||
@@ -45,7 +46,7 @@
|
||||
Label="@this.InputLabel"
|
||||
Placeholder="@this.ProviderPlaceholder"
|
||||
Adornment="Adornment.End"
|
||||
AdornmentIcon="@Icons.Material.Filled.Send"
|
||||
AdornmentIcon="@(this.MediaTranscriptionService.IsBusy(this.CurrentMediaImportOwner) ? Icons.Material.Filled.HourglassTop : Icons.Material.Filled.Send)"
|
||||
OnAdornmentClick="() => this.SendMessage()"
|
||||
Disabled="@this.IsInputForbidden()"
|
||||
Immediate="@true"
|
||||
@@ -100,7 +101,7 @@
|
||||
</MudTooltip>
|
||||
}
|
||||
|
||||
<AttachDocuments Name="File Attachments" Layer="@DropLayers.PAGES" DocumentPaths="@this.ComposerState.FileAttachments" DocumentPathsChanged="@this.ComposerAttachmentsChanged" CatchAllDocuments="true" UseSmallForm="true" Provider="@this.Provider"/>
|
||||
<AttachDocuments Name="File Attachments" Layer="@DropLayers.PAGES" DocumentPaths="@this.ComposerState.FileAttachments" DocumentPathsChanged="@this.ComposerAttachmentsChanged" CatchAllDocuments="true" UseSmallForm="true" ShowMediaStatus="false" Provider="@this.Provider" OwnerChat="@this.ChatThread" EnsureOwnerChatAsync="@this.EnsureMediaImportChatAsync" Disabled="@this.MediaTranscriptionService.IsBusy(this.CurrentMediaImportOwner)"/>
|
||||
|
||||
<MudDivider Vertical="true" Style="height: 24px; align-self: center;"/>
|
||||
|
||||
|
||||
@@ -4,6 +4,8 @@ using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.AIJobs;
|
||||
using AIStudio.Tools.Media;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
using Microsoft.AspNetCore.Components;
|
||||
using Microsoft.AspNetCore.Components.Web;
|
||||
@@ -14,6 +16,7 @@ namespace AIStudio.Components;
|
||||
|
||||
public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
{
|
||||
private readonly Guid draftMediaOwnerId = Guid.NewGuid();
|
||||
private const string CHAT_INPUT_ID = "chat-user-input";
|
||||
private const string MARKDOWN_CODE = "code";
|
||||
private const string MARKDOWN_BOLD = "bold";
|
||||
@@ -54,6 +57,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
[Inject]
|
||||
private AIJobService AIJobService { get; init; } = null!;
|
||||
|
||||
[Inject]
|
||||
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
|
||||
|
||||
private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top;
|
||||
private static readonly Dictionary<string, object?> USER_INPUT_ATTRIBUTES = new();
|
||||
|
||||
@@ -81,6 +87,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
private Guid foregroundChatId = Guid.Empty;
|
||||
private int workspaceHeaderSyncVersion;
|
||||
|
||||
private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForChat(this.ChatThread?.ChatId ?? this.draftMediaOwnerId);
|
||||
|
||||
// Unfortunately, we need the input field reference to blur the focus away. Without
|
||||
// this, we cannot clear the input field.
|
||||
private MudTextField<string> inputField = null!;
|
||||
@@ -104,6 +112,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged;
|
||||
|
||||
// Apply the filters for the message bus:
|
||||
this.ApplyFilters([], [ Event.HAS_CHAT_UNSAVED_CHANGES, Event.RESET_CHAT_STATE, Event.CHAT_STREAMING_DONE, Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED, Event.CHAT_GENERATION_CHANGED, Event.WORKSPACE_RENAMED, Event.CONFIGURATION_CHANGED ]);
|
||||
|
||||
@@ -243,9 +253,50 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
// Select the correct provider:
|
||||
await this.SelectProviderWhenLoadingChat();
|
||||
await this.SyncForegroundChatAsync();
|
||||
await this.ConsumeMediaOutcomeAsync();
|
||||
await base.OnInitializedAsync();
|
||||
}
|
||||
|
||||
/// <summary>Refreshes send and attachment controls when the media import lane changes.</summary>
|
||||
private void OnMediaImportStateChanged(MediaImportOwner owner)
|
||||
{
|
||||
if (owner == this.CurrentMediaImportOwner)
|
||||
_ = this.InvokeAsync(async () =>
|
||||
{
|
||||
await this.ConsumeMediaOutcomeAsync();
|
||||
this.StateHasChanged();
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Consumes a terminal media notification when its chat is visible.</summary>
|
||||
private async Task ConsumeMediaOutcomeAsync()
|
||||
{
|
||||
var outcome = this.MediaTranscriptionService.TryConsumeOutcome(this.CurrentMediaImportOwner);
|
||||
if (outcome is null)
|
||||
return;
|
||||
|
||||
if (outcome.Failures.Count > 0)
|
||||
{
|
||||
var message = string.Join(Environment.NewLine, outcome.Failures.Select(failure => $"{failure.FileName}: {failure.UserMessage}"));
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, message));
|
||||
}
|
||||
else if (outcome.Status is MediaImportStatus.FAILED)
|
||||
{
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed.")));
|
||||
}
|
||||
|
||||
if (outcome.Warnings.Count > 0)
|
||||
{
|
||||
var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}"));
|
||||
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message));
|
||||
}
|
||||
|
||||
if (outcome.Status is MediaImportStatus.CANCELLED)
|
||||
{
|
||||
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled.")));
|
||||
}
|
||||
}
|
||||
|
||||
protected override async Task OnAfterRenderAsync(bool firstRender)
|
||||
{
|
||||
if (firstRender && this.ChatThread is not null && this.mustStoreChat)
|
||||
@@ -314,6 +365,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
|
||||
await this.ApplyLoadedChatParameterAsync();
|
||||
await this.SyncForegroundChatAsync();
|
||||
await this.ConsumeMediaOutcomeAsync();
|
||||
await base.OnParametersSetAsync();
|
||||
}
|
||||
|
||||
@@ -680,9 +732,43 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
this.ComposerState.MarkUserDraft();
|
||||
this.hasUnsavedChanges = true;
|
||||
}
|
||||
|
||||
/// <summary>Creates and stores a stable draft immediately after media import confirmation.</summary>
|
||||
private async Task<ChatThread?> EnsureMediaImportChatAsync(string firstMediaPath)
|
||||
{
|
||||
if (this.ChatThread is not null)
|
||||
return this.ChatThread;
|
||||
|
||||
this.RefreshCurrentProfileAndChatTemplate();
|
||||
var promptName = this.ExtractThreadName(this.ComposerState.UserInput);
|
||||
this.ChatThread = new()
|
||||
{
|
||||
IncludeDateTime = true,
|
||||
SelectedProvider = this.Provider.Id,
|
||||
SelectedProfile = this.currentProfile.Id,
|
||||
SelectedChatTemplate = this.currentChatTemplate.Id,
|
||||
SystemPrompt = SystemPrompts.DEFAULT,
|
||||
WorkspaceId = this.currentWorkspaceId,
|
||||
ChatId = Guid.NewGuid(),
|
||||
DataSourceOptions = this.earlyDataSourceOptions,
|
||||
Name = string.IsNullOrWhiteSpace(this.ComposerState.UserInput)
|
||||
? $"Transkription: {Path.GetFileName(firstMediaPath)}"
|
||||
: promptName,
|
||||
Blocks = this.currentChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE ? [] : this.currentChatTemplate.ExampleConversation.Select(block => block.DeepClone()).ToList(),
|
||||
};
|
||||
|
||||
await WorkspaceBehaviour.StoreChatAsync(this.ChatThread);
|
||||
this.MarkCurrentChatAsLoadedParameter();
|
||||
await this.ChatThreadChanged.InvokeAsync(this.ChatThread);
|
||||
await this.SyncForegroundChatAsync();
|
||||
return this.ChatThread;
|
||||
}
|
||||
|
||||
private async Task SendMessage(bool reuseLastUserPrompt = false)
|
||||
{
|
||||
if (this.MediaTranscriptionService.IsBusy(this.CurrentMediaImportOwner))
|
||||
return;
|
||||
|
||||
if (!this.IsProviderSelected)
|
||||
return;
|
||||
|
||||
@@ -745,6 +831,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
Text = this.ComposerState.UserInput,
|
||||
FileAttachments = normalizedAttachments,
|
||||
};
|
||||
this.ChatThread.PendingMediaTranscripts.Clear();
|
||||
|
||||
//
|
||||
// Add the user message to the thread:
|
||||
@@ -986,12 +1073,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
if (workspaceId == Guid.Empty)
|
||||
return;
|
||||
|
||||
// Delete the chat from the current workspace or the temporary storage:
|
||||
await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, this.ChatThread!.WorkspaceId, this.ChatThread.ChatId, askForConfirmation: false);
|
||||
|
||||
this.ChatThread!.WorkspaceId = workspaceId;
|
||||
await WorkspaceBehaviour.MoveChatAsync(this.ChatThread!, workspaceId);
|
||||
this.MarkCurrentChatAsLoadedParameter();
|
||||
await this.SaveThread();
|
||||
|
||||
await this.SyncWorkspaceHeaderWithChatThreadAsync();
|
||||
}
|
||||
@@ -1209,6 +1292,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged;
|
||||
if(this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is WorkspaceStorageBehavior.STORE_CHATS_AUTOMATICALLY)
|
||||
{
|
||||
await this.SaveThread();
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
@inherits MSGComponentBase
|
||||
@inject MediaTranscriptionService MediaTranscriptionService
|
||||
@using AIStudio.Tools.Services
|
||||
|
||||
@if (this.Snapshot is { IsBusy: true } snapshot)
|
||||
{
|
||||
@if (this.Compact)
|
||||
{
|
||||
<MudStack Row="true" AlignItems="AlignItems.Center" Class="pa-1">
|
||||
<MudProgressCircular Size="Size.Small" Indeterminate="@(snapshot.Progress is null)" Value="@((snapshot.Progress ?? 0) * 100)"/>
|
||||
<MudText Typo="Typo.body2">
|
||||
@this.StatusText
|
||||
</MudText>
|
||||
<MudTooltip Text="@T("Stop media transcription")">
|
||||
<MudIconButton Size="Size.Small" Icon="@Icons.Material.Filled.Stop" Color="Color.Error" OnClick="@(async () => await this.MediaTranscriptionService.StopAsync(this.Owner))"/>
|
||||
</MudTooltip>
|
||||
</MudStack>
|
||||
}
|
||||
else
|
||||
{
|
||||
<MudPaper Outlined="true" Class="pa-2 mb-2">
|
||||
<MudStack Row="true" AlignItems="AlignItems.Center">
|
||||
<MudProgressCircular Size="Size.Small" Indeterminate="@(snapshot.Progress is null)" Value="@((snapshot.Progress ?? 0) * 100)"/>
|
||||
<MudText Typo="Typo.body2">
|
||||
@this.StatusText
|
||||
</MudText>
|
||||
<MudSpacer/>
|
||||
<MudTooltip Text="@T("Stop media transcription")">
|
||||
<MudIconButton Icon="@Icons.Material.Filled.Stop" Color="Color.Error" OnClick="@(async () => await this.MediaTranscriptionService.StopAsync(this.Owner))"/>
|
||||
</MudTooltip>
|
||||
</MudStack>
|
||||
</MudPaper>
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
using AIStudio.Tools.Media;
|
||||
using Microsoft.AspNetCore.Components;
|
||||
|
||||
namespace AIStudio.Components;
|
||||
|
||||
public partial class MediaTranscriptionStatus
|
||||
{
|
||||
/// <summary>The surface owner whose operation is rendered.</summary>
|
||||
[Parameter]
|
||||
public MediaImportOwner Owner { get; set; }
|
||||
|
||||
/// <summary>Optional target filter used by embedded file controls.</summary>
|
||||
[Parameter]
|
||||
public string TargetId { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>Renders the status without an enclosing paper surface.</summary>
|
||||
[Parameter]
|
||||
public bool Compact { get; set; }
|
||||
|
||||
private MediaImportSnapshot? Snapshot
|
||||
{
|
||||
get
|
||||
{
|
||||
var snapshot = this.MediaTranscriptionService.GetSnapshot(this.Owner);
|
||||
return string.IsNullOrWhiteSpace(this.TargetId) || snapshot?.Target.TargetId == this.TargetId
|
||||
? snapshot
|
||||
: null;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Gets the localized visible status for the active import.</summary>
|
||||
private string StatusText
|
||||
{
|
||||
get
|
||||
{
|
||||
var snapshot = this.Snapshot;
|
||||
if (snapshot is null)
|
||||
return string.Empty;
|
||||
|
||||
return snapshot.Phase switch
|
||||
{
|
||||
MediaTranscriptionPhase.QUEUED => $"{this.T("Waiting to prepare media")}: {snapshot.CurrentFileName}",
|
||||
MediaTranscriptionPhase.PROBING => $"{this.T("Inspecting media")}: {snapshot.CurrentFileName}",
|
||||
MediaTranscriptionPhase.TRANSCODING => $"{this.T("Preparing audio")}: {snapshot.CurrentFileName}",
|
||||
MediaTranscriptionPhase.UPLOADING => $"{this.T("Transcribing")}: {snapshot.CurrentFileName}",
|
||||
MediaTranscriptionPhase.CANCELING => $"{this.T("Stopping media transcription")}: {snapshot.CurrentFileName}",
|
||||
|
||||
_ => snapshot.CurrentFileName,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Subscribes to singleton import state changes.</summary>
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged += this.OnStateChanged;
|
||||
await base.OnInitializedAsync();
|
||||
}
|
||||
|
||||
/// <summary>Schedules a render after an import state transition.</summary>
|
||||
private void OnStateChanged(MediaImportOwner owner)
|
||||
{
|
||||
if (owner == this.Owner)
|
||||
_ = this.InvokeAsync(this.StateHasChanged);
|
||||
}
|
||||
|
||||
/// <summary>Unsubscribes from singleton import state changes.</summary>
|
||||
protected override void DisposeResources()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged -= this.OnStateChanged;
|
||||
base.DisposeResources();
|
||||
}
|
||||
}
|
||||
@@ -5,19 +5,29 @@
|
||||
<div @onmouseenter="@this.OnMouseEnter" @onmouseleave="@this.OnMouseLeave">
|
||||
<MudPaper Outlined="true" Class="@this.dragClass">
|
||||
<MudStack Row="true" AlignItems="AlignItems.Center" StretchItems="StretchItems.None" Wrap="Wrap.Wrap">
|
||||
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Disabled="@this.Disabled">
|
||||
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Disabled="@this.IsUnavailable">
|
||||
@this.ButtonText
|
||||
</MudButton>
|
||||
<MudText Typo="Typo.body2">
|
||||
@T("Drop one file here to load its content.")
|
||||
</MudText>
|
||||
@if (this.IsCurrentTargetBusy)
|
||||
{
|
||||
<MediaTranscriptionStatus Owner="@this.EffectiveImportOwner" TargetId="@this.EffectiveMediaImportTarget.TargetId" Compact="true"/>
|
||||
}
|
||||
else
|
||||
{
|
||||
<MudText Typo="Typo.body2">
|
||||
@T("Drop one file here to load its content.")
|
||||
</MudText>
|
||||
}
|
||||
</MudStack>
|
||||
</MudPaper>
|
||||
</div>
|
||||
}
|
||||
else
|
||||
{
|
||||
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Class="mb-3" Disabled="@this.Disabled">
|
||||
@this.ButtonText
|
||||
</MudButton>
|
||||
<MudStack Row="true" AlignItems="AlignItems.Center" StretchItems="StretchItems.None" Wrap="Wrap.Wrap" Class="mb-3">
|
||||
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Disabled="@this.IsUnavailable">
|
||||
@this.ButtonText
|
||||
</MudButton>
|
||||
<MediaTranscriptionStatus Owner="@this.EffectiveImportOwner" TargetId="@this.EffectiveMediaImportTarget.TargetId" Compact="true"/>
|
||||
</MudStack>
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
using AIStudio.Dialogs;
|
||||
using AIStudio.Tools.Media;
|
||||
using AIStudio.Tools.Rust;
|
||||
using AIStudio.Tools.Services;
|
||||
using AIStudio.Tools.Validation;
|
||||
@@ -8,6 +10,22 @@ namespace AIStudio.Components;
|
||||
|
||||
public partial class ReadFileContent : MSGComponentBase
|
||||
{
|
||||
private readonly MediaImportOwner fallbackMediaImportOwner = new(MediaImportOwnerKind.ASSISTANT, $"read-file-content:{Guid.NewGuid():N}");
|
||||
|
||||
[CascadingParameter]
|
||||
private MediaImportOwner? ImportOwner { get; set; }
|
||||
|
||||
private MediaImportOwner EffectiveImportOwner => this.ImportOwner ?? this.fallbackMediaImportOwner;
|
||||
|
||||
[Parameter]
|
||||
public string MediaImportTargetId { get; set; } = string.Empty;
|
||||
|
||||
private string EffectiveMediaImportTargetId => string.IsNullOrWhiteSpace(this.MediaImportTargetId)
|
||||
? string.IsNullOrWhiteSpace(this.Text) ? "primary" : this.Text
|
||||
: this.MediaImportTargetId;
|
||||
|
||||
private MediaImportTarget EffectiveMediaImportTarget => new(this.EffectiveImportOwner, this.EffectiveMediaImportTargetId);
|
||||
|
||||
[Parameter]
|
||||
public string Text { get; set; } = string.Empty;
|
||||
|
||||
@@ -47,17 +65,24 @@ public partial class ReadFileContent : MSGComponentBase
|
||||
[Inject]
|
||||
private PandocAvailabilityService PandocAvailabilityService { get; init; } = null!;
|
||||
|
||||
[Inject]
|
||||
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
|
||||
|
||||
private const string DEFAULT_DRAG_CLASS = "relative rounded-lg border-2 border-dashed pa-3 mb-3 mud-width-full";
|
||||
|
||||
private string ButtonText => string.IsNullOrWhiteSpace(this.Text) ? T("Use file content as input") : this.Text;
|
||||
private string dragClass = DEFAULT_DRAG_CLASS;
|
||||
private uint numDropAreasAboveThis;
|
||||
private bool isComponentHovered;
|
||||
private bool IsCurrentTargetBusy => this.MediaTranscriptionService.GetSnapshot(this.EffectiveImportOwner) is { IsBusy: true } snapshot
|
||||
&& snapshot.Target == this.EffectiveMediaImportTarget;
|
||||
private bool IsUnavailable => this.Disabled || this.MediaTranscriptionService.IsBusy(this.EffectiveImportOwner);
|
||||
|
||||
#region Overrides of MSGComponentBase
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged;
|
||||
if (this.EnableDragDrop)
|
||||
{
|
||||
this.ApplyFilters([], [ Event.TAURI_EVENT_RECEIVED, Event.REGISTER_FILE_DROP_AREA, Event.UNREGISTER_FILE_DROP_AREA ]);
|
||||
@@ -65,6 +90,69 @@ public partial class ReadFileContent : MSGComponentBase
|
||||
}
|
||||
|
||||
await base.OnInitializedAsync();
|
||||
await this.SyncCompletedMediaTextAsync();
|
||||
}
|
||||
|
||||
/// <summary>Refreshes disabled controls when the shared import lane changes.</summary>
|
||||
private void OnMediaImportStateChanged(MediaImportOwner owner)
|
||||
{
|
||||
if (owner == this.EffectiveImportOwner)
|
||||
_ = this.InvokeAsync(async () =>
|
||||
{
|
||||
await this.SyncCompletedMediaTextAsync();
|
||||
await this.ConsumeStandaloneMediaOutcomeAsync();
|
||||
this.StateHasChanged();
|
||||
});
|
||||
}
|
||||
|
||||
/// <summary>Consumes outcomes for dialog-local controls that have no assistant owner surface.</summary>
|
||||
private async Task ConsumeStandaloneMediaOutcomeAsync()
|
||||
{
|
||||
if (this.ImportOwner is not null)
|
||||
return;
|
||||
|
||||
var outcome = this.MediaTranscriptionService.TryConsumeOutcome(this.EffectiveImportOwner);
|
||||
if (outcome is null)
|
||||
return;
|
||||
|
||||
if (outcome.Failures.Count > 0)
|
||||
{
|
||||
var message = string.Join(Environment.NewLine, outcome.Failures.Select(failure => $"{failure.FileName}: {failure.UserMessage}"));
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, message));
|
||||
}
|
||||
else if (outcome.Status is MediaImportStatus.FAILED)
|
||||
{
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed.")));
|
||||
}
|
||||
|
||||
if (outcome.Warnings.Count > 0)
|
||||
{
|
||||
var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}"));
|
||||
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message));
|
||||
}
|
||||
|
||||
if (outcome.Status is MediaImportStatus.CANCELLED)
|
||||
{
|
||||
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled.")));
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>Applies a completed target transcript after progress or navigation.</summary>
|
||||
private async Task SyncCompletedMediaTextAsync()
|
||||
{
|
||||
var delivery = this.MediaTranscriptionService.GetPendingDelivery(this.EffectiveMediaImportTarget);
|
||||
if (delivery is null || delivery.Text is not { } text)
|
||||
return;
|
||||
|
||||
await this.FileContentChanged.InvokeAsync(text);
|
||||
this.MediaTranscriptionService.AcknowledgeDelivery(delivery);
|
||||
}
|
||||
|
||||
/// <summary>Unsubscribes from the singleton media service.</summary>
|
||||
protected override void DisposeResources()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged;
|
||||
base.DisposeResources();
|
||||
}
|
||||
|
||||
protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
|
||||
@@ -72,7 +160,7 @@ public partial class ReadFileContent : MSGComponentBase
|
||||
if (!this.EnableDragDrop)
|
||||
return;
|
||||
|
||||
if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
|
||||
if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
|
||||
return;
|
||||
|
||||
switch (triggeredEvent)
|
||||
@@ -126,10 +214,7 @@ public partial class ReadFileContent : MSGComponentBase
|
||||
|
||||
private async Task SelectFile()
|
||||
{
|
||||
if (this.Disabled)
|
||||
return;
|
||||
|
||||
if (!await this.EnsurePandocAvailability())
|
||||
if (this.IsUnavailable)
|
||||
return;
|
||||
|
||||
var selectedFile = await this.RustService.SelectFile(T("Select file to read its content"));
|
||||
@@ -161,9 +246,6 @@ public partial class ReadFileContent : MSGComponentBase
|
||||
|
||||
private async Task LoadFirstValidFile(List<string> paths)
|
||||
{
|
||||
if (!await this.EnsurePandocAvailability())
|
||||
return;
|
||||
|
||||
foreach (var path in paths)
|
||||
{
|
||||
if (await this.LoadFileIfValid(path))
|
||||
@@ -179,6 +261,12 @@ public partial class ReadFileContent : MSGComponentBase
|
||||
return false;
|
||||
}
|
||||
|
||||
if (FileTypes.IsAllowedPath(filePath, FileTypes.AUDIO) || FileTypes.IsAllowedPath(filePath, FileTypes.VIDEO))
|
||||
return await this.LoadMediaTranscriptAsync(filePath);
|
||||
|
||||
if (!await this.EnsurePandocAvailability())
|
||||
return false;
|
||||
|
||||
if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.DIRECTLY_LOADING_CONTENT, filePath))
|
||||
{
|
||||
this.Logger.LogWarning("User attempted to load unsupported file: {FilePath}", filePath);
|
||||
@@ -200,6 +288,42 @@ public partial class ReadFileContent : MSGComponentBase
|
||||
}
|
||||
}
|
||||
|
||||
private async Task<bool> LoadMediaTranscriptAsync(string filePath)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider))
|
||||
{
|
||||
await this.MessageBus.SendWarning(new(
|
||||
Icons.Material.Filled.VoiceChat,
|
||||
this.T("Media files require a configured transcription provider. Configure one in the transcription settings.")));
|
||||
return false;
|
||||
}
|
||||
|
||||
var message = this.T("The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider.");
|
||||
var dialogParameters = new DialogParameters<ConfirmDialog>
|
||||
{
|
||||
{
|
||||
x => x.MarkdownBody,
|
||||
$"""
|
||||
{message}
|
||||
|
||||
- {Markdown.EscapeInlineText(Path.GetFileName(filePath))}
|
||||
"""
|
||||
},
|
||||
};
|
||||
var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(
|
||||
this.T("Transcribe media file"),
|
||||
dialogParameters,
|
||||
Dialogs.DialogOptions.FULLSCREEN);
|
||||
|
||||
var dialogResult = await dialogReference.Result;
|
||||
if (dialogResult is null || dialogResult.Canceled)
|
||||
return false;
|
||||
|
||||
return this.MediaTranscriptionService.TryStartTextImport(
|
||||
filePath,
|
||||
this.EffectiveMediaImportTarget);
|
||||
}
|
||||
|
||||
private bool CanCatchDroppedFile() => this.numDropAreasAboveThis is 0 && (this.isComponentHovered || this.CatchAllDocuments);
|
||||
|
||||
private void SetDragClass() => this.dragClass = $"{DEFAULT_DRAG_CLASS} mud-border-primary border-2";
|
||||
@@ -208,7 +332,7 @@ public partial class ReadFileContent : MSGComponentBase
|
||||
|
||||
private void OnMouseEnter(EventArgs _)
|
||||
{
|
||||
if(this.Disabled || this.numDropAreasAboveThis > 0)
|
||||
if(this.IsUnavailable || this.numDropAreasAboveThis > 0)
|
||||
return;
|
||||
|
||||
this.Logger.LogDebug("Read file content component is hovered.");
|
||||
@@ -219,7 +343,7 @@ public partial class ReadFileContent : MSGComponentBase
|
||||
|
||||
private void OnMouseLeave(EventArgs _)
|
||||
{
|
||||
if(this.Disabled)
|
||||
if(this.IsUnavailable)
|
||||
return;
|
||||
|
||||
this.Logger.LogDebug("Read file content component is no longer hovered.");
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
using AIStudio.Provider;
|
||||
using System.Buffers.Binary;
|
||||
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.MIME;
|
||||
using AIStudio.Tools.Media;
|
||||
using AIStudio.Tools.Rust;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
@@ -10,6 +11,8 @@ namespace AIStudio.Components;
|
||||
|
||||
public partial class VoiceRecorder : MSGComponentBase
|
||||
{
|
||||
private const int PCM_WAV_HEADER_SIZE = 44;
|
||||
|
||||
[Inject]
|
||||
private ILogger<VoiceRecorder> Logger { get; init; } = null!;
|
||||
|
||||
@@ -25,6 +28,9 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
[Inject]
|
||||
private VoiceRecordingAvailabilityService VoiceRecordingAvailabilityService { get; init; } = null!;
|
||||
|
||||
[Inject]
|
||||
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
|
||||
|
||||
#region Overrides of MSGComponentBase
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
@@ -93,7 +99,6 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
private bool isTranscribing;
|
||||
private FileStream? currentRecordingStream;
|
||||
private string? currentRecordingPath;
|
||||
private string? currentRecordingMimeType;
|
||||
private string? finalRecordingPath;
|
||||
private DotNetObjectReference<VoiceRecorder>? dotNetReference;
|
||||
|
||||
@@ -131,17 +136,7 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
return;
|
||||
}
|
||||
|
||||
var mimeTypes = GetPreferredMimeTypes(
|
||||
Builder.Create().UseAudio().UseSubtype(AudioSubtype.WEBM).Build(),
|
||||
Builder.Create().UseAudio().UseSubtype(AudioSubtype.OGG).Build(),
|
||||
Builder.Create().UseAudio().UseSubtype(AudioSubtype.AAC).Build(),
|
||||
Builder.Create().UseAudio().UseSubtype(AudioSubtype.MP3).Build(),
|
||||
Builder.Create().UseAudio().UseSubtype(AudioSubtype.AIFF).Build(),
|
||||
Builder.Create().UseAudio().UseSubtype(AudioSubtype.WAV).Build(),
|
||||
Builder.Create().UseAudio().UseSubtype(AudioSubtype.FLAC).Build()
|
||||
);
|
||||
|
||||
this.Logger.LogInformation("Starting audio recording with preferred MIME types: '{PreferredMimeTypes}'.", string.Join<MIMEType>(", ", mimeTypes));
|
||||
this.Logger.LogInformation("Starting PCM/WAV audio recording.");
|
||||
|
||||
// Create a DotNetObjectReference to pass to JavaScript:
|
||||
this.dotNetReference = DotNetObjectReference.Create(this);
|
||||
@@ -151,13 +146,8 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
|
||||
try
|
||||
{
|
||||
var mimeTypeStrings = mimeTypes.ToStringArray();
|
||||
var actualMimeType = await this.JsRuntime.InvokeAsync<string>("audioRecorder.start", this.dotNetReference, mimeTypeStrings);
|
||||
|
||||
// Store the MIME type for later use:
|
||||
this.currentRecordingMimeType = actualMimeType;
|
||||
|
||||
this.Logger.LogInformation("Audio recording started with MIME type: '{ActualMimeType}'.", actualMimeType);
|
||||
await this.JsRuntime.InvokeVoidAsync("audioRecorder.start", this.dotNetReference);
|
||||
this.Logger.LogInformation("PCM/WAV audio recording started.");
|
||||
this.isPreparing = false;
|
||||
this.isRecording = true;
|
||||
}
|
||||
@@ -168,6 +158,7 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
|
||||
// Clean up the recording stream if starting failed:
|
||||
await this.FinalizeRecordingStream();
|
||||
await this.ReleaseMicrophoneAsync();
|
||||
}
|
||||
finally
|
||||
{
|
||||
@@ -176,11 +167,11 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
}
|
||||
else
|
||||
{
|
||||
var recordingStoppedSuccessfully = false;
|
||||
try
|
||||
{
|
||||
var result = await this.JsRuntime.InvokeAsync<AudioRecordingResult>("audioRecorder.stop");
|
||||
if (result.ChangedMimeType)
|
||||
this.Logger.LogWarning("The recorded audio MIME type was changed to '{ResultMimeType}'.", result.MimeType);
|
||||
await this.JsRuntime.InvokeVoidAsync("audioRecorder.stop");
|
||||
recordingStoppedSuccessfully = true;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
@@ -194,28 +185,21 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
this.isRecording = false;
|
||||
this.StateHasChanged();
|
||||
|
||||
// Start transcription if we have a recording and a configured provider:
|
||||
if (this.finalRecordingPath is not null)
|
||||
await this.TranscribeRecordingAsync();
|
||||
}
|
||||
}
|
||||
if (!recordingStoppedSuccessfully || this.finalRecordingPath is null)
|
||||
{
|
||||
if (recordingStoppedSuccessfully)
|
||||
{
|
||||
this.Logger.LogWarning("The audio recorder did not produce any data.");
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.MicOff, this.T("Failed to stop audio recording.")));
|
||||
}
|
||||
|
||||
private static MIMEType[] GetPreferredMimeTypes(params MIMEType[] mimeTypes)
|
||||
{
|
||||
// Default list if no parameters provided:
|
||||
if (mimeTypes.Length is 0)
|
||||
{
|
||||
var audioBuilder = Builder.Create().UseAudio();
|
||||
return
|
||||
[
|
||||
audioBuilder.UseSubtype(AudioSubtype.WEBM).Build(),
|
||||
audioBuilder.UseSubtype(AudioSubtype.OGG).Build(),
|
||||
audioBuilder.UseSubtype(AudioSubtype.MP4).Build(),
|
||||
audioBuilder.UseSubtype(AudioSubtype.MPEG).Build(),
|
||||
];
|
||||
}
|
||||
this.DeleteFinalRecording();
|
||||
await this.ReleaseMicrophoneAsync();
|
||||
return;
|
||||
}
|
||||
|
||||
return mimeTypes;
|
||||
await this.TranscribeRecordingAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private async Task InitializeRecordingStream()
|
||||
@@ -226,7 +210,7 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
if (!Directory.Exists(recordingDirectory))
|
||||
Directory.CreateDirectory(recordingDirectory);
|
||||
|
||||
var fileName = $"recording_{DateTime.UtcNow:yyyyMMdd_HHmmss}.audio";
|
||||
var fileName = $"recording_{DateTime.UtcNow:yyyyMMdd_HHmmss}.wav";
|
||||
this.currentRecordingPath = Path.Combine(recordingDirectory, fileName);
|
||||
this.currentRecordingStream = new FileStream(this.currentRecordingPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 8192, useAsync: true);
|
||||
|
||||
@@ -253,6 +237,7 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
catch (Exception ex)
|
||||
{
|
||||
this.Logger.LogError(ex, "Error writing audio chunk to stream.");
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -262,45 +247,56 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
if (this.currentRecordingStream is not null)
|
||||
{
|
||||
await this.currentRecordingStream.FlushAsync();
|
||||
var hasPcmAudioData = await this.FinalizePcmWavHeaderAsync(this.currentRecordingStream);
|
||||
await this.currentRecordingStream.DisposeAsync();
|
||||
this.currentRecordingStream = null;
|
||||
|
||||
// Rename the file with the correct extension based on MIME type:
|
||||
if (this.currentRecordingPath is not null && this.currentRecordingMimeType is not null)
|
||||
if (this.currentRecordingPath is not null && File.Exists(this.currentRecordingPath))
|
||||
{
|
||||
var extension = GetFileExtension(this.currentRecordingMimeType);
|
||||
var newPath = Path.ChangeExtension(this.currentRecordingPath, extension);
|
||||
var fileSize = new FileInfo(this.currentRecordingPath).Length;
|
||||
|
||||
if (File.Exists(this.currentRecordingPath))
|
||||
if (hasPcmAudioData)
|
||||
{
|
||||
File.Move(this.currentRecordingPath, newPath, overwrite: true);
|
||||
this.finalRecordingPath = newPath;
|
||||
this.Logger.LogInformation("Finalized audio recording over {NumChunks} streamed audio chunks to the file '{RecordingPath}'.", this.numReceivedChunks, newPath);
|
||||
this.finalRecordingPath = this.currentRecordingPath;
|
||||
this.Logger.LogInformation("Finalized audio recording over {NumChunks} streamed audio chunks to the file '{RecordingPath}' with {FileSize} bytes.", this.numReceivedChunks, this.currentRecordingPath, fileSize);
|
||||
}
|
||||
else
|
||||
{
|
||||
this.Logger.LogWarning("Discarding a PCM/WAV audio recording without audio data ({FileSize} bytes).", fileSize);
|
||||
File.Delete(this.currentRecordingPath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
this.currentRecordingPath = null;
|
||||
this.currentRecordingMimeType = null;
|
||||
|
||||
// Dispose the .NET reference:
|
||||
this.dotNetReference?.Dispose();
|
||||
this.dotNetReference = null;
|
||||
}
|
||||
|
||||
private static string GetFileExtension(string mimeType)
|
||||
private async Task<bool> FinalizePcmWavHeaderAsync(FileStream recordingStream)
|
||||
{
|
||||
var baseMimeType = mimeType.Split(';')[0].Trim().ToLowerInvariant();
|
||||
return baseMimeType switch
|
||||
{
|
||||
"audio/webm" => ".webm",
|
||||
"audio/ogg" => ".ogg",
|
||||
"audio/mp4" => ".m4a",
|
||||
"audio/mpeg" => ".mp3",
|
||||
"audio/wav" => ".wav",
|
||||
"audio/x-wav" => ".wav",
|
||||
_ => ".audio" // Fallback
|
||||
};
|
||||
if (recordingStream.Length <= PCM_WAV_HEADER_SIZE)
|
||||
return false;
|
||||
|
||||
var pcmDataSize = recordingStream.Length - PCM_WAV_HEADER_SIZE;
|
||||
if (pcmDataSize > uint.MaxValue - 36)
|
||||
throw new InvalidDataException("The streamed PCM recording exceeds the WAV size limit.");
|
||||
|
||||
var valueBuffer = new byte[sizeof(uint)];
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(valueBuffer, checked((uint)(36 + pcmDataSize)));
|
||||
recordingStream.Seek(4, SeekOrigin.Begin);
|
||||
await recordingStream.WriteAsync(valueBuffer);
|
||||
|
||||
BinaryPrimitives.WriteUInt32LittleEndian(valueBuffer, checked((uint)pcmDataSize));
|
||||
recordingStream.Seek(40, SeekOrigin.Begin);
|
||||
await recordingStream.WriteAsync(valueBuffer);
|
||||
recordingStream.Seek(0, SeekOrigin.End);
|
||||
await recordingStream.FlushAsync();
|
||||
|
||||
this.Logger.LogInformation("Finalized a streamed PCM/WAV header for {PcmDataSize} bytes of audio data.", pcmDataSize);
|
||||
return true;
|
||||
}
|
||||
|
||||
private async Task TranscribeRecordingAsync()
|
||||
@@ -317,58 +313,22 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
|
||||
try
|
||||
{
|
||||
// Get the configured transcription provider ID:
|
||||
var transcriptionProviderId = this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider;
|
||||
if (string.IsNullOrWhiteSpace(transcriptionProviderId))
|
||||
var transcriptionResult = await this.MediaTranscriptionService.TranscribeVoiceAsync(this.finalRecordingPath);
|
||||
if (transcriptionResult.Status is not MediaTranscriptionResultStatus.SUCCEEDED)
|
||||
{
|
||||
this.Logger.LogWarning("No transcription provider is configured.");
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("No transcription provider is configured.")));
|
||||
return;
|
||||
}
|
||||
if (transcriptionResult.Status is MediaTranscriptionResultStatus.CANCELLED)
|
||||
return;
|
||||
|
||||
// Find the transcription provider in the list of configured providers:
|
||||
var transcriptionProviderSettings = this.SettingsManager.ConfigurationData.TranscriptionProviders
|
||||
.FirstOrDefault(x => x.Id == transcriptionProviderId);
|
||||
if (transcriptionResult.Status is MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL)
|
||||
{
|
||||
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, transcriptionResult.UserMessage));
|
||||
return;
|
||||
}
|
||||
|
||||
if (transcriptionProviderSettings is null)
|
||||
{
|
||||
this.Logger.LogWarning("The configured transcription provider with ID '{ProviderId}' was not found.", transcriptionProviderId);
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The configured transcription provider was not found.")));
|
||||
return;
|
||||
}
|
||||
|
||||
// Check the confidence level:
|
||||
var minimumLevel = this.SettingsManager.GetMinimumConfidenceLevel(Tools.Components.NONE);
|
||||
var providerConfidence = transcriptionProviderSettings.UsedLLMProvider.GetConfidence(this.SettingsManager);
|
||||
if (providerConfidence.Level < minimumLevel)
|
||||
{
|
||||
this.Logger.LogWarning(
|
||||
"The configured transcription provider '{ProviderName}' has a confidence level of '{ProviderLevel}', which is below the minimum required level of '{MinimumLevel}'.",
|
||||
transcriptionProviderSettings.UsedLLMProvider,
|
||||
providerConfidence.Level,
|
||||
minimumLevel);
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The configured transcription provider does not meet the minimum confidence level.")));
|
||||
return;
|
||||
}
|
||||
|
||||
// Create the provider instance:
|
||||
var provider = transcriptionProviderSettings.CreateProvider();
|
||||
if (provider.Provider is LLMProviders.NONE)
|
||||
{
|
||||
this.Logger.LogError("Failed to create the transcription provider instance.");
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("Failed to create the transcription provider.")));
|
||||
return;
|
||||
}
|
||||
|
||||
// Call the transcription API:
|
||||
this.Logger.LogInformation("Starting transcription with provider '{ProviderName}' and model '{ModelName}'.", transcriptionProviderSettings.UsedLLMProvider, transcriptionProviderSettings.Model.ToString());
|
||||
var transcriptionResult = await provider.TranscribeAudioAsync(transcriptionProviderSettings.Model, this.finalRecordingPath, this.SettingsManager);
|
||||
if (!transcriptionResult.Success)
|
||||
{
|
||||
this.Logger.LogWarning("The transcription request failed.");
|
||||
var userMessage = string.IsNullOrWhiteSpace(transcriptionResult.ErrorMessage)
|
||||
var userMessage = string.IsNullOrWhiteSpace(transcriptionResult.UserMessage)
|
||||
? this.T("Unfortunately, there was an error communicating with the AI system.")
|
||||
: transcriptionResult.ErrorMessage;
|
||||
: transcriptionResult.UserMessage;
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, userMessage));
|
||||
return;
|
||||
}
|
||||
@@ -406,19 +366,6 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
// Copy the transcribed text to the clipboard:
|
||||
await this.RustService.CopyText2Clipboard(this.Snackbar, transcribedText);
|
||||
|
||||
// Delete the recording file:
|
||||
try
|
||||
{
|
||||
if (File.Exists(this.finalRecordingPath))
|
||||
{
|
||||
File.Delete(this.finalRecordingPath);
|
||||
this.Logger.LogInformation("Deleted the recording file '{RecordingPath}'.", this.finalRecordingPath);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
this.Logger.LogError(ex, "Failed to delete the recording file '{RecordingPath}'.", this.finalRecordingPath);
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
@@ -428,13 +375,31 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
finally
|
||||
{
|
||||
await this.ReleaseMicrophoneAsync();
|
||||
|
||||
this.finalRecordingPath = null;
|
||||
this.DeleteFinalRecording();
|
||||
this.isTranscribing = false;
|
||||
this.StateHasChanged();
|
||||
}
|
||||
}
|
||||
|
||||
private void DeleteFinalRecording()
|
||||
{
|
||||
var recordingPath = this.finalRecordingPath;
|
||||
this.finalRecordingPath = null;
|
||||
|
||||
if (recordingPath is null)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
if (File.Exists(recordingPath))
|
||||
File.Delete(recordingPath);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
this.Logger.LogError(ex, "Failed to delete the recording file '{RecordingPath}'.", recordingPath);
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReleaseMicrophoneAsync()
|
||||
{
|
||||
// Wait a moment for any queued sounds to finish playing, then release the microphone.
|
||||
@@ -530,4 +495,4 @@ public partial class VoiceRecorder : MSGComponentBase
|
||||
}
|
||||
|
||||
#endregion
|
||||
}
|
||||
}
|
||||
@@ -4,6 +4,8 @@ using System.Text.Json;
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Dialogs;
|
||||
using AIStudio.Tools.AIJobs;
|
||||
using AIStudio.Tools.Media;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
using Microsoft.AspNetCore.Components;
|
||||
|
||||
@@ -21,6 +23,9 @@ public partial class Workspaces : MSGComponentBase
|
||||
|
||||
[Inject]
|
||||
private AIJobService AIJobService { get; init; } = null!;
|
||||
|
||||
[Inject]
|
||||
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
|
||||
|
||||
[Parameter]
|
||||
public ChatThread? CurrentChatThread { get; set; }
|
||||
@@ -55,6 +60,7 @@ public partial class Workspaces : MSGComponentBase
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged;
|
||||
await base.OnInitializedAsync();
|
||||
this.ApplyFilters([], [ Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED, Event.CHAT_GENERATION_CHANGED, Event.WORKSPACE_CREATED ]);
|
||||
_ = this.LoadTreeItemsAsync(startPrefetch: true);
|
||||
@@ -376,12 +382,26 @@ public partial class Workspaces : MSGComponentBase
|
||||
|
||||
private bool IsChatTreeItemBusy(TreeItemData treeItem)
|
||||
{
|
||||
return treeItem.Type is TreeItemType.CHAT && this.AIJobService.IsChatGenerationActive(treeItem.ChatId);
|
||||
return treeItem.Type is TreeItemType.CHAT
|
||||
&& (this.AIJobService.IsChatGenerationActive(treeItem.ChatId)
|
||||
|| this.MediaTranscriptionService.IsBusy(MediaImportOwner.ForChat(treeItem.ChatId)));
|
||||
}
|
||||
|
||||
private string GetChatTreeItemTextStyle(TreeItemData treeItem)
|
||||
{
|
||||
return this.IsCurrentChatTreeItem(treeItem) ? "justify-self: start; font-weight: 700;" : "justify-self: start;";
|
||||
var status = this.MediaTranscriptionService.GetSnapshot(MediaImportOwner.ForChat(treeItem.ChatId))?.Status;
|
||||
var color = status switch
|
||||
{
|
||||
MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => " color: var(--mud-palette-info);",
|
||||
MediaImportStatus.SUCCEEDED => " color: var(--mud-palette-success);",
|
||||
MediaImportStatus.WARNING => " color: var(--mud-palette-warning);",
|
||||
MediaImportStatus.FAILED => " color: var(--mud-palette-error);",
|
||||
MediaImportStatus.CANCELLED => " color: var(--mud-palette-warning);",
|
||||
_ => string.Empty,
|
||||
};
|
||||
|
||||
var weight = this.IsCurrentChatTreeItem(treeItem) ? " font-weight: 700;" : string.Empty;
|
||||
return $"justify-self: start;{weight}{color}";
|
||||
}
|
||||
|
||||
private bool IsCurrentChatTreeItem(TreeItemData treeItem)
|
||||
@@ -394,6 +414,22 @@ public partial class Workspaces : MSGComponentBase
|
||||
|
||||
private string GetChatTreeIcon(Guid chatId, string defaultIcon)
|
||||
{
|
||||
var mediaStatus = this.MediaTranscriptionService.GetSnapshot(MediaImportOwner.ForChat(chatId))?.Status;
|
||||
if (mediaStatus is not null)
|
||||
{
|
||||
return mediaStatus switch
|
||||
{
|
||||
MediaImportStatus.QUEUED => Icons.Material.Filled.HourglassTop,
|
||||
MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => Icons.Material.Filled.ChangeCircle,
|
||||
MediaImportStatus.SUCCEEDED => Icons.Material.Filled.TaskAlt,
|
||||
MediaImportStatus.WARNING => Icons.Material.Filled.WarningAmber,
|
||||
MediaImportStatus.FAILED => Icons.Material.Filled.Error,
|
||||
MediaImportStatus.CANCELLED => Icons.Material.Filled.Cancel,
|
||||
|
||||
_ => defaultIcon,
|
||||
};
|
||||
}
|
||||
|
||||
var snapshot = this.AIJobService.TryGetChatSnapshot(chatId);
|
||||
if (snapshot is null || !snapshot.IsActive)
|
||||
return defaultIcon;
|
||||
@@ -406,6 +442,12 @@ public partial class Workspaces : MSGComponentBase
|
||||
};
|
||||
}
|
||||
|
||||
private void OnMediaImportStateChanged(MediaImportOwner owner)
|
||||
{
|
||||
if (owner.Kind is MediaImportOwnerKind.CHAT)
|
||||
_ = this.SafeStateHasChanged();
|
||||
}
|
||||
|
||||
private async Task SafeStateHasChanged()
|
||||
{
|
||||
if (this.isDisposed)
|
||||
@@ -668,7 +710,8 @@ public partial class Workspaces : MSGComponentBase
|
||||
if (chat is null)
|
||||
return;
|
||||
|
||||
if (this.AIJobService.IsChatGenerationActive(chat.ChatId))
|
||||
var mediaOwner = MediaImportOwner.ForChat(chat.ChatId);
|
||||
if (this.AIJobService.IsChatGenerationActive(chat.ChatId) || this.MediaTranscriptionService.IsBusy(mediaOwner))
|
||||
return;
|
||||
|
||||
if (askForConfirmation)
|
||||
@@ -692,6 +735,7 @@ public partial class Workspaces : MSGComponentBase
|
||||
}
|
||||
|
||||
await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, chat.WorkspaceId, chat.ChatId, askForConfirmation: false);
|
||||
this.MediaTranscriptionService.ClearOwnerState(mediaOwner);
|
||||
await this.LoadTreeItemsAsync(startPrefetch: false);
|
||||
|
||||
if (unloadChat && this.CurrentChatThread?.ChatId == chat.ChatId)
|
||||
@@ -845,16 +889,13 @@ public partial class Workspaces : MSGComponentBase
|
||||
if (workspaceId == Guid.Empty)
|
||||
return;
|
||||
|
||||
await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, chat.WorkspaceId, chat.ChatId, askForConfirmation: false);
|
||||
|
||||
chat.WorkspaceId = workspaceId;
|
||||
await WorkspaceBehaviour.MoveChatAsync(chat, workspaceId);
|
||||
if (this.CurrentChatThread?.ChatId == chat.ChatId)
|
||||
{
|
||||
this.CurrentChatThread = chat;
|
||||
await this.CurrentChatThreadChanged.InvokeAsync(this.CurrentChatThread);
|
||||
}
|
||||
|
||||
await WorkspaceBehaviour.StoreChatAsync(chat);
|
||||
|
||||
await this.LoadTreeItemsAsync(startPrefetch: false);
|
||||
}
|
||||
|
||||
@@ -914,6 +955,7 @@ public partial class Workspaces : MSGComponentBase
|
||||
|
||||
protected override void DisposeResources()
|
||||
{
|
||||
this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged;
|
||||
this.isDisposed = true;
|
||||
this.prefetchCancellationTokenSource?.Cancel();
|
||||
this.prefetchCancellationTokenSource?.Dispose();
|
||||
|
||||
Reference in new issue
Block a user