Added audio and video transcription for chats and assistants (#856)
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2026-07-15 12:53:30 +02:00
1 parent 5dc6282908
commit aaf77b6882
83 files changed
+5985 -442

No files matched your search

@@ -1,6 +1,9 @@
using AIStudio.Dialogs.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.AssistantSessions;
using AIStudio.Tools.Media;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
using DialogOptions = AIStudio.Dialogs.DialogOptions;
@@ -60,6 +63,9 @@ public partial class AssistantBlock<TSettings> : MSGComponentBase where TSetting
[Inject]
private AssistantSessionService AssistantSessionService { get; init; } = null!;
[Inject]
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
private async Task OpenSettingsDialog()
{
@@ -71,7 +77,7 @@ public partial class AssistantBlock<TSettings> : MSGComponentBase where TSetting
await this.DialogService.ShowAsync<TSettings>(T("Open Settings"), dialogParameters, DialogOptions.FULLSCREEN);
}
private string BorderColor => this.AssistantSessionSnapshot?.IsActive is true ? this.ColorTheme.GetActivityIndicatorColor(this.SettingsManager) : this.SettingsManager.IsDarkMode switch
private string BorderColor => this.AssistantSessionSnapshot?.IsActive is true || this.MediaImportSnapshot?.IsBusy is true ? this.ColorTheme.GetActivityIndicatorColor(this.SettingsManager) : this.SettingsManager.IsDarkMode switch
{
true => this.ColorTheme.GetCurrentPalette(this.SettingsManager).GrayDefault,
false => this.ColorTheme.GetCurrentPalette(this.SettingsManager).GrayDefault,
@@ -92,10 +98,29 @@ public partial class AssistantBlock<TSettings> : MSGComponentBase where TSetting
? this.AssistantSessionService.GetSnapshots().FirstOrDefault(snapshot => snapshot.Key.Component == this.Component)
: this.AssistantSessionService.GetSnapshots().FirstOrDefault(snapshot => snapshot.Key.InstanceId == this.AssistantSessionInstanceId);
private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForAssistant(new AssistantSessionKey(this.Component, this.AssistantSessionInstanceId));
private MediaImportSnapshot? MediaImportSnapshot => string.IsNullOrWhiteSpace(this.AssistantSessionInstanceId)
? this.MediaTranscriptionService.GetSnapshots().FirstOrDefault(snapshot =>
snapshot.Owner.Kind is MediaImportOwnerKind.ASSISTANT
&& snapshot.Owner.Id.StartsWith($"{this.Component}:", StringComparison.Ordinal))
: this.MediaTranscriptionService.GetSnapshot(this.CurrentMediaImportOwner);
/// <summary>
/// Gets the assistant session indicator shown on top of the assistant icon.
/// </summary>
private AssistantSessionIndicatorData? AssistantSessionIndicator => this.AssistantSessionSnapshot?.Status switch
private AssistantSessionIndicatorData? AssistantSessionIndicator => this.MediaImportSnapshot?.Status switch
{
MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => new(Icons.Material.Filled.ChangeCircle, Color.Info, this.T("Media is still being prepared.")),
MediaImportStatus.SUCCEEDED => new(Icons.Material.Filled.TaskAlt, Color.Success, this.T("The media transcript is ready.")),
MediaImportStatus.WARNING => new(Icons.Material.Filled.WarningAmber, Color.Warning, this.T("Media transcription completed with a warning. Open the assistant to review it.")),
MediaImportStatus.FAILED => new(Icons.Material.Filled.Error, Color.Error, this.T("Media transcription failed. Open the assistant to review it.")),
MediaImportStatus.CANCELLED => new(Icons.Material.Filled.Cancel, Color.Warning, this.T("Media transcription was canceled. Open the assistant to review it.")),
_ => this.AssistantSessionIndicatorWithoutMedia,
};
private AssistantSessionIndicatorData? AssistantSessionIndicatorWithoutMedia => this.AssistantSessionSnapshot?.Status switch
{
AssistantSessionStatus.RUNNING or AssistantSessionStatus.CANCELING => new(Icons.Material.Filled.ChangeCircle, Color.Info, this.T("Assistant is still running.")),
AssistantSessionStatus.COMPLETED => new(Icons.Material.Filled.TaskAlt, Color.Success, this.T("The result is ready.")),
@@ -104,6 +129,28 @@ public partial class AssistantBlock<TSettings> : MSGComponentBase where TSetting
_ => null,
};
protected override async Task OnInitializedAsync()
{
this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged;
await base.OnInitializedAsync();
}
private void OnMediaImportStateChanged(MediaImportOwner owner)
{
var matches = string.IsNullOrWhiteSpace(this.AssistantSessionInstanceId)
? owner.Kind is MediaImportOwnerKind.ASSISTANT && owner.Id.StartsWith($"{this.Component}:", StringComparison.Ordinal)
: owner == this.CurrentMediaImportOwner;
if (matches)
_ = this.InvokeAsync(this.StateHasChanged);
}
protected override void DisposeResources()
{
this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged;
base.DisposeResources();
}
/// <summary>
/// Refreshes the block when assistant session activity changes.
/// </summary>
@@ -2,57 +2,66 @@
@if (this.UseSmallForm)
{
<div @onmouseenter="@this.OnMouseEnter" @onmouseleave="@this.OnMouseLeave">
@if (this.isDraggingOver)
{
<MudBadge
Content="@this.DocumentPaths.Count"
Color="Color.Primary"
Overlap="true"
Class="cursor-pointer"
OnClick="@this.OpenAttachmentsDialog">
<MudLink OnClick="@this.AddFilesManually" Style="text-decoration: none;">
<MudTextField T="string"
Text="@DROP_FILES_HERE_TEXT"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
Typo="Typo.body2"
Variant="Variant.Outlined"
ReadOnly="true"
/>
</MudLink>
</MudBadge>
}
else if (this.DocumentPaths.Any())
{
<MudTooltip Text="@T("Click the paperclip to attach files, or click the number to see your attached files.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
<MudStack Spacing="0">
<div @onmouseenter="@this.OnMouseEnter" @onmouseleave="@this.OnMouseLeave">
@if (this.isDraggingOver)
{
<MudBadge
Content="@this.DocumentPaths.Count"
Color="Color.Primary"
Overlap="true"
Class="cursor-pointer"
OnClick="@this.OpenAttachmentsDialog">
<MudLink OnClick="@this.AddFilesManually" Style="text-decoration: none;">
<MudTextField T="string"
Text="@DROP_FILES_HERE_TEXT"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
Typo="Typo.body2"
Variant="Variant.Outlined"
ReadOnly="true"
Disabled="@this.IsUnavailable"
/>
</MudLink>
</MudBadge>
}
else if (this.DocumentPaths.Any())
{
<MudTooltip Text="@T("Click the paperclip to attach files, or click the number to see your attached files.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
<MudBadge
Content="@this.DocumentPaths.Count"
Color="Color.Primary"
Overlap="true"
Class="cursor-pointer"
OnClick="@this.OpenAttachmentsDialog">
<MudIconButton
Icon="@Icons.Material.Filled.AttachFile"
Color="Color.Default"
Disabled="@this.IsUnavailable"
OnClick="@this.AddFilesManually"/>
</MudBadge>
</MudTooltip>
}
else
{
<MudTooltip Text="@T("Click here to attach files.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
<MudIconButton
Icon="@Icons.Material.Filled.AttachFile"
Color="Color.Default"
Disabled="@this.IsUnavailable"
OnClick="@this.AddFilesManually"/>
</MudBadge>
</MudTooltip>
}
else
</MudTooltip>
}
</div>
@if (this.ShowMediaStatus)
{
<MudTooltip Text="@T("Click here to attach files.")" Placement="@TOOLBAR_TOOLTIP_PLACEMENT">
<MudIconButton
Icon="@Icons.Material.Filled.AttachFile"
Color="Color.Default"
OnClick="@this.AddFilesManually"/>
</MudTooltip>
<MediaTranscriptionStatus Owner="@this.EffectiveImportOwner" TargetId="@this.EffectiveMediaImportTarget.TargetId" Compact="true"/>
}
</div>
</MudStack>
}
else
{
@if (!this.Disabled)
@if (!this.IsUnavailable)
{
<MudStack Row="true" AlignItems="AlignItems.Center" StretchItems="StretchItems.None" Wrap="Wrap.Wrap">
<MudText Typo="Typo.body1" Inline="true">
@@ -69,11 +78,15 @@ else
</MudButton>
</MudStack>
}
@if (this.ShowMediaStatus)
{
<MediaTranscriptionStatus Owner="@this.EffectiveImportOwner" TargetId="@this.EffectiveMediaImportTarget.TargetId"/>
}
<div @onmouseenter="@this.OnMouseEnter" @onmouseleave="@this.OnMouseLeave">
<MudPaper Height="20em" Outlined="true" Class="@this.dragClass" Style="overflow-y: auto;">
@foreach (var fileAttachment in this.DocumentPaths)
{
@if (this.Disabled)
@if (this.IsUnavailable)
{
<MudChip T="string" Color="Color.Dark" Text="@fileAttachment.FileName" tabindex="-1" Icon="@Icons.Material.Filled.Search" OnClick="@(() => this.InvestigateFile(fileAttachment))"/>
}
@@ -84,7 +97,7 @@ else
}
</MudPaper>
</div>
@if (!this.Disabled)
@if (!this.IsUnavailable)
{
<MudButton OnClick="@(async () => await this.ClearAllFiles())" Variant="Variant.Filled" Color="Color.Info" Class="mt-2" StartIcon="@Icons.Material.Filled.Delete">
@T("Clear file list")
@@ -1,5 +1,6 @@
using AIStudio.Chat;
using AIStudio.Dialogs;
using AIStudio.Tools.Media;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
@@ -13,6 +14,11 @@ using DialogOptions = Dialogs.DialogOptions;
public partial class AttachDocuments : MSGComponentBase
{
private readonly MediaImportOwner fallbackMediaImportOwner = new(MediaImportOwnerKind.CHAT, $"attachments:{Guid.NewGuid():N}");
[CascadingParameter]
private MediaImportOwner? ImportOwner { get; set; }
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(AttachDocuments).Namespace, nameof(AttachDocuments));
[Parameter]
@@ -48,6 +54,10 @@ public partial class AttachDocuments : MSGComponentBase
[Parameter]
public bool UseSmallForm { get; set; }
/// <summary>Whether this control renders its own media status.</summary>
[Parameter]
public bool ShowMediaStatus { get; set; } = true;
[Parameter]
public bool Disabled { get; set; }
@@ -63,6 +73,14 @@ public partial class AttachDocuments : MSGComponentBase
[Parameter]
public AIStudio.Settings.Provider? Provider { get; set; }
/// <summary>Optional persisted chat that can own transcript files immediately.</summary>
[Parameter]
public ChatThread? OwnerChat { get; set; }
/// <summary>Creates and persists a draft owner after media import confirmation.</summary>
[Parameter]
public Func<string, Task<ChatThread?>> EnsureOwnerChatAsync { get; set; } = _ => Task.FromResult<ChatThread?>(null);
[Inject]
private ILogger<AttachDocuments> Logger { get; set; } = null!;
@@ -75,17 +93,28 @@ public partial class AttachDocuments : MSGComponentBase
[Inject]
private PandocAvailabilityService PandocAvailabilityService { get; init; } = null!;
[Inject]
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top;
private static readonly string DROP_FILES_HERE_TEXT = TB("Drop files here to attach them.");
private uint numDropAreasAboveThis;
private bool isComponentHovered;
private bool isDraggingOver;
private MediaImportOwner EffectiveImportOwner => this.OwnerChat is not null
? MediaImportOwner.ForChat(this.OwnerChat.ChatId)
: this.ImportOwner ?? this.fallbackMediaImportOwner;
private MediaImportTarget EffectiveMediaImportTarget => new(this.EffectiveImportOwner, string.IsNullOrWhiteSpace(this.Name) ? "attachments" : this.Name);
private bool IsUnavailable => this.Disabled || this.MediaTranscriptionService.IsBusy(this.EffectiveImportOwner);
#region Overrides of MSGComponentBase
protected override async Task OnInitializedAsync()
{
this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged;
this.ApplyFilters([], [ Event.TAURI_EVENT_RECEIVED, Event.REGISTER_FILE_DROP_AREA, Event.UNREGISTER_FILE_DROP_AREA ]);
// Register this drop area:
@@ -93,9 +122,101 @@ public partial class AttachDocuments : MSGComponentBase
await base.OnInitializedAsync();
}
/// <summary>Rehydrates results after the component is assigned another chat or target.</summary>
protected override async Task OnParametersSetAsync()
{
await base.OnParametersSetAsync();
await this.SyncCompletedMediaAttachmentsAsync();
}
/// <summary>Refreshes disabled controls when the shared import lane changes.</summary>
private void OnMediaImportStateChanged(MediaImportOwner owner)
{
if (owner == this.EffectiveImportOwner)
_ = this.InvokeAsync(async () =>
{
await this.SyncCompletedMediaAttachmentsAsync();
await this.ConsumeStandaloneMediaOutcomeAsync();
this.StateHasChanged();
});
}
/// <summary>Consumes outcomes for dialog-local controls that have no chat or assistant owner surface.</summary>
private async Task ConsumeStandaloneMediaOutcomeAsync()
{
if (this.ImportOwner is not null || this.OwnerChat is not null)
return;
var outcome = this.MediaTranscriptionService.TryConsumeOutcome(this.EffectiveImportOwner);
if (outcome is null)
return;
if (outcome.Failures.Count > 0)
{
var message = string.Join(Environment.NewLine, outcome.Failures.Select(failure => $"{failure.FileName}: {failure.UserMessage}"));
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, message));
}
else if (outcome.Status is MediaImportStatus.FAILED)
{
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed.")));
}
if (outcome.Warnings.Count > 0)
{
var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}"));
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message));
}
if (outcome.Status is MediaImportStatus.CANCELLED)
{
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled.")));
}
}
/// <summary>Reattaches completed owner results after progress updates or navigation.</summary>
private async Task SyncCompletedMediaAttachmentsAsync()
{
var delivery = this.MediaTranscriptionService.GetPendingDelivery(this.EffectiveMediaImportTarget);
var completed = delivery?.Attachments ?? [];
var pending = this.OwnerChat?.PendingMediaTranscripts ?? [];
var changed = false;
var ownerPendingChanged = false;
foreach (var attachment in completed.Concat(pending))
changed |= this.DocumentPaths.Add(attachment);
if (this.OwnerChat is not null)
{
foreach (var attachment in completed.OfType<ManagedTranscriptAttachment>())
{
if (this.OwnerChat.PendingMediaTranscripts.All(existing => existing.FilePath != attachment.FilePath))
{
this.OwnerChat.PendingMediaTranscripts.Add(attachment);
ownerPendingChanged = true;
}
}
}
if (changed || ownerPendingChanged)
{
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
await this.OnChange(this.DocumentPaths);
}
if (delivery is not null)
this.MediaTranscriptionService.AcknowledgeDelivery(delivery);
}
/// <summary>Unsubscribes from the singleton media service.</summary>
protected override void DisposeResources()
{
this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged;
base.DisposeResources();
}
protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
{
if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
return;
switch (triggeredEvent)
@@ -168,29 +289,7 @@ public partial class AttachDocuments : MSGComponentBase
return;
}
// Ensure that Pandoc is installed and ready:
var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync(
showSuccessMessage: false,
showDialog: true);
// If Pandoc is not available (user cancelled installation), abort file drop:
if (!pandocState.IsAvailable)
{
this.Logger.LogWarning("The user cancelled the Pandoc installation or Pandoc is not available. Aborting file drop.");
this.isDraggingOver = false;
this.ClearDragClass();
this.StateHasChanged();
return;
}
foreach (var path in paths)
{
if(!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, path, this.ValidateMediaFileTypes, this.Provider))
continue;
this.DocumentPaths.Add(FileAttachment.FromPath(path));
}
await this.AddFileBatchAsync(paths);
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
await this.OnChange(this.DocumentPaths);
this.isDraggingOver = false;
@@ -208,54 +307,41 @@ public partial class AttachDocuments : MSGComponentBase
private async Task AddFilesManually()
{
if (this.Disabled)
if (this.IsUnavailable)
return;
// Ensure that Pandoc is installed and ready:
var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync(
showSuccessMessage: false,
showDialog: true);
// If Pandoc is not available (user cancelled installation), abort file selection:
if (!pandocState.IsAvailable)
{
this.Logger.LogWarning("The user cancelled the Pandoc installation or Pandoc is not available. Aborting file selection.");
return;
}
var selectFiles = await this.RustService.SelectFiles(T("Select files to attach"));
if (selectFiles.UserCancelled)
return;
foreach (var selectedFilePath in selectFiles.SelectedFilePaths)
{
if (!File.Exists(selectedFilePath))
continue;
if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, selectedFilePath, this.ValidateMediaFileTypes, this.Provider))
continue;
this.DocumentPaths.Add(FileAttachment.FromPath(selectedFilePath));
}
await this.AddFileBatchAsync(selectFiles.SelectedFilePaths);
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
await this.OnChange(this.DocumentPaths);
}
private async Task OpenAttachmentsDialog()
{
if (this.Disabled)
if (this.IsUnavailable)
return;
var previousAttachments = this.DocumentPaths.ToHashSet();
this.DocumentPaths = await ReviewAttachmentsDialog.OpenDialogAsync(this.DialogService, this.DocumentPaths);
foreach (var removedAttachment in previousAttachments.Except(this.DocumentPaths))
ManagedTranscriptAttachment.TryDeleteOwnedFile(removedAttachment);
this.ReconcileOwnerPendingTranscripts();
}
private async Task ClearAllFiles()
{
if (this.Disabled)
if (this.IsUnavailable)
return;
foreach (var attachment in this.DocumentPaths)
ManagedTranscriptAttachment.TryDeleteOwnedFile(attachment);
this.DocumentPaths.Clear();
this.ReconcileOwnerPendingTranscripts();
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
await this.OnChange(this.DocumentPaths);
}
@@ -266,7 +352,7 @@ public partial class AttachDocuments : MSGComponentBase
private void OnMouseEnter(EventArgs _)
{
if(this.Disabled || this.PauseCatchingDrops)
if(this.IsUnavailable || this.PauseCatchingDrops)
return;
this.Logger.LogDebug("Attach documents component '{Name}' is hovered.", this.Name);
@@ -277,7 +363,7 @@ public partial class AttachDocuments : MSGComponentBase
private void OnMouseLeave(EventArgs _)
{
if(this.Disabled || this.PauseCatchingDrops)
if(this.IsUnavailable || this.PauseCatchingDrops)
return;
this.Logger.LogDebug("Attach documents component '{Name}' is no longer hovered.", this.Name);
@@ -288,15 +374,98 @@ public partial class AttachDocuments : MSGComponentBase
private async Task RemoveDocument(FileAttachment fileAttachment)
{
if (this.Disabled)
if (this.IsUnavailable)
return;
this.DocumentPaths.Remove(fileAttachment);
ManagedTranscriptAttachment.TryDeleteOwnedFile(fileAttachment);
this.ReconcileOwnerPendingTranscripts();
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
await this.OnChange(this.DocumentPaths);
}
/// <summary>Keeps persisted chat-draft transcript references aligned with the composer.</summary>
private void ReconcileOwnerPendingTranscripts()
{
if (this.OwnerChat is null)
return;
var retainedPaths = this.DocumentPaths.Select(attachment => attachment.FilePath).ToHashSet(StringComparer.Ordinal);
this.OwnerChat.PendingMediaTranscripts.RemoveAll(attachment => !retainedPaths.Contains(attachment.FilePath));
}
private async Task AddFileBatchAsync(IEnumerable<string> paths)
{
var existingPaths = paths.Where(File.Exists).ToList();
var mediaPaths = existingPaths.Where(IsTranscribableMedia).ToList();
var regularPaths = existingPaths.Except(mediaPaths).ToList();
var canAddRegularFiles = true;
if (regularPaths.Count > 0)
{
var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync(
showSuccessMessage: false,
showDialog: true);
canAddRegularFiles = pandocState.IsAvailable;
}
foreach (var path in regularPaths)
{
if (!canAddRegularFiles)
break;
if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(
FileExtensionValidation.UseCase.ATTACHING_CONTENT,
path,
this.ValidateMediaFileTypes,
this.Provider))
continue;
this.DocumentPaths.Add(FileAttachment.FromPath(path));
}
if (mediaPaths.Count is 0)
return;
if (string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider))
{
await this.MessageBus.SendWarning(new(
Icons.Material.Filled.VoiceChat,
this.T("Media files require a configured transcription provider. Configure one in the transcription settings.")));
return;
}
var names = string.Join('\n', mediaPaths.Select(path => $"- {Markdown.EscapeInlineText(Path.GetFileName(path))}"));
var message = this.T("The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider.");
var dialogParameters = new DialogParameters<ConfirmDialog>
{
{
x => x.MarkdownBody,
$"""
{message}
{names}
"""
},
};
var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(
this.T("Transcribe media files"),
dialogParameters,
DialogOptions.FULLSCREEN);
var dialogResult = await dialogReference.Result;
if (dialogResult is null || dialogResult.Canceled)
return;
if (this.OwnerChat is null)
this.OwnerChat = await this.EnsureOwnerChatAsync(mediaPaths[0]);
this.MediaTranscriptionService.TryStartAttachmentBatch(mediaPaths, this.EffectiveMediaImportTarget, this.OwnerChat);
}
private static bool IsTranscribableMedia(string path) => FileTypes.IsAllowedPath(path, FileTypes.AUDIO) || FileTypes.IsAllowedPath(path, FileTypes.VIDEO);
/// <summary>
/// The user might want to check what we actually extract from his file and therefore give the LLM as an input.
/// </summary>
@@ -33,6 +33,7 @@
}
</ChildContent>
<FooterContent>
<MediaTranscriptionStatus Owner="@this.CurrentMediaImportOwner"/>
<MudElement Style="flex: 0 0 auto;">
<MudTextField
T="string"
@@ -45,7 +46,7 @@
Label="@this.InputLabel"
Placeholder="@this.ProviderPlaceholder"
Adornment="Adornment.End"
AdornmentIcon="@Icons.Material.Filled.Send"
AdornmentIcon="@(this.MediaTranscriptionService.IsBusy(this.CurrentMediaImportOwner) ? Icons.Material.Filled.HourglassTop : Icons.Material.Filled.Send)"
OnAdornmentClick="() => this.SendMessage()"
Disabled="@this.IsInputForbidden()"
Immediate="@true"
@@ -100,7 +101,7 @@
</MudTooltip>
}
<AttachDocuments Name="File Attachments" Layer="@DropLayers.PAGES" DocumentPaths="@this.ComposerState.FileAttachments" DocumentPathsChanged="@this.ComposerAttachmentsChanged" CatchAllDocuments="true" UseSmallForm="true" Provider="@this.Provider"/>
<AttachDocuments Name="File Attachments" Layer="@DropLayers.PAGES" DocumentPaths="@this.ComposerState.FileAttachments" DocumentPathsChanged="@this.ComposerAttachmentsChanged" CatchAllDocuments="true" UseSmallForm="true" ShowMediaStatus="false" Provider="@this.Provider" OwnerChat="@this.ChatThread" EnsureOwnerChatAsync="@this.EnsureMediaImportChatAsync" Disabled="@this.MediaTranscriptionService.IsBusy(this.CurrentMediaImportOwner)"/>
<MudDivider Vertical="true" Style="height: 24px; align-self: center;"/>
@@ -4,6 +4,8 @@ using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.AIJobs;
using AIStudio.Tools.Media;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
@@ -14,6 +16,7 @@ namespace AIStudio.Components;
public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
{
private readonly Guid draftMediaOwnerId = Guid.NewGuid();
private const string CHAT_INPUT_ID = "chat-user-input";
private const string MARKDOWN_CODE = "code";
private const string MARKDOWN_BOLD = "bold";
@@ -54,6 +57,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
[Inject]
private AIJobService AIJobService { get; init; } = null!;
[Inject]
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top;
private static readonly Dictionary<string, object?> USER_INPUT_ATTRIBUTES = new();
@@ -81,6 +87,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
private Guid foregroundChatId = Guid.Empty;
private int workspaceHeaderSyncVersion;
private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForChat(this.ChatThread?.ChatId ?? this.draftMediaOwnerId);
// Unfortunately, we need the input field reference to blur the focus away. Without
// this, we cannot clear the input field.
private MudTextField<string> inputField = null!;
@@ -104,6 +112,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
protected override async Task OnInitializedAsync()
{
this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged;
// Apply the filters for the message bus:
this.ApplyFilters([], [ Event.HAS_CHAT_UNSAVED_CHANGES, Event.RESET_CHAT_STATE, Event.CHAT_STREAMING_DONE, Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED, Event.CHAT_GENERATION_CHANGED, Event.WORKSPACE_RENAMED, Event.CONFIGURATION_CHANGED ]);
@@ -243,9 +253,50 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
// Select the correct provider:
await this.SelectProviderWhenLoadingChat();
await this.SyncForegroundChatAsync();
await this.ConsumeMediaOutcomeAsync();
await base.OnInitializedAsync();
}
/// <summary>Refreshes send and attachment controls when the media import lane changes.</summary>
private void OnMediaImportStateChanged(MediaImportOwner owner)
{
if (owner == this.CurrentMediaImportOwner)
_ = this.InvokeAsync(async () =>
{
await this.ConsumeMediaOutcomeAsync();
this.StateHasChanged();
});
}
/// <summary>Consumes a terminal media notification when its chat is visible.</summary>
private async Task ConsumeMediaOutcomeAsync()
{
var outcome = this.MediaTranscriptionService.TryConsumeOutcome(this.CurrentMediaImportOwner);
if (outcome is null)
return;
if (outcome.Failures.Count > 0)
{
var message = string.Join(Environment.NewLine, outcome.Failures.Select(failure => $"{failure.FileName}: {failure.UserMessage}"));
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, message));
}
else if (outcome.Status is MediaImportStatus.FAILED)
{
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed.")));
}
if (outcome.Warnings.Count > 0)
{
var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}"));
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message));
}
if (outcome.Status is MediaImportStatus.CANCELLED)
{
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled.")));
}
}
protected override async Task OnAfterRenderAsync(bool firstRender)
{
if (firstRender && this.ChatThread is not null && this.mustStoreChat)
@@ -314,6 +365,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
await this.ApplyLoadedChatParameterAsync();
await this.SyncForegroundChatAsync();
await this.ConsumeMediaOutcomeAsync();
await base.OnParametersSetAsync();
}
@@ -680,9 +732,43 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
this.ComposerState.MarkUserDraft();
this.hasUnsavedChanges = true;
}
/// <summary>Creates and stores a stable draft immediately after media import confirmation.</summary>
private async Task<ChatThread?> EnsureMediaImportChatAsync(string firstMediaPath)
{
if (this.ChatThread is not null)
return this.ChatThread;
this.RefreshCurrentProfileAndChatTemplate();
var promptName = this.ExtractThreadName(this.ComposerState.UserInput);
this.ChatThread = new()
{
IncludeDateTime = true,
SelectedProvider = this.Provider.Id,
SelectedProfile = this.currentProfile.Id,
SelectedChatTemplate = this.currentChatTemplate.Id,
SystemPrompt = SystemPrompts.DEFAULT,
WorkspaceId = this.currentWorkspaceId,
ChatId = Guid.NewGuid(),
DataSourceOptions = this.earlyDataSourceOptions,
Name = string.IsNullOrWhiteSpace(this.ComposerState.UserInput)
? $"Transkription: {Path.GetFileName(firstMediaPath)}"
: promptName,
Blocks = this.currentChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE ? [] : this.currentChatTemplate.ExampleConversation.Select(block => block.DeepClone()).ToList(),
};
await WorkspaceBehaviour.StoreChatAsync(this.ChatThread);
this.MarkCurrentChatAsLoadedParameter();
await this.ChatThreadChanged.InvokeAsync(this.ChatThread);
await this.SyncForegroundChatAsync();
return this.ChatThread;
}
private async Task SendMessage(bool reuseLastUserPrompt = false)
{
if (this.MediaTranscriptionService.IsBusy(this.CurrentMediaImportOwner))
return;
if (!this.IsProviderSelected)
return;
@@ -745,6 +831,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
Text = this.ComposerState.UserInput,
FileAttachments = normalizedAttachments,
};
this.ChatThread.PendingMediaTranscripts.Clear();
//
// Add the user message to the thread:
@@ -986,12 +1073,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
if (workspaceId == Guid.Empty)
return;
// Delete the chat from the current workspace or the temporary storage:
await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, this.ChatThread!.WorkspaceId, this.ChatThread.ChatId, askForConfirmation: false);
this.ChatThread!.WorkspaceId = workspaceId;
await WorkspaceBehaviour.MoveChatAsync(this.ChatThread!, workspaceId);
this.MarkCurrentChatAsLoadedParameter();
await this.SaveThread();
await this.SyncWorkspaceHeaderWithChatThreadAsync();
}
@@ -1209,6 +1292,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
public async ValueTask DisposeAsync()
{
this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged;
if(this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is WorkspaceStorageBehavior.STORE_CHATS_AUTOMATICALLY)
{
await this.SaveThread();
@@ -0,0 +1,34 @@
@inherits MSGComponentBase
@inject MediaTranscriptionService MediaTranscriptionService
@using AIStudio.Tools.Services
@if (this.Snapshot is { IsBusy: true } snapshot)
{
@if (this.Compact)
{
<MudStack Row="true" AlignItems="AlignItems.Center" Class="pa-1">
<MudProgressCircular Size="Size.Small" Indeterminate="@(snapshot.Progress is null)" Value="@((snapshot.Progress ?? 0) * 100)"/>
<MudText Typo="Typo.body2">
@this.StatusText
</MudText>
<MudTooltip Text="@T("Stop media transcription")">
<MudIconButton Size="Size.Small" Icon="@Icons.Material.Filled.Stop" Color="Color.Error" OnClick="@(async () => await this.MediaTranscriptionService.StopAsync(this.Owner))"/>
</MudTooltip>
</MudStack>
}
else
{
<MudPaper Outlined="true" Class="pa-2 mb-2">
<MudStack Row="true" AlignItems="AlignItems.Center">
<MudProgressCircular Size="Size.Small" Indeterminate="@(snapshot.Progress is null)" Value="@((snapshot.Progress ?? 0) * 100)"/>
<MudText Typo="Typo.body2">
@this.StatusText
</MudText>
<MudSpacer/>
<MudTooltip Text="@T("Stop media transcription")">
<MudIconButton Icon="@Icons.Material.Filled.Stop" Color="Color.Error" OnClick="@(async () => await this.MediaTranscriptionService.StopAsync(this.Owner))"/>
</MudTooltip>
</MudStack>
</MudPaper>
}
}
@@ -0,0 +1,73 @@
using AIStudio.Tools.Media;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Components;
public partial class MediaTranscriptionStatus
{
/// <summary>The surface owner whose operation is rendered.</summary>
[Parameter]
public MediaImportOwner Owner { get; set; }
/// <summary>Optional target filter used by embedded file controls.</summary>
[Parameter]
public string TargetId { get; set; } = string.Empty;
/// <summary>Renders the status without an enclosing paper surface.</summary>
[Parameter]
public bool Compact { get; set; }
private MediaImportSnapshot? Snapshot
{
get
{
var snapshot = this.MediaTranscriptionService.GetSnapshot(this.Owner);
return string.IsNullOrWhiteSpace(this.TargetId) || snapshot?.Target.TargetId == this.TargetId
? snapshot
: null;
}
}
/// <summary>Gets the localized visible status for the active import.</summary>
private string StatusText
{
get
{
var snapshot = this.Snapshot;
if (snapshot is null)
return string.Empty;
return snapshot.Phase switch
{
MediaTranscriptionPhase.QUEUED => $"{this.T("Waiting to prepare media")}: {snapshot.CurrentFileName}",
MediaTranscriptionPhase.PROBING => $"{this.T("Inspecting media")}: {snapshot.CurrentFileName}",
MediaTranscriptionPhase.TRANSCODING => $"{this.T("Preparing audio")}: {snapshot.CurrentFileName}",
MediaTranscriptionPhase.UPLOADING => $"{this.T("Transcribing")}: {snapshot.CurrentFileName}",
MediaTranscriptionPhase.CANCELING => $"{this.T("Stopping media transcription")}: {snapshot.CurrentFileName}",
_ => snapshot.CurrentFileName,
};
}
}
/// <summary>Subscribes to singleton import state changes.</summary>
protected override async Task OnInitializedAsync()
{
this.MediaTranscriptionService.StateChanged += this.OnStateChanged;
await base.OnInitializedAsync();
}
/// <summary>Schedules a render after an import state transition.</summary>
private void OnStateChanged(MediaImportOwner owner)
{
if (owner == this.Owner)
_ = this.InvokeAsync(this.StateHasChanged);
}
/// <summary>Unsubscribes from singleton import state changes.</summary>
protected override void DisposeResources()
{
this.MediaTranscriptionService.StateChanged -= this.OnStateChanged;
base.DisposeResources();
}
}
@@ -5,19 +5,29 @@
<div @onmouseenter="@this.OnMouseEnter" @onmouseleave="@this.OnMouseLeave">
<MudPaper Outlined="true" Class="@this.dragClass">
<MudStack Row="true" AlignItems="AlignItems.Center" StretchItems="StretchItems.None" Wrap="Wrap.Wrap">
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Disabled="@this.Disabled">
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Disabled="@this.IsUnavailable">
@this.ButtonText
</MudButton>
<MudText Typo="Typo.body2">
@T("Drop one file here to load its content.")
</MudText>
@if (this.IsCurrentTargetBusy)
{
<MediaTranscriptionStatus Owner="@this.EffectiveImportOwner" TargetId="@this.EffectiveMediaImportTarget.TargetId" Compact="true"/>
}
else
{
<MudText Typo="Typo.body2">
@T("Drop one file here to load its content.")
</MudText>
}
</MudStack>
</MudPaper>
</div>
}
else
{
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Class="mb-3" Disabled="@this.Disabled">
@this.ButtonText
</MudButton>
<MudStack Row="true" AlignItems="AlignItems.Center" StretchItems="StretchItems.None" Wrap="Wrap.Wrap" Class="mb-3">
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Disabled="@this.IsUnavailable">
@this.ButtonText
</MudButton>
<MediaTranscriptionStatus Owner="@this.EffectiveImportOwner" TargetId="@this.EffectiveMediaImportTarget.TargetId" Compact="true"/>
</MudStack>
}
@@ -1,3 +1,5 @@
using AIStudio.Dialogs;
using AIStudio.Tools.Media;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
@@ -8,6 +10,22 @@ namespace AIStudio.Components;
public partial class ReadFileContent : MSGComponentBase
{
private readonly MediaImportOwner fallbackMediaImportOwner = new(MediaImportOwnerKind.ASSISTANT, $"read-file-content:{Guid.NewGuid():N}");
[CascadingParameter]
private MediaImportOwner? ImportOwner { get; set; }
private MediaImportOwner EffectiveImportOwner => this.ImportOwner ?? this.fallbackMediaImportOwner;
[Parameter]
public string MediaImportTargetId { get; set; } = string.Empty;
private string EffectiveMediaImportTargetId => string.IsNullOrWhiteSpace(this.MediaImportTargetId)
? string.IsNullOrWhiteSpace(this.Text) ? "primary" : this.Text
: this.MediaImportTargetId;
private MediaImportTarget EffectiveMediaImportTarget => new(this.EffectiveImportOwner, this.EffectiveMediaImportTargetId);
[Parameter]
public string Text { get; set; } = string.Empty;
@@ -47,17 +65,24 @@ public partial class ReadFileContent : MSGComponentBase
[Inject]
private PandocAvailabilityService PandocAvailabilityService { get; init; } = null!;
[Inject]
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
private const string DEFAULT_DRAG_CLASS = "relative rounded-lg border-2 border-dashed pa-3 mb-3 mud-width-full";
private string ButtonText => string.IsNullOrWhiteSpace(this.Text) ? T("Use file content as input") : this.Text;
private string dragClass = DEFAULT_DRAG_CLASS;
private uint numDropAreasAboveThis;
private bool isComponentHovered;
private bool IsCurrentTargetBusy => this.MediaTranscriptionService.GetSnapshot(this.EffectiveImportOwner) is { IsBusy: true } snapshot
&& snapshot.Target == this.EffectiveMediaImportTarget;
private bool IsUnavailable => this.Disabled || this.MediaTranscriptionService.IsBusy(this.EffectiveImportOwner);
#region Overrides of MSGComponentBase
protected override async Task OnInitializedAsync()
{
this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged;
if (this.EnableDragDrop)
{
this.ApplyFilters([], [ Event.TAURI_EVENT_RECEIVED, Event.REGISTER_FILE_DROP_AREA, Event.UNREGISTER_FILE_DROP_AREA ]);
@@ -65,6 +90,69 @@ public partial class ReadFileContent : MSGComponentBase
}
await base.OnInitializedAsync();
await this.SyncCompletedMediaTextAsync();
}
/// <summary>Refreshes disabled controls when the shared import lane changes.</summary>
private void OnMediaImportStateChanged(MediaImportOwner owner)
{
if (owner == this.EffectiveImportOwner)
_ = this.InvokeAsync(async () =>
{
await this.SyncCompletedMediaTextAsync();
await this.ConsumeStandaloneMediaOutcomeAsync();
this.StateHasChanged();
});
}
/// <summary>Consumes outcomes for dialog-local controls that have no assistant owner surface.</summary>
private async Task ConsumeStandaloneMediaOutcomeAsync()
{
if (this.ImportOwner is not null)
return;
var outcome = this.MediaTranscriptionService.TryConsumeOutcome(this.EffectiveImportOwner);
if (outcome is null)
return;
if (outcome.Failures.Count > 0)
{
var message = string.Join(Environment.NewLine, outcome.Failures.Select(failure => $"{failure.FileName}: {failure.UserMessage}"));
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, message));
}
else if (outcome.Status is MediaImportStatus.FAILED)
{
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed.")));
}
if (outcome.Warnings.Count > 0)
{
var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}"));
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message));
}
if (outcome.Status is MediaImportStatus.CANCELLED)
{
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled.")));
}
}
/// <summary>Applies a completed target transcript after progress or navigation.</summary>
private async Task SyncCompletedMediaTextAsync()
{
var delivery = this.MediaTranscriptionService.GetPendingDelivery(this.EffectiveMediaImportTarget);
if (delivery is null || delivery.Text is not { } text)
return;
await this.FileContentChanged.InvokeAsync(text);
this.MediaTranscriptionService.AcknowledgeDelivery(delivery);
}
/// <summary>Unsubscribes from the singleton media service.</summary>
protected override void DisposeResources()
{
this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged;
base.DisposeResources();
}
protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
@@ -72,7 +160,7 @@ public partial class ReadFileContent : MSGComponentBase
if (!this.EnableDragDrop)
return;
if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
return;
switch (triggeredEvent)
@@ -126,10 +214,7 @@ public partial class ReadFileContent : MSGComponentBase
private async Task SelectFile()
{
if (this.Disabled)
return;
if (!await this.EnsurePandocAvailability())
if (this.IsUnavailable)
return;
var selectedFile = await this.RustService.SelectFile(T("Select file to read its content"));
@@ -161,9 +246,6 @@ public partial class ReadFileContent : MSGComponentBase
private async Task LoadFirstValidFile(List<string> paths)
{
if (!await this.EnsurePandocAvailability())
return;
foreach (var path in paths)
{
if (await this.LoadFileIfValid(path))
@@ -179,6 +261,12 @@ public partial class ReadFileContent : MSGComponentBase
return false;
}
if (FileTypes.IsAllowedPath(filePath, FileTypes.AUDIO) || FileTypes.IsAllowedPath(filePath, FileTypes.VIDEO))
return await this.LoadMediaTranscriptAsync(filePath);
if (!await this.EnsurePandocAvailability())
return false;
if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.DIRECTLY_LOADING_CONTENT, filePath))
{
this.Logger.LogWarning("User attempted to load unsupported file: {FilePath}", filePath);
@@ -200,6 +288,42 @@ public partial class ReadFileContent : MSGComponentBase
}
}
private async Task<bool> LoadMediaTranscriptAsync(string filePath)
{
if (string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider))
{
await this.MessageBus.SendWarning(new(
Icons.Material.Filled.VoiceChat,
this.T("Media files require a configured transcription provider. Configure one in the transcription settings.")));
return false;
}
var message = this.T("The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider.");
var dialogParameters = new DialogParameters<ConfirmDialog>
{
{
x => x.MarkdownBody,
$"""
{message}
- {Markdown.EscapeInlineText(Path.GetFileName(filePath))}
"""
},
};
var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(
this.T("Transcribe media file"),
dialogParameters,
Dialogs.DialogOptions.FULLSCREEN);
var dialogResult = await dialogReference.Result;
if (dialogResult is null || dialogResult.Canceled)
return false;
return this.MediaTranscriptionService.TryStartTextImport(
filePath,
this.EffectiveMediaImportTarget);
}
private bool CanCatchDroppedFile() => this.numDropAreasAboveThis is 0 && (this.isComponentHovered || this.CatchAllDocuments);
private void SetDragClass() => this.dragClass = $"{DEFAULT_DRAG_CLASS} mud-border-primary border-2";
@@ -208,7 +332,7 @@ public partial class ReadFileContent : MSGComponentBase
private void OnMouseEnter(EventArgs _)
{
if(this.Disabled || this.numDropAreasAboveThis > 0)
if(this.IsUnavailable || this.numDropAreasAboveThis > 0)
return;
this.Logger.LogDebug("Read file content component is hovered.");
@@ -219,7 +343,7 @@ public partial class ReadFileContent : MSGComponentBase
private void OnMouseLeave(EventArgs _)
{
if(this.Disabled)
if(this.IsUnavailable)
return;
this.Logger.LogDebug("Read file content component is no longer hovered.");
@@ -1,6 +1,7 @@
using AIStudio.Provider;
using System.Buffers.Binary;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.MIME;
using AIStudio.Tools.Media;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
@@ -10,6 +11,8 @@ namespace AIStudio.Components;
public partial class VoiceRecorder : MSGComponentBase
{
private const int PCM_WAV_HEADER_SIZE = 44;
[Inject]
private ILogger<VoiceRecorder> Logger { get; init; } = null!;
@@ -25,6 +28,9 @@ public partial class VoiceRecorder : MSGComponentBase
[Inject]
private VoiceRecordingAvailabilityService VoiceRecordingAvailabilityService { get; init; } = null!;
[Inject]
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
#region Overrides of MSGComponentBase
protected override async Task OnInitializedAsync()
@@ -93,7 +99,6 @@ public partial class VoiceRecorder : MSGComponentBase
private bool isTranscribing;
private FileStream? currentRecordingStream;
private string? currentRecordingPath;
private string? currentRecordingMimeType;
private string? finalRecordingPath;
private DotNetObjectReference<VoiceRecorder>? dotNetReference;
@@ -131,17 +136,7 @@ public partial class VoiceRecorder : MSGComponentBase
return;
}
var mimeTypes = GetPreferredMimeTypes(
Builder.Create().UseAudio().UseSubtype(AudioSubtype.WEBM).Build(),
Builder.Create().UseAudio().UseSubtype(AudioSubtype.OGG).Build(),
Builder.Create().UseAudio().UseSubtype(AudioSubtype.AAC).Build(),
Builder.Create().UseAudio().UseSubtype(AudioSubtype.MP3).Build(),
Builder.Create().UseAudio().UseSubtype(AudioSubtype.AIFF).Build(),
Builder.Create().UseAudio().UseSubtype(AudioSubtype.WAV).Build(),
Builder.Create().UseAudio().UseSubtype(AudioSubtype.FLAC).Build()
);
this.Logger.LogInformation("Starting audio recording with preferred MIME types: '{PreferredMimeTypes}'.", string.Join<MIMEType>(", ", mimeTypes));
this.Logger.LogInformation("Starting PCM/WAV audio recording.");
// Create a DotNetObjectReference to pass to JavaScript:
this.dotNetReference = DotNetObjectReference.Create(this);
@@ -151,13 +146,8 @@ public partial class VoiceRecorder : MSGComponentBase
try
{
var mimeTypeStrings = mimeTypes.ToStringArray();
var actualMimeType = await this.JsRuntime.InvokeAsync<string>("audioRecorder.start", this.dotNetReference, mimeTypeStrings);
// Store the MIME type for later use:
this.currentRecordingMimeType = actualMimeType;
this.Logger.LogInformation("Audio recording started with MIME type: '{ActualMimeType}'.", actualMimeType);
await this.JsRuntime.InvokeVoidAsync("audioRecorder.start", this.dotNetReference);
this.Logger.LogInformation("PCM/WAV audio recording started.");
this.isPreparing = false;
this.isRecording = true;
}
@@ -168,6 +158,7 @@ public partial class VoiceRecorder : MSGComponentBase
// Clean up the recording stream if starting failed:
await this.FinalizeRecordingStream();
await this.ReleaseMicrophoneAsync();
}
finally
{
@@ -176,11 +167,11 @@ public partial class VoiceRecorder : MSGComponentBase
}
else
{
var recordingStoppedSuccessfully = false;
try
{
var result = await this.JsRuntime.InvokeAsync<AudioRecordingResult>("audioRecorder.stop");
if (result.ChangedMimeType)
this.Logger.LogWarning("The recorded audio MIME type was changed to '{ResultMimeType}'.", result.MimeType);
await this.JsRuntime.InvokeVoidAsync("audioRecorder.stop");
recordingStoppedSuccessfully = true;
}
catch (Exception e)
{
@@ -194,28 +185,21 @@ public partial class VoiceRecorder : MSGComponentBase
this.isRecording = false;
this.StateHasChanged();
// Start transcription if we have a recording and a configured provider:
if (this.finalRecordingPath is not null)
await this.TranscribeRecordingAsync();
}
}
if (!recordingStoppedSuccessfully || this.finalRecordingPath is null)
{
if (recordingStoppedSuccessfully)
{
this.Logger.LogWarning("The audio recorder did not produce any data.");
await this.MessageBus.SendError(new(Icons.Material.Filled.MicOff, this.T("Failed to stop audio recording.")));
}
private static MIMEType[] GetPreferredMimeTypes(params MIMEType[] mimeTypes)
{
// Default list if no parameters provided:
if (mimeTypes.Length is 0)
{
var audioBuilder = Builder.Create().UseAudio();
return
[
audioBuilder.UseSubtype(AudioSubtype.WEBM).Build(),
audioBuilder.UseSubtype(AudioSubtype.OGG).Build(),
audioBuilder.UseSubtype(AudioSubtype.MP4).Build(),
audioBuilder.UseSubtype(AudioSubtype.MPEG).Build(),
];
}
this.DeleteFinalRecording();
await this.ReleaseMicrophoneAsync();
return;
}
return mimeTypes;
await this.TranscribeRecordingAsync();
}
}
private async Task InitializeRecordingStream()
@@ -226,7 +210,7 @@ public partial class VoiceRecorder : MSGComponentBase
if (!Directory.Exists(recordingDirectory))
Directory.CreateDirectory(recordingDirectory);
var fileName = $"recording_{DateTime.UtcNow:yyyyMMdd_HHmmss}.audio";
var fileName = $"recording_{DateTime.UtcNow:yyyyMMdd_HHmmss}.wav";
this.currentRecordingPath = Path.Combine(recordingDirectory, fileName);
this.currentRecordingStream = new FileStream(this.currentRecordingPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 8192, useAsync: true);
@@ -253,6 +237,7 @@ public partial class VoiceRecorder : MSGComponentBase
catch (Exception ex)
{
this.Logger.LogError(ex, "Error writing audio chunk to stream.");
throw;
}
}
@@ -262,45 +247,56 @@ public partial class VoiceRecorder : MSGComponentBase
if (this.currentRecordingStream is not null)
{
await this.currentRecordingStream.FlushAsync();
var hasPcmAudioData = await this.FinalizePcmWavHeaderAsync(this.currentRecordingStream);
await this.currentRecordingStream.DisposeAsync();
this.currentRecordingStream = null;
// Rename the file with the correct extension based on MIME type:
if (this.currentRecordingPath is not null && this.currentRecordingMimeType is not null)
if (this.currentRecordingPath is not null && File.Exists(this.currentRecordingPath))
{
var extension = GetFileExtension(this.currentRecordingMimeType);
var newPath = Path.ChangeExtension(this.currentRecordingPath, extension);
var fileSize = new FileInfo(this.currentRecordingPath).Length;
if (File.Exists(this.currentRecordingPath))
if (hasPcmAudioData)
{
File.Move(this.currentRecordingPath, newPath, overwrite: true);
this.finalRecordingPath = newPath;
this.Logger.LogInformation("Finalized audio recording over {NumChunks} streamed audio chunks to the file '{RecordingPath}'.", this.numReceivedChunks, newPath);
this.finalRecordingPath = this.currentRecordingPath;
this.Logger.LogInformation("Finalized audio recording over {NumChunks} streamed audio chunks to the file '{RecordingPath}' with {FileSize} bytes.", this.numReceivedChunks, this.currentRecordingPath, fileSize);
}
else
{
this.Logger.LogWarning("Discarding a PCM/WAV audio recording without audio data ({FileSize} bytes).", fileSize);
File.Delete(this.currentRecordingPath);
}
}
}
this.currentRecordingPath = null;
this.currentRecordingMimeType = null;
// Dispose the .NET reference:
this.dotNetReference?.Dispose();
this.dotNetReference = null;
}
private static string GetFileExtension(string mimeType)
private async Task<bool> FinalizePcmWavHeaderAsync(FileStream recordingStream)
{
var baseMimeType = mimeType.Split(';')[0].Trim().ToLowerInvariant();
return baseMimeType switch
{
"audio/webm" => ".webm",
"audio/ogg" => ".ogg",
"audio/mp4" => ".m4a",
"audio/mpeg" => ".mp3",
"audio/wav" => ".wav",
"audio/x-wav" => ".wav",
_ => ".audio" // Fallback
};
if (recordingStream.Length <= PCM_WAV_HEADER_SIZE)
return false;
var pcmDataSize = recordingStream.Length - PCM_WAV_HEADER_SIZE;
if (pcmDataSize > uint.MaxValue - 36)
throw new InvalidDataException("The streamed PCM recording exceeds the WAV size limit.");
var valueBuffer = new byte[sizeof(uint)];
BinaryPrimitives.WriteUInt32LittleEndian(valueBuffer, checked((uint)(36 + pcmDataSize)));
recordingStream.Seek(4, SeekOrigin.Begin);
await recordingStream.WriteAsync(valueBuffer);
BinaryPrimitives.WriteUInt32LittleEndian(valueBuffer, checked((uint)pcmDataSize));
recordingStream.Seek(40, SeekOrigin.Begin);
await recordingStream.WriteAsync(valueBuffer);
recordingStream.Seek(0, SeekOrigin.End);
await recordingStream.FlushAsync();
this.Logger.LogInformation("Finalized a streamed PCM/WAV header for {PcmDataSize} bytes of audio data.", pcmDataSize);
return true;
}
private async Task TranscribeRecordingAsync()
@@ -317,58 +313,22 @@ public partial class VoiceRecorder : MSGComponentBase
try
{
// Get the configured transcription provider ID:
var transcriptionProviderId = this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider;
if (string.IsNullOrWhiteSpace(transcriptionProviderId))
var transcriptionResult = await this.MediaTranscriptionService.TranscribeVoiceAsync(this.finalRecordingPath);
if (transcriptionResult.Status is not MediaTranscriptionResultStatus.SUCCEEDED)
{
this.Logger.LogWarning("No transcription provider is configured.");
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("No transcription provider is configured.")));
return;
}
if (transcriptionResult.Status is MediaTranscriptionResultStatus.CANCELLED)
return;
// Find the transcription provider in the list of configured providers:
var transcriptionProviderSettings = this.SettingsManager.ConfigurationData.TranscriptionProviders
.FirstOrDefault(x => x.Id == transcriptionProviderId);
if (transcriptionResult.Status is MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL)
{
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, transcriptionResult.UserMessage));
return;
}
if (transcriptionProviderSettings is null)
{
this.Logger.LogWarning("The configured transcription provider with ID '{ProviderId}' was not found.", transcriptionProviderId);
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The configured transcription provider was not found.")));
return;
}
// Check the confidence level:
var minimumLevel = this.SettingsManager.GetMinimumConfidenceLevel(Tools.Components.NONE);
var providerConfidence = transcriptionProviderSettings.UsedLLMProvider.GetConfidence(this.SettingsManager);
if (providerConfidence.Level < minimumLevel)
{
this.Logger.LogWarning(
"The configured transcription provider '{ProviderName}' has a confidence level of '{ProviderLevel}', which is below the minimum required level of '{MinimumLevel}'.",
transcriptionProviderSettings.UsedLLMProvider,
providerConfidence.Level,
minimumLevel);
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The configured transcription provider does not meet the minimum confidence level.")));
return;
}
// Create the provider instance:
var provider = transcriptionProviderSettings.CreateProvider();
if (provider.Provider is LLMProviders.NONE)
{
this.Logger.LogError("Failed to create the transcription provider instance.");
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("Failed to create the transcription provider.")));
return;
}
// Call the transcription API:
this.Logger.LogInformation("Starting transcription with provider '{ProviderName}' and model '{ModelName}'.", transcriptionProviderSettings.UsedLLMProvider, transcriptionProviderSettings.Model.ToString());
var transcriptionResult = await provider.TranscribeAudioAsync(transcriptionProviderSettings.Model, this.finalRecordingPath, this.SettingsManager);
if (!transcriptionResult.Success)
{
this.Logger.LogWarning("The transcription request failed.");
var userMessage = string.IsNullOrWhiteSpace(transcriptionResult.ErrorMessage)
var userMessage = string.IsNullOrWhiteSpace(transcriptionResult.UserMessage)
? this.T("Unfortunately, there was an error communicating with the AI system.")
: transcriptionResult.ErrorMessage;
: transcriptionResult.UserMessage;
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, userMessage));
return;
}
@@ -406,19 +366,6 @@ public partial class VoiceRecorder : MSGComponentBase
// Copy the transcribed text to the clipboard:
await this.RustService.CopyText2Clipboard(this.Snackbar, transcribedText);
// Delete the recording file:
try
{
if (File.Exists(this.finalRecordingPath))
{
File.Delete(this.finalRecordingPath);
this.Logger.LogInformation("Deleted the recording file '{RecordingPath}'.", this.finalRecordingPath);
}
}
catch (Exception ex)
{
this.Logger.LogError(ex, "Failed to delete the recording file '{RecordingPath}'.", this.finalRecordingPath);
}
}
catch (Exception ex)
{
@@ -428,13 +375,31 @@ public partial class VoiceRecorder : MSGComponentBase
finally
{
await this.ReleaseMicrophoneAsync();
this.finalRecordingPath = null;
this.DeleteFinalRecording();
this.isTranscribing = false;
this.StateHasChanged();
}
}
private void DeleteFinalRecording()
{
var recordingPath = this.finalRecordingPath;
this.finalRecordingPath = null;
if (recordingPath is null)
return;
try
{
if (File.Exists(recordingPath))
File.Delete(recordingPath);
}
catch (Exception ex)
{
this.Logger.LogError(ex, "Failed to delete the recording file '{RecordingPath}'.", recordingPath);
}
}
private async Task ReleaseMicrophoneAsync()
{
// Wait a moment for any queued sounds to finish playing, then release the microphone.
@@ -530,4 +495,4 @@ public partial class VoiceRecorder : MSGComponentBase
}
#endregion
}
}
@@ -4,6 +4,8 @@ using System.Text.Json;
using AIStudio.Chat;
using AIStudio.Dialogs;
using AIStudio.Tools.AIJobs;
using AIStudio.Tools.Media;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
@@ -21,6 +23,9 @@ public partial class Workspaces : MSGComponentBase
[Inject]
private AIJobService AIJobService { get; init; } = null!;
[Inject]
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
[Parameter]
public ChatThread? CurrentChatThread { get; set; }
@@ -55,6 +60,7 @@ public partial class Workspaces : MSGComponentBase
protected override async Task OnInitializedAsync()
{
this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged;
await base.OnInitializedAsync();
this.ApplyFilters([], [ Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED, Event.CHAT_GENERATION_CHANGED, Event.WORKSPACE_CREATED ]);
_ = this.LoadTreeItemsAsync(startPrefetch: true);
@@ -376,12 +382,26 @@ public partial class Workspaces : MSGComponentBase
private bool IsChatTreeItemBusy(TreeItemData treeItem)
{
return treeItem.Type is TreeItemType.CHAT && this.AIJobService.IsChatGenerationActive(treeItem.ChatId);
return treeItem.Type is TreeItemType.CHAT
&& (this.AIJobService.IsChatGenerationActive(treeItem.ChatId)
|| this.MediaTranscriptionService.IsBusy(MediaImportOwner.ForChat(treeItem.ChatId)));
}
private string GetChatTreeItemTextStyle(TreeItemData treeItem)
{
return this.IsCurrentChatTreeItem(treeItem) ? "justify-self: start; font-weight: 700;" : "justify-self: start;";
var status = this.MediaTranscriptionService.GetSnapshot(MediaImportOwner.ForChat(treeItem.ChatId))?.Status;
var color = status switch
{
MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => " color: var(--mud-palette-info);",
MediaImportStatus.SUCCEEDED => " color: var(--mud-palette-success);",
MediaImportStatus.WARNING => " color: var(--mud-palette-warning);",
MediaImportStatus.FAILED => " color: var(--mud-palette-error);",
MediaImportStatus.CANCELLED => " color: var(--mud-palette-warning);",
_ => string.Empty,
};
var weight = this.IsCurrentChatTreeItem(treeItem) ? " font-weight: 700;" : string.Empty;
return $"justify-self: start;{weight}{color}";
}
private bool IsCurrentChatTreeItem(TreeItemData treeItem)
@@ -394,6 +414,22 @@ public partial class Workspaces : MSGComponentBase
private string GetChatTreeIcon(Guid chatId, string defaultIcon)
{
var mediaStatus = this.MediaTranscriptionService.GetSnapshot(MediaImportOwner.ForChat(chatId))?.Status;
if (mediaStatus is not null)
{
return mediaStatus switch
{
MediaImportStatus.QUEUED => Icons.Material.Filled.HourglassTop,
MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => Icons.Material.Filled.ChangeCircle,
MediaImportStatus.SUCCEEDED => Icons.Material.Filled.TaskAlt,
MediaImportStatus.WARNING => Icons.Material.Filled.WarningAmber,
MediaImportStatus.FAILED => Icons.Material.Filled.Error,
MediaImportStatus.CANCELLED => Icons.Material.Filled.Cancel,
_ => defaultIcon,
};
}
var snapshot = this.AIJobService.TryGetChatSnapshot(chatId);
if (snapshot is null || !snapshot.IsActive)
return defaultIcon;
@@ -406,6 +442,12 @@ public partial class Workspaces : MSGComponentBase
};
}
private void OnMediaImportStateChanged(MediaImportOwner owner)
{
if (owner.Kind is MediaImportOwnerKind.CHAT)
_ = this.SafeStateHasChanged();
}
private async Task SafeStateHasChanged()
{
if (this.isDisposed)
@@ -668,7 +710,8 @@ public partial class Workspaces : MSGComponentBase
if (chat is null)
return;
if (this.AIJobService.IsChatGenerationActive(chat.ChatId))
var mediaOwner = MediaImportOwner.ForChat(chat.ChatId);
if (this.AIJobService.IsChatGenerationActive(chat.ChatId) || this.MediaTranscriptionService.IsBusy(mediaOwner))
return;
if (askForConfirmation)
@@ -692,6 +735,7 @@ public partial class Workspaces : MSGComponentBase
}
await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, chat.WorkspaceId, chat.ChatId, askForConfirmation: false);
this.MediaTranscriptionService.ClearOwnerState(mediaOwner);
await this.LoadTreeItemsAsync(startPrefetch: false);
if (unloadChat && this.CurrentChatThread?.ChatId == chat.ChatId)
@@ -845,16 +889,13 @@ public partial class Workspaces : MSGComponentBase
if (workspaceId == Guid.Empty)
return;
await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, chat.WorkspaceId, chat.ChatId, askForConfirmation: false);
chat.WorkspaceId = workspaceId;
await WorkspaceBehaviour.MoveChatAsync(chat, workspaceId);
if (this.CurrentChatThread?.ChatId == chat.ChatId)
{
this.CurrentChatThread = chat;
await this.CurrentChatThreadChanged.InvokeAsync(this.CurrentChatThread);
}
await WorkspaceBehaviour.StoreChatAsync(chat);
await this.LoadTreeItemsAsync(startPrefetch: false);
}
@@ -914,6 +955,7 @@ public partial class Workspaces : MSGComponentBase
protected override void DisposeResources()
{
this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged;
this.isDisposed = true;
this.prefetchCancellationTokenSource?.Cancel();
this.prefetchCancellationTokenSource?.Dispose();