diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor b/app/MindWork AI Studio/Assistants/AssistantBase.razor index 42902a41..0a3fdd9b 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor @@ -31,6 +31,7 @@ @if (this.Body is not null) { + @this.Body @@ -38,7 +39,7 @@ - + @this.SubmitText @if (this.IsProcessing) diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs index 577ce61f..535f3866 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs @@ -47,6 +47,9 @@ public abstract partial class AssistantBase : AssistantLowerBase wher /// [Inject] protected AIJobService AIJobService { get; init; } = null!; + + [Inject] + protected MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; protected abstract string Title { get; } diff --git a/app/MindWork AI Studio/Chat/ChatThread.cs b/app/MindWork AI Studio/Chat/ChatThread.cs index 2c9bb720..542e74bd 100644 --- a/app/MindWork AI Studio/Chat/ChatThread.cs +++ b/app/MindWork AI Studio/Chat/ChatThread.cs @@ -24,6 +24,11 @@ public sealed record ChatThread /// public Guid WorkspaceId { get; set; } + /// + /// The monotonically increasing number used for managed media transcript filenames. + /// + public ulong LastMediaTranscriptNumber { get; set; } + /// /// Specifies the provider selected for the chat thread. /// @@ -240,14 +245,28 @@ public sealed record ChatThread { var previousBlock = sortedBlocks[index - 1]; if (previousBlock.Role is ChatRole.USER && previousBlock.HideFromUser) + { + DeleteManagedAttachments(previousBlock); this.Blocks.Remove(previousBlock); + } } } + DeleteManagedAttachments(block); + // Remove the block from the chat thread: this.Blocks.Remove(block); } + private static void DeleteManagedAttachments(ContentBlock block) + { + if (block.Content is not ContentText textContent) + return; + + foreach (var attachment in textContent.FileAttachments) + ManagedTranscriptAttachment.TryDeleteOwnedFile(attachment); + } + /// /// Transforms this chat thread to an ERI chat thread. /// diff --git a/app/MindWork AI Studio/Chat/FileAttachment.cs b/app/MindWork AI Studio/Chat/FileAttachment.cs index bdc9651d..ce093592 100644 --- a/app/MindWork AI Studio/Chat/FileAttachment.cs +++ b/app/MindWork AI Studio/Chat/FileAttachment.cs @@ -14,6 +14,7 @@ namespace AIStudio.Chat; [JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] [JsonDerivedType(typeof(FileAttachment), typeDiscriminator: "file")] [JsonDerivedType(typeof(FileAttachmentImage), typeDiscriminator: "image")] +[JsonDerivedType(typeof(ManagedTranscriptAttachment), typeDiscriminator: "managed_transcript")] public record FileAttachment(FileAttachmentType Type, string FileName, string FilePath, long FileSizeBytes) { /// @@ -56,7 +57,7 @@ public record FileAttachment(FileAttachmentType Type, string FileName, string Fi /// /// Rebuilds the attachment from its current file path so file type detection uses the latest rules. /// - public FileAttachment Normalize() => FromPath(this.FilePath); + public virtual FileAttachment Normalize() => FromPath(this.FilePath); /// /// Creates a FileAttachment from a file path by automatically determining the type, diff --git a/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs b/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs new file mode 100644 index 00000000..cbbf96d9 --- /dev/null +++ b/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs @@ -0,0 +1,93 @@ +using System.Text; + +using AIStudio.Settings; + +namespace AIStudio.Chat; + +public sealed record ManagedTranscriptAttachment(string FileName, string FilePath, long FileSizeBytes, string OriginalFileName, bool IsStaged) + : FileAttachment(FileAttachmentType.DOCUMENT, FileName, FilePath, FileSizeBytes) +{ + public override FileAttachment Normalize() + { + var size = File.Exists(this.FilePath) ? new FileInfo(this.FilePath).Length : 0; + return this with { FileName = Path.GetFileName(this.FilePath), FileSizeBytes = size }; + } + + public static async Task CreateStagedAsync(string originalPath, string transcript) + { + var operationDirectory = Path.Combine(SettingsManager.DataDirectory!, "media-staging", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(operationDirectory); + + var originalFileName = Path.GetFileName(originalPath); + var stagingPath = Path.Combine(operationDirectory, $"{Guid.NewGuid():N}.md"); + var markdown = $""" + # Transcription of {originalFileName} + + {transcript.Trim()} + """; + + await File.WriteAllTextAsync(stagingPath, markdown, new UTF8Encoding(false)); + return new ManagedTranscriptAttachment( + Path.GetFileName(stagingPath), + stagingPath, + new FileInfo(stagingPath).Length, + originalFileName, + true); + } + + public static bool TryDeleteOwnedFile(FileAttachment attachment) + { + if (attachment is not ManagedTranscriptAttachment managed || !File.Exists(managed.FilePath)) + return false; + + var fileInfo = new FileInfo(managed.FilePath); + var parentInfo = fileInfo.Directory!; + var canonicalParent = parentInfo.ResolveLinkTarget(true)?.FullName ?? parentInfo.FullName; + var fullPath = fileInfo.ResolveLinkTarget(true)?.FullName ?? Path.Combine(canonicalParent, fileInfo.Name); + var dataDirectory = new DirectoryInfo(SettingsManager.DataDirectory!); + + var canonicalDataRoot = EnsureTrailingSeparator(dataDirectory.ResolveLinkTarget(true)?.FullName ?? dataDirectory.FullName); + var stagingDirectory = new DirectoryInfo(Path.Combine(SettingsManager.DataDirectory!, "media-staging")); + var stagingRoot = EnsureTrailingSeparator(stagingDirectory.ResolveLinkTarget(true)?.FullName ?? stagingDirectory.FullName); + + var transcriptsSegment = $"{Path.DirectorySeparatorChar}attachments{Path.DirectorySeparatorChar}transcripts{Path.DirectorySeparatorChar}"; + var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + var owned = fullPath.StartsWith(stagingRoot, comparison) + || (fullPath.StartsWith(canonicalDataRoot, comparison) + && fullPath.Contains(transcriptsSegment, comparison)); + + if (!owned) + return false; + + File.Delete(fullPath); + var parent = Path.GetDirectoryName(fullPath); + if (managed.IsStaged && parent is not null && Directory.Exists(parent) && !Directory.EnumerateFileSystemEntries(parent).Any()) + Directory.Delete(parent); + + return true; + } + + internal static string NormalizeOriginalStem(string originalFileName) + { + var stem = Path.GetFileNameWithoutExtension(originalFileName).Normalize(NormalizationForm.FormC); + var invalid = Path.GetInvalidFileNameChars().ToHashSet(); + var normalized = new StringBuilder(); + + foreach (var character in stem) + { + normalized.Append(char.IsControl(character) + || invalid.Contains(character) + || character is '/' or '\\' + ? '-' + : character); + } + + var result = normalized.ToString().Trim(' ', '.', '-'); + if (string.IsNullOrWhiteSpace(result)) + result = "media"; + + return result.Length <= 80 ? result : result[..80]; + } + + private static string EnsureTrailingSeparator(string path) => path.EndsWith(Path.DirectorySeparatorChar) ? path : path + Path.DirectorySeparatorChar; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/AttachDocuments.razor b/app/MindWork AI Studio/Components/AttachDocuments.razor index e96825c3..4c3980a0 100644 --- a/app/MindWork AI Studio/Components/AttachDocuments.razor +++ b/app/MindWork AI Studio/Components/AttachDocuments.razor @@ -52,7 +52,7 @@ } else { - @if (!this.Disabled) + @if (!this.IsUnavailable) { @@ -73,7 +73,7 @@ else @foreach (var fileAttachment in this.DocumentPaths) { - @if (this.Disabled) + @if (this.IsUnavailable) { } @@ -84,7 +84,7 @@ else } - @if (!this.Disabled) + @if (!this.IsUnavailable) { @T("Clear file list") diff --git a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs index dc72d2e9..f90ff2cc 100644 --- a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs +++ b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs @@ -75,12 +75,16 @@ public partial class AttachDocuments : MSGComponentBase [Inject] private PandocAvailabilityService PandocAvailabilityService { get; init; } = null!; + [Inject] + private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; + private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top; private static readonly string DROP_FILES_HERE_TEXT = TB("Drop files here to attach them."); private uint numDropAreasAboveThis; private bool isComponentHovered; private bool isDraggingOver; + private bool IsUnavailable => this.Disabled || this.MediaTranscriptionService.IsBusy; #region Overrides of MSGComponentBase @@ -95,7 +99,7 @@ public partial class AttachDocuments : MSGComponentBase protected override async Task ProcessIncomingMessage(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default { - if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED) + if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED) return; switch (triggeredEvent) @@ -168,29 +172,7 @@ public partial class AttachDocuments : MSGComponentBase return; } - // Ensure that Pandoc is installed and ready: - var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync( - showSuccessMessage: false, - showDialog: true); - - // If Pandoc is not available (user cancelled installation), abort file drop: - if (!pandocState.IsAvailable) - { - this.Logger.LogWarning("The user cancelled the Pandoc installation or Pandoc is not available. Aborting file drop."); - this.isDraggingOver = false; - this.ClearDragClass(); - this.StateHasChanged(); - return; - } - - foreach (var path in paths) - { - if(!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, path, this.ValidateMediaFileTypes, this.Provider)) - continue; - - this.DocumentPaths.Add(FileAttachment.FromPath(path)); - } - + await this.AddFileBatchAsync(paths); await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths); await this.OnChange(this.DocumentPaths); this.isDraggingOver = false; @@ -208,53 +190,36 @@ public partial class AttachDocuments : MSGComponentBase private async Task AddFilesManually() { - if (this.Disabled) + if (this.IsUnavailable) return; - // Ensure that Pandoc is installed and ready: - var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync( - showSuccessMessage: false, - showDialog: true); - - // If Pandoc is not available (user cancelled installation), abort file selection: - if (!pandocState.IsAvailable) - { - this.Logger.LogWarning("The user cancelled the Pandoc installation or Pandoc is not available. Aborting file selection."); - return; - } - var selectFiles = await this.RustService.SelectFiles(T("Select files to attach")); if (selectFiles.UserCancelled) return; - foreach (var selectedFilePath in selectFiles.SelectedFilePaths) - { - if (!File.Exists(selectedFilePath)) - continue; - - if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, selectedFilePath, this.ValidateMediaFileTypes, this.Provider)) - continue; - - this.DocumentPaths.Add(FileAttachment.FromPath(selectedFilePath)); - } - + await this.AddFileBatchAsync(selectFiles.SelectedFilePaths); await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths); await this.OnChange(this.DocumentPaths); } private async Task OpenAttachmentsDialog() { - if (this.Disabled) + if (this.IsUnavailable) return; + var previousAttachments = this.DocumentPaths.ToHashSet(); this.DocumentPaths = await ReviewAttachmentsDialog.OpenDialogAsync(this.DialogService, this.DocumentPaths); + foreach (var removedAttachment in previousAttachments.Except(this.DocumentPaths)) + ManagedTranscriptAttachment.TryDeleteOwnedFile(removedAttachment); } private async Task ClearAllFiles() { - if (this.Disabled) + if (this.IsUnavailable) return; + foreach (var attachment in this.DocumentPaths) + ManagedTranscriptAttachment.TryDeleteOwnedFile(attachment); this.DocumentPaths.Clear(); await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths); await this.OnChange(this.DocumentPaths); @@ -266,7 +231,7 @@ public partial class AttachDocuments : MSGComponentBase private void OnMouseEnter(EventArgs _) { - if(this.Disabled || this.PauseCatchingDrops) + if(this.IsUnavailable || this.PauseCatchingDrops) return; this.Logger.LogDebug("Attach documents component '{Name}' is hovered.", this.Name); @@ -277,7 +242,7 @@ public partial class AttachDocuments : MSGComponentBase private void OnMouseLeave(EventArgs _) { - if(this.Disabled || this.PauseCatchingDrops) + if(this.IsUnavailable || this.PauseCatchingDrops) return; this.Logger.LogDebug("Attach documents component '{Name}' is no longer hovered.", this.Name); @@ -288,15 +253,97 @@ public partial class AttachDocuments : MSGComponentBase private async Task RemoveDocument(FileAttachment fileAttachment) { - if (this.Disabled) + if (this.IsUnavailable) return; this.DocumentPaths.Remove(fileAttachment); + ManagedTranscriptAttachment.TryDeleteOwnedFile(fileAttachment); await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths); await this.OnChange(this.DocumentPaths); } + private async Task AddFileBatchAsync(IEnumerable paths) + { + var existingPaths = paths.Where(File.Exists).ToList(); + var mediaPaths = existingPaths.Where(IsTranscribableMedia).ToList(); + var regularPaths = existingPaths.Except(mediaPaths).ToList(); + + var canAddRegularFiles = true; + if (regularPaths.Count > 0) + { + var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync( + showSuccessMessage: false, + showDialog: true); + canAddRegularFiles = pandocState.IsAvailable; + } + + foreach (var path in regularPaths) + { + if (!canAddRegularFiles) + break; + + if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync( + FileExtensionValidation.UseCase.ATTACHING_CONTENT, + path, + this.ValidateMediaFileTypes, + this.Provider)) + continue; + this.DocumentPaths.Add(FileAttachment.FromPath(path)); + } + + if (mediaPaths.Count is 0) + return; + + if (string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider)) + { + await this.MessageBus.SendWarning(new( + Icons.Material.Filled.VoiceChat, + this.T("Media files require a configured transcription provider. Configure one in the transcription settings."))); + return; + } + + var names = string.Join(Environment.NewLine, mediaPaths.Select(path => $"• {Path.GetFileName(path)}")); + var dialogParameters = new DialogParameters + { + { + x => x.Message, + $"{this.T("The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider.")}{Environment.NewLine}{Environment.NewLine}{names}" + }, + }; + + var dialogReference = await this.DialogService.ShowAsync( + this.T("Transcribe media files"), + dialogParameters, + DialogOptions.FULLSCREEN); + + var dialogResult = await dialogReference.Result; + if (dialogResult is null || dialogResult.Canceled) + return; + + var failures = new List(); + foreach (var mediaPath in mediaPaths) + { + var result = await this.MediaTranscriptionService.TranscribeAsync(mediaPath); + if (!result.Success) + { + if (result.ErrorMessage == "The media transcription was cancelled.") + break; + + failures.Add($"{Path.GetFileName(mediaPath)}: {result.ErrorMessage}"); + continue; + } + + var attachment = await ManagedTranscriptAttachment.CreateStagedAsync(mediaPath, result.Text); + this.DocumentPaths.Add(attachment); + } + + if (failures.Count > 0) + await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, string.Join(Environment.NewLine, failures))); + } + + private static bool IsTranscribableMedia(string path) => FileTypes.IsAllowedPath(path, FileTypes.AUDIO) || FileTypes.IsAllowedPath(path, FileTypes.VIDEO); + /// /// The user might want to check what we actually extract from his file and therefore give the LLM as an input. /// diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor b/app/MindWork AI Studio/Components/ChatComponent.razor index 45c0584c..48d9b948 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor +++ b/app/MindWork AI Studio/Components/ChatComponent.razor @@ -33,6 +33,7 @@ } + } - + diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index 06b6fb92..b50b607e 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -4,6 +4,7 @@ using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.AIJobs; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components.Web; @@ -54,6 +55,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable [Inject] private AIJobService AIJobService { get; init; } = null!; + [Inject] + private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; + private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top; private static readonly Dictionary USER_INPUT_ATTRIBUTES = new(); @@ -683,6 +687,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable private async Task SendMessage(bool reuseLastUserPrompt = false) { + if (this.MediaTranscriptionService.IsBusy) + return; + if (!this.IsProviderSelected) return; @@ -986,12 +993,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable if (workspaceId == Guid.Empty) return; - // Delete the chat from the current workspace or the temporary storage: - await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, this.ChatThread!.WorkspaceId, this.ChatThread.ChatId, askForConfirmation: false); - - this.ChatThread!.WorkspaceId = workspaceId; + await WorkspaceBehaviour.MoveChatAsync(this.ChatThread!, workspaceId); this.MarkCurrentChatAsLoadedParameter(); - await this.SaveThread(); await this.SyncWorkspaceHeaderWithChatThreadAsync(); } diff --git a/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor b/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor new file mode 100644 index 00000000..2c6001d2 --- /dev/null +++ b/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor @@ -0,0 +1,19 @@ +@inherits MSGComponentBase +@inject MediaTranscriptionService MediaTranscriptionService +@using AIStudio.Tools.Services + +@if (this.MediaTranscriptionService.IsBusy) +{ + + + + + @this.StatusText + + + + + + + +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor.cs b/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor.cs new file mode 100644 index 00000000..402b0083 --- /dev/null +++ b/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor.cs @@ -0,0 +1,28 @@ +using AIStudio.Tools.Services; + +namespace AIStudio.Components; + +public partial class MediaTranscriptionStatus +{ + private string StatusText => this.MediaTranscriptionService.Phase switch + { + MediaTranscriptionPhase.PROBING => $"{this.T("Inspecting media")}: {this.MediaTranscriptionService.CurrentFileName}", + MediaTranscriptionPhase.TRANSCODING => $"{this.T("Preparing audio")}: {this.MediaTranscriptionService.CurrentFileName}", + MediaTranscriptionPhase.UPLOADING => $"{this.T("Transcribing")}: {this.MediaTranscriptionService.CurrentFileName}", + + _ => this.MediaTranscriptionService.CurrentFileName, + }; + + protected override async Task OnInitializedAsync() + { + this.MediaTranscriptionService.StateChanged += this.OnStateChanged; + await base.OnInitializedAsync(); + } + + private void OnStateChanged() => _ = this.InvokeAsync(this.StateHasChanged); + + protected override void DisposeResources() + { + this.MediaTranscriptionService.StateChanged -= this.OnStateChanged; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ReadFileContent.razor b/app/MindWork AI Studio/Components/ReadFileContent.razor index 27f979b0..82f7b1f6 100644 --- a/app/MindWork AI Studio/Components/ReadFileContent.razor +++ b/app/MindWork AI Studio/Components/ReadFileContent.razor @@ -1,11 +1,13 @@ @inherits MSGComponentBase + + @if (this.EnableDragDrop) {
- + @this.ButtonText @@ -17,7 +19,7 @@ } else { - + @this.ButtonText } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ReadFileContent.razor.cs b/app/MindWork AI Studio/Components/ReadFileContent.razor.cs index c301a541..31fa36fe 100644 --- a/app/MindWork AI Studio/Components/ReadFileContent.razor.cs +++ b/app/MindWork AI Studio/Components/ReadFileContent.razor.cs @@ -1,6 +1,7 @@ using AIStudio.Tools.Rust; using AIStudio.Tools.Services; using AIStudio.Tools.Validation; +using AIStudio.Dialogs; using Microsoft.AspNetCore.Components; @@ -47,12 +48,16 @@ public partial class ReadFileContent : MSGComponentBase [Inject] private PandocAvailabilityService PandocAvailabilityService { get; init; } = null!; + [Inject] + private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; + private const string DEFAULT_DRAG_CLASS = "relative rounded-lg border-2 border-dashed pa-3 mb-3 mud-width-full"; private string ButtonText => string.IsNullOrWhiteSpace(this.Text) ? T("Use file content as input") : this.Text; private string dragClass = DEFAULT_DRAG_CLASS; private uint numDropAreasAboveThis; private bool isComponentHovered; + private bool IsUnavailable => this.Disabled || this.MediaTranscriptionService.IsBusy; #region Overrides of MSGComponentBase @@ -72,7 +77,7 @@ public partial class ReadFileContent : MSGComponentBase if (!this.EnableDragDrop) return; - if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED) + if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED) return; switch (triggeredEvent) @@ -126,10 +131,7 @@ public partial class ReadFileContent : MSGComponentBase private async Task SelectFile() { - if (this.Disabled) - return; - - if (!await this.EnsurePandocAvailability()) + if (this.IsUnavailable) return; var selectedFile = await this.RustService.SelectFile(T("Select file to read its content")); @@ -161,9 +163,6 @@ public partial class ReadFileContent : MSGComponentBase private async Task LoadFirstValidFile(List paths) { - if (!await this.EnsurePandocAvailability()) - return; - foreach (var path in paths) { if (await this.LoadFileIfValid(path)) @@ -179,6 +178,12 @@ public partial class ReadFileContent : MSGComponentBase return false; } + if (FileTypes.IsAllowedPath(filePath, FileTypes.AUDIO) || FileTypes.IsAllowedPath(filePath, FileTypes.VIDEO)) + return await this.LoadMediaTranscriptAsync(filePath); + + if (!await this.EnsurePandocAvailability()) + return false; + if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.DIRECTLY_LOADING_CONTENT, filePath)) { this.Logger.LogWarning("User attempted to load unsupported file: {FilePath}", filePath); @@ -200,6 +205,43 @@ public partial class ReadFileContent : MSGComponentBase } } + private async Task LoadMediaTranscriptAsync(string filePath) + { + if (string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider)) + { + await this.MessageBus.SendWarning(new( + Icons.Material.Filled.VoiceChat, + this.T("Media files require a configured transcription provider. Configure one in the transcription settings."))); + return false; + } + + var dialogParameters = new DialogParameters + { + { + x => x.Message, + $"{this.T("The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider.")}{Environment.NewLine}{Environment.NewLine}{Path.GetFileName(filePath)}" + }, + }; + var dialogReference = await this.DialogService.ShowAsync( + this.T("Transcribe media file"), + dialogParameters, + Dialogs.DialogOptions.FULLSCREEN); + + var dialogResult = await dialogReference.Result; + if (dialogResult is null || dialogResult.Canceled) + return false; + + var result = await this.MediaTranscriptionService.TranscribeAsync(filePath); + if (!result.Success) + { + await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, result.ErrorMessage)); + return false; + } + + await this.FileContentChanged.InvokeAsync(result.Text); + return true; + } + private bool CanCatchDroppedFile() => this.numDropAreasAboveThis is 0 && (this.isComponentHovered || this.CatchAllDocuments); private void SetDragClass() => this.dragClass = $"{DEFAULT_DRAG_CLASS} mud-border-primary border-2"; @@ -208,7 +250,7 @@ public partial class ReadFileContent : MSGComponentBase private void OnMouseEnter(EventArgs _) { - if(this.Disabled || this.numDropAreasAboveThis > 0) + if(this.IsUnavailable || this.numDropAreasAboveThis > 0) return; this.Logger.LogDebug("Read file content component is hovered."); @@ -219,7 +261,7 @@ public partial class ReadFileContent : MSGComponentBase private void OnMouseLeave(EventArgs _) { - if(this.Disabled) + if(this.IsUnavailable) return; this.Logger.LogDebug("Read file content component is no longer hovered."); diff --git a/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs b/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs index 669932f6..5d2fcaa9 100644 --- a/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs +++ b/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs @@ -1,4 +1,3 @@ -using AIStudio.Provider; using AIStudio.Settings.DataModel; using AIStudio.Tools.MIME; using AIStudio.Tools.Rust; @@ -25,6 +24,9 @@ public partial class VoiceRecorder : MSGComponentBase [Inject] private VoiceRecordingAvailabilityService VoiceRecordingAvailabilityService { get; init; } = null!; + [Inject] + private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; + #region Overrides of MSGComponentBase protected override async Task OnInitializedAsync() @@ -101,7 +103,8 @@ public partial class VoiceRecorder : MSGComponentBase && !string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider); private bool IsVoiceRecordingAvailable => this.ShouldRenderVoiceRecording - && this.VoiceRecordingAvailabilityService.IsAvailable; + && this.VoiceRecordingAvailabilityService.IsAvailable + && !this.MediaTranscriptionService.IsBusy; private string Tooltip => !this.VoiceRecordingAvailabilityService.IsAvailable ? T("Voice recording is unavailable because the client could not initialize audio playback.") @@ -151,7 +154,7 @@ public partial class VoiceRecorder : MSGComponentBase try { - var mimeTypeStrings = mimeTypes.ToStringArray(); + string[] mimeTypeStrings = ["audio/webm;codecs=opus", .. mimeTypes.ToStringArray()]; var actualMimeType = await this.JsRuntime.InvokeAsync("audioRecorder.start", this.dotNetReference, mimeTypeStrings); // Store the MIME type for later use: @@ -317,52 +320,7 @@ public partial class VoiceRecorder : MSGComponentBase try { - // Get the configured transcription provider ID: - var transcriptionProviderId = this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider; - if (string.IsNullOrWhiteSpace(transcriptionProviderId)) - { - this.Logger.LogWarning("No transcription provider is configured."); - await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("No transcription provider is configured."))); - return; - } - - // Find the transcription provider in the list of configured providers: - var transcriptionProviderSettings = this.SettingsManager.ConfigurationData.TranscriptionProviders - .FirstOrDefault(x => x.Id == transcriptionProviderId); - - if (transcriptionProviderSettings is null) - { - this.Logger.LogWarning("The configured transcription provider with ID '{ProviderId}' was not found.", transcriptionProviderId); - await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The configured transcription provider was not found."))); - return; - } - - // Check the confidence level: - var minimumLevel = this.SettingsManager.GetMinimumConfidenceLevel(Tools.Components.NONE); - var providerConfidence = transcriptionProviderSettings.UsedLLMProvider.GetConfidence(this.SettingsManager); - if (providerConfidence.Level < minimumLevel) - { - this.Logger.LogWarning( - "The configured transcription provider '{ProviderName}' has a confidence level of '{ProviderLevel}', which is below the minimum required level of '{MinimumLevel}'.", - transcriptionProviderSettings.UsedLLMProvider, - providerConfidence.Level, - minimumLevel); - await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The configured transcription provider does not meet the minimum confidence level."))); - return; - } - - // Create the provider instance: - var provider = transcriptionProviderSettings.CreateProvider(); - if (provider.Provider is LLMProviders.NONE) - { - this.Logger.LogError("Failed to create the transcription provider instance."); - await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("Failed to create the transcription provider."))); - return; - } - - // Call the transcription API: - this.Logger.LogInformation("Starting transcription with provider '{ProviderName}' and model '{ModelName}'.", transcriptionProviderSettings.UsedLLMProvider, transcriptionProviderSettings.Model.ToString()); - var transcriptionResult = await provider.TranscribeAudioAsync(transcriptionProviderSettings.Model, this.finalRecordingPath, this.SettingsManager); + var transcriptionResult = await this.MediaTranscriptionService.TranscribeAsync(this.finalRecordingPath); if (!transcriptionResult.Success) { this.Logger.LogWarning("The transcription request failed."); @@ -406,19 +364,6 @@ public partial class VoiceRecorder : MSGComponentBase // Copy the transcribed text to the clipboard: await this.RustService.CopyText2Clipboard(this.Snackbar, transcribedText); - // Delete the recording file: - try - { - if (File.Exists(this.finalRecordingPath)) - { - File.Delete(this.finalRecordingPath); - this.Logger.LogInformation("Deleted the recording file '{RecordingPath}'.", this.finalRecordingPath); - } - } - catch (Exception ex) - { - this.Logger.LogError(ex, "Failed to delete the recording file '{RecordingPath}'.", this.finalRecordingPath); - } } catch (Exception ex) { @@ -429,6 +374,15 @@ public partial class VoiceRecorder : MSGComponentBase { await this.ReleaseMicrophoneAsync(); + try + { + if (File.Exists(this.finalRecordingPath)) + File.Delete(this.finalRecordingPath); + } + catch (Exception ex) + { + this.Logger.LogError(ex, "Failed to delete the recording file '{RecordingPath}'.", this.finalRecordingPath); + } this.finalRecordingPath = null; this.isTranscribing = false; this.StateHasChanged(); diff --git a/app/MindWork AI Studio/Pages/Information.razor b/app/MindWork AI Studio/Pages/Information.razor index 18863903..f3858a04 100644 --- a/app/MindWork AI Studio/Pages/Information.razor +++ b/app/MindWork AI Studio/Pages/Information.razor @@ -308,7 +308,11 @@ - + + + + + diff --git a/app/MindWork AI Studio/Program.cs b/app/MindWork AI Studio/Program.cs index d5cdaf5c..c50ebeeb 100644 --- a/app/MindWork AI Studio/Program.cs +++ b/app/MindWork AI Studio/Program.cs @@ -136,6 +136,7 @@ internal sealed class Program builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -148,6 +149,7 @@ internal sealed class Program builder.Services.AddTransient(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); + builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddSingleton(); builder.Services.AddHostedService(); diff --git a/app/MindWork AI Studio/Tools/Rust/CreateMediaJobRequest.cs b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobRequest.cs new file mode 100644 index 00000000..7726bb72 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobRequest.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Rust; + +public sealed record CreateMediaJobRequest(string InputPath, string OutputPath, ulong? MaxPassThroughBytes = null); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/CreateMediaJobResponse.cs b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobResponse.cs new file mode 100644 index 00000000..7ff6e4fe --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobResponse.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Rust; + +public sealed record CreateMediaJobResponse(string JobId); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/FileTypes.cs b/app/MindWork AI Studio/Tools/Rust/FileTypes.cs index 1e388109..5f29bc11 100644 --- a/app/MindWork AI Studio/Tools/Rust/FileTypes.cs +++ b/app/MindWork AI Studio/Tools/Rust/FileTypes.cs @@ -61,7 +61,7 @@ public static class FileTypes public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"), "jpg", "jpeg", "png", "gif", "bmp", "tiff", "svg", "webp", "heic"); public static readonly FileTypeFilter AUDIO = FileTypeFilter.Leaf(TB("Audio"), - "mp3", "wav", "wave", "aac", "flac", "ogg", "m4a", "wma", "alac", "aiff", "m4b"); + "mp3", "wav", "wave", "aac", "flac", "ogg", "opus", "m4a", "m4b", "wma", "alac", "aif", "aiff", "caf"); public static readonly FileTypeFilter VIDEO = FileTypeFilter.Leaf(TB("Video"), "mp4", "m4v", "avi", "mkv", "mov", "wmv", "flv", "webm"); diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobError.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobError.cs new file mode 100644 index 00000000..9c721283 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobError.cs @@ -0,0 +1,3 @@ +namespace AIStudio.Tools.Rust; + +public sealed record MediaJobError(string Code, string Message); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobEvent.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobEvent.cs new file mode 100644 index 00000000..dda2de50 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobEvent.cs @@ -0,0 +1,7 @@ +namespace AIStudio.Tools.Rust; + +public sealed record MediaJobEvent( + MediaJobPhase Phase, + double? Progress, + MediaJobResult? Result, + MediaJobError? Error); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobPhase.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobPhase.cs new file mode 100644 index 00000000..5540bb8c --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobPhase.cs @@ -0,0 +1,11 @@ +namespace AIStudio.Tools.Rust; + +public enum MediaJobPhase +{ + UNKNOWN, + PROBING, + TRANSCODING, + COMPLETED, + FAILED, + CANCELLED, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs new file mode 100644 index 00000000..50bfb175 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Tools.Rust; + +public sealed record MediaJobResult( + string OutputPath, + string DetectedFormat, + string DetectedCodec, + ulong DurationMs, + bool PassThrough); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionPhase.cs b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionPhase.cs new file mode 100644 index 00000000..46369c8a --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionPhase.cs @@ -0,0 +1,9 @@ +namespace AIStudio.Tools.Services; + +public enum MediaTranscriptionPhase +{ + IDLE, + PROBING, + TRANSCODING, + UPLOADING, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs new file mode 100644 index 00000000..7906476e --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs @@ -0,0 +1,173 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Tools.Rust; + +namespace AIStudio.Tools.Services; + +public sealed class MediaTranscriptionService(RustService rustService, SettingsManager settingsManager, ILogger logger) : IDisposable +{ + private readonly SemaphoreSlim queue = new(1, 1); + private readonly Lock stateLock = new(); + private CancellationTokenSource? currentCancellation; + private string? currentJobId; + + public event Action? StateChanged; + + public bool IsBusy { get; private set; } + + public string CurrentFileName { get; private set; } = string.Empty; + + public MediaTranscriptionPhase Phase { get; private set; } = MediaTranscriptionPhase.IDLE; + + public double? Progress { get; private set; } + + public async Task TranscribeAsync(string mediaPath, CancellationToken token = default) + { + await this.queue.WaitAsync(token); + var normalizedPath = Path.Combine( + Path.GetTempPath(), + "mindwork-ai-studio-media", + $"{Guid.NewGuid():N}.webm"); + + Directory.CreateDirectory(Path.GetDirectoryName(normalizedPath)!); + using var operationCancellation = CancellationTokenSource.CreateLinkedTokenSource(token); + lock (this.stateLock) + this.currentCancellation = operationCancellation; + + try + { + this.UpdateState(true, Path.GetFileName(mediaPath), MediaTranscriptionPhase.PROBING, 0.0); + var jobId = await rustService.StartMediaJobAsync(mediaPath, normalizedPath, operationCancellation.Token); + lock (this.stateLock) + this.currentJobId = jobId; + + MediaJobResult? normalized = null; + await foreach (var mediaEvent in rustService.StreamMediaJobEventsAsync(jobId, operationCancellation.Token)) + { + switch (mediaEvent.Phase) + { + case MediaJobPhase.PROBING: + this.UpdateState(true, this.CurrentFileName, MediaTranscriptionPhase.PROBING, mediaEvent.Progress); + break; + + case MediaJobPhase.TRANSCODING: + this.UpdateState(true, this.CurrentFileName, MediaTranscriptionPhase.TRANSCODING, mediaEvent.Progress); + break; + + case MediaJobPhase.COMPLETED: + normalized = mediaEvent.Result; + break; + + case MediaJobPhase.CANCELLED: + throw new OperationCanceledException(operationCancellation.Token); + + case MediaJobPhase.FAILED: + return TranscriptionResult.Failure(mediaEvent.Error?.Message ?? "The media file could not be prepared."); + } + } + + if (normalized is null) + return TranscriptionResult.Failure("The media pipeline ended without an output file."); + + var providerSettings = this.ResolveProvider(); + if (providerSettings is null) + return TranscriptionResult.Failure("No usable transcription provider is configured."); + + this.UpdateState(true, this.CurrentFileName, MediaTranscriptionPhase.UPLOADING, null); + + var provider = providerSettings.CreateProvider(); + if (provider.Provider is LLMProviders.NONE) + return TranscriptionResult.Failure("The configured transcription provider could not be created."); + + logger.LogInformation( + "Transcribing normalized media '{MediaPath}' with provider '{Provider}' and model '{Model}'.", + mediaPath, + providerSettings.UsedLLMProvider, + providerSettings.Model); + + var result = await provider.TranscribeAudioAsync( + providerSettings.Model, + normalized.OutputPath, + settingsManager, + operationCancellation.Token); + + return result.Success + ? TranscriptionResult.FromText(result.Text.Trim()) + : result; + } + catch (OperationCanceledException) + { + return TranscriptionResult.Failure("The media transcription was cancelled."); + } + catch (Exception exception) + { + logger.LogError(exception, "Media transcription failed for '{MediaPath}'.", mediaPath); + return TranscriptionResult.Failure("The media file could not be transcribed."); + } + finally + { + lock (this.stateLock) + { + this.currentJobId = null; + this.currentCancellation = null; + } + try + { + if (File.Exists(normalizedPath)) + File.Delete(normalizedPath); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not delete normalized media file '{NormalizedPath}'.", normalizedPath); + } + + this.UpdateState(false, string.Empty, MediaTranscriptionPhase.IDLE, null); + this.queue.Release(); + } + } + + public async Task StopAsync() + { + string? jobId; + lock (this.stateLock) + { + this.currentCancellation?.Cancel(); + jobId = this.currentJobId; + } + + if (!string.IsNullOrWhiteSpace(jobId)) + await rustService.CancelMediaJobAsync(jobId); + } + + private TranscriptionProvider? ResolveProvider() + { + var providerId = settingsManager.ConfigurationData.App.UseTranscriptionProvider; + if (string.IsNullOrWhiteSpace(providerId)) + return null; + + var providerSettings = settingsManager.ConfigurationData.TranscriptionProviders.FirstOrDefault(x => x.Id == providerId); + if (providerSettings is null) + return null; + + var minimumLevel = settingsManager.GetMinimumConfidenceLevel(Components.NONE); + return providerSettings.UsedLLMProvider.GetConfidence(settingsManager).Level >= minimumLevel + ? providerSettings + : null; + } + + private void UpdateState(bool isBusy, string fileName, MediaTranscriptionPhase phase, double? progress) + { + this.IsBusy = isBusy; + this.CurrentFileName = fileName; + this.Phase = phase; + this.Progress = progress; + this.StateChanged?.Invoke(); + } + + public void Dispose() + { + this.currentCancellation?.Cancel(); + this.currentCancellation?.Dispose(); + this.queue.Dispose(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Media.cs b/app/MindWork AI Studio/Tools/Services/RustService.Media.cs new file mode 100644 index 00000000..fb01eebe --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/RustService.Media.cs @@ -0,0 +1,54 @@ +using System.Runtime.CompilerServices; +using System.Text.Json; + +using AIStudio.Tools.Rust; + +namespace AIStudio.Tools.Services; + +public partial class RustService +{ + public async Task StartMediaJobAsync(string inputPath, string outputPath, CancellationToken token = default) + { + using var response = await this.http.PostAsJsonAsync( + "/media/jobs", + new CreateMediaJobRequest(inputPath, outputPath), + this.jsonRustSerializerOptions, + token); + + response.EnsureSuccessStatusCode(); + var result = await response.Content.ReadFromJsonAsync(this.jsonRustSerializerOptions, token); + return result?.JobId ?? throw new InvalidOperationException("The Rust runtime did not return a media job ID."); + } + + public async IAsyncEnumerable StreamMediaJobEventsAsync(string jobId, [EnumeratorCancellation] CancellationToken token = default) + { + using var request = new HttpRequestMessage(HttpMethod.Get, $"/media/jobs/{Uri.EscapeDataString(jobId)}/events"); + using var response = await this.http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(token); + using var reader = new StreamReader(stream); + + while (!reader.EndOfStream && !token.IsCancellationRequested) + { + var line = await reader.ReadLineAsync(token); + if (line is null || !line.StartsWith("data:", StringComparison.Ordinal)) + continue; + + var json = line["data:".Length..].Trim(); + var mediaEvent = JsonSerializer.Deserialize(json, this.jsonRustSerializerOptions); + if (mediaEvent is not null) + yield return mediaEvent; + + if (mediaEvent?.Phase is MediaJobPhase.COMPLETED or MediaJobPhase.FAILED or MediaJobPhase.CANCELLED) + yield break; + } + } + + public async Task CancelMediaJobAsync(string jobId, CancellationToken token = default) + { + using var response = await this.http.DeleteAsync($"/media/jobs/{Uri.EscapeDataString(jobId)}", token); + if (response is { IsSuccessStatusCode: false, StatusCode: not System.Net.HttpStatusCode.NotFound }) + response.EnsureSuccessStatusCode(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/TranscriptStagingCleanupService.cs b/app/MindWork AI Studio/Tools/Services/TranscriptStagingCleanupService.cs new file mode 100644 index 00000000..0387b5e4 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/TranscriptStagingCleanupService.cs @@ -0,0 +1,31 @@ +using AIStudio.Settings; + +namespace AIStudio.Tools.Services; + +public sealed class TranscriptStagingCleanupService(ILogger logger) : BackgroundService +{ + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (string.IsNullOrWhiteSpace(SettingsManager.DataDirectory) && !stoppingToken.IsCancellationRequested) + await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); + + if (stoppingToken.IsCancellationRequested) + return; + + var stagingRoot = Path.Combine(SettingsManager.DataDirectory!, "media-staging"); + if (!Directory.Exists(stagingRoot)) + return; + + foreach (var directory in Directory.EnumerateDirectories(stagingRoot)) + { + try + { + Directory.Delete(directory, true); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not remove orphaned media staging directory '{Directory}'.", directory); + } + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs index d45601e5..b665d629 100644 --- a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs +++ b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs @@ -754,6 +754,7 @@ public static class WorkspaceBehaviour Directory.CreateDirectory(chatDirectory); + await FinalizeStagedTranscriptsAsync(chat, chatDirectory); var chatNamePath = Path.Join(chatDirectory, "name"); await File.WriteAllTextAsync(chatNamePath, chat.Name); @@ -769,6 +770,99 @@ public static class WorkspaceBehaviour } } + public static async Task MoveChatAsync(ChatThread chat, Guid targetWorkspaceId) + { + if (chat.WorkspaceId == targetWorkspaceId) + return; + + var sourceDirectory = chat.WorkspaceId == Guid.Empty + ? Path.Join(SettingsManager.DataDirectory, "tempChats", chat.ChatId.ToString()) + : Path.Join(SettingsManager.DataDirectory, "workspaces", chat.WorkspaceId.ToString(), chat.ChatId.ToString()); + + var targetDirectory = targetWorkspaceId == Guid.Empty + ? Path.Join(SettingsManager.DataDirectory, "tempChats", chat.ChatId.ToString()) + : Path.Join(SettingsManager.DataDirectory, "workspaces", targetWorkspaceId.ToString(), chat.ChatId.ToString()); + + if (Directory.Exists(sourceDirectory)) + { + Directory.CreateDirectory(Path.GetDirectoryName(targetDirectory)!); + if (Directory.Exists(targetDirectory)) + throw new IOException($"The target chat directory already exists: '{targetDirectory}'."); + + Directory.Move(sourceDirectory, targetDirectory); + UpdateAttachmentPathsAfterMove(chat, sourceDirectory, targetDirectory); + } + + chat.WorkspaceId = targetWorkspaceId; + await StoreChatAsync(chat); + InvalidateWorkspaceTreeCache(); + } + + private static void UpdateAttachmentPathsAfterMove(ChatThread chat, string sourceDirectory, string targetDirectory) + { + var sourcePrefix = sourceDirectory.EndsWith(Path.DirectorySeparatorChar) + ? sourceDirectory + : sourceDirectory + Path.DirectorySeparatorChar; + + var pathComparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + foreach (var content in chat.Blocks.Select(block => block.Content).OfType()) + { + for (var index = 0; index < content.FileAttachments.Count; index++) + { + var attachment = content.FileAttachments[index]; + if (!Path.GetFullPath(attachment.FilePath).StartsWith(sourcePrefix, pathComparison)) + continue; + + var relativePath = Path.GetRelativePath(sourceDirectory, attachment.FilePath); + var movedPath = Path.Combine(targetDirectory, relativePath); + + content.FileAttachments[index] = attachment switch + { + ManagedTranscriptAttachment managed => managed with { FilePath = movedPath }, + FileAttachmentImage image => image with { FilePath = movedPath }, + _ => attachment with { FilePath = movedPath }, + }; + } + } + } + + private static async Task FinalizeStagedTranscriptsAsync(ChatThread chat, string chatDirectory) + { + var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts"); + foreach (var content in chat.Blocks.Select(block => block.Content).OfType()) + { + for (var index = 0; index < content.FileAttachments.Count; index++) + { + if (content.FileAttachments[index] is not ManagedTranscriptAttachment { IsStaged: true } staged + || !File.Exists(staged.FilePath)) + continue; + + Directory.CreateDirectory(transcriptDirectory); + string targetPath; + do + { + chat.LastMediaTranscriptNumber++; + var stem = ManagedTranscriptAttachment.NormalizeOriginalStem(staged.OriginalFileName); + targetPath = Path.Combine(transcriptDirectory, $"{stem}-transcript-{chat.LastMediaTranscriptNumber:D4}.md"); + } while (File.Exists(targetPath)); + + File.Move(staged.FilePath, targetPath); + var sourceDirectory = Path.GetDirectoryName(staged.FilePath); + if (sourceDirectory is not null && Directory.Exists(sourceDirectory) && !Directory.EnumerateFileSystemEntries(sourceDirectory).Any()) + Directory.Delete(sourceDirectory); + + content.FileAttachments[index] = new ManagedTranscriptAttachment( + Path.GetFileName(targetPath), + targetPath, + new FileInfo(targetPath).Length, + staged.OriginalFileName, + false); + } + } + + await Task.CompletedTask; + } + public static async Task LoadChatAsync(LoadChat loadChat) { var (acquired, semaphore) = await TryAcquireChatSemaphoreAsync(loadChat.WorkspaceId, loadChat.ChatId, nameof(LoadChatAsync)); diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.7.3.md b/app/MindWork AI Studio/wwwroot/changelog/v26.7.3.md index 346091e4..5537ead0 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.7.3.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.7.3.md @@ -1,5 +1,6 @@ # v26.7.3, build 245 (2026-07-xx xx:xx UTC) - Added support for OpenAI GPT-5.6 Sol, Terra, and Luna; Anthropic Claude Fable 5 and Mythos 5; and Google Gemini 3 Flash, Gemini 3.1 Flash-Lite, Gemini 3.1 Pro, and Gemini 3.5 Flash. +- Added audio and video transcription for chats and assistants. AI Studio now prepares supported media locally, sends only normalized audio to the configured transcription provider, and attaches the resulting transcript instead of the original media. - Improved the "My Tasks Assistant": you can now provide one or more documents in addition to text or use documents alone when asking to identify tasks. - Improved update guidance for Flatpak installations and added an enterprise option that lets organizations manage updates entirely through their IT department. - Fixed an issue that could leave AI Studio unresponsive after waking the computer from sleep. diff --git a/runtime/Cargo.lock b/runtime/Cargo.lock index f0956ce7..0c8bd4de 100644 --- a/runtime/Cargo.lock +++ b/runtime/Cargo.lock @@ -490,6 +490,43 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "audio-core" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ebbf82d06013f4c41fe71303feb980cddd78496d904d06be627972de51a24" + +[[package]] +name = "audioadapter" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75c3943c6c7279bb25a449a8d1727480730ab2efd7b6fd5d6ca51927096e6e4" +dependencies = [ + "audio-core", + "num-traits", +] + +[[package]] +name = "audioadapter-buffers" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece3390b6eb40379094843a1da5aaccc34bc0d85a8cbf68d09fe092fee6de29e" +dependencies = [ + "audioadapter", + "audioadapter-sample", + "num-traits", +] + +[[package]] +name = "audioadapter-sample" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1592f90413568e259413c21a41a3d571feb1774255c209e7966d98f9db708c90" +dependencies = [ + "audio-core", + "num-traits", +] + [[package]] name = "autocfg" version = "1.3.0" @@ -1923,6 +1960,35 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ebml-iterable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5173ac3752f08b526a6991509615e1a345b221ec3c58c7633433e8c9582312" +dependencies = [ + "ebml-iterable-specification", + "ebml-iterable-specification-derive", + "futures", +] + +[[package]] +name = "ebml-iterable-specification" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56467af159a98735d44231f53eaa505e919e6003266f103b99649a93f106784" + +[[package]] +name = "ebml-iterable-specification-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b066b81018300fdce40f71c4db355a102699324af96fad28f25ab1b5f87de066" +dependencies = [ + "ebml-iterable-specification", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "ecow" version = "0.3.0" @@ -2112,6 +2178,12 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fast-float2" version = "0.2.3" @@ -4023,11 +4095,14 @@ dependencies = [ "rand 0.10.2", "rand_chacha 0.10.0", "rcgen", + "ropus", + "rubato", "rustls", "serde", "serde_json", "sha2 0.11.0", "strum_macros", + "symphonia", "sys-locale", "sysinfo 0.39.6", "tauri", @@ -4043,6 +4118,7 @@ dependencies = [ "tokio", "tokio-stream", "webkit2gtk", + "webm-iterable", "whoami", "windows-native-keyring-store", "windows-registry", @@ -5126,6 +5202,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -5561,6 +5646,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -5613,6 +5707,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.10" @@ -5740,6 +5840,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839" +[[package]] +name = "ropus" +version = "0.12.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80804dadbfa2851c95fe45ff9ae8f4328d6371fdc0d51b14740d49a8b41d3758" +dependencies = [ + "cc", + "wide", +] + [[package]] name = "roxmltree" version = "0.20.0" @@ -5757,6 +5867,22 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rubato" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f57c655d11e929f05a8663b323ff553f8d9773be05dfdc087795955bedeb8d92" +dependencies = [ + "audioadapter", + "audioadapter-buffers", + "num-complex", + "num-integer", + "num-traits", + "realfft", + "visibility", + "windowfunctions", +] + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -5778,6 +5904,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -5902,6 +6042,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + [[package]] name = "same-file" version = "1.0.6" @@ -6532,6 +6681,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "string_cache" version = "0.9.0" @@ -6606,6 +6761,192 @@ dependencies = [ "serde_json", ] +[[package]] +name = "symphonia" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1758d6c853020a7244de03cc3e0185eaea3f58715122422dd3cc7452e6d4c16a" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-caf", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee69ad01236a67260b82fd1ff9790dd75ead29f2f46af145e63b7e72273e0e03" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350f1f2f2e19ad4dd315db94304d1eb361b29af070681f94e51b8fdaad769546" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1979c515a76371b186aad2feff5f23e21cbec775bf95de08bf1e3af92a2ad76" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a149cbfc7fb5c405d123a273227d31de17138419552112bf1aa7b73e65827b8" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50baee168f0e9dcf6ba7fc06e8b57eb62072a4490cc7cf13af77e72baae5d328" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45b07b4423cd8e0fc472575909a5554b12c2f58e3c190b38c24f042e732fd8de" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8257891ffa7f05e02b58f4761e2abf7e5278c8744fd59e981559e050f86eef55" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ec293b5f288383b72a7bffcade6b2860b642cf66f28b3bd5967349a49938b1" +dependencies = [ + "bitflags 2.11.1", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "smallvec", +] + +[[package]] +name = "symphonia-format-caf" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde3ca76633d3400ab57195456c09f8a58d775ff5452329f3f212b6efc8622f5" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d179a01305b3505940135a9f0180d6ef4b487912748fe97554756f120fbd05e" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb17713e134f5ad316c2690fa3104590ccc85842cdbcf82c3cd1a845cb08aa74" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05a67e02b1e4fca1a261ba4fe06910a9357489ad8c36aafdd2960e9c6559433" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17424452a777666d3eaf09a5c651029b15b6a333812fcc5b5474f2a3f0cff3f0" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31acf5cd623398a6208e2225d18f4b20f761c55098a796a5247ad516a4a8681" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" @@ -7356,6 +7697,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] @@ -7603,6 +7945,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tray-icon" version = "0.24.1" @@ -7896,6 +8248,17 @@ version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "vswhom" version = "0.1.0" @@ -8174,6 +8537,15 @@ dependencies = [ "system-deps", ] +[[package]] +name = "webm-iterable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9fbf173b4b38f2f8bbb0082a0d4cb21f263a70811f5fccb1663c421c66d9f9" +dependencies = [ + "ebml-iterable", +] + [[package]] name = "webpki-root-certs" version = "1.0.4" @@ -8248,6 +8620,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi" version = "0.3.9" @@ -8294,6 +8676,15 @@ dependencies = [ "windows-version", ] +[[package]] +name = "windowfunctions" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a" +dependencies = [ + "num-traits", +] + [[package]] name = "windows" version = "0.61.3" diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index ec61e3d2..e1480599 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -20,7 +20,7 @@ serde_json = "1.0.150" keyring-core = "1.0.0" arboard = "3.6.1" tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "macros", "process"] } -tokio-stream = "0.1.18" +tokio-stream = { version = "0.1.18", features = ["sync"] } futures = "0.3.32" async-stream = "0.3.6" flexi_logger = "0.31.9" @@ -39,6 +39,10 @@ hmac = "0.13.0" sha2 = "0.11.0" rcgen = { version = "0.14.8", features = ["pem"] } file-format = "0.29.0" +symphonia = { version = "0.6", default-features = false, features = ["aac", "aiff", "alac", "caf", "flac", "isomp4", "mkv", "mp1", "mp2", "mp3", "ogg", "pcm", "vorbis", "wav"] } +ropus = "=0.12.18" +rubato = { version = "4", default-features = false, features = ["fft_resampler"] } +webm-iterable = "0.6.4" calamine = "0.36.0" pdfium-render = "0.9.1" sys-locale = "0.3.2" diff --git a/runtime/resources/notices/THIRD_PARTY_MEDIA_NOTICES.md b/runtime/resources/notices/THIRD_PARTY_MEDIA_NOTICES.md new file mode 100644 index 00000000..f995d4f7 --- /dev/null +++ b/runtime/resources/notices/THIRD_PARTY_MEDIA_NOTICES.md @@ -0,0 +1,162 @@ +# Media pipeline third-party notices + +These notices are bundled offline with MindWork AI Studio. + +## Symphonia 0.6.0 + +Copyright (c) 2019-2026 The Project Symphonia Developers. + +MindWork AI Studio uses the unmodified Symphonia 0.6.0 crates. The exact corresponding source is: + +- https://github.com/pdeljanov/Symphonia/tree/v0.6.0 +- https://crates.io/api/v1/crates/symphonia/0.6.0/download + +If a future AI Studio release modifies MPL-covered Symphonia files, those modifications must be identified and made available separately under MPL-2.0. No such modifications are present in this release. + +Mozilla Public License Version 2.0 + +1. Definitions + +1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. + +1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” means Covered Software of a particular Contributor. + +1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. + +1.5. “Incompatible With Secondary Licenses” means that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. + +1.6. “Executable Form” means any form of the work other than Source Code Form. + +1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. + +1.8. “License” means this document. + +1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. + +1.10. “Modifications” means any of the following: any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. + +1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” means the form of the work preferred for making modifications. + +1.14. “You” means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. “Control” means ownership of more than fifty percent of the outstanding shares or beneficial ownership of such entity. + +2. License Grants and Conditions + +2.1. Grants. Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. + +2.2. Effective Date. The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. + +2.3. Limitations on Grant Scope. No license is granted in the trademarks, service marks, or logos of any Contributor. Except as otherwise provided in this License, no Contributor grants additional rights by implication, estoppel, or otherwise. + +2.4. Subsequent Licenses. No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License or the terms of a Secondary License. + +2.5. Representation. Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use. This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions. Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. + +3. Responsibilities + +3.1. Distribution of Source Form. All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form. If You distribute Covered Software in Executable Form then such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work. You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). + +3.4. Notices. You may not remove or alter the substance of any license notices contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. You must include a copy of this License with every copy of the Covered Software You distribute. You may add additional accurate notices of copyright ownership. + +3.5. Application of Additional Terms. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. + +4. Inability to Comply Due to Statute or Regulation + +If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must comply with the terms of this License to the maximum extent possible and describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent infringement claim alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 will terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. + +6. Disclaimer of Warranty + +Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. + +7. Limitation of Liability + +Under no circumstances and under no legal theory, whether tort, contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. + +8. Litigation + +Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + +This License represents the complete agreement concerning the subject matter hereof. If any provision is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. + +10. Versions of the License + +10.1. New Versions. Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. + +10.2. Effect of New Versions. You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. + +10.3. Modified Versions. If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses. If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + +This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + +This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. + +## Ropus 0.12.18 + +Copyright 2001-2023 Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo, Mozilla, Amazon + +Copyright (c) 2026 Martin Davidson (Rust port additions) + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the names of specific contributors, may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Opus is subject to the royalty-free patent licenses specified at: + +- Xiph.Org Foundation: https://datatracker.ietf.org/ipr/1524/ +- Microsoft Corporation: https://datatracker.ietf.org/ipr/1914/ +- Broadcom Corporation: https://datatracker.ietf.org/ipr/1526/ + +## Rubato 4.0.0 (MIT option) + +Copyright (c) 2020 Henrik Enquist + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +## webm-iterable 0.6.4 + +MIT License + +Copyright (c) 2021 Austin Blake + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index ac9f9250..353c808e 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -11,6 +11,7 @@ pub mod runtime_api; pub mod runtime_certificate; pub mod file_data; pub mod metadata; +pub mod media; pub mod pdfium; pub mod pandoc; pub mod qdrant_edge_database; diff --git a/runtime/src/media.rs b/runtime/src/media.rs new file mode 100644 index 00000000..641ef37d --- /dev/null +++ b/runtime/src/media.rs @@ -0,0 +1,745 @@ +use std::collections::HashMap; +use std::convert::Infallible; +use std::fs::{self, File}; +use std::path::{Path as FilePath, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; + +use axum::extract::Path; +use axum::http::StatusCode; +use axum::response::sse::{Event, KeepAlive, Sse}; +use axum::response::IntoResponse; +use axum::Json; +use file_format::{FileFormat, Kind}; +use futures::Stream; +use once_cell::sync::Lazy; +use ropus::{Application, Bitrate, Channels as OpusChannels, DecodeMode, Decoder as OpusDecoder, Encoder as OpusEncoder}; +use rubato::audioadapter::Adapter; +use rubato::audioadapter_buffers::direct::SequentialSliceOfVecs; +use rubato::{Fft, FixedSync, Resampler}; +use serde::{Deserialize, Serialize}; +use symphonia::core::audio::sample::Sample; +use symphonia::core::codecs::audio::{well_known::CODEC_ID_OPUS, AudioDecoder, AudioDecoderOptions}; +use symphonia::core::codecs::CodecParameters; +use symphonia::core::errors::Error as SymphoniaError; +use symphonia::core::formats::{FormatOptions, Track, TrackFlags, TrackType}; +use symphonia::core::io::MediaSourceStream; +use symphonia::core::meta::MetadataOptions; +use symphonia::core::formats::probe::Hint; +use symphonia::core::units::Timestamp; +use tokio::sync::broadcast; +use tokio_stream::wrappers::BroadcastStream; +use tokio_stream::StreamExt; +use webm_iterable::matroska_spec::{Master, MatroskaSpec, SimpleBlock}; +use webm_iterable::{WebmIterator, WebmWriter, WriteOptions}; + +use crate::api_token::APIToken; + +const OUTPUT_SAMPLE_RATE: u32 = 48_000; +const OPUS_FRAME_SAMPLES: usize = 960; +const OPUS_BITRATE: u32 = 32_000; +const CLUSTER_DURATION_MS: u64 = 30_000; +const OPUS_PRE_SKIP: u16 = 312; +const DEFAULT_MAX_PASS_THROUGH_BYTES: u64 = 25 * 1024 * 1024; + +static JOBS: Lazy>>> = Lazy::new(|| RwLock::new(HashMap::new())); + +#[derive(Debug, Deserialize)] +pub struct CreateMediaJobRequest { + pub input_path: String, + pub output_path: Option, + pub max_pass_through_bytes: Option, +} + +#[derive(Debug, Serialize)] +pub struct CreateMediaJobResponse { + pub job_id: String, +} + +#[derive(Clone, Debug, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MediaJobPhase { + Probing, + Transcoding, + Completed, + Failed, + Cancelled, +} + +#[derive(Clone, Debug, Serialize)] +pub struct MediaJobEvent { + pub phase: MediaJobPhase, + pub progress: Option, + pub result: Option, + pub error: Option, +} + +#[derive(Clone, Debug, Serialize)] +pub struct MediaJobResult { + pub output_path: String, + pub detected_format: String, + pub detected_codec: String, + pub duration_ms: u64, + pub pass_through: bool, +} + +#[derive(Clone, Debug, Serialize)] +pub struct MediaError { + pub code: String, + pub message: String, +} + +impl MediaError { + fn new(code: &str, message: impl Into) -> Self { + Self { code: code.to_string(), message: message.into() } + } +} + +struct MediaJob { + cancelled: AtomicBool, + current: Mutex, + events: broadcast::Sender, +} + +impl MediaJob { + fn new() -> Self { + let initial = MediaJobEvent { + phase: MediaJobPhase::Probing, + progress: Some(0.0), + result: None, + error: None, + }; + + let (events, _) = broadcast::channel(32); + Self { cancelled: AtomicBool::new(false), current: Mutex::new(initial), events } + } + + fn publish(&self, event: MediaJobEvent) { + *self.current.lock().unwrap() = event.clone(); + let _ = self.events.send(event); + } + + fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Relaxed) + } +} + +pub async fn create_job( + _token: APIToken, + Json(request): Json, +) -> Result, (StatusCode, Json)> { + let input_path = PathBuf::from(&request.input_path); + if !input_path.is_file() { + return Err((StatusCode::BAD_REQUEST, Json(MediaError::new("file_not_found", "The selected media file does not exist.")))); + } + + let output_path = request.output_path.map(PathBuf::from).unwrap_or_else(|| { + let parent = input_path.parent().unwrap_or_else(|| FilePath::new(".")); + let stem = input_path.file_stem().and_then(|value| value.to_str()).unwrap_or("media"); + parent.join(format!("{stem}-normalized.webm")) + }); + + let job_id = format!("{}-{}", std::process::id(), rand::random::()); + let job = Arc::new(MediaJob::new()); + JOBS.write().unwrap().insert(job_id.clone(), Arc::clone(&job)); + let completed_job_id = job_id.clone(); + + tauri::async_runtime::spawn(async move { + let max_pass_through_bytes = request.max_pass_through_bytes.unwrap_or(DEFAULT_MAX_PASS_THROUGH_BYTES); + let task_job = Arc::clone(&job); + let result = tokio::task::spawn_blocking(move || normalize_media(&input_path, &output_path, max_pass_through_bytes, &task_job)).await; + match result { + Ok(Ok(result)) => job.publish(MediaJobEvent { + phase: MediaJobPhase::Completed, + progress: Some(1.0), + result: Some(result), + error: None, + }), + Ok(Err(error)) if error.code == "cancelled" => job.publish(MediaJobEvent { + phase: MediaJobPhase::Cancelled, + progress: None, + result: None, + error: None, + }), + Ok(Err(error)) => job.publish(MediaJobEvent { + phase: MediaJobPhase::Failed, + progress: None, + result: None, + error: Some(error), + }), + Err(error) => job.publish(MediaJobEvent { + phase: MediaJobPhase::Failed, + progress: None, + result: None, + error: Some(MediaError::new("internal_error", format!("The media worker failed: {error}"))), + }), + } + tokio::time::sleep(std::time::Duration::from_secs(600)).await; + JOBS.write().unwrap().remove(&completed_job_id); + }); + + Ok(Json(CreateMediaJobResponse { job_id })) +} + +pub async fn get_job_events( + _token: APIToken, + Path(job_id): Path, +) -> Result>>, StatusCode> { + let job = JOBS.read().unwrap().get(&job_id).cloned().ok_or(StatusCode::NOT_FOUND)?; + let current = job.current.lock().unwrap().clone(); + let initial = tokio_stream::once(current); + let updates = BroadcastStream::new(job.events.subscribe()).filter_map(|event| event.ok()); + let stream = initial.chain(updates).map(|event| { + let data = serde_json::to_string(&event).unwrap_or_else(|_| "{}".to_string()); + Ok(Event::default().event(phase_name(&event.phase)).data(data)) + }); + + Ok(Sse::new(stream).keep_alive(KeepAlive::default())) +} + +pub async fn cancel_job(_token: APIToken, Path(job_id): Path) -> impl IntoResponse { + match JOBS.read().unwrap().get(&job_id) { + Some(job) => { + job.cancelled.store(true, Ordering::Relaxed); + StatusCode::NO_CONTENT + } + + None => StatusCode::NOT_FOUND, + } +} + +fn phase_name(phase: &MediaJobPhase) -> &'static str { + match phase { + MediaJobPhase::Probing => "probing", + MediaJobPhase::Transcoding => "transcoding", + MediaJobPhase::Completed => "completed", + MediaJobPhase::Failed => "failed", + MediaJobPhase::Cancelled => "cancelled", + } +} + +fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_through_bytes: u64, job: &MediaJob) -> Result { + check_cancelled(job)?; + + let detected = FileFormat::from_file(input_path) + .map_err(|error| MediaError::new("unknown_format", format!("The file type could not be identified: {error}")))?; + + if detected.kind() == Kind::Executable { + return Err(MediaError::new("unsafe_file", "The selected file contains executable data and cannot be processed as media.")); + } + + if !matches!(detected.kind(), Kind::Audio | Kind::Video) + && !matches!(detected, FileFormat::ExtensibleBinaryMetaLanguage) + { + return Err(MediaError::new("not_media", format!("The selected file is not supported media (detected as {detected:?})."))); + } + + let file = File::open(input_path).map_err(|error| MediaError::new("file_open_failed", error.to_string()))?; + let mss = MediaSourceStream::new(Box::new(file), Default::default()); + + let mut hint = Hint::new(); + if let Some(extension) = input_path.extension().and_then(|value| value.to_str()) { + hint.with_extension(extension); + } + + let mut format = symphonia::default::get_probe() + .probe(&hint, mss, FormatOptions::default(), MetadataOptions::default()) + .map_err(map_probe_error)?; + + let detected_format = format!("{detected:?} / {}", format.format_info().long_name); + + let tracks = format.tracks(); + let mut audio_tracks: Vec<&Track> = tracks.iter().filter(|track| { + matches!(track.codec_params, Some(CodecParameters::Audio(_))) + }).collect(); + + if audio_tracks.is_empty() { + return Err(MediaError::new("no_audio_track", "The selected media file does not contain an audio track.")); + } + + audio_tracks.sort_by_key(|track| !track.flags.contains(TrackFlags::DEFAULT)); + let selected = audio_tracks.into_iter().find(|track| is_decodable(track)) + .ok_or_else(|| MediaError::new("unsupported_codec", "None of the audio tracks uses a supported codec."))?; + + let track_id = selected.id; + let params = selected.codec_params.as_ref().and_then(CodecParameters::audio).unwrap().clone(); + let detected_codec = if params.codec == CODEC_ID_OPUS { "opus".to_string() } else { format!("{}", params.codec) }; + let duration_ms = selected.time_base.zip(selected.duration).and_then(|(time_base, duration)| { + let timestamp = Timestamp::new(i64::try_from(duration.get()).ok()?); + let time = time_base.calc_time(timestamp)?; + Some((time.as_secs_f64() * 1000.0).max(0.0).round() as u64) + }).unwrap_or(0); + + let audio_only = tracks.iter().all(|track| track.track_type() != Some(TrackType::Video)); + let channels = params.channels.as_ref().map(|value| value.count()).unwrap_or(0); + let pass_through = is_webm_container(input_path) + && audio_only + && tracks.iter().filter(|track| track.track_type() == Some(TrackType::Audio)).count() == 1 + && params.codec == CODEC_ID_OPUS + && params.sample_rate == Some(OUTPUT_SAMPLE_RATE) + && channels == 1 + && input_path.metadata().map(|metadata| metadata.len() <= max_pass_through_bytes).unwrap_or(false); + + let partial_path = partial_path(output_path); + if let Some(parent) = partial_path.parent() { + fs::create_dir_all(parent).map_err(|error| MediaError::new("output_create_failed", error.to_string()))?; + } + + let result = if pass_through { + check_cancelled(job)?; + fs::copy(input_path, &partial_path).map_err(|error| MediaError::new("output_write_failed", error.to_string()))?; + check_cancelled(job)?; + Ok(MediaJobResult { + output_path: output_path.to_string_lossy().into_owned(), + detected_format: detected_format.clone(), + detected_codec, + duration_ms, + pass_through: true, + }) + } else { + job.publish(MediaJobEvent { + phase: MediaJobPhase::Transcoding, + progress: Some(0.0), + result: None, + error: None, + }); + + transcode(&mut *format, track_id, ¶ms, &partial_path, output_path, detected_format, detected_codec, duration_ms, job) + }; + + match result { + Ok(result) => { + if let Err(error) = fs::rename(&partial_path, output_path) { + let _ = fs::remove_file(&partial_path); + return Err(MediaError::new("output_commit_failed", error.to_string())); + } + + Ok(result) + } + + Err(error) => { + let _ = fs::remove_file(&partial_path); + Err(error) + } + } +} + +fn is_decodable(track: &Track) -> bool { + let Some(params) = track.codec_params.as_ref().and_then(CodecParameters::audio) else { return false; }; + params.codec == CODEC_ID_OPUS || symphonia::default::get_codecs().make_audio_decoder(params, &AudioDecoderOptions::default()).is_ok() +} + +fn transcode( + format: &mut dyn symphonia::core::formats::FormatReader, + track_id: u32, + params: &symphonia::core::codecs::audio::AudioCodecParameters, + partial_path: &FilePath, + output_path: &FilePath, + detected_format: String, + detected_codec: String, + expected_duration_ms: u64, + job: &MediaJob, +) -> Result { + let input_rate = params.sample_rate.unwrap_or(OUTPUT_SAMPLE_RATE); + let input_channels = params.channels.as_ref().map(|value| value.count()).unwrap_or(1).max(1); + + let mut symphonia_decoder: Option> = if params.codec == CODEC_ID_OPUS { + None + } else { + Some(symphonia::default::get_codecs().make_audio_decoder(params, &AudioDecoderOptions::default()) + .map_err(|_| MediaError::new("unsupported_codec", "The selected audio codec is not supported."))?) + }; + + let mut opus_decoder = if params.codec == CODEC_ID_OPUS { + let channels = if input_channels == 1 { OpusChannels::Mono } else { OpusChannels::Stereo }; + Some(OpusDecoder::new(OUTPUT_SAMPLE_RATE, channels) + .map_err(|error| MediaError::new("decoder_init_failed", error.to_string()))?) + } else { None }; + + let mut opus_encoder = OpusEncoder::builder(OUTPUT_SAMPLE_RATE, OpusChannels::Mono, Application::Audio) + .bitrate(Bitrate::Bits(OPUS_BITRATE)) + .vbr(true) + .build() + .map_err(|error| MediaError::new("encoder_init_failed", error.to_string()))?; + + let file = File::create(partial_path).map_err(|error| MediaError::new("output_create_failed", error.to_string()))?; + let mut writer = WebmOpusWriter::new(file)?; + let mut pending = Vec::::with_capacity(OPUS_FRAME_SAMPLES * 3); + let mut interleaved = Vec::::new(); + let mut opus_pcm = vec![0i16; 5_760 * input_channels]; + let mut encoded = [0u8; 4_000]; + let mut produced_samples = 0u64; + let mut input_samples = 0u64; + let mut resampled_samples = 0u64; + + loop { + check_cancelled(job)?; + let packet = match format.next_packet() { + Ok(Some(packet)) => packet, + Ok(None) => break, + + Err(SymphoniaError::ResetRequired) => return Err(MediaError::new("stream_reset", "The media stream changed unexpectedly.")), + Err(SymphoniaError::IoError(error)) if error.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(error) => return Err(MediaError::new("damaged_container", format!("The media container is damaged: {error}"))), + }; + + if packet.track_id != track_id { + continue; + } + + let mono = if let Some(decoder) = symphonia_decoder.as_mut() { + let decoded = match decoder.decode(&packet) { + Ok(decoded) => decoded, + Err(SymphoniaError::DecodeError(_)) => continue, + Err(error) => return Err(MediaError::new("decode_failed", format!("Audio decoding failed: {error}"))), + }; + + interleaved.resize(decoded.samples_interleaved(), f32::MID); + decoded.copy_to_slice_interleaved(&mut interleaved); + downmix_to_mono(&interleaved, decoded.num_planes()) + } else { + let decoder = opus_decoder.as_mut().unwrap(); + let frames = decoder.decode(&packet.data, &mut opus_pcm, DecodeMode::Normal) + .map_err(|error| MediaError::new("decode_failed", error.to_string()))?; + + let samples = &opus_pcm[..frames * input_channels]; + if input_channels == 1 { + samples.iter().map(|sample| f32::from(*sample) / 32768.0).collect() + } else { + samples.chunks_exact(input_channels).map(|frame| { + frame.iter().map(|sample| f32::from(*sample) / 32768.0).sum::() / input_channels as f32 + }).collect() + } + }; + + input_samples += mono.len() as u64; + let expected_resampled_samples = input_samples.saturating_mul(u64::from(OUTPUT_SAMPLE_RATE)) / u64::from(input_rate); + let expected_chunk_samples = expected_resampled_samples.saturating_sub(resampled_samples) as usize; + let mut resampled = resample_chunk(&mono, input_rate)?; + resampled.resize(expected_chunk_samples, 0.0); + resampled.truncate(expected_chunk_samples); + resampled_samples += resampled.len() as u64; + pending.extend_from_slice(&resampled); + + let mut consumed = 0usize; + while pending.len() - consumed >= OPUS_FRAME_SAMPLES { + check_cancelled(job)?; + let frame = &pending[consumed..consumed + OPUS_FRAME_SAMPLES]; + let length = opus_encoder.encode_float(frame, &mut encoded) + .map_err(|error| MediaError::new("encode_failed", error.to_string()))?; + + writer.write_packet(&encoded[..length], produced_samples)?; + produced_samples += OPUS_FRAME_SAMPLES as u64; + consumed += OPUS_FRAME_SAMPLES; + } + + if consumed > 0 { + pending.drain(..consumed); + + } + + if expected_duration_ms > 0 { + let current_ms = produced_samples.saturating_mul(1000) / u64::from(OUTPUT_SAMPLE_RATE); + job.publish(MediaJobEvent { + phase: MediaJobPhase::Transcoding, + progress: Some((current_ms as f64 / expected_duration_ms as f64).clamp(0.0, 0.99)), + result: None, + error: None, + }); + } + } + + if !pending.is_empty() { + pending.resize(OPUS_FRAME_SAMPLES, 0.0); + let length = opus_encoder.encode_float(&pending, &mut encoded) + .map_err(|error| MediaError::new("encode_failed", error.to_string()))?; + writer.write_packet(&encoded[..length], produced_samples)?; + produced_samples += OPUS_FRAME_SAMPLES as u64; + } + + writer.finish()?; + check_cancelled(job)?; + + Ok(MediaJobResult { + output_path: output_path.to_string_lossy().into_owned(), + detected_format, + detected_codec, + duration_ms: produced_samples.saturating_mul(1000) / u64::from(OUTPUT_SAMPLE_RATE), + pass_through: false, + }) +} + +fn downmix_to_mono(samples: &[f32], channels: usize) -> Vec { + if channels <= 1 { + return samples.to_vec(); + } + + samples.chunks_exact(channels).map(|frame| frame.iter().copied().sum::() / channels as f32).collect() +} + +fn resample_chunk(samples: &[f32], input_rate: u32) -> Result, MediaError> { + if input_rate == OUTPUT_SAMPLE_RATE || samples.is_empty() { + return Ok(samples.to_vec()); + } + + let input_data = vec![samples.to_vec()]; + let input = SequentialSliceOfVecs::new(&input_data, 1, samples.len()) + .map_err(|error| MediaError::new("resample_failed", error.to_string()))?; + + let mut resampler = Fft::::new(input_rate as usize, OUTPUT_SAMPLE_RATE as usize, samples.len().max(256), 1, FixedSync::Input) + .map_err(|error| MediaError::new("resample_failed", error.to_string()))?; + + let output = resampler.process_all(&input, samples.len(), None) + .map_err(|error| MediaError::new("resample_failed", error.to_string()))?; + + let mut result = Vec::with_capacity(output.frames()); + for frame in 0..output.frames() { + result.push(output.read_sample(0, frame).unwrap_or(0.0)); + } + + Ok(result) +} + +struct WebmOpusWriter { + writer: WebmWriter, + cluster_start_ms: Option, +} + +impl WebmOpusWriter { + fn new(file: File) -> Result { + let mut writer = WebmWriter::new(file); + write_tags(&mut writer, &[ + MatroskaSpec::Ebml(Master::Start), + MatroskaSpec::EbmlVersion(1), + MatroskaSpec::EbmlReadVersion(1), + MatroskaSpec::EbmlMaxIdLength(4), + MatroskaSpec::EbmlMaxSizeLength(8), + MatroskaSpec::DocType("webm".to_string()), + MatroskaSpec::DocTypeVersion(4), + MatroskaSpec::DocTypeReadVersion(2), + MatroskaSpec::Ebml(Master::End), + ])?; + + writer.write_advanced(&MatroskaSpec::Segment(Master::Start), WriteOptions::is_unknown_sized_element()).map_err(webm_error)?; + + write_tags(&mut writer, &[ + MatroskaSpec::Info(Master::Start), + MatroskaSpec::TimestampScale(1_000_000), + MatroskaSpec::MuxingApp("MindWork AI Studio".to_string()), + MatroskaSpec::WritingApp("MindWork AI Studio".to_string()), + MatroskaSpec::Info(Master::End), + MatroskaSpec::Tracks(Master::Start), + MatroskaSpec::TrackEntry(Master::Start), + MatroskaSpec::TrackNumber(1), + MatroskaSpec::TrackUID(1), + MatroskaSpec::TrackType(2), + MatroskaSpec::FlagDefault(1), + MatroskaSpec::CodecID("A_OPUS".to_string()), + MatroskaSpec::CodecPrivate(opus_head()), + MatroskaSpec::CodecDelay(u64::from(OPUS_PRE_SKIP) * 1_000_000_000 / u64::from(OUTPUT_SAMPLE_RATE)), + MatroskaSpec::SeekPreRoll(80_000_000), + MatroskaSpec::Audio(Master::Start), + MatroskaSpec::SamplingFrequency(f64::from(OUTPUT_SAMPLE_RATE)), + MatroskaSpec::Channels(1), + MatroskaSpec::Audio(Master::End), + MatroskaSpec::TrackEntry(Master::End), + MatroskaSpec::Tracks(Master::End), + ])?; + + Ok(Self { writer, cluster_start_ms: None }) + } + + fn write_packet(&mut self, packet: &[u8], sample_position: u64) -> Result<(), MediaError> { + let timestamp_ms = sample_position.saturating_mul(1000) / u64::from(OUTPUT_SAMPLE_RATE); + let rotate = self.cluster_start_ms.map(|start| timestamp_ms.saturating_sub(start) >= CLUSTER_DURATION_MS).unwrap_or(true); + + if rotate { + if self.cluster_start_ms.is_some() { + self.writer.write(&MatroskaSpec::Cluster(Master::End)).map_err(webm_error)?; + } + + self.writer.write(&MatroskaSpec::Cluster(Master::Start)).map_err(webm_error)?; + self.writer.write(&MatroskaSpec::Timestamp(timestamp_ms)).map_err(webm_error)?; + self.cluster_start_ms = Some(timestamp_ms); + } + + let relative = timestamp_ms.saturating_sub(self.cluster_start_ms.unwrap_or(timestamp_ms)); + if relative > i16::MAX as u64 { + return Err(MediaError::new("webm_timestamp_overflow", "The WebM cluster timestamp exceeded its safe range.")); + } + + let block: MatroskaSpec = SimpleBlock::new_uncheked(packet, 1, relative as i16, false, None, false, true).into(); + self.writer.write(&block).map_err(webm_error) + } + + fn finish(mut self) -> Result<(), MediaError> { + if self.cluster_start_ms.is_some() { + self.writer.write(&MatroskaSpec::Cluster(Master::End)).map_err(webm_error)?; + } + + self.writer.write(&MatroskaSpec::Segment(Master::End)).map_err(webm_error)?; + self.writer.into_inner().map_err(webm_error)?; + Ok(()) + } +} + +fn write_tags(writer: &mut WebmWriter, tags: &[MatroskaSpec]) -> Result<(), MediaError> { + for tag in tags { + writer.write(tag).map_err(webm_error)?; + } + + Ok(()) +} + +fn opus_head() -> Vec { + let mut data = b"OpusHead".to_vec(); + data.push(1); + data.push(1); + data.extend_from_slice(&OPUS_PRE_SKIP.to_le_bytes()); + data.extend_from_slice(&OUTPUT_SAMPLE_RATE.to_le_bytes()); + data.extend_from_slice(&0i16.to_le_bytes()); + data.push(0); + data +} + +fn partial_path(output_path: &FilePath) -> PathBuf { + let mut name = output_path.file_name().unwrap_or_default().to_os_string(); + name.push(".partial"); + output_path.with_file_name(name) +} + +fn is_webm_container(path: &FilePath) -> bool { + let Ok(file) = File::open(path) else { return false; }; + WebmIterator::new(file, &[]).take(16).filter_map(Result::ok).any(|tag| { + matches!(tag, MatroskaSpec::DocType(doc_type) if doc_type.eq_ignore_ascii_case("webm")) + }) +} + +fn check_cancelled(job: &MediaJob) -> Result<(), MediaError> { + if job.is_cancelled() { + Err(MediaError::new("cancelled", "The media job was cancelled.")) + } else { + Ok(()) + } +} + +fn map_probe_error(error: SymphoniaError) -> MediaError { + match error { + SymphoniaError::Unsupported(_) => MediaError::new("unsupported_container", "This media container or codec is not supported."), + _ => MediaError::new("damaged_container", format!("The media container could not be read: {error}")), + } +} + +fn webm_error(error: impl std::fmt::Display) -> MediaError { + MediaError::new("webm_write_failed", format!("The WebM output could not be written: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + + #[test] + fn opus_head_describes_48_khz_mono() { + let head = opus_head(); + assert_eq!(&head[..8], b"OpusHead"); + assert_eq!(head[9], 1); + assert_eq!(u32::from_le_bytes(head[12..16].try_into().unwrap()), 48_000); + } + + #[test] + fn webm_writer_rotates_clusters_before_relative_timestamp_overflow() { + let path = std::env::temp_dir().join(format!("ai-studio-media-writer-{}.webm", rand::random::())); + let file = File::create(&path).unwrap(); + let mut writer = WebmOpusWriter::new(file).unwrap(); + writer.write_packet(&[0xf8, 0xff, 0xfe], 0).unwrap(); + writer.write_packet(&[0xf8, 0xff, 0xfe], 31 * 48_000).unwrap(); + writer.finish().unwrap(); + let bytes = fs::read(&path).unwrap(); + let clusters = WebmIterator::new(Cursor::new(bytes), &[]) + .filter_map(Result::ok) + .filter(|tag| matches!(tag, MatroskaSpec::Cluster(Master::Start))) + .count(); + let _ = fs::remove_file(path); + assert_eq!(clusters, 2); + } + + #[test] + fn downmix_is_bounded_and_balanced() { + assert_eq!(downmix_to_mono(&[1.0, -1.0, 0.5, 0.5], 2), vec![0.0, 0.5]); + } + + #[test] + fn wav_is_normalized_to_one_mono_opus_track_with_frame_bounded_duration() { + let directory = std::env::temp_dir().join(format!("ai-studio-media-test-{}", rand::random::())); + fs::create_dir_all(&directory).unwrap(); + let input = directory.join("input.wav"); + let output = directory.join("output.webm"); + fs::write(&input, wav_silence(44_100, 4_410)).unwrap(); + let job = MediaJob::new(); + let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap(); + assert!(!result.pass_through); + assert!(result.duration_ms.abs_diff(100) <= 20); + + let file = File::open(&output).unwrap(); + let tags: Vec<_> = WebmIterator::new(file, &[]) + .filter_map(Result::ok) + .collect(); + assert_eq!(tags.iter().filter(|tag| matches!(tag, MatroskaSpec::TrackEntry(Master::Start))).count(), 1); + assert!(tags.iter().any(|tag| matches!(tag, MatroskaSpec::CodecID(codec) if codec == "A_OPUS"))); + assert!(tags.iter().any(|tag| matches!(tag, MatroskaSpec::Channels(1)))); + assert!(tags.iter().any(|tag| matches!(tag, MatroskaSpec::SamplingFrequency(rate) if *rate == 48_000.0))); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn cancellation_does_not_leave_an_output_file() { + let directory = std::env::temp_dir().join(format!("ai-studio-media-cancel-{}", rand::random::())); + fs::create_dir_all(&directory).unwrap(); + let input = directory.join("input.wav"); + let output = directory.join("output.webm"); + fs::write(&input, wav_silence(48_000, 960)).unwrap(); + let job = MediaJob::new(); + job.cancelled.store(true, Ordering::Relaxed); + let error = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap_err(); + assert_eq!(error.code, "cancelled"); + assert!(!output.exists()); + assert!(!partial_path(&output).exists()); + let _ = fs::remove_dir_all(directory); + } + + #[test] + fn suitable_audio_only_webm_opus_is_passed_through() { + let directory = std::env::temp_dir().join(format!("ai-studio-media-pass-through-{}", rand::random::())); + fs::create_dir_all(&directory).unwrap(); + let input = directory.join("input.webm"); + let output = directory.join("output.webm"); + let mut writer = WebmOpusWriter::new(File::create(&input).unwrap()).unwrap(); + writer.write_packet(&[0xf8, 0xff, 0xfe], 0).unwrap(); + writer.finish().unwrap(); + let job = MediaJob::new(); + let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap(); + assert!(result.pass_through); + assert_eq!(fs::read(input).unwrap(), fs::read(output).unwrap()); + let _ = fs::remove_dir_all(directory); + } + + fn wav_silence(sample_rate: u32, samples: u32) -> Vec { + let data_size = samples * 2; + let mut wav = Vec::with_capacity(44 + data_size as usize); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(36 + data_size).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16u32.to_le_bytes()); + wav.extend_from_slice(&1u16.to_le_bytes()); + wav.extend_from_slice(&1u16.to_le_bytes()); + wav.extend_from_slice(&sample_rate.to_le_bytes()); + wav.extend_from_slice(&(sample_rate * 2).to_le_bytes()); + wav.extend_from_slice(&2u16.to_le_bytes()); + wav.extend_from_slice(&16u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_size.to_le_bytes()); + wav.resize(44 + data_size as usize, 0); + wav + } +} \ No newline at end of file diff --git a/runtime/src/runtime_api.rs b/runtime/src/runtime_api.rs index 087c0ffd..c7de61d4 100644 --- a/runtime/src/runtime_api.rs +++ b/runtime/src/runtime_api.rs @@ -1,6 +1,6 @@ use log::info; use once_cell::sync::Lazy; -use axum::routing::{get, post}; +use axum::routing::{delete, get, post}; use axum::Router; use axum_server::tls_rustls::RustlsConfig; use std::net::SocketAddr; @@ -59,6 +59,9 @@ pub fn start_runtime_api() { .route("/system/enterprise/config/encryption_secret", get(crate::environment::read_enterprise_env_config_encryption_secret)) .route("/system/enterprise/configs", get(crate::environment::read_enterprise_configs)) .route("/retrieval/fs/extract", get(crate::file_data::extract_data)) + .route("/media/jobs", post(crate::media::create_job)) + .route("/media/jobs/{id}/events", get(crate::media::get_job_events)) + .route("/media/jobs/{id}", delete(crate::media::cancel_job)) .route("/log/paths", get(crate::log::get_log_paths)) .route("/log/event", post(crate::log::log_event)) .route("/shortcuts/register", post(crate::app_window::register_shortcut)) diff --git a/runtime/tauri.conf.json b/runtime/tauri.conf.json index 78849800..4fc34089 100644 --- a/runtime/tauri.conf.json +++ b/runtime/tauri.conf.json @@ -27,7 +27,8 @@ "../app/MindWork AI Studio/bin/dist/mindworkAIStudioServer" ], "resources": [ - "resources/libraries/*" + "resources/libraries/*", + "resources/notices/*" ], "macOS": { "exceptionDomain": "localhost"