From 8aaa2dc40cac5a0e69eec738051c2c10859d0da7 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Tue, 14 Jul 2026 09:00:30 +0200 Subject: [PATCH] Rework after first review --- .../Assistants/AssistantBase.razor.cs | 8 + .../Assistants/I18N/allTexts.lua | 96 +- .../Chat/ManagedTranscriptAttachment.cs | 168 ++- .../Components/AttachDocuments.razor.cs | 41 +- .../Components/ChatComponent.razor | 4 +- .../Components/ChatComponent.razor.cs | 6 + .../MediaTranscriptionStatus.razor.cs | 7 +- .../Components/ReadFileContent.razor.cs | 23 +- .../Components/VoiceRecorder.razor.cs | 13 +- .../Components/Workspaces.razor.cs | 7 +- .../Tools/Rust/CreateMediaJobRequest.cs | 4 + .../Tools/Rust/CreateMediaJobResponse.cs | 2 + .../Tools/Rust/MediaJobError.cs | 7 +- .../Tools/Rust/MediaJobErrorCode.cs | 85 ++ .../Tools/Rust/MediaJobEvent.cs | 5 + .../Tools/Rust/MediaJobPhase.cs | 12 + .../Tools/Rust/MediaJobResult.cs | 6 + .../Tools/Services/MediaTranscriptionPhase.cs | 8 + .../Services/MediaTranscriptionResult.cs | 25 + .../MediaTranscriptionResultStatus.cs | 16 + .../Services/MediaTranscriptionService.cs | 396 +++++-- .../Tools/Services/RustService.Media.cs | 16 +- .../TranscriptStagingCleanupService.cs | 49 +- .../Tools/WorkspaceBehaviour.cs | 177 ++- runtime/src/media.rs | 1014 ++++++++++++++--- runtime/tests/fixtures/media/audio-only.webm | Bin 0 -> 1133 bytes runtime/tests/fixtures/media/damaged.bin | 1 + runtime/tests/fixtures/media/no-audio.webm | Bin 0 -> 539 bytes runtime/tests/fixtures/media/sample.aiff | Bin 0 -> 10638 bytes runtime/tests/fixtures/media/sample.caf | Bin 0 -> 10714 bytes runtime/tests/fixtures/media/sample.flac | Bin 0 -> 9811 bytes runtime/tests/fixtures/media/sample.m4a | Bin 0 -> 1449 bytes runtime/tests/fixtures/media/sample.mkv | Bin 0 -> 1222 bytes runtime/tests/fixtures/media/sample.mov | Bin 0 -> 1500 bytes runtime/tests/fixtures/media/sample.mp3 | Bin 0 -> 854 bytes runtime/tests/fixtures/media/sample.mp4 | Bin 0 -> 1469 bytes runtime/tests/fixtures/media/sample.ogg | Bin 0 -> 719 bytes runtime/tests/fixtures/media/sample.wav | Bin 0 -> 10662 bytes runtime/tests/fixtures/media/subtitle.vtt | 4 + runtime/tests/fixtures/media/subtitle.webm | Bin 0 -> 1158 bytes .../tests/fixtures/media/unknown-codec.mkv | Bin 0 -> 1109 bytes runtime/tests/fixtures/media/video.webm | Bin 0 -> 1658 bytes 42 files changed, 1821 insertions(+), 379 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Rust/MediaJobErrorCode.cs create mode 100644 app/MindWork AI Studio/Tools/Services/MediaTranscriptionResult.cs create mode 100644 app/MindWork AI Studio/Tools/Services/MediaTranscriptionResultStatus.cs create mode 100644 runtime/tests/fixtures/media/audio-only.webm create mode 100644 runtime/tests/fixtures/media/damaged.bin create mode 100644 runtime/tests/fixtures/media/no-audio.webm create mode 100644 runtime/tests/fixtures/media/sample.aiff create mode 100644 runtime/tests/fixtures/media/sample.caf create mode 100644 runtime/tests/fixtures/media/sample.flac create mode 100644 runtime/tests/fixtures/media/sample.m4a create mode 100644 runtime/tests/fixtures/media/sample.mkv create mode 100644 runtime/tests/fixtures/media/sample.mov create mode 100644 runtime/tests/fixtures/media/sample.mp3 create mode 100644 runtime/tests/fixtures/media/sample.mp4 create mode 100644 runtime/tests/fixtures/media/sample.ogg create mode 100644 runtime/tests/fixtures/media/sample.wav create mode 100644 runtime/tests/fixtures/media/subtitle.vtt create mode 100644 runtime/tests/fixtures/media/subtitle.webm create mode 100644 runtime/tests/fixtures/media/unknown-codec.mkv create mode 100644 runtime/tests/fixtures/media/video.webm diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs index 535f3866..0804eb13 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs @@ -157,6 +157,7 @@ public abstract partial class AssistantBase : AssistantLowerBase wher protected override async Task OnInitializedAsync() { + this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; await base.OnInitializedAsync(); if (!this.SettingsManager.IsAssistantVisible(this.Component, assistantName: this.Title)) @@ -226,6 +227,9 @@ public abstract partial class AssistantBase : AssistantLowerBase wher private async Task Start() { + if (this.MediaTranscriptionService.IsBusy) + return; + var activeSession = this.AssistantSessionService.TryGetSnapshot(this.assistantSessionKey); if (activeSession?.IsActive ?? false) { @@ -675,6 +679,7 @@ public abstract partial class AssistantBase : AssistantLowerBase wher protected override void DisposeResources() { + this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; this.isDisposed = true; try { @@ -689,6 +694,9 @@ public abstract partial class AssistantBase : AssistantLowerBase wher base.DisposeResources(); } + /// Refreshes assistant actions when the shared import lane changes. + private void OnMediaImportStateChanged() => _ = this.InvokeAsync(this.StateHasChanged); + #endregion #region Assistant sessions diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 3ff17464..25e7e9bd 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -2290,6 +2290,9 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T3219823625"] = "The lo -- The image at the URL is too large (>10 MB). Skipping the image. UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T349928509"] = "The image at the URL is too large (>10 MB). Skipping the image." +-- Transcription: {0} +UI_TEXT_CONTENT["AISTUDIO::CHAT::MANAGEDTRANSCRIPTATTACHMENT::T1004696017"] = "Transcription: {0}" + -- Open Settings UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T1172211894"] = "Open Settings" @@ -2401,6 +2404,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T143112277"] = "Drop fil -- Click here to attach files. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T1875575968"] = "Click here to attach files." +-- Transcribe media files +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T2178031033"] = "Transcribe media files" + -- Drag and drop files into the marked area or click here to attach documents: UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T230755331"] = "Drag and drop files into the marked area or click here to attach documents:" @@ -2410,6 +2416,12 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T2495931372"] = "Select -- Document Preview UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T285154968"] = "Document Preview" +-- Media files require a configured transcription provider. Configure one in the transcription settings. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T3172443094"] = "Media files require a configured transcription provider. Configure one in the transcription settings." + +-- The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T322693339"] = "The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider." + -- Clear file list UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T3759696136"] = "Clear file list" @@ -2671,6 +2683,18 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANDATORYINFODISPLAY::T3511160492"] = "Ac -- Please review this text again. The content was changed. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANDATORYINFODISPLAY::T941885055"] = "Please review this text again. The content was changed." +-- Stop media transcription +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T1868377405"] = "Stop media transcription" + +-- Inspecting media +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T2431421733"] = "Inspecting media" + +-- Transcribing +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T2938661425"] = "Transcribing" + +-- Preparing audio +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T3200155905"] = "Preparing audio" + -- Given that my employer's workplace uses both Windows and Linux, I wanted a cross-platform solution that would work seamlessly across all major operating systems, including macOS. Additionally, I wanted to demonstrate that it is possible to create modern, efficient, cross-platform applications without resorting to Electron bloatware. The combination of .NET and Rust with Tauri proved to be an excellent technology stack for building such robust applications. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MOTIVATION::T1057189794"] = "Given that my employer's workplace uses both Windows and Linux, I wanted a cross-platform solution that would work seamlessly across all major operating systems, including macOS. Additionally, I wanted to demonstrate that it is possible to create modern, efficient, cross-platform applications without resorting to Electron bloatware. The combination of .NET and Rust with Tauri proved to be an excellent technology stack for building such robust applications." @@ -2794,12 +2818,21 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T1989554334"] = "Failed -- Drop one file here to load its content. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T2274562398"] = "Drop one file here to load its content." +-- The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T2839709466"] = "The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider." + +-- Media files require a configured transcription provider. Configure one in the transcription settings. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3172443094"] = "Media files require a configured transcription provider. Configure one in the transcription settings." + -- Use file content as input UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3499386973"] = "Use file content as input" -- Select file to read its content UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T354817589"] = "Select file to read its content" +-- Transcribe media file +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3554222960"] = "Transcribe media file" + -- The content is cleaned using an LLM agent: the main content is extracted, advertisements and other irrelevant things are attempted to be removed; relative links are attempted to be converted into absolute links so that they can be used. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READWEBCONTENT::T1164201762"] = "The content is cleaned using an LLM agent: the main content is extracted, advertisements and other irrelevant things are attempted to be removed; relative links are attempted to be converted into absolute links so that they can be used." @@ -3538,9 +3571,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VISION::T586430036"] = "Useful assistants -- Voice recording has been disabled for this session because audio playback could not be initialized on the client. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T1123032432"] = "Voice recording has been disabled for this session because audio playback could not be initialized on the client." --- Failed to create the transcription provider. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T1689988905"] = "Failed to create the transcription provider." - -- Failed to start audio recording. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T2144994226"] = "Failed to start audio recording." @@ -3559,21 +3589,12 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T2851219233"] = "Transcrip -- Unfortunately, there was an error communicating with the AI system. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3236134591"] = "Unfortunately, there was an error communicating with the AI system." --- The configured transcription provider was not found. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T331613105"] = "The configured transcription provider was not found." - -- Failed to stop audio recording. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3462568264"] = "Failed to stop audio recording." --- The configured transcription provider does not meet the minimum confidence level. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3834149033"] = "The configured transcription provider does not meet the minimum confidence level." - -- An error occurred during transcription. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T588743762"] = "An error occurred during transcription." --- No transcription provider is configured. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T663630295"] = "No transcription provider is configured." - -- The transcription result is empty. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T974954792"] = "The transcription result is empty." @@ -6748,9 +6769,6 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2037899437"] = "Copies the serve -- This library is used to create temporary folders in runtime tests and supporting filesystem operations. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2160280545"] = "This library is used to create temporary folders in runtime tests and supporting filesystem operations." --- This library is used to determine the file type of a file. This is necessary, e.g., when we want to stream a file. -UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2173617769"] = "This library is used to determine the file type of a file. This is necessary, e.g., when we want to stream a file." - -- For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2174764529"] = "For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose." @@ -6769,6 +6787,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2301484629"] = "Configuration pl -- The C# language is used for the implementation of the user interface and the backend. To implement the user interface with C#, the Blazor technology from ASP.NET Core is used. All these technologies are integrated into the .NET SDK. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2329884315"] = "The C# language is used for the implementation of the user interface and the backend. To implement the user interface with C#, the Blazor technology from ASP.NET Core is used. All these technologies are integrated into the .NET SDK." +-- We use Rubato to resample the decoded audio to 48 kHz before the Opus encoding. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2345444286"] = "We use Rubato to resample the decoded audio to 48 kHz before the Opus encoding." + -- Linux AppImages bundle GStreamer components to support microphone access and WebM audio recording in the embedded WebKitGTK web view. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T234598990"] = "Linux AppImages bundle GStreamer components to support microphone access and WebM audio recording in the embedded WebKitGTK web view." @@ -6850,6 +6871,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2929232062"] = "Copies the confi -- Copies the root certificate fingerprint to the clipboard UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2989678330"] = "Copies the root certificate fingerprint to the clipboard" +-- This library identifies files by their content. It is used for document streaming and as the first safety and media classification step before local audio processing. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3002755581"] = "This library identifies files by their content. It is used for document streaming and as the first safety and media classification step before local audio processing." + -- Changelog UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3017574265"] = "Changelog" @@ -6895,6 +6919,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3315279770"] = "External HTTPS c -- User-language provided by the OS UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3334355246"] = "User-language provided by the OS" +-- webm-iterable provides the EBML and WebM writing path for normalized audio. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3385332793"] = "webm-iterable provides the EBML and WebM writing path for normalized audio." + -- Status: UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3396815215"] = "Status:" @@ -6955,6 +6982,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T396609403"] = "not applicable" -- Copies the allowed host configuration to the clipboard UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3970230163"] = "Copies the allowed host configuration to the clipboard" +-- Symphonia is used for media container demuxing and audio decoding. The exact MPL-covered source is available from the repository linked and is identified in the offline notices bundled with AI Studio. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3971563979"] = "Symphonia is used for media container demuxing and audio decoding. The exact MPL-covered source is available from the repository linked and is identified in the offline notices bundled with AI Studio." + -- Installed Pandoc version UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3983971016"] = "Installed Pandoc version" @@ -6973,6 +7003,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4060906280"] = "This library is -- This library is used to create asynchronous streams in Rust. It allows us to work with streams of data that can be produced asynchronously, making it easier to handle events or data that arrive over time. We use this, e.g., to stream arbitrary data from the file system to the embedding system. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4079152443"] = "This library is used to create asynchronous streams in Rust. It allows us to work with streams of data that can be produced asynchronously, making it easier to handle events or data that arrive over time. We use this, e.g., to stream arbitrary data from the file system to the embedding system." +-- Ropus provides the Opus encoder and decoder used by the media pipeline. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4113556626"] = "Ropus provides the Opus encoder and decoder used by the media pipeline." + -- Community & Code UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4158546761"] = "Community & Code" @@ -8485,6 +8518,39 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source like p -- Document UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document" +-- The configured transcription provider could not be created. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1235984176"] = "The configured transcription provider could not be created." + +-- The selected media file no longer exists. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T129859547"] = "The selected media file no longer exists." + +-- The selected media file does not contain an audio track. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T134825479"] = "The selected media file does not contain an audio track." + +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1543974632"] = "The media file could not be transcribed." + +-- The selected file cannot be processed as media. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1707342767"] = "The selected file cannot be processed as media." + +-- The media file is damaged or its format could not be identified. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2004316549"] = "The media file is damaged or its format could not be identified." + +-- This media format or audio codec is not supported. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2142564510"] = "This media format or audio codec is not supported." + +-- No usable transcription provider is configured. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2282521655"] = "No usable transcription provider is configured." + +-- The media file could not be prepared for transcription. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2749117459"] = "The media file could not be prepared for transcription." + +-- The transcription provider could not transcribe the media file. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T3091669215"] = "The transcription provider could not transcribe the media file." + +-- The media pipeline ended without an output file. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T632852430"] = "The media pipeline ended without an output file." + -- Pandoc Installation UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::PANDOCAVAILABILITYSERVICE::T185447014"] = "Pandoc Installation" diff --git a/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs b/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs index cbbf96d9..915d2b8c 100644 --- a/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs +++ b/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs @@ -1,93 +1,173 @@ using System.Text; using AIStudio.Settings; +using AIStudio.Tools.PluginSystem; namespace AIStudio.Chat; +/// +/// Attachment whose Markdown file is owned and lifecycle-managed by the media feature. +/// +/// Display file name. +/// Absolute staged or chat-owned path. +/// Current file size. +/// Original media file name used in the title and stem. +/// Whether the file still lives in operation staging. public sealed record ManagedTranscriptAttachment(string FileName, string FilePath, long FileSizeBytes, string OriginalFileName, bool IsStaged) : FileAttachment(FileAttachmentType.DOCUMENT, FileName, FilePath, FileSizeBytes) { + /// Refreshes the path-derived name and current file size. public override FileAttachment Normalize() { var size = File.Exists(this.FilePath) ? new FileInfo(this.FilePath).Length : 0; return this with { FileName = Path.GetFileName(this.FilePath), FileSizeBytes = size }; } + /// Creates a transcript in an operation-specific staging directory. + /// Original media path. + /// Provider transcript. + /// The staged managed attachment. public static async Task CreateStagedAsync(string originalPath, string transcript) { var operationDirectory = Path.Combine(SettingsManager.DataDirectory!, "media-staging", Guid.NewGuid().ToString("N")); Directory.CreateDirectory(operationDirectory); - var originalFileName = Path.GetFileName(originalPath); var stagingPath = Path.Combine(operationDirectory, $"{Guid.NewGuid():N}.md"); - var markdown = $""" - # Transcription of {originalFileName} - - {transcript.Trim()} - """; - - await File.WriteAllTextAsync(stagingPath, markdown, new UTF8Encoding(false)); - return new ManagedTranscriptAttachment( - Path.GetFileName(stagingPath), - stagingPath, - new FileInfo(stagingPath).Length, - originalFileName, - true); + await WriteMarkdownAsync(stagingPath, originalFileName, transcript); + return FromPath(stagingPath, originalFileName, isStaged: true); } + /// Writes transcript Markdown to a temporary file and atomically publishes it. + /// Final managed target path. + /// Original media file name. + /// Provider transcript. + /// The chat-owned managed attachment. + internal static async Task CreateAtomicAsync(string targetPath, string originalFileName, string transcript) + { + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + var temporaryPath = Path.Combine(Path.GetDirectoryName(targetPath)!, $".{Guid.NewGuid():N}.tmp"); + try + { + await WriteMarkdownAsync(temporaryPath, originalFileName, transcript); + File.Move(temporaryPath, targetPath); + return FromPath(targetPath, originalFileName, isStaged: false); + } + finally + { + if (File.Exists(temporaryPath)) + File.Delete(temporaryPath); + } + } + + /// Deletes a file only when its canonical path has an exact managed structure. + /// Candidate managed attachment. + /// Whether an owned file was deleted. public static bool TryDeleteOwnedFile(FileAttachment attachment) { if (attachment is not ManagedTranscriptAttachment managed || !File.Exists(managed.FilePath)) return false; var fileInfo = new FileInfo(managed.FilePath); - var parentInfo = fileInfo.Directory!; - var canonicalParent = parentInfo.ResolveLinkTarget(true)?.FullName ?? parentInfo.FullName; - var fullPath = fileInfo.ResolveLinkTarget(true)?.FullName ?? Path.Combine(canonicalParent, fileInfo.Name); - var dataDirectory = new DirectoryInfo(SettingsManager.DataDirectory!); + var fullFilePath = Path.GetFullPath(fileInfo.FullName); + var fullDataRoot = Path.GetFullPath(SettingsManager.DataDirectory!); + var relative = Path.GetRelativePath(fullDataRoot, fullFilePath); - var canonicalDataRoot = EnsureTrailingSeparator(dataDirectory.ResolveLinkTarget(true)?.FullName ?? dataDirectory.FullName); - var stagingDirectory = new DirectoryInfo(Path.Combine(SettingsManager.DataDirectory!, "media-staging")); - var stagingRoot = EnsureTrailingSeparator(stagingDirectory.ResolveLinkTarget(true)?.FullName ?? stagingDirectory.FullName); + if (Path.IsPathRooted(relative) || relative == ".." || relative.StartsWith($"..{Path.DirectorySeparatorChar}", PathComparison)) + return false; - var transcriptsSegment = $"{Path.DirectorySeparatorChar}attachments{Path.DirectorySeparatorChar}transcripts{Path.DirectorySeparatorChar}"; - var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; - var owned = fullPath.StartsWith(stagingRoot, comparison) - || (fullPath.StartsWith(canonicalDataRoot, comparison) - && fullPath.Contains(transcriptsSegment, comparison)); - - if (!owned) + if (fileInfo.LinkTarget is not null || HasLinkedDirectory(fileInfo.Directory, fullDataRoot)) return false; - File.Delete(fullPath); - var parent = Path.GetDirectoryName(fullPath); - if (managed.IsStaged && parent is not null && Directory.Exists(parent) && !Directory.EnumerateFileSystemEntries(parent).Any()) + var segments = relative.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var isStaging = segments is ["media-staging", _, _] + && Guid.TryParseExact(segments[1], "N", out _) + && !string.IsNullOrWhiteSpace(segments[2]); + + var isTemporaryChatTranscript = segments is ["tempChats", _, _, _, _] + && Guid.TryParse(segments[1], out _) + && segments[2] == "attachments" + && segments[3] == "transcripts"; + + var isWorkspaceChatTranscript = segments is ["workspaces", _, _, _, _, _] + && Guid.TryParse(segments[1], out _) + && Guid.TryParse(segments[2], out _) + && segments[3] == "attachments" + && segments[4] == "transcripts"; + + if (!isStaging && !isTemporaryChatTranscript && !isWorkspaceChatTranscript) + return false; + + File.Delete(fullFilePath); + var parent = Path.GetDirectoryName(fullFilePath); + if (isStaging && parent is not null && Directory.Exists(parent) && !Directory.EnumerateFileSystemEntries(parent).Any()) Directory.Delete(parent); return true; } + /// Rejects paths traversing any symbolic-link or junction directory below the data root. + private static bool HasLinkedDirectory(DirectoryInfo? directory, string fullDataRoot) + { + while (directory is not null && !string.Equals(Path.GetFullPath(directory.FullName), fullDataRoot, PathComparison)) + { + if (directory.LinkTarget is not null) + return true; + + directory = directory.Parent; + } + + return directory is null; + } + + /// Normalizes an original stem using Unicode scalar values and cross-platform rules. + /// Original media file name. + /// A non-empty stem containing at most 80 Unicode text characters. internal static string NormalizeOriginalStem(string originalFileName) { var stem = Path.GetFileNameWithoutExtension(originalFileName).Normalize(NormalizationForm.FormC); - var invalid = Path.GetInvalidFileNameChars().ToHashSet(); var normalized = new StringBuilder(); - foreach (var character in stem) + var textCharacters = 0; + foreach (var rune in stem.EnumerateRunes()) { - normalized.Append(char.IsControl(character) - || invalid.Contains(character) - || character is '/' or '\\' - ? '-' - : character); + if (textCharacters == 80) + break; + + var replacement = Rune.IsControl(rune) || rune.Value is '/' or '\\' or '<' or '>' or ':' or '"' or '|' or '?' or '*' + ? new Rune('-') + : rune; + + normalized.Append(replacement); + textCharacters++; } - + var result = normalized.ToString().Trim(' ', '.', '-'); - if (string.IsNullOrWhiteSpace(result)) - result = "media"; - - return result.Length <= 80 ? result : result[..80]; + return string.IsNullOrWhiteSpace(result) ? "media" : result; } - private static string EnsureTrailingSeparator(string path) => path.EndsWith(Path.DirectorySeparatorChar) ? path : path + Path.DirectorySeparatorChar; + /// Creates an attachment record from a file already written to disk. + private static ManagedTranscriptAttachment FromPath(string path, string originalFileName, bool isStaged) => new( + Path.GetFileName(path), + path, + new FileInfo(path).Length, + originalFileName, + isStaged); + + /// Writes localized transcript Markdown without a UTF-8 byte-order mark. + private static async Task WriteMarkdownAsync(string path, string originalFileName, string transcript) + { + var markdown = $""" + # Transcription: {originalFileName} + + {transcript.Trim()} + """; + + await File.WriteAllTextAsync(path, markdown, new UTF8Encoding(false)); + } + + /// Gets the platform path comparison used for canonical containment checks. + private static StringComparison PathComparison => OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + + /// Returns localized text while registering the US-English fallback with I18N. + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ManagedTranscriptAttachment).Namespace, nameof(ManagedTranscriptAttachment)); } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs index f90ff2cc..52fbd408 100644 --- a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs +++ b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs @@ -63,6 +63,10 @@ public partial class AttachDocuments : MSGComponentBase [Parameter] public AIStudio.Settings.Provider? Provider { get; set; } + /// Optional persisted chat that can own transcript files immediately. + [Parameter] + public ChatThread? OwnerChat { get; set; } + [Inject] private ILogger Logger { get; set; } = null!; @@ -90,6 +94,7 @@ public partial class AttachDocuments : MSGComponentBase protected override async Task OnInitializedAsync() { + this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; this.ApplyFilters([], [ Event.TAURI_EVENT_RECEIVED, Event.REGISTER_FILE_DROP_AREA, Event.UNREGISTER_FILE_DROP_AREA ]); // Register this drop area: @@ -97,6 +102,16 @@ public partial class AttachDocuments : MSGComponentBase await base.OnInitializedAsync(); } + /// Refreshes disabled controls when the shared import lane changes. + private void OnMediaImportStateChanged() => _ = this.InvokeAsync(this.StateHasChanged); + + /// Unsubscribes from the singleton media service. + protected override void DisposeResources() + { + this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; + base.DisposeResources(); + } + protected override async Task ProcessIncomingMessage(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default { if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED) @@ -277,12 +292,12 @@ public partial class AttachDocuments : MSGComponentBase showDialog: true); canAddRegularFiles = pandocState.IsAvailable; } - + foreach (var path in regularPaths) { if (!canAddRegularFiles) break; - + if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync( FileExtensionValidation.UseCase.ATTACHING_CONTENT, path, @@ -311,12 +326,12 @@ public partial class AttachDocuments : MSGComponentBase $"{this.T("The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider.")}{Environment.NewLine}{Environment.NewLine}{names}" }, }; - + var dialogReference = await this.DialogService.ShowAsync( this.T("Transcribe media files"), dialogParameters, DialogOptions.FULLSCREEN); - + var dialogResult = await dialogReference.Result; if (dialogResult is null || dialogResult.Canceled) return; @@ -324,17 +339,21 @@ public partial class AttachDocuments : MSGComponentBase var failures = new List(); foreach (var mediaPath in mediaPaths) { - var result = await this.MediaTranscriptionService.TranscribeAsync(mediaPath); - if (!result.Success) + var result = await this.MediaTranscriptionService.TranscribeImportAsync(mediaPath); + if (result.Status is MediaTranscriptionResultStatus.CANCELLED) + break; + + if (result.Status is MediaTranscriptionResultStatus.FAILED) { - if (result.ErrorMessage == "The media transcription was cancelled.") - break; - - failures.Add($"{Path.GetFileName(mediaPath)}: {result.ErrorMessage}"); + failures.Add($"{Path.GetFileName(mediaPath)}: {result.UserMessage}"); continue; } + + var attachment = this.OwnerChat is not null + && WorkspaceBehaviour.IsChatExisting(new LoadChat(this.OwnerChat.WorkspaceId, this.OwnerChat.ChatId)) + ? await WorkspaceBehaviour.CreateManagedTranscriptAsync(this.OwnerChat, mediaPath, result.Text) + : await ManagedTranscriptAttachment.CreateStagedAsync(mediaPath, result.Text); - var attachment = await ManagedTranscriptAttachment.CreateStagedAsync(mediaPath, result.Text); this.DocumentPaths.Add(attachment); } diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor b/app/MindWork AI Studio/Components/ChatComponent.razor index 48d9b948..4229b54b 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor +++ b/app/MindWork AI Studio/Components/ChatComponent.razor @@ -46,7 +46,7 @@ Label="@this.InputLabel" Placeholder="@this.ProviderPlaceholder" Adornment="Adornment.End" - AdornmentIcon="@Icons.Material.Filled.Send" + AdornmentIcon="@(this.MediaTranscriptionService.IsBusy ? Icons.Material.Filled.HourglassTop : Icons.Material.Filled.Send)" OnAdornmentClick="() => this.SendMessage()" Disabled="@this.IsInputForbidden()" Immediate="@true" @@ -101,7 +101,7 @@ } - + diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index b50b607e..8dccb301 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -108,6 +108,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable protected override async Task OnInitializedAsync() { + this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; + // Apply the filters for the message bus: this.ApplyFilters([], [ Event.HAS_CHAT_UNSAVED_CHANGES, Event.RESET_CHAT_STATE, Event.CHAT_STREAMING_DONE, Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED, Event.CHAT_GENERATION_CHANGED, Event.WORKSPACE_RENAMED, Event.CONFIGURATION_CHANGED ]); @@ -250,6 +252,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable await base.OnInitializedAsync(); } + /// Refreshes send and attachment controls when the media import lane changes. + private void OnMediaImportStateChanged() => _ = this.InvokeAsync(this.StateHasChanged); + protected override async Task OnAfterRenderAsync(bool firstRender) { if (firstRender && this.ChatThread is not null && this.mustStoreChat) @@ -1212,6 +1217,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable public async ValueTask DisposeAsync() { + this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; if(this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is WorkspaceStorageBehavior.STORE_CHATS_AUTOMATICALLY) { await this.SaveThread(); diff --git a/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor.cs b/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor.cs index 402b0083..4d087ff7 100644 --- a/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor.cs +++ b/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor.cs @@ -4,25 +4,30 @@ namespace AIStudio.Components; public partial class MediaTranscriptionStatus { + /// Gets the localized visible status for the active import. private string StatusText => this.MediaTranscriptionService.Phase switch { MediaTranscriptionPhase.PROBING => $"{this.T("Inspecting media")}: {this.MediaTranscriptionService.CurrentFileName}", MediaTranscriptionPhase.TRANSCODING => $"{this.T("Preparing audio")}: {this.MediaTranscriptionService.CurrentFileName}", MediaTranscriptionPhase.UPLOADING => $"{this.T("Transcribing")}: {this.MediaTranscriptionService.CurrentFileName}", - + _ => this.MediaTranscriptionService.CurrentFileName, }; + /// Subscribes to singleton import state changes. protected override async Task OnInitializedAsync() { this.MediaTranscriptionService.StateChanged += this.OnStateChanged; await base.OnInitializedAsync(); } + /// Schedules a render after an import state transition. private void OnStateChanged() => _ = this.InvokeAsync(this.StateHasChanged); + /// Unsubscribes from singleton import state changes. protected override void DisposeResources() { this.MediaTranscriptionService.StateChanged -= this.OnStateChanged; + base.DisposeResources(); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ReadFileContent.razor.cs b/app/MindWork AI Studio/Components/ReadFileContent.razor.cs index 31fa36fe..cdca6779 100644 --- a/app/MindWork AI Studio/Components/ReadFileContent.razor.cs +++ b/app/MindWork AI Studio/Components/ReadFileContent.razor.cs @@ -63,6 +63,7 @@ public partial class ReadFileContent : MSGComponentBase protected override async Task OnInitializedAsync() { + this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; if (this.EnableDragDrop) { this.ApplyFilters([], [ Event.TAURI_EVENT_RECEIVED, Event.REGISTER_FILE_DROP_AREA, Event.UNREGISTER_FILE_DROP_AREA ]); @@ -72,6 +73,16 @@ public partial class ReadFileContent : MSGComponentBase await base.OnInitializedAsync(); } + /// Refreshes disabled controls when the shared import lane changes. + private void OnMediaImportStateChanged() => _ = this.InvokeAsync(this.StateHasChanged); + + /// Unsubscribes from the singleton media service. + protected override void DisposeResources() + { + this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; + base.DisposeResources(); + } + protected override async Task ProcessIncomingMessage(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default { if (!this.EnableDragDrop) @@ -226,18 +237,20 @@ public partial class ReadFileContent : MSGComponentBase this.T("Transcribe media file"), dialogParameters, Dialogs.DialogOptions.FULLSCREEN); - + var dialogResult = await dialogReference.Result; if (dialogResult is null || dialogResult.Canceled) return false; - var result = await this.MediaTranscriptionService.TranscribeAsync(filePath); - if (!result.Success) + var result = await this.MediaTranscriptionService.TranscribeImportAsync(filePath); + if (result.Status is not MediaTranscriptionResultStatus.SUCCEEDED) { - await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, result.ErrorMessage)); + if (result.Status is MediaTranscriptionResultStatus.FAILED) + await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, result.UserMessage)); + return false; } - + await this.FileContentChanged.InvokeAsync(result.Text); return true; } diff --git a/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs b/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs index 5d2fcaa9..78c0c6e7 100644 --- a/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs +++ b/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs @@ -103,8 +103,7 @@ public partial class VoiceRecorder : MSGComponentBase && !string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider); private bool IsVoiceRecordingAvailable => this.ShouldRenderVoiceRecording - && this.VoiceRecordingAvailabilityService.IsAvailable - && !this.MediaTranscriptionService.IsBusy; + && this.VoiceRecordingAvailabilityService.IsAvailable; private string Tooltip => !this.VoiceRecordingAvailabilityService.IsAvailable ? T("Voice recording is unavailable because the client could not initialize audio playback.") @@ -320,13 +319,15 @@ public partial class VoiceRecorder : MSGComponentBase try { - var transcriptionResult = await this.MediaTranscriptionService.TranscribeAsync(this.finalRecordingPath); - if (!transcriptionResult.Success) + var transcriptionResult = await this.MediaTranscriptionService.TranscribeVoiceAsync(this.finalRecordingPath); + if (transcriptionResult.Status is not MediaTranscriptionResultStatus.SUCCEEDED) { this.Logger.LogWarning("The transcription request failed."); - var userMessage = string.IsNullOrWhiteSpace(transcriptionResult.ErrorMessage) + if (transcriptionResult.Status is MediaTranscriptionResultStatus.CANCELLED) + return; + var userMessage = string.IsNullOrWhiteSpace(transcriptionResult.UserMessage) ? this.T("Unfortunately, there was an error communicating with the AI system.") - : transcriptionResult.ErrorMessage; + : transcriptionResult.UserMessage; await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, userMessage)); return; } diff --git a/app/MindWork AI Studio/Components/Workspaces.razor.cs b/app/MindWork AI Studio/Components/Workspaces.razor.cs index 0848fa34..1d4f0eb5 100644 --- a/app/MindWork AI Studio/Components/Workspaces.razor.cs +++ b/app/MindWork AI Studio/Components/Workspaces.razor.cs @@ -845,16 +845,13 @@ public partial class Workspaces : MSGComponentBase if (workspaceId == Guid.Empty) return; - await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, chat.WorkspaceId, chat.ChatId, askForConfirmation: false); - - chat.WorkspaceId = workspaceId; + await WorkspaceBehaviour.MoveChatAsync(chat, workspaceId); if (this.CurrentChatThread?.ChatId == chat.ChatId) { this.CurrentChatThread = chat; await this.CurrentChatThreadChanged.InvokeAsync(this.CurrentChatThread); } - - await WorkspaceBehaviour.StoreChatAsync(chat); + await this.LoadTreeItemsAsync(startPrefetch: false); } diff --git a/app/MindWork AI Studio/Tools/Rust/CreateMediaJobRequest.cs b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobRequest.cs index 7726bb72..1c89e491 100644 --- a/app/MindWork AI Studio/Tools/Rust/CreateMediaJobRequest.cs +++ b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobRequest.cs @@ -1,3 +1,7 @@ namespace AIStudio.Tools.Rust; +/// Request body used to start a Rust media normalization job. +/// Absolute source media path. +/// Absolute operation-owned output path. +/// Optional pass-through size ceiling. public sealed record CreateMediaJobRequest(string InputPath, string OutputPath, ulong? MaxPassThroughBytes = null); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/CreateMediaJobResponse.cs b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobResponse.cs index 7ff6e4fe..2e47855b 100644 --- a/app/MindWork AI Studio/Tools/Rust/CreateMediaJobResponse.cs +++ b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobResponse.cs @@ -1,3 +1,5 @@ namespace AIStudio.Tools.Rust; +/// Response returned after a Rust media job is registered. +/// Opaque runtime job identifier. public sealed record CreateMediaJobResponse(string JobId); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobError.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobError.cs index 9c721283..be0dd2f5 100644 --- a/app/MindWork AI Studio/Tools/Rust/MediaJobError.cs +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobError.cs @@ -1,3 +1,8 @@ namespace AIStudio.Tools.Rust; -public sealed record MediaJobError(string Code, string Message); \ No newline at end of file +/// +/// Runtime media error containing a stable code and an English log diagnostic. +/// +/// Stable machine-readable error category. +/// US-English diagnostic intended for logs. +public sealed record MediaJobError(MediaJobErrorCode Code, string Message); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobErrorCode.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobErrorCode.cs new file mode 100644 index 00000000..68f29b39 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobErrorCode.cs @@ -0,0 +1,85 @@ +namespace AIStudio.Tools.Rust; + +/// +/// Stable failure categories returned by the Rust media pipeline. +/// +public enum MediaJobErrorCode +{ + /// The runtime returned an unrecognized code. + UNKNOWN, + + /// The input file does not exist. + FILE_NOT_FOUND, + + /// The file type could not be identified. + UNKNOWN_FORMAT, + + /// Executable input was rejected. + UNSAFE_FILE, + + /// The input is not media. + NOT_MEDIA, + + /// The input file could not be opened. + FILE_OPEN_FAILED, + + /// The container is unsupported. + UNSUPPORTED_CONTAINER, + + /// The media has no audio track. + NO_AUDIO_TRACK, + + /// No audio track has a supported decoder. + UNSUPPORTED_CODEC, + + /// Decoded audio parameters are absent or inconsistent. + INVALID_AUDIO_PARAMETERS, + + /// The Opus identification header is invalid. + INVALID_OPUS_HEADER, + + /// The Opus mapping requires unsupported multistream decoding. + UNSUPPORTED_OPUS_MAPPING, + + /// The decoder could not be initialized. + DECODER_INIT_FAILED, + + /// The encoder could not be initialized. + ENCODER_INIT_FAILED, + + /// The stream changed unexpectedly. + STREAM_RESET, + + /// The container is damaged. + DAMAGED_CONTAINER, + + /// Audio decoding failed. + DECODE_FAILED, + + /// Audio resampling failed. + RESAMPLE_FAILED, + + /// Opus encoding failed. + ENCODE_FAILED, + + /// The output directory or file could not be created. + OUTPUT_CREATE_FAILED, + + /// The output could not be written. + OUTPUT_WRITE_FAILED, + + /// The partial output could not be committed. + OUTPUT_COMMIT_FAILED, + + /// A WebM relative timestamp overflowed. + WEBM_TIMESTAMP_OVERFLOW, + + /// WebM serialization failed. + WEBM_WRITE_FAILED, + + /// The job was cancelled. + CANCELLED, + + /// The runtime worker failed unexpectedly. + INTERNAL_ERROR, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobEvent.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobEvent.cs index dda2de50..81b1400b 100644 --- a/app/MindWork AI Studio/Tools/Rust/MediaJobEvent.cs +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobEvent.cs @@ -1,5 +1,10 @@ namespace AIStudio.Tools.Rust; +/// Snapshot emitted by the Rust media job event stream. +/// Current job phase. +/// Optional progress fraction. +/// Completed result. +/// Failure diagnostic. public sealed record MediaJobEvent( MediaJobPhase Phase, double? Progress, diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobPhase.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobPhase.cs index 5540bb8c..bf09d744 100644 --- a/app/MindWork AI Studio/Tools/Rust/MediaJobPhase.cs +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobPhase.cs @@ -1,11 +1,23 @@ namespace AIStudio.Tools.Rust; +/// Lifecycle phases exposed by the Rust media API. public enum MediaJobPhase { + /// An unknown future value received from Rust. UNKNOWN, + + /// The runtime is identifying the input and selecting audio. PROBING, + + /// The runtime is normalizing audio. TRANSCODING, + + /// The output was committed successfully. COMPLETED, + + /// The job failed. FAILED, + + /// Cancellation and temporary-output cleanup completed. CANCELLED, } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs index 50bfb175..5fc4f70b 100644 --- a/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs @@ -1,5 +1,11 @@ namespace AIStudio.Tools.Rust; +/// Successful terminal result returned by Rust media normalization. +/// Committed normalized output path. +/// Detected container diagnostic. +/// Selected codec diagnostic. +/// Normalized duration in milliseconds. +/// Whether the source was copied unchanged. public sealed record MediaJobResult( string OutputPath, string DetectedFormat, diff --git a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionPhase.cs b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionPhase.cs index 46369c8a..af5fa0bd 100644 --- a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionPhase.cs +++ b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionPhase.cs @@ -1,9 +1,17 @@ namespace AIStudio.Tools.Services; +/// Visible phases of the serialized media import lane. public enum MediaTranscriptionPhase { + /// No import is active. IDLE, + + /// The runtime is inspecting the input. PROBING, + + /// The runtime is preparing normalized audio. TRANSCODING, + + /// The normalized audio is being transcribed by the provider. UPLOADING, } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionResult.cs b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionResult.cs new file mode 100644 index 00000000..278fc593 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionResult.cs @@ -0,0 +1,25 @@ +using AIStudio.Tools.Rust; + +namespace AIStudio.Tools.Services; + +/// +/// Typed terminal result returned by media import and voice operations. +/// +/// Terminal operation status. +/// Transcript text for a successful operation. +/// Localized message suitable for display after failure. +/// Optional stable runtime failure category. +public sealed record MediaTranscriptionResult(MediaTranscriptionResultStatus Status, string Text, string UserMessage, MediaJobErrorCode? ErrorCode = null) +{ + /// Creates a successful result. + /// Provider transcript. + public static MediaTranscriptionResult Succeeded(string text) => new(MediaTranscriptionResultStatus.SUCCEEDED, text, string.Empty); + + /// Creates a failed result. + /// Localized visible message. + /// Optional runtime error category. + public static MediaTranscriptionResult Failed(string userMessage, MediaJobErrorCode? errorCode = null) => new(MediaTranscriptionResultStatus.FAILED, string.Empty, userMessage, errorCode); + + /// Creates a cancelled result without relying on visible text. + public static MediaTranscriptionResult Cancelled() => new(MediaTranscriptionResultStatus.CANCELLED, string.Empty, string.Empty, MediaJobErrorCode.CANCELLED); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionResultStatus.cs b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionResultStatus.cs new file mode 100644 index 00000000..63fe1eb3 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionResultStatus.cs @@ -0,0 +1,16 @@ +namespace AIStudio.Tools.Services; + +/// +/// Terminal outcome of a media transcription operation. +/// +public enum MediaTranscriptionResultStatus +{ + /// The provider returned a usable transcript. + SUCCEEDED, + + /// The operation failed. + FAILED, + + /// The caller or user cancelled the operation. + CANCELLED, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs index 7906476e..95c9e1de 100644 --- a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs +++ b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs @@ -1,144 +1,241 @@ using AIStudio.Provider; using AIStudio.Settings; +using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; namespace AIStudio.Tools.Services; +/// +/// Coordinates serialized visible media imports and independent voice transcriptions. +/// public sealed class MediaTranscriptionService(RustService rustService, SettingsManager settingsManager, ILogger logger) : IDisposable { - private readonly SemaphoreSlim queue = new(1, 1); - private readonly Lock stateLock = new(); - private CancellationTokenSource? currentCancellation; - private string? currentJobId; + /// Serializes attachment and file-content imports. + private readonly SemaphoreSlim importQueue = new(1, 1); + /// Protects operation ownership and visible import state. + private readonly Lock stateLock = new(); + + /// All operations retained so disposal can cancel voice and import work. + private readonly HashSet operations = []; + + /// The currently visible import operation, if any. + private MediaOperation? currentImport; + + /// Prevents new work after disposal. + private bool disposed; + + /// Raised whenever visible import state changes. public event Action? StateChanged; + /// Gets whether the serialized import lane is active. public bool IsBusy { get; private set; } + /// Gets the file name shown for the active import. public string CurrentFileName { get; private set; } = string.Empty; + /// Gets the active import phase. public MediaTranscriptionPhase Phase { get; private set; } = MediaTranscriptionPhase.IDLE; + /// Gets optional import progress between zero and one. public double? Progress { get; private set; } - public async Task TranscribeAsync(string mediaPath, CancellationToken token = default) + /// + /// Transcribes an attachment or file-content import on the serialized visible lane. + /// + /// Source media path. + /// Caller cancellation token. + /// A typed terminal result. + public async Task TranscribeImportAsync(string mediaPath, CancellationToken token = default) { - await this.queue.WaitAsync(token); - var normalizedPath = Path.Combine( - Path.GetTempPath(), - "mindwork-ai-studio-media", - $"{Guid.NewGuid():N}.webm"); - - Directory.CreateDirectory(Path.GetDirectoryName(normalizedPath)!); - using var operationCancellation = CancellationTokenSource.CreateLinkedTokenSource(token); + this.ThrowIfDisposed(); + await this.importQueue.WaitAsync(token); + var operation = this.CreateOperation(token); lock (this.stateLock) - this.currentCancellation = operationCancellation; + this.currentImport = operation; try { - this.UpdateState(true, Path.GetFileName(mediaPath), MediaTranscriptionPhase.PROBING, 0.0); - var jobId = await rustService.StartMediaJobAsync(mediaPath, normalizedPath, operationCancellation.Token); - lock (this.stateLock) - this.currentJobId = jobId; - - MediaJobResult? normalized = null; - await foreach (var mediaEvent in rustService.StreamMediaJobEventsAsync(jobId, operationCancellation.Token)) - { - switch (mediaEvent.Phase) - { - case MediaJobPhase.PROBING: - this.UpdateState(true, this.CurrentFileName, MediaTranscriptionPhase.PROBING, mediaEvent.Progress); - break; - - case MediaJobPhase.TRANSCODING: - this.UpdateState(true, this.CurrentFileName, MediaTranscriptionPhase.TRANSCODING, mediaEvent.Progress); - break; - - case MediaJobPhase.COMPLETED: - normalized = mediaEvent.Result; - break; - - case MediaJobPhase.CANCELLED: - throw new OperationCanceledException(operationCancellation.Token); - - case MediaJobPhase.FAILED: - return TranscriptionResult.Failure(mediaEvent.Error?.Message ?? "The media file could not be prepared."); - } - } - - if (normalized is null) - return TranscriptionResult.Failure("The media pipeline ended without an output file."); - - var providerSettings = this.ResolveProvider(); - if (providerSettings is null) - return TranscriptionResult.Failure("No usable transcription provider is configured."); - - this.UpdateState(true, this.CurrentFileName, MediaTranscriptionPhase.UPLOADING, null); - - var provider = providerSettings.CreateProvider(); - if (provider.Provider is LLMProviders.NONE) - return TranscriptionResult.Failure("The configured transcription provider could not be created."); - - logger.LogInformation( - "Transcribing normalized media '{MediaPath}' with provider '{Provider}' and model '{Model}'.", - mediaPath, - providerSettings.UsedLLMProvider, - providerSettings.Model); - - var result = await provider.TranscribeAudioAsync( - providerSettings.Model, - normalized.OutputPath, - settingsManager, - operationCancellation.Token); - - return result.Success - ? TranscriptionResult.FromText(result.Text.Trim()) - : result; - } - catch (OperationCanceledException) - { - return TranscriptionResult.Failure("The media transcription was cancelled."); - } - catch (Exception exception) - { - logger.LogError(exception, "Media transcription failed for '{MediaPath}'.", mediaPath); - return TranscriptionResult.Failure("The media file could not be transcribed."); + this.UpdateImportState(true, Path.GetFileName(mediaPath), MediaTranscriptionPhase.PROBING, 0.0); + return await this.TranscribeCoreAsync(mediaPath, operation, updateImportState: true); } finally { lock (this.stateLock) { - this.currentJobId = null; - this.currentCancellation = null; + if (ReferenceEquals(this.currentImport, operation)) + this.currentImport = null; } - try - { - if (File.Exists(normalizedPath)) - File.Delete(normalizedPath); - } - catch (Exception exception) - { - logger.LogWarning(exception, "Could not delete normalized media file '{NormalizedPath}'.", normalizedPath); - } - - this.UpdateState(false, string.Empty, MediaTranscriptionPhase.IDLE, null); - this.queue.Release(); + + this.ReleaseOperation(operation); + this.UpdateImportState(false, string.Empty, MediaTranscriptionPhase.IDLE, null); + this.importQueue.Release(); } } + /// + /// Transcribes a voice recording independently of the visible import lane. + /// + /// Voice recording path. + /// Caller cancellation token. + /// A typed terminal result. + public async Task TranscribeVoiceAsync(string mediaPath, CancellationToken token = default) + { + this.ThrowIfDisposed(); + var operation = this.CreateOperation(token); + + try + { + return await this.TranscribeCoreAsync(mediaPath, operation, updateImportState: false); + } + finally + { + this.ReleaseOperation(operation); + } + } + + /// Cancels only the active visible import operation. public async Task StopAsync() { - string? jobId; + MediaOperation? operation; lock (this.stateLock) { - this.currentCancellation?.Cancel(); - jobId = this.currentJobId; + operation = this.currentImport; + operation?.Cancellation.Cancel(); } - - if (!string.IsNullOrWhiteSpace(jobId)) - await rustService.CancelMediaJobAsync(jobId); + + if (!string.IsNullOrWhiteSpace(operation?.JobId)) + await rustService.CancelMediaJobAsync(operation.JobId); } + /// Runs normalization, provider resolution, and upload for one owned operation. + /// Source media path. + /// Operation-specific cancellation and Rust-job state. + /// Whether progress belongs to the visible import lane. + /// A typed terminal result. + private async Task TranscribeCoreAsync(string mediaPath, MediaOperation operation, bool updateImportState) + { + var normalizedPath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-media", $"{operation.Id:N}.webm"); + Directory.CreateDirectory(Path.GetDirectoryName(normalizedPath)!); + + try + { + var normalized = await this.NormalizeAsync(mediaPath, normalizedPath, operation, updateImportState); + if (normalized.Result is null) + return normalized.Error is null + ? MediaTranscriptionResult.Failed(TB("The media pipeline ended without an output file.")) + : MediaTranscriptionResult.Failed(UserMessageFor(normalized.Error.Code), normalized.Error.Code); + + var providerSettings = this.ResolveProvider(); + if (providerSettings is null) + return MediaTranscriptionResult.Failed(TB("No usable transcription provider is configured.")); + + if (updateImportState) + this.UpdateImportState(true, this.CurrentFileName, MediaTranscriptionPhase.UPLOADING, null); + + var provider = providerSettings.CreateProvider(); + if (provider.Provider is LLMProviders.NONE) + return MediaTranscriptionResult.Failed(TB("The configured transcription provider could not be created.")); + + logger.LogInformation("Transcribing normalized media '{MediaPath}' with provider '{Provider}' and model '{Model}'.", + mediaPath, + providerSettings.UsedLLMProvider, + providerSettings.Model); + + var providerResult = await provider.TranscribeAudioAsync(providerSettings.Model, normalized.Result.OutputPath, settingsManager, operation.Cancellation.Token); + if (!providerResult.Success) + { + logger.LogWarning("The transcription provider failed for '{MediaPath}': {Diagnostic}", mediaPath, providerResult.ErrorMessage); + return MediaTranscriptionResult.Failed(TB("The transcription provider could not transcribe the media file.")); + } + + return MediaTranscriptionResult.Succeeded(providerResult.Text.Trim()); + } + catch (OperationCanceledException) + { + return MediaTranscriptionResult.Cancelled(); + } + catch (Exception exception) + { + logger.LogError(exception, "Media transcription failed for '{MediaPath}'.", mediaPath); + return MediaTranscriptionResult.Failed(TB("The media file could not be transcribed.")); + } + finally + { + // NormalizeAsync does not return from cancellation until Rust has reached a terminal + // phase, so deleting both paths here cannot race a still-writing worker. + this.DeleteTemporaryFile(normalizedPath); + this.DeleteTemporaryFile(normalizedPath + ".partial"); + } + } + + /// Runs the Rust normalization job and drains cancellation to a terminal event. + /// Source media path. + /// Owned temporary output path. + /// Operation-specific state. + /// Whether progress belongs to the import lane. + /// The terminal runtime result or error. + private async Task<(MediaJobResult? Result, MediaJobError? Error)> NormalizeAsync( + string mediaPath, + string normalizedPath, + MediaOperation operation, + bool updateImportState) + { + // The quick POST is intentionally not cancelled: losing its response could orphan a job + // whose ID the client never received. Cancellation is applied immediately after ownership. + var jobId = await rustService.StartMediaJobAsync(mediaPath, normalizedPath, CancellationToken.None); + operation.JobId = jobId; + + try + { + operation.Cancellation.Token.ThrowIfCancellationRequested(); + await foreach (var mediaEvent in rustService.StreamMediaJobEventsAsync(jobId, operation.Cancellation.Token)) + { + if (updateImportState && mediaEvent.Phase is MediaJobPhase.PROBING or MediaJobPhase.TRANSCODING) + { + var phase = mediaEvent.Phase is MediaJobPhase.PROBING + ? MediaTranscriptionPhase.PROBING + : MediaTranscriptionPhase.TRANSCODING; + this.UpdateImportState(true, this.CurrentFileName, phase, mediaEvent.Progress); + } + + switch (mediaEvent.Phase) + { + case MediaJobPhase.COMPLETED: + return (mediaEvent.Result, null); + + case MediaJobPhase.FAILED: + if (mediaEvent.Error is not null) + logger.LogWarning("Rust media normalization failed for '{MediaPath}' with {Code}: {Diagnostic}", mediaPath, mediaEvent.Error.Code, mediaEvent.Error.Message); + + return (null, mediaEvent.Error); + + case MediaJobPhase.CANCELLED: + throw new OperationCanceledException(operation.Cancellation.Token); + } + } + + return (null, null); + } + catch (OperationCanceledException) + { + await rustService.CancelMediaJobAsync(jobId, CancellationToken.None); + await this.DrainTerminalEventAsync(jobId); + throw; + } + } + + /// Waits for Rust cleanup after cooperative cancellation. + /// Owned Rust job identifier. + private async Task DrainTerminalEventAsync(string jobId) + { + await foreach (var _ in rustService.StreamMediaJobEventsAsync(jobId, CancellationToken.None)) + { + // The stream itself ends immediately after the first terminal snapshot or event. + } + } + + /// Resolves the configured provider after confidence validation. private TranscriptionProvider? ResolveProvider() { var providerId = settingsManager.ConfigurationData.App.UseTranscriptionProvider; @@ -155,7 +252,30 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM : null; } - private void UpdateState(bool isBusy, string fileName, MediaTranscriptionPhase phase, double? progress) + /// Creates and registers operation-owned cancellation state. + /// Caller token linked to the operation. + /// The registered operation. + private MediaOperation CreateOperation(CancellationToken token) + { + var operation = new MediaOperation(token); + lock (this.stateLock) + this.operations.Add(operation); + + return operation; + } + + /// Unregisters and disposes completed operation state. + /// Completed operation. + private void ReleaseOperation(MediaOperation operation) + { + lock (this.stateLock) + this.operations.Remove(operation); + + operation.Dispose(); + } + + /// Updates state exposed only by the serialized import lane. + private void UpdateImportState(bool isBusy, string fileName, MediaTranscriptionPhase phase, double? progress) { this.IsBusy = isBusy; this.CurrentFileName = fileName; @@ -164,10 +284,76 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM this.StateChanged?.Invoke(); } + /// Maps runtime codes to localized user-facing fallback text. + private static string UserMessageFor(MediaJobErrorCode code) => code switch + { + MediaJobErrorCode.FILE_NOT_FOUND => TB("The selected media file no longer exists."), + MediaJobErrorCode.UNSAFE_FILE or MediaJobErrorCode.NOT_MEDIA => TB("The selected file cannot be processed as media."), + MediaJobErrorCode.NO_AUDIO_TRACK => TB("The selected media file does not contain an audio track."), + MediaJobErrorCode.UNSUPPORTED_CONTAINER or MediaJobErrorCode.UNSUPPORTED_CODEC or MediaJobErrorCode.UNSUPPORTED_OPUS_MAPPING => TB("This media format or audio codec is not supported."), + MediaJobErrorCode.UNKNOWN_FORMAT or MediaJobErrorCode.DAMAGED_CONTAINER => TB("The media file is damaged or its format could not be identified."), + + _ => TB("The media file could not be prepared for transcription."), + }; + + /// Deletes one operation-owned temporary file on a best-effort basis. + private void DeleteTemporaryFile(string path) + { + try + { + if (File.Exists(path)) + File.Delete(path); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not delete operation-owned temporary media file '{Path}'.", path); + } + } + + /// Returns localized text while registering the US-English fallback with I18N. + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(MediaTranscriptionService).Namespace, nameof(MediaTranscriptionService)); + + /// Throws when a caller attempts to start work after disposal. + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(this.disposed, this); + + /// Cancels every active import and voice operation and releases owned resources. public void Dispose() { - this.currentCancellation?.Cancel(); - this.currentCancellation?.Dispose(); - this.queue.Dispose(); + MediaOperation[] active; + lock (this.stateLock) + { + if (this.disposed) + return; + + this.disposed = true; + active = [.. this.operations]; + } + foreach (var operation in active) + operation.Cancellation.Cancel(); + + // The semaphore may still be released by an operation unwinding after cancellation. + } + + /// Cancellation and runtime-job ownership for exactly one media operation. + private sealed class MediaOperation : IDisposable + { + /// Creates operation state linked to a caller token. + /// Caller cancellation token. + public MediaOperation(CancellationToken token) + { + this.Cancellation = CancellationTokenSource.CreateLinkedTokenSource(token); + } + + /// Gets the unique temporary-path identifier. + public Guid Id { get; } = Guid.NewGuid(); + + /// Gets the operation-owned cancellation source. + public CancellationTokenSource Cancellation { get; } + + /// Gets or sets the Rust job after its POST response establishes ownership. + public string? JobId { get; set; } + + /// Disposes operation-owned cancellation state. + public void Dispose() => this.Cancellation.Dispose(); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Media.cs b/app/MindWork AI Studio/Tools/Services/RustService.Media.cs index fb01eebe..82bb87ef 100644 --- a/app/MindWork AI Studio/Tools/Services/RustService.Media.cs +++ b/app/MindWork AI Studio/Tools/Services/RustService.Media.cs @@ -7,6 +7,11 @@ namespace AIStudio.Tools.Services; public partial class RustService { + /// Starts a Rust media normalization job. + /// Absolute source path. + /// Absolute operation-owned output path. + /// Request cancellation token. + /// The opaque runtime job identifier. public async Task StartMediaJobAsync(string inputPath, string outputPath, CancellationToken token = default) { using var response = await this.http.PostAsJsonAsync( @@ -14,18 +19,22 @@ public partial class RustService new CreateMediaJobRequest(inputPath, outputPath), this.jsonRustSerializerOptions, token); - + response.EnsureSuccessStatusCode(); var result = await response.Content.ReadFromJsonAsync(this.jsonRustSerializerOptions, token); return result?.JobId ?? throw new InvalidOperationException("The Rust runtime did not return a media job ID."); } + /// Streams replayed and live snapshots until the media job becomes terminal. + /// Runtime job identifier. + /// Stream cancellation token. + /// Asynchronous media job snapshots. public async IAsyncEnumerable StreamMediaJobEventsAsync(string jobId, [EnumeratorCancellation] CancellationToken token = default) { using var request = new HttpRequestMessage(HttpMethod.Get, $"/media/jobs/{Uri.EscapeDataString(jobId)}/events"); using var response = await this.http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token); response.EnsureSuccessStatusCode(); - + await using var stream = await response.Content.ReadAsStreamAsync(token); using var reader = new StreamReader(stream); @@ -45,6 +54,9 @@ public partial class RustService } } + /// Requests cooperative cancellation of a Rust media job. + /// Runtime job identifier. + /// Request cancellation token. public async Task CancelMediaJobAsync(string jobId, CancellationToken token = default) { using var response = await this.http.DeleteAsync($"/media/jobs/{Uri.EscapeDataString(jobId)}", token); diff --git a/app/MindWork AI Studio/Tools/Services/TranscriptStagingCleanupService.cs b/app/MindWork AI Studio/Tools/Services/TranscriptStagingCleanupService.cs index 0387b5e4..a4ca3570 100644 --- a/app/MindWork AI Studio/Tools/Services/TranscriptStagingCleanupService.cs +++ b/app/MindWork AI Studio/Tools/Services/TranscriptStagingCleanupService.cs @@ -2,30 +2,75 @@ using AIStudio.Settings; namespace AIStudio.Tools.Services; +/// +/// One-shot startup service that removes transcript staging left by crashes or forced shutdowns. +/// public sealed class TranscriptStagingCleanupService(ILogger logger) : BackgroundService { + /// Waits for the data directory, performs one cleanup pass, and then exits. + /// Host shutdown token. protected override async Task ExecuteAsync(CancellationToken stoppingToken) { while (string.IsNullOrWhiteSpace(SettingsManager.DataDirectory) && !stoppingToken.IsCancellationRequested) await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); - + if (stoppingToken.IsCancellationRequested) return; var stagingRoot = Path.Combine(SettingsManager.DataDirectory!, "media-staging"); if (!Directory.Exists(stagingRoot)) + { + logger.LogInformation("Media transcript staging does not exist; startup cleanup has nothing to remove."); return; + } - foreach (var directory in Directory.EnumerateDirectories(stagingRoot)) + var directories = Directory.EnumerateDirectories(stagingRoot).ToArray(); + var files = Directory.EnumerateFiles(stagingRoot).ToArray(); + logger.LogInformation("Media transcript startup cleanup found {DirectoryCount} directories and {FileCount} loose files.", directories.Length, files.Length); + + if (directories.Length == 0 && files.Length == 0) + { + logger.LogInformation("Media transcript staging is empty."); + return; + } + + var deletedDirectories = 0; + var deletedFiles = 0; + var failures = 0; + + foreach (var directory in directories) { try { Directory.Delete(directory, true); + deletedDirectories++; + logger.LogInformation("Removed orphaned media staging directory '{Directory}'.", directory); } catch (Exception exception) { + failures++; logger.LogWarning(exception, "Could not remove orphaned media staging directory '{Directory}'.", directory); } } + + foreach (var file in files) + { + try + { + File.Delete(file); + deletedFiles++; + logger.LogInformation("Removed orphaned media staging file '{File}'.", file); + } + catch (Exception exception) + { + failures++; + logger.LogWarning(exception, "Could not remove orphaned media staging file '{File}'.", file); + } + } + + logger.LogInformation("Media transcript startup cleanup removed {DirectoryCount} directories and {FileCount} files with {FailureCount} failures.", + deletedDirectories, + deletedFiles, + failures); } } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs index b665d629..0cb407b0 100644 --- a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs +++ b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs @@ -736,7 +736,7 @@ public static class WorkspaceBehaviour var chatPath = loadChat.WorkspaceId == Guid.Empty ? Path.Join(SettingsManager.DataDirectory, "tempChats", loadChat.ChatId.ToString()) : Path.Join(SettingsManager.DataDirectory, "workspaces", loadChat.WorkspaceId.ToString(), loadChat.ChatId.ToString()); - + return Directory.Exists(chatPath); } @@ -770,40 +770,128 @@ public static class WorkspaceBehaviour } } + /// Creates a transcript atomically inside an already persisted chat. + /// Persisted chat that owns the transcript counter. + /// Original media path. + /// Provider transcript. + /// The chat-owned managed attachment. + public static async Task CreateManagedTranscriptAsync(ChatThread chat, string originalPath, string transcript) + { + var (acquired, semaphore) = await TryAcquireChatSemaphoreAsync(chat.WorkspaceId, chat.ChatId, nameof(CreateManagedTranscriptAsync)); + if (!acquired) + throw new IOException("The chat transcript directory is busy."); + + try + { + var chatDirectory = GetChatDirectory(chat.WorkspaceId, chat.ChatId); + if (!Directory.Exists(chatDirectory)) + throw new DirectoryNotFoundException($"The owning chat directory does not exist: '{chatDirectory}'."); + + var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts"); + Directory.CreateDirectory(transcriptDirectory); + ReconcileTranscriptCounter(chat, transcriptDirectory); + + var targetPath = NextTranscriptPath(chat, transcriptDirectory, Path.GetFileName(originalPath)); + return await ManagedTranscriptAttachment.CreateAtomicAsync(targetPath, Path.GetFileName(originalPath), transcript); + } + finally + { + semaphore.Release(); + } + } + public static async Task MoveChatAsync(ChatThread chat, Guid targetWorkspaceId) { if (chat.WorkspaceId == targetWorkspaceId) return; - var sourceDirectory = chat.WorkspaceId == Guid.Empty - ? Path.Join(SettingsManager.DataDirectory, "tempChats", chat.ChatId.ToString()) - : Path.Join(SettingsManager.DataDirectory, "workspaces", chat.WorkspaceId.ToString(), chat.ChatId.ToString()); + var sourceWorkspaceId = chat.WorkspaceId; + var sourceDirectory = GetChatDirectory(sourceWorkspaceId, chat.ChatId); + var targetDirectory = GetChatDirectory(targetWorkspaceId, chat.ChatId); + var sourceSemaphore = GetChatSemaphore(sourceWorkspaceId, chat.ChatId); + var targetSemaphore = GetChatSemaphore(targetWorkspaceId, chat.ChatId); + var orderedSemaphores = string.CompareOrdinal(sourceWorkspaceId.ToString("N"), targetWorkspaceId.ToString("N")) <= 0 + ? new[] { sourceSemaphore, targetSemaphore } + : new[] { targetSemaphore, sourceSemaphore }; - var targetDirectory = targetWorkspaceId == Guid.Empty - ? Path.Join(SettingsManager.DataDirectory, "tempChats", chat.ChatId.ToString()) - : Path.Join(SettingsManager.DataDirectory, "workspaces", targetWorkspaceId.ToString(), chat.ChatId.ToString()); - - if (Directory.Exists(sourceDirectory)) + await orderedSemaphores[0].WaitAsync(); + await orderedSemaphores[1].WaitAsync(); + + var moved = false; + try { + if (!Directory.Exists(sourceDirectory)) + throw new DirectoryNotFoundException($"The source chat directory does not exist: '{sourceDirectory}'."); + Directory.CreateDirectory(Path.GetDirectoryName(targetDirectory)!); if (Directory.Exists(targetDirectory)) throw new IOException($"The target chat directory already exists: '{targetDirectory}'."); - - Directory.Move(sourceDirectory, targetDirectory); - UpdateAttachmentPathsAfterMove(chat, sourceDirectory, targetDirectory); - } - chat.WorkspaceId = targetWorkspaceId; - await StoreChatAsync(chat); - InvalidateWorkspaceTreeCache(); + Directory.Move(sourceDirectory, targetDirectory); + moved = true; + + UpdateAttachmentPathsAfterMove(chat, sourceDirectory, targetDirectory); + chat.WorkspaceId = targetWorkspaceId; + + await FinalizeStagedTranscriptsAsync(chat, targetDirectory); + await StoreMovedChatFilesAsync(chat, targetDirectory); + } + catch + { + if (moved) + { + try + { + UpdateAttachmentPathsAfterMove(chat, targetDirectory, sourceDirectory); + chat.WorkspaceId = sourceWorkspaceId; + + if (Directory.Exists(targetDirectory) && !Directory.Exists(sourceDirectory)) + Directory.Move(targetDirectory, sourceDirectory); + + if (Directory.Exists(sourceDirectory)) + await StoreMovedChatFilesAsync(chat, sourceDirectory); + } + catch (Exception rollbackError) + { + LOG.LogError(rollbackError, "Could not roll back moving chat '{ChatId}' to workspace '{WorkspaceId}'.", chat.ChatId, targetWorkspaceId); + } + } + throw; + } + finally + { + orderedSemaphores[1].Release(); + orderedSemaphores[0].Release(); + InvalidateWorkspaceTreeCache(); + } } + /// Atomically stores the name and thread after a directory move. + private static async Task StoreMovedChatFilesAsync(ChatThread chat, string chatDirectory) + { + await File.WriteAllTextAsync(Path.Join(chatDirectory, "name"), chat.Name); + var chatPath = Path.Join(chatDirectory, "thread.json"); + var temporaryPath = Path.Join(chatDirectory, $".thread-{Guid.NewGuid():N}.tmp"); + + try + { + await File.WriteAllTextAsync(temporaryPath, JsonSerializer.Serialize(chat, JSON_OPTIONS), Encoding.UTF8); + File.Move(temporaryPath, chatPath, overwrite: true); + } + finally + { + if (File.Exists(temporaryPath)) + File.Delete(temporaryPath); + } + } + + /// Rewrites absolute attachment paths after moving the complete chat directory. private static void UpdateAttachmentPathsAfterMove(ChatThread chat, string sourceDirectory, string targetDirectory) { var sourcePrefix = sourceDirectory.EndsWith(Path.DirectorySeparatorChar) ? sourceDirectory : sourceDirectory + Path.DirectorySeparatorChar; - + var pathComparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; foreach (var content in chat.Blocks.Select(block => block.Content).OfType()) { @@ -812,10 +900,10 @@ public static class WorkspaceBehaviour var attachment = content.FileAttachments[index]; if (!Path.GetFullPath(attachment.FilePath).StartsWith(sourcePrefix, pathComparison)) continue; - + var relativePath = Path.GetRelativePath(sourceDirectory, attachment.FilePath); var movedPath = Path.Combine(targetDirectory, relativePath); - + content.FileAttachments[index] = attachment switch { ManagedTranscriptAttachment managed => managed with { FilePath = movedPath }, @@ -829,6 +917,7 @@ public static class WorkspaceBehaviour private static async Task FinalizeStagedTranscriptsAsync(ChatThread chat, string chatDirectory) { var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts"); + ReconcileTranscriptCounter(chat, transcriptDirectory); foreach (var content in chat.Blocks.Select(block => block.Content).OfType()) { for (var index = 0; index < content.FileAttachments.Count; index++) @@ -838,19 +927,13 @@ public static class WorkspaceBehaviour continue; Directory.CreateDirectory(transcriptDirectory); - string targetPath; - do - { - chat.LastMediaTranscriptNumber++; - var stem = ManagedTranscriptAttachment.NormalizeOriginalStem(staged.OriginalFileName); - targetPath = Path.Combine(transcriptDirectory, $"{stem}-transcript-{chat.LastMediaTranscriptNumber:D4}.md"); - } while (File.Exists(targetPath)); + var targetPath = NextTranscriptPath(chat, transcriptDirectory, staged.OriginalFileName); File.Move(staged.FilePath, targetPath); var sourceDirectory = Path.GetDirectoryName(staged.FilePath); if (sourceDirectory is not null && Directory.Exists(sourceDirectory) && !Directory.EnumerateFileSystemEntries(sourceDirectory).Any()) Directory.Delete(sourceDirectory); - + content.FileAttachments[index] = new ManagedTranscriptAttachment( Path.GetFileName(targetPath), targetPath, @@ -859,10 +942,48 @@ public static class WorkspaceBehaviour false); } } - + await Task.CompletedTask; } + /// Raises the persisted counter to the highest transcript suffix found chat-wide. + private static void ReconcileTranscriptCounter(ChatThread chat, string transcriptDirectory) + { + if (!Directory.Exists(transcriptDirectory)) + return; + + ulong highest = 0; + foreach (var path in Directory.EnumerateFiles(transcriptDirectory, "*-transcript-*.md", SearchOption.TopDirectoryOnly)) + { + var name = Path.GetFileNameWithoutExtension(path); + var marker = name.LastIndexOf("-transcript-", StringComparison.Ordinal); + + if (marker >= 0 && ulong.TryParse(name[(marker + "-transcript-".Length)..], out var number)) + highest = Math.Max(highest, number); + } + + chat.LastMediaTranscriptNumber = Math.Max(chat.LastMediaTranscriptNumber, highest); + } + + /// Allocates the next globally monotonic transcript path for one chat. + private static string NextTranscriptPath(ChatThread chat, string transcriptDirectory, string originalFileName) + { + string targetPath; + do + { + chat.LastMediaTranscriptNumber++; + var stem = ManagedTranscriptAttachment.NormalizeOriginalStem(originalFileName); + targetPath = Path.Combine(transcriptDirectory, $"{stem}-transcript-{chat.LastMediaTranscriptNumber:D4}.md"); + } while (File.Exists(targetPath)); + + return targetPath; + } + + /// Returns the canonical storage directory for a chat identity. + private static string GetChatDirectory(Guid workspaceId, Guid chatId) => workspaceId == Guid.Empty + ? Path.Join(SettingsManager.DataDirectory, "tempChats", chatId.ToString()) + : Path.Join(SettingsManager.DataDirectory, "workspaces", workspaceId.ToString(), chatId.ToString()); + public static async Task LoadChatAsync(LoadChat loadChat) { var (acquired, semaphore) = await TryAcquireChatSemaphoreAsync(loadChat.WorkspaceId, loadChat.ChatId, nameof(LoadChatAsync)); diff --git a/runtime/src/media.rs b/runtime/src/media.rs index 641ef37d..bc594d0a 100644 --- a/runtime/src/media.rs +++ b/runtime/src/media.rs @@ -1,6 +1,9 @@ +//! Asynchronous media normalization jobs producing bounded mono WebM/Opus output. + use std::collections::HashMap; use std::convert::Infallible; use std::fs::{self, File}; +use std::io::{Read, Write}; use std::path::{Path as FilePath, PathBuf}; use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex, RwLock}; @@ -14,9 +17,8 @@ use file_format::{FileFormat, Kind}; use futures::Stream; use once_cell::sync::Lazy; use ropus::{Application, Bitrate, Channels as OpusChannels, DecodeMode, Decoder as OpusDecoder, Encoder as OpusEncoder}; -use rubato::audioadapter::Adapter; use rubato::audioadapter_buffers::direct::SequentialSliceOfVecs; -use rubato::{Fft, FixedSync, Resampler}; +use rubato::{Fft, FixedSync, Indexing, Resampler}; use serde::{Deserialize, Serialize}; use symphonia::core::audio::sample::Sample; use symphonia::core::codecs::audio::{well_known::CODEC_ID_OPUS, AudioDecoder, AudioDecoderOptions}; @@ -35,73 +37,222 @@ use webm_iterable::{WebmIterator, WebmWriter, WriteOptions}; use crate::api_token::APIToken; +/// Sample rate required by the normalized WebM/Opus output contract. const OUTPUT_SAMPLE_RATE: u32 = 48_000; + +/// Number of samples in one 20 ms Opus frame at 48 kHz. const OPUS_FRAME_SAMPLES: usize = 960; + +/// Target bitrate for mono speech-oriented Opus output. const OPUS_BITRATE: u32 = 32_000; + +/// Maximum duration of a WebM cluster before rotating it. const CLUSTER_DURATION_MS: u64 = 30_000; + +/// Encoder look-ahead advertised as the output track's codec delay. const OPUS_PRE_SKIP: u16 = 312; + +/// Default size ceiling for copying an already-normalized file unchanged. const DEFAULT_MAX_PASS_THROUGH_BYTES: u64 = 25 * 1024 * 1024; +/// Bounded input block used for streaming resampling. +const RESAMPLE_INPUT_BLOCK_SAMPLES: usize = 2_048; + +/// Bounded block used by the cancellation-aware pass-through copy. +const COPY_BLOCK_BYTES: usize = 64 * 1024; + +/// Time a terminal job remains available for late SSE subscribers. +const TERMINAL_JOB_RETENTION: std::time::Duration = std::time::Duration::from_secs(10 * 60); + +/// In-memory registry of running and recently completed media jobs. static JOBS: Lazy>>> = Lazy::new(|| RwLock::new(HashMap::new())); +/// Request body for starting a media normalization job. #[derive(Debug, Deserialize)] pub struct CreateMediaJobRequest { + /// Absolute path of the source media file. pub input_path: String, + + /// Optional absolute output path; a sibling WebM path is derived when omitted. pub output_path: Option, + + /// Optional size ceiling for pass-through files. pub max_pass_through_bytes: Option, } +/// Response returned immediately after a media job has been registered. #[derive(Debug, Serialize)] pub struct CreateMediaJobResponse { + /// Opaque identifier used by the event and cancellation routes. pub job_id: String, } -#[derive(Clone, Debug, Serialize)] +/// Observable lifecycle phases of a media job. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] #[serde(rename_all = "snake_case")] pub enum MediaJobPhase { + /// The runtime is identifying the container and selecting an audio track. Probing, + + /// The runtime is decoding and normalizing the selected track. Transcoding, + + /// The normalized output was committed atomically. Completed, + + /// The job ended with a stable media error. Failed, + + /// Cancellation completed and temporary output has been removed. Cancelled, } +/// Stable, machine-readable failure categories returned by the media API. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MediaErrorCode { + /// The requested input file does not exist. + FileNotFound, + + /// The file type could not be identified. + UnknownFormat, + + /// Executable input was rejected. + UnsafeFile, + + /// The identified input is not audio or video. + NotMedia, + + /// The input file could not be opened. + FileOpenFailed, + + /// Symphonia does not support the container. + UnsupportedContainer, + + /// The container has no audio track. + NoAudioTrack, + + /// No audio track has a supported decoder. + UnsupportedCodec, + + /// Required decoded stream parameters are absent or changed. + InvalidAudioParameters, + + /// The Opus identification header is invalid. + InvalidOpusHeader, + + /// The Opus mapping requires unsupported multistream decoding. + UnsupportedOpusMapping, + + /// The decoder could not be initialized. + DecoderInitFailed, + + /// The encoder could not be initialized. + EncoderInitFailed, + + /// The stream requested a decoder reset. + StreamReset, + + /// The media container is truncated or malformed. + DamagedContainer, + + /// Audio decoding failed. + DecodeFailed, + + /// Audio resampling failed. + ResampleFailed, + + /// Opus encoding failed. + EncodeFailed, + + /// The output directory could not be created. + OutputCreateFailed, + + /// Output bytes could not be written. + OutputWriteFailed, + + /// The partial output could not be committed. + OutputCommitFailed, + + /// A WebM relative timestamp exceeded its safe range. + WebmTimestampOverflow, + + /// WebM output serialization failed. + WebmWriteFailed, + + /// The job was cancelled. + Cancelled, + + /// The worker task terminated unexpectedly. + InternalError, +} + +/// Snapshot delivered through the media job SSE stream. #[derive(Clone, Debug, Serialize)] pub struct MediaJobEvent { + /// Current lifecycle phase. pub phase: MediaJobPhase, + + /// Optional progress fraction between zero and one. pub progress: Option, + + /// Terminal result, present only for completed jobs. pub result: Option, + + /// Terminal diagnostic, present only for failed jobs. pub error: Option, } +/// Successful normalized-media result. #[derive(Clone, Debug, Serialize)] pub struct MediaJobResult { + /// Path at which the normalized output was committed. pub output_path: String, + + /// Human-readable detected container description for diagnostics. pub detected_format: String, + + /// Human-readable selected codec description for diagnostics. pub detected_codec: String, + + /// Duration of the normalized playable audio. pub duration_ms: u64, + + /// Whether the input was copied unchanged. pub pass_through: bool, } +/// Stable error code plus an English diagnostic intended for logs. #[derive(Clone, Debug, Serialize)] pub struct MediaError { - pub code: String, + /// Machine-readable failure category used for localization by the client. + pub code: MediaErrorCode, + + /// US-English diagnostic detail for logging and support. pub message: String, } impl MediaError { - fn new(code: &str, message: impl Into) -> Self { - Self { code: code.to_string(), message: message.into() } + /// Creates a media error without exposing free-form codes on the wire. + fn new(code: MediaErrorCode, message: impl Into) -> Self { + Self { code, message: message.into() } } } +/// Mutable state shared by the request routes and blocking worker. struct MediaJob { + /// Cooperative cancellation flag checked at bounded intervals. cancelled: AtomicBool, + + /// Latest snapshot replayed to a newly connected SSE subscriber. current: Mutex, + + /// Fan-out channel for live state changes. events: broadcast::Sender, } impl MediaJob { + /// Creates a job in the probing phase before its worker is scheduled. fn new() -> Self { let initial = MediaJobEvent { phase: MediaJobPhase::Probing, @@ -114,23 +265,26 @@ impl MediaJob { Self { cancelled: AtomicBool::new(false), current: Mutex::new(initial), events } } + /// Replaces the replay snapshot and notifies all live subscribers. fn publish(&self, event: MediaJobEvent) { *self.current.lock().unwrap() = event.clone(); let _ = self.events.send(event); } + /// Returns whether cooperative cancellation was requested. fn is_cancelled(&self) -> bool { self.cancelled.load(Ordering::Relaxed) } } +/// Registers and immediately schedules a media normalization job. pub async fn create_job( _token: APIToken, Json(request): Json, ) -> Result, (StatusCode, Json)> { let input_path = PathBuf::from(&request.input_path); if !input_path.is_file() { - return Err((StatusCode::BAD_REQUEST, Json(MediaError::new("file_not_found", "The selected media file does not exist.")))); + return Err((StatusCode::BAD_REQUEST, Json(MediaError::new(MediaErrorCode::FileNotFound, "The selected media file does not exist.")))); } let output_path = request.output_path.map(PathBuf::from).unwrap_or_else(|| { @@ -155,32 +309,45 @@ pub async fn create_job( result: Some(result), error: None, }), - Ok(Err(error)) if error.code == "cancelled" => job.publish(MediaJobEvent { + + Ok(Err(error)) if error.code == MediaErrorCode::Cancelled => job.publish(MediaJobEvent { phase: MediaJobPhase::Cancelled, progress: None, result: None, error: None, }), + Ok(Err(error)) => job.publish(MediaJobEvent { phase: MediaJobPhase::Failed, progress: None, result: None, error: Some(error), }), + Err(error) => job.publish(MediaJobEvent { phase: MediaJobPhase::Failed, progress: None, result: None, - error: Some(MediaError::new("internal_error", format!("The media worker failed: {error}"))), + error: Some(MediaError::new(MediaErrorCode::InternalError, format!("The media worker failed: {error}"))), }), } - tokio::time::sleep(std::time::Duration::from_secs(600)).await; - JOBS.write().unwrap().remove(&completed_job_id); + + retain_terminal_job(completed_job_id).await; }); Ok(Json(CreateMediaJobResponse { job_id })) } +/// Retains a terminal job for late SSE subscribers, then removes it asynchronously. +/// +/// Retention starts only after the worker has published a terminal event. Sleeping here neither +/// blocks the originating request nor the blocking media worker. +async fn retain_terminal_job(job_id: String) { + tokio::time::sleep(TERMINAL_JOB_RETENTION).await; + JOBS.write().unwrap().remove(&job_id); +} + +/// Streams the current snapshot followed by live media job events. pub async fn get_job_events( _token: APIToken, Path(job_id): Path, @@ -197,6 +364,7 @@ pub async fn get_job_events( Ok(Sse::new(stream).keep_alive(KeepAlive::default())) } +/// Requests cooperative cancellation of a running media job. pub async fn cancel_job(_token: APIToken, Path(job_id): Path) -> impl IntoResponse { match JOBS.read().unwrap().get(&job_id) { Some(job) => { @@ -208,6 +376,7 @@ pub async fn cancel_job(_token: APIToken, Path(job_id): Path) -> impl In } } +/// Maps a phase to the corresponding SSE event name. fn phase_name(phase: &MediaJobPhase) -> &'static str { match phase { MediaJobPhase::Probing => "probing", @@ -218,23 +387,32 @@ fn phase_name(phase: &MediaJobPhase) -> &'static str { } } +/// Probes, normalizes, and atomically commits one media file. fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_through_bytes: u64, job: &MediaJob) -> Result { check_cancelled(job)?; let detected = FileFormat::from_file(input_path) - .map_err(|error| MediaError::new("unknown_format", format!("The file type could not be identified: {error}")))?; + .map_err(|error| MediaError::new(MediaErrorCode::UnknownFormat, format!("The file type could not be identified: {error}")))?; if detected.kind() == Kind::Executable { - return Err(MediaError::new("unsafe_file", "The selected file contains executable data and cannot be processed as media.")); + return Err(MediaError::new(MediaErrorCode::UnsafeFile, "The selected file contains executable data and cannot be processed as media.")); } if !matches!(detected.kind(), Kind::Audio | Kind::Video) - && !matches!(detected, FileFormat::ExtensibleBinaryMetaLanguage) + && !matches!( + detected, + FileFormat::ExtensibleBinaryMetaLanguage + | FileFormat::Id3v2 + | FileFormat::Mpeg4Part14 + | FileFormat::Mpeg4Part14Audio + | FileFormat::Mpeg4Part14Video + ) + && !has_supported_media_extension(input_path) { - return Err(MediaError::new("not_media", format!("The selected file is not supported media (detected as {detected:?})."))); + return Err(MediaError::new(MediaErrorCode::NotMedia, format!("The selected file is not supported media (detected as {detected:?})."))); } - let file = File::open(input_path).map_err(|error| MediaError::new("file_open_failed", error.to_string()))?; + let file = File::open(input_path).map_err(|error| MediaError::new(MediaErrorCode::FileOpenFailed, error.to_string()))?; let mss = MediaSourceStream::new(Box::new(file), Default::default()); let mut hint = Hint::new(); @@ -249,32 +427,29 @@ fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_throu let detected_format = format!("{detected:?} / {}", format.format_info().long_name); let tracks = format.tracks(); - let mut audio_tracks: Vec<&Track> = tracks.iter().filter(|track| { - matches!(track.codec_params, Some(CodecParameters::Audio(_))) - }).collect(); - - if audio_tracks.is_empty() { - return Err(MediaError::new("no_audio_track", "The selected media file does not contain an audio track.")); + if !tracks.iter().any(is_audio_track) { + return Err(MediaError::new(MediaErrorCode::NoAudioTrack, "The selected media file does not contain an audio track.")); } - audio_tracks.sort_by_key(|track| !track.flags.contains(TrackFlags::DEFAULT)); - let selected = audio_tracks.into_iter().find(|track| is_decodable(track)) - .ok_or_else(|| MediaError::new("unsupported_codec", "None of the audio tracks uses a supported codec."))?; + let selected = select_audio_track(tracks) + .ok_or_else(|| MediaError::new(MediaErrorCode::UnsupportedCodec, "None of the audio tracks uses a supported codec."))?; let track_id = selected.id; let params = selected.codec_params.as_ref().and_then(CodecParameters::audio).unwrap().clone(); let detected_codec = if params.codec == CODEC_ID_OPUS { "opus".to_string() } else { format!("{}", params.codec) }; - let duration_ms = selected.time_base.zip(selected.duration).and_then(|(time_base, duration)| { - let timestamp = Timestamp::new(i64::try_from(duration.get()).ok()?); - let time = time_base.calc_time(timestamp)?; - Some((time.as_secs_f64() * 1000.0).max(0.0).round() as u64) - }).unwrap_or(0); + let duration_ms = selected.num_frames.zip(params.sample_rate) + .map(|(frames, rate)| frames.saturating_mul(1_000) / u64::from(rate)) + .or_else(|| selected.time_base.zip(selected.duration).and_then(|(time_base, duration)| { + let timestamp = Timestamp::new(i64::try_from(duration.get()).ok()?); + let time = time_base.calc_time(timestamp)?; + Some((time.as_secs_f64() * 1000.0).max(0.0).round() as u64) + })) + .unwrap_or(0); - let audio_only = tracks.iter().all(|track| track.track_type() != Some(TrackType::Video)); let channels = params.channels.as_ref().map(|value| value.count()).unwrap_or(0); let pass_through = is_webm_container(input_path) - && audio_only - && tracks.iter().filter(|track| track.track_type() == Some(TrackType::Audio)).count() == 1 + && tracks.len() == 1 + && selected.track_type() == Some(TrackType::Audio) && params.codec == CODEC_ID_OPUS && params.sample_rate == Some(OUTPUT_SAMPLE_RATE) && channels == 1 @@ -282,13 +457,11 @@ fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_throu let partial_path = partial_path(output_path); if let Some(parent) = partial_path.parent() { - fs::create_dir_all(parent).map_err(|error| MediaError::new("output_create_failed", error.to_string()))?; + fs::create_dir_all(parent).map_err(|error| MediaError::new(MediaErrorCode::OutputCreateFailed, error.to_string()))?; } let result = if pass_through { - check_cancelled(job)?; - fs::copy(input_path, &partial_path).map_err(|error| MediaError::new("output_write_failed", error.to_string()))?; - check_cancelled(job)?; + copy_with_cancellation(input_path, &partial_path, job)?; Ok(MediaJobResult { output_path: output_path.to_string_lossy().into_owned(), detected_format: detected_format.clone(), @@ -304,14 +477,26 @@ fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_throu error: None, }); - transcode(&mut *format, track_id, ¶ms, &partial_path, output_path, detected_format, detected_codec, duration_ms, job) + let context = TranscodeContext { + track_id, + track_delay: selected.delay.unwrap_or(0), + track_padding: selected.padding.unwrap_or(0), + params, + partial_path: &partial_path, + output_path, + detected_format, + detected_codec, + expected_duration_ms: duration_ms, + job, + }; + transcode(&mut *format, context) }; match result { Ok(result) => { if let Err(error) = fs::rename(&partial_path, output_path) { let _ = fs::remove_file(&partial_path); - return Err(MediaError::new("output_commit_failed", error.to_string())); + return Err(MediaError::new(MediaErrorCode::OutputCommitFailed, error.to_string())); } Ok(result) @@ -324,151 +509,189 @@ fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_throu } } +/// Recognizes extensions for containers supported by the configured Symphonia readers. +fn has_supported_media_extension(path: &FilePath) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| matches!( + extension.to_ascii_lowercase().as_str(), + "aac" | "aif" | "aiff" | "caf" | "flac" | "m4a" | "mka" | "mkv" | "mov" + | "mp1" | "mp2" | "mp3" | "mp4" | "oga" | "ogg" | "opus" | "wav" | "webm" + )) +} + +/// Returns whether a track explicitly contains audio codec parameters. +fn is_audio_track(track: &Track) -> bool { + matches!(track.codec_params, Some(CodecParameters::Audio(_))) +} + +/// Selects the decodable default audio track, falling back to the first decodable audio track. +fn select_audio_track(tracks: &[Track]) -> Option<&Track> { + tracks + .iter() + .filter(|track| is_audio_track(track) && is_decodable(track)) + .min_by_key(|track| !track.flags.contains(TrackFlags::DEFAULT)) +} + +/// Checks whether the runtime can construct a decoder for the track. fn is_decodable(track: &Track) -> bool { let Some(params) = track.codec_params.as_ref().and_then(CodecParameters::audio) else { return false; }; params.codec == CODEC_ID_OPUS || symphonia::default::get_codecs().make_audio_decoder(params, &AudioDecoderOptions::default()).is_ok() } +/// Immutable inputs shared across a single transcoding operation. +struct TranscodeContext<'a> { + /// Selected input track identifier. + track_id: u32, + + /// Leading decoded frames to discard. + track_delay: u32, + + /// Trailing decoded frames to discard. + track_padding: u32, + + /// Selected track codec parameters. + params: symphonia::core::codecs::audio::AudioCodecParameters, + + /// Temporary output path used until the job succeeds. + partial_path: &'a FilePath, + + /// Final output path returned in the result. + output_path: &'a FilePath, + + /// Detected container diagnostic. + detected_format: String, + + /// Detected codec diagnostic. + detected_codec: String, + + /// Container duration used for progress reporting. + expected_duration_ms: u64, + + /// Cancellation and progress state for the job. + job: &'a MediaJob, +} + +/// Decodes a selected track and writes bounded 20 ms mono Opus frames. fn transcode( format: &mut dyn symphonia::core::formats::FormatReader, - track_id: u32, - params: &symphonia::core::codecs::audio::AudioCodecParameters, - partial_path: &FilePath, - output_path: &FilePath, - detected_format: String, - detected_codec: String, - expected_duration_ms: u64, - job: &MediaJob, + context: TranscodeContext<'_>, ) -> Result { - let input_rate = params.sample_rate.unwrap_or(OUTPUT_SAMPLE_RATE); - let input_channels = params.channels.as_ref().map(|value| value.count()).unwrap_or(1).max(1); - - let mut symphonia_decoder: Option> = if params.codec == CODEC_ID_OPUS { - None - } else { - Some(symphonia::default::get_codecs().make_audio_decoder(params, &AudioDecoderOptions::default()) - .map_err(|_| MediaError::new("unsupported_codec", "The selected audio codec is not supported."))?) - }; - - let mut opus_decoder = if params.codec == CODEC_ID_OPUS { - let channels = if input_channels == 1 { OpusChannels::Mono } else { OpusChannels::Stereo }; - Some(OpusDecoder::new(OUTPUT_SAMPLE_RATE, channels) - .map_err(|error| MediaError::new("decoder_init_failed", error.to_string()))?) - } else { None }; - + let mut decoder = StreamDecoder::new(&context.params, context.track_delay)?; let mut opus_encoder = OpusEncoder::builder(OUTPUT_SAMPLE_RATE, OpusChannels::Mono, Application::Audio) .bitrate(Bitrate::Bits(OPUS_BITRATE)) .vbr(true) .build() - .map_err(|error| MediaError::new("encoder_init_failed", error.to_string()))?; + .map_err(|error| MediaError::new(MediaErrorCode::EncoderInitFailed, error.to_string()))?; - let file = File::create(partial_path).map_err(|error| MediaError::new("output_create_failed", error.to_string()))?; + let file = File::create(context.partial_path).map_err(|error| MediaError::new(MediaErrorCode::OutputCreateFailed, error.to_string()))?; let mut writer = WebmOpusWriter::new(file)?; let mut pending = Vec::::with_capacity(OPUS_FRAME_SAMPLES * 3); - let mut interleaved = Vec::::new(); - let mut opus_pcm = vec![0i16; 5_760 * input_channels]; + let mut resampler: Option = None; + let mut decoded_tail = Vec::::new(); let mut encoded = [0u8; 4_000]; let mut produced_samples = 0u64; - let mut input_samples = 0u64; - let mut resampled_samples = 0u64; loop { - check_cancelled(job)?; + check_cancelled(context.job)?; let packet = match format.next_packet() { Ok(Some(packet)) => packet, Ok(None) => break, - Err(SymphoniaError::ResetRequired) => return Err(MediaError::new("stream_reset", "The media stream changed unexpectedly.")), + Err(SymphoniaError::ResetRequired) => return Err(MediaError::new(MediaErrorCode::StreamReset, "The media stream changed unexpectedly.")), Err(SymphoniaError::IoError(error)) if error.kind() == std::io::ErrorKind::UnexpectedEof => break, - Err(error) => return Err(MediaError::new("damaged_container", format!("The media container is damaged: {error}"))), + Err(error) => return Err(MediaError::new(MediaErrorCode::DamagedContainer, format!("The media container is damaged: {error}"))), }; - if packet.track_id != track_id { + if packet.track_id != context.track_id { continue; } - let mono = if let Some(decoder) = symphonia_decoder.as_mut() { - let decoded = match decoder.decode(&packet) { - Ok(decoded) => decoded, - Err(SymphoniaError::DecodeError(_)) => continue, - Err(error) => return Err(MediaError::new("decode_failed", format!("Audio decoding failed: {error}"))), - }; - - interleaved.resize(decoded.samples_interleaved(), f32::MID); - decoded.copy_to_slice_interleaved(&mut interleaved); - downmix_to_mono(&interleaved, decoded.num_planes()) - } else { - let decoder = opus_decoder.as_mut().unwrap(); - let frames = decoder.decode(&packet.data, &mut opus_pcm, DecodeMode::Normal) - .map_err(|error| MediaError::new("decode_failed", error.to_string()))?; - - let samples = &opus_pcm[..frames * input_channels]; - if input_channels == 1 { - samples.iter().map(|sample| f32::from(*sample) / 32768.0).collect() - } else { - samples.chunks_exact(input_channels).map(|frame| { - frame.iter().map(|sample| f32::from(*sample) / 32768.0).sum::() / input_channels as f32 - }).collect() - } + let Some((mono, sample_rate)) = decoder.decode(&packet)? else { + continue; }; - input_samples += mono.len() as u64; - let expected_resampled_samples = input_samples.saturating_mul(u64::from(OUTPUT_SAMPLE_RATE)) / u64::from(input_rate); - let expected_chunk_samples = expected_resampled_samples.saturating_sub(resampled_samples) as usize; - let mut resampled = resample_chunk(&mono, input_rate)?; - resampled.resize(expected_chunk_samples, 0.0); - resampled.truncate(expected_chunk_samples); - resampled_samples += resampled.len() as u64; - pending.extend_from_slice(&resampled); + let stream_resampler = match resampler.as_mut() { + Some(existing) if existing.input_rate() == sample_rate => existing, + Some(_) => return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The decoded audio sample rate changed during the stream.")), + None => resampler.insert(StreamResampler::new(sample_rate)?), + }; - let mut consumed = 0usize; - while pending.len() - consumed >= OPUS_FRAME_SAMPLES { - check_cancelled(job)?; - let frame = &pending[consumed..consumed + OPUS_FRAME_SAMPLES]; - let length = opus_encoder.encode_float(frame, &mut encoded) - .map_err(|error| MediaError::new("encode_failed", error.to_string()))?; - - writer.write_packet(&encoded[..length], produced_samples)?; - produced_samples += OPUS_FRAME_SAMPLES as u64; - consumed += OPUS_FRAME_SAMPLES; + // Retain only the possible end padding so it can never be emitted prematurely. + decoded_tail.extend_from_slice(&mono); + let padding = context.track_padding as usize; + let emit_len = decoded_tail.len().saturating_sub(padding); + if emit_len > 0 { + let emit: Vec<_> = decoded_tail.drain(..emit_len).collect(); + append_duration_bounded( + &mut pending, + &stream_resampler.push(&emit)?, + produced_samples, + context.expected_duration_ms, + ); } - if consumed > 0 { - pending.drain(..consumed); + encode_complete_frames(&mut pending, &mut opus_encoder, &mut writer, &mut encoded, &mut produced_samples, context.job)?; - } - - if expected_duration_ms > 0 { + if context.expected_duration_ms > 0 { let current_ms = produced_samples.saturating_mul(1000) / u64::from(OUTPUT_SAMPLE_RATE); - job.publish(MediaJobEvent { + context.job.publish(MediaJobEvent { phase: MediaJobPhase::Transcoding, - progress: Some((current_ms as f64 / expected_duration_ms as f64).clamp(0.0, 0.99)), + progress: Some((current_ms as f64 / context.expected_duration_ms as f64).clamp(0.0, 0.99)), result: None, error: None, }); } } + if let Some(stream_resampler) = resampler.as_mut() { + // Discard the retained decoded tail (container padding), then flush the filter delay. + append_duration_bounded( + &mut pending, + &stream_resampler.finish()?, + produced_samples, + context.expected_duration_ms, + ); + } else { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The selected audio track did not yield decoded audio parameters.")); + } + + encode_complete_frames(&mut pending, &mut opus_encoder, &mut writer, &mut encoded, &mut produced_samples, context.job)?; + if !pending.is_empty() { pending.resize(OPUS_FRAME_SAMPLES, 0.0); let length = opus_encoder.encode_float(&pending, &mut encoded) - .map_err(|error| MediaError::new("encode_failed", error.to_string()))?; + .map_err(|error| MediaError::new(MediaErrorCode::EncodeFailed, error.to_string()))?; writer.write_packet(&encoded[..length], produced_samples)?; produced_samples += OPUS_FRAME_SAMPLES as u64; } writer.finish()?; - check_cancelled(job)?; + check_cancelled(context.job)?; Ok(MediaJobResult { - output_path: output_path.to_string_lossy().into_owned(), - detected_format, - detected_codec, + output_path: context.output_path.to_string_lossy().into_owned(), + detected_format: context.detected_format, + detected_codec: context.detected_codec, duration_ms: produced_samples.saturating_mul(1000) / u64::from(OUTPUT_SAMPLE_RATE), pass_through: false, }) } +/// Appends resampled PCM without exceeding a reliable container duration. +fn append_duration_bounded(pending: &mut Vec, samples: &[f32], produced_samples: u64, expected_duration_ms: u64) { + if expected_duration_ms == 0 { + pending.extend_from_slice(samples); + return; + } + + let target_samples = expected_duration_ms.saturating_mul(u64::from(OUTPUT_SAMPLE_RATE)).div_ceil(1_000); + let accepted = produced_samples.saturating_add(pending.len() as u64); + let remaining = target_samples.saturating_sub(accepted) as usize; + pending.extend_from_slice(&samples[..samples.len().min(remaining)]); +} + +/// Downmixes interleaved PCM to mono using a deterministic arithmetic mean. fn downmix_to_mono(samples: &[f32], channels: usize) -> Vec { if channels <= 1 { return samples.to_vec(); @@ -477,35 +700,373 @@ fn downmix_to_mono(samples: &[f32], channels: usize) -> Vec { samples.chunks_exact(channels).map(|frame| frame.iter().copied().sum::() / channels as f32).collect() } -fn resample_chunk(samples: &[f32], input_rate: u32) -> Result, MediaError> { - if input_rate == OUTPUT_SAMPLE_RATE || samples.is_empty() { - return Ok(samples.to_vec()); - } +/// Parsed subset of an Opus identification header supported by the single-stream decoder. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct OpusHeader { + /// Mono or stereo channel count. + channels: usize, - let input_data = vec![samples.to_vec()]; - let input = SequentialSliceOfVecs::new(&input_data, 1, samples.len()) - .map_err(|error| MediaError::new("resample_failed", error.to_string()))?; - - let mut resampler = Fft::::new(input_rate as usize, OUTPUT_SAMPLE_RATE as usize, samples.len().max(256), 1, FixedSync::Input) - .map_err(|error| MediaError::new("resample_failed", error.to_string()))?; - - let output = resampler.process_all(&input, samples.len(), None) - .map_err(|error| MediaError::new("resample_failed", error.to_string()))?; - - let mut result = Vec::with_capacity(output.frames()); - for frame in 0..output.frames() { - result.push(output.read_sample(0, frame).unwrap_or(0.0)); - } - - Ok(result) + /// Number of leading decoded frames to discard at 48 kHz. + pre_skip: u16, } +impl OpusHeader { + /// Parses and validates a mono/stereo, mapping-family-zero `OpusHead` packet. + fn parse(data: &[u8]) -> Result { + if data.len() < 19 || &data[..8] != b"OpusHead" || data[8] > 15 || !matches!(data[9], 1 | 2) { + return Err(MediaError::new(MediaErrorCode::InvalidOpusHeader, "The Opus identification header is invalid.")); + } + + if data[18] != 0 { + return Err(MediaError::new(MediaErrorCode::UnsupportedOpusMapping, "The Opus channel mapping requires unsupported multistream decoding.")); + } + + Ok(Self { + channels: usize::from(data[9]), + pre_skip: u16::from_le_bytes([data[10], data[11]]), + }) + } +} + +/// Ropus adapter that validates Symphonia parameters and applies Opus pre-skip exactly once. +struct RopusOpusDecoder { + /// Underlying libopus single-stream decoder. + decoder: OpusDecoder, + + /// Number of interleaved channels produced by the decoder. + channels: usize, + + /// Remaining leading frames to discard. + delay_remaining: usize, + + /// Reused bounded PCM output buffer. + pcm: Vec, +} + +impl RopusOpusDecoder { + /// Builds a mono/stereo decoder from the codec's `OpusHead` private data. + fn new(params: &symphonia::core::codecs::audio::AudioCodecParameters, track_delay: u32) -> Result { + let header = OpusHeader::parse(params.extra_data.as_deref().unwrap_or_default())?; + let declared_channels = params.channels.as_ref().map(|channels| channels.count()) + .ok_or_else(|| MediaError::new(MediaErrorCode::InvalidAudioParameters, "The Opus stream does not declare its channel count."))?; + + if declared_channels != header.channels || params.sample_rate != Some(OUTPUT_SAMPLE_RATE) { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The Opus stream parameters do not match its identification header.")); + } + + let opus_channels = if header.channels == 1 { OpusChannels::Mono } else { OpusChannels::Stereo }; + let decoder = OpusDecoder::new(OUTPUT_SAMPLE_RATE, opus_channels) + .map_err(|error| MediaError::new(MediaErrorCode::DecoderInitFailed, error.to_string()))?; + + let delay = if track_delay == 0 { u32::from(header.pre_skip) } else { track_delay }; + + Ok(Self { + decoder, + channels: header.channels, + delay_remaining: delay as usize, + pcm: vec![0; 5_760 * header.channels], + }) + } + + /// Decodes one Opus packet, downmixes it, and removes leading codec delay. + fn decode(&mut self, packet: &[u8]) -> Result, MediaError> { + let frames = self.decoder.decode(packet, &mut self.pcm, DecodeMode::Normal) + .map_err(|error| MediaError::new(MediaErrorCode::DecodeFailed, error.to_string()))?; + + let skip = self.delay_remaining.min(frames); + self.delay_remaining -= skip; + + let samples = &self.pcm[skip * self.channels..frames * self.channels]; + let interleaved: Vec<_> = samples.iter().map(|sample| f32::from(*sample) / 32_768.0).collect(); + Ok(downmix_to_mono(&interleaved, self.channels)) + } +} + +/// Decoder abstraction allowing Symphonia demuxing with either its native decoders or Ropus. +enum StreamDecoder { + /// Decoder supplied by Symphonia for non-Opus codecs. + Symphonia { + /// Stateful codec decoder. + decoder: Box, + + /// Reused interleaved PCM buffer. + interleaved: Vec, + + /// First observed decoded sample rate. + sample_rate: Option, + + /// First observed decoded channel count. + channels: Option, + + /// Remaining track delay to discard. + delay_remaining: usize, + }, + + /// Symphonia-compatible Opus packet adapter backed by Ropus. + Opus(Box), +} + +impl StreamDecoder { + /// Creates the appropriate decoder without guessing missing stream parameters. + fn new(params: &symphonia::core::codecs::audio::AudioCodecParameters, track_delay: u32) -> Result { + if params.codec == CODEC_ID_OPUS { + return Ok(Self::Opus(Box::new(RopusOpusDecoder::new(params, track_delay)?))); + } + + let decoder = symphonia::default::get_codecs().make_audio_decoder(params, &AudioDecoderOptions::default()) + .map_err(|_| MediaError::new(MediaErrorCode::UnsupportedCodec, "The selected audio codec is not supported."))?; + + Ok(Self::Symphonia { + decoder, + interleaved: Vec::new(), + sample_rate: None, + channels: None, + delay_remaining: track_delay as usize, + }) + } + + /// Decodes one packet and returns mono PCM plus the actual decoder sample rate. + fn decode(&mut self, packet: &symphonia::core::packet::Packet) -> Result, u32)>, MediaError> { + match self { + Self::Opus(decoder) => Ok(Some((decoder.decode(&packet.data)?, OUTPUT_SAMPLE_RATE))), + + Self::Symphonia { decoder, interleaved, sample_rate, channels, delay_remaining } => { + let decoded = match decoder.decode(packet) { + Ok(decoded) => decoded, + Err(SymphoniaError::DecodeError(_)) => return Ok(None), + Err(error) => return Err(MediaError::new(MediaErrorCode::DecodeFailed, format!("Audio decoding failed: {error}"))), + }; + + let actual_rate = decoded.spec().rate(); + let actual_channels = decoded.spec().channels().count(); + + if actual_rate == 0 || actual_channels == 0 { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The decoder returned invalid audio parameters.")); + } + + if sample_rate.is_some_and(|rate| rate != actual_rate) || channels.is_some_and(|count| count != actual_channels) { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The decoded audio parameters changed during the stream.")); + } + + *sample_rate = Some(actual_rate); + *channels = Some(actual_channels); + interleaved.resize(decoded.samples_interleaved(), f32::MID); + decoded.copy_to_slice_interleaved(&mut *interleaved); + + let mono = downmix_to_mono(interleaved, actual_channels); + let skip = (*delay_remaining).min(mono.len()); + + *delay_remaining -= skip; + Ok(Some((mono[skip..].to_vec(), actual_rate))) + } + } + } +} + +/// Stateful, bounded mono resampler whose filter history spans decoder packets. +enum StreamResampler { + /// Zero-copy-rate path for already-48-kHz decoded PCM. + Passthrough { + /// Input rate retained for stream consistency checks. + input_rate: u32, + }, + + /// Rubato FFT resampler and its bounded pending input. + Rubato { + /// Input rate retained for stream consistency checks. + input_rate: u32, + + /// Stateful resampler instance used for the entire stream. + inner: Box>, + + /// Samples waiting to fill the next fixed input block. + pending: Vec, + + /// Startup-delay output frames still to discard. + delay_remaining: usize, + + /// Total real input frames accepted. + total_input: u64, + + /// Total trimmed output frames returned to the encoder. + total_output: u64, + }, +} + +impl StreamResampler { + /// Creates one resampler for the decoded stream. + fn new(input_rate: u32) -> Result { + if input_rate == 0 { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The decoded audio sample rate is missing.")); + } + + if input_rate == OUTPUT_SAMPLE_RATE { + return Ok(Self::Passthrough { input_rate }); + } + + let inner = Fft::::new( + input_rate as usize, + OUTPUT_SAMPLE_RATE as usize, + RESAMPLE_INPUT_BLOCK_SAMPLES, + 1, + FixedSync::Input, + ).map_err(|error| MediaError::new(MediaErrorCode::ResampleFailed, error.to_string()))?; + + let delay_remaining = inner.output_delay(); + + Ok(Self::Rubato { + input_rate, + inner: Box::new(inner), + pending: Vec::with_capacity(RESAMPLE_INPUT_BLOCK_SAMPLES * 2), + delay_remaining, + total_input: 0, + total_output: 0, + }) + } + + /// Returns the configured input sample rate. + fn input_rate(&self) -> u32 { + match self { + Self::Passthrough { input_rate } | Self::Rubato { input_rate, .. } => *input_rate, + } + } + + /// Accepts arbitrary packet-sized PCM and processes all complete bounded blocks. + fn push(&mut self, samples: &[f32]) -> Result, MediaError> { + match self { + Self::Passthrough { .. } => Ok(samples.to_vec()), + + Self::Rubato { inner, pending, delay_remaining, total_input, total_output, .. } => { + *total_input += samples.len() as u64; + pending.extend_from_slice(samples); + + let mut output = Vec::new(); + loop { + let block_len = inner.input_frames_next(); + if pending.len() < block_len { + break; + } + + let block: Vec<_> = pending.drain(..block_len).collect(); + append_resampled(inner, &block, None, delay_remaining, &mut output)?; + } + + *total_output += output.len() as u64; + Ok(output) + } + } + } + + /// Flushes the last partial block and filter delay, returning the exact rounded duration. + fn finish(&mut self) -> Result, MediaError> { + match self { + Self::Passthrough { .. } => Ok(Vec::new()), + + Self::Rubato { input_rate, inner, pending, delay_remaining, total_input, total_output } => { + let target = total_input.saturating_mul(u64::from(OUTPUT_SAMPLE_RATE)).div_ceil(u64::from(*input_rate)); + let mut output = Vec::new(); + + if !pending.is_empty() { + let valid = pending.len(); + let block_len = inner.input_frames_next(); + pending.resize(block_len, 0.0); + append_resampled(inner, pending, Some(valid), delay_remaining, &mut output)?; + pending.clear(); + } + + while *total_output + (output.len() as u64) < target { + let zeros = vec![0.0; inner.input_frames_next()]; + append_resampled(inner, &zeros, Some(0), delay_remaining, &mut output)?; + } + + output.truncate(target.saturating_sub(*total_output) as usize); + *total_output += output.len() as u64; + Ok(output) + } + } + } +} + +/// Processes one Rubato block and removes the resampler's startup delay. +fn append_resampled( + resampler: &mut Fft, + block: &[f32], + partial_len: Option, + delay_remaining: &mut usize, + destination: &mut Vec, +) -> Result<(), MediaError> { + let input_data = vec![block.to_vec()]; + let input = SequentialSliceOfVecs::new(&input_data, 1, block.len()) + .map_err(|error| MediaError::new(MediaErrorCode::ResampleFailed, error.to_string()))?; + + let indexing = partial_len.map(|length| Indexing::new().partial_len(length)); + let output = resampler.process(&input, indexing.as_ref()) + .map_err(|error| MediaError::new(MediaErrorCode::ResampleFailed, error.to_string()))?; + + let data = output.take_data(); + let skip = (*delay_remaining).min(data.len()); + *delay_remaining -= skip; + destination.extend_from_slice(&data[skip..]); + Ok(()) +} + +/// Encodes every complete 20 ms frame currently buffered. +fn encode_complete_frames( + pending: &mut Vec, + encoder: &mut OpusEncoder, + writer: &mut WebmOpusWriter, + encoded: &mut [u8], + produced_samples: &mut u64, + job: &MediaJob, +) -> Result<(), MediaError> { + let mut consumed = 0usize; + while pending.len().saturating_sub(consumed) >= OPUS_FRAME_SAMPLES { + check_cancelled(job)?; + let length = encoder.encode_float(&pending[consumed..consumed + OPUS_FRAME_SAMPLES], encoded) + .map_err(|error| MediaError::new(MediaErrorCode::EncodeFailed, error.to_string()))?; + writer.write_packet(&encoded[..length], *produced_samples)?; + *produced_samples += OPUS_FRAME_SAMPLES as u64; + consumed += OPUS_FRAME_SAMPLES; + } + + if consumed > 0 { + pending.drain(..consumed); + } + + Ok(()) +} + +/// Copies a pass-through input in bounded blocks with cooperative cancellation checks. +fn copy_with_cancellation(input_path: &FilePath, output_path: &FilePath, job: &MediaJob) -> Result<(), MediaError> { + let mut input = File::open(input_path).map_err(|error| MediaError::new(MediaErrorCode::FileOpenFailed, error.to_string()))?; + let mut output = File::create(output_path).map_err(|error| MediaError::new(MediaErrorCode::OutputCreateFailed, error.to_string()))?; + let mut buffer = [0u8; COPY_BLOCK_BYTES]; + + loop { + check_cancelled(job)?; + let count = input.read(&mut buffer).map_err(|error| MediaError::new(MediaErrorCode::FileOpenFailed, error.to_string()))?; + if count == 0 { + break; + } + + output.write_all(&buffer[..count]).map_err(|error| MediaError::new(MediaErrorCode::OutputWriteFailed, error.to_string()))?; + } + + output.flush().map_err(|error| MediaError::new(MediaErrorCode::OutputWriteFailed, error.to_string()))?; + check_cancelled(job) +} + +/// Minimal streaming WebM writer for one mono Opus track. struct WebmOpusWriter { + /// EBML writer owning the partial output file. writer: WebmWriter, + + /// Absolute timestamp of the current cluster in milliseconds. cluster_start_ms: Option, } impl WebmOpusWriter { + /// Writes EBML, segment, info, and the single-track header. fn new(file: File) -> Result { let mut writer = WebmWriter::new(file); write_tags(&mut writer, &[ @@ -549,6 +1110,7 @@ impl WebmOpusWriter { Ok(Self { writer, cluster_start_ms: None }) } + /// Writes one Opus packet and rotates clusters before timestamp overflow. fn write_packet(&mut self, packet: &[u8], sample_position: u64) -> Result<(), MediaError> { let timestamp_ms = sample_position.saturating_mul(1000) / u64::from(OUTPUT_SAMPLE_RATE); let rotate = self.cluster_start_ms.map(|start| timestamp_ms.saturating_sub(start) >= CLUSTER_DURATION_MS).unwrap_or(true); @@ -565,13 +1127,14 @@ impl WebmOpusWriter { let relative = timestamp_ms.saturating_sub(self.cluster_start_ms.unwrap_or(timestamp_ms)); if relative > i16::MAX as u64 { - return Err(MediaError::new("webm_timestamp_overflow", "The WebM cluster timestamp exceeded its safe range.")); + return Err(MediaError::new(MediaErrorCode::WebmTimestampOverflow, "The WebM cluster timestamp exceeded its safe range.")); } let block: MatroskaSpec = SimpleBlock::new_uncheked(packet, 1, relative as i16, false, None, false, true).into(); self.writer.write(&block).map_err(webm_error) } + /// Closes the active cluster and finalizes the segment and output file. fn finish(mut self) -> Result<(), MediaError> { if self.cluster_start_ms.is_some() { self.writer.write(&MatroskaSpec::Cluster(Master::End)).map_err(webm_error)?; @@ -583,6 +1146,7 @@ impl WebmOpusWriter { } } +/// Writes a sequence of Matroska tags with a consistent error mapping. fn write_tags(writer: &mut WebmWriter, tags: &[MatroskaSpec]) -> Result<(), MediaError> { for tag in tags { writer.write(tag).map_err(webm_error)?; @@ -591,6 +1155,7 @@ fn write_tags(writer: &mut WebmWriter, tags: &[MatroskaSpec]) -> Result<() Ok(()) } +/// Builds the mono 48-kHz output track's Opus identification header. fn opus_head() -> Vec { let mut data = b"OpusHead".to_vec(); data.push(1); @@ -602,12 +1167,14 @@ fn opus_head() -> Vec { data } +/// Derives the operation-owned partial path adjacent to the final output. fn partial_path(output_path: &FilePath) -> PathBuf { let mut name = output_path.file_name().unwrap_or_default().to_os_string(); name.push(".partial"); output_path.with_file_name(name) } +/// Checks the EBML document type rather than trusting the input extension. fn is_webm_container(path: &FilePath) -> bool { let Ok(file) = File::open(path) else { return false; }; WebmIterator::new(file, &[]).take(16).filter_map(Result::ok).any(|tag| { @@ -615,30 +1182,49 @@ fn is_webm_container(path: &FilePath) -> bool { }) } +/// Converts the cooperative cancellation flag to a stable terminal error. fn check_cancelled(job: &MediaJob) -> Result<(), MediaError> { if job.is_cancelled() { - Err(MediaError::new("cancelled", "The media job was cancelled.")) + Err(MediaError::new(MediaErrorCode::Cancelled, "The media job was cancelled.")) } else { Ok(()) } } +/// Maps probe failures to stable public media error categories. fn map_probe_error(error: SymphoniaError) -> MediaError { match error { - SymphoniaError::Unsupported(_) => MediaError::new("unsupported_container", "This media container or codec is not supported."), - _ => MediaError::new("damaged_container", format!("The media container could not be read: {error}")), + SymphoniaError::Unsupported(_) => MediaError::new(MediaErrorCode::UnsupportedContainer, "This media container or codec is not supported."), + _ => MediaError::new(MediaErrorCode::DamagedContainer, format!("The media container could not be read: {error}")), } } +/// Maps WebM writer failures to a stable public media error category. fn webm_error(error: impl std::fmt::Display) -> MediaError { - MediaError::new("webm_write_failed", format!("The WebM output could not be written: {error}")) + MediaError::new(MediaErrorCode::WebmWriteFailed, format!("The WebM output could not be written: {error}")) } #[cfg(test)] mod tests { use super::*; use std::io::Cursor; + use symphonia::core::audio::{Channels, Position}; + use symphonia::core::codecs::audio::well_known::{CODEC_ID_AC3, CODEC_ID_PCM_S16LE}; + use symphonia::core::codecs::audio::AudioCodecParameters; + /// Returns the checked-in, FFmpeg-free-at-test-time media fixture directory. + fn fixtures() -> PathBuf { + FilePath::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/media") + } + + /// Creates a temporary output and normalizes one checked-in fixture. + fn normalize_fixture(name: &str) -> Result<(MediaJobResult, PathBuf), MediaError> { + let output = std::env::temp_dir().join(format!("ai-studio-fixture-{}.webm", rand::random::())); + let result = normalize_media(&fixtures().join(name), &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &MediaJob::new())?; + Ok((result, output)) + } + + /// Verifies the generated output identification header. #[test] fn opus_head_describes_48_khz_mono() { let head = opus_head(); @@ -647,6 +1233,55 @@ mod tests { assert_eq!(u32::from_le_bytes(head[12..16].try_into().unwrap()), 48_000); } + /// Verifies both single-stream channel layouts accepted by the adapter. + #[test] + fn opus_adapter_accepts_single_stream_mono_and_stereo_headers() { + for channels in [1, 2] { + let mut head = opus_head(); + head[9] = channels; + let parsed = OpusHeader::parse(&head).unwrap(); + assert_eq!(parsed.channels, usize::from(channels)); + assert_eq!(parsed.pre_skip, OPUS_PRE_SKIP); + } + } + + /// Verifies unsupported multistream mappings retain their stable code. + #[test] + fn opus_adapter_rejects_multistream_mapping_with_stable_code() { + let mut head = opus_head(); + head[18] = 1; + assert_eq!(OpusHeader::parse(&head).unwrap_err().code, MediaErrorCode::UnsupportedOpusMapping); + } + + /// Verifies a decodable default track wins over an earlier fallback. + #[test] + fn track_selection_prefers_decodable_default_audio() { + let mut first_params = AudioCodecParameters::new(); + first_params.for_codec(CODEC_ID_PCM_S16LE).with_sample_rate(48_000).with_channels(Channels::Positioned(Position::FRONT_LEFT)); + let mut default_params = first_params.clone(); + default_params.for_codec(CODEC_ID_PCM_S16LE); + let mut first = Track::new(1); + first.with_codec_params(CodecParameters::Audio(first_params)); + let mut preferred = Track::new(2); + preferred.with_codec_params(CodecParameters::Audio(default_params)).with_flags(TrackFlags::DEFAULT); + assert_eq!(select_audio_track(&[first, preferred]).unwrap().id, 2); + } + + /// Verifies an undecodable default track does not mask a usable fallback. + #[test] + fn track_selection_skips_undecodable_default_audio() { + let mut unsupported = AudioCodecParameters::new(); + unsupported.for_codec(CODEC_ID_AC3).with_sample_rate(48_000).with_channels(Channels::Positioned(Position::FRONT_LEFT)); + let mut supported = AudioCodecParameters::new(); + supported.for_codec(CODEC_ID_PCM_S16LE).with_sample_rate(48_000).with_channels(Channels::Positioned(Position::FRONT_LEFT)); + let mut default = Track::new(1); + default.with_codec_params(CodecParameters::Audio(unsupported)).with_flags(TrackFlags::DEFAULT); + let mut fallback = Track::new(2); + fallback.with_codec_params(CodecParameters::Audio(supported)); + assert_eq!(select_audio_track(&[default, fallback]).unwrap().id, 2); + } + + /// Verifies long output rotates clusters before signed relative timestamps overflow. #[test] fn webm_writer_rotates_clusters_before_relative_timestamp_overflow() { let path = std::env::temp_dir().join(format!("ai-studio-media-writer-{}.webm", rand::random::())); @@ -664,11 +1299,13 @@ mod tests { assert_eq!(clusters, 2); } + /// Verifies deterministic arithmetic-mean downmixing. #[test] fn downmix_is_bounded_and_balanced() { assert_eq!(downmix_to_mono(&[1.0, -1.0, 0.5, 0.5], 2), vec![0.0, 0.5]); } + /// Verifies output shape and at-most-one-frame duration rounding. #[test] fn wav_is_normalized_to_one_mono_opus_track_with_frame_bounded_duration() { let directory = std::env::temp_dir().join(format!("ai-studio-media-test-{}", rand::random::())); @@ -692,6 +1329,7 @@ mod tests { let _ = fs::remove_dir_all(directory); } + /// Verifies cancellation removes both final and partial outputs. #[test] fn cancellation_does_not_leave_an_output_file() { let directory = std::env::temp_dir().join(format!("ai-studio-media-cancel-{}", rand::random::())); @@ -702,12 +1340,13 @@ mod tests { let job = MediaJob::new(); job.cancelled.store(true, Ordering::Relaxed); let error = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap_err(); - assert_eq!(error.code, "cancelled"); + assert_eq!(error.code, MediaErrorCode::Cancelled); assert!(!output.exists()); assert!(!partial_path(&output).exists()); let _ = fs::remove_dir_all(directory); } + /// Verifies an exactly compliant one-track WebM is copied unchanged. #[test] fn suitable_audio_only_webm_opus_is_passed_through() { let directory = std::env::temp_dir().join(format!("ai-studio-media-pass-through-{}", rand::random::())); @@ -724,6 +1363,77 @@ mod tests { let _ = fs::remove_dir_all(directory); } + /// Exercises every checked-in supported audio container without FFmpeg at test time. + #[test] + fn checked_in_audio_fixtures_normalize_without_external_tools() { + for name in [ + "sample.m4a", + "sample.mov", + "sample.mp4", + "sample.mkv", + "sample.ogg", + "sample.mp3", + "sample.flac", + "sample.wav", + "sample.aiff", + "sample.caf", + ] { + let (result, output) = normalize_fixture(name).unwrap_or_else(|error| panic!("{name}: {error:?}")); + assert!(result.duration_ms > 0 && result.duration_ms <= 200, "{name}: {} ms", result.duration_ms); + assert!(output.is_file(), "{name}"); + let _ = fs::remove_file(output); + } + } + + /// Verifies video and subtitle tracks independently disable pass-through. + #[test] + fn pass_through_requires_exactly_one_audio_track() { + let (audio_only, audio_output) = normalize_fixture("audio-only.webm").unwrap(); + assert!(audio_only.pass_through); + let _ = fs::remove_file(audio_output); + + let (video, video_output) = normalize_fixture("video.webm").unwrap(); + assert!(!video.pass_through); + let _ = fs::remove_file(video_output); + + let (subtitle, subtitle_output) = normalize_fixture("subtitle.webm").unwrap(); + assert!(!subtitle.pass_through); + let _ = fs::remove_file(subtitle_output); + } + + /// Verifies malformed, audio-less, and unknown-codec fixtures return stable categories. + #[test] + fn fixture_errors_are_stable() { + let damaged_output = std::env::temp_dir().join(format!("ai-studio-damaged-{}.webm", rand::random::())); + let damaged = normalize_media(&fixtures().join("damaged.bin"), &damaged_output, DEFAULT_MAX_PASS_THROUGH_BYTES, &MediaJob::new()).unwrap_err(); + assert!(matches!(damaged.code, MediaErrorCode::UnknownFormat | MediaErrorCode::NotMedia | MediaErrorCode::DamagedContainer)); + + let no_audio_output = std::env::temp_dir().join(format!("ai-studio-no-audio-{}.webm", rand::random::())); + let no_audio = normalize_media(&fixtures().join("no-audio.webm"), &no_audio_output, DEFAULT_MAX_PASS_THROUGH_BYTES, &MediaJob::new()).unwrap_err(); + assert_eq!(no_audio.code, MediaErrorCode::NoAudioTrack); + + let unknown_output = std::env::temp_dir().join(format!("ai-studio-unknown-{}.webm", rand::random::())); + let unknown = normalize_media(&fixtures().join("unknown-codec.mkv"), &unknown_output, DEFAULT_MAX_PASS_THROUGH_BYTES, &MediaJob::new()).unwrap_err(); + assert_eq!(unknown.code, MediaErrorCode::UnsupportedCodec); + } + + /// Verifies long streaming input never grows the resampler's pending buffer unboundedly. + #[test] + fn long_stream_resampling_keeps_pending_input_bounded() { + let mut resampler = StreamResampler::new(44_100).unwrap(); + let chunk = vec![0.0; 441]; + let mut produced = 0usize; + for _ in 0..6_000 { + produced += resampler.push(&chunk).unwrap().len(); + if let StreamResampler::Rubato { inner, pending, .. } = &resampler { + assert!(pending.len() < inner.input_frames_next()); + } + } + produced += resampler.finish().unwrap().len(); + assert_eq!(produced, 2_880_000); + } + + /// Constructs a minimal mono 16-bit PCM WAV fixture in memory. fn wav_silence(sample_rate: u32, samples: u32) -> Vec { let data_size = samples * 2; let mut wav = Vec::with_capacity(44 + data_size as usize); diff --git a/runtime/tests/fixtures/media/audio-only.webm b/runtime/tests/fixtures/media/audio-only.webm new file mode 100644 index 0000000000000000000000000000000000000000..dd4b32b39ef5f4a6dc6e110bcbec3bf8c7610468 GIT binary patch literal 1133 zcmb1gy}x+AQ(GgW({~{L)X3uWxsk)EsiizMDc7kT$Zc(8k_c`{XJh~Y7F$8z*(JMt zcXtP`ZV~ldnHIdJaUp~!7P2lI$h+aYy9X#I6mmE{cufEXor zjf`769G(||4NOXD6i8k)*}tH)*dsMDg^|&Mv0*n@1{5@Zi;EW|A81HFG0SyvyQ`nG zzl&>-OFOFLii@u&ALvd-I3xYUB4pK5fnG}nddt~(-|@w1RHdiEmpfG{_Wge$1A>HVX5}kSK4uxmwQ~4_{1H< z=pVmun&P^94}ux4rKZ};FFU=neE;;MgF@$f57qu>6Rpm3wJ}^8NfcyOp08T&do0^!DMSCOQgk zuQ_;s{=esi}Pf2*cOx9yo3Yk%cw5<@_bGEd_HgBgq$vOllsP`5K!VQ@-9 zfn~z~f8oz!wQ^sq`0ca!QX@ku+|$91rRVkw&3$!f^1lO{PfgNy>a~_Oeg682UY_pB zi-#N~I^Hb$Z=By^GXJ5=+^U3xpksSBuh5Lu5n6xo0>e-JYuo?NN{dP1U2tmQzD9;h zh6*<4XDzHB4cnGIeq^_tdx3|(ZR#GwiDFMhZ=7yYxhvpwZquvFPtKp3Fe~n+hnd>k uqEHdx;vJKe&hpJ}WG`LR!f^Iq0w}{z+}*#qkzsomuwePv#CYUGBQpS*Imk%> literal 0 HcmV?d00001 diff --git a/runtime/tests/fixtures/media/damaged.bin b/runtime/tests/fixtures/media/damaged.bin new file mode 100644 index 00000000..8d44e9d0 --- /dev/null +++ b/runtime/tests/fixtures/media/damaged.bin @@ -0,0 +1 @@ +not a valid media container diff --git a/runtime/tests/fixtures/media/no-audio.webm b/runtime/tests/fixtures/media/no-audio.webm new file mode 100644 index 0000000000000000000000000000000000000000..bdbc75e40e8f7a12a3e31ad18db1d29f209e4500 GIT binary patch literal 539 zcmb1gy}x+AQ(GgW({~{L)X3uWxsk)EsiizMDc7mJk;$pGkx3%BA)S!{1Q=fn`pz!d z<-5B(cy)`Y=gPF;HH`})Jh6~<*+AY6-`zbxIiZll>A`E77*!!$nc&?($tK3DZy@F{ zM1qZ@1p#u^CavomoB5p_d>eXw63f!ejPwkF$iOJv5tZ-K+2N20aRkWuC)Pn-XMeqs zvG{05=GJt@nKeGY<}@-WZJpCxnwQei$k5gr79SR1+1$ve{J6P=`}HA)2NO0l3hZhW znAOM>x}ha>XCouX_YMuk#S4-TG^C%H<+`}t)z8`A#Wl#K9n}HF#kZ0VbSERMN2JVB&0I0K-P#GxilNqVCI7V2{pW*Ja z0V^{l=rYvvN{^Yk5uG3}>)C)7oP`G^-go`U$(grpdSDT2hXg_En}c-JR-5_9A-4J)@ps@1URmj`y)U)md&I zwC-D_%x=cl`eVI=wo9vtcjC6_AzFs|p_fn!q7Z{hphjpg+Jt=kCO)mT)h_DUhH4x# z$5`oh5&NNY*4^c;@fQbk!dcPGcqTE6oJGx~r!y1T;oLyJgHTeM=W&hwTb|-j4`~gAFuv^qM?n-2mU8yc~ zXQmz7jH|(y;2q&t@mpz#oF?Zhlhv~722d3&1}^Lbzk)v{C*dcZrNJw36C4UFfm5Kr z`nNhzIj_``CrPIyR{W3fHa~@1#%^a0(kG}hVqUv$SmdHy(+LqJwBOdKHyNe9~Dms*1XxndlO(g%@hFHb<|dA2EiQFPL|& z?e;V$%kAJb@T&#Shoz&EaY>>SS&Aw}7iR#=vKDunKOw9ZM@lWESl+GlQ(SchkikT7 z7dC;P!F}*nqK+8a@K3l0j)x7vpI|hg)kz9cGUW}DF1;+iC(P&fa(V21#-t;fp*WH! zxR{H$PzZou%B$iwak|^1thMGfvy}0czE^)vo37FLzqlH@fTp3&s4|k0It27QYJN~g|6Xt_yb)cGZxA=g8&p1hg*ne2;lAhR z3OQmMsi<^Go~1NWuBwC80`M;Q1-=MBhFjn@Xe4!b@E$w@XTo+s2XjCLb)}l3jFK

sO04TN*R;e7%YG zotBE%<5$pav=H@1H4#kuibc<%rf4YIfoYtDFKL3Mwb7b*EwPqdORc7tGYi=-xKH^0LUXaCcvspe4^oOMyVVZrub>my2TQ;~a5cOL z3lgOV@G(3CzlOaa4>o|t>Or-h@{MfE?WCFF3BeS~@lCi)b`Ud+{)GCJ97~Li$3|nq zals^ix;Nkb*7?!Sw<)Wh`Ju7Hp!ELQ&suYQ40lH+T8G|24N!4JCs$GQG8%|hp#t0q z@7L;U2lZ~cWvnv?Sd}f^KI?3D7kHEXPl6A_!BKXcO=OdUscbrn>BnYrE&1w#BHS1E zO5^3m@?&L=`l7lU)CH>`2{YjW_%nQ%R1rZ0e}n5_HY^JcgYIg++Fd!Ul$Ud)y`m;o z7CQ4ox#{duW<9-)+D+~u_QZRl-Qk{KpMTgp=vg>UF9P3ab<|gs|!I%Fatb+E#Y){5Z-}q(pe_V zhdbaXSQ}ghLseTHuKcC6l9x!grBv|^VG_TFJIG#SZqg5^f5-x&ATEfWh6MrkZJ+dt zxK*5%c9u2AJZZ8<7k#}hX=AX1r{d?)DKrUnKoyag)FCFnuc684G)~3Su&0gJReh7u z%@obk)&e`*Y2&7OC49~g0y{J!BQ^+=G$>3zVeYV(xug7d!W6NWR9U(wFH_nn_tZ~R z59ENW@Fh4J?o9r_nbhIJzu=GXbC?bufoY(aI$vd!!SWGVkvfZ0h3))h4s#q^l1Zhj zQfXuwQ9VwJ(!v@+9lx>H*6rhruoqf~P2H?xjMmTTHMON$al9PYM}MHNP0R)T2ZzGkXn(w)$R%^Bee`Z- zGrN+T#Sa%U#Wc~D4#*>vROOi3L%jidfTOTH91Oop`U(H5kcaRjoDaK#7_0y;7u2@O4h6^qq;=vgv8d35@56n}PG#oOOQ>b!3SvdP zJX#*E2v++Wyj|`I=eq6LsaA$L-8gHM)<|eTf!m zL-9X&3@(X|qOquT;vN6%g_Y6kXgbQnX?TvtYBTh*`hFwJEMw+d>+Nw)Z@0Nu-7gmu z3qd5u5+RccCDS59GcNm>yTTt37K=lqhLSCBR^CuFbqZj?7;pH>Y1ImYhcR`Xv7?~2VNQ93J6QR*lc z)j{e*@D?}&E5l*%dw4ZDi)UVVA0C7Mg|7h{ECiL+b?S4<1o@g=MS4$MDg46gd$(NZ*NnOP9sN~pi&hnH#x2o3v;@70 zY9;yVSz0WG8lw-;78K(F_!q5{c2&FD90d zOQ=QkJmyPwJU4{zCe#-dF<)9Gzo|&d7PXan4zvTiU>u@u`x5=gepFw&H;~Mg*0wr7*w@4%9y7C=mhFVG8 z3TlF7AcPrk9y|dbBxx~9=GG;+9)18Tfa9RIdRy(IoK&jFqou=QAl4Ro@}s$V>^f#A zeSkVl9wm;(N1`L)(crj$#=Gp^b3ErcyP5UAx!SmE)YZS#3beQJUpNP=C>M=H>8R|# zUZ^B9I1|lA*Kl3DR7=qo>1p~YV}x15d}`&|bDW`WcQ4(4IY`wNbWIzLHJ&rAhnly`9~#7CDgA#5ik`zfX(4lcp!O?dX^R~cpdJ7V_;qII~cBp z>KNsr(otR|J(AMI{=%31X6|P;pLs;o^_lq1ICKf(eXX z`c^$fn}j1g16M+4(G=7XRr=$c+Wnm-ZUs;B zd>;po!w1oW_yO^dd`R7=Z!_1}Gu&Q&nJ`AokjhFogLwwvfx;r(Z}21J9nepEkf z6g2f)dtKcuXM(-bI&B7KV`GwjNw24^(#qg9xH0+@%|ks6kvT+8JS-X(<^&`Bao!j1*Uk<*&vLCaGt1bhdwNgpsMZMo zh%*pIYtY-M9xC>)7ZyYHQ5IT@blepm(wb;L>3#Ij*kWc|wXM*;;_P)-c(eRT!KdNK zXm~uF7(tGphSNFBV74FEiEkj36bi({(lq&1SyL9PFR8hp5m*OV*aI$xr;`evrNsi6 z2iL-PU@4FbGStgzXJwyKQqGpPiVwxILMuLt8_#~lETg}rHjtZ$P4UKPL%1>6;_vbf zyBC~$4r8ZTz05hrZ$?FZtae*_17F4OVHWK|IjC8363@I4pc<$rT7YiiCiq*eytY3d8NL3Xu#j$HaG%S2YDb{)zlA_e5I*8SNdHlC3X}(;TLl| z*%Qof^mXbsd55?g---ST?*xDQPdv+G-7-!?yN@;9JZOYQdwr!&Ya_6RC*m^bIGT{C zqC)a6MJ6hF0d+#3qw}~jo~aSq6ur2-f?T?)dx4xI^rvySsMMM-630x^8 zCxzssken2flR|P*NKOjLNg+8YBqxRBq>!8xl9NJmlDtn9l9NJmQbO>IDArst&pyDO#tzcWdY5f#vO;t6w;J|WQvp!s$$^|M?Uxc_TVXuEfjiCKV|+$R zl%Xq7&yvp)&qkHQ%0aSU!>jAIaJtz;tvTjiJ<#(= zMkHcUCDa;?K?g9zBk>(IQ&qKjdL#XB;~R5?)!MFPN6tg{5AVExGB_R{kB$>3$P?5t z`f%bqW;45lo5c4Q8j6JYlQcnoK|ZAnP;P^_z;T!ihrvzo5>#ObJ@_{~1{c8BfCrX= zTFSRd6?ukqQ>rUw3!C^WT)P58^K8pvS0Bu3Y2LKHvzIuN-C^D!zjyFP*gfh_^dP%auhU%< z9hjDEZLSPw^S=sTi^HT8sX(5pR8%&CnqUcVU>Eoq{3$*OFU~9#UWS|Da99nT1Ot`7 zm2~->Tvy5!Pl}B2AO0O~8oQj?kvK>nr%sdSh;z}|@N96_zu;YVZ#snzV^_1og2 z{jvTZZGmd4S@>@}8ka-|(Kz%9s*Jcevm{g#y^iLfi?}ZST$R-MS~cy6KFoN|xMS|H zX4sidC%3Wpy#H)aJ}euRCCZWIsB(1a1Yj7(WN&fD`8C35@ntcR_Q(TdN0|*IFa_L! z&EdyzKfD>EBZ3zE6YhnRU}Nwn7zb!2S4MIVX``r#4TN|3Mch6%pShPX=#WlOEXfgU z#D;9Z`@k#b)^M8JJ*~0kI^(KQPJdh5r#++2RB8NQ{5(33W}vR9IuheJc=Rl4hbE#Q z@iTZLE>u5MBke1_t!^9p&5x~~cB)g_iQGc(u75MQ9$t&C6W7V>Q~`ZCagI5{e#VOsEt-`?xy_IF0uv#d?#uO?%()@N%4T66Uq zH5qTfFQHrLbJQ2rK`_oM20epXpy6mIrg0{|sP<6*)W+(S^k0lQW)G{n71}qQQ||ZP zR)0gVE?gU}Bi51Ys5SJ8#OKT`b__R=ZzYr!?ueVD47sGdN9m;e3c7&(unf$AYv2V~ z7$e<>kKk$eCF}z^un{y<4k``huOv(AAkGnv^9EmuYtHsyG7=-`G1N!o1Y$xoAsin} z^mDzL?jq-F`vlxF<5udNdd{Mx_xQkCLbXN=K_vA?|{| zR~xAZwKp_VUvCUDtDBm2#@^yAcBgt{{11YmVOEqyWRXLuEIKnWfa$@u=4$aWe^1ya zPLi5QkL3Bv3(6i)AFPHX>;V_UpW%ZziV!0B8(a^wU`22k^i&Fzp7LS2vXm|E6I7u( z-<2EA&SaJ)HqhItJ>*_uZ?reu6YTZ(dxzcA&L1|i6|13{ZhWa1=qcJ%^}hNRzJ}k& z0@{a0qE@IvTz5&NpcM2bnvbsGCU})vPF=30Y8Uj0MkB*Ak6Fv?an3-uv-gtUI7kia zMfHe!qYa29C9gl519Y2dsqFmGoRYgJ^hY-)NqN(TRmLbT z$Oc#7i*Ov=75{%Dj>Ca}!5`o!FbzBeGe9Y2k&=*yN=GDF>?%y>cW{?j%(6_`L^55I zN+DB-T2V@v64drny=HEEr=Oi;eQq8$G$U0Xr=8X6sLRyScm-~R{y?9hp6G@6EQ+|W z0(u$cp#50FAK)A6K=qOKiC#y)VXQC*TTShfw&C1x&wI!GLqT5nee^w%N9Ixc={<=p z%qn&+H;V5eqzIOHK+2Jm5cD5$p~kuoAqe>{M#X z^Q8My6Y(Qq7k`WAxEgFrrfZ@v{Wg_BW)hiEMwk&~`a`{Jcarm&y}|m~EHtYc{q;4P zroFD_sSWUVxHI|(twL|3`Y|?&5K=nE$`G^xS-2NIrlzToDxkTa>-n33Oyl_!(_3&xtIB5-VsiUpGmJuuDnudq8tG&!DcAHzHk{l z10Tms7{Vv;JX{UaVJWZ=ysDg6+RHm-APo}N3pa(5d~>cJJCd26SU@kOmXj-qmC=fD zMX=Ie<85?zJIC#7mTM)O-He&~8NIxgqZX*$a6WzqlW05o05wPD;(A5Jb*m2QhnAq* z_+`9Vt)_0$T4^`*xkfuvFfUtM?Rm~9H{I*$cMdv)?V@%>d$K*%j&7Z3!PIA~aw7K+ ze^OW`W=K_~-{kR18D%Y~3Kjwbc7$``(YWTiab{_84c-Djgf+ogkfA(Q-jy%Pjigy( zz6gZY{QKNIb_0`_I7#PIm&vQd)#yriCAjJrc(>d~j_Z`L>sYUwlZ@TEuBT~BRbL&B z|H0#NS#%UlKy730_?HW-qu0<(l#f&Je3emWYZbNc^-QCJQDAPcCfa?SR&Fh?l3ywS zp%jUPM9P#z3kf>mFpt>F+yQ=xFidPBTGAHzO<7f@0S1f*H(^sa8SaBOVje8w!W-~A zm}rKSTZQ>m-ni-#kgry(X+IJS`~GkD&R%9Hu??CM%_@2n9s>LmddCN`UoAurSMq%P<>Bzw8eT$ z9U0rraaL#hc}H+`_m206e=*1pFGTr7KABIQr_UsQWcIRaxLN!Vp`|E@XQXLzs(eAo zP#%D{!D(0>j)337EAd$rap67qBm6IX6>PCUdwBiacb)O}B5SvK%>>4)`U35?_KLbyt%wgt24Znz%5KGCW z)ED%^#HY+8b{O{t-$;;!0&%tUmMqF!l{U&*&;jg*NpKKc1%HXLQKS|h$GY(=_!gAF z7Vxt2lhRCHCq+^>ak20VAM!Q1c5HuUcw#*LG4%;KgP0M063z%_`g6R6?n-C7{j+t~ zlFgRJXgyCCw882*wGI9m_eBocgtAbRSnn38#Uzx92B8(`5$=fhsP)u+S{Ln!zTD_# zRx}@4N9{GvYL2q?yO*51wrf9QwKU%|*64Ti`r4;zq52N~3uj{m<)P6i4OM)~ zg>qbjd!TvfDz1;0sY&V=T8eg3&oOEnPs}`PzCGOO>85!N{FI=2SS_kXJV!o9Ril#= z6_^q%$v)!F^WO@S#f}mu{UB#5gz_0E0j7iduoavR55)JVBDHA3Yj8Il59@>9!6+qA z#>)@n&eCe}p_n2JjysN$!-NF)$VK!HB!8V`}%mL+(g$_1Pq*YN9q>pK3%lCK^YL!bU+; zzlGPABiPt)aSFt$^3!X6R3}5cNW}5Qve^pz`P?^dZ`VIXo2q zuJ%#yXp{Bo`ekFW+25*b^Y%mMqI=By-rp5$54T0zh;8IHYAd}lv6@-Te#~X@9fc~w z6LFXHzFb~DpuDbJ0o}n5usj?BzlOiY`l?7RK7=RX7qBO!!8*`L`A(@LFO~|WR^n9Q z0RM>6j@-roKsYwchGcwJH7q zcS9JhMem@7sMJ#~EQK1OOtcPZxH~?iHdlYr`e}i_)yOjInSpiL-sh}z=X$yRN5SZD zR5XgnA#FHRj9mg~DNRhV+W0%1e|Nl|0ZCtcMKj1((26aRfzbu@L6N zb#O2&2l7BS<&x4>-Y=Jxvczq|1EC_{hRb9pF`p%t(_d2?$<4&(Xj8Z`*yL~ZcDsk2 z^Y&dkVWpUTjQRR+dR1+LdP{v1U%~HU2JJ@KsAYT-MO+9_ZPXhrMmKPC{IyzHU8gnG zuIST^myF0fXRWiRIz!!FUPnJIXc0Dxni0*(7F2V(X`((;gDuMu+-?4lut4l5JtLiw zM=3yA4l03p0K<0hQ+PPes3I=Z;qP!e%z?E)KFCs3nX*K`@Dg5AXGXU;SZ>Ve)-Tcy!z4p#9LTmcXke`vogX~X=uT9W))Wj!SGy^M zR_v9~qaxHG=@M*GF(X|4+~KYQMJc2Yf%Yk-P;`0Fw3|H>KePoGD!RHO4=wJhP^l<( zCy)IT3cKvS>~D5<<}=^fr}cQPoXKP+E@UobG6Ux`XaAP{rQquS6mMr$2W&FPR0CNkQsP7_{Evyv;P?S>sP|wFY?d-J-PVXKf0H$U;6haB$Gp}GG^#; z7`fJZodJ4?SNbSv#6;^@iH4NchNe=5A@sOzqepl}0?CDObK?YmjYGm3KSif)3bv?k zu1>;~#{`#zi{+Tk1WWkM@MwCrQnh%TFw+l3eTqGwvPtPckpK}c6-#;GATFVZYos)B zqH0^L#9h+j!d0Qhl^S-zMjv2LJyMa|MsmlhxUd@4X{+K&x)HXR0(u2P)$&70NB*3X zn&21NgO|eK!Wo!;L>!^@F%eq32a|n%bDP-228rYRgArUU>Xd+M5?E z`YL7sq0?ibsE+y(kDJRH(O^V%Ks-!{U11FDaQY&_t=O#eZK}?%GxjK&yiyDV_Gb`B*!B6vw+& zM}r8MWxC+MvJ1J+$+<%a!xMCm6;V_J;qLolu1TPoc*4hlceB}>WoGFrb&9|1fTk(} zqD2)VH;!@LF)u;ds*{70bC!FLwP_f3ZFtp{2#_`m^^qhP-v%&E==7o`*wqmn%%i^1 zqebE88@5)TvBY!Lu5Zna*c+i>4k0kItv>VEWZ0(KX*F_5(6CXi^c zlD0NMQ%Wy?o{M9#?-R12_g7Y6>cSLoDE6He&%h6dE_HVJopA;(qb`iALV=+OZnhSd~Z}^iS=7}8z zRl8MoREudFOpJ39OiXpl4I!EDSik=8>2YItVDQ5q@-OZ_x%2c+=I+4V!8>P~y<+L( zp!ChZJec~)kAL@GX3*WceSYHXwRZxw9o>*-I1dho*UjJ&WQWdVYwLV7chC_pUC%Cj zdTw}?@yG8L3WL@Q=8@eOO1opj&C#LNSdNZu>eiUPKL&>dd-C$ayTysG?rkkjEg#G* zY%PddF+09`ZSwN<;oBD$vkg{j{Ac<7>|*x0-1dwsn4DTRJ*{r9vrW9QTicv|?iu>v z{k4f95PQ<%YuO9yK5AZmaY&-=wLD`pVvX$O)!|5)Z0?y-r!;YK(0rGQrJc=UflfN7 Y+vguTKfkVL)|U#Sm9eZh_U&){7f>+5ZU6uP literal 0 HcmV?d00001 diff --git a/runtime/tests/fixtures/media/sample.m4a b/runtime/tests/fixtures/media/sample.m4a new file mode 100644 index 0000000000000000000000000000000000000000..205fd8fa193bcbd54eef7df7d14b05209ff6fb68 GIT binary patch literal 1449 zcmZQzV30{GsVwj{aa3SnU}6B#nZ^0JKy1Xoz`&7Kl$r_@&&y3oEV;+T;FDOEY-Xfq zWT9thV8pa~jdlu7z$RHVz9Kg6xRvHP90?#fL0$Co}= z-&=q3L9S0|a?Z6`pI@u0&03YTvum;1lubb=GncCMOwn}nG741NV%B!yz_GK<%t|gi zIX5?FOjMe3b914qNFWS(bZIQ=(wHbHC?qHnDCjDr)TIOzdBA8f<<`mADZ#5XuTFY- z{`1>ovu(2e|GNJEy42(F_5Zg%{`vXXj;pW#Kl=W!=Ht4pt9k!zGpQAk;9lUfp>_Vs zYx$bJI}fB=@bJ77HZU;W;o$)x42*ZTw5*Ykki2`y$QlGW z^}g!A^Y?E*x+`<(tEDF%Px4e!;d}o5-0aP1X7xpqc09(v8z(>h#_NVsS+tVvU;OZ<8~xXMYZU@b>2%MvI`yQrGQv|5Jbc{Pv^u;d(LCRSK4DdZb{_&?uHx z^Sgk7!QkP;w>uixWGX7ZH!!yM*lurNIDAX4_6>u?|=SY?8$ujYdECv>Y$-pGg_zo~}2(6NuQc?_L%B1ELmq4XJI)LV~K;6u^GdCqO5h$jR zn}Y85H7*Pc5^F+_fOYC-q~sJq&0r|bFU^CoL3A*XpW>02my(kTQp1>%o0*peWC#@J zq8O@@0uh%n1Q*Wp+Oa>j0H%GJBWgW7+QaTFe^}mRfr?(DFYjj#R!ZF28H4h za4>-sf%t6T;DOk(2FRBIx+EDS2?QXUL6{v%8vy01fNU55iO&S$ML=u=#CAaJ2*jYM zW(8uA;*#WiAPoW_b98`MsyMwM1;i=J&jI-y4$Jur&lC10w+a Cr=sWp literal 0 HcmV?d00001 diff --git a/runtime/tests/fixtures/media/sample.mkv b/runtime/tests/fixtures/media/sample.mkv new file mode 100644 index 0000000000000000000000000000000000000000..25a84e40b90d3f5fee9f68754c6516be13729e3c GIT binary patch literal 1222 zcmb1gy}x*|Q(GgW({~{L)X3uWxsk)EsUtVBq$s~QJJG2fDAd}>BoW+@&d2})ERzI% zXO|q<-@^QL&n@5G-NCC{L_Jrg1+Qsb2;qr^tjh-SKKkzN0m=!598M2j)5I76;mQQ> z21+)ul)ZtN#t;lPl)41SJ*W4#{92#guXTN6GryCAZ$qz7Vp*D*k)9zC85o5-qVkKE zOt`V?+-I@2jRL{@4#|1T-Su?o>~KhY0CpJ2N4NI3u$*|aWF6Gm-q#x$i;s4kaPQjl zA<6ynoJIzvt#g`7^HMq*8QNMM;~gEH!&bF0Ci#7CZe)5maegD?)((fK9StD!l2RHK zLfWM&~CIfwHWC8N2LSkZa`iX_Ei#uFGgB(LV{rz0pCmR@8fsvk}iLn6~ zFvvS^D|MQ^zlAL)>EH84hQ$tZ8yOfH?lCc-n8v`Aury@Ctsa~jd zlu7z$RHVz9Kg6xRvHP90?#fL0$Co}=-&=q3L9S0|a?Z6`pI@u0&03YTvum;1lubb= zGncCMOwn}nG741NV%B!yz_GK<%t|giIX5?FOjMe3b914qNFWS(bZIQ=(wHbHC?qHn zDCjDr)TIOzd9c`FStEmZ1Ea;1TPI_u1h3Y-I_c&4&u@>-w#oYc>-ziaQjf#e|KIxf z=jUTPuD<^N==;B#kL$Ls=KZ(Lq*g?Ndx6h}*7+;1$A_Y-XFHqB8YCrjwIrTtRZ$ zAt$fqv*u#0tq<3WnXXc>WYZ%Bdxl1_w3^=q z3=9SjAHLnuz$Q~s`MrU$y~lQY1H<84aTsi>FS^SEmg1lZRgJq^6k$L)JZLWw|{T(thUtbhQ*zY o4BZWk5{+95KqbfY{VfrXq%?Q;Z*FAR-UTeNJ~lC4{m{q>0D}$-4gdfE literal 0 HcmV?d00001 diff --git a/runtime/tests/fixtures/media/sample.mov b/runtime/tests/fixtures/media/sample.mov new file mode 100644 index 0000000000000000000000000000000000000000..6fe861e8d3b27c3cadf5f7ed084e22b7acb17d93 GIT binary patch literal 1500 zcmZQzU=T?wsVpcgQBYuDU}AvK3>@W|DXBnyUT#Wa$vq|ppTx3cGb24C3q3;vBL=2~ zr6Ch;6&Za#zI>i(T+!22uT|WqOwvE2B3;h>A#U}J-S4DzS7ypTzVx~J-ujada(zOR zbFR(${909Q)~ckPU5nMGYzjJ=xm2xZil&>FQJ~ruv$hKdj-730R&wFVxw$!GqSBO` zn+sh<0%6FbOJh-&#za9uAwiKqL02KAE+wGI14fG}w@$`R30|#vb<)f8pWhyvZIku? z*Y)?;r5=Z`|G)L|&(FtpTz&oj(f5BfAJ=VN&HHbgNv((k_X3{{t@Br2%h&APc_7__ zhv%KJfr0T34-W`oV7$AfWsQV{XWW#vFP-w!@&YC;*~~UY zMP=%POeZJLxPs)gLrz}JX%7w@1JWB$A2AgPbX);+?Va3`zypj{pSq>4_f`L$zkmDD zU71T?Ej{sglBbdi-}CS1W^Yb2t1ptY<1zl-IQhY0`#s8!AMpHpQ1FAg<^n?U(weeB@$kkoRBqe!8KH^UK}wr=P4? zC_VeXRot>5UHy~4rRtTx?fm&czWw=uI;rLF_U|p8)s~vwz$nqU1?V2;pxpfYG6n{Q zoZPaE6d;WRm|p-n6%33FAcO!Y9H1~7-vOZ#8;c2;!uOUGC1wL9V@k5Yb}}O=gqc=> zECv>Y$-pGg_zo~}2(6NuQc?_L%B1ELmq4XJI)LV~K;6t_nwyfD2ozJuO+okj8W#oz zi8b~A!8&y_QgVtwtlW&8qT>A0Jg_K~;0Xo_qE`;fsmzED?0rfGk z0rhG!M4^FdbT9`E>_rFn(7}H+0J0hA$Yh`+jVwSM7*JsmfiRghgEbf1gW3Oo3mjo! zU?~v}4Vlnju#9uPxs(DE1LImt2L{GSE8Bno4mM844>j`arOY{ zDkH1E_X2~$?B=DVeGhyZfHFH299UIZ5)3!|@Lbwz+39H6)Zn4Qq%%Q-Df{l-|Ldc*e21V_OdALagUlL6Whe;wVaH>8?D*@?+5zc_x>`23y=9^4KFv_fjq?0 z#L6Ns!TVduVN>&x%=sxk%?9hG9|xS1F?lWR%=sYK^b8YQ-ip$?Y_^k27oNZVg&`zB zg2BF1EQurkT*3v>wsh?sX&$1-coX6Zb&^FA`x0%MCw=h$ZTx@x|NpCi-Z{XSzyR_P za}#TXy!x8oO$v*dmwn00;bT~!p~l9Sd?F!1UD!D<;)Fxf?5*1qC+@5eX{$T`y?*(-!}9ZeGHwAa5^<37 z0C|YHfi*|gaO(vIgB5*mzl#(wF|51)(Lv&1L&M=m2IsT`qQVl_9obZAA3wA5-;=ET z|G(}2t$(zDf$a(pJ9FcK13;4)*%wAV_K0X#{Np$OV|5+hp|byKktgaO9al*I?D$Km zk#QzCh8S30xJF1SH2`&(G&eA?=rg_2QB~HdR_I&fpw{2R&$a&O!2|oXWQ;#FY$*Pm UFk|}{MlBylUsn@7gH^-=08za~jdlu7z$RHVz9Kg6xRvHP90?#fL0$Co}= z-&=q3L9S0|a?Z6`pI@u0&03YTvum;1lubb=GncCMOwn}nG741NV%B!yz_GK<%t|gi zIX5?FOjMe3b914qNFWS(bZIQ=(wHbHC?qHnDCjDr)TIOzdBA8f<<`mADZ#5XuTFY- z{`1>ovu(2e|GNJEy42(F_5Zg%{`vXXj;pW#Kl=W!=Ht4pt9k!zGpQAk;9lUfp>_Vs zYx$bJI}fB=@bJ77HZU;W;o$)x42*ZTw5*Ykki2`y$QlGW z^}g!A^Y?E*x+`<(tEDF%Px4e!;d}o5-0aP1X7xpqc09(v8z(>h#_NVsS+tVvU;OZ<8~xXMYZU@b>2%MvI`yQrGQv|5Jbc{Pv^u;d(LCRSK4DdZb{_&?uHx z^Sgk7!QkP;w>uixWGX7ZH!!yM*lurNIDAX4_6>u?|=SY?8$ujYdECv>Y$-pGg_zo~}2(6NuQc?_L%B1ELmq4XJI)LV~K;6uEHa8_R5h$jR zn}Y85H7*Pc5^F+_fOYC-q~sJq&0r|bFU^CoL3A*XpW>02my(kTQo~r0o0*peWC#@J zq8O@@0uh%s$54v<|}Tv7~+43L^UU=aawCfgt2T_m^L+cL^W(A6{3UP!zWncrc7=cm2Ad*y4 z1PvaD8lad$aS1r+KpH`6*uX&sv3U)UF9UR4GDs2#K&}B{b|`HCl&b==VE`mP6Nnc9 zu?-O00kI8%)TgTOO+WuTGm%GiXU>l2uc~{_U2cxB5 zO!qD1k(~W&_qM$iGY`d=&iUhQ<=UwF>&qIag1#M^PCK4W7G_)=uAJ3M4A3kcLqu}ma>oD1y(-oTM%QpA*Std2F-tMBnePPYYDS7-X5%$vT3565pGj7#z zJ?*jO@HSqkF^0zbs~aBusi}Pf2*cOx9yo3Yk%cw5<@_b zGEd_HgBgq$vOllsP`5K!VQ@-9fn~z~f8oz!wQ^sq`0WGpVz6WBx&1t))$$zrLcEr+f0^AxDXhH;euo=eL;5f9Nu|Dj^~0*q+TRG-Gvy)?d89@KgWV z_W!feVp4c3*qooWuzoabTlV;o-E!^)9{RSadkiOvJr%uix=H1(fYZ56uP#41e`><4 VxSJkkYIBQ1MTCoYOj0__2LQU}GM4}V literal 0 HcmV?d00001 diff --git a/runtime/tests/fixtures/media/sample.wav b/runtime/tests/fixtures/media/sample.wav new file mode 100644 index 0000000000000000000000000000000000000000..19a3e1b39db17933f9508768491f28ea9037f0f1 GIT binary patch literal 10662 zcmeI2_jeW5*T$#!IfqcCNkBkDGxRS-1nB|>0@9?2L=B1%B^2obQUt!z1VlO_MS2s1 zGzFvzQJMw>q)3yTGri9{_`d(en>%-{d)NKpuG#DCXFtz;*3RtMuHDKsg6P$%XWMr^ z$W1Lq5CoZAZMG1^%-$qH5ygnk9lQ4^m;B!GwRRaDySM8BlJA`dje4h1y*hQ1cX~Y{ zYf$c>-pRitFES=W%AubyH`%k?4t}oimRMVQB7H5traV^0tFbx~6oC409NYu1Lkq^x zfwy2DoC=%6dteFxV6GY}gOvR;Be#*piW`K}{5>vUWwsPkj((PUmVB0|7*~u^!)ifo zznPce4smANd#wlN^Jb25T(7P#)S$K$r{l|L4$4H&BL$I&LlsafGzuL+5PyX4;4Dqk z=IRZMzm0Fq;Z`fVf)hIr-9NnZ{>k8Ycsx2DpCC?<$Ed^fcg$vX2{(c7C)5`S=_hHN z{DN{y>95{a-vGx!DjW(o!Anqs5%l5T@EDvAUxPka25NwB)k^Ag<)&O)&XG2WSA>u+ z&)4H#VY@Ir=w4KBvNzEy?iKY3`v-6PA9>T<70yBXo>j)`ZhmPz(mU$AwOZOP+zvlL z%Ta&yB1%CNVo)j67!5(2k&oWMr*J#%f|jkT#vx;@+0rUzKd{d@yWO?k5`SJWC!8J4 zj^_|_$l25kdJ;2&9mI9yYY3G1tGGn!E>p@kN^|vu+6rt3D(nZB!?W-o7$(<0@ElwT z2S5mSf;Qlk+EU%5a7u4^g>*#}#B?Epf0z51{gjzY&!-lU3yAsg{AfYAI9Ta#^77n# zr_e5Kx3wmjKN=~|E5>yvThZpotS zK@}07be4>&qpoN+x`=AyFR-l5)2ir4^r6Oc<{fi~HQml~I=T(L=ly4cvSI0{bX>J)F_@I_s)g z#(dM*r$3|5&}i+y_<4LDO-Ef&RU{>K2aehJ?~U!Xpy7J^A%vFI7p z3=Km&5skC(MVzVqsg2ev7{3^^%}lGR9oaXXQ||ZPR)0gVE?gU}i`NnB$Tido`U_?z zJBk~?w-8E;cf?K7V7a)mN9m~ks&)qZK`A&Gu7MX|VWRW^K7yy=m#{bF;YQE|98~M8 zUn#cSUY;!-7fqpp(3H>Q2D8JNQS`^uIC2~@E*=|=4<`pR{Dt1v?hj6ZP1*IW56qng zWem`N)>>#k;vU#U>(Se&Au5UJZ;oey;hwNhoFFxkAIbBS7t}p!U9cLEFcU6< zKf?z}6%j=6H@F^V!}9Pj=m84U9_nGGqLL%;lQgla*hLt|&)}A^8<_3%9%?VSm)INc ziS~y3gTwx5?++I_s#D({WPNEC7-_~7{l4~wb`8IeMZ6Dvgj%3-$=oH8iqg>QXdb$T z8sSyAjJ8}$*DvVfjRvM|9Q2f1!us6@HTXl&N5*E+zCg+I`9%025fbN`j^sLSt{R> zQl;0$$--LxAa{Yi!Q7|+p$f@DqA-3E6^1ylebO)HRdZW8S@t~Zgvpv+jSae_kJTJJ z4L^%dqRFTus*J>>4l((B6-_~>P%55|Jv>2E_04*BLo`pBi>z$Bt&`@K@;E>6?Z60) z$cRnCATjlrzRg@>k8+9w^ zU)YB&-Ap&f7-#ib`ZBGgwgNZ6f1uA%5A;HE7Eisf9C{h$qWwt5AK)8!fc8lHRIg>+ zFjkmvTaE4Fj_KTR&wI!GLqT5nee`{tN92+FsXg=-W)(Y!8^LD^X`(G2kaFczd4LC;YEM9>5cDA?yZYuoAoocB<9YxypUHk^Hf=OS~oULN&fQ*M;rF zyh#tHvdAoAa6CB53Wo$a{sixHcZ2h@U1(Ld`k8ACUGJ*rY4x=4a3}l^T7}+3brWqo zRY=K1EAOBU$VNT!G2Bu+tqsyyW3Ta{+1TRkoAwcBle@&5?N19Phm)d7@nm8$If)ua z=Q7!BFYaYNRd9uq;^)$!q7waj~h;m;Z>H#?EJ!(#xrpiydqi|t_e2!yS?M?HOI44?F?&% zdB!MfUo zt3B5l;STb8_??3GVcV!}+>U5RwxwFp&6v7uWlrM%;ZF+7#KBT!`8Rp2Qc7K`Rt5`z z2|Bq^kwQQ zd6l>lUx}`U1;H)|O^+*(dodxEvwG>n$UQa#XyY5(A{xHLYB#-TQecl@sxRz98;K7-^}-wQJ2)9Ofq#JsKm^lOOX;iZkbS9{ z^r5&yIKp4!FdH)x1L@*a39>km5*Lq3hGl~)e!ADj9q4>wZ?SHgmCS78pk7IztBKk| zToeC>W}ysJE%9?Qsih)njXp+)PzgL5Kg928uC_>TZXjd3ImYT_KktaH;okB7@Gl1W z;e{wa&L{H8^VAvoM`ka(hMUR1BQzI9>5Md0PFF4{gVhJ>o8UC43WvjQ;g#epo_gUu z_#^x;d==W@3s4oTSD#TQDp%!d^1IS1@fShoQ~2k(#_TIh2f8!Wh3rgpiaSMJ!pxwr z|E@RIUFht#uUWu))tqnK)?d-LYSpzZxD~#OmZCRM?Id44O^anv6ZAgXiefYn|AISf zSF{|xlySnCVs^C3*}8qnIpl8j*7{!sOT)#{l6Wbxlw3?Lpg&_KutT}-d;>ud3&hpZ z8?vNqRa&cO)%IXFNPz?4D)>vHji+hxagrOqf^R?>ZUHZYpVTJmIwh7ft=P@2k!GGD8gJ|8wAR|sxDR&G zCX|gDC3*L0T1-LdXdqgF9-$6+53Zx_(>m)<^yNlRv%K}tI%=MYAvuFgdhVhfXCteBrQhC+`0%i z!1rM#cntIbx75Ds38k7cMm{VBVjZ!UFos{it!H;J2k67pQSvBpBt8-y4UYw<{Y&0m z*K?k6n%nPLYs@=FUE?#oPoiadBpt88elhdD!G#9d)R*z+z;6P;D|DJw9Wn3Ii* zdVPJhR!&=so8Uju0@M@LKp;^%i^`&x(1&Ob;_(ptJMOLB(I)CujmyR&v!7Ml7MzF9 zMfaHZy}v8i9&U@a#oLH&a5R7V;G7fcQvA5z_hgTyJ(T^Bz5%${}-z z;qmY&C(I4T`!l^S-JMRp?OJJ8mbuCB^j`W=t+Dn4&cGP0MQ@?{sKmcsSOPUbS!f;7 zQ8#=DH`RX9`s$&+)yOvMSfO><-sh}z=XjI-kAsonh-gHdOXQLxs2q9-)1U3kHRMYR zg~DNRy7Y>y$xD+QBS?l zfWO1-Fc;Q<`5+r;>IZ6p(oC5z|1Oo0I*Fr%CHyY#IQtuOjlMh~lV-oD`9hB63nhPKwA$5jiO$Cq?9>h@2FW ulOl3bL{5sxNf9|IA}2-Uq==jpk&~jGlcJrIqMehXos*)SlmBl!C;tQG=cG6Q literal 0 HcmV?d00001 diff --git a/runtime/tests/fixtures/media/subtitle.vtt b/runtime/tests/fixtures/media/subtitle.vtt new file mode 100644 index 00000000..07059e48 --- /dev/null +++ b/runtime/tests/fixtures/media/subtitle.vtt @@ -0,0 +1,4 @@ +WEBVTT + +00:00.000 --> 00:00.100 +fixture diff --git a/runtime/tests/fixtures/media/subtitle.webm b/runtime/tests/fixtures/media/subtitle.webm new file mode 100644 index 0000000000000000000000000000000000000000..2f1bf360cd1e0f03252e31dc1af043b9877123a9 GIT binary patch literal 1158 zcmb1gy}x+AQ(GgW({~{L)X3uWxsk)EsiizMDc7kT$Zc(8k_c`{XJh~YmM}rz*(JMt zcXtP`ZV~ldnHIdJaUp~!7P2lI$h+aYy9X#I6mmE{cuf;y9fT_ryc;Ol#A5gcVh%$j z*f?4cAXjbDy1ucQ-^szZq1Pv|EX~YF&k%?VjKUpJ`7WIu4v`LESAd-FuwWh3c`?@; z8HIrLA+COY>4X8X4N!9OL~1LW9FrH7DyW2;1Gla%2lbb0gEk ziSrv7w{|!@F8~{ul+q}WylAq2L20o^YGMi_qXlEbZmC(a0v}^4Ds~$b7`M! zU|y152f+;2Qd8~amz~~OzJGesL80@#hid;a zH2nW{tYh0ffgS$Gl(S#8pJZJyX<=^@Fp9Xi(lRSbN{dowH>NB;)yN>)P%%$#;qyOJ zef?HRu-Wa8e9Cb$;GCFEG(#?{`;Cy`#A(+=7O9n6$JKP&{#-AYyUnU#8;gc{SKw?1 zqorR=_bud+oc(L}w!MqbH!^58RLndSUpnWHx0P$7>aQ!o$Oru^Pxwn6ci*Wc&=rdGcWlf5}zp?SV+b6=ljQuFHVE(+Wi)~uY8 z$IlXBFU_7%IAK2HRt?wF9$OA?<6V5Uks%Ofn4$6h>V`-Ecs=sU-Mow>HahLwT0cSQ zde(N+->Rw6ZF^?M+FyB^#1PP<%+q+lUJ~T3K@BQA$1Yy@~YD@tDb>+TW literal 0 HcmV?d00001 diff --git a/runtime/tests/fixtures/media/unknown-codec.mkv b/runtime/tests/fixtures/media/unknown-codec.mkv new file mode 100644 index 0000000000000000000000000000000000000000..a7f02a248a7ddbb599047d70053875468b2514b7 GIT binary patch literal 1109 zcmb1gy}x*|Q(GgW({~{L)X3uWxsk)EsUtVBq$s~QJJG2fDAd}>BoW+@&d2})EQ*4@ zvr7)_Z&B>2t@hpB9lW|l)N^H8@S4Vj5T01bx@;itqwnq>pqx<1;q>4&O^p5!u1xT5 zpkxyZ>l=t^48dSSsY`&|b9#RZ&*H_|TGux=^E)~CHuU-=mZg~)=@|l%fl;_4D!+Ki zgj-2-=0A5c)hqreHvg--kf=*%heI61T_7Lb+~2aoI_1MUsI$GUH!>C??;=z{>b@LBOrbE z&f)T$=hvopR9b9UysGb!o?A$&JG;r5mKCSE6aptz{Bv2(d3%Qc>aEqW-|@>h(?^(k$aW-4&3zu;{BlBI_+BZ5`!%>>?)QSVw7 zt-G-G|I6rYZXWFNZk0MK-gk+{d=K6etiW)6I@lQwKxb^UM>ylC`IAh$T3+s3*IwJ! zd%inz666dQmRo{KPK!_8N!ng(meP>kaogTSz3B74`Ru}zj2vtw_rKgLu&+_i{>jR3 zHx<|~#@IYr%pzy{!~gb@H4}I|P9B0PkDw3+#m_!|;6&Nge z7C3Y*1jS1N19$nn-$*X0@L##K?xpsu_ggcUd-b!N>e)YApoLe?vPXrX{eanfp9!1` zVhy`(9TbB)Pu*rU5IMe^fAwMJ5BwK&KQ;?+9Fa)zxVB}6h};wN^YY^@A79IIM= z+(bOfLSF4MugPVv49x}0T1~E;Hsy8p0480Tl-Z3Epo~0ye~WGFvWdIoJIzvt#g`7^HMq*8QNOI;==+gn;RLGA2+vfzdpq9V8Vt*fnALP zvl^K~H?)NAY-EI)6LYssn$zYO?;j8v9JZ=CS#Lqu?iQ9KTNs)f znI2A@-^jSN!{K=W*kws6jRMJwCi@qZ7JH;7rZ6&EFgEN4%Yeez;Z||+g5(1Y=_h8n zE^c@AbM|*}4RUEm^=5JLt>gpU$q4_XpIC~ldIm6%l7WF_WC02!g`CW!vVscTvI5KW z6AN7zcesQGIfi)p`?<7FHZZUPBRwMnC}1cqz78=Q=E?LEi;xYUieY#@&@1>%wtyJT zAn&}b)cNr9Mux@Tn-~}xDjKVg36^~+Wnf_MJC%8R_Zg2nVtZ#_j!>_dyvN)n?!4}F zhRg=R1|4RL)vk$uJGa;Iitkrgs=f7f)g9HW!f&Ksgdh{?XKGnn^+E6i1ZsGGkQ+@qbNwC@Nk9^8;GT@w;O*BI;tNV?R z;KXUyL>8%)TgTOO+WuTGm%GiXU>l2uc~{_U2cxB5O!qD1k(~W&_qM%@&o?n>HdM?! z6kj^$kGGX;qw23OYn%%Dc4#{7cs5y>adEhcPJ7Bu%Y}vOj;1wuHM>Tba^+sypnN|+ z&Ti%B1y`y!9KC(`sELk(+iMQqpZ{;WE|Sof0d+v}xuyQci!W41x@m)GCt z|E5;I4wJn(U7>ltY;#|qWm5C%?Jf%37uKwtlE=>yVK2>|P&i>e<5msV(;iz6Z{uBj zwTU4RW|*Pz{_2KD|9Cy}%H6z-BsMzj+gd+C>3Y_7)8DG8(QSKX#@b(bn#2&$qs-HI zz+eXBh3wC3I@Ik9Rv4U;P+*zx|6lmCSgqU_D}MVdzSP8!3ioudW9hm5LUUgon*8s; z=2MgOoqDaMO`pHMqL-(8^5P*!iH^gCzP9a`nV_~O_! zv&Y7#PgZ2RJo2;BKl*Ujq%?^mf5ljsCL1+GE&Z~P$8_K7-H9{zyDUD}#Lx>f)Y!`P zmh=&J&C0!drQKWpru_RF{0-b1n^a&Xrm^yUHbPA~zqM z42hig+t0sIFlSwSwuxaO%skz<$M;;9`r3Q=dzI`2{w*8znhh7bOq{CT&$G_WXw|(< z?xEX1Onav``R93c!FTg(J~rgnhpj7|(4-Z9AkTGY%HeIkAm?pu$N(lDU|bvW9b5P* z#Ci3+wo6sBgd(pjSQ41%+3-#Fj=r5k&EMWh$A88~o>hy{K45Wn;+I_E)LPD9;kmU7 zCprKB89zgs>E7XuJ_}AQ+}Ff#k|Be&Yr2)*{|Ap}zg6PZ7ghH=((4dYU|FJEs~sqA zv{hPsTf6<{e