diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor b/app/MindWork AI Studio/Assistants/AssistantBase.razor index 42902a41..b1d3ef12 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor @@ -31,14 +31,16 @@ @if (this.Body is not null) { - - - @this.Body + + + + @this.Body + - + @this.SubmitText @if (this.IsProcessing) @@ -158,7 +160,7 @@ @if (this.ShowReset) { - + @TB("Reset") } diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs index 577ce61f..bbf0291d 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs @@ -4,6 +4,7 @@ using AIStudio.Settings; using AIStudio.Dialogs.Settings; using AIStudio.Tools.AIJobs; using AIStudio.Tools.AssistantSessions; +using AIStudio.Tools.Media; using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; @@ -47,6 +48,9 @@ public abstract partial class AssistantBase : AssistantLowerBase wher /// [Inject] protected AIJobService AIJobService { get; init; } = null!; + + [Inject] + protected MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; protected abstract string Title { get; } @@ -132,6 +136,7 @@ public abstract partial class AssistantBase : AssistantLowerBase wher protected CancellationTokenSource? CancellationTokenSource; private bool isDisposed; private AssistantSessionKey assistantSessionKey; + private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForAssistant(this.assistantSessionKey); private Guid? assistantSessionId; private AssistantSessionSnapshot? pendingRenderedAssistantSessionSnapshot; @@ -145,6 +150,9 @@ public abstract partial class AssistantBase : AssistantLowerBase wher /// protected bool HasAssistantSession => this.assistantSessionId is not null; + /// Gets whether this assistant currently owns active media work. + protected bool IsMediaImportBusy => this.MediaTranscriptionService.IsBusy(this.CurrentMediaImportOwner); + /// /// Gets the assistant-specific identifier used to distinguish session slots. /// @@ -154,6 +162,7 @@ public abstract partial class AssistantBase : AssistantLowerBase wher protected override async Task OnInitializedAsync() { + this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; await base.OnInitializedAsync(); if (!this.SettingsManager.IsAssistantVisible(this.Component, assistantName: this.Title)) @@ -176,6 +185,7 @@ public abstract partial class AssistantBase : AssistantLowerBase wher this.CurrentChatTemplate = this.SettingsManager.GetPreselectedChatTemplate(this.Component); this.assistantSessionKey = new(this.Component, this.AssistantSessionInstanceId); await this.AttachAssistantSessionIfAvailable(); + await this.ConsumeMediaOutcomeAsync(); } protected override async Task OnParametersSetAsync() @@ -223,6 +233,9 @@ public abstract partial class AssistantBase : AssistantLowerBase wher private async Task Start() { + if (this.MediaTranscriptionService.IsBusy(this.CurrentMediaImportOwner)) + return; + var activeSession = this.AssistantSessionService.TryGetSnapshot(this.assistantSessionKey); if (activeSession?.IsActive ?? false) { @@ -634,10 +647,12 @@ public abstract partial class AssistantBase : AssistantLowerBase wher private async Task InnerResetForm() { - if (this.AssistantSessionService.TryGetSnapshot(this.assistantSessionKey)?.IsActive ?? false) + if ((this.AssistantSessionService.TryGetSnapshot(this.assistantSessionKey)?.IsActive ?? false) + || this.MediaTranscriptionService.IsBusy(this.CurrentMediaImportOwner)) return; await this.AssistantSessionService.ClearAsync(this.assistantSessionKey); + this.MediaTranscriptionService.ClearOwnerState(this.CurrentMediaImportOwner); this.assistantSessionId = null; this.ResultingContentBlock = null; this.ProviderSettings = Settings.Provider.NONE; @@ -672,6 +687,7 @@ public abstract partial class AssistantBase : AssistantLowerBase wher protected override void DisposeResources() { + this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; this.isDisposed = true; try { @@ -686,6 +702,46 @@ public abstract partial class AssistantBase : AssistantLowerBase wher base.DisposeResources(); } + /// Refreshes assistant actions when the shared import lane changes. + private void OnMediaImportStateChanged(MediaImportOwner owner) + { + if (owner == this.CurrentMediaImportOwner) + _ = this.InvokeAsync(async () => + { + await this.ConsumeMediaOutcomeAsync(); + this.StateHasChanged(); + }); + } + + /// Consumes a terminal media notification when this assistant is visible. + private async Task ConsumeMediaOutcomeAsync() + { + var outcome = this.MediaTranscriptionService.TryConsumeOutcome(this.CurrentMediaImportOwner); + if (outcome is null) + return; + + if (outcome.Failures.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Failures.Select(failure => $"{failure.FileName}: {failure.UserMessage}")); + await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, message)); + } + else if (outcome.Status is MediaImportStatus.FAILED) + { + await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.TB("The media file could not be transcribed."))); + } + + if (outcome.Warnings.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}")); + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message)); + } + + if (outcome.Status is MediaImportStatus.CANCELLED) + { + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.TB("The media transcription was canceled."))); + } + } + #endregion #region Assistant sessions diff --git a/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor b/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor index 89f8e04c..be60a4c8 100644 --- a/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor +++ b/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor @@ -21,7 +21,7 @@ } else { - + @foreach (var policy in this.SettingsManager.ConfigurationData.DocumentAnalysis.Policies) { @if (policy.IsEnterpriseConfiguration) @@ -44,10 +44,10 @@ else } - + @T("Add policy") - + @T("Delete this policy") diff --git a/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor.cs b/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor.cs index 436c5c4d..d896d315 100644 --- a/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor.cs +++ b/app/MindWork AI Studio/Assistants/DocumentAnalysis/DocumentAnalysisAssistant.razor.cs @@ -333,9 +333,14 @@ public partial class DocumentAnalysisAssistant : AssistantBaseCore this.selectedPolicy is null || this.selectedPolicy.IsProtected; private bool IsNoPolicySelected => this.selectedPolicy is null; + + private bool ArePolicyControlsDisabled => this.IsProcessing || this.IsMediaImportBusy; private void SelectedPolicyChanged(DataDocumentAnalysisPolicy? policy) { + if (this.ArePolicyControlsDisabled) + return; + this.selectedPolicy = policy; this.ResetForm(); this.policyDefinitionExpanded = !this.selectedPolicy?.IsProtected ?? true; @@ -353,6 +358,9 @@ public partial class DocumentAnalysisAssistant : AssistantBaseCore - + } break; diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index 3ff17464..d5b95dd6 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -304,6 +304,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::AGENDA::NUMBERPARTICIPANTSEXTENSIONS::T81 -- Stop generation UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1317408357"] = "Stop generation" +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1543974632"] = "The media file could not be transcribed." + -- Reset UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T180921696"] = "Reset" @@ -313,6 +316,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1809312323"] = "Please se -- The assistant failed. The message is: '{0}' UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1833836281"] = "The assistant failed. The message is: '{0}'" +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T241403726"] = "The media transcription was canceled." + -- This assistant is already running. AI Studio opens the running session instead. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T2575715765"] = "This assistant is already running. AI Studio opens the running session instead." @@ -2293,9 +2299,24 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T349928509"] = "The ima -- Open Settings UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T1172211894"] = "Open Settings" +-- Media transcription was canceled. Open the assistant to review it. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T1233815302"] = "Media transcription was canceled. Open the assistant to review it." + +-- Media transcription failed. Open the assistant to review it. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2177964639"] = "Media transcription failed. Open the assistant to review it." + +-- Media transcription completed with a warning. Open the assistant to review it. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2217674098"] = "Media transcription completed with a warning. Open the assistant to review it." + +-- Media is still being prepared. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2600900617"] = "Media is still being prepared." + -- Assistant is still running. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2719896610"] = "Assistant is still running." +-- The media transcript is ready. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T3248321953"] = "The media transcript is ready." + -- Assistant was canceled. Open it to review the result. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T3390934621"] = "Assistant was canceled. Open it to review the result." @@ -2398,18 +2419,33 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T1358313858"] = "Click t -- Drop files here to attach them. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T143112277"] = "Drop files here to attach them." +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T1543974632"] = "The media file could not be transcribed." + -- Click here to attach files. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T1875575968"] = "Click here to attach files." +-- Transcribe media files +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T2178031033"] = "Transcribe media files" + -- Drag and drop files into the marked area or click here to attach documents: UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T230755331"] = "Drag and drop files into the marked area or click here to attach documents:" +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T241403726"] = "The media transcription was canceled." + -- Select files to attach UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T2495931372"] = "Select files to attach" -- Document Preview UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T285154968"] = "Document Preview" +-- Media files require a configured transcription provider. Configure one in the transcription settings. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T3172443094"] = "Media files require a configured transcription provider. Configure one in the transcription settings." + +-- The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T322693339"] = "The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider." + -- Clear file list UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T3759696136"] = "Clear file list" @@ -2434,6 +2470,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1317408357"] = "Stop gene -- Save chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1516264254"] = "Save chat" +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "The media file could not be transcribed." + -- Type your input here... UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your input here..." @@ -2446,6 +2485,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2036185364"] = "Code" -- Italic UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2377171085"] = "Italic" +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T241403726"] = "The media transcription was canceled." + -- Profile usage is disabled according to your chat template settings. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2670286472"] = "Profile usage is disabled according to your chat template settings." @@ -2671,6 +2713,24 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANDATORYINFODISPLAY::T3511160492"] = "Ac -- Please review this text again. The content was changed. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANDATORYINFODISPLAY::T941885055"] = "Please review this text again. The content was changed." +-- Waiting to prepare media +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T1167267986"] = "Waiting to prepare media" + +-- Stop media transcription +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T1868377405"] = "Stop media transcription" + +-- Stopping media transcription +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T1878101489"] = "Stopping media transcription" + +-- Inspecting media +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T2431421733"] = "Inspecting media" + +-- Transcribing +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T2938661425"] = "Transcribing" + +-- Preparing audio +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T3200155905"] = "Preparing audio" + -- Given that my employer's workplace uses both Windows and Linux, I wanted a cross-platform solution that would work seamlessly across all major operating systems, including macOS. Additionally, I wanted to demonstrate that it is possible to create modern, efficient, cross-platform applications without resorting to Electron bloatware. The combination of .NET and Rust with Tauri proved to be an excellent technology stack for building such robust applications. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MOTIVATION::T1057189794"] = "Given that my employer's workplace uses both Windows and Linux, I wanted a cross-platform solution that would work seamlessly across all major operating systems, including macOS. Additionally, I wanted to demonstrate that it is possible to create modern, efficient, cross-platform applications without resorting to Electron bloatware. The combination of .NET and Rust with Tauri proved to be an excellent technology stack for building such robust applications." @@ -2788,18 +2848,33 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::PROVIDERSELECTION::T4279448758"] = "Uses -- Provider UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::PROVIDERSELECTION::T900237532"] = "Provider" +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T1543974632"] = "The media file could not be transcribed." + -- Failed to load file content UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T1989554334"] = "Failed to load file content" -- Drop one file here to load its content. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T2274562398"] = "Drop one file here to load its content." +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T241403726"] = "The media transcription was canceled." + +-- The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T2839709466"] = "The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider." + +-- Media files require a configured transcription provider. Configure one in the transcription settings. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3172443094"] = "Media files require a configured transcription provider. Configure one in the transcription settings." + -- Use file content as input UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3499386973"] = "Use file content as input" -- Select file to read its content UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T354817589"] = "Select file to read its content" +-- Transcribe media file +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3554222960"] = "Transcribe media file" + -- The content is cleaned using an LLM agent: the main content is extracted, advertisements and other irrelevant things are attempted to be removed; relative links are attempted to be converted into absolute links so that they can be used. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READWEBCONTENT::T1164201762"] = "The content is cleaned using an LLM agent: the main content is extracted, advertisements and other irrelevant things are attempted to be removed; relative links are attempted to be converted into absolute links so that they can be used." @@ -3538,9 +3613,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VISION::T586430036"] = "Useful assistants -- Voice recording has been disabled for this session because audio playback could not be initialized on the client. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T1123032432"] = "Voice recording has been disabled for this session because audio playback could not be initialized on the client." --- Failed to create the transcription provider. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T1689988905"] = "Failed to create the transcription provider." - -- Failed to start audio recording. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T2144994226"] = "Failed to start audio recording." @@ -3559,21 +3631,12 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T2851219233"] = "Transcrip -- Unfortunately, there was an error communicating with the AI system. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3236134591"] = "Unfortunately, there was an error communicating with the AI system." --- The configured transcription provider was not found. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T331613105"] = "The configured transcription provider was not found." - -- Failed to stop audio recording. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3462568264"] = "Failed to stop audio recording." --- The configured transcription provider does not meet the minimum confidence level. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3834149033"] = "The configured transcription provider does not meet the minimum confidence level." - -- An error occurred during transcription. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T588743762"] = "An error occurred during transcription." --- No transcription provider is configured. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T663630295"] = "No transcription provider is configured." - -- The transcription result is empty. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T974954792"] = "The transcription result is empty." @@ -6748,9 +6811,6 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2037899437"] = "Copies the serve -- This library is used to create temporary folders in runtime tests and supporting filesystem operations. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2160280545"] = "This library is used to create temporary folders in runtime tests and supporting filesystem operations." --- This library is used to determine the file type of a file. This is necessary, e.g., when we want to stream a file. -UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2173617769"] = "This library is used to determine the file type of a file. This is necessary, e.g., when we want to stream a file." - -- For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2174764529"] = "For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose." @@ -6769,6 +6829,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2301484629"] = "Configuration pl -- The C# language is used for the implementation of the user interface and the backend. To implement the user interface with C#, the Blazor technology from ASP.NET Core is used. All these technologies are integrated into the .NET SDK. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2329884315"] = "The C# language is used for the implementation of the user interface and the backend. To implement the user interface with C#, the Blazor technology from ASP.NET Core is used. All these technologies are integrated into the .NET SDK." +-- We use Rubato to resample the decoded audio to 48 kHz before the Opus encoding. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2345444286"] = "We use Rubato to resample the decoded audio to 48 kHz before the Opus encoding." + -- Linux AppImages bundle GStreamer components to support microphone access and WebM audio recording in the embedded WebKitGTK web view. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T234598990"] = "Linux AppImages bundle GStreamer components to support microphone access and WebM audio recording in the embedded WebKitGTK web view." @@ -6850,6 +6913,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2929232062"] = "Copies the confi -- Copies the root certificate fingerprint to the clipboard UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2989678330"] = "Copies the root certificate fingerprint to the clipboard" +-- This library identifies files by their content. It is used for document streaming and as the first safety and media classification step before local audio processing. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3002755581"] = "This library identifies files by their content. It is used for document streaming and as the first safety and media classification step before local audio processing." + -- Changelog UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3017574265"] = "Changelog" @@ -6895,6 +6961,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3315279770"] = "External HTTPS c -- User-language provided by the OS UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3334355246"] = "User-language provided by the OS" +-- webm-iterable provides the EBML and WebM writing path for normalized audio. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3385332793"] = "webm-iterable provides the EBML and WebM writing path for normalized audio." + -- Status: UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3396815215"] = "Status:" @@ -6955,6 +7024,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T396609403"] = "not applicable" -- Copies the allowed host configuration to the clipboard UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3970230163"] = "Copies the allowed host configuration to the clipboard" +-- Symphonia is used for media container demuxing and audio decoding. The exact MPL-covered source is available from the repository linked and is identified in the offline notices bundled with AI Studio. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3971563979"] = "Symphonia is used for media container demuxing and audio decoding. The exact MPL-covered source is available from the repository linked and is identified in the offline notices bundled with AI Studio." + -- Installed Pandoc version UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3983971016"] = "Installed Pandoc version" @@ -6973,6 +7045,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4060906280"] = "This library is -- This library is used to create asynchronous streams in Rust. It allows us to work with streams of data that can be produced asynchronously, making it easier to handle events or data that arrive over time. We use this, e.g., to stream arbitrary data from the file system to the embedding system. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4079152443"] = "This library is used to create asynchronous streams in Rust. It allows us to work with streams of data that can be produced asynchronously, making it easier to handle events or data that arrive over time. We use this, e.g., to stream arbitrary data from the file system to the embedding system." +-- Ropus provides the Opus encoder and decoder used by the media pipeline. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4113556626"] = "Ropus provides the Opus encoder and decoder used by the media pipeline." + -- Community & Code UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4158546761"] = "Community & Code" @@ -8485,6 +8560,42 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source like p -- Document UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document" +-- The configured transcription provider could not be created. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1235984176"] = "The configured transcription provider could not be created." + +-- The selected media file no longer exists. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T129859547"] = "The selected media file no longer exists." + +-- The selected media file does not contain an audio track. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T134825479"] = "The selected media file does not contain an audio track." + +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1543974632"] = "The media file could not be transcribed." + +-- The selected file cannot be processed as media. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1707342767"] = "The selected file cannot be processed as media." + +-- The audio track contains no audible signal, so there is nothing to transcribe. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1988190152"] = "The audio track contains no audible signal, so there is nothing to transcribe." + +-- The media file is damaged or its format could not be identified. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2004316549"] = "The media file is damaged or its format could not be identified." + +-- This media format or audio codec is not supported. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2142564510"] = "This media format or audio codec is not supported." + +-- No usable transcription provider is configured. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2282521655"] = "No usable transcription provider is configured." + +-- The media file could not be prepared for transcription. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2749117459"] = "The media file could not be prepared for transcription." + +-- The transcription provider could not transcribe the media file. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T3091669215"] = "The transcription provider could not transcribe the media file." + +-- The media pipeline ended without an output file. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T632852430"] = "The media pipeline ended without an output file." + -- Pandoc Installation UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::PANDOCAVAILABILITYSERVICE::T185447014"] = "Pandoc Installation" diff --git a/app/MindWork AI Studio/Chat/ChatThread.cs b/app/MindWork AI Studio/Chat/ChatThread.cs index 2c9bb720..3b00805a 100644 --- a/app/MindWork AI Studio/Chat/ChatThread.cs +++ b/app/MindWork AI Studio/Chat/ChatThread.cs @@ -24,6 +24,17 @@ public sealed record ChatThread /// public Guid WorkspaceId { get; set; } + /// + /// The monotonically increasing number used for managed media transcript filenames. + /// + public ulong LastMediaTranscriptNumber { get; set; } + + /// + /// Managed transcript attachments prepared for the composer but not sent yet. + /// Empty by default so older serialized threads require no migration. + /// + public List PendingMediaTranscripts { get; set; } = []; + /// /// Specifies the provider selected for the chat thread. /// @@ -240,14 +251,28 @@ public sealed record ChatThread { var previousBlock = sortedBlocks[index - 1]; if (previousBlock.Role is ChatRole.USER && previousBlock.HideFromUser) + { + DeleteManagedAttachments(previousBlock); this.Blocks.Remove(previousBlock); + } } } + DeleteManagedAttachments(block); + // Remove the block from the chat thread: this.Blocks.Remove(block); } + private static void DeleteManagedAttachments(ContentBlock block) + { + if (block.Content is not ContentText textContent) + return; + + foreach (var attachment in textContent.FileAttachments) + ManagedTranscriptAttachment.TryDeleteOwnedFile(attachment); + } + /// /// Transforms this chat thread to an ERI chat thread. /// diff --git a/app/MindWork AI Studio/Chat/FileAttachment.cs b/app/MindWork AI Studio/Chat/FileAttachment.cs index bdc9651d..ce093592 100644 --- a/app/MindWork AI Studio/Chat/FileAttachment.cs +++ b/app/MindWork AI Studio/Chat/FileAttachment.cs @@ -14,6 +14,7 @@ namespace AIStudio.Chat; [JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")] [JsonDerivedType(typeof(FileAttachment), typeDiscriminator: "file")] [JsonDerivedType(typeof(FileAttachmentImage), typeDiscriminator: "image")] +[JsonDerivedType(typeof(ManagedTranscriptAttachment), typeDiscriminator: "managed_transcript")] public record FileAttachment(FileAttachmentType Type, string FileName, string FilePath, long FileSizeBytes) { /// @@ -56,7 +57,7 @@ public record FileAttachment(FileAttachmentType Type, string FileName, string Fi /// /// Rebuilds the attachment from its current file path so file type detection uses the latest rules. /// - public FileAttachment Normalize() => FromPath(this.FilePath); + public virtual FileAttachment Normalize() => FromPath(this.FilePath); /// /// Creates a FileAttachment from a file path by automatically determining the type, diff --git a/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs b/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs new file mode 100644 index 00000000..4b811734 --- /dev/null +++ b/app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs @@ -0,0 +1,169 @@ +using System.Text; + +using AIStudio.Settings; + +namespace AIStudio.Chat; + +/// +/// Attachment whose Markdown file is owned and lifecycle-managed by the media feature. +/// +/// Display file name. +/// Absolute staged or chat-owned path. +/// Current file size. +/// Original media file name used in the title and stem. +/// Whether the file still lives in operation staging. +public sealed record ManagedTranscriptAttachment(string FileName, string FilePath, long FileSizeBytes, string OriginalFileName, bool IsStaged) + : FileAttachment(FileAttachmentType.DOCUMENT, FileName, FilePath, FileSizeBytes) +{ + /// Refreshes the path-derived name and current file size. + public override FileAttachment Normalize() + { + var size = File.Exists(this.FilePath) ? new FileInfo(this.FilePath).Length : 0; + return this with { FileName = Path.GetFileName(this.FilePath), FileSizeBytes = size }; + } + + /// Creates a transcript in an operation-specific staging directory. + /// Original media path. + /// Provider transcript. + /// The staged managed attachment. + public static async Task CreateStagedAsync(string originalPath, string transcript) + { + var operationDirectory = Path.Combine(SettingsManager.DataDirectory!, "media-staging", Guid.NewGuid().ToString("N")); + Directory.CreateDirectory(operationDirectory); + var originalFileName = Path.GetFileName(originalPath); + var stagingPath = Path.Combine(operationDirectory, $"{Guid.NewGuid():N}.md"); + await WriteMarkdownAsync(stagingPath, originalFileName, transcript); + return FromPath(stagingPath, originalFileName, isStaged: true); + } + + /// Writes transcript Markdown to a temporary file and atomically publishes it. + /// Final managed target path. + /// Original media file name. + /// Provider transcript. + /// The chat-owned managed attachment. + internal static async Task CreateAtomicAsync(string targetPath, string originalFileName, string transcript) + { + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + var temporaryPath = Path.Combine(Path.GetDirectoryName(targetPath)!, $".{Guid.NewGuid():N}.tmp"); + try + { + await WriteMarkdownAsync(temporaryPath, originalFileName, transcript); + File.Move(temporaryPath, targetPath); + return FromPath(targetPath, originalFileName, isStaged: false); + } + finally + { + if (File.Exists(temporaryPath)) + File.Delete(temporaryPath); + } + } + + /// Deletes a file only when its canonical path has an exact managed structure. + /// Candidate managed attachment. + /// Whether an owned file was deleted. + public static bool TryDeleteOwnedFile(FileAttachment attachment) + { + if (attachment is not ManagedTranscriptAttachment managed || !File.Exists(managed.FilePath)) + return false; + + var fileInfo = new FileInfo(managed.FilePath); + var fullFilePath = Path.GetFullPath(fileInfo.FullName); + var fullDataRoot = Path.GetFullPath(SettingsManager.DataDirectory!); + var relative = Path.GetRelativePath(fullDataRoot, fullFilePath); + + if (Path.IsPathRooted(relative) || relative == ".." || relative.StartsWith($"..{Path.DirectorySeparatorChar}", PathComparison)) + return false; + + if (fileInfo.LinkTarget is not null || HasLinkedDirectory(fileInfo.Directory, fullDataRoot)) + return false; + + var segments = relative.Split(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar); + var isStaging = segments is ["media-staging", _, _] + && Guid.TryParseExact(segments[1], "N", out _) + && !string.IsNullOrWhiteSpace(segments[2]); + + var isTemporaryChatTranscript = segments is ["tempChats", _, _, _, _] + && Guid.TryParse(segments[1], out _) + && segments[2] == "attachments" + && segments[3] == "transcripts"; + + var isWorkspaceChatTranscript = segments is ["workspaces", _, _, _, _, _] + && Guid.TryParse(segments[1], out _) + && Guid.TryParse(segments[2], out _) + && segments[3] == "attachments" + && segments[4] == "transcripts"; + + if (!isStaging && !isTemporaryChatTranscript && !isWorkspaceChatTranscript) + return false; + + File.Delete(fullFilePath); + var parent = Path.GetDirectoryName(fullFilePath); + if (isStaging && parent is not null && Directory.Exists(parent) && !Directory.EnumerateFileSystemEntries(parent).Any()) + Directory.Delete(parent); + + return true; + } + + /// Rejects paths traversing any symbolic-link or junction directory below the data root. + private static bool HasLinkedDirectory(DirectoryInfo? directory, string fullDataRoot) + { + while (directory is not null && !string.Equals(Path.GetFullPath(directory.FullName), fullDataRoot, PathComparison)) + { + if (directory.LinkTarget is not null) + return true; + + directory = directory.Parent; + } + + return directory is null; + } + + /// Normalizes an original stem using Unicode scalar values and cross-platform rules. + /// Original media file name. + /// A non-empty stem containing at most 80 Unicode text characters. + internal static string NormalizeOriginalStem(string originalFileName) + { + var stem = Path.GetFileNameWithoutExtension(originalFileName).Normalize(NormalizationForm.FormC); + var normalized = new StringBuilder(); + + var textCharacters = 0; + foreach (var rune in stem.EnumerateRunes()) + { + if (textCharacters == 80) + break; + + var replacement = Rune.IsControl(rune) || rune.Value is '/' or '\\' or '<' or '>' or ':' or '"' or '|' or '?' or '*' + ? new Rune('-') + : rune; + + normalized.Append(replacement); + textCharacters++; + } + + var result = normalized.ToString().Trim(' ', '.', '-'); + return string.IsNullOrWhiteSpace(result) ? "media" : result; + } + + /// Creates an attachment record from a file already written to disk. + private static ManagedTranscriptAttachment FromPath(string path, string originalFileName, bool isStaged) => new( + Path.GetFileName(path), + path, + new FileInfo(path).Length, + originalFileName, + isStaged); + + /// Writes localized transcript Markdown without a UTF-8 byte-order mark. + private static async Task WriteMarkdownAsync(string path, string originalFileName, string transcript) + { + var markdown = $""" + # Transcription: {originalFileName} + + {transcript.Trim()} + """; + + await File.WriteAllTextAsync(path, markdown, new UTF8Encoding(false)); + } + + /// Gets the platform path comparison used for canonical containment checks. + private static StringComparison PathComparison => OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/AssistantBlock.razor.cs b/app/MindWork AI Studio/Components/AssistantBlock.razor.cs index 985cf659..48672332 100644 --- a/app/MindWork AI Studio/Components/AssistantBlock.razor.cs +++ b/app/MindWork AI Studio/Components/AssistantBlock.razor.cs @@ -1,6 +1,9 @@ using AIStudio.Dialogs.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.AssistantSessions; +using AIStudio.Tools.Media; +using AIStudio.Tools.Services; + using Microsoft.AspNetCore.Components; using DialogOptions = AIStudio.Dialogs.DialogOptions; @@ -60,6 +63,9 @@ public partial class AssistantBlock : MSGComponentBase where TSetting [Inject] private AssistantSessionService AssistantSessionService { get; init; } = null!; + + [Inject] + private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; private async Task OpenSettingsDialog() { @@ -71,7 +77,7 @@ public partial class AssistantBlock : MSGComponentBase where TSetting await this.DialogService.ShowAsync(T("Open Settings"), dialogParameters, DialogOptions.FULLSCREEN); } - private string BorderColor => this.AssistantSessionSnapshot?.IsActive is true ? this.ColorTheme.GetActivityIndicatorColor(this.SettingsManager) : this.SettingsManager.IsDarkMode switch + private string BorderColor => this.AssistantSessionSnapshot?.IsActive is true || this.MediaImportSnapshot?.IsBusy is true ? this.ColorTheme.GetActivityIndicatorColor(this.SettingsManager) : this.SettingsManager.IsDarkMode switch { true => this.ColorTheme.GetCurrentPalette(this.SettingsManager).GrayDefault, false => this.ColorTheme.GetCurrentPalette(this.SettingsManager).GrayDefault, @@ -92,10 +98,29 @@ public partial class AssistantBlock : MSGComponentBase where TSetting ? this.AssistantSessionService.GetSnapshots().FirstOrDefault(snapshot => snapshot.Key.Component == this.Component) : this.AssistantSessionService.GetSnapshots().FirstOrDefault(snapshot => snapshot.Key.InstanceId == this.AssistantSessionInstanceId); + private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForAssistant(new AssistantSessionKey(this.Component, this.AssistantSessionInstanceId)); + + private MediaImportSnapshot? MediaImportSnapshot => string.IsNullOrWhiteSpace(this.AssistantSessionInstanceId) + ? this.MediaTranscriptionService.GetSnapshots().FirstOrDefault(snapshot => + snapshot.Owner.Kind is MediaImportOwnerKind.ASSISTANT + && snapshot.Owner.Id.StartsWith($"{this.Component}:", StringComparison.Ordinal)) + : this.MediaTranscriptionService.GetSnapshot(this.CurrentMediaImportOwner); + /// /// Gets the assistant session indicator shown on top of the assistant icon. /// - private AssistantSessionIndicatorData? AssistantSessionIndicator => this.AssistantSessionSnapshot?.Status switch + private AssistantSessionIndicatorData? AssistantSessionIndicator => this.MediaImportSnapshot?.Status switch + { + MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => new(Icons.Material.Filled.ChangeCircle, Color.Info, this.T("Media is still being prepared.")), + MediaImportStatus.SUCCEEDED => new(Icons.Material.Filled.TaskAlt, Color.Success, this.T("The media transcript is ready.")), + MediaImportStatus.WARNING => new(Icons.Material.Filled.WarningAmber, Color.Warning, this.T("Media transcription completed with a warning. Open the assistant to review it.")), + MediaImportStatus.FAILED => new(Icons.Material.Filled.Error, Color.Error, this.T("Media transcription failed. Open the assistant to review it.")), + MediaImportStatus.CANCELLED => new(Icons.Material.Filled.Cancel, Color.Warning, this.T("Media transcription was canceled. Open the assistant to review it.")), + + _ => this.AssistantSessionIndicatorWithoutMedia, + }; + + private AssistantSessionIndicatorData? AssistantSessionIndicatorWithoutMedia => this.AssistantSessionSnapshot?.Status switch { AssistantSessionStatus.RUNNING or AssistantSessionStatus.CANCELING => new(Icons.Material.Filled.ChangeCircle, Color.Info, this.T("Assistant is still running.")), AssistantSessionStatus.COMPLETED => new(Icons.Material.Filled.TaskAlt, Color.Success, this.T("The result is ready.")), @@ -104,6 +129,28 @@ public partial class AssistantBlock : MSGComponentBase where TSetting _ => null, }; + protected override async Task OnInitializedAsync() + { + this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; + await base.OnInitializedAsync(); + } + + private void OnMediaImportStateChanged(MediaImportOwner owner) + { + var matches = string.IsNullOrWhiteSpace(this.AssistantSessionInstanceId) + ? owner.Kind is MediaImportOwnerKind.ASSISTANT && owner.Id.StartsWith($"{this.Component}:", StringComparison.Ordinal) + : owner == this.CurrentMediaImportOwner; + + if (matches) + _ = this.InvokeAsync(this.StateHasChanged); + } + + protected override void DisposeResources() + { + this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; + base.DisposeResources(); + } + /// /// Refreshes the block when assistant session activity changes. /// diff --git a/app/MindWork AI Studio/Components/AttachDocuments.razor b/app/MindWork AI Studio/Components/AttachDocuments.razor index e96825c3..b707f064 100644 --- a/app/MindWork AI Studio/Components/AttachDocuments.razor +++ b/app/MindWork AI Studio/Components/AttachDocuments.razor @@ -2,57 +2,66 @@ @if (this.UseSmallForm) { -
- @if (this.isDraggingOver) - { - - - - - - } - else if (this.DocumentPaths.Any()) - { - + +
+ @if (this.isDraggingOver) + { + + + + + } + else if (this.DocumentPaths.Any()) + { + + + + + + } + else + { + - - - } - else + + } +
+ @if (this.ShowMediaStatus) { - - - + } -
+ } else { - @if (!this.Disabled) + @if (!this.IsUnavailable) { @@ -69,11 +78,15 @@ else } + @if (this.ShowMediaStatus) + { + + }
@foreach (var fileAttachment in this.DocumentPaths) { - @if (this.Disabled) + @if (this.IsUnavailable) { } @@ -84,7 +97,7 @@ else }
- @if (!this.Disabled) + @if (!this.IsUnavailable) { @T("Clear file list") diff --git a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs index dc72d2e9..c52bd115 100644 --- a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs +++ b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs @@ -1,5 +1,6 @@ using AIStudio.Chat; using AIStudio.Dialogs; +using AIStudio.Tools.Media; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; using AIStudio.Tools.Services; @@ -13,6 +14,11 @@ using DialogOptions = Dialogs.DialogOptions; public partial class AttachDocuments : MSGComponentBase { + private readonly MediaImportOwner fallbackMediaImportOwner = new(MediaImportOwnerKind.CHAT, $"attachments:{Guid.NewGuid():N}"); + + [CascadingParameter] + private MediaImportOwner? ImportOwner { get; set; } + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(AttachDocuments).Namespace, nameof(AttachDocuments)); [Parameter] @@ -48,6 +54,10 @@ public partial class AttachDocuments : MSGComponentBase [Parameter] public bool UseSmallForm { get; set; } + /// Whether this control renders its own media status. + [Parameter] + public bool ShowMediaStatus { get; set; } = true; + [Parameter] public bool Disabled { get; set; } @@ -63,6 +73,14 @@ public partial class AttachDocuments : MSGComponentBase [Parameter] public AIStudio.Settings.Provider? Provider { get; set; } + /// Optional persisted chat that can own transcript files immediately. + [Parameter] + public ChatThread? OwnerChat { get; set; } + + /// Creates and persists a draft owner after media import confirmation. + [Parameter] + public Func> EnsureOwnerChatAsync { get; set; } = _ => Task.FromResult(null); + [Inject] private ILogger Logger { get; set; } = null!; @@ -75,17 +93,28 @@ public partial class AttachDocuments : MSGComponentBase [Inject] private PandocAvailabilityService PandocAvailabilityService { get; init; } = null!; + [Inject] + private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; + private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top; private static readonly string DROP_FILES_HERE_TEXT = TB("Drop files here to attach them."); private uint numDropAreasAboveThis; private bool isComponentHovered; private bool isDraggingOver; + private MediaImportOwner EffectiveImportOwner => this.OwnerChat is not null + ? MediaImportOwner.ForChat(this.OwnerChat.ChatId) + : this.ImportOwner ?? this.fallbackMediaImportOwner; + + private MediaImportTarget EffectiveMediaImportTarget => new(this.EffectiveImportOwner, string.IsNullOrWhiteSpace(this.Name) ? "attachments" : this.Name); + + private bool IsUnavailable => this.Disabled || this.MediaTranscriptionService.IsBusy(this.EffectiveImportOwner); #region Overrides of MSGComponentBase protected override async Task OnInitializedAsync() { + this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; this.ApplyFilters([], [ Event.TAURI_EVENT_RECEIVED, Event.REGISTER_FILE_DROP_AREA, Event.UNREGISTER_FILE_DROP_AREA ]); // Register this drop area: @@ -93,9 +122,101 @@ public partial class AttachDocuments : MSGComponentBase await base.OnInitializedAsync(); } + /// Rehydrates results after the component is assigned another chat or target. + protected override async Task OnParametersSetAsync() + { + await base.OnParametersSetAsync(); + await this.SyncCompletedMediaAttachmentsAsync(); + } + + /// Refreshes disabled controls when the shared import lane changes. + private void OnMediaImportStateChanged(MediaImportOwner owner) + { + if (owner == this.EffectiveImportOwner) + _ = this.InvokeAsync(async () => + { + await this.SyncCompletedMediaAttachmentsAsync(); + await this.ConsumeStandaloneMediaOutcomeAsync(); + this.StateHasChanged(); + }); + } + + /// Consumes outcomes for dialog-local controls that have no chat or assistant owner surface. + private async Task ConsumeStandaloneMediaOutcomeAsync() + { + if (this.ImportOwner is not null || this.OwnerChat is not null) + return; + + var outcome = this.MediaTranscriptionService.TryConsumeOutcome(this.EffectiveImportOwner); + if (outcome is null) + return; + + if (outcome.Failures.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Failures.Select(failure => $"{failure.FileName}: {failure.UserMessage}")); + await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, message)); + } + else if (outcome.Status is MediaImportStatus.FAILED) + { + await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed."))); + } + + if (outcome.Warnings.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}")); + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message)); + } + + if (outcome.Status is MediaImportStatus.CANCELLED) + { + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled."))); + } + } + + /// Reattaches completed owner results after progress updates or navigation. + private async Task SyncCompletedMediaAttachmentsAsync() + { + var delivery = this.MediaTranscriptionService.GetPendingDelivery(this.EffectiveMediaImportTarget); + var completed = delivery?.Attachments ?? []; + var pending = this.OwnerChat?.PendingMediaTranscripts ?? []; + var changed = false; + var ownerPendingChanged = false; + + foreach (var attachment in completed.Concat(pending)) + changed |= this.DocumentPaths.Add(attachment); + + if (this.OwnerChat is not null) + { + foreach (var attachment in completed.OfType()) + { + if (this.OwnerChat.PendingMediaTranscripts.All(existing => existing.FilePath != attachment.FilePath)) + { + this.OwnerChat.PendingMediaTranscripts.Add(attachment); + ownerPendingChanged = true; + } + } + } + + if (changed || ownerPendingChanged) + { + await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths); + await this.OnChange(this.DocumentPaths); + } + + if (delivery is not null) + this.MediaTranscriptionService.AcknowledgeDelivery(delivery); + } + + /// Unsubscribes from the singleton media service. + protected override void DisposeResources() + { + this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; + base.DisposeResources(); + } + protected override async Task ProcessIncomingMessage(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default { - if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED) + if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED) return; switch (triggeredEvent) @@ -168,29 +289,7 @@ public partial class AttachDocuments : MSGComponentBase return; } - // Ensure that Pandoc is installed and ready: - var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync( - showSuccessMessage: false, - showDialog: true); - - // If Pandoc is not available (user cancelled installation), abort file drop: - if (!pandocState.IsAvailable) - { - this.Logger.LogWarning("The user cancelled the Pandoc installation or Pandoc is not available. Aborting file drop."); - this.isDraggingOver = false; - this.ClearDragClass(); - this.StateHasChanged(); - return; - } - - foreach (var path in paths) - { - if(!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, path, this.ValidateMediaFileTypes, this.Provider)) - continue; - - this.DocumentPaths.Add(FileAttachment.FromPath(path)); - } - + await this.AddFileBatchAsync(paths); await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths); await this.OnChange(this.DocumentPaths); this.isDraggingOver = false; @@ -208,54 +307,41 @@ public partial class AttachDocuments : MSGComponentBase private async Task AddFilesManually() { - if (this.Disabled) + if (this.IsUnavailable) return; - // Ensure that Pandoc is installed and ready: - var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync( - showSuccessMessage: false, - showDialog: true); - - // If Pandoc is not available (user cancelled installation), abort file selection: - if (!pandocState.IsAvailable) - { - this.Logger.LogWarning("The user cancelled the Pandoc installation or Pandoc is not available. Aborting file selection."); - return; - } - var selectFiles = await this.RustService.SelectFiles(T("Select files to attach")); if (selectFiles.UserCancelled) return; - foreach (var selectedFilePath in selectFiles.SelectedFilePaths) - { - if (!File.Exists(selectedFilePath)) - continue; - - if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, selectedFilePath, this.ValidateMediaFileTypes, this.Provider)) - continue; - - this.DocumentPaths.Add(FileAttachment.FromPath(selectedFilePath)); - } - + await this.AddFileBatchAsync(selectFiles.SelectedFilePaths); await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths); await this.OnChange(this.DocumentPaths); } private async Task OpenAttachmentsDialog() { - if (this.Disabled) + if (this.IsUnavailable) return; + var previousAttachments = this.DocumentPaths.ToHashSet(); this.DocumentPaths = await ReviewAttachmentsDialog.OpenDialogAsync(this.DialogService, this.DocumentPaths); + foreach (var removedAttachment in previousAttachments.Except(this.DocumentPaths)) + ManagedTranscriptAttachment.TryDeleteOwnedFile(removedAttachment); + + this.ReconcileOwnerPendingTranscripts(); } private async Task ClearAllFiles() { - if (this.Disabled) + if (this.IsUnavailable) return; + foreach (var attachment in this.DocumentPaths) + ManagedTranscriptAttachment.TryDeleteOwnedFile(attachment); + this.DocumentPaths.Clear(); + this.ReconcileOwnerPendingTranscripts(); await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths); await this.OnChange(this.DocumentPaths); } @@ -266,7 +352,7 @@ public partial class AttachDocuments : MSGComponentBase private void OnMouseEnter(EventArgs _) { - if(this.Disabled || this.PauseCatchingDrops) + if(this.IsUnavailable || this.PauseCatchingDrops) return; this.Logger.LogDebug("Attach documents component '{Name}' is hovered.", this.Name); @@ -277,7 +363,7 @@ public partial class AttachDocuments : MSGComponentBase private void OnMouseLeave(EventArgs _) { - if(this.Disabled || this.PauseCatchingDrops) + if(this.IsUnavailable || this.PauseCatchingDrops) return; this.Logger.LogDebug("Attach documents component '{Name}' is no longer hovered.", this.Name); @@ -288,15 +374,98 @@ public partial class AttachDocuments : MSGComponentBase private async Task RemoveDocument(FileAttachment fileAttachment) { - if (this.Disabled) + if (this.IsUnavailable) return; this.DocumentPaths.Remove(fileAttachment); + ManagedTranscriptAttachment.TryDeleteOwnedFile(fileAttachment); + this.ReconcileOwnerPendingTranscripts(); await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths); await this.OnChange(this.DocumentPaths); } + /// Keeps persisted chat-draft transcript references aligned with the composer. + private void ReconcileOwnerPendingTranscripts() + { + if (this.OwnerChat is null) + return; + + var retainedPaths = this.DocumentPaths.Select(attachment => attachment.FilePath).ToHashSet(StringComparer.Ordinal); + this.OwnerChat.PendingMediaTranscripts.RemoveAll(attachment => !retainedPaths.Contains(attachment.FilePath)); + } + + private async Task AddFileBatchAsync(IEnumerable paths) + { + var existingPaths = paths.Where(File.Exists).ToList(); + var mediaPaths = existingPaths.Where(IsTranscribableMedia).ToList(); + var regularPaths = existingPaths.Except(mediaPaths).ToList(); + + var canAddRegularFiles = true; + if (regularPaths.Count > 0) + { + var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync( + showSuccessMessage: false, + showDialog: true); + canAddRegularFiles = pandocState.IsAvailable; + } + + foreach (var path in regularPaths) + { + if (!canAddRegularFiles) + break; + + if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync( + FileExtensionValidation.UseCase.ATTACHING_CONTENT, + path, + this.ValidateMediaFileTypes, + this.Provider)) + continue; + this.DocumentPaths.Add(FileAttachment.FromPath(path)); + } + + if (mediaPaths.Count is 0) + return; + + if (string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider)) + { + await this.MessageBus.SendWarning(new( + Icons.Material.Filled.VoiceChat, + this.T("Media files require a configured transcription provider. Configure one in the transcription settings."))); + return; + } + + var names = string.Join('\n', mediaPaths.Select(path => $"- {Markdown.EscapeInlineText(Path.GetFileName(path))}")); + var message = this.T("The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider."); + var dialogParameters = new DialogParameters + { + { + x => x.MarkdownBody, + $""" + {message} + + {names} + """ + }, + }; + + var dialogReference = await this.DialogService.ShowAsync( + this.T("Transcribe media files"), + dialogParameters, + DialogOptions.FULLSCREEN); + + var dialogResult = await dialogReference.Result; + if (dialogResult is null || dialogResult.Canceled) + return; + + if (this.OwnerChat is null) + this.OwnerChat = await this.EnsureOwnerChatAsync(mediaPaths[0]); + + this.MediaTranscriptionService.TryStartAttachmentBatch(mediaPaths, this.EffectiveMediaImportTarget, this.OwnerChat); + } + + private static bool IsTranscribableMedia(string path) => FileTypes.IsAllowedPath(path, FileTypes.AUDIO) || FileTypes.IsAllowedPath(path, FileTypes.VIDEO); + /// /// The user might want to check what we actually extract from his file and therefore give the LLM as an input. /// diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor b/app/MindWork AI Studio/Components/ChatComponent.razor index 45c0584c..1d622ec3 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor +++ b/app/MindWork AI Studio/Components/ChatComponent.razor @@ -33,6 +33,7 @@ } + } - + diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index 06b6fb92..2cee066a 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -4,6 +4,8 @@ using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.AIJobs; +using AIStudio.Tools.Media; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; using Microsoft.AspNetCore.Components.Web; @@ -14,6 +16,7 @@ namespace AIStudio.Components; public partial class ChatComponent : MSGComponentBase, IAsyncDisposable { + private readonly Guid draftMediaOwnerId = Guid.NewGuid(); private const string CHAT_INPUT_ID = "chat-user-input"; private const string MARKDOWN_CODE = "code"; private const string MARKDOWN_BOLD = "bold"; @@ -54,6 +57,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable [Inject] private AIJobService AIJobService { get; init; } = null!; + [Inject] + private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; + private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top; private static readonly Dictionary USER_INPUT_ATTRIBUTES = new(); @@ -81,6 +87,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable private Guid foregroundChatId = Guid.Empty; private int workspaceHeaderSyncVersion; + private MediaImportOwner CurrentMediaImportOwner => MediaImportOwner.ForChat(this.ChatThread?.ChatId ?? this.draftMediaOwnerId); + // Unfortunately, we need the input field reference to blur the focus away. Without // this, we cannot clear the input field. private MudTextField inputField = null!; @@ -104,6 +112,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable protected override async Task OnInitializedAsync() { + this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; + // Apply the filters for the message bus: this.ApplyFilters([], [ Event.HAS_CHAT_UNSAVED_CHANGES, Event.RESET_CHAT_STATE, Event.CHAT_STREAMING_DONE, Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED, Event.CHAT_GENERATION_CHANGED, Event.WORKSPACE_RENAMED, Event.CONFIGURATION_CHANGED ]); @@ -243,9 +253,50 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable // Select the correct provider: await this.SelectProviderWhenLoadingChat(); await this.SyncForegroundChatAsync(); + await this.ConsumeMediaOutcomeAsync(); await base.OnInitializedAsync(); } + /// Refreshes send and attachment controls when the media import lane changes. + private void OnMediaImportStateChanged(MediaImportOwner owner) + { + if (owner == this.CurrentMediaImportOwner) + _ = this.InvokeAsync(async () => + { + await this.ConsumeMediaOutcomeAsync(); + this.StateHasChanged(); + }); + } + + /// Consumes a terminal media notification when its chat is visible. + private async Task ConsumeMediaOutcomeAsync() + { + var outcome = this.MediaTranscriptionService.TryConsumeOutcome(this.CurrentMediaImportOwner); + if (outcome is null) + return; + + if (outcome.Failures.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Failures.Select(failure => $"{failure.FileName}: {failure.UserMessage}")); + await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, message)); + } + else if (outcome.Status is MediaImportStatus.FAILED) + { + await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed."))); + } + + if (outcome.Warnings.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}")); + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message)); + } + + if (outcome.Status is MediaImportStatus.CANCELLED) + { + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled."))); + } + } + protected override async Task OnAfterRenderAsync(bool firstRender) { if (firstRender && this.ChatThread is not null && this.mustStoreChat) @@ -314,6 +365,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable await this.ApplyLoadedChatParameterAsync(); await this.SyncForegroundChatAsync(); + await this.ConsumeMediaOutcomeAsync(); await base.OnParametersSetAsync(); } @@ -680,9 +732,43 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable this.ComposerState.MarkUserDraft(); this.hasUnsavedChanges = true; } + + /// Creates and stores a stable draft immediately after media import confirmation. + private async Task EnsureMediaImportChatAsync(string firstMediaPath) + { + if (this.ChatThread is not null) + return this.ChatThread; + + this.RefreshCurrentProfileAndChatTemplate(); + var promptName = this.ExtractThreadName(this.ComposerState.UserInput); + this.ChatThread = new() + { + IncludeDateTime = true, + SelectedProvider = this.Provider.Id, + SelectedProfile = this.currentProfile.Id, + SelectedChatTemplate = this.currentChatTemplate.Id, + SystemPrompt = SystemPrompts.DEFAULT, + WorkspaceId = this.currentWorkspaceId, + ChatId = Guid.NewGuid(), + DataSourceOptions = this.earlyDataSourceOptions, + Name = string.IsNullOrWhiteSpace(this.ComposerState.UserInput) + ? $"Transkription: {Path.GetFileName(firstMediaPath)}" + : promptName, + Blocks = this.currentChatTemplate == ChatTemplate.NO_CHAT_TEMPLATE ? [] : this.currentChatTemplate.ExampleConversation.Select(block => block.DeepClone()).ToList(), + }; + + await WorkspaceBehaviour.StoreChatAsync(this.ChatThread); + this.MarkCurrentChatAsLoadedParameter(); + await this.ChatThreadChanged.InvokeAsync(this.ChatThread); + await this.SyncForegroundChatAsync(); + return this.ChatThread; + } private async Task SendMessage(bool reuseLastUserPrompt = false) { + if (this.MediaTranscriptionService.IsBusy(this.CurrentMediaImportOwner)) + return; + if (!this.IsProviderSelected) return; @@ -745,6 +831,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable Text = this.ComposerState.UserInput, FileAttachments = normalizedAttachments, }; + this.ChatThread.PendingMediaTranscripts.Clear(); // // Add the user message to the thread: @@ -986,12 +1073,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable if (workspaceId == Guid.Empty) return; - // Delete the chat from the current workspace or the temporary storage: - await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, this.ChatThread!.WorkspaceId, this.ChatThread.ChatId, askForConfirmation: false); - - this.ChatThread!.WorkspaceId = workspaceId; + await WorkspaceBehaviour.MoveChatAsync(this.ChatThread!, workspaceId); this.MarkCurrentChatAsLoadedParameter(); - await this.SaveThread(); await this.SyncWorkspaceHeaderWithChatThreadAsync(); } @@ -1209,6 +1292,7 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable public async ValueTask DisposeAsync() { + this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; if(this.SettingsManager.ConfigurationData.Workspace.StorageBehavior is WorkspaceStorageBehavior.STORE_CHATS_AUTOMATICALLY) { await this.SaveThread(); diff --git a/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor b/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor new file mode 100644 index 00000000..e2acc313 --- /dev/null +++ b/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor @@ -0,0 +1,34 @@ +@inherits MSGComponentBase +@inject MediaTranscriptionService MediaTranscriptionService +@using AIStudio.Tools.Services + +@if (this.Snapshot is { IsBusy: true } snapshot) +{ + @if (this.Compact) + { + + + + @this.StatusText + + + + + + } + else + { + + + + + @this.StatusText + + + + + + + + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor.cs b/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor.cs new file mode 100644 index 00000000..1a048d61 --- /dev/null +++ b/app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor.cs @@ -0,0 +1,73 @@ +using AIStudio.Tools.Media; +using Microsoft.AspNetCore.Components; + +namespace AIStudio.Components; + +public partial class MediaTranscriptionStatus +{ + /// The surface owner whose operation is rendered. + [Parameter] + public MediaImportOwner Owner { get; set; } + + /// Optional target filter used by embedded file controls. + [Parameter] + public string TargetId { get; set; } = string.Empty; + + /// Renders the status without an enclosing paper surface. + [Parameter] + public bool Compact { get; set; } + + private MediaImportSnapshot? Snapshot + { + get + { + var snapshot = this.MediaTranscriptionService.GetSnapshot(this.Owner); + return string.IsNullOrWhiteSpace(this.TargetId) || snapshot?.Target.TargetId == this.TargetId + ? snapshot + : null; + } + } + + /// Gets the localized visible status for the active import. + private string StatusText + { + get + { + var snapshot = this.Snapshot; + if (snapshot is null) + return string.Empty; + + return snapshot.Phase switch + { + MediaTranscriptionPhase.QUEUED => $"{this.T("Waiting to prepare media")}: {snapshot.CurrentFileName}", + MediaTranscriptionPhase.PROBING => $"{this.T("Inspecting media")}: {snapshot.CurrentFileName}", + MediaTranscriptionPhase.TRANSCODING => $"{this.T("Preparing audio")}: {snapshot.CurrentFileName}", + MediaTranscriptionPhase.UPLOADING => $"{this.T("Transcribing")}: {snapshot.CurrentFileName}", + MediaTranscriptionPhase.CANCELING => $"{this.T("Stopping media transcription")}: {snapshot.CurrentFileName}", + + _ => snapshot.CurrentFileName, + }; + } + } + + /// Subscribes to singleton import state changes. + protected override async Task OnInitializedAsync() + { + this.MediaTranscriptionService.StateChanged += this.OnStateChanged; + await base.OnInitializedAsync(); + } + + /// Schedules a render after an import state transition. + private void OnStateChanged(MediaImportOwner owner) + { + if (owner == this.Owner) + _ = this.InvokeAsync(this.StateHasChanged); + } + + /// Unsubscribes from singleton import state changes. + protected override void DisposeResources() + { + this.MediaTranscriptionService.StateChanged -= this.OnStateChanged; + base.DisposeResources(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ReadFileContent.razor b/app/MindWork AI Studio/Components/ReadFileContent.razor index 27f979b0..c06fd5b5 100644 --- a/app/MindWork AI Studio/Components/ReadFileContent.razor +++ b/app/MindWork AI Studio/Components/ReadFileContent.razor @@ -5,19 +5,29 @@
- + @this.ButtonText - - @T("Drop one file here to load its content.") - + @if (this.IsCurrentTargetBusy) + { + + } + else + { + + @T("Drop one file here to load its content.") + + }
} else { - - @this.ButtonText - + + + @this.ButtonText + + + } \ No newline at end of file diff --git a/app/MindWork AI Studio/Components/ReadFileContent.razor.cs b/app/MindWork AI Studio/Components/ReadFileContent.razor.cs index c301a541..4a200f1f 100644 --- a/app/MindWork AI Studio/Components/ReadFileContent.razor.cs +++ b/app/MindWork AI Studio/Components/ReadFileContent.razor.cs @@ -1,3 +1,5 @@ +using AIStudio.Dialogs; +using AIStudio.Tools.Media; using AIStudio.Tools.Rust; using AIStudio.Tools.Services; using AIStudio.Tools.Validation; @@ -8,6 +10,22 @@ namespace AIStudio.Components; public partial class ReadFileContent : MSGComponentBase { + private readonly MediaImportOwner fallbackMediaImportOwner = new(MediaImportOwnerKind.ASSISTANT, $"read-file-content:{Guid.NewGuid():N}"); + + [CascadingParameter] + private MediaImportOwner? ImportOwner { get; set; } + + private MediaImportOwner EffectiveImportOwner => this.ImportOwner ?? this.fallbackMediaImportOwner; + + [Parameter] + public string MediaImportTargetId { get; set; } = string.Empty; + + private string EffectiveMediaImportTargetId => string.IsNullOrWhiteSpace(this.MediaImportTargetId) + ? string.IsNullOrWhiteSpace(this.Text) ? "primary" : this.Text + : this.MediaImportTargetId; + + private MediaImportTarget EffectiveMediaImportTarget => new(this.EffectiveImportOwner, this.EffectiveMediaImportTargetId); + [Parameter] public string Text { get; set; } = string.Empty; @@ -47,17 +65,24 @@ public partial class ReadFileContent : MSGComponentBase [Inject] private PandocAvailabilityService PandocAvailabilityService { get; init; } = null!; + [Inject] + private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; + private const string DEFAULT_DRAG_CLASS = "relative rounded-lg border-2 border-dashed pa-3 mb-3 mud-width-full"; private string ButtonText => string.IsNullOrWhiteSpace(this.Text) ? T("Use file content as input") : this.Text; private string dragClass = DEFAULT_DRAG_CLASS; private uint numDropAreasAboveThis; private bool isComponentHovered; + private bool IsCurrentTargetBusy => this.MediaTranscriptionService.GetSnapshot(this.EffectiveImportOwner) is { IsBusy: true } snapshot + && snapshot.Target == this.EffectiveMediaImportTarget; + private bool IsUnavailable => this.Disabled || this.MediaTranscriptionService.IsBusy(this.EffectiveImportOwner); #region Overrides of MSGComponentBase protected override async Task OnInitializedAsync() { + this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; if (this.EnableDragDrop) { this.ApplyFilters([], [ Event.TAURI_EVENT_RECEIVED, Event.REGISTER_FILE_DROP_AREA, Event.UNREGISTER_FILE_DROP_AREA ]); @@ -65,6 +90,69 @@ public partial class ReadFileContent : MSGComponentBase } await base.OnInitializedAsync(); + await this.SyncCompletedMediaTextAsync(); + } + + /// Refreshes disabled controls when the shared import lane changes. + private void OnMediaImportStateChanged(MediaImportOwner owner) + { + if (owner == this.EffectiveImportOwner) + _ = this.InvokeAsync(async () => + { + await this.SyncCompletedMediaTextAsync(); + await this.ConsumeStandaloneMediaOutcomeAsync(); + this.StateHasChanged(); + }); + } + + /// Consumes outcomes for dialog-local controls that have no assistant owner surface. + private async Task ConsumeStandaloneMediaOutcomeAsync() + { + if (this.ImportOwner is not null) + return; + + var outcome = this.MediaTranscriptionService.TryConsumeOutcome(this.EffectiveImportOwner); + if (outcome is null) + return; + + if (outcome.Failures.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Failures.Select(failure => $"{failure.FileName}: {failure.UserMessage}")); + await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, message)); + } + else if (outcome.Status is MediaImportStatus.FAILED) + { + await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed."))); + } + + if (outcome.Warnings.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}")); + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message)); + } + + if (outcome.Status is MediaImportStatus.CANCELLED) + { + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled."))); + } + } + + /// Applies a completed target transcript after progress or navigation. + private async Task SyncCompletedMediaTextAsync() + { + var delivery = this.MediaTranscriptionService.GetPendingDelivery(this.EffectiveMediaImportTarget); + if (delivery is null || delivery.Text is not { } text) + return; + + await this.FileContentChanged.InvokeAsync(text); + this.MediaTranscriptionService.AcknowledgeDelivery(delivery); + } + + /// Unsubscribes from the singleton media service. + protected override void DisposeResources() + { + this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; + base.DisposeResources(); } protected override async Task ProcessIncomingMessage(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default @@ -72,7 +160,7 @@ public partial class ReadFileContent : MSGComponentBase if (!this.EnableDragDrop) return; - if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED) + if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED) return; switch (triggeredEvent) @@ -126,10 +214,7 @@ public partial class ReadFileContent : MSGComponentBase private async Task SelectFile() { - if (this.Disabled) - return; - - if (!await this.EnsurePandocAvailability()) + if (this.IsUnavailable) return; var selectedFile = await this.RustService.SelectFile(T("Select file to read its content")); @@ -161,9 +246,6 @@ public partial class ReadFileContent : MSGComponentBase private async Task LoadFirstValidFile(List paths) { - if (!await this.EnsurePandocAvailability()) - return; - foreach (var path in paths) { if (await this.LoadFileIfValid(path)) @@ -179,6 +261,12 @@ public partial class ReadFileContent : MSGComponentBase return false; } + if (FileTypes.IsAllowedPath(filePath, FileTypes.AUDIO) || FileTypes.IsAllowedPath(filePath, FileTypes.VIDEO)) + return await this.LoadMediaTranscriptAsync(filePath); + + if (!await this.EnsurePandocAvailability()) + return false; + if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.DIRECTLY_LOADING_CONTENT, filePath)) { this.Logger.LogWarning("User attempted to load unsupported file: {FilePath}", filePath); @@ -200,6 +288,42 @@ public partial class ReadFileContent : MSGComponentBase } } + private async Task LoadMediaTranscriptAsync(string filePath) + { + if (string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider)) + { + await this.MessageBus.SendWarning(new( + Icons.Material.Filled.VoiceChat, + this.T("Media files require a configured transcription provider. Configure one in the transcription settings."))); + return false; + } + + var message = this.T("The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider."); + var dialogParameters = new DialogParameters + { + { + x => x.MarkdownBody, + $""" + {message} + + - {Markdown.EscapeInlineText(Path.GetFileName(filePath))} + """ + }, + }; + var dialogReference = await this.DialogService.ShowAsync( + this.T("Transcribe media file"), + dialogParameters, + Dialogs.DialogOptions.FULLSCREEN); + + var dialogResult = await dialogReference.Result; + if (dialogResult is null || dialogResult.Canceled) + return false; + + return this.MediaTranscriptionService.TryStartTextImport( + filePath, + this.EffectiveMediaImportTarget); + } + private bool CanCatchDroppedFile() => this.numDropAreasAboveThis is 0 && (this.isComponentHovered || this.CatchAllDocuments); private void SetDragClass() => this.dragClass = $"{DEFAULT_DRAG_CLASS} mud-border-primary border-2"; @@ -208,7 +332,7 @@ public partial class ReadFileContent : MSGComponentBase private void OnMouseEnter(EventArgs _) { - if(this.Disabled || this.numDropAreasAboveThis > 0) + if(this.IsUnavailable || this.numDropAreasAboveThis > 0) return; this.Logger.LogDebug("Read file content component is hovered."); @@ -219,7 +343,7 @@ public partial class ReadFileContent : MSGComponentBase private void OnMouseLeave(EventArgs _) { - if(this.Disabled) + if(this.IsUnavailable) return; this.Logger.LogDebug("Read file content component is no longer hovered."); diff --git a/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs b/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs index 669932f6..f754695f 100644 --- a/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs +++ b/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs @@ -1,6 +1,7 @@ -using AIStudio.Provider; +using System.Buffers.Binary; + using AIStudio.Settings.DataModel; -using AIStudio.Tools.MIME; +using AIStudio.Tools.Media; using AIStudio.Tools.Rust; using AIStudio.Tools.Services; @@ -10,6 +11,8 @@ namespace AIStudio.Components; public partial class VoiceRecorder : MSGComponentBase { + private const int PCM_WAV_HEADER_SIZE = 44; + [Inject] private ILogger Logger { get; init; } = null!; @@ -25,6 +28,9 @@ public partial class VoiceRecorder : MSGComponentBase [Inject] private VoiceRecordingAvailabilityService VoiceRecordingAvailabilityService { get; init; } = null!; + [Inject] + private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; + #region Overrides of MSGComponentBase protected override async Task OnInitializedAsync() @@ -93,7 +99,6 @@ public partial class VoiceRecorder : MSGComponentBase private bool isTranscribing; private FileStream? currentRecordingStream; private string? currentRecordingPath; - private string? currentRecordingMimeType; private string? finalRecordingPath; private DotNetObjectReference? dotNetReference; @@ -131,17 +136,7 @@ public partial class VoiceRecorder : MSGComponentBase return; } - var mimeTypes = GetPreferredMimeTypes( - Builder.Create().UseAudio().UseSubtype(AudioSubtype.WEBM).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.OGG).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.AAC).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.MP3).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.AIFF).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.WAV).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.FLAC).Build() - ); - - this.Logger.LogInformation("Starting audio recording with preferred MIME types: '{PreferredMimeTypes}'.", string.Join(", ", mimeTypes)); + this.Logger.LogInformation("Starting PCM/WAV audio recording."); // Create a DotNetObjectReference to pass to JavaScript: this.dotNetReference = DotNetObjectReference.Create(this); @@ -151,13 +146,8 @@ public partial class VoiceRecorder : MSGComponentBase try { - var mimeTypeStrings = mimeTypes.ToStringArray(); - var actualMimeType = await this.JsRuntime.InvokeAsync("audioRecorder.start", this.dotNetReference, mimeTypeStrings); - - // Store the MIME type for later use: - this.currentRecordingMimeType = actualMimeType; - - this.Logger.LogInformation("Audio recording started with MIME type: '{ActualMimeType}'.", actualMimeType); + await this.JsRuntime.InvokeVoidAsync("audioRecorder.start", this.dotNetReference); + this.Logger.LogInformation("PCM/WAV audio recording started."); this.isPreparing = false; this.isRecording = true; } @@ -168,6 +158,7 @@ public partial class VoiceRecorder : MSGComponentBase // Clean up the recording stream if starting failed: await this.FinalizeRecordingStream(); + await this.ReleaseMicrophoneAsync(); } finally { @@ -176,11 +167,11 @@ public partial class VoiceRecorder : MSGComponentBase } else { + var recordingStoppedSuccessfully = false; try { - var result = await this.JsRuntime.InvokeAsync("audioRecorder.stop"); - if (result.ChangedMimeType) - this.Logger.LogWarning("The recorded audio MIME type was changed to '{ResultMimeType}'.", result.MimeType); + await this.JsRuntime.InvokeVoidAsync("audioRecorder.stop"); + recordingStoppedSuccessfully = true; } catch (Exception e) { @@ -194,28 +185,21 @@ public partial class VoiceRecorder : MSGComponentBase this.isRecording = false; this.StateHasChanged(); - // Start transcription if we have a recording and a configured provider: - if (this.finalRecordingPath is not null) - await this.TranscribeRecordingAsync(); - } - } + if (!recordingStoppedSuccessfully || this.finalRecordingPath is null) + { + if (recordingStoppedSuccessfully) + { + this.Logger.LogWarning("The audio recorder did not produce any data."); + await this.MessageBus.SendError(new(Icons.Material.Filled.MicOff, this.T("Failed to stop audio recording."))); + } - private static MIMEType[] GetPreferredMimeTypes(params MIMEType[] mimeTypes) - { - // Default list if no parameters provided: - if (mimeTypes.Length is 0) - { - var audioBuilder = Builder.Create().UseAudio(); - return - [ - audioBuilder.UseSubtype(AudioSubtype.WEBM).Build(), - audioBuilder.UseSubtype(AudioSubtype.OGG).Build(), - audioBuilder.UseSubtype(AudioSubtype.MP4).Build(), - audioBuilder.UseSubtype(AudioSubtype.MPEG).Build(), - ]; - } + this.DeleteFinalRecording(); + await this.ReleaseMicrophoneAsync(); + return; + } - return mimeTypes; + await this.TranscribeRecordingAsync(); + } } private async Task InitializeRecordingStream() @@ -226,7 +210,7 @@ public partial class VoiceRecorder : MSGComponentBase if (!Directory.Exists(recordingDirectory)) Directory.CreateDirectory(recordingDirectory); - var fileName = $"recording_{DateTime.UtcNow:yyyyMMdd_HHmmss}.audio"; + var fileName = $"recording_{DateTime.UtcNow:yyyyMMdd_HHmmss}.wav"; this.currentRecordingPath = Path.Combine(recordingDirectory, fileName); this.currentRecordingStream = new FileStream(this.currentRecordingPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 8192, useAsync: true); @@ -253,6 +237,7 @@ public partial class VoiceRecorder : MSGComponentBase catch (Exception ex) { this.Logger.LogError(ex, "Error writing audio chunk to stream."); + throw; } } @@ -262,45 +247,56 @@ public partial class VoiceRecorder : MSGComponentBase if (this.currentRecordingStream is not null) { await this.currentRecordingStream.FlushAsync(); + var hasPcmAudioData = await this.FinalizePcmWavHeaderAsync(this.currentRecordingStream); await this.currentRecordingStream.DisposeAsync(); this.currentRecordingStream = null; - // Rename the file with the correct extension based on MIME type: - if (this.currentRecordingPath is not null && this.currentRecordingMimeType is not null) + if (this.currentRecordingPath is not null && File.Exists(this.currentRecordingPath)) { - var extension = GetFileExtension(this.currentRecordingMimeType); - var newPath = Path.ChangeExtension(this.currentRecordingPath, extension); + var fileSize = new FileInfo(this.currentRecordingPath).Length; - if (File.Exists(this.currentRecordingPath)) + if (hasPcmAudioData) { - File.Move(this.currentRecordingPath, newPath, overwrite: true); - this.finalRecordingPath = newPath; - this.Logger.LogInformation("Finalized audio recording over {NumChunks} streamed audio chunks to the file '{RecordingPath}'.", this.numReceivedChunks, newPath); + this.finalRecordingPath = this.currentRecordingPath; + this.Logger.LogInformation("Finalized audio recording over {NumChunks} streamed audio chunks to the file '{RecordingPath}' with {FileSize} bytes.", this.numReceivedChunks, this.currentRecordingPath, fileSize); + } + else + { + this.Logger.LogWarning("Discarding a PCM/WAV audio recording without audio data ({FileSize} bytes).", fileSize); + File.Delete(this.currentRecordingPath); } } } this.currentRecordingPath = null; - this.currentRecordingMimeType = null; // Dispose the .NET reference: this.dotNetReference?.Dispose(); this.dotNetReference = null; } - private static string GetFileExtension(string mimeType) + private async Task FinalizePcmWavHeaderAsync(FileStream recordingStream) { - var baseMimeType = mimeType.Split(';')[0].Trim().ToLowerInvariant(); - return baseMimeType switch - { - "audio/webm" => ".webm", - "audio/ogg" => ".ogg", - "audio/mp4" => ".m4a", - "audio/mpeg" => ".mp3", - "audio/wav" => ".wav", - "audio/x-wav" => ".wav", - _ => ".audio" // Fallback - }; + if (recordingStream.Length <= PCM_WAV_HEADER_SIZE) + return false; + + var pcmDataSize = recordingStream.Length - PCM_WAV_HEADER_SIZE; + if (pcmDataSize > uint.MaxValue - 36) + throw new InvalidDataException("The streamed PCM recording exceeds the WAV size limit."); + + var valueBuffer = new byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(valueBuffer, checked((uint)(36 + pcmDataSize))); + recordingStream.Seek(4, SeekOrigin.Begin); + await recordingStream.WriteAsync(valueBuffer); + + BinaryPrimitives.WriteUInt32LittleEndian(valueBuffer, checked((uint)pcmDataSize)); + recordingStream.Seek(40, SeekOrigin.Begin); + await recordingStream.WriteAsync(valueBuffer); + recordingStream.Seek(0, SeekOrigin.End); + await recordingStream.FlushAsync(); + + this.Logger.LogInformation("Finalized a streamed PCM/WAV header for {PcmDataSize} bytes of audio data.", pcmDataSize); + return true; } private async Task TranscribeRecordingAsync() @@ -317,58 +313,22 @@ public partial class VoiceRecorder : MSGComponentBase try { - // Get the configured transcription provider ID: - var transcriptionProviderId = this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider; - if (string.IsNullOrWhiteSpace(transcriptionProviderId)) + var transcriptionResult = await this.MediaTranscriptionService.TranscribeVoiceAsync(this.finalRecordingPath); + if (transcriptionResult.Status is not MediaTranscriptionResultStatus.SUCCEEDED) { - this.Logger.LogWarning("No transcription provider is configured."); - await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("No transcription provider is configured."))); - return; - } + if (transcriptionResult.Status is MediaTranscriptionResultStatus.CANCELLED) + return; - // Find the transcription provider in the list of configured providers: - var transcriptionProviderSettings = this.SettingsManager.ConfigurationData.TranscriptionProviders - .FirstOrDefault(x => x.Id == transcriptionProviderId); + if (transcriptionResult.Status is MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL) + { + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, transcriptionResult.UserMessage)); + return; + } - if (transcriptionProviderSettings is null) - { - this.Logger.LogWarning("The configured transcription provider with ID '{ProviderId}' was not found.", transcriptionProviderId); - await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The configured transcription provider was not found."))); - return; - } - - // Check the confidence level: - var minimumLevel = this.SettingsManager.GetMinimumConfidenceLevel(Tools.Components.NONE); - var providerConfidence = transcriptionProviderSettings.UsedLLMProvider.GetConfidence(this.SettingsManager); - if (providerConfidence.Level < minimumLevel) - { - this.Logger.LogWarning( - "The configured transcription provider '{ProviderName}' has a confidence level of '{ProviderLevel}', which is below the minimum required level of '{MinimumLevel}'.", - transcriptionProviderSettings.UsedLLMProvider, - providerConfidence.Level, - minimumLevel); - await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The configured transcription provider does not meet the minimum confidence level."))); - return; - } - - // Create the provider instance: - var provider = transcriptionProviderSettings.CreateProvider(); - if (provider.Provider is LLMProviders.NONE) - { - this.Logger.LogError("Failed to create the transcription provider instance."); - await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("Failed to create the transcription provider."))); - return; - } - - // Call the transcription API: - this.Logger.LogInformation("Starting transcription with provider '{ProviderName}' and model '{ModelName}'.", transcriptionProviderSettings.UsedLLMProvider, transcriptionProviderSettings.Model.ToString()); - var transcriptionResult = await provider.TranscribeAudioAsync(transcriptionProviderSettings.Model, this.finalRecordingPath, this.SettingsManager); - if (!transcriptionResult.Success) - { this.Logger.LogWarning("The transcription request failed."); - var userMessage = string.IsNullOrWhiteSpace(transcriptionResult.ErrorMessage) + var userMessage = string.IsNullOrWhiteSpace(transcriptionResult.UserMessage) ? this.T("Unfortunately, there was an error communicating with the AI system.") - : transcriptionResult.ErrorMessage; + : transcriptionResult.UserMessage; await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, userMessage)); return; } @@ -406,19 +366,6 @@ public partial class VoiceRecorder : MSGComponentBase // Copy the transcribed text to the clipboard: await this.RustService.CopyText2Clipboard(this.Snackbar, transcribedText); - // Delete the recording file: - try - { - if (File.Exists(this.finalRecordingPath)) - { - File.Delete(this.finalRecordingPath); - this.Logger.LogInformation("Deleted the recording file '{RecordingPath}'.", this.finalRecordingPath); - } - } - catch (Exception ex) - { - this.Logger.LogError(ex, "Failed to delete the recording file '{RecordingPath}'.", this.finalRecordingPath); - } } catch (Exception ex) { @@ -428,13 +375,31 @@ public partial class VoiceRecorder : MSGComponentBase finally { await this.ReleaseMicrophoneAsync(); - - this.finalRecordingPath = null; + this.DeleteFinalRecording(); this.isTranscribing = false; this.StateHasChanged(); } } + private void DeleteFinalRecording() + { + var recordingPath = this.finalRecordingPath; + this.finalRecordingPath = null; + + if (recordingPath is null) + return; + + try + { + if (File.Exists(recordingPath)) + File.Delete(recordingPath); + } + catch (Exception ex) + { + this.Logger.LogError(ex, "Failed to delete the recording file '{RecordingPath}'.", recordingPath); + } + } + private async Task ReleaseMicrophoneAsync() { // Wait a moment for any queued sounds to finish playing, then release the microphone. @@ -530,4 +495,4 @@ public partial class VoiceRecorder : MSGComponentBase } #endregion -} \ No newline at end of file +} diff --git a/app/MindWork AI Studio/Components/Workspaces.razor.cs b/app/MindWork AI Studio/Components/Workspaces.razor.cs index 0848fa34..8ec4165a 100644 --- a/app/MindWork AI Studio/Components/Workspaces.razor.cs +++ b/app/MindWork AI Studio/Components/Workspaces.razor.cs @@ -4,6 +4,8 @@ using System.Text.Json; using AIStudio.Chat; using AIStudio.Dialogs; using AIStudio.Tools.AIJobs; +using AIStudio.Tools.Media; +using AIStudio.Tools.Services; using Microsoft.AspNetCore.Components; @@ -21,6 +23,9 @@ public partial class Workspaces : MSGComponentBase [Inject] private AIJobService AIJobService { get; init; } = null!; + + [Inject] + private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; [Parameter] public ChatThread? CurrentChatThread { get; set; } @@ -55,6 +60,7 @@ public partial class Workspaces : MSGComponentBase protected override async Task OnInitializedAsync() { + this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; await base.OnInitializedAsync(); this.ApplyFilters([], [ Event.AI_JOB_CHANGED, Event.AI_JOB_FINISHED, Event.CHAT_GENERATION_CHANGED, Event.WORKSPACE_CREATED ]); _ = this.LoadTreeItemsAsync(startPrefetch: true); @@ -376,12 +382,26 @@ public partial class Workspaces : MSGComponentBase private bool IsChatTreeItemBusy(TreeItemData treeItem) { - return treeItem.Type is TreeItemType.CHAT && this.AIJobService.IsChatGenerationActive(treeItem.ChatId); + return treeItem.Type is TreeItemType.CHAT + && (this.AIJobService.IsChatGenerationActive(treeItem.ChatId) + || this.MediaTranscriptionService.IsBusy(MediaImportOwner.ForChat(treeItem.ChatId))); } private string GetChatTreeItemTextStyle(TreeItemData treeItem) { - return this.IsCurrentChatTreeItem(treeItem) ? "justify-self: start; font-weight: 700;" : "justify-self: start;"; + var status = this.MediaTranscriptionService.GetSnapshot(MediaImportOwner.ForChat(treeItem.ChatId))?.Status; + var color = status switch + { + MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => " color: var(--mud-palette-info);", + MediaImportStatus.SUCCEEDED => " color: var(--mud-palette-success);", + MediaImportStatus.WARNING => " color: var(--mud-palette-warning);", + MediaImportStatus.FAILED => " color: var(--mud-palette-error);", + MediaImportStatus.CANCELLED => " color: var(--mud-palette-warning);", + _ => string.Empty, + }; + + var weight = this.IsCurrentChatTreeItem(treeItem) ? " font-weight: 700;" : string.Empty; + return $"justify-self: start;{weight}{color}"; } private bool IsCurrentChatTreeItem(TreeItemData treeItem) @@ -394,6 +414,22 @@ public partial class Workspaces : MSGComponentBase private string GetChatTreeIcon(Guid chatId, string defaultIcon) { + var mediaStatus = this.MediaTranscriptionService.GetSnapshot(MediaImportOwner.ForChat(chatId))?.Status; + if (mediaStatus is not null) + { + return mediaStatus switch + { + MediaImportStatus.QUEUED => Icons.Material.Filled.HourglassTop, + MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => Icons.Material.Filled.ChangeCircle, + MediaImportStatus.SUCCEEDED => Icons.Material.Filled.TaskAlt, + MediaImportStatus.WARNING => Icons.Material.Filled.WarningAmber, + MediaImportStatus.FAILED => Icons.Material.Filled.Error, + MediaImportStatus.CANCELLED => Icons.Material.Filled.Cancel, + + _ => defaultIcon, + }; + } + var snapshot = this.AIJobService.TryGetChatSnapshot(chatId); if (snapshot is null || !snapshot.IsActive) return defaultIcon; @@ -406,6 +442,12 @@ public partial class Workspaces : MSGComponentBase }; } + private void OnMediaImportStateChanged(MediaImportOwner owner) + { + if (owner.Kind is MediaImportOwnerKind.CHAT) + _ = this.SafeStateHasChanged(); + } + private async Task SafeStateHasChanged() { if (this.isDisposed) @@ -668,7 +710,8 @@ public partial class Workspaces : MSGComponentBase if (chat is null) return; - if (this.AIJobService.IsChatGenerationActive(chat.ChatId)) + var mediaOwner = MediaImportOwner.ForChat(chat.ChatId); + if (this.AIJobService.IsChatGenerationActive(chat.ChatId) || this.MediaTranscriptionService.IsBusy(mediaOwner)) return; if (askForConfirmation) @@ -692,6 +735,7 @@ public partial class Workspaces : MSGComponentBase } await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, chat.WorkspaceId, chat.ChatId, askForConfirmation: false); + this.MediaTranscriptionService.ClearOwnerState(mediaOwner); await this.LoadTreeItemsAsync(startPrefetch: false); if (unloadChat && this.CurrentChatThread?.ChatId == chat.ChatId) @@ -845,16 +889,13 @@ public partial class Workspaces : MSGComponentBase if (workspaceId == Guid.Empty) return; - await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, chat.WorkspaceId, chat.ChatId, askForConfirmation: false); - - chat.WorkspaceId = workspaceId; + await WorkspaceBehaviour.MoveChatAsync(chat, workspaceId); if (this.CurrentChatThread?.ChatId == chat.ChatId) { this.CurrentChatThread = chat; await this.CurrentChatThreadChanged.InvokeAsync(this.CurrentChatThread); } - - await WorkspaceBehaviour.StoreChatAsync(chat); + await this.LoadTreeItemsAsync(startPrefetch: false); } @@ -914,6 +955,7 @@ public partial class Workspaces : MSGComponentBase protected override void DisposeResources() { + this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; this.isDisposed = true; this.prefetchCancellationTokenSource?.Cancel(); this.prefetchCancellationTokenSource?.Dispose(); diff --git a/app/MindWork AI Studio/Dialogs/ConfirmDialog.razor b/app/MindWork AI Studio/Dialogs/ConfirmDialog.razor index 9e55a4b3..6f48c798 100644 --- a/app/MindWork AI Studio/Dialogs/ConfirmDialog.razor +++ b/app/MindWork AI Studio/Dialogs/ConfirmDialog.razor @@ -1,9 +1,16 @@ @inherits MSGComponentBase - - @this.Message - + @if (!string.IsNullOrWhiteSpace(this.MarkdownBody)) + { + + } + else + { + + @this.Message + + } diff --git a/app/MindWork AI Studio/Dialogs/ConfirmDialog.razor.cs b/app/MindWork AI Studio/Dialogs/ConfirmDialog.razor.cs index f022152e..696d6fa4 100644 --- a/app/MindWork AI Studio/Dialogs/ConfirmDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/ConfirmDialog.razor.cs @@ -15,6 +15,12 @@ public partial class ConfirmDialog : MSGComponentBase [Parameter] public string Message { get; set; } = string.Empty; + /// + /// Optional Markdown content rendered instead of using the message property. + /// + [Parameter] + public string MarkdownBody { get; set; } = string.Empty; + private void Cancel() => this.MudDialog.Cancel(); private void Confirm() => this.MudDialog.Close(DialogResult.Ok(true)); diff --git a/app/MindWork AI Studio/Layout/MainLayout.razor.cs b/app/MindWork AI Studio/Layout/MainLayout.razor.cs index 2bac1fd8..ad0bf3e5 100644 --- a/app/MindWork AI Studio/Layout/MainLayout.razor.cs +++ b/app/MindWork AI Studio/Layout/MainLayout.razor.cs @@ -3,6 +3,7 @@ using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.AIJobs; using AIStudio.Tools.AssistantSessions; +using AIStudio.Tools.Media; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; using AIStudio.Tools.Services; @@ -37,6 +38,9 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan [Inject] private AssistantSessionService AssistantSessionService { get; init; } = null!; + + [Inject] + private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!; [Inject] private ISnackbar Snackbar { get; init; } = null!; @@ -75,6 +79,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan protected override async Task OnInitializedAsync() { this.NavigationManager.RegisterLocationChangingHandler(this.OnLocationChanging); + this.MediaTranscriptionService.StateChanged += this.OnMediaImportStateChanged; // // We use the Tauri API (Rust) to get the data and config directories @@ -348,6 +353,16 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan { this.navItems = new List(this.GetNavItems()); } + + /// Refreshes navigation activity colors when a media import changes state. + private void OnMediaImportStateChanged(MediaImportOwner owner) + { + _ = this.InvokeAsync(() => + { + this.LoadNavItems(); + this.StateHasChanged(); + }); + } private IEnumerable GetNavItems() { @@ -356,10 +371,15 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan var activityIndicatorDarkColor = this.ColorTheme.GetActivityIndicatorDarkColor(); var defaultLightColor = palette.DarkLighten; var defaultDarkColor = palette.GrayLight; - var chatLightColor = this.AIJobService.HasActiveJobs ? activityIndicatorLightColor : defaultLightColor; - var chatDarkColor = this.AIJobService.HasActiveJobs ? activityIndicatorDarkColor : defaultDarkColor; - var assistantsLightColor = this.AssistantSessionService.HasActiveSessions ? activityIndicatorLightColor : defaultLightColor; - var assistantsDarkColor = this.AssistantSessionService.HasActiveSessions ? activityIndicatorDarkColor : defaultDarkColor; + var mediaSnapshots = this.MediaTranscriptionService.GetSnapshots(); + var hasActiveChatMedia = mediaSnapshots.Any(snapshot => snapshot.IsBusy && snapshot.Owner.Kind is MediaImportOwnerKind.CHAT); + var hasActiveAssistantMedia = mediaSnapshots.Any(snapshot => snapshot.IsBusy && snapshot.Owner.Kind is MediaImportOwnerKind.ASSISTANT); + var hasActiveChatWork = this.AIJobService.HasActiveJobs || hasActiveChatMedia; + var hasActiveAssistantWork = this.AssistantSessionService.HasActiveSessions || hasActiveAssistantMedia; + var chatLightColor = hasActiveChatWork ? activityIndicatorLightColor : defaultLightColor; + var chatDarkColor = hasActiveChatWork ? activityIndicatorDarkColor : defaultDarkColor; + var assistantsLightColor = hasActiveAssistantWork ? activityIndicatorLightColor : defaultLightColor; + var assistantsDarkColor = hasActiveAssistantWork ? activityIndicatorDarkColor : defaultDarkColor; yield return new(T("Home"), Icons.Material.Filled.Home, defaultLightColor, defaultDarkColor, Routes.HOME, true); yield return new(T("Chat"), Icons.Material.Filled.Chat, chatLightColor, chatDarkColor, Routes.CHAT, false); @@ -535,6 +555,7 @@ public partial class MainLayout : LayoutComponentBase, IMessageBusReceiver, ILan public void Dispose() { + this.MediaTranscriptionService.StateChanged -= this.OnMediaImportStateChanged; this.MessageBus.Unregister(this); this.mandatoryInfoDialogSemaphore.Dispose(); } diff --git a/app/MindWork AI Studio/Pages/Information.razor b/app/MindWork AI Studio/Pages/Information.razor index 18863903..f3858a04 100644 --- a/app/MindWork AI Studio/Pages/Information.razor +++ b/app/MindWork AI Studio/Pages/Information.razor @@ -308,7 +308,11 @@ - + + + + + diff --git a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua index 98099068..68f299b5 100644 --- a/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua @@ -306,6 +306,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::AGENDA::NUMBERPARTICIPANTSEXTENSIONS::T81 -- Stop generation UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1317408357"] = "Generierung stoppen" +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1543974632"] = "Die Mediendatei konnte nicht transkribiert werden." + -- Reset UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T180921696"] = "Zurücksetzen" @@ -315,6 +318,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1809312323"] = "Bitte wä -- The assistant failed. The message is: '{0}' UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1833836281"] = "Der Assistent ist fehlgeschlagen. Die Meldung lautet: „{0}“" +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T241403726"] = "Die Transkription des Mediums wurde abgebrochen." + -- This assistant is already running. AI Studio opens the running session instead. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T2575715765"] = "Dieser Assistent läuft bereits. AI Studio öffnet stattdessen die laufende Sitzung." @@ -2295,9 +2301,24 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T349928509"] = "Das Bil -- Open Settings UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T1172211894"] = "Einstellungen öffnen" +-- Media transcription was canceled. Open the assistant to review it. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T1233815302"] = "Die Medientranskription wurde abgebrochen. Öffnen Sie den Assistenten, um sie zu überprüfen." + +-- Media transcription failed. Open the assistant to review it. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2177964639"] = "Die Transkription der Medieninhalte ist fehlgeschlagen. Öffnen Sie den Assistenten, um sie zu überprüfen." + +-- Media transcription completed with a warning. Open the assistant to review it. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2217674098"] = "Die Medientranskription wurde mit einer Warnung abgeschlossen. Öffnen Sie den Assistenten, um sie zu überprüfen." + +-- Media is still being prepared. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2600900617"] = "Die Medien werden noch vorbereitet." + -- Assistant is still running. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2719896610"] = "Assistent läuft noch." +-- The media transcript is ready. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T3248321953"] = "Das Medientranskript ist fertig." + -- Assistant was canceled. Open it to review the result. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T3390934621"] = "Assistent wurde abgebrochen. Öffnen Sie ihn, um das Ergebnis zu überprüfen." @@ -2400,18 +2421,33 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T1358313858"] = "Klicken -- Drop files here to attach them. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T143112277"] = "Dateien hier ablegen, um sie anzuhängen." +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T1543974632"] = "Die Mediendatei konnte nicht transkribiert werden." + -- Click here to attach files. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T1875575968"] = "Klicken Sie hier, um Dateien anzuhängen." +-- Transcribe media files +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T2178031033"] = "Mediendateien transkribieren" + -- Drag and drop files into the marked area or click here to attach documents: UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T230755331"] = "Ziehen Sie Dateien in den markierten Bereich oder klicken Sie hier, um Dokumente anzuhängen:" +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T241403726"] = "Die Transkription des Mediums wurde abgebrochen." + -- Select files to attach UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T2495931372"] = "Dateien zum Anhängen auswählen" -- Document Preview UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T285154968"] = "Dokumentenvorschau" +-- Media files require a configured transcription provider. Configure one in the transcription settings. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T3172443094"] = "Für Mediendateien muss ein Transkriptionsanbieter eingerichtet sein. Richten Sie in den Einstellungen der Transkriptionen einen Anbieter ein." + +-- The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T322693339"] = "Die ausgewählten Audio- und Videodateien werden lokal vorbereitet. Anschließend werden die Audiodaten an den konfigurierten Transkriptionsanbieter hochgeladen." + -- Clear file list UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T3759696136"] = "Dateiliste löschen" @@ -2436,6 +2472,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1317408357"] = "Generieru -- Save chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1516264254"] = "Chat speichern" +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "Die Mediendatei konnte nicht transkribiert werden." + -- Type your input here... UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Geben Sie hier Ihre Eingabe ein..." @@ -2448,6 +2487,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2036185364"] = "Code" -- Italic UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2377171085"] = "Kursiv" +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T241403726"] = "Die Transkription der Mediendatei wurde abgebrochen." + -- Profile usage is disabled according to your chat template settings. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2670286472"] = "Die Profilnutzung ist gemäß den Einstellungen ihrer Chat-Vorlage deaktiviert." @@ -2673,6 +2715,24 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANDATORYINFODISPLAY::T3511160492"] = "Ak -- Please review this text again. The content was changed. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANDATORYINFODISPLAY::T941885055"] = "Bitte lesen Sie diesen Text erneut durch. Der Inhalt wurde geändert." +-- Waiting to prepare media +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T1167267986"] = "Warten, bis die Medien vorbereitet sind" + +-- Stop media transcription +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T1868377405"] = "Medientranskription stoppen" + +-- Stopping media transcription +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T1878101489"] = "Transkription von Medien wird beendet" + +-- Inspecting media +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T2431421733"] = "Medien werden geprüft" + +-- Transcribing +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T2938661425"] = "Transkribieren" + +-- Preparing audio +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T3200155905"] = "Audio wird vorbereitet" + -- Given that my employer's workplace uses both Windows and Linux, I wanted a cross-platform solution that would work seamlessly across all major operating systems, including macOS. Additionally, I wanted to demonstrate that it is possible to create modern, efficient, cross-platform applications without resorting to Electron bloatware. The combination of .NET and Rust with Tauri proved to be an excellent technology stack for building such robust applications. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MOTIVATION::T1057189794"] = "Da mein Arbeitgeber sowohl Windows als auch Linux am Arbeitsplatz nutzt, wollte ich eine plattformübergreifende Lösung, die nahtlos auf allen wichtigen Betriebssystemen, einschließlich macOS, funktioniert. Außerdem wollte ich zeigen, dass es möglich ist, moderne, effiziente und plattformübergreifende Anwendungen zu erstellen, ohne auf Software-Ballast, wie z.B. das Electron-Framework, zurückzugreifen. Die Kombination aus .NET und Rust mit Tauri hat sich dabei als hervorragender Technologie-Stack für den Bau solch robuster Anwendungen erwiesen." @@ -2790,18 +2850,33 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::PROVIDERSELECTION::T4279448758"] = "Nutzt -- Provider UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::PROVIDERSELECTION::T900237532"] = "Anbieter" +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T1543974632"] = "Die Mediendatei konnte nicht transkribiert werden." + -- Failed to load file content UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T1989554334"] = "Laden des Dateiinhalts fehlgeschlagen" -- Drop one file here to load its content. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T2274562398"] = "Datei hier ablegen, um ihren Inhalt zu laden." +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T241403726"] = "Die Transkription des Mediums wurde abgebrochen." + +-- The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T2839709466"] = "Die ausgewählte Mediendatei wird lokal vorbereitet. Anschließend wird die Audiospur an den konfigurierten Transkriptionsanbieter hochgeladen." + +-- Media files require a configured transcription provider. Configure one in the transcription settings. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3172443094"] = "Für Mediendateien muss ein Transkriptionsanbieter eingerichtet sein. Richten Sie in den Transkriptionseinstellungen einen Anbieter ein." + -- Use file content as input UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3499386973"] = "Dokumenteninhalt als Eingabe verwenden" -- Select file to read its content UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T354817589"] = "Datei auswählen, um den Inhalt zu lesen" +-- Transcribe media file +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3554222960"] = "Mediendatei transkribieren" + -- The content is cleaned using an LLM agent: the main content is extracted, advertisements and other irrelevant things are attempted to be removed; relative links are attempted to be converted into absolute links so that they can be used. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READWEBCONTENT::T1164201762"] = "Der Inhalt wird mithilfe eines LLM-Agents bereinigt: Der Hauptinhalt wird extrahiert, Werbung und andere irrelevante Elemente werden nach Möglichkeit entfernt. Relative Links werden nach Möglichkeit in absolute Links umgewandelt, damit sie verwendet werden können." @@ -3540,9 +3615,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VISION::T586430036"] = "Nützliche Assist -- Voice recording has been disabled for this session because audio playback could not be initialized on the client. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T1123032432"] = "Die Sprachaufnahme wurde für diese Sitzung deaktiviert, da die Audiowiedergabe auf dem Client nicht initialisiert werden konnte." --- Failed to create the transcription provider. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T1689988905"] = "Der Anbieter für die Transkription konnte nicht erstellt werden." - -- Failed to start audio recording. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T2144994226"] = "Audioaufnahme konnte nicht gestartet werden." @@ -3561,21 +3633,12 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T2851219233"] = "Transkrip -- Unfortunately, there was an error communicating with the AI system. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3236134591"] = "Leider ist bei der Kommunikation mit dem KI-System ein Fehler aufgetreten." --- The configured transcription provider was not found. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T331613105"] = "Der konfigurierte Anbieter für die Transkription wurde nicht gefunden." - -- Failed to stop audio recording. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3462568264"] = "Audioaufnahme konnte nicht beendet werden." --- The configured transcription provider does not meet the minimum confidence level. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3834149033"] = "Der konfigurierte Anbieter für die Transkription erfüllt nicht das erforderliche Mindestmaß an Vertrauenswürdigkeit." - -- An error occurred during transcription. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T588743762"] = "Während der Transkription ist ein Fehler aufgetreten." --- No transcription provider is configured. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T663630295"] = "Es ist kein Anbieter für die Transkription konfiguriert." - -- The transcription result is empty. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T974954792"] = "Das Ergebnis der Transkription ist leer." @@ -6750,9 +6813,6 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2037899437"] = "Kopiert die Serv -- This library is used to create temporary folders in runtime tests and supporting filesystem operations. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2160280545"] = "Diese Bibliothek wird verwendet, um temporäre Ordner bei Laufzeittests zu erstellen und Dateisystemoperationen zu unterstützen." --- This library is used to determine the file type of a file. This is necessary, e.g., when we want to stream a file. -UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2173617769"] = "Diese Bibliothek wird verwendet, um den Dateityp einer Datei zu bestimmen. Das ist zum Beispiel notwendig, wenn wir eine Datei streamen möchten." - -- For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2174764529"] = "Für die sichere Kommunikation zwischen der Benutzeroberfläche und der Laufzeit müssen wir Zertifikate erstellen. Diese Rust-Bibliothek eignet sich hervorragend dafür." @@ -6771,6 +6831,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2301484629"] = "Konfigurations-P -- The C# language is used for the implementation of the user interface and the backend. To implement the user interface with C#, the Blazor technology from ASP.NET Core is used. All these technologies are integrated into the .NET SDK. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2329884315"] = "Die Programmiersprache C# wird für die Umsetzung der Benutzeroberfläche und des Backends verwendet. Für die Entwicklung der Benutzeroberfläche mit C# kommt die Blazor-Technologie aus ASP.NET Core zum Einsatz. Alle diese Technologien sind im .NET SDK integriert." +-- We use Rubato to resample the decoded audio to 48 kHz before the Opus encoding. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2345444286"] = "Wir verwenden Rubato, um das dekodierte Audiosignal vor der Opus-Kodierung auf 48 kHz neu abzutasten." + -- Linux AppImages bundle GStreamer components to support microphone access and WebM audio recording in the embedded WebKitGTK web view. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T234598990"] = "Linux-AppImages bündeln GStreamer-Komponenten, um den Mikrofonzugriff und WebM-Audioaufnahmen in der eingebetteten WebKitGTK-Webansicht zu unterstützen." @@ -6852,6 +6915,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2929232062"] = "Kopiert die Quel -- Copies the root certificate fingerprint to the clipboard UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2989678330"] = "Kopiert den Fingerabdruck des Stammzertifikats in die Zwischenablage" +-- This library identifies files by their content. It is used for document streaming and as the first safety and media classification step before local audio processing. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3002755581"] = "Diese Bibliothek identifiziert Dateien anhand ihres Inhalts. Sie wird für das Streaming von Dokumenten sowie als erste Sicherheits- und Medienklassifizierungsstufe vor der lokalen Audioverarbeitung verwendet." + -- Changelog UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3017574265"] = "Änderungsprotokoll" @@ -6897,6 +6963,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3315279770"] = "Externe HTTPS-St -- User-language provided by the OS UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3334355246"] = "Vom Betriebssystem bereitgestellte Sprache" +-- webm-iterable provides the EBML and WebM writing path for normalized audio. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3385332793"] = "webm-iterable stellt den EBML- und WebM-Schreibpfad für normalisiertes Audio bereit." + -- Status: UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3396815215"] = "Status:" @@ -6957,6 +7026,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T396609403"] = "nicht zutreffend" -- Copies the allowed host configuration to the clipboard UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3970230163"] = "Kopiert die zulässige Host-Konfiguration in die Zwischenablage" +-- Symphonia is used for media container demuxing and audio decoding. The exact MPL-covered source is available from the repository linked and is identified in the offline notices bundled with AI Studio. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3971563979"] = "Symphonia wird zum Demultiplexen von Mediencontainern und zur Audiodekodierung verwendet. Der genaue, unter der MPL lizenzierte Quellcode ist im verlinkten Repository verfügbar und in den mit AI Studio gebündelten Offline-Hinweisen angegeben." + -- Installed Pandoc version UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3983971016"] = "Installierte Pandoc-Version" @@ -6975,6 +7047,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4060906280"] = "Diese Bibliothek -- This library is used to create asynchronous streams in Rust. It allows us to work with streams of data that can be produced asynchronously, making it easier to handle events or data that arrive over time. We use this, e.g., to stream arbitrary data from the file system to the embedding system. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4079152443"] = "Diese Bibliothek wird verwendet, um asynchrone Datenströme in Rust zu erstellen. Sie ermöglicht es uns, mit Datenströmen zu arbeiten, die asynchron bereitgestellt werden, wodurch sich Ereignisse oder Daten, die nach und nach eintreffen, leichter verarbeiten lassen. Wir nutzen dies zum Beispiel, um beliebige Daten aus dem Dateisystem an das Einbettungssystem zu übertragen." +-- Ropus provides the Opus encoder and decoder used by the media pipeline. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4113556626"] = "Ropus stellt den Opus-Encoder und -Decoder bereit, die von der Medienpipeline verwendet werden." + -- Community & Code UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4158546761"] = "Community & Code" @@ -7045,7 +7120,7 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T864851737"] = "Axum wird verwend UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T870640199"] = "Für einige Datenübertragungen müssen wir die Daten in Base64 kodieren. Diese Rust-Bibliothek eignet sich dafür hervorragend." -- How to update -UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T906183311"] = "Update-Anleitung " +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T906183311"] = "Update-Anleitung" -- Install Pandoc UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T986578435"] = "Pandoc installieren" @@ -8487,6 +8562,42 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source Code -- Document UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Dokument" +-- The configured transcription provider could not be created. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1235984176"] = "Der konfigurierte Transkriptionsanbieter konnte nicht erstellt werden." + +-- The selected media file no longer exists. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T129859547"] = "Die ausgewählte Mediendatei ist nicht mehr vorhanden." + +-- The selected media file does not contain an audio track. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T134825479"] = "Die ausgewählte Mediendatei enthält keine Audiospur." + +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1543974632"] = "Die Mediendatei konnte nicht transkribiert werden." + +-- The selected file cannot be processed as media. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1707342767"] = "Die ausgewählte Datei kann nicht als Medium verarbeitet werden." + +-- The audio track contains no audible signal, so there is nothing to transcribe. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1988190152"] = "Die Audiospur enthält kein hörbares Signal. Daher gibt es nichts zu transkribieren." + +-- The media file is damaged or its format could not be identified. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2004316549"] = "Die Mediendatei ist beschädigt oder ihr Format konnte nicht erkannt werden." + +-- This media format or audio codec is not supported. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2142564510"] = "Dieses Medienformat oder dieser Audiocodec wird nicht unterstützt." + +-- No usable transcription provider is configured. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2282521655"] = "Es ist kein nutzbarer Transkriptionsanbieter konfiguriert." + +-- The media file could not be prepared for transcription. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2749117459"] = "Die Mediendatei konnte nicht für die Transkription vorbereitet werden." + +-- The transcription provider could not transcribe the media file. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T3091669215"] = "Der Transkriptionsanbieter konnte die Mediendatei nicht transkribieren." + +-- The media pipeline ended without an output file. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T632852430"] = "Die Medienpipeline wurde beendet, ohne eine Ausgabedatei zu erzeugen." + -- Pandoc Installation UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::PANDOCAVAILABILITYSERVICE::T185447014"] = "Pandoc-Installation" diff --git a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua index 90677d36..2162562a 100644 --- a/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua +++ b/app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua @@ -306,6 +306,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::AGENDA::NUMBERPARTICIPANTSEXTENSIONS::T81 -- Stop generation UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1317408357"] = "Stop generation" +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1543974632"] = "The media file could not be transcribed." + -- Reset UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T180921696"] = "Reset" @@ -315,6 +318,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1809312323"] = "Please se -- The assistant failed. The message is: '{0}' UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T1833836281"] = "The assistant failed. The message is: '{0}'" +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T241403726"] = "The media transcription was canceled." + -- This assistant is already running. AI Studio opens the running session instead. UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T2575715765"] = "This assistant is already running. AI Studio opens the running session instead." @@ -2295,9 +2301,24 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T349928509"] = "The ima -- Open Settings UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T1172211894"] = "Open Settings" +-- Media transcription was canceled. Open the assistant to review it. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T1233815302"] = "Media transcription was canceled. Open the assistant to review it." + +-- Media transcription failed. Open the assistant to review it. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2177964639"] = "Media transcription failed. Open the assistant to review it." + +-- Media transcription completed with a warning. Open the assistant to review it. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2217674098"] = "Media transcription completed with a warning. Open the assistant to review it." + +-- Media is still being prepared. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2600900617"] = "Media is still being prepared." + -- Assistant is still running. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2719896610"] = "Assistant is still running." +-- The media transcript is ready. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T3248321953"] = "The media transcript is ready." + -- Assistant was canceled. Open it to review the result. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T3390934621"] = "Assistant was canceled. Open it to review the result." @@ -2400,18 +2421,33 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T1358313858"] = "Click t -- Drop files here to attach them. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T143112277"] = "Drop files here to attach them." +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T1543974632"] = "The media file could not be transcribed." + -- Click here to attach files. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T1875575968"] = "Click here to attach files." +-- Transcribe media files +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T2178031033"] = "Transcribe media files" + -- Drag and drop files into the marked area or click here to attach documents: UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T230755331"] = "Drag and drop files into the marked area or click here to attach documents:" +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T241403726"] = "The media transcription was canceled." + -- Select files to attach UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T2495931372"] = "Select files to attach" -- Document Preview UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T285154968"] = "Document Preview" +-- Media files require a configured transcription provider. Configure one in the transcription settings. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T3172443094"] = "Media files require a configured transcription provider. Configure one in the transcription settings." + +-- The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T322693339"] = "The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider." + -- Clear file list UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ATTACHDOCUMENTS::T3759696136"] = "Clear file list" @@ -2436,6 +2472,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1317408357"] = "Stop gene -- Save chat UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1516264254"] = "Save chat" +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1543974632"] = "The media file could not be transcribed." + -- Type your input here... UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T1849313532"] = "Type your input here..." @@ -2448,6 +2487,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2036185364"] = "Code" -- Italic UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2377171085"] = "Italic" +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T241403726"] = "The media transcription was canceled." + -- Profile usage is disabled according to your chat template settings. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CHATCOMPONENT::T2670286472"] = "Profile usage is disabled according to your chat template settings." @@ -2673,6 +2715,24 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANDATORYINFODISPLAY::T3511160492"] = "Ac -- Please review this text again. The content was changed. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MANDATORYINFODISPLAY::T941885055"] = "Please review this text again. The content was changed." +-- Waiting to prepare media +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T1167267986"] = "Waiting to prepare media" + +-- Stop media transcription +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T1868377405"] = "Stop media transcription" + +-- Stopping media transcription +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T1878101489"] = "Stopping media transcription" + +-- Inspecting media +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T2431421733"] = "Inspecting media" + +-- Transcribing +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T2938661425"] = "Transcribing" + +-- Preparing audio +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MEDIATRANSCRIPTIONSTATUS::T3200155905"] = "Preparing audio" + -- Given that my employer's workplace uses both Windows and Linux, I wanted a cross-platform solution that would work seamlessly across all major operating systems, including macOS. Additionally, I wanted to demonstrate that it is possible to create modern, efficient, cross-platform applications without resorting to Electron bloatware. The combination of .NET and Rust with Tauri proved to be an excellent technology stack for building such robust applications. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::MOTIVATION::T1057189794"] = "Given that my employer's workplace uses both Windows and Linux, I wanted a cross-platform solution that would work seamlessly across all major operating systems, including macOS. Additionally, I wanted to demonstrate that it is possible to create modern, efficient, cross-platform applications without resorting to Electron bloatware. The combination of .NET and Rust with Tauri proved to be an excellent technology stack for building such robust applications." @@ -2790,18 +2850,33 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::PROVIDERSELECTION::T4279448758"] = "Uses -- Provider UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::PROVIDERSELECTION::T900237532"] = "Provider" +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T1543974632"] = "The media file could not be transcribed." + -- Failed to load file content UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T1989554334"] = "Failed to load file content" -- Drop one file here to load its content. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T2274562398"] = "Drop one file here to load its content." +-- The media transcription was canceled. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T241403726"] = "The media transcription was canceled." + +-- The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T2839709466"] = "The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider." + +-- Media files require a configured transcription provider. Configure one in the transcription settings. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3172443094"] = "Media files require a configured transcription provider. Configure one in the transcription settings." + -- Use file content as input UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3499386973"] = "Use file content as input" -- Select file to read its content UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T354817589"] = "Select file to read its content" +-- Transcribe media file +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READFILECONTENT::T3554222960"] = "Transcribe media file" + -- The content is cleaned using an LLM agent: the main content is extracted, advertisements and other irrelevant things are attempted to be removed; relative links are attempted to be converted into absolute links so that they can be used. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::READWEBCONTENT::T1164201762"] = "The content is cleaned using an LLM agent: the main content is extracted, advertisements and other irrelevant things are attempted to be removed; relative links are attempted to be converted into absolute links so that they can be used." @@ -3540,9 +3615,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VISION::T586430036"] = "Useful assistants -- Voice recording has been disabled for this session because audio playback could not be initialized on the client. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T1123032432"] = "Voice recording has been disabled for this session because audio playback could not be initialized on the client." --- Failed to create the transcription provider. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T1689988905"] = "Failed to create the transcription provider." - -- Failed to start audio recording. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T2144994226"] = "Failed to start audio recording." @@ -3561,21 +3633,12 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T2851219233"] = "Transcrip -- Unfortunately, there was an error communicating with the AI system. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3236134591"] = "Unfortunately, there was an error communicating with the AI system." --- The configured transcription provider was not found. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T331613105"] = "The configured transcription provider was not found." - -- Failed to stop audio recording. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3462568264"] = "Failed to stop audio recording." --- The configured transcription provider does not meet the minimum confidence level. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T3834149033"] = "The configured transcription provider does not meet the minimum confidence level." - -- An error occurred during transcription. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T588743762"] = "An error occurred during transcription." --- No transcription provider is configured. -UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T663630295"] = "No transcription provider is configured." - -- The transcription result is empty. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::VOICERECORDER::T974954792"] = "The transcription result is empty." @@ -6750,9 +6813,6 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2037899437"] = "Copies the serve -- This library is used to create temporary folders in runtime tests and supporting filesystem operations. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2160280545"] = "This library is used to create temporary folders in runtime tests and supporting filesystem operations." --- This library is used to determine the file type of a file. This is necessary, e.g., when we want to stream a file. -UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2173617769"] = "This library is used to determine the file type of a file. This is necessary, e.g., when we want to stream a file." - -- For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2174764529"] = "For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose." @@ -6771,6 +6831,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2301484629"] = "Configuration pl -- The C# language is used for the implementation of the user interface and the backend. To implement the user interface with C#, the Blazor technology from ASP.NET Core is used. All these technologies are integrated into the .NET SDK. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2329884315"] = "The C# language is used for the implementation of the user interface and the backend. To implement the user interface with C#, the Blazor technology from ASP.NET Core is used. All these technologies are integrated into the .NET SDK." +-- We use Rubato to resample the decoded audio to 48 kHz before the Opus encoding. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2345444286"] = "We use Rubato to resample the decoded audio to 48 kHz before the Opus encoding." + -- Linux AppImages bundle GStreamer components to support microphone access and WebM audio recording in the embedded WebKitGTK web view. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T234598990"] = "Linux AppImages bundle GStreamer components to support microphone access and WebM audio recording in the embedded WebKitGTK web view." @@ -6852,6 +6915,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2929232062"] = "Copies the confi -- Copies the root certificate fingerprint to the clipboard UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2989678330"] = "Copies the root certificate fingerprint to the clipboard" +-- This library identifies files by their content. It is used for document streaming and as the first safety and media classification step before local audio processing. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3002755581"] = "This library identifies files by their content. It is used for document streaming and as the first safety and media classification step before local audio processing." + -- Changelog UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3017574265"] = "Changelog" @@ -6897,6 +6963,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3315279770"] = "External HTTPS c -- User-language provided by the OS UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3334355246"] = "User-language provided by the OS" +-- webm-iterable provides the EBML and WebM writing path for normalized audio. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3385332793"] = "webm-iterable provides the EBML and WebM writing path for normalized audio." + -- Status: UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3396815215"] = "Status:" @@ -6957,6 +7026,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T396609403"] = "not applicable" -- Copies the allowed host configuration to the clipboard UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3970230163"] = "Copies the allowed host configuration to the clipboard" +-- Symphonia is used for media container demuxing and audio decoding. The exact MPL-covered source is available from the repository linked and is identified in the offline notices bundled with AI Studio. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3971563979"] = "Symphonia is used for media container demuxing and audio decoding. The exact MPL-covered source is available from the repository linked and is identified in the offline notices bundled with AI Studio." + -- Installed Pandoc version UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T3983971016"] = "Installed Pandoc version" @@ -6975,6 +7047,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4060906280"] = "This library is -- This library is used to create asynchronous streams in Rust. It allows us to work with streams of data that can be produced asynchronously, making it easier to handle events or data that arrive over time. We use this, e.g., to stream arbitrary data from the file system to the embedding system. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4079152443"] = "This library is used to create asynchronous streams in Rust. It allows us to work with streams of data that can be produced asynchronously, making it easier to handle events or data that arrive over time. We use this, e.g., to stream arbitrary data from the file system to the embedding system." +-- Ropus provides the Opus encoder and decoder used by the media pipeline. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4113556626"] = "Ropus provides the Opus encoder and decoder used by the media pipeline." + -- Community & Code UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T4158546761"] = "Community & Code" @@ -8487,6 +8562,42 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source like p -- Document UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document" +-- The configured transcription provider could not be created. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1235984176"] = "The configured transcription provider could not be created." + +-- The selected media file no longer exists. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T129859547"] = "The selected media file no longer exists." + +-- The selected media file does not contain an audio track. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T134825479"] = "The selected media file does not contain an audio track." + +-- The media file could not be transcribed. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1543974632"] = "The media file could not be transcribed." + +-- The selected file cannot be processed as media. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1707342767"] = "The selected file cannot be processed as media." + +-- The audio track contains no audible signal, so there is nothing to transcribe. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1988190152"] = "The audio track contains no audible signal, so there is nothing to transcribe." + +-- The media file is damaged or its format could not be identified. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2004316549"] = "The media file is damaged or its format could not be identified." + +-- This media format or audio codec is not supported. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2142564510"] = "This media format or audio codec is not supported." + +-- No usable transcription provider is configured. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2282521655"] = "No usable transcription provider is configured." + +-- The media file could not be prepared for transcription. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2749117459"] = "The media file could not be prepared for transcription." + +-- The transcription provider could not transcribe the media file. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T3091669215"] = "The transcription provider could not transcribe the media file." + +-- The media pipeline ended without an output file. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T632852430"] = "The media pipeline ended without an output file." + -- Pandoc Installation UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::PANDOCAVAILABILITYSERVICE::T185447014"] = "Pandoc Installation" diff --git a/app/MindWork AI Studio/Program.cs b/app/MindWork AI Studio/Program.cs index d5cdaf5c..c50ebeeb 100644 --- a/app/MindWork AI Studio/Program.cs +++ b/app/MindWork AI Studio/Program.cs @@ -136,6 +136,7 @@ internal sealed class Program builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); + builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); builder.Services.AddSingleton(); @@ -148,6 +149,7 @@ internal sealed class Program builder.Services.AddTransient(); builder.Services.AddHostedService(); builder.Services.AddHostedService(); + builder.Services.AddHostedService(); builder.Services.AddHostedService(); builder.Services.AddSingleton(); builder.Services.AddHostedService(); diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index e679f795..4ad26580 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -1069,7 +1069,11 @@ public abstract class BaseProvider : IProvider, ISecretId request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(Program.ENCRYPTION)); break; } - + + this.logger.LogInformation("Uploading transcription media '{FileName}' with content type '{ContentType}' and {FileSize} bytes.", + Path.GetFileName(audioFilePath), + mimeType.TextRepresentation, + fileStream.Length); using var response = await this.HttpClient.SendAsync(request, token); var responseBody = await response.Content.ReadAsStringAsync(token); @@ -1089,6 +1093,10 @@ public abstract class BaseProvider : IProvider, ISecretId return TranscriptionResult.FromText(transcriptionResponse.Text); } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + throw; + } catch (Exception e) { if (this.IsTimeoutException(e, token)) diff --git a/app/MindWork AI Studio/Tools/AudioRecordingResult.cs b/app/MindWork AI Studio/Tools/AudioRecordingResult.cs deleted file mode 100644 index cdde82ac..00000000 --- a/app/MindWork AI Studio/Tools/AudioRecordingResult.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace AIStudio.Tools; - -public sealed class AudioRecordingResult -{ - public string MimeType { get; init; } = string.Empty; - - public bool ChangedMimeType { get; init; } -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Markdown.cs b/app/MindWork AI Studio/Tools/Markdown.cs index e1f87d9c..c523795b 100644 --- a/app/MindWork AI Studio/Tools/Markdown.cs +++ b/app/MindWork AI Studio/Tools/Markdown.cs @@ -34,6 +34,30 @@ public static class Markdown } }; + /// Escapes arbitrary text for literal display inside Markdown. + public static string EscapeInlineText(string value) + { + if (string.IsNullOrEmpty(value)) + return string.Empty; + + var escaped = new StringBuilder(value.Length); + foreach (var character in value) + { + if (character is '\r' or '\n' or '\t' || char.IsControl(character)) + { + escaped.Append(' '); + continue; + } + + if (character is >= '!' and <= '/' or >= ':' and <= '@' or >= '[' and <= '`' or >= '{' and <= '~') + escaped.Append('\\'); + + escaped.Append(character); + } + + return escaped.ToString(); + } + public static string RemoveSharedIndentation(string value) { if (string.IsNullOrWhiteSpace(value)) diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportDelivery.cs b/app/MindWork AI Studio/Tools/Media/MediaImportDelivery.cs new file mode 100644 index 00000000..d2987776 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaImportDelivery.cs @@ -0,0 +1,15 @@ +using AIStudio.Chat; + +namespace AIStudio.Tools.Media; + +/// Pending media results waiting for one concrete UI target. +public sealed record MediaImportDelivery +{ + public required MediaImportTarget Target { get; init; } + + public IReadOnlyList Attachments { get; init; } = []; + + public string? Text { get; init; } + + public bool IsEmpty => this.Attachments.Count is 0 && this.Text is null; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportFailure.cs b/app/MindWork AI Studio/Tools/Media/MediaImportFailure.cs new file mode 100644 index 00000000..99a84a04 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaImportFailure.cs @@ -0,0 +1,6 @@ +using AIStudio.Tools.Rust; + +namespace AIStudio.Tools.Media; + +/// One user-visible failure retained until its owner is displayed. +public sealed record MediaImportFailure(string FileName, string UserMessage, MediaJobErrorCode? ErrorCode = null); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportOutcome.cs b/app/MindWork AI Studio/Tools/Media/MediaImportOutcome.cs new file mode 100644 index 00000000..33eeb159 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaImportOutcome.cs @@ -0,0 +1,13 @@ +namespace AIStudio.Tools.Media; + +/// Terminal batch outcome retained until its owner is displayed. +public sealed record MediaImportOutcome +{ + public required MediaImportOwner Owner { get; init; } + + public required MediaImportStatus Status { get; init; } + + public IReadOnlyList Failures { get; init; } = []; + + public IReadOnlyList Warnings { get; init; } = []; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportOwner.cs b/app/MindWork AI Studio/Tools/Media/MediaImportOwner.cs new file mode 100644 index 00000000..09cb2cdd --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaImportOwner.cs @@ -0,0 +1,11 @@ +using AIStudio.Tools.AssistantSessions; + +namespace AIStudio.Tools.Media; + +/// Identifies the chat or assistant that owns a media import. +public readonly record struct MediaImportOwner(MediaImportOwnerKind Kind, string Id) +{ + public static MediaImportOwner ForChat(Guid chatId) => new(MediaImportOwnerKind.CHAT, chatId.ToString("N")); + + public static MediaImportOwner ForAssistant(AssistantSessionKey key) => new(MediaImportOwnerKind.ASSISTANT, key.ToString()); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportOwnerKind.cs b/app/MindWork AI Studio/Tools/Media/MediaImportOwnerKind.cs new file mode 100644 index 00000000..e5a58a97 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaImportOwnerKind.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Tools.Media; + +/// Supported persistent media-operation owners. +public enum MediaImportOwnerKind +{ + CHAT, + ASSISTANT, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportSnapshot.cs b/app/MindWork AI Studio/Tools/Media/MediaImportSnapshot.cs new file mode 100644 index 00000000..92957d91 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaImportSnapshot.cs @@ -0,0 +1,19 @@ +namespace AIStudio.Tools.Media; + +/// Copied owner-specific state suitable for rendering after navigation. +public sealed record MediaImportSnapshot +{ + public required MediaImportOwner Owner { get; init; } + + public required MediaImportTarget Target { get; init; } + + public required MediaTranscriptionPhase Phase { get; init; } + + public required MediaImportStatus Status { get; init; } + + public string CurrentFileName { get; init; } = string.Empty; + + public double? Progress { get; init; } + + public bool IsBusy => this.Status is MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportStatus.cs b/app/MindWork AI Studio/Tools/Media/MediaImportStatus.cs new file mode 100644 index 00000000..7207a58d --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaImportStatus.cs @@ -0,0 +1,13 @@ +namespace AIStudio.Tools.Media; + +/// Lifecycle status retained independently for each owner. +public enum MediaImportStatus +{ + QUEUED, + RUNNING, + CANCELING, + SUCCEEDED, + WARNING, + FAILED, + CANCELLED, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportTarget.cs b/app/MindWork AI Studio/Tools/Media/MediaImportTarget.cs new file mode 100644 index 00000000..9ea4e68c --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaImportTarget.cs @@ -0,0 +1,4 @@ +namespace AIStudio.Tools.Media; + +/// Identifies the concrete attachment or file-content field inside an owner. +public readonly record struct MediaImportTarget(MediaImportOwner Owner, string TargetId); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportWarning.cs b/app/MindWork AI Studio/Tools/Media/MediaImportWarning.cs new file mode 100644 index 00000000..d0ac9c0f --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaImportWarning.cs @@ -0,0 +1,4 @@ +namespace AIStudio.Tools.Media; + +/// One user-visible media warning retained until its owner is displayed. +public sealed record MediaImportWarning(string FileName, string UserMessage); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaTranscriptionPhase.cs b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionPhase.cs new file mode 100644 index 00000000..290282d2 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionPhase.cs @@ -0,0 +1,23 @@ +namespace AIStudio.Tools.Media; + +/// Visible phases of the serialized media import lane. +public enum MediaTranscriptionPhase +{ + /// No import is active. + IDLE, + + /// The operation is waiting for the serialized runtime lane. + QUEUED, + + /// The runtime is inspecting the input. + PROBING, + + /// The runtime is preparing normalized audio. + TRANSCODING, + + /// The normalized audio is being transcribed by the provider. + UPLOADING, + + /// Cancellation was requested and runtime cleanup is in progress. + CANCELING, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs new file mode 100644 index 00000000..b486480f --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs @@ -0,0 +1,32 @@ +using AIStudio.Tools.Rust; + +namespace AIStudio.Tools.Media; + +/// +/// Typed terminal result returned by media import and voice operations. +/// +/// Terminal operation status. +/// Transcript text for a successful operation. +/// Localized message suitable for display after a warning or failure. +/// Optional stable runtime failure category. +public sealed record MediaTranscriptionResult(MediaTranscriptionResultStatus Status, string Text, string UserMessage, MediaJobErrorCode? ErrorCode = null) +{ + /// Creates a successful result. + /// Provider transcript. + public static MediaTranscriptionResult Succeeded(string text) => new(MediaTranscriptionResultStatus.SUCCEEDED, text, string.Empty); + + /// Creates a failed result. + /// Localized visible message. + /// Optional runtime error category. + public static MediaTranscriptionResult Failed(string userMessage, MediaJobErrorCode? errorCode = null) => new(MediaTranscriptionResultStatus.FAILED, string.Empty, userMessage, errorCode); + + /// Creates a warning result for media without an audible signal. + /// Localized visible warning. + public static MediaTranscriptionResult NoAudibleSignal(string userMessage) => new( + MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL, + string.Empty, + userMessage); + + /// Creates a cancelled result without relying on visible text. + public static MediaTranscriptionResult Cancelled() => new(MediaTranscriptionResultStatus.CANCELLED, string.Empty, string.Empty, MediaJobErrorCode.CANCELLED); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResultStatus.cs b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResultStatus.cs new file mode 100644 index 00000000..02ff300b --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResultStatus.cs @@ -0,0 +1,19 @@ +namespace AIStudio.Tools.Media; + +/// +/// Terminal outcome of a media transcription operation. +/// +public enum MediaTranscriptionResultStatus +{ + /// The provider returned a usable transcript. + SUCCEEDED, + + /// The operation failed. + FAILED, + + /// The media contains no signal above the practical-silence threshold. + NO_AUDIBLE_SIGNAL, + + /// The caller or user cancelled the operation. + CANCELLED, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/CreateMediaJobRequest.cs b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobRequest.cs new file mode 100644 index 00000000..1c89e491 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobRequest.cs @@ -0,0 +1,7 @@ +namespace AIStudio.Tools.Rust; + +/// Request body used to start a Rust media normalization job. +/// Absolute source media path. +/// Absolute operation-owned output path. +/// Optional pass-through size ceiling. +public sealed record CreateMediaJobRequest(string InputPath, string OutputPath, ulong? MaxPassThroughBytes = null); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/CreateMediaJobResponse.cs b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobResponse.cs new file mode 100644 index 00000000..2e47855b --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/CreateMediaJobResponse.cs @@ -0,0 +1,5 @@ +namespace AIStudio.Tools.Rust; + +/// Response returned after a Rust media job is registered. +/// Opaque runtime job identifier. +public sealed record CreateMediaJobResponse(string JobId); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/FileTypes.cs b/app/MindWork AI Studio/Tools/Rust/FileTypes.cs index 1e388109..5f29bc11 100644 --- a/app/MindWork AI Studio/Tools/Rust/FileTypes.cs +++ b/app/MindWork AI Studio/Tools/Rust/FileTypes.cs @@ -61,7 +61,7 @@ public static class FileTypes public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"), "jpg", "jpeg", "png", "gif", "bmp", "tiff", "svg", "webp", "heic"); public static readonly FileTypeFilter AUDIO = FileTypeFilter.Leaf(TB("Audio"), - "mp3", "wav", "wave", "aac", "flac", "ogg", "m4a", "wma", "alac", "aiff", "m4b"); + "mp3", "wav", "wave", "aac", "flac", "ogg", "opus", "m4a", "m4b", "wma", "alac", "aif", "aiff", "caf"); public static readonly FileTypeFilter VIDEO = FileTypeFilter.Leaf(TB("Video"), "mp4", "m4v", "avi", "mkv", "mov", "wmv", "flv", "webm"); diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobError.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobError.cs new file mode 100644 index 00000000..be0dd2f5 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobError.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Tools.Rust; + +/// +/// Runtime media error containing a stable code and an English log diagnostic. +/// +/// Stable machine-readable error category. +/// US-English diagnostic intended for logs. +public sealed record MediaJobError(MediaJobErrorCode Code, string Message); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobErrorCode.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobErrorCode.cs new file mode 100644 index 00000000..68f29b39 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobErrorCode.cs @@ -0,0 +1,85 @@ +namespace AIStudio.Tools.Rust; + +/// +/// Stable failure categories returned by the Rust media pipeline. +/// +public enum MediaJobErrorCode +{ + /// The runtime returned an unrecognized code. + UNKNOWN, + + /// The input file does not exist. + FILE_NOT_FOUND, + + /// The file type could not be identified. + UNKNOWN_FORMAT, + + /// Executable input was rejected. + UNSAFE_FILE, + + /// The input is not media. + NOT_MEDIA, + + /// The input file could not be opened. + FILE_OPEN_FAILED, + + /// The container is unsupported. + UNSUPPORTED_CONTAINER, + + /// The media has no audio track. + NO_AUDIO_TRACK, + + /// No audio track has a supported decoder. + UNSUPPORTED_CODEC, + + /// Decoded audio parameters are absent or inconsistent. + INVALID_AUDIO_PARAMETERS, + + /// The Opus identification header is invalid. + INVALID_OPUS_HEADER, + + /// The Opus mapping requires unsupported multistream decoding. + UNSUPPORTED_OPUS_MAPPING, + + /// The decoder could not be initialized. + DECODER_INIT_FAILED, + + /// The encoder could not be initialized. + ENCODER_INIT_FAILED, + + /// The stream changed unexpectedly. + STREAM_RESET, + + /// The container is damaged. + DAMAGED_CONTAINER, + + /// Audio decoding failed. + DECODE_FAILED, + + /// Audio resampling failed. + RESAMPLE_FAILED, + + /// Opus encoding failed. + ENCODE_FAILED, + + /// The output directory or file could not be created. + OUTPUT_CREATE_FAILED, + + /// The output could not be written. + OUTPUT_WRITE_FAILED, + + /// The partial output could not be committed. + OUTPUT_COMMIT_FAILED, + + /// A WebM relative timestamp overflowed. + WEBM_TIMESTAMP_OVERFLOW, + + /// WebM serialization failed. + WEBM_WRITE_FAILED, + + /// The job was cancelled. + CANCELLED, + + /// The runtime worker failed unexpectedly. + INTERNAL_ERROR, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobEvent.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobEvent.cs new file mode 100644 index 00000000..81b1400b --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobEvent.cs @@ -0,0 +1,12 @@ +namespace AIStudio.Tools.Rust; + +/// Snapshot emitted by the Rust media job event stream. +/// Current job phase. +/// Optional progress fraction. +/// Completed result. +/// Failure diagnostic. +public sealed record MediaJobEvent( + MediaJobPhase Phase, + double? Progress, + MediaJobResult? Result, + MediaJobError? Error); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobPhase.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobPhase.cs new file mode 100644 index 00000000..bf09d744 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobPhase.cs @@ -0,0 +1,23 @@ +namespace AIStudio.Tools.Rust; + +/// Lifecycle phases exposed by the Rust media API. +public enum MediaJobPhase +{ + /// An unknown future value received from Rust. + UNKNOWN, + + /// The runtime is identifying the input and selecting audio. + PROBING, + + /// The runtime is normalizing audio. + TRANSCODING, + + /// The output was committed successfully. + COMPLETED, + + /// The job failed. + FAILED, + + /// Cancellation and temporary-output cleanup completed. + CANCELLED, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs new file mode 100644 index 00000000..3dc9d551 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs @@ -0,0 +1,20 @@ +namespace AIStudio.Tools.Rust; + +/// Successful terminal result returned by Rust media normalization. +/// Committed normalized output path. +/// Stable normalized container used for provider uploads. +/// Stable normalized audio codec used for provider uploads. +/// Detected container diagnostic. +/// Selected codec diagnostic. +/// Normalized duration in milliseconds. +/// Whether the source was copied unchanged. +/// Whether the normalized audio exceeds the practical-silence threshold. +public sealed record MediaJobResult( + string OutputPath, + string OutputFormat, + string OutputCodec, + string DetectedFormat, + string DetectedCodec, + ulong DurationMs, + bool PassThrough, + bool HasAudibleSignal); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs new file mode 100644 index 00000000..726bfbb9 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs @@ -0,0 +1,888 @@ +using AIStudio.Chat; +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Tools.Media; +using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.Rust; + +namespace AIStudio.Tools.Services; + +/// +/// Coordinates serialized visible media imports and independent voice transcriptions. +/// +public sealed class MediaTranscriptionService(RustService rustService, SettingsManager settingsManager, ILogger logger) : IDisposable +{ + private const string NORMALIZED_OUTPUT_EXTENSION = ".webm"; + private const string NORMALIZED_OUTPUT_FORMAT = "webm"; + private const string NORMALIZED_OUTPUT_CODEC = "opus"; + private static readonly byte[] WEBM_EBML_SIGNATURE = [0x1A, 0x45, 0xDF, 0xA3]; + + /// Serializes attachment and file-content imports. + private readonly SemaphoreSlim importQueue = new(1, 1); + + /// Protects operation ownership and owner-specific import state. + private readonly Lock stateLock = new(); + + /// All operations retained so disposal can cancel voice and import work. + private readonly HashSet operations = []; + + /// The active or queued import operation for each owner. + private readonly Dictionary currentImports = []; + + /// The latest active or unacknowledged terminal state for each owner. + private readonly Dictionary snapshots = []; + + /// Successful results waiting for their concrete UI target. + private readonly Dictionary pendingDeliveries = []; + + /// Terminal notifications waiting for their owner surface to be displayed. + private readonly Dictionary outcomes = []; + + /// Owners whose complete file batches are managed by this service. + private readonly HashSet activeBatches = []; + + /// Batch-level cancellation keeps Stop effective between two files. + private readonly Dictionary batchCancellations = []; + + /// Prevents new work after disposal. + private bool disposed; + + /// Raised only with the owner whose copied state changed. + public event Action? StateChanged; + + /// Gets whether one owner has queued, running, or canceling media work. + public bool IsBusy(MediaImportOwner owner) + { + lock (this.stateLock) + return this.activeBatches.Contains(owner); + } + + /// Gets the last retained state for one owner. + public MediaImportSnapshot? GetSnapshot(MediaImportOwner owner) + { + lock (this.stateLock) + return this.snapshots.GetValueOrDefault(owner); + } + + /// Gets copied retained snapshots for navigation indicators. + public IReadOnlyCollection GetSnapshots() + { + lock (this.stateLock) + return [.. this.snapshots.Values]; + } + + /// Gets copied results that have not yet been applied by one target. + public MediaImportDelivery? GetPendingDelivery(MediaImportTarget target) + { + lock (this.stateLock) + { + if (!this.pendingDeliveries.TryGetValue(target, out var pending)) + return null; + + return new() + { + Target = target, + Attachments = [.. pending.Attachments], + Text = pending.Text, + }; + } + } + + /// Removes exactly the results that one target applied successfully. + public void AcknowledgeDelivery(MediaImportDelivery delivery) + { + lock (this.stateLock) + { + if (!this.pendingDeliveries.TryGetValue(delivery.Target, out var pending)) + return; + + var acknowledgedPaths = delivery.Attachments.Select(attachment => attachment.FilePath).ToHashSet(StringComparer.Ordinal); + pending.Attachments.RemoveAll(attachment => acknowledgedPaths.Contains(attachment.FilePath)); + + if (delivery.Text is not null && string.Equals(pending.Text, delivery.Text, StringComparison.Ordinal)) + pending.Text = null; + + if (pending.Attachments.Count is 0 && pending.Text is null) + this.pendingDeliveries.Remove(delivery.Target); + } + } + + /// Consumes one terminal notification when its owner surface is displayed. + public MediaImportOutcome? TryConsumeOutcome(MediaImportOwner owner) + { + MediaImportOutcome? outcome; + lock (this.stateLock) + { + if (!this.outcomes.Remove(owner, out outcome)) + return null; + + if (this.snapshots.GetValueOrDefault(owner) is { IsBusy: false }) + this.snapshots.Remove(owner); + } + + this.NotifyStateChanged(owner); + return outcome; + } + + /// Discards retained inactive state and deletes unclaimed managed transcript files. + public void ClearOwnerState(MediaImportOwner owner) + { + List discardedAttachments = []; + lock (this.stateLock) + { + if (this.activeBatches.Contains(owner)) + return; + + this.snapshots.Remove(owner); + this.outcomes.Remove(owner); + + foreach (var target in this.pendingDeliveries.Keys.Where(target => target.Owner == owner).ToList()) + { + discardedAttachments.AddRange(this.pendingDeliveries[target].Attachments); + this.pendingDeliveries.Remove(target); + } + } + + foreach (var attachment in discardedAttachments) + ManagedTranscriptAttachment.TryDeleteOwnedFile(attachment); + + this.NotifyStateChanged(owner); + } + + /// Starts an owner-managed attachment batch and returns without holding the UI event handler. + public bool TryStartAttachmentBatch(IReadOnlyList mediaPaths, MediaImportTarget target, ChatThread? ownerChat = null) + { + this.ThrowIfDisposed(); + if (mediaPaths.Count is 0) + return false; + + lock (this.stateLock) + { + if (!this.activeBatches.Add(target.Owner)) + return false; + + this.batchCancellations[target.Owner] = new(); + this.outcomes.Remove(target.Owner); + } + + this.UpdateImportState(target, Path.GetFileName(mediaPaths[0]), MediaTranscriptionPhase.QUEUED, null, MediaImportStatus.QUEUED); + _ = Task.Run(() => this.RunAttachmentBatchAsync(mediaPaths, target, ownerChat)); + return true; + } + + /// Starts a reattachable file-content import for one stable assistant field. + public bool TryStartTextImport(string mediaPath, MediaImportTarget target) + { + this.ThrowIfDisposed(); + lock (this.stateLock) + { + if (!this.activeBatches.Add(target.Owner)) + return false; + + this.batchCancellations[target.Owner] = new(); + this.outcomes.Remove(target.Owner); + } + + this.UpdateImportState(target, Path.GetFileName(mediaPath), MediaTranscriptionPhase.QUEUED, null, MediaImportStatus.QUEUED); + _ = Task.Run(() => this.RunTextImportAsync(mediaPath, target)); + return true; + } + + /// Completes a field import independently of the originating Blazor component. + private async Task RunTextImportAsync(string mediaPath, MediaImportTarget target) + { + CancellationTokenSource cancellation; + lock (this.stateLock) + cancellation = this.batchCancellations[target.Owner]; + + var status = MediaImportStatus.SUCCEEDED; + List failures = []; + List warnings = []; + try + { + var result = await this.TranscribeImportAsync(mediaPath, target, cancellation.Token); + if (result.Status is MediaTranscriptionResultStatus.SUCCEEDED) + this.AddCompletedText(target, result.Text); + else if (result.Status is MediaTranscriptionResultStatus.CANCELLED) + status = MediaImportStatus.CANCELLED; + else if (result.Status is MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL) + { + status = MediaImportStatus.WARNING; + warnings.Add(new(Path.GetFileName(mediaPath), result.UserMessage)); + } + else + { + status = MediaImportStatus.FAILED; + failures.Add(new(Path.GetFileName(mediaPath), result.UserMessage, result.ErrorCode)); + } + } + catch (OperationCanceledException) + { + status = MediaImportStatus.CANCELLED; + } + catch (Exception exception) + { + logger.LogError(exception, "Owner media text import failed for '{Owner}' and target '{TargetId}'.", target.Owner, target.TargetId); + status = MediaImportStatus.FAILED; + failures.Add(new(Path.GetFileName(mediaPath), TB("The media file could not be transcribed."))); + } + finally + { + lock (this.stateLock) + { + this.activeBatches.Remove(target.Owner); + if (this.batchCancellations.Remove(target.Owner, out var ownedCancellation)) + ownedCancellation.Dispose(); + } + + this.CompleteImport(target, Path.GetFileName(mediaPath), status, failures, warnings); + } + } + + /// Stores a completed field transcript for reattachment after navigation. + private void AddCompletedText(MediaImportTarget target, string text) + { + lock (this.stateLock) + { + if (!this.pendingDeliveries.TryGetValue(target, out var pending)) + this.pendingDeliveries[target] = pending = new(); + + pending.Text = text; + } + + this.NotifyStateChanged(target.Owner); + } + + /// Serially transcribes a complete owner batch while retaining every successful result. + private async Task RunAttachmentBatchAsync(IReadOnlyList mediaPaths, MediaImportTarget target, ChatThread? ownerChat) + { + CancellationToken batchToken; + lock (this.stateLock) + batchToken = this.batchCancellations[target.Owner].Token; + + var status = MediaImportStatus.SUCCEEDED; + var currentFileName = Path.GetFileName(mediaPaths[0]); + List failures = []; + List warnings = []; + + try + { + foreach (var mediaPath in mediaPaths) + { + currentFileName = Path.GetFileName(mediaPath); + batchToken.ThrowIfCancellationRequested(); + var result = await this.TranscribeImportAsync(mediaPath, target, batchToken); + if (result.Status is MediaTranscriptionResultStatus.CANCELLED) + { + status = MediaImportStatus.CANCELLED; + break; + } + + if (result.Status is MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL) + { + if (status is MediaImportStatus.SUCCEEDED) + status = MediaImportStatus.WARNING; + + warnings.Add(new(currentFileName, result.UserMessage)); + continue; + } + + if (result.Status is not MediaTranscriptionResultStatus.SUCCEEDED) + { + status = MediaImportStatus.FAILED; + failures.Add(new(currentFileName, result.UserMessage, result.ErrorCode)); + continue; + } + + var isPersistedChat = ownerChat is not null && WorkspaceBehaviour.IsChatExisting(new LoadChat(ownerChat.WorkspaceId, ownerChat.ChatId)); + var attachment = isPersistedChat + ? await WorkspaceBehaviour.CreateManagedTranscriptAsync(ownerChat!, mediaPath, result.Text) + : await ManagedTranscriptAttachment.CreateStagedAsync(mediaPath, result.Text); + + if (ownerChat is not null && attachment is { } managed + && ownerChat.PendingMediaTranscripts.All(existing => existing.FilePath != managed.FilePath)) + ownerChat.PendingMediaTranscripts.Add(managed); + + if (isPersistedChat) + await WorkspaceBehaviour.StoreChatAsync(ownerChat!); + + this.AddCompletedAttachment(target, attachment); + } + } + catch (OperationCanceledException) + { + status = MediaImportStatus.CANCELLED; + } + catch (Exception exception) + { + logger.LogError(exception, "Owner media batch failed for '{Owner}'.", target.Owner); + status = MediaImportStatus.FAILED; + failures.Add(new(currentFileName, TB("The media file could not be transcribed."))); + } + finally + { + lock (this.stateLock) + { + this.activeBatches.Remove(target.Owner); + if (this.batchCancellations.Remove(target.Owner, out var cancellation)) + cancellation.Dispose(); + } + + this.CompleteImport(target, currentFileName, status, failures, warnings); + } + } + + /// Adds a successful partial result to the retained owner snapshot. + private void AddCompletedAttachment(MediaImportTarget target, FileAttachment attachment) + { + lock (this.stateLock) + { + if (!this.pendingDeliveries.TryGetValue(target, out var pending)) + this.pendingDeliveries[target] = pending = new(); + + if (pending.Attachments.All(existing => existing.FilePath != attachment.FilePath)) + pending.Attachments.Add(attachment); + } + + this.NotifyStateChanged(target.Owner); + } + + /// + /// Transcribes an attachment or file-content import on the serialized visible lane. + /// + /// Source media path. + /// Media import target. + /// Caller cancellation token. + /// A typed terminal result. + private async Task TranscribeImportAsync(string mediaPath, MediaImportTarget target, CancellationToken token = default) + { + this.ThrowIfDisposed(); + var operation = this.CreateOperation(target, token); + lock (this.stateLock) + { + if (!this.currentImports.TryAdd(target.Owner, operation)) + throw new InvalidOperationException($"Media owner '{target.Owner}' already has an active operation."); + } + + this.UpdateImportState(target, Path.GetFileName(mediaPath), MediaTranscriptionPhase.QUEUED, null, MediaImportStatus.QUEUED); + + try + { + await this.importQueue.WaitAsync(operation.Cancellation.Token); + operation.HasQueueLease = true; + + this.UpdateImportState(target, Path.GetFileName(mediaPath), MediaTranscriptionPhase.PROBING, 0.0, MediaImportStatus.RUNNING); + return await this.TranscribeCoreAsync(mediaPath, operation, updateImportState: true); + } + catch (OperationCanceledException) + { + return MediaTranscriptionResult.Cancelled(); + } + finally + { + lock (this.stateLock) + { + if (this.currentImports.GetValueOrDefault(target.Owner) == operation) + this.currentImports.Remove(target.Owner); + } + + this.ReleaseOperation(operation); + if (operation.HasQueueLease) + this.importQueue.Release(); + } + } + + /// + /// Transcribes a voice recording independently of the visible import lane. + /// + /// Voice recording path. + /// Caller cancellation token. + /// A typed terminal result. + public async Task TranscribeVoiceAsync(string mediaPath, CancellationToken token = default) + { + this.ThrowIfDisposed(); + var operation = this.CreateOperation(null, token); + + try + { + return await this.TranscribeCoreAsync(mediaPath, operation, updateImportState: false); + } + finally + { + this.ReleaseOperation(operation); + } + } + + /// Cancels only the queued or active operation belonging to one owner. + public async Task StopAsync(MediaImportOwner owner) + { + MediaOperation? operation; + MediaImportSnapshot? snapshot; + lock (this.stateLock) + { + operation = this.currentImports.GetValueOrDefault(owner); + this.batchCancellations.GetValueOrDefault(owner)?.Cancel(); + operation?.Cancellation.Cancel(); + snapshot = this.snapshots.GetValueOrDefault(owner); + } + + if (snapshot is not null && this.IsBusy(owner)) + this.UpdateImportState(snapshot.Target, snapshot.CurrentFileName, MediaTranscriptionPhase.CANCELING, null, MediaImportStatus.CANCELING); + + if (!string.IsNullOrWhiteSpace(operation?.JobId)) + await rustService.CancelMediaJobAsync(operation.JobId); + } + + /// Runs normalization, provider resolution, and upload for one owned operation. + /// Source media path. + /// Operation-specific cancellation and Rust-job state. + /// Whether progress belongs to the visible import lane. + /// A typed terminal result. + private async Task TranscribeCoreAsync(string mediaPath, MediaOperation operation, bool updateImportState) + { + var normalizedPath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-media", $"{operation.Id:N}.webm"); + Directory.CreateDirectory(Path.GetDirectoryName(normalizedPath)!); + + try + { + var normalized = await this.NormalizeAsync(mediaPath, normalizedPath, operation, updateImportState); + if (normalized.Result is null) + return normalized.Error is null + ? MediaTranscriptionResult.Failed(TB("The media pipeline ended without an output file.")) + : MediaTranscriptionResult.Failed(UserMessageFor(normalized.Error.Code), normalized.Error.Code); + + var uploadContractError = await ValidateNormalizedProviderUploadAsync(normalized.Result, normalizedPath, operation.Cancellation.Token); + if (uploadContractError is not null) + { + logger.LogError("Refusing the transcription provider upload because the normalized media contract validation failed: {Diagnostic}", uploadContractError); + return MediaTranscriptionResult.Failed(TB("The media pipeline ended without an output file.")); + } + + if (!normalized.Result.HasAudibleSignal) + { + logger.LogInformation("Skipping transcription for '{MediaPath}' because its maximum audio peak does not exceed the practical-silence threshold.", mediaPath); + return MediaTranscriptionResult.NoAudibleSignal(TB("The audio track contains no audible signal, so there is nothing to transcribe.")); + } + + var providerSettings = this.ResolveProvider(); + if (providerSettings is null) + return MediaTranscriptionResult.Failed(TB("No usable transcription provider is configured.")); + + if (updateImportState) + this.UpdateImportState(operation.Target!.Value, Path.GetFileName(mediaPath), MediaTranscriptionPhase.UPLOADING, null, MediaImportStatus.RUNNING); + + var provider = providerSettings.CreateProvider(); + if (provider.Provider is LLMProviders.NONE) + return MediaTranscriptionResult.Failed(TB("The configured transcription provider could not be created.")); + + var sourceSize = File.Exists(mediaPath) ? new FileInfo(mediaPath).Length : 0; + var normalizedSize = new FileInfo(normalizedPath).Length; + var reductionPercent = sourceSize > 0 + ? (1.0 - (double)normalizedSize / sourceSize) * 100.0 + : 0.0; + logger.LogInformation("Transcribing normalized WebM/Opus media '{NormalizedPath}' ({NormalizedSize} bytes; source '{SourcePath}' {SourceSize} bytes; size reduction {ReductionPercent:F1}%) with provider '{Provider}' and model '{Model}'.", + normalizedPath, + normalizedSize, + mediaPath, + sourceSize, + reductionPercent, + providerSettings.UsedLLMProvider, + providerSettings.Model); + + var providerResult = await provider.TranscribeAudioAsync(providerSettings.Model, normalizedPath, settingsManager, operation.Cancellation.Token); + operation.Cancellation.Token.ThrowIfCancellationRequested(); + if (!providerResult.Success) + { + logger.LogWarning("The transcription provider failed for '{MediaPath}': {Diagnostic}", mediaPath, providerResult.ErrorMessage); + return MediaTranscriptionResult.Failed(TB("The transcription provider could not transcribe the media file.")); + } + + return MediaTranscriptionResult.Succeeded(providerResult.Text.Trim()); + } + catch (OperationCanceledException) + { + return MediaTranscriptionResult.Cancelled(); + } + catch (Exception exception) + { + logger.LogError(exception, "Media transcription failed for '{MediaPath}'.", mediaPath); + return MediaTranscriptionResult.Failed(TB("The media file could not be transcribed.")); + } + finally + { + // NormalizeAsync does not return from cancellation until Rust has reached a terminal + // phase, so deleting both paths here cannot race a still-writing worker. + if (!this.RetainNormalizedMediaIfRequested(normalizedPath, operation.Id)) + this.DeleteTemporaryFile(normalizedPath); + + this.DeleteTemporaryFile(normalizedPath + ".partial"); + } + } + + /// Validates the fail-closed WebM/Opus contract before provider upload. + private static async Task ValidateNormalizedProviderUploadAsync(MediaJobResult result, string expectedOutputPath, CancellationToken token) + { + if (string.IsNullOrWhiteSpace(result.OutputPath)) + return "Rust returned an empty normalized output path."; + + string actualFullPath; + string expectedFullPath; + try + { + actualFullPath = Path.GetFullPath(result.OutputPath); + expectedFullPath = Path.GetFullPath(expectedOutputPath); + } + catch (Exception exception) + { + return $"The normalized output path is invalid: {exception.Message}"; + } + + var pathComparison = OperatingSystem.IsWindows() + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; + + if (!string.Equals(actualFullPath, expectedFullPath, pathComparison)) + return $"Rust returned the unexpected output path '{result.OutputPath}' instead of '{expectedOutputPath}'."; + + if (!string.Equals(Path.GetExtension(actualFullPath), NORMALIZED_OUTPUT_EXTENSION, StringComparison.OrdinalIgnoreCase)) + return $"The normalized output path '{actualFullPath}' does not use the required '{NORMALIZED_OUTPUT_EXTENSION}' extension."; + + if (!string.Equals(result.OutputFormat, NORMALIZED_OUTPUT_FORMAT, StringComparison.Ordinal)) + return $"Rust returned output format '{result.OutputFormat}' instead of '{NORMALIZED_OUTPUT_FORMAT}'."; + + if (!string.Equals(result.OutputCodec, NORMALIZED_OUTPUT_CODEC, StringComparison.Ordinal)) + return $"Rust returned output codec '{result.OutputCodec}' instead of '{NORMALIZED_OUTPUT_CODEC}'."; + + if (!File.Exists(actualFullPath)) + return $"The normalized output file '{actualFullPath}' does not exist."; + + var header = new byte[WEBM_EBML_SIGNATURE.Length]; + var bytesRead = 0; + try + { + await using var stream = File.OpenRead(actualFullPath); + while (bytesRead < header.Length) + { + var count = await stream.ReadAsync(header.AsMemory(bytesRead), token); + if (count is 0) + break; + + bytesRead += count; + } + } + catch (IOException exception) + { + return $"The normalized output file '{actualFullPath}' could not be read: {exception.Message}"; + } + catch (UnauthorizedAccessException exception) + { + return $"The normalized output file '{actualFullPath}' could not be read: {exception.Message}"; + } + + if (bytesRead != header.Length || !header.AsSpan().SequenceEqual(WEBM_EBML_SIGNATURE)) + return $"The normalized output file '{actualFullPath}' does not begin with the WebM/Matroska EBML signature."; + + return null; + } + + /// Runs the Rust normalization job and drains cancellation to a terminal event. + /// Source media path. + /// Owned temporary output path. + /// Operation-specific state. + /// Whether progress belongs to the import lane. + /// The terminal runtime result or error. + private async Task<(MediaJobResult? Result, MediaJobError? Error)> NormalizeAsync( + string mediaPath, + string normalizedPath, + MediaOperation operation, + bool updateImportState) + { + // The quick POST is intentionally not cancelled: losing its response could orphan a job + // whose ID the client never received. Cancellation is applied immediately after ownership. + var jobId = await rustService.StartMediaJobAsync(mediaPath, normalizedPath, CancellationToken.None); + operation.JobId = jobId; + + try + { + operation.Cancellation.Token.ThrowIfCancellationRequested(); + await foreach (var mediaEvent in rustService.StreamMediaJobEventsAsync(jobId, operation.Cancellation.Token)) + { + if (updateImportState && mediaEvent.Phase is MediaJobPhase.PROBING or MediaJobPhase.TRANSCODING) + { + var phase = mediaEvent.Phase is MediaJobPhase.PROBING + ? MediaTranscriptionPhase.PROBING + : MediaTranscriptionPhase.TRANSCODING; + this.UpdateImportState(operation.Target!.Value, Path.GetFileName(mediaPath), phase, mediaEvent.Progress, MediaImportStatus.RUNNING); + } + + switch (mediaEvent.Phase) + { + case MediaJobPhase.COMPLETED: + return (mediaEvent.Result, null); + + case MediaJobPhase.FAILED: + if (mediaEvent.Error is not null) + logger.LogWarning("Rust media normalization failed for '{MediaPath}' with {Code}: {Diagnostic}", mediaPath, mediaEvent.Error.Code, mediaEvent.Error.Message); + + return (null, mediaEvent.Error); + + case MediaJobPhase.CANCELLED: + throw new OperationCanceledException(operation.Cancellation.Token); + } + } + + return (null, null); + } + catch (OperationCanceledException) + { + await rustService.CancelMediaJobAsync(jobId, CancellationToken.None); + await this.DrainTerminalEventAsync(jobId); + throw; + } + } + + /// Waits for Rust cleanup after cooperative cancellation. + /// Owned Rust job identifier. + private async Task DrainTerminalEventAsync(string jobId) + { + await foreach (var _ in rustService.StreamMediaJobEventsAsync(jobId, CancellationToken.None)) + { + // The stream itself ends immediately after the first terminal snapshot or event. + } + } + + /// Resolves the configured provider after confidence validation. + private TranscriptionProvider? ResolveProvider() + { + var providerId = settingsManager.ConfigurationData.App.UseTranscriptionProvider; + if (string.IsNullOrWhiteSpace(providerId)) + return null; + + var providerSettings = settingsManager.ConfigurationData.TranscriptionProviders.FirstOrDefault(x => x.Id == providerId); + if (providerSettings is null) + return null; + + var minimumLevel = settingsManager.GetMinimumConfidenceLevel(Components.NONE); + return providerSettings.UsedLLMProvider.GetConfidence(settingsManager).Level >= minimumLevel + ? providerSettings + : null; + } + + /// Creates and registers operation-owned cancellation state. + /// Optional visible media import target. + /// Caller token linked to the operation. + /// The registered operation. + private MediaOperation CreateOperation(MediaImportTarget? target, CancellationToken token) + { + var operation = new MediaOperation(target, token); + lock (this.stateLock) + this.operations.Add(operation); + + return operation; + } + + /// Unregisters and disposes completed operation state. + /// Completed operation. + private void ReleaseOperation(MediaOperation operation) + { + lock (this.stateLock) + this.operations.Remove(operation); + + operation.Dispose(); + } + + /// Updates and publishes copied state for exactly one owner. + private void UpdateImportState(MediaImportTarget target, string fileName, MediaTranscriptionPhase phase, double? progress, MediaImportStatus status) + { + var snapshot = new MediaImportSnapshot + { + Owner = target.Owner, + Target = target, + CurrentFileName = fileName, + Phase = phase, + Progress = progress, + Status = status, + }; + + lock (this.stateLock) + this.snapshots[target.Owner] = snapshot; + + this.NotifyStateChanged(target.Owner); + } + + /// Publishes one retained terminal result after an entire target batch ended. + private void CompleteImport( + MediaImportTarget target, + string fileName, + MediaImportStatus status, + IReadOnlyList failures, + IReadOnlyList warnings) + { + lock (this.stateLock) + { + this.snapshots[target.Owner] = new() + { + Owner = target.Owner, + Target = target, + CurrentFileName = fileName, + Phase = MediaTranscriptionPhase.IDLE, + Progress = null, + Status = status, + }; + + this.outcomes[target.Owner] = new() + { + Owner = target.Owner, + Status = status, + Failures = [.. failures], + Warnings = [.. warnings], + }; + } + + this.NotifyStateChanged(target.Owner); + } + + /// Publishes state changes without allowing one stale UI subscriber to fault a worker. + private void NotifyStateChanged(MediaImportOwner owner) + { + if (this.StateChanged is not { } stateChanged) + return; + + foreach (var @delegate in stateChanged.GetInvocationList()) + { + var handler = (Action)@delegate; + + try + { + handler(owner); + } + catch (Exception exception) + { + logger.LogWarning(exception, "A media state subscriber failed for owner '{Owner}'.", owner); + } + } + } + + /// Maps runtime codes to localized user-facing fallback text. + private static string UserMessageFor(MediaJobErrorCode code) => code switch + { + MediaJobErrorCode.FILE_NOT_FOUND => TB("The selected media file no longer exists."), + MediaJobErrorCode.UNSAFE_FILE or MediaJobErrorCode.NOT_MEDIA => TB("The selected file cannot be processed as media."), + MediaJobErrorCode.NO_AUDIO_TRACK => TB("The selected media file does not contain an audio track."), + MediaJobErrorCode.UNSUPPORTED_CONTAINER or MediaJobErrorCode.UNSUPPORTED_CODEC or MediaJobErrorCode.UNSUPPORTED_OPUS_MAPPING => TB("This media format or audio codec is not supported."), + MediaJobErrorCode.UNKNOWN_FORMAT or MediaJobErrorCode.DAMAGED_CONTAINER => TB("The media file is damaged or its format could not be identified."), + + _ => TB("The media file could not be prepared for transcription."), + }; + + /// Deletes one operation-owned temporary file on a best-effort basis. + private void DeleteTemporaryFile(string path) + { + try + { + if (File.Exists(path)) + File.Delete(path); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not delete operation-owned temporary media file '{Path}'.", path); + } + } + + /// Retains the exact provider upload only for opt-in debug diagnostics. + private bool RetainNormalizedMediaIfRequested(string normalizedPath, Guid operationId) + { +#if DEBUG + if (!string.Equals(Environment.GetEnvironmentVariable("MINDWORK_AI_RETAIN_NORMALIZED_MEDIA"), "true", StringComparison.OrdinalIgnoreCase) + || !File.Exists(normalizedPath)) + return false; + + try + { + var diagnosticDirectory = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-media", "diagnostics"); + Directory.CreateDirectory(diagnosticDirectory); + var diagnosticPath = Path.Combine(diagnosticDirectory, $"{operationId:N}.webm"); + File.Move(normalizedPath, diagnosticPath, overwrite: true); + + foreach (var oldPath in new DirectoryInfo(diagnosticDirectory).EnumerateFiles("*.webm").OrderByDescending(file => file.LastWriteTimeUtc).Skip(10)) + oldPath.Delete(); + + logger.LogInformation("Retained normalized media diagnostic '{DiagnosticPath}'.", diagnosticPath); + return true; + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not retain normalized media diagnostic for operation '{OperationId}'.", operationId); + } +#endif + return false; + } + + /// Returns localized text while registering the US-English fallback with I18N. + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(MediaTranscriptionService).Namespace, nameof(MediaTranscriptionService)); + + /// Throws when a caller attempts to start work after disposal. + private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(this.disposed, this); + + /// Cancels every active import and voice operation and releases owned resources. + public void Dispose() + { + MediaOperation[] active; + CancellationTokenSource[] batches; + lock (this.stateLock) + { + if (this.disposed) + return; + + this.disposed = true; + active = [.. this.operations]; + batches = [.. this.batchCancellations.Values]; + } + foreach (var operation in active) + operation.Cancellation.Cancel(); + + foreach (var batch in batches) + batch.Cancel(); + + // The semaphore may still be released by an operation unwinding after cancellation. + } + + /// Cancellation and runtime-job ownership for exactly one media operation. + private sealed class MediaOperation : IDisposable + { + /// Creates operation state linked to a caller token. + /// Optional visible media import target. + /// Caller cancellation token. + public MediaOperation(MediaImportTarget? target, CancellationToken token) + { + this.Target = target; + this.Cancellation = CancellationTokenSource.CreateLinkedTokenSource(token); + } + + /// Gets the optional visible import target; voice operations have none. + public MediaImportTarget? Target { get; } + + /// Gets the unique temporary-path identifier. + public Guid Id { get; } = Guid.NewGuid(); + + /// Gets the operation-owned cancellation source. + public CancellationTokenSource Cancellation { get; } + + /// Gets or sets the Rust job after its POST response establishes ownership. + public string? JobId { get; set; } + + /// Gets or sets whether this operation currently owns the serialized lane. + public bool HasQueueLease { get; set; } + + /// Disposes operation-owned cancellation state. + public void Dispose() => this.Cancellation.Dispose(); + } + + /// Mutable successful results waiting for acknowledgement by one target. + private sealed class PendingDelivery + { + public List Attachments { get; } = []; + + public string? Text { get; set; } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/RustService.Media.cs b/app/MindWork AI Studio/Tools/Services/RustService.Media.cs new file mode 100644 index 00000000..6e575836 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/RustService.Media.cs @@ -0,0 +1,69 @@ +using System.Runtime.CompilerServices; +using System.Text.Json; + +using AIStudio.Tools.Rust; + +namespace AIStudio.Tools.Services; + +public partial class RustService +{ + /// Starts a Rust media normalization job. + /// Absolute source path. + /// Absolute operation-owned output path. + /// Request cancellation token. + /// The opaque runtime job identifier. + public async Task StartMediaJobAsync(string inputPath, string outputPath, CancellationToken token = default) + { + using var response = await this.http.PostAsJsonAsync( + "/media/jobs", + new CreateMediaJobRequest(inputPath, outputPath), + this.jsonRustSerializerOptions, + token); + + response.EnsureSuccessStatusCode(); + var result = await response.Content.ReadFromJsonAsync(this.jsonRustSerializerOptions, token); + return result?.JobId ?? throw new InvalidOperationException("The Rust runtime did not return a media job ID."); + } + + /// Streams replayed and live snapshots until the media job becomes terminal. + /// Runtime job identifier. + /// Stream cancellation token. + /// Asynchronous media job snapshots. + public async IAsyncEnumerable StreamMediaJobEventsAsync(string jobId, [EnumeratorCancellation] CancellationToken token = default) + { + using var request = new HttpRequestMessage(HttpMethod.Get, $"/media/jobs/{Uri.EscapeDataString(jobId)}/events"); + using var response = await this.http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token); + response.EnsureSuccessStatusCode(); + + await using var stream = await response.Content.ReadAsStreamAsync(token); + using var reader = new StreamReader(stream); + + while (!token.IsCancellationRequested) + { + var line = await reader.ReadLineAsync(token); + if (line is null) + yield break; + + if (!line.StartsWith("data:", StringComparison.Ordinal)) + continue; + + var json = line["data:".Length..].Trim(); + var mediaEvent = JsonSerializer.Deserialize(json, this.jsonRustSerializerOptions); + if (mediaEvent is not null) + yield return mediaEvent; + + if (mediaEvent?.Phase is MediaJobPhase.COMPLETED or MediaJobPhase.FAILED or MediaJobPhase.CANCELLED) + yield break; + } + } + + /// Requests cooperative cancellation of a Rust media job. + /// Runtime job identifier. + /// Request cancellation token. + public async Task CancelMediaJobAsync(string jobId, CancellationToken token = default) + { + using var response = await this.http.DeleteAsync($"/media/jobs/{Uri.EscapeDataString(jobId)}", token); + if (response is { IsSuccessStatusCode: false, StatusCode: not System.Net.HttpStatusCode.NotFound }) + response.EnsureSuccessStatusCode(); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/TranscriptStagingCleanupService.cs b/app/MindWork AI Studio/Tools/Services/TranscriptStagingCleanupService.cs new file mode 100644 index 00000000..a4ca3570 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/TranscriptStagingCleanupService.cs @@ -0,0 +1,76 @@ +using AIStudio.Settings; + +namespace AIStudio.Tools.Services; + +/// +/// One-shot startup service that removes transcript staging left by crashes or forced shutdowns. +/// +public sealed class TranscriptStagingCleanupService(ILogger logger) : BackgroundService +{ + /// Waits for the data directory, performs one cleanup pass, and then exits. + /// Host shutdown token. + protected override async Task ExecuteAsync(CancellationToken stoppingToken) + { + while (string.IsNullOrWhiteSpace(SettingsManager.DataDirectory) && !stoppingToken.IsCancellationRequested) + await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken); + + if (stoppingToken.IsCancellationRequested) + return; + + var stagingRoot = Path.Combine(SettingsManager.DataDirectory!, "media-staging"); + if (!Directory.Exists(stagingRoot)) + { + logger.LogInformation("Media transcript staging does not exist; startup cleanup has nothing to remove."); + return; + } + + var directories = Directory.EnumerateDirectories(stagingRoot).ToArray(); + var files = Directory.EnumerateFiles(stagingRoot).ToArray(); + logger.LogInformation("Media transcript startup cleanup found {DirectoryCount} directories and {FileCount} loose files.", directories.Length, files.Length); + + if (directories.Length == 0 && files.Length == 0) + { + logger.LogInformation("Media transcript staging is empty."); + return; + } + + var deletedDirectories = 0; + var deletedFiles = 0; + var failures = 0; + + foreach (var directory in directories) + { + try + { + Directory.Delete(directory, true); + deletedDirectories++; + logger.LogInformation("Removed orphaned media staging directory '{Directory}'.", directory); + } + catch (Exception exception) + { + failures++; + logger.LogWarning(exception, "Could not remove orphaned media staging directory '{Directory}'.", directory); + } + } + + foreach (var file in files) + { + try + { + File.Delete(file); + deletedFiles++; + logger.LogInformation("Removed orphaned media staging file '{File}'.", file); + } + catch (Exception exception) + { + failures++; + logger.LogWarning(exception, "Could not remove orphaned media staging file '{File}'.", file); + } + } + + logger.LogInformation("Media transcript startup cleanup removed {DirectoryCount} directories and {FileCount} files with {FailureCount} failures.", + deletedDirectories, + deletedFiles, + failures); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs index d45601e5..55c279f2 100644 --- a/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs +++ b/app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs @@ -736,7 +736,7 @@ public static class WorkspaceBehaviour var chatPath = loadChat.WorkspaceId == Guid.Empty ? Path.Join(SettingsManager.DataDirectory, "tempChats", loadChat.ChatId.ToString()) : Path.Join(SettingsManager.DataDirectory, "workspaces", loadChat.WorkspaceId.ToString(), loadChat.ChatId.ToString()); - + return Directory.Exists(chatPath); } @@ -754,6 +754,7 @@ public static class WorkspaceBehaviour Directory.CreateDirectory(chatDirectory); + await FinalizeStagedTranscriptsAsync(chat, chatDirectory); var chatNamePath = Path.Join(chatDirectory, "name"); await File.WriteAllTextAsync(chatNamePath, chat.Name); @@ -769,6 +770,225 @@ public static class WorkspaceBehaviour } } + /// Creates a transcript atomically inside an already persisted chat. + /// Persisted chat that owns the transcript counter. + /// Original media path. + /// Provider transcript. + /// The chat-owned managed attachment. + public static async Task CreateManagedTranscriptAsync(ChatThread chat, string originalPath, string transcript) + { + var (acquired, semaphore) = await TryAcquireChatSemaphoreAsync(chat.WorkspaceId, chat.ChatId, nameof(CreateManagedTranscriptAsync)); + if (!acquired) + throw new IOException("The chat transcript directory is busy."); + + try + { + var chatDirectory = GetChatDirectory(chat.WorkspaceId, chat.ChatId); + if (!Directory.Exists(chatDirectory)) + throw new DirectoryNotFoundException($"The owning chat directory does not exist: '{chatDirectory}'."); + + var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts"); + Directory.CreateDirectory(transcriptDirectory); + ReconcileTranscriptCounter(chat, transcriptDirectory); + + var targetPath = NextTranscriptPath(chat, transcriptDirectory, Path.GetFileName(originalPath)); + return await ManagedTranscriptAttachment.CreateAtomicAsync(targetPath, Path.GetFileName(originalPath), transcript); + } + finally + { + semaphore.Release(); + } + } + + public static async Task MoveChatAsync(ChatThread chat, Guid targetWorkspaceId) + { + if (chat.WorkspaceId == targetWorkspaceId) + return; + + var sourceWorkspaceId = chat.WorkspaceId; + var sourceDirectory = GetChatDirectory(sourceWorkspaceId, chat.ChatId); + var targetDirectory = GetChatDirectory(targetWorkspaceId, chat.ChatId); + var sourceSemaphore = GetChatSemaphore(sourceWorkspaceId, chat.ChatId); + var targetSemaphore = GetChatSemaphore(targetWorkspaceId, chat.ChatId); + // Always acquire both workspace/chat locks in canonical workspace-ID order. This prevents + // opposing moves of the same chat from waiting on one another with reversed lock order. + var orderedSemaphores = string.CompareOrdinal(sourceWorkspaceId.ToString("N"), targetWorkspaceId.ToString("N")) <= 0 + ? new[] { sourceSemaphore, targetSemaphore } + : new[] { targetSemaphore, sourceSemaphore }; + + await orderedSemaphores[0].WaitAsync(); + await orderedSemaphores[1].WaitAsync(); + + var moved = false; + try + { + if (!Directory.Exists(sourceDirectory)) + throw new DirectoryNotFoundException($"The source chat directory does not exist: '{sourceDirectory}'."); + + // Only the workspace parent is created here. Directory.Move requires the chat target + // directory itself not to exist so an existing destination is never merged silently. + var targetWorkspaceDirectory = Path.GetDirectoryName(targetDirectory)!; + Directory.CreateDirectory(targetWorkspaceDirectory); + if (Directory.Exists(targetDirectory)) + throw new IOException($"The target chat directory already exists: '{targetDirectory}'."); + + Directory.Move(sourceDirectory, targetDirectory); + moved = true; + + UpdateAttachmentPathsAfterMove(chat, sourceDirectory, targetDirectory); + chat.WorkspaceId = targetWorkspaceId; + + await FinalizeStagedTranscriptsAsync(chat, targetDirectory); + await StoreMovedChatFilesAsync(chat, targetDirectory); + } + catch + { + if (moved) + { + try + { + UpdateAttachmentPathsAfterMove(chat, targetDirectory, sourceDirectory); + chat.WorkspaceId = sourceWorkspaceId; + + if (Directory.Exists(targetDirectory) && !Directory.Exists(sourceDirectory)) + Directory.Move(targetDirectory, sourceDirectory); + + if (Directory.Exists(sourceDirectory)) + await StoreMovedChatFilesAsync(chat, sourceDirectory); + } + catch (Exception rollbackError) + { + LOG.LogError(rollbackError, "Could not roll back moving chat '{ChatId}' to workspace '{WorkspaceId}'.", chat.ChatId, targetWorkspaceId); + } + } + throw; + } + finally + { + orderedSemaphores[1].Release(); + orderedSemaphores[0].Release(); + InvalidateWorkspaceTreeCache(); + } + } + + /// Atomically stores the name and thread after a directory move. + private static async Task StoreMovedChatFilesAsync(ChatThread chat, string chatDirectory) + { + await File.WriteAllTextAsync(Path.Join(chatDirectory, "name"), chat.Name); + var chatPath = Path.Join(chatDirectory, "thread.json"); + var temporaryPath = Path.Join(chatDirectory, $".thread-{Guid.NewGuid():N}.tmp"); + + try + { + await File.WriteAllTextAsync(temporaryPath, JsonSerializer.Serialize(chat, JSON_OPTIONS), Encoding.UTF8); + File.Move(temporaryPath, chatPath, overwrite: true); + } + finally + { + if (File.Exists(temporaryPath)) + File.Delete(temporaryPath); + } + } + + /// Rewrites absolute attachment paths after moving the complete chat directory. + private static void UpdateAttachmentPathsAfterMove(ChatThread chat, string sourceDirectory, string targetDirectory) + { + var sourcePrefix = sourceDirectory.EndsWith(Path.DirectorySeparatorChar) + ? sourceDirectory + : sourceDirectory + Path.DirectorySeparatorChar; + + var pathComparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal; + foreach (var content in chat.Blocks.Select(block => block.Content).OfType()) + { + for (var index = 0; index < content.FileAttachments.Count; index++) + { + var attachment = content.FileAttachments[index]; + if (!Path.GetFullPath(attachment.FilePath).StartsWith(sourcePrefix, pathComparison)) + continue; + + var relativePath = Path.GetRelativePath(sourceDirectory, attachment.FilePath); + var movedPath = Path.Combine(targetDirectory, relativePath); + + content.FileAttachments[index] = attachment switch + { + ManagedTranscriptAttachment managed => managed with { FilePath = movedPath }, + FileAttachmentImage image => image with { FilePath = movedPath }, + _ => attachment with { FilePath = movedPath }, + }; + } + } + } + + private static async Task FinalizeStagedTranscriptsAsync(ChatThread chat, string chatDirectory) + { + var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts"); + ReconcileTranscriptCounter(chat, transcriptDirectory); + foreach (var content in chat.Blocks.Select(block => block.Content).OfType()) + { + for (var index = 0; index < content.FileAttachments.Count; index++) + { + if (content.FileAttachments[index] is not ManagedTranscriptAttachment { IsStaged: true } staged + || !File.Exists(staged.FilePath)) + continue; + + Directory.CreateDirectory(transcriptDirectory); + var targetPath = NextTranscriptPath(chat, transcriptDirectory, staged.OriginalFileName); + + File.Move(staged.FilePath, targetPath); + var sourceDirectory = Path.GetDirectoryName(staged.FilePath); + if (sourceDirectory is not null && Directory.Exists(sourceDirectory) && !Directory.EnumerateFileSystemEntries(sourceDirectory).Any()) + Directory.Delete(sourceDirectory); + + content.FileAttachments[index] = new ManagedTranscriptAttachment( + Path.GetFileName(targetPath), + targetPath, + new FileInfo(targetPath).Length, + staged.OriginalFileName, + false); + } + } + + await Task.CompletedTask; + } + + /// Raises the persisted counter to the highest transcript suffix found chat-wide. + private static void ReconcileTranscriptCounter(ChatThread chat, string transcriptDirectory) + { + if (!Directory.Exists(transcriptDirectory)) + return; + + ulong highest = 0; + foreach (var path in Directory.EnumerateFiles(transcriptDirectory, "*-transcript-*.md", SearchOption.TopDirectoryOnly)) + { + var name = Path.GetFileNameWithoutExtension(path); + var marker = name.LastIndexOf("-transcript-", StringComparison.Ordinal); + + if (marker >= 0 && ulong.TryParse(name[(marker + "-transcript-".Length)..], out var number)) + highest = Math.Max(highest, number); + } + + chat.LastMediaTranscriptNumber = Math.Max(chat.LastMediaTranscriptNumber, highest); + } + + /// Allocates the next globally monotonic transcript path for one chat. + private static string NextTranscriptPath(ChatThread chat, string transcriptDirectory, string originalFileName) + { + string targetPath; + do + { + chat.LastMediaTranscriptNumber++; + var stem = ManagedTranscriptAttachment.NormalizeOriginalStem(originalFileName); + targetPath = Path.Combine(transcriptDirectory, $"{stem}-transcript-{chat.LastMediaTranscriptNumber:D4}.md"); + } while (File.Exists(targetPath)); + + return targetPath; + } + + /// Returns the canonical storage directory for a chat identity. + private static string GetChatDirectory(Guid workspaceId, Guid chatId) => workspaceId == Guid.Empty + ? Path.Join(SettingsManager.DataDirectory, "tempChats", chatId.ToString()) + : Path.Join(SettingsManager.DataDirectory, "workspaces", workspaceId.ToString(), chatId.ToString()); + public static async Task LoadChatAsync(LoadChat loadChat) { var (acquired, semaphore) = await TryAcquireChatSemaphoreAsync(loadChat.WorkspaceId, loadChat.ChatId, nameof(LoadChatAsync)); diff --git a/app/MindWork AI Studio/wwwroot/audio-recorder-worklet.js b/app/MindWork AI Studio/wwwroot/audio-recorder-worklet.js new file mode 100644 index 00000000..c6a10219 --- /dev/null +++ b/app/MindWork AI Studio/wwwroot/audio-recorder-worklet.js @@ -0,0 +1,61 @@ +class PCMRecorderProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + + const chunkDurationSeconds = options.processorOptions?.chunkDurationSeconds || 3; + this.chunkSamples = Math.max(128, Math.round(sampleRate * chunkDurationSeconds)); + this.samples = new Int16Array(this.chunkSamples); + this.numSamples = 0; + + this.port.onmessage = event => { + if (event.data?.type === 'flush') { + this.flush(); + this.port.postMessage({ type: 'flushed' }); + } + }; + } + + process(inputs) { + const channels = inputs[0]; + if (!channels || channels.length === 0) + return true; + + const numFrames = channels[0].length; + for (let frame = 0; frame < numFrames; frame++) { + let monoSample = 0; + for (const channel of channels) { + monoSample += channel[frame] || 0; + } + + monoSample = Math.max(-1, Math.min(1, monoSample / channels.length)); + this.samples[this.numSamples++] = monoSample < 0 + ? Math.round(monoSample * 0x8000) + : Math.round(monoSample * 0x7fff); + + if (this.numSamples === this.chunkSamples) + this.flush(); + } + + return true; + } + + flush() { + if (this.numSamples === 0) + return; + + const buffer = new ArrayBuffer(this.numSamples * 2); + const view = new DataView(buffer); + for (let index = 0; index < this.numSamples; index++) { + view.setInt16(index * 2, this.samples[index], true); + } + + this.port.postMessage({ + type: 'chunk', + buffer: buffer, + sampleCount: this.numSamples, + }, [buffer]); + this.numSamples = 0; + } +} + +registerProcessor('pcm-recorder-processor', PCMRecorderProcessor); \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/audio.js b/app/MindWork AI Studio/wwwroot/audio.js index 4e9f40b5..a2fd4d8f 100644 --- a/app/MindWork AI Studio/wwwroot/audio.js +++ b/app/MindWork AI Studio/wwwroot/audio.js @@ -180,22 +180,208 @@ window.playSound = async function(soundPath) { } }; -let mediaRecorder; -let actualRecordingMimeType; -let changedMimeType = false; let pendingChunkUploads = 0; +let chunkUploadPromise = Promise.resolve(); +let chunkUploadError = null; +let recordingError = null; +let captureAudioContext = null; +let captureSourceNode = null; +let captureWorkletNode = null; +let captureSilentGainNode = null; +let pcmFlushResolve = null; +let pcmSamplesReceived = 0; // Store the media stream so we can close the microphone later: let activeMediaStream = null; // Delay in milliseconds to wait after getUserMedia() for Bluetooth profile switch (A2DP → HFP): const BLUETOOTH_PROFILE_SWITCH_DELAY_MS = 1_600; +const PCM_SAMPLE_RATE = 48_000; +const PCM_CHUNK_DURATION_SECONDS = 3; +const PCM_FLUSH_TIMEOUT_MS = 5_000; + +function queueAudioChunkUpload(upload) { + pendingChunkUploads++; + chunkUploadPromise = chunkUploadPromise + .then(upload) + .catch(error => { + chunkUploadError ??= error; + console.error('Error sending audio chunk to .NET:', error); + }) + .finally(() => pendingChunkUploads--); +} + +async function waitForAudioChunkUploads() { + let observedUploadPromise; + do { + observedUploadPromise = chunkUploadPromise; + await observedUploadPromise; + } while (pendingChunkUploads > 0 || observedUploadPromise !== chunkUploadPromise); +} + +function createPcmWavHeader(sampleRate) { + const buffer = new ArrayBuffer(44); + const view = new DataView(buffer); + + const writeAscii = (offset, value) => { + for (let index = 0; index < value.length; index++) { + view.setUint8(offset + index, value.charCodeAt(index)); + } + }; + + writeAscii(0, 'RIFF'); + view.setUint32(4, 0, true); // Finalized by .NET after all PCM data was written. + writeAscii(8, 'WAVE'); + writeAscii(12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); // PCM + view.setUint16(22, 1, true); // Mono + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + writeAscii(36, 'data'); + view.setUint32(40, 0, true); // Finalized by .NET after all PCM data was written. + + return new Uint8Array(buffer); +} + +function observeAudioTrack(track) { + console.log('Audio recording - microphone track state:', { + label: track.label, + enabled: track.enabled, + muted: track.muted, + readyState: track.readyState, + settings: typeof track.getSettings === 'function' ? track.getSettings() : null, + }); + + track.addEventListener('mute', () => console.warn('Audio recording - microphone track was muted.')); + track.addEventListener('unmute', () => console.log('Audio recording - microphone track was unmuted.')); + track.addEventListener('ended', () => console.warn('Audio recording - microphone track ended.')); +} + +async function startPcmRecording(stream, dotnetRef) { + const AudioContextClass = window.AudioContext || window.webkitAudioContext; + if (!AudioContextClass || typeof AudioWorkletNode === 'undefined') { + throw new Error('PCM audio capture is unavailable because AudioWorklet is not supported.'); + } + + try { + captureAudioContext = new AudioContextClass({ + latencyHint: 'interactive', + sampleRate: PCM_SAMPLE_RATE, + }); + + if (!captureAudioContext.audioWorklet) { + throw new Error('PCM audio capture is unavailable because AudioWorklet is not supported.'); + } + + await captureAudioContext.audioWorklet.addModule('/audio-recorder-worklet.js'); + + const actualSampleRate = captureAudioContext.sampleRate; + console.log(`Audio recording - starting PCM/WAV capture at ${actualSampleRate} Hz mono.`); + + if (captureAudioContext.state === 'suspended') { + await captureAudioContext.resume(); + } + + captureSourceNode = captureAudioContext.createMediaStreamSource(stream); + captureWorkletNode = new AudioWorkletNode(captureAudioContext, 'pcm-recorder-processor', { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + processorOptions: { + chunkDurationSeconds: PCM_CHUNK_DURATION_SECONDS, + }, + }); + + captureSilentGainNode = captureAudioContext.createGain(); + captureSilentGainNode.gain.value = 0; + + captureWorkletNode.port.onmessage = event => { + if (event.data?.type === 'chunk') { + const chunkBytes = new Uint8Array(event.data.buffer); + pcmSamplesReceived += event.data.sampleCount; + console.debug(`Audio recording - received ${event.data.sampleCount} PCM samples from AudioWorklet.`); + queueAudioChunkUpload(() => dotnetRef.invokeMethodAsync('OnAudioChunkReceived', chunkBytes)); + } else if (event.data?.type === 'flushed') { + pcmFlushResolve?.(); + pcmFlushResolve = null; + } + }; + + captureWorkletNode.onprocessorerror = event => { + recordingError ??= event.error || new Error('The PCM audio processor failed.'); + console.error('Audio recording - AudioWorklet error:', recordingError); + }; + + captureSourceNode.connect(captureWorkletNode); + captureWorkletNode.connect(captureSilentGainNode); + captureSilentGainNode.connect(captureAudioContext.destination); + queueAudioChunkUpload(() => dotnetRef.invokeMethodAsync('OnAudioChunkReceived', createPcmWavHeader(actualSampleRate))); + } catch (error) { + await cleanupPcmCapture(); + throw error; + } +} + +async function flushPcmRecording() { + if (!captureWorkletNode) { + throw new Error('The PCM audio processor is unavailable.'); + } + + await new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + pcmFlushResolve = null; + reject(new Error('Timed out while flushing PCM audio data.')); + }, PCM_FLUSH_TIMEOUT_MS); + + pcmFlushResolve = () => { + clearTimeout(timeoutId); + resolve(); + }; + captureWorkletNode.port.postMessage({ type: 'flush' }); + }); +} + +async function cleanupPcmCapture() { + captureSourceNode?.disconnect(); + captureWorkletNode?.disconnect(); + captureSilentGainNode?.disconnect(); + captureSourceNode = null; + captureWorkletNode = null; + captureSilentGainNode = null; + pcmFlushResolve = null; + + if (captureAudioContext && captureAudioContext.state !== 'closed') { + await captureAudioContext.close(); + } + captureAudioContext = null; +} window.audioRecorder = { - start: async function (dotnetRef, desiredMimeTypes = []) { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + start: async function (dotnetRef) { + // Reset the upload and recorder state: + pendingChunkUploads = 0; + chunkUploadPromise = Promise.resolve(); + chunkUploadError = null; + recordingError = null; + pcmSamplesReceived = 0; + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + sampleRate: { ideal: PCM_SAMPLE_RATE }, + channelCount: { ideal: 1 }, + }, + }); activeMediaStream = stream; + const audioTracks = stream.getAudioTracks(); + if (audioTracks.length === 0) { + throw new Error('The microphone stream does not contain an audio track.'); + } + observeAudioTrack(audioTracks[0]); + // Wait for Bluetooth headsets to complete the profile switch from A2DP to HFP. // This prevents the first sound from being cut off during the switch: console.log('Audio recording - waiting for Bluetooth profile switch...'); @@ -204,121 +390,55 @@ window.audioRecorder = { // Play start recording sound effect: await window.playSound('/sounds/start_recording.ogg'); - // When only one mime type is provided as a string, convert it to an array: - if (typeof desiredMimeTypes === 'string') { - desiredMimeTypes = [desiredMimeTypes]; - } - - // Log sent mime types for debugging: - console.log('Audio recording - requested mime types: ', desiredMimeTypes); - - let mimeTypes = desiredMimeTypes.filter(type => typeof type === 'string' && type.trim() !== ''); - - // Next, we have to ensure that we have some default mime types to check as well. - // In case the provided list does not contain these, we append them: - // Use provided mime types or fallback to a default list: - const defaultMimeTypes = [ - 'audio/webm', - 'audio/ogg', - 'audio/mp4', - 'audio/mpeg', - ''// Fallback to browser default - ]; - - defaultMimeTypes.forEach(type => { - if (!mimeTypes.includes(type)) { - mimeTypes.push(type); - } - }); - - console.log('Audio recording - final mime types to check (included defaults): ', mimeTypes); - - // Find the first supported mime type: - actualRecordingMimeType = mimeTypes.find(type => - type === '' || MediaRecorder.isTypeSupported(type) - ) || ''; - - console.log('Audio recording - the browser selected the following mime type for recording: ', actualRecordingMimeType); - const options = actualRecordingMimeType ? { mimeType: actualRecordingMimeType } : {}; - mediaRecorder = new MediaRecorder(stream, options); - - // In case the browser changed the mime type: - actualRecordingMimeType = mediaRecorder.mimeType; - console.log('Audio recording - actual mime type used by the browser: ', actualRecordingMimeType); - - // Check the list of desired mime types against the actual one: - if (!desiredMimeTypes.includes(actualRecordingMimeType)) { - changedMimeType = true; - console.warn(`Audio recording - requested mime types ('${desiredMimeTypes.join(', ')}') do not include the actual mime type used by the browser ('${actualRecordingMimeType}').`); - } else { - changedMimeType = false; - } - - // Reset the pending uploads counter: - pendingChunkUploads = 0; - - // Stream each chunk directly to .NET as it becomes available: - mediaRecorder.ondataavailable = async (event) => { - if (event.data.size > 0) { - pendingChunkUploads++; - try { - const arrayBuffer = await event.data.arrayBuffer(); - const uint8Array = new Uint8Array(arrayBuffer); - await dotnetRef.invokeMethodAsync('OnAudioChunkReceived', uint8Array); - } catch (error) { - console.error('Error sending audio chunk to .NET:', error); - } finally { - pendingChunkUploads--; - } - } - }; - - mediaRecorder.start(3000); // read the recorded data in 3-second chunks - return actualRecordingMimeType; + await startPcmRecording(stream, dotnetRef); }, stop: async function () { - return new Promise((resolve) => { + let stopError = null; - // Add an event listener to handle the stop event: - mediaRecorder.onstop = async () => { + try { + try { + await flushPcmRecording(); + } finally { + await cleanupPcmCapture(); + } - // Wait for all pending chunk uploads to complete before finalizing: - console.log(`Audio recording - waiting for ${pendingChunkUploads} pending uploads.`); - while (pendingChunkUploads > 0) { - await new Promise(r => setTimeout(r, 10)); // wait 10 ms before checking again - } + console.log(`Audio recording - PCM/WAV capture produced ${pcmSamplesReceived} samples.`); + if (pcmSamplesReceived === 0) { + throw new Error('The microphone did not produce any PCM audio samples.'); + } + } catch (error) { + stopError = error; + } - console.log('Audio recording - all chunks uploaded, finalizing.'); + console.log(`Audio recording - waiting for ${pendingChunkUploads} pending uploads.`); + await waitForAudioChunkUploads(); + console.log('Audio recording - all chunks uploaded, finalizing.'); - // Play stop recording sound effect: - await window.playSound('/sounds/stop_recording.ogg'); + // Play stop recording sound effect: + await window.playSound('/sounds/stop_recording.ogg'); - // - // IMPORTANT: Do NOT release the microphone here! - // Bluetooth headsets switch profiles (HFP → A2DP) when the microphone is released, - // which causes audio to be interrupted. We keep the microphone open so that the - // stop_recording and transcription_done sounds can play without interruption. - // - // Call window.audioRecorder.releaseMicrophone() after the last sound has played. - // + // + // IMPORTANT: Do NOT release the microphone here! + // Bluetooth headsets switch profiles (HFP → A2DP) when the microphone is released, + // which causes audio to be interrupted. We keep the microphone open so that the + // stop_recording and transcription_done sounds can play without interruption. + // + // Call window.audioRecorder.releaseMicrophone() after the last sound has played. + // - // No need to process data here anymore, just signal completion: - resolve({ - mimeType: actualRecordingMimeType, - changedMimeType: changedMimeType, - }); - }; - - // Finally, stop the recording (which will actually trigger the onstop event): - mediaRecorder.stop(); - }); + const error = stopError || recordingError || chunkUploadError; + if (error) { + throw error; + } }, // Release the microphone after all sounds have been played. // This should be called after the transcription_done sound to allow // Bluetooth headsets to switch back to A2DP profile without interrupting audio: - releaseMicrophone: function () { + releaseMicrophone: async function () { + await cleanupPcmCapture(); + if (activeMediaStream) { console.log('Audio recording - releasing microphone (Bluetooth will switch back to A2DP)'); activeMediaStream.getTracks().forEach(track => track.stop()); diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.7.3.md b/app/MindWork AI Studio/wwwroot/changelog/v26.7.3.md index 346091e4..628ef17b 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.7.3.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.7.3.md @@ -1,10 +1,12 @@ # v26.7.3, build 245 (2026-07-xx xx:xx UTC) - Added support for OpenAI GPT-5.6 Sol, Terra, and Luna; Anthropic Claude Fable 5 and Mythos 5; and Google Gemini 3 Flash, Gemini 3.1 Flash-Lite, Gemini 3.1 Pro, and Gemini 3.5 Flash. +- Added audio and video transcription for chats and assistants. AI Studio now prepares supported media locally, sends only normalized audio to the configured transcription provider, and attaches the resulting transcript instead of the original media. - Improved the "My Tasks Assistant": you can now provide one or more documents in addition to text or use documents alone when asking to identify tasks. - Improved update guidance for Flatpak installations and added an enterprise option that lets organizations manage updates entirely through their IT department. -- Fixed an issue that could leave AI Studio unresponsive after waking the computer from sleep. -- Fixed enterprise configuration plugins from Windows-created ZIP files not loading correctly on Linux when the ZIP contained plugin files inside a folder. -- Fixed voice recording not starting on Linux. +- Fixed an issue that could leave AI Studio unresponsive after waking the computer from sleep. Yes, we know this was an annoying bug, and we apologize for the inconvenience. +- Fixed enterprise configuration plugins from Windows-created ZIP files may not load correctly on Linux when the ZIP contained plugin files inside a folder. +- Fixed voice recording and transcription on Linux. +- Fixed being able to switch document analysis policies while an analysis or media transcription was still in progress. - Upgraded Rust to v1.97.0. - Upgraded Tauri to v2.11.5. - Upgraded common dependencies. \ No newline at end of file diff --git a/runtime/.idea/runtime.iml b/runtime/.idea/runtime.iml index cf84ae4a..bbe0a70f 100644 --- a/runtime/.idea/runtime.iml +++ b/runtime/.idea/runtime.iml @@ -3,6 +3,7 @@ + diff --git a/runtime/Cargo.lock b/runtime/Cargo.lock index f0956ce7..0c8bd4de 100644 --- a/runtime/Cargo.lock +++ b/runtime/Cargo.lock @@ -490,6 +490,43 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "audio-core" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ebbf82d06013f4c41fe71303feb980cddd78496d904d06be627972de51a24" + +[[package]] +name = "audioadapter" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c75c3943c6c7279bb25a449a8d1727480730ab2efd7b6fd5d6ca51927096e6e4" +dependencies = [ + "audio-core", + "num-traits", +] + +[[package]] +name = "audioadapter-buffers" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ece3390b6eb40379094843a1da5aaccc34bc0d85a8cbf68d09fe092fee6de29e" +dependencies = [ + "audioadapter", + "audioadapter-sample", + "num-traits", +] + +[[package]] +name = "audioadapter-sample" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1592f90413568e259413c21a41a3d571feb1774255c209e7966d98f9db708c90" +dependencies = [ + "audio-core", + "num-traits", +] + [[package]] name = "autocfg" version = "1.3.0" @@ -1923,6 +1960,35 @@ dependencies = [ "num-traits", ] +[[package]] +name = "ebml-iterable" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b5173ac3752f08b526a6991509615e1a345b221ec3c58c7633433e8c9582312" +dependencies = [ + "ebml-iterable-specification", + "ebml-iterable-specification-derive", + "futures", +] + +[[package]] +name = "ebml-iterable-specification" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f56467af159a98735d44231f53eaa505e919e6003266f103b99649a93f106784" + +[[package]] +name = "ebml-iterable-specification-derive" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b066b81018300fdce40f71c4db355a102699324af96fad28f25ab1b5f87de066" +dependencies = [ + "ebml-iterable-specification", + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "ecow" version = "0.3.0" @@ -2112,6 +2178,12 @@ dependencies = [ "zune-inflate", ] +[[package]] +name = "extended" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365" + [[package]] name = "fast-float2" version = "0.2.3" @@ -4023,11 +4095,14 @@ dependencies = [ "rand 0.10.2", "rand_chacha 0.10.0", "rcgen", + "ropus", + "rubato", "rustls", "serde", "serde_json", "sha2 0.11.0", "strum_macros", + "symphonia", "sys-locale", "sysinfo 0.39.6", "tauri", @@ -4043,6 +4118,7 @@ dependencies = [ "tokio", "tokio-stream", "webkit2gtk", + "webm-iterable", "whoami", "windows-native-keyring-store", "windows-registry", @@ -5126,6 +5202,15 @@ dependencies = [ "syn 2.0.117", ] +[[package]] +name = "primal-check" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08" +dependencies = [ + "num-integer", +] + [[package]] name = "proc-macro-crate" version = "1.3.1" @@ -5561,6 +5646,15 @@ dependencies = [ "yasna", ] +[[package]] +name = "realfft" +version = "3.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677" +dependencies = [ + "rustfft", +] + [[package]] name = "redox_syscall" version = "0.4.1" @@ -5613,6 +5707,12 @@ dependencies = [ "regex-syntax", ] +[[package]] +name = "regex-lite" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973" + [[package]] name = "regex-syntax" version = "0.8.10" @@ -5740,6 +5840,16 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839" +[[package]] +name = "ropus" +version = "0.12.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "80804dadbfa2851c95fe45ff9ae8f4328d6371fdc0d51b14740d49a8b41d3758" +dependencies = [ + "cc", + "wide", +] + [[package]] name = "roxmltree" version = "0.20.0" @@ -5757,6 +5867,22 @@ dependencies = [ "smallvec", ] +[[package]] +name = "rubato" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f57c655d11e929f05a8663b323ff553f8d9773be05dfdc087795955bedeb8d92" +dependencies = [ + "audioadapter", + "audioadapter-buffers", + "num-complex", + "num-integer", + "num-traits", + "realfft", + "visibility", + "windowfunctions", +] + [[package]] name = "rustc-demangle" version = "0.1.27" @@ -5778,6 +5904,20 @@ dependencies = [ "semver", ] +[[package]] +name = "rustfft" +version = "6.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89" +dependencies = [ + "num-complex", + "num-integer", + "num-traits", + "primal-check", + "strength_reduce", + "transpose", +] + [[package]] name = "rusticata-macros" version = "4.1.0" @@ -5902,6 +6042,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "safe_arch" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323" +dependencies = [ + "bytemuck", +] + [[package]] name = "same-file" version = "1.0.6" @@ -6532,6 +6681,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +[[package]] +name = "strength_reduce" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82" + [[package]] name = "string_cache" version = "0.9.0" @@ -6606,6 +6761,192 @@ dependencies = [ "serde_json", ] +[[package]] +name = "symphonia" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1758d6c853020a7244de03cc3e0185eaea3f58715122422dd3cc7452e6d4c16a" +dependencies = [ + "lazy_static", + "symphonia-bundle-flac", + "symphonia-bundle-mp3", + "symphonia-codec-aac", + "symphonia-codec-alac", + "symphonia-codec-pcm", + "symphonia-codec-vorbis", + "symphonia-core", + "symphonia-format-caf", + "symphonia-format-isomp4", + "symphonia-format-mkv", + "symphonia-format-ogg", + "symphonia-format-riff", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-flac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee69ad01236a67260b82fd1ff9790dd75ead29f2f46af145e63b7e72273e0e03" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-bundle-mp3" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "350f1f2f2e19ad4dd315db94304d1eb361b29af070681f94e51b8fdaad769546" +dependencies = [ + "lazy_static", + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-aac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1979c515a76371b186aad2feff5f23e21cbec775bf95de08bf1e3af92a2ad76" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-alac" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a149cbfc7fb5c405d123a273227d31de17138419552112bf1aa7b73e65827b8" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-pcm" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50baee168f0e9dcf6ba7fc06e8b57eb62072a4490cc7cf13af77e72baae5d328" +dependencies = [ + "log", + "symphonia-core", +] + +[[package]] +name = "symphonia-codec-vorbis" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45b07b4423cd8e0fc472575909a5554b12c2f58e3c190b38c24f042e732fd8de" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-common" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8257891ffa7f05e02b58f4761e2abf7e5278c8744fd59e981559e050f86eef55" +dependencies = [ + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-core" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95ec293b5f288383b72a7bffcade6b2860b642cf66f28b3bd5967349a49938b1" +dependencies = [ + "bitflags 2.11.1", + "bytemuck", + "lazy_static", + "log", + "num-complex", + "smallvec", +] + +[[package]] +name = "symphonia-format-caf" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cde3ca76633d3400ab57195456c09f8a58d775ff5452329f3f212b6efc8622f5" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-isomp4" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d179a01305b3505940135a9f0180d6ef4b487912748fe97554756f120fbd05e" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-mkv" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb17713e134f5ad316c2690fa3104590ccc85842cdbcf82c3cd1a845cb08aa74" +dependencies = [ + "lazy_static", + "log", + "symphonia-common", + "symphonia-core", +] + +[[package]] +name = "symphonia-format-ogg" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05a67e02b1e4fca1a261ba4fe06910a9357489ad8c36aafdd2960e9c6559433" +dependencies = [ + "log", + "symphonia-common", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-format-riff" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17424452a777666d3eaf09a5c651029b15b6a333812fcc5b5474f2a3f0cff3f0" +dependencies = [ + "extended", + "log", + "symphonia-core", + "symphonia-metadata", +] + +[[package]] +name = "symphonia-metadata" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31acf5cd623398a6208e2225d18f4b20f761c55098a796a5247ad516a4a8681" +dependencies = [ + "lazy_static", + "log", + "regex-lite", + "smallvec", + "symphonia-core", +] + [[package]] name = "syn" version = "1.0.109" @@ -7356,6 +7697,7 @@ dependencies = [ "futures-core", "pin-project-lite", "tokio", + "tokio-util", ] [[package]] @@ -7603,6 +7945,16 @@ dependencies = [ "once_cell", ] +[[package]] +name = "transpose" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e" +dependencies = [ + "num-integer", + "strength_reduce", +] + [[package]] name = "tray-icon" version = "0.24.1" @@ -7896,6 +8248,17 @@ version = "0.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1" +[[package]] +name = "visibility" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] + [[package]] name = "vswhom" version = "0.1.0" @@ -8174,6 +8537,15 @@ dependencies = [ "system-deps", ] +[[package]] +name = "webm-iterable" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd9fbf173b4b38f2f8bbb0082a0d4cb21f263a70811f5fccb1663c421c66d9f9" +dependencies = [ + "ebml-iterable", +] + [[package]] name = "webpki-root-certs" version = "1.0.4" @@ -8248,6 +8620,16 @@ dependencies = [ "web-sys", ] +[[package]] +name = "wide" +version = "0.7.33" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "winapi" version = "0.3.9" @@ -8294,6 +8676,15 @@ dependencies = [ "windows-version", ] +[[package]] +name = "windowfunctions" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a" +dependencies = [ + "num-traits", +] + [[package]] name = "windows" version = "0.61.3" diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index ec61e3d2..f86dbfad 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -20,7 +20,7 @@ serde_json = "1.0.150" keyring-core = "1.0.0" arboard = "3.6.1" tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "macros", "process"] } -tokio-stream = "0.1.18" +tokio-stream = { version = "0.1.18", features = ["sync"] } futures = "0.3.32" async-stream = "0.3.6" flexi_logger = "0.31.9" @@ -39,6 +39,10 @@ hmac = "0.13.0" sha2 = "0.11.0" rcgen = { version = "0.14.8", features = ["pem"] } file-format = "0.29.0" +symphonia = { version = "0.6", default-features = false, features = ["aac", "aiff", "alac", "caf", "flac", "isomp4", "mkv", "mp1", "mp2", "mp3", "ogg", "pcm", "vorbis", "wav"] } +ropus = "=0.12.18" +rubato = { version = "4", default-features = false, features = ["fft_resampler"] } +webm-iterable = "0.6.4" calamine = "0.36.0" pdfium-render = "0.9.1" sys-locale = "0.3.2" @@ -77,3 +81,20 @@ tauri-plugin-updater = "2.10.1" [features] custom-protocol = ["tauri/custom-protocol"] + +# Media normalization is CPU-heavy even when the application itself is built for development. +# Keep release settings untouched while optimizing the hot decoder/resampler/container crates. +[profile.dev.package.symphonia-core] +opt-level = 3 + +[profile.dev.package.symphonia-format-mkv] +opt-level = 3 + +[profile.dev.package.ropus] +opt-level = 3 + +[profile.dev.package.rubato] +opt-level = 3 + +[profile.dev.package.webm-iterable] +opt-level = 3 diff --git a/runtime/resources/notices/THIRD_PARTY_MEDIA_NOTICES.md b/runtime/resources/notices/THIRD_PARTY_MEDIA_NOTICES.md new file mode 100644 index 00000000..f995d4f7 --- /dev/null +++ b/runtime/resources/notices/THIRD_PARTY_MEDIA_NOTICES.md @@ -0,0 +1,162 @@ +# Media pipeline third-party notices + +These notices are bundled offline with MindWork AI Studio. + +## Symphonia 0.6.0 + +Copyright (c) 2019-2026 The Project Symphonia Developers. + +MindWork AI Studio uses the unmodified Symphonia 0.6.0 crates. The exact corresponding source is: + +- https://github.com/pdeljanov/Symphonia/tree/v0.6.0 +- https://crates.io/api/v1/crates/symphonia/0.6.0/download + +If a future AI Studio release modifies MPL-covered Symphonia files, those modifications must be identified and made available separately under MPL-2.0. No such modifications are present in this release. + +Mozilla Public License Version 2.0 + +1. Definitions + +1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software. + +1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” means Covered Software of a particular Contributor. + +1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof. + +1.5. “Incompatible With Secondary Licenses” means that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License. + +1.6. “Executable Form” means any form of the work other than Source Code Form. + +1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software. + +1.8. “License” means this document. + +1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License. + +1.10. “Modifications” means any of the following: any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version. + +1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” means the form of the work preferred for making modifications. + +1.14. “You” means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. “Control” means ownership of more than fifty percent of the outstanding shares or beneficial ownership of such entity. + +2. License Grants and Conditions + +2.1. Grants. Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version. + +2.2. Effective Date. The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution. + +2.3. Limitations on Grant Scope. No license is granted in the trademarks, service marks, or logos of any Contributor. Except as otherwise provided in this License, no Contributor grants additional rights by implication, estoppel, or otherwise. + +2.4. Subsequent Licenses. No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License or the terms of a Secondary License. + +2.5. Representation. Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use. This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions. Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1. + +3. Responsibilities + +3.1. Distribution of Source Form. All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form. If You distribute Covered Software in Executable Form then such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work. You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s). + +3.4. Notices. You may not remove or alter the substance of any license notices contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. You must include a copy of this License with every copy of the Covered Software You distribute. You may add additional accurate notices of copyright ownership. + +3.5. Application of Additional Terms. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor. + +4. Inability to Comply Due to Statute or Regulation + +If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must comply with the terms of this License to the maximum extent possible and describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent infringement claim alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 will terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination. + +6. Disclaimer of Warranty + +Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You. + +7. Limitation of Liability + +Under no circumstances and under no legal theory, whether tort, contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages. + +8. Litigation + +Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + +This License represents the complete agreement concerning the subject matter hereof. If any provision is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor. + +10. Versions of the License + +10.1. New Versions. Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License. + +10.2. Effect of New Versions. You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward. + +10.3. Modified Versions. If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses. If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + +This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + +This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0. + +## Ropus 0.12.18 + +Copyright 2001-2023 Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo, Mozilla, Amazon + +Copyright (c) 2026 Martin Davidson (Rust port additions) + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. +- Neither the name of Internet Society, IETF or IETF Trust, nor the names of specific contributors, may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Opus is subject to the royalty-free patent licenses specified at: + +- Xiph.Org Foundation: https://datatracker.ietf.org/ipr/1524/ +- Microsoft Corporation: https://datatracker.ietf.org/ipr/1914/ +- Broadcom Corporation: https://datatracker.ietf.org/ipr/1526/ + +## Rubato 4.0.0 (MIT option) + +Copyright (c) 2020 Henrik Enquist + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +## webm-iterable 0.6.4 + +MIT License + +Copyright (c) 2021 Austin Blake + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/runtime/src/lib.rs b/runtime/src/lib.rs index ac9f9250..353c808e 100644 --- a/runtime/src/lib.rs +++ b/runtime/src/lib.rs @@ -11,6 +11,7 @@ pub mod runtime_api; pub mod runtime_certificate; pub mod file_data; pub mod metadata; +pub mod media; pub mod pdfium; pub mod pandoc; pub mod qdrant_edge_database; diff --git a/runtime/src/log.rs b/runtime/src/log.rs index bf94fc33..22741f0e 100644 --- a/runtime/src/log.rs +++ b/runtime/src/log.rs @@ -43,6 +43,7 @@ pub fn init_logging() { log_config.push_str("tower_http=info, "); log_config.push_str("rustls=info, "); log_config.push_str("tokio_rustls=info, "); + log_config.push_str("symphonia_format_mkv=info, "); log_config.push_str("reqwest=info"); // Configure the initial filename. On Unix systems, the file should start diff --git a/runtime/src/media.rs b/runtime/src/media.rs new file mode 100644 index 00000000..b10d8bf6 --- /dev/null +++ b/runtime/src/media.rs @@ -0,0 +1,1971 @@ +//! Asynchronous media normalization jobs producing bounded mono WebM/Opus output. + +use std::collections::HashMap; +use std::convert::Infallible; +use std::fs::{self, File}; +use std::io::{Read, Seek, SeekFrom, Write}; +use std::path::{Path as FilePath, PathBuf}; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock}; +use std::time::{Duration as StdDuration, Instant}; + +use axum::extract::Path; +use axum::http::StatusCode; +use axum::response::sse::{Event, KeepAlive, Sse}; +use axum::response::IntoResponse; +use axum::Json; +use file_format::{FileFormat, Kind}; +use futures::Stream; +use once_cell::sync::Lazy; +use ropus::{Application, Bitrate, Channels as OpusChannels, DecodeMode, Decoder as OpusDecoder, Encoder as OpusEncoder}; +use rubato::audioadapter_buffers::direct::SequentialSliceOfVecs; +use rubato::{Fft, FixedSync, Indexing, Resampler}; +use serde::{Deserialize, Serialize}; +use symphonia::core::audio::sample::Sample; +use symphonia::core::codecs::audio::{well_known::CODEC_ID_OPUS, AudioDecoder, AudioDecoderOptions}; +use symphonia::core::codecs::CodecParameters; +use symphonia::core::errors::Error as SymphoniaError; +use symphonia::core::formats::{FormatOptions, Track, TrackFlags, TrackType}; +use symphonia::core::io::{MediaSource, MediaSourceStream}; +use symphonia::core::meta::MetadataOptions; +use symphonia::core::formats::probe::Hint; +use symphonia::core::units::{TimeBase, Timestamp}; +use tokio::sync::broadcast; +use tokio_stream::wrappers::BroadcastStream; +use tokio_stream::StreamExt; +use webm_iterable::matroska_spec::{Master, MatroskaSpec, SimpleBlock}; +use webm_iterable::{WebmIterator, WebmWriter, WriteOptions}; + +use crate::api_token::APIToken; + +/// Sample rate required by the normalized WebM/Opus output contract. +const OUTPUT_SAMPLE_RATE: u32 = 48_000; + +/// Number of samples in one 20 ms Opus frame at 48 kHz. +const OPUS_FRAME_SAMPLES: usize = 960; + +/// Target bitrate for mono speech-oriented Opus output. +const OPUS_BITRATE: u32 = 32_000; + +/// Stable normalized container name returned to upload clients. +const OUTPUT_FORMAT: &str = "webm"; + +/// Stable normalized codec name returned to upload clients. +const OUTPUT_CODEC: &str = "opus"; + +/// Maximum duration of a WebM cluster before rotating it. +const CLUSTER_DURATION_MS: u64 = 30_000; + +/// Encoder look-ahead advertised as the output track's codec delay. +const OPUS_PRE_SKIP: u16 = 312; + +/// Default size ceiling for copying an already-normalized file unchanged. +const DEFAULT_MAX_PASS_THROUGH_BYTES: u64 = 25 * 1024 * 1024; + +/// Bounded input block used for streaming resampling. +const RESAMPLE_INPUT_BLOCK_SAMPLES: usize = 2_048; + +/// Bounded block used by the cancellation-aware pass-through copy. +const COPY_BLOCK_BYTES: usize = 64 * 1024; + +/// Minimum interval between non-terminal progress events in one phase. +const PROGRESS_EVENT_INTERVAL: StdDuration = StdDuration::from_secs(6); + +/// Timestamp differences above this threshold are recorded as discontinuities. +const LARGE_DISCONTINUITY_MS: i64 = 1_000; + +/// Maximum full-scale peak still treated as practical silence. +const SILENCE_MAX_PEAK_DBFS: f32 = -60.0; + +/// Time a terminal job remains available for late SSE subscribers. +const TERMINAL_JOB_RETENTION: std::time::Duration = std::time::Duration::from_secs(10 * 60); + +/// In-memory registry of running and recently completed media jobs. +static JOBS: Lazy>>> = Lazy::new(|| RwLock::new(HashMap::new())); + +/// Request body for starting a media normalization job. +#[derive(Debug, Deserialize)] +pub struct CreateMediaJobRequest { + /// Absolute path of the source media file. + pub input_path: String, + + /// Optional absolute output path; a sibling WebM path is derived when omitted. + pub output_path: Option, + + /// Optional size ceiling for pass-through files. + pub max_pass_through_bytes: Option, +} + +/// Response returned immediately after a media job has been registered. +#[derive(Debug, Serialize)] +pub struct CreateMediaJobResponse { + /// Opaque identifier used by the event and cancellation routes. + pub job_id: String, +} + +/// Observable lifecycle phases of a media job. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MediaJobPhase { + /// The runtime is identifying the container and selecting an audio track. + Probing, + + /// The runtime is decoding and normalizing the selected track. + Transcoding, + + /// The normalized output was committed atomically. + Completed, + + /// The job ended with a stable media error. + Failed, + + /// Cancellation completed and temporary output has been removed. + Cancelled, +} + +/// Stable, machine-readable failure categories returned by the media API. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Serialize)] +#[serde(rename_all = "snake_case")] +pub enum MediaErrorCode { + /// The requested input file does not exist. + FileNotFound, + + /// The file type could not be identified. + UnknownFormat, + + /// Executable input was rejected. + UnsafeFile, + + /// The identified input is not audio or video. + NotMedia, + + /// The input file could not be opened. + FileOpenFailed, + + /// Symphonia does not support the container. + UnsupportedContainer, + + /// The container has no audio track. + NoAudioTrack, + + /// No audio track has a supported decoder. + UnsupportedCodec, + + /// Required decoded stream parameters are absent or changed. + InvalidAudioParameters, + + /// The Opus identification header is invalid. + InvalidOpusHeader, + + /// The Opus mapping requires unsupported multistream decoding. + UnsupportedOpusMapping, + + /// The decoder could not be initialized. + DecoderInitFailed, + + /// The encoder could not be initialized. + EncoderInitFailed, + + /// The stream requested a decoder reset. + StreamReset, + + /// The media container is truncated or malformed. + DamagedContainer, + + /// Audio decoding failed. + DecodeFailed, + + /// Audio resampling failed. + ResampleFailed, + + /// Opus encoding failed. + EncodeFailed, + + /// The output directory could not be created. + OutputCreateFailed, + + /// Output bytes could not be written. + OutputWriteFailed, + + /// The partial output could not be committed. + OutputCommitFailed, + + /// A WebM relative timestamp exceeded its safe range. + WebmTimestampOverflow, + + /// WebM output serialization failed. + WebmWriteFailed, + + /// The job was cancelled. + Cancelled, + + /// The worker task terminated unexpectedly. + InternalError, +} + +/// Snapshot delivered through the media job SSE stream. +#[derive(Clone, Debug, Serialize)] +pub struct MediaJobEvent { + /// Current lifecycle phase. + pub phase: MediaJobPhase, + + /// Optional progress fraction between zero and one. + pub progress: Option, + + /// Terminal result, present only for completed jobs. + pub result: Option, + + /// Terminal diagnostic, present only for failed jobs. + pub error: Option, +} + +/// Successful normalized-media result. +#[derive(Clone, Debug, Serialize)] +pub struct MediaJobResult { + /// Path at which the normalized output was committed. + pub output_path: String, + + /// Stable container produced for provider uploads. + pub output_format: String, + + /// Stable audio codec produced for provider uploads. + pub output_codec: String, + + /// Human-readable detected container description for diagnostics. + pub detected_format: String, + + /// Human-readable selected codec description for diagnostics. + pub detected_codec: String, + + /// Duration of the normalized playable audio. + pub duration_ms: u64, + + /// Whether the input was copied unchanged. + pub pass_through: bool, + + /// Whether the normalized audio exceeds the practical-silence threshold. + pub has_audible_signal: bool, +} + +/// Stable error code plus an English diagnostic intended for logs. +#[derive(Clone, Debug, Serialize)] +pub struct MediaError { + /// Machine-readable failure category used for localization by the client. + pub code: MediaErrorCode, + + /// US-English diagnostic detail for logging and support. + pub message: String, +} + +impl MediaError { + /// Creates a media error without exposing free-form codes on the wire. + fn new(code: MediaErrorCode, message: impl Into) -> Self { + Self { code, message: message.into() } + } +} + +/// Mutable state shared by the request routes and blocking worker. +struct MediaJob { + /// Cooperative cancellation flag checked at bounded intervals. + cancelled: Arc, + + /// The latest snapshot replayed to a newly connected SSE subscriber. + current: Mutex, + + /// Fan-out channel for live state changes. + events: broadcast::Sender, + + /// Last running progress publication, used to protect Blazor from render storms. + last_progress: Mutex>, +} + +impl MediaJob { + /// Creates a job in the probing phase before its worker is scheduled. + fn new() -> Self { + let initial = MediaJobEvent { + phase: MediaJobPhase::Probing, + progress: Some(0.0), + result: None, + error: None, + }; + + let (events, _) = broadcast::channel(32); + Self { + cancelled: Arc::new(AtomicBool::new(false)), + current: Mutex::new(initial), + events, + last_progress: Mutex::new(None), + } + } + + /// Replaces the replay snapshot and notifies all live subscribers. + fn publish(&self, event: MediaJobEvent) { + *self.current.lock().unwrap() = event.clone(); + let _ = self.events.send(event); + } + + /// Publishes running progress no more than once per interval and always on a phase change. + fn publish_progress(&self, phase: MediaJobPhase, progress: Option) { + let now = Instant::now(); + let mut last = self.last_progress.lock().unwrap(); + if last.is_some_and(|(last_phase, last_at)| last_phase == phase && now.duration_since(last_at) < PROGRESS_EVENT_INTERVAL) { + return; + } + + *last = Some((phase, now)); + drop(last); + self.publish(MediaJobEvent { phase, progress, result: None, error: None }); + } + + /// Returns whether cooperative cancellation was requested. + fn is_cancelled(&self) -> bool { + self.cancelled.load(Ordering::Relaxed) + } +} + +/// Registers and immediately schedules a media normalization job. +pub async fn create_job( + _token: APIToken, + Json(request): Json, +) -> Result, (StatusCode, Json)> { + let input_path = PathBuf::from(&request.input_path); + if !input_path.is_file() { + return Err((StatusCode::BAD_REQUEST, Json(MediaError::new(MediaErrorCode::FileNotFound, "The selected media file does not exist.")))); + } + + let output_path = request.output_path.map(PathBuf::from).unwrap_or_else(|| { + let parent = input_path.parent().unwrap_or_else(|| FilePath::new(".")); + let stem = input_path.file_stem().and_then(|value| value.to_str()).unwrap_or("media"); + parent.join(format!("{stem}-normalized.webm")) + }); + + let job_id = format!("{}-{}", std::process::id(), rand::random::()); + let job = Arc::new(MediaJob::new()); + JOBS.write().unwrap().insert(job_id.clone(), Arc::clone(&job)); + let completed_job_id = job_id.clone(); + + tauri::async_runtime::spawn(async move { + let started_at = Instant::now(); + log::info!("media job registered: job_id={completed_job_id}"); + let max_pass_through_bytes = request.max_pass_through_bytes.unwrap_or(DEFAULT_MAX_PASS_THROUGH_BYTES); + let task_job = Arc::clone(&job); + let result = tokio::task::spawn_blocking(move || normalize_media(&input_path, &output_path, max_pass_through_bytes, &task_job)).await; + match result { + Ok(Ok(result)) => { + log::info!("media job completed: job_id={completed_job_id}, elapsed_ms={}", started_at.elapsed().as_millis()); + job.publish(MediaJobEvent { + phase: MediaJobPhase::Completed, + progress: Some(1.0), + result: Some(result), + error: None, + }); + } + + Ok(Err(error)) if error.code == MediaErrorCode::Cancelled => { + log::info!("media job cancelled: job_id={completed_job_id}, elapsed_ms={}", started_at.elapsed().as_millis()); + job.publish(MediaJobEvent { + phase: MediaJobPhase::Cancelled, + progress: None, + result: None, + error: None, + }); + } + + Ok(Err(error)) => { + log::error!("media job failed: job_id={completed_job_id}, code={:?}, diagnostic={}, elapsed_ms={}", error.code, error.message, started_at.elapsed().as_millis()); + job.publish(MediaJobEvent { + phase: MediaJobPhase::Failed, + progress: None, + result: None, + error: Some(error), + }); + } + + Err(error) => job.publish(MediaJobEvent { + phase: MediaJobPhase::Failed, + progress: None, + result: None, + error: Some(MediaError::new(MediaErrorCode::InternalError, format!("The media worker failed: {error}"))), + }), + } + + retain_terminal_job(completed_job_id).await; + }); + + Ok(Json(CreateMediaJobResponse { job_id })) +} + +/// Retains a terminal job for late SSE subscribers, then removes it asynchronously. +/// +/// Retention starts only after the worker has published a terminal event. Sleeping here neither +/// blocks the originating request nor the blocking media worker. +async fn retain_terminal_job(job_id: String) { + tokio::time::sleep(TERMINAL_JOB_RETENTION).await; + JOBS.write().unwrap().remove(&job_id); +} + +/// Streams the current snapshot followed by live media job events. +pub async fn get_job_events( + _token: APIToken, + Path(job_id): Path, +) -> Result>>, StatusCode> { + let job = JOBS.read().unwrap().get(&job_id).cloned().ok_or(StatusCode::NOT_FOUND)?; + let current = job.current.lock().unwrap().clone(); + let initial = tokio_stream::once(current); + let updates = BroadcastStream::new(job.events.subscribe()).filter_map(|event| event.ok()); + let stream = initial.chain(updates).map(|event| { + let data = serde_json::to_string(&event).unwrap_or_else(|_| "{}".to_string()); + Ok(Event::default().event(phase_name(&event.phase)).data(data)) + }); + + Ok(Sse::new(stream).keep_alive(KeepAlive::default())) +} + +/// Requests cooperative cancellation of a running media job. +pub async fn cancel_job(_token: APIToken, Path(job_id): Path) -> impl IntoResponse { + match JOBS.read().unwrap().get(&job_id) { + Some(job) => { + job.cancelled.store(true, Ordering::Relaxed); + StatusCode::NO_CONTENT + } + + None => StatusCode::NOT_FOUND, + } +} + +/// Maps a phase to the corresponding SSE event name. +fn phase_name(phase: &MediaJobPhase) -> &'static str { + match phase { + MediaJobPhase::Probing => "probing", + MediaJobPhase::Transcoding => "transcoding", + MediaJobPhase::Completed => "completed", + MediaJobPhase::Failed => "failed", + MediaJobPhase::Cancelled => "cancelled", + } +} + +/// Shared byte position retained after the source is moved into Symphonia. +#[derive(Clone)] +struct SourceProgress { + bytes_read: Arc, + length: u64, +} + +impl SourceProgress { + /// Returns monotonically clamped sequential read progress. + fn fraction(&self) -> Option { + (self.length > 0).then(|| (self.bytes_read.load(Ordering::Relaxed) as f64 / self.length as f64).clamp(0.0, 0.99)) + } +} + +/// File source that checks cancellation inside every read and seek operation. +struct CancellationMediaSource { + file: File, + cancelled: Arc, + bytes_read: Arc, + length: u64, +} + +impl CancellationMediaSource { + /// Wraps a regular file and exposes a progress handle to the transcoder. + fn new(file: File, cancelled: Arc) -> std::io::Result<(Self, SourceProgress)> { + let length = file.metadata()?.len(); + let bytes_read = Arc::new(AtomicU64::new(0)); + let progress = SourceProgress { bytes_read: Arc::clone(&bytes_read), length }; + Ok((Self { file, cancelled, bytes_read, length }, progress)) + } + + /// Converts cancellation into an interrupted I/O operation understood by the reader. + fn check_cancelled(&self) -> std::io::Result<()> { + if self.cancelled.load(Ordering::Relaxed) { + Err(std::io::Error::new(std::io::ErrorKind::Interrupted, "media job cancelled")) + } else { + Ok(()) + } + } +} + +impl Read for CancellationMediaSource { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + self.check_cancelled()?; + let count = self.file.read(buffer)?; + self.bytes_read.fetch_add(count as u64, Ordering::Relaxed); + self.check_cancelled()?; + Ok(count) + } +} + +impl Seek for CancellationMediaSource { + fn seek(&mut self, position: SeekFrom) -> std::io::Result { + self.check_cancelled()?; + let position = self.file.seek(position)?; + self.check_cancelled()?; + Ok(position) + } +} + +impl MediaSource for CancellationMediaSource { + fn is_seekable(&self) -> bool { + true + } + + fn byte_len(&self) -> Option { + Some(self.length) + } +} + +/// Probes, normalizes, and atomically commits one media file. +fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_through_bytes: u64, job: &MediaJob) -> Result { + check_cancelled(job)?; + + let detected = FileFormat::from_file(input_path) + .map_err(|error| MediaError::new(MediaErrorCode::UnknownFormat, format!("The file type could not be identified: {error}")))?; + + if detected.kind() == Kind::Executable { + return Err(MediaError::new(MediaErrorCode::UnsafeFile, "The selected file contains executable data and cannot be processed as media.")); + } + + if !matches!(detected.kind(), Kind::Audio | Kind::Video) + && !matches!( + detected, + FileFormat::ExtensibleBinaryMetaLanguage + | FileFormat::Id3v2 + | FileFormat::Mpeg4Part14 + | FileFormat::Mpeg4Part14Audio + | FileFormat::Mpeg4Part14Video + ) + && !has_supported_media_extension(input_path) + { + return Err(MediaError::new(MediaErrorCode::NotMedia, format!("The selected file is not supported media (detected as {detected:?})."))); + } + + let file_size = input_path.metadata().map(|metadata| metadata.len()).unwrap_or(0); + log::info!("media job started: input='{}', output='{}', size_bytes={}, detected_file_format={detected:?}", input_path.display(), output_path.display(), file_size); + + let file = File::open(input_path).map_err(|error| MediaError::new(MediaErrorCode::FileOpenFailed, error.to_string()))?; + let (source, source_progress) = CancellationMediaSource::new(file, Arc::clone(&job.cancelled)) + .map_err(|error| MediaError::new(MediaErrorCode::FileOpenFailed, error.to_string()))?; + let mss = MediaSourceStream::new(Box::new(source), Default::default()); + + let mut hint = Hint::new(); + if let Some(extension) = input_path.extension().and_then(|value| value.to_str()) { + hint.with_extension(extension); + } + + let mut format = match symphonia::default::get_probe() + .probe(&hint, mss, FormatOptions::default(), MetadataOptions::default()) + { + Ok(format) => format, + Err(_) if job.is_cancelled() => return Err(MediaError::new(MediaErrorCode::Cancelled, "The media job was cancelled.")), + Err(error) => return Err(map_probe_error(error)), + }; + + let detected_format = format!("{detected:?} / {}", format.format_info().long_name); + + let tracks = format.tracks(); + for track in tracks { + if let Some(params) = track.codec_params.as_ref().and_then(CodecParameters::audio) { + log::info!( + "media track: id={}, type={:?}, default={}, codec={}, sample_rate={:?}, channels={:?}, duration={:?}, time_base={:?}", + track.id, + track.track_type(), + track.flags.contains(TrackFlags::DEFAULT), + params.codec, + params.sample_rate, + params.channels.as_ref().map(|channels| channels.count()), + track.duration, + track.time_base, + ); + } else { + log::info!("media track: id={}, type={:?}, default={}, non_audio=true", track.id, track.track_type(), track.flags.contains(TrackFlags::DEFAULT)); + } + } + if !tracks.iter().any(is_audio_track) { + return Err(MediaError::new(MediaErrorCode::NoAudioTrack, "The selected media file does not contain an audio track.")); + } + + let selected = select_audio_track(tracks) + .ok_or_else(|| MediaError::new(MediaErrorCode::UnsupportedCodec, "None of the audio tracks uses a supported codec."))?; + + let track_id = selected.id; + let track_delay = selected.delay.unwrap_or(0); + let track_padding = selected.padding.unwrap_or(0); + let track_time_base = selected.time_base; + let params = selected.codec_params.as_ref().and_then(CodecParameters::audio).unwrap().clone(); + let detected_codec = if params.codec == CODEC_ID_OPUS { "opus".to_string() } else { format!("{}", params.codec) }; + let track_duration_ms = selected.num_frames.zip(params.sample_rate) + .map(|(frames, rate)| frames.saturating_mul(1_000) / u64::from(rate)) + .or_else(|| selected.time_base.zip(selected.duration).and_then(|(time_base, duration)| { + let timestamp = Timestamp::new(i64::try_from(duration.get()).ok()?); + let time = time_base.calc_time(timestamp)?; + Some((time.as_secs_f64() * 1000.0).max(0.0).round() as u64) + })); + let container_duration_ms = format.media_info().time_base.zip(format.media_info().duration).and_then(|(time_base, duration)| { + let timestamp = Timestamp::new(i64::try_from(duration.get()).ok()?); + let time = time_base.calc_time(timestamp)?; + Some((time.as_secs_f64() * 1000.0).max(0.0).round() as u64) + }); + let duration_ms = track_duration_ms.or(container_duration_ms).unwrap_or(0); + log::info!( + "media audio selection: track_id={}, default={}, track_duration_ms={:?}, container_duration_ms={:?}, progress_duration_ms={}", + track_id, + selected.flags.contains(TrackFlags::DEFAULT), + track_duration_ms, + container_duration_ms, + duration_ms, + ); + + let channels = params.channels.as_ref().map(|value| value.count()).unwrap_or(0); + let pass_through = is_webm_container(input_path) + && tracks.len() == 1 + && selected.track_type() == Some(TrackType::Audio) + && params.codec == CODEC_ID_OPUS + && params.sample_rate == Some(OUTPUT_SAMPLE_RATE) + && channels == 1 + && input_path.metadata().map(|metadata| metadata.len() <= max_pass_through_bytes).unwrap_or(false); + log::info!("media normalization decision: track_id={track_id}, pass_through={pass_through}, codec={detected_codec}, channels={channels}"); + + let partial_path = partial_path(output_path); + if let Some(parent) = partial_path.parent() { + fs::create_dir_all(parent).map_err(|error| MediaError::new(MediaErrorCode::OutputCreateFailed, error.to_string()))?; + } + + let result = if pass_through { + job.publish_progress(MediaJobPhase::Transcoding, Some(0.0)); + let analysis_context = SignalAnalysisContext { + track_id, + params: ¶ms, + track_delay, + track_padding, + expected_duration_ms: duration_ms, + time_base: track_time_base, + source_progress: &source_progress, + job, + }; + let signal = analyze_audio_signal(&mut *format, analysis_context)?; + copy_with_cancellation(input_path, &partial_path, job)?; + Ok(MediaJobResult { + output_path: output_path.to_string_lossy().into_owned(), + output_format: OUTPUT_FORMAT.to_string(), + output_codec: OUTPUT_CODEC.to_string(), + detected_format: detected_format.clone(), + detected_codec, + duration_ms, + pass_through: true, + has_audible_signal: signal.has_audible_signal(), + }) + } else { + job.publish_progress(MediaJobPhase::Transcoding, Some(0.0)); + + let context = TranscodeContext { + track_id, + track_delay, + track_padding, + params, + partial_path: &partial_path, + output_path, + detected_format, + detected_codec, + expected_duration_ms: duration_ms, + time_base: track_time_base, + source_progress, + job, + }; + transcode(&mut *format, context) + }; + + match result { + Ok(result) => { + if let Err(error) = fs::rename(&partial_path, output_path) { + let _ = fs::remove_file(&partial_path); + return Err(MediaError::new(MediaErrorCode::OutputCommitFailed, error.to_string())); + } + + Ok(result) + } + + Err(error) => { + let _ = fs::remove_file(&partial_path); + Err(error) + } + } +} + +/// Recognizes extensions for containers supported by the configured Symphonia readers. +fn has_supported_media_extension(path: &FilePath) -> bool { + path.extension() + .and_then(|extension| extension.to_str()) + .is_some_and(|extension| matches!( + extension.to_ascii_lowercase().as_str(), + "aac" | "aif" | "aiff" | "caf" | "flac" | "m4a" | "mka" | "mkv" | "mov" + | "mp1" | "mp2" | "mp3" | "mp4" | "oga" | "ogg" | "opus" | "wav" | "webm" + )) +} + +/// Returns whether a track explicitly contains audio codec parameters. +fn is_audio_track(track: &Track) -> bool { + matches!(track.codec_params, Some(CodecParameters::Audio(_))) +} + +/// Selects the decodable default audio track, falling back to the first decodable audio track. +fn select_audio_track(tracks: &[Track]) -> Option<&Track> { + tracks + .iter() + .filter(|track| is_audio_track(track) && is_decodable(track)) + .min_by_key(|track| !track.flags.contains(TrackFlags::DEFAULT)) +} + +/// Checks whether the runtime can construct a decoder for the track. +fn is_decodable(track: &Track) -> bool { + let Some(params) = track.codec_params.as_ref().and_then(CodecParameters::audio) else { return false; }; + params.codec == CODEC_ID_OPUS || symphonia::default::get_codecs().make_audio_decoder(params, &AudioDecoderOptions::default()).is_ok() +} + +/// Streaming peak measurement over normalized full-scale floating-point samples. +#[derive(Default)] +struct AudioPeakDetector { + /// Highest absolute sample observed so far. + max_amplitude: f32, +} + +impl AudioPeakDetector { + /// Includes one bounded sample block in the maximum-peak measurement. + fn observe(&mut self, samples: &[f32]) { + for sample in samples { + let amplitude = sample.abs(); + self.max_amplitude = if amplitude.is_finite() { + self.max_amplitude.max(amplitude) + } else { + f32::INFINITY + }; + } + } + + /// Returns whether any retained sample exceeds the configured silence ceiling. + fn has_audible_signal(&self) -> bool { + self.max_amplitude > 10.0_f32.powf(SILENCE_MAX_PEAK_DBFS / 20.0) + } + + /// Returns the measured full-scale peak for diagnostics. + fn max_peak_dbfs(&self) -> f32 { + if self.max_amplitude == 0.0 { + f32::NEG_INFINITY + } else { + 20.0 * self.max_amplitude.log10() + } + } +} + +/// Inputs required to scan an otherwise pass-through-compatible audio track. +struct SignalAnalysisContext<'a> { + /// Selected track identifier. + track_id: u32, + + /// Selected track codec parameters. + params: &'a symphonia::core::codecs::audio::AudioCodecParameters, + + /// Leading decoded frames to discard. + track_delay: u32, + + /// Trailing decoded frames to discard. + track_padding: u32, + + /// Container duration used for progress reporting. + expected_duration_ms: u64, + + /// Selected track timebase used for progress reporting. + time_base: Option, + + /// Sequential byte progress fallback when the track has no duration. + source_progress: &'a SourceProgress, + + /// Cancellation and progress state for the job. + job: &'a MediaJob, +} + +/// Decodes an otherwise pass-through-compatible track solely to classify practical silence. +fn analyze_audio_signal( + format: &mut dyn symphonia::core::formats::FormatReader, + context: SignalAnalysisContext<'_>, +) -> Result { + let mut decoder = StreamDecoder::new(context.params, context.track_delay)?; + let mut detector = AudioPeakDetector::default(); + let mut decoded_tail = Vec::::new(); + let mut first_packet_pts = None::; + let mut decoded_packets = 0u64; + let mut last_progress = 0.0f64; + + loop { + check_cancelled(context.job)?; + let packet = match format.next_packet() { + Ok(Some(packet)) => packet, + Ok(None) => break, + + Err(SymphoniaError::ResetRequired) => return Err(MediaError::new(MediaErrorCode::StreamReset, "The media stream changed unexpectedly.")), + Err(SymphoniaError::IoError(error)) if error.kind() == std::io::ErrorKind::Interrupted && context.job.is_cancelled() => { + return Err(MediaError::new(MediaErrorCode::Cancelled, "The media job was cancelled.")); + } + + Err(SymphoniaError::IoError(error)) if error.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(error) => return Err(MediaError::new(MediaErrorCode::DamagedContainer, format!("The media container is damaged: {error}"))), + }; + + if packet.track_id != context.track_id { + continue; + } + + let packet_pts = packet.pts.get(); + first_packet_pts.get_or_insert(packet_pts); + let Some((mono, _)) = decoder.decode(&packet)? else { continue; }; + decoded_packets += 1; + + decoded_tail.extend_from_slice(&mono); + let emit_len = decoded_tail.len().saturating_sub(context.track_padding as usize); + if emit_len > 0 { + detector.observe(&decoded_tail[..emit_len]); + drop(decoded_tail.drain(..emit_len)); + } + + let timestamp_ms = packet_timestamp_ms(packet_pts, first_packet_pts, context.time_base); + let progress = if context.expected_duration_ms > 0 { + timestamp_ms.map(|current_ms| (current_ms as f64 / context.expected_duration_ms as f64).clamp(0.0, 0.99)) + } else { + context.source_progress.fraction() + }; + + if let Some(progress) = progress { + last_progress = last_progress.max(progress); + } + + context.job.publish_progress(MediaJobPhase::Transcoding, progress.map(|_| last_progress)); + } + + if decoded_packets == 0 { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The selected audio track did not yield decoded audio samples.")); + } + + log::info!( + "media signal analysis completed: track_id={}, max_peak_dbfs={}, silence_threshold_dbfs={}, has_audible_signal={}", + context.track_id, + detector.max_peak_dbfs(), + SILENCE_MAX_PEAK_DBFS, + detector.has_audible_signal(), + ); + + Ok(detector) +} + +/// Immutable inputs shared across a single transcoding operation. +struct TranscodeContext<'a> { + /// Selected input track identifier. + track_id: u32, + + /// Leading decoded frames to discard. + track_delay: u32, + + /// Trailing decoded frames to discard. + track_padding: u32, + + /// Selected track codec parameters. + params: symphonia::core::codecs::audio::AudioCodecParameters, + + /// Temporary output path used until the job succeeds. + partial_path: &'a FilePath, + + /// Final output path returned in the result. + output_path: &'a FilePath, + + /// Detected container diagnostic. + detected_format: String, + + /// Detected codec diagnostic. + detected_codec: String, + + /// Container duration used for progress reporting. + expected_duration_ms: u64, + + /// Selected track timebase used to align packet presentation timestamps. + time_base: Option, + + /// Sequential byte progress fallback when the selected track has no duration. + source_progress: SourceProgress, + + /// Cancellation and progress state for the job. + job: &'a MediaJob, +} + +/// Decodes a selected track and writes timestamp-aligned 20 ms mono Opus frames. +fn transcode( + format: &mut dyn symphonia::core::formats::FormatReader, + context: TranscodeContext<'_>, +) -> Result { + let mut decoder = StreamDecoder::new(&context.params, context.track_delay)?; + let mut opus_encoder = OpusEncoder::builder(OUTPUT_SAMPLE_RATE, OpusChannels::Mono, Application::Audio) + .bitrate(Bitrate::Bits(OPUS_BITRATE)) + .vbr(true) + .build() + .map_err(|error| MediaError::new(MediaErrorCode::EncoderInitFailed, error.to_string()))?; + + let file = File::create(context.partial_path).map_err(|error| MediaError::new(MediaErrorCode::OutputCreateFailed, error.to_string()))?; + let mut writer = WebmOpusWriter::new(file)?; + let mut pending = Vec::::with_capacity(OPUS_FRAME_SAMPLES * 3); + let mut signal = AudioPeakDetector::default(); + let mut resampler: Option = None; + let mut decoded_tail = Vec::::new(); + let mut encoded = [0u8; 4_000]; + let mut produced_samples = 0u64; + let mut first_packet_pts = None::; + let mut last_packet_pts = None::; + let mut decoded_packets = 0u64; + let mut discarded_packets = 0u64; + let mut decode_errors = 0u64; + let mut discontinuities = 0u64; + let mut last_progress = 0.0f64; + + loop { + check_cancelled(context.job)?; + let packet = match format.next_packet() { + Ok(Some(packet)) => packet, + Ok(None) => break, + + Err(SymphoniaError::ResetRequired) => return Err(MediaError::new(MediaErrorCode::StreamReset, "The media stream changed unexpectedly.")), + Err(SymphoniaError::IoError(error)) if error.kind() == std::io::ErrorKind::Interrupted && context.job.is_cancelled() => { + return Err(MediaError::new(MediaErrorCode::Cancelled, "The media job was cancelled.")); + } + + Err(SymphoniaError::IoError(error)) if error.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(error) => return Err(MediaError::new(MediaErrorCode::DamagedContainer, format!("The media container is damaged: {error}"))), + }; + + if packet.track_id != context.track_id { + discarded_packets += 1; + continue; + } + + let packet_pts = packet.pts.get(); + first_packet_pts.get_or_insert(packet_pts); + last_packet_pts = Some(packet_pts); + let Some((mono, sample_rate)) = decoder.decode(&packet)? else { + decode_errors += 1; + continue; + }; + + decoded_packets += 1; + let stream_resampler = match resampler.as_mut() { + Some(existing) if existing.input_rate() == sample_rate => existing, + Some(_) => return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The decoded audio sample rate changed during the stream.")), + + None => { + log::info!("media resampling: input_rate={sample_rate}, output_rate={OUTPUT_SAMPLE_RATE}, enabled={}", sample_rate != OUTPUT_SAMPLE_RATE); + resampler.insert(StreamResampler::new(sample_rate)?) + } + }; + + // Retain only the possible end padding so it can never be emitted prematurely. + decoded_tail.extend_from_slice(&mono); + let padding = context.track_padding as usize; + let emit_len = decoded_tail.len().saturating_sub(padding); + if emit_len > 0 { + let emit: Vec<_> = decoded_tail.drain(..emit_len).collect(); + let resampled = stream_resampler.push(&emit)?; + + // FFT resamplers buffer across packet boundaries, so their returned samples no longer + // begin at the current packet PTS. Native 48-kHz streams retain exact packet alignment. + let desired_start = (sample_rate == OUTPUT_SAMPLE_RATE) + .then(|| packet_output_start(packet_pts, first_packet_pts, context.time_base)) + .flatten(); + + let current_start = produced_samples.saturating_add(pending.len() as u64); + let previous_pending_len = pending.len(); + append_timestamp_aligned( + &mut pending, + &resampled, + current_start, + desired_start, + context.track_id, + packet_pts, + &mut discontinuities, + ); + + signal.observe(&pending[previous_pending_len..]); + } + + encode_complete_frames(&mut pending, &mut opus_encoder, &mut writer, &mut encoded, &mut produced_samples, context.job)?; + + let timestamp_ms = packet_timestamp_ms(packet_pts, first_packet_pts, context.time_base); + let progress = if context.expected_duration_ms > 0 { + timestamp_ms.map(|current_ms| (current_ms as f64 / context.expected_duration_ms as f64).clamp(0.0, 0.99)) + } else { + context.source_progress.fraction() + }; + + if let Some(progress) = progress { + last_progress = last_progress.max(progress); + } + + context.job.publish_progress(MediaJobPhase::Transcoding, progress.map(|_| last_progress)); + } + + if let Some(stream_resampler) = resampler.as_mut() { + // Discard the retained decoded tail (container padding), then flush the filter delay. + let flushed = stream_resampler.finish()?; + signal.observe(&flushed); + pending.extend_from_slice(&flushed); + } else { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The selected audio track did not yield decoded audio parameters.")); + } + + encode_complete_frames(&mut pending, &mut opus_encoder, &mut writer, &mut encoded, &mut produced_samples, context.job)?; + + if !pending.is_empty() { + pending.resize(OPUS_FRAME_SAMPLES, 0.0); + let length = opus_encoder.encode_float(&pending, &mut encoded) + .map_err(|error| MediaError::new(MediaErrorCode::EncodeFailed, error.to_string()))?; + writer.write_packet(&encoded[..length], produced_samples)?; + produced_samples += OPUS_FRAME_SAMPLES as u64; + } + + writer.finish()?; + check_cancelled(context.job)?; + + let output_size = fs::metadata(context.partial_path).map(|metadata| metadata.len()).unwrap_or(0); + let output_duration_ms = produced_samples.saturating_mul(1000) / u64::from(OUTPUT_SAMPLE_RATE); + if context.expected_duration_ms > 0 && output_duration_ms.abs_diff(context.expected_duration_ms) > 2_000 { + log::warn!( + "media duration mismatch: track_id={}, expected_duration_ms={}, decoded_duration_ms={}", + context.track_id, + context.expected_duration_ms, + output_duration_ms, + ); + } + + log::info!( + "media transcode completed: track_id={}, decoded_packets={}, discarded_packets={}, recoverable_decode_errors={}, first_pts={:?}, last_pts={:?}, discontinuities={}, output_bytes={}, duration_ms={}, max_peak_dbfs={}, silence_threshold_dbfs={}, has_audible_signal={}", + context.track_id, + decoded_packets, + discarded_packets, + decode_errors, + first_packet_pts, + last_packet_pts, + discontinuities, + output_size, + output_duration_ms, + signal.max_peak_dbfs(), + SILENCE_MAX_PEAK_DBFS, + signal.has_audible_signal(), + ); + + Ok(MediaJobResult { + output_path: context.output_path.to_string_lossy().into_owned(), + output_format: OUTPUT_FORMAT.to_string(), + output_codec: OUTPUT_CODEC.to_string(), + detected_format: context.detected_format, + detected_codec: context.detected_codec, + duration_ms: output_duration_ms, + pass_through: false, + has_audible_signal: signal.has_audible_signal(), + }) +} + +/// Converts a packet PTS to its output sample offset relative to the first audio packet. +fn packet_output_start(packet_pts: i64, first_packet_pts: Option, time_base: Option) -> Option { + packet_timestamp_ms(packet_pts, first_packet_pts, time_base) + .map(|milliseconds| milliseconds.saturating_mul(u64::from(OUTPUT_SAMPLE_RATE)) / 1_000) +} + +/// Converts a packet PTS to milliseconds relative to the first selected-track packet. +fn packet_timestamp_ms(packet_pts: i64, first_packet_pts: Option, time_base: Option) -> Option { + let delta = packet_pts.checked_sub(first_packet_pts?)?; + if delta < 0 { + return Some(0); + } + + let time = time_base?.calc_time(Timestamp::new(delta))?; + Some((time.as_secs_f64() * 1_000.0).max(0.0).round() as u64) +} + +/// Inserts silence for forward timestamp gaps and trims overlapping decoded samples. +fn append_timestamp_aligned( + pending: &mut Vec, + samples: &[f32], + current_start: u64, + desired_start: Option, + track_id: u32, + packet_pts: i64, + discontinuities: &mut u64, +) { + let Some(desired_start) = desired_start else { + pending.extend_from_slice(samples); + return; + }; + + let delta = i128::from(desired_start) - i128::from(current_start); + let delta_ms = delta.saturating_mul(1_000) / i128::from(OUTPUT_SAMPLE_RATE); + if delta_ms.unsigned_abs() >= LARGE_DISCONTINUITY_MS as u128 { + *discontinuities += 1; + log::warn!("audio timestamp discontinuity: track_id={track_id}, pts={packet_pts}, delta_ms={delta_ms}"); + } + + if delta > 0 { + let silence = usize::try_from(delta).unwrap_or(usize::MAX); + pending.resize(pending.len().saturating_add(silence), 0.0); + pending.extend_from_slice(samples); + } else { + let overlap = usize::try_from(delta.unsigned_abs()).unwrap_or(usize::MAX).min(samples.len()); + pending.extend_from_slice(&samples[overlap..]); + } +} + +/// Downmixes interleaved PCM to mono using a deterministic arithmetic mean. +fn downmix_to_mono(samples: &[f32], channels: usize) -> Vec { + if channels <= 1 { + return samples.to_vec(); + } + + samples.chunks_exact(channels).map(|frame| frame.iter().copied().sum::() / channels as f32).collect() +} + +/// Parsed subset of an Opus identification header supported by the single-stream decoder. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +struct OpusHeader { + /// Mono or stereo channel count. + channels: usize, + + /// Number of leading decoded frames to discard at 48 kHz. + pre_skip: u16, +} + +impl OpusHeader { + /// Parses and validates a mono/stereo, mapping-family-zero `OpusHead` packet. + fn parse(data: &[u8]) -> Result { + if data.len() < 19 || &data[..8] != b"OpusHead" || data[8] > 15 || !matches!(data[9], 1 | 2) { + return Err(MediaError::new(MediaErrorCode::InvalidOpusHeader, "The Opus identification header is invalid.")); + } + + if data[18] != 0 { + return Err(MediaError::new(MediaErrorCode::UnsupportedOpusMapping, "The Opus channel mapping requires unsupported multistream decoding.")); + } + + Ok(Self { + channels: usize::from(data[9]), + pre_skip: u16::from_le_bytes([data[10], data[11]]), + }) + } +} + +/// Ropus adapter that validates Symphonia parameters and applies Opus pre-skip exactly once. +struct RopusOpusDecoder { + /// Underlying libopus single-stream decoder. + decoder: OpusDecoder, + + /// Number of interleaved channels produced by the decoder. + channels: usize, + + /// Remaining leading frames to discard. + delay_remaining: usize, + + /// Reused bounded PCM output buffer. + pcm: Vec, +} + +impl RopusOpusDecoder { + /// Builds a mono/stereo decoder from the codec's `OpusHead` private data. + fn new(params: &symphonia::core::codecs::audio::AudioCodecParameters, track_delay: u32) -> Result { + let header = OpusHeader::parse(params.extra_data.as_deref().unwrap_or_default())?; + let declared_channels = params.channels.as_ref().map(|channels| channels.count()) + .ok_or_else(|| MediaError::new(MediaErrorCode::InvalidAudioParameters, "The Opus stream does not declare its channel count."))?; + + if declared_channels != header.channels || params.sample_rate != Some(OUTPUT_SAMPLE_RATE) { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The Opus stream parameters do not match its identification header.")); + } + + let opus_channels = if header.channels == 1 { OpusChannels::Mono } else { OpusChannels::Stereo }; + let decoder = OpusDecoder::new(OUTPUT_SAMPLE_RATE, opus_channels) + .map_err(|error| MediaError::new(MediaErrorCode::DecoderInitFailed, error.to_string()))?; + + let delay = if track_delay == 0 { u32::from(header.pre_skip) } else { track_delay }; + + Ok(Self { + decoder, + channels: header.channels, + delay_remaining: delay as usize, + pcm: vec![0; 5_760 * header.channels], + }) + } + + /// Decodes one Opus packet, downmixes it, and removes leading codec delay. + fn decode(&mut self, packet: &[u8]) -> Result, MediaError> { + let frames = self.decoder.decode(packet, &mut self.pcm, DecodeMode::Normal) + .map_err(|error| MediaError::new(MediaErrorCode::DecodeFailed, error.to_string()))?; + + let skip = self.delay_remaining.min(frames); + self.delay_remaining -= skip; + + let samples = &self.pcm[skip * self.channels..frames * self.channels]; + let interleaved: Vec<_> = samples.iter().map(|sample| f32::from(*sample) / 32_768.0).collect(); + Ok(downmix_to_mono(&interleaved, self.channels)) + } +} + +/// Decoder abstraction allowing Symphonia demuxing with either its native decoders or Ropus. +enum StreamDecoder { + /// Decoder supplied by Symphonia for non-Opus codecs. + Symphonia { + /// Stateful codec decoder. + decoder: Box, + + /// Reused interleaved PCM buffer. + interleaved: Vec, + + /// First observed decoded sample rate. + sample_rate: Option, + + /// First observed decoded channel count. + channels: Option, + + /// Remaining track delay to discard. + delay_remaining: usize, + }, + + /// Symphonia-compatible Opus packet adapter backed by Ropus. + Opus(Box), +} + +impl StreamDecoder { + /// Creates the appropriate decoder without guessing missing stream parameters. + fn new(params: &symphonia::core::codecs::audio::AudioCodecParameters, track_delay: u32) -> Result { + if params.codec == CODEC_ID_OPUS { + return Ok(Self::Opus(Box::new(RopusOpusDecoder::new(params, track_delay)?))); + } + + let decoder = symphonia::default::get_codecs().make_audio_decoder(params, &AudioDecoderOptions::default()) + .map_err(|_| MediaError::new(MediaErrorCode::UnsupportedCodec, "The selected audio codec is not supported."))?; + + Ok(Self::Symphonia { + decoder, + interleaved: Vec::new(), + sample_rate: None, + channels: None, + delay_remaining: track_delay as usize, + }) + } + + /// Decodes one packet and returns mono PCM plus the actual decoder sample rate. + fn decode(&mut self, packet: &symphonia::core::packet::Packet) -> Result, u32)>, MediaError> { + match self { + Self::Opus(decoder) => Ok(Some((decoder.decode(&packet.data)?, OUTPUT_SAMPLE_RATE))), + + Self::Symphonia { decoder, interleaved, sample_rate, channels, delay_remaining } => { + let decoded = match decoder.decode(packet) { + Ok(decoded) => decoded, + Err(SymphoniaError::DecodeError(_)) => return Ok(None), + Err(error) => return Err(MediaError::new(MediaErrorCode::DecodeFailed, format!("Audio decoding failed: {error}"))), + }; + + let actual_rate = decoded.spec().rate(); + let actual_channels = decoded.spec().channels().count(); + + if actual_rate == 0 || actual_channels == 0 { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The decoder returned invalid audio parameters.")); + } + + if sample_rate.is_some_and(|rate| rate != actual_rate) || channels.is_some_and(|count| count != actual_channels) { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The decoded audio parameters changed during the stream.")); + } + + *sample_rate = Some(actual_rate); + *channels = Some(actual_channels); + interleaved.resize(decoded.samples_interleaved(), f32::MID); + decoded.copy_to_slice_interleaved(&mut *interleaved); + + let mono = downmix_to_mono(interleaved, actual_channels); + let skip = (*delay_remaining).min(mono.len()); + + *delay_remaining -= skip; + Ok(Some((mono[skip..].to_vec(), actual_rate))) + } + } + } +} + +/// Stateful, bounded mono resampler whose filter history spans decoder packets. +enum StreamResampler { + /// Zero-copy-rate path for already-48-kHz decoded PCM. + Passthrough { + /// Input rate retained for stream consistency checks. + input_rate: u32, + }, + + /// Rubato FFT resampler and its bounded pending input. + Rubato { + /// Input rate retained for stream consistency checks. + input_rate: u32, + + /// Stateful resampler instance used for the entire stream. + inner: Box>, + + /// Samples waiting to fill the next fixed input block. + pending: Vec, + + /// Startup-delay output frames still to discard. + delay_remaining: usize, + + /// Total real input frames accepted. + total_input: u64, + + /// Total trimmed output frames returned to the encoder. + total_output: u64, + }, +} + +impl StreamResampler { + /// Creates one resampler for the decoded stream. + fn new(input_rate: u32) -> Result { + if input_rate == 0 { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The decoded audio sample rate is missing.")); + } + + if input_rate == OUTPUT_SAMPLE_RATE { + return Ok(Self::Passthrough { input_rate }); + } + + let inner = Fft::::new( + input_rate as usize, + OUTPUT_SAMPLE_RATE as usize, + RESAMPLE_INPUT_BLOCK_SAMPLES, + 1, + FixedSync::Input, + ).map_err(|error| MediaError::new(MediaErrorCode::ResampleFailed, error.to_string()))?; + + let delay_remaining = inner.output_delay(); + + Ok(Self::Rubato { + input_rate, + inner: Box::new(inner), + pending: Vec::with_capacity(RESAMPLE_INPUT_BLOCK_SAMPLES * 2), + delay_remaining, + total_input: 0, + total_output: 0, + }) + } + + /// Returns the configured input sample rate. + fn input_rate(&self) -> u32 { + match self { + Self::Passthrough { input_rate } | Self::Rubato { input_rate, .. } => *input_rate, + } + } + + /// Accepts arbitrary packet-sized PCM and processes all complete bounded blocks. + fn push(&mut self, samples: &[f32]) -> Result, MediaError> { + match self { + Self::Passthrough { .. } => Ok(samples.to_vec()), + + Self::Rubato { inner, pending, delay_remaining, total_input, total_output, .. } => { + *total_input += samples.len() as u64; + pending.extend_from_slice(samples); + + let mut output = Vec::new(); + loop { + let block_len = inner.input_frames_next(); + if pending.len() < block_len { + break; + } + + let block: Vec<_> = pending.drain(..block_len).collect(); + append_resampled(inner, &block, None, delay_remaining, &mut output)?; + } + + *total_output += output.len() as u64; + Ok(output) + } + } + } + + /// Flushes the last partial block and filter delay, returning the exact rounded duration. + fn finish(&mut self) -> Result, MediaError> { + match self { + Self::Passthrough { .. } => Ok(Vec::new()), + + Self::Rubato { input_rate, inner, pending, delay_remaining, total_input, total_output } => { + let target = total_input.saturating_mul(u64::from(OUTPUT_SAMPLE_RATE)).div_ceil(u64::from(*input_rate)); + let mut output = Vec::new(); + + if !pending.is_empty() { + let valid = pending.len(); + let block_len = inner.input_frames_next(); + pending.resize(block_len, 0.0); + append_resampled(inner, pending, Some(valid), delay_remaining, &mut output)?; + pending.clear(); + } + + while *total_output + (output.len() as u64) < target { + let zeros = vec![0.0; inner.input_frames_next()]; + append_resampled(inner, &zeros, Some(0), delay_remaining, &mut output)?; + } + + output.truncate(target.saturating_sub(*total_output) as usize); + *total_output += output.len() as u64; + Ok(output) + } + } + } +} + +/// Processes one Rubato block and removes the resampler's startup delay. +fn append_resampled( + resampler: &mut Fft, + block: &[f32], + partial_len: Option, + delay_remaining: &mut usize, + destination: &mut Vec, +) -> Result<(), MediaError> { + let input_data = vec![block.to_vec()]; + let input = SequentialSliceOfVecs::new(&input_data, 1, block.len()) + .map_err(|error| MediaError::new(MediaErrorCode::ResampleFailed, error.to_string()))?; + + let indexing = partial_len.map(|length| Indexing::new().partial_len(length)); + let output = resampler.process(&input, indexing.as_ref()) + .map_err(|error| MediaError::new(MediaErrorCode::ResampleFailed, error.to_string()))?; + + let data = output.take_data(); + let skip = (*delay_remaining).min(data.len()); + *delay_remaining -= skip; + destination.extend_from_slice(&data[skip..]); + Ok(()) +} + +/// Encodes every complete 20 ms frame currently buffered. +fn encode_complete_frames( + pending: &mut Vec, + encoder: &mut OpusEncoder, + writer: &mut WebmOpusWriter, + encoded: &mut [u8], + produced_samples: &mut u64, + job: &MediaJob, +) -> Result<(), MediaError> { + let mut consumed = 0usize; + while pending.len().saturating_sub(consumed) >= OPUS_FRAME_SAMPLES { + check_cancelled(job)?; + let length = encoder.encode_float(&pending[consumed..consumed + OPUS_FRAME_SAMPLES], encoded) + .map_err(|error| MediaError::new(MediaErrorCode::EncodeFailed, error.to_string()))?; + writer.write_packet(&encoded[..length], *produced_samples)?; + *produced_samples += OPUS_FRAME_SAMPLES as u64; + consumed += OPUS_FRAME_SAMPLES; + } + + if consumed > 0 { + pending.drain(..consumed); + } + + Ok(()) +} + +/// Copies a pass-through input in bounded blocks with cooperative cancellation checks. +fn copy_with_cancellation(input_path: &FilePath, output_path: &FilePath, job: &MediaJob) -> Result<(), MediaError> { + let mut input = File::open(input_path).map_err(|error| MediaError::new(MediaErrorCode::FileOpenFailed, error.to_string()))?; + let mut output = File::create(output_path).map_err(|error| MediaError::new(MediaErrorCode::OutputCreateFailed, error.to_string()))?; + let mut buffer = [0u8; COPY_BLOCK_BYTES]; + + loop { + check_cancelled(job)?; + let count = input.read(&mut buffer).map_err(|error| MediaError::new(MediaErrorCode::FileOpenFailed, error.to_string()))?; + if count == 0 { + break; + } + + output.write_all(&buffer[..count]).map_err(|error| MediaError::new(MediaErrorCode::OutputWriteFailed, error.to_string()))?; + } + + output.flush().map_err(|error| MediaError::new(MediaErrorCode::OutputWriteFailed, error.to_string()))?; + check_cancelled(job) +} + +/// Minimal streaming WebM writer for one mono Opus track. +struct WebmOpusWriter { + /// EBML writer owning the partial output file. + writer: WebmWriter, + + /// Absolute timestamp of the current cluster in milliseconds. + cluster_start_ms: Option, +} + +impl WebmOpusWriter { + /// Writes EBML, segment, info, and the single-track header. + fn new(file: File) -> Result { + let mut writer = WebmWriter::new(file); + write_tags(&mut writer, &[ + MatroskaSpec::Ebml(Master::Start), + MatroskaSpec::EbmlVersion(1), + MatroskaSpec::EbmlReadVersion(1), + MatroskaSpec::EbmlMaxIdLength(4), + MatroskaSpec::EbmlMaxSizeLength(8), + MatroskaSpec::DocType("webm".to_string()), + MatroskaSpec::DocTypeVersion(4), + MatroskaSpec::DocTypeReadVersion(2), + MatroskaSpec::Ebml(Master::End), + ])?; + + writer.write_advanced(&MatroskaSpec::Segment(Master::Start), WriteOptions::is_unknown_sized_element()).map_err(webm_error)?; + + write_tags(&mut writer, &[ + MatroskaSpec::Info(Master::Start), + MatroskaSpec::TimestampScale(1_000_000), + MatroskaSpec::MuxingApp("MindWork AI Studio".to_string()), + MatroskaSpec::WritingApp("MindWork AI Studio".to_string()), + MatroskaSpec::Info(Master::End), + MatroskaSpec::Tracks(Master::Start), + MatroskaSpec::TrackEntry(Master::Start), + MatroskaSpec::TrackNumber(1), + MatroskaSpec::TrackUID(1), + MatroskaSpec::TrackType(2), + MatroskaSpec::FlagDefault(1), + MatroskaSpec::CodecID("A_OPUS".to_string()), + MatroskaSpec::CodecPrivate(opus_head()), + MatroskaSpec::CodecDelay(u64::from(OPUS_PRE_SKIP) * 1_000_000_000 / u64::from(OUTPUT_SAMPLE_RATE)), + MatroskaSpec::SeekPreRoll(80_000_000), + MatroskaSpec::Audio(Master::Start), + MatroskaSpec::SamplingFrequency(f64::from(OUTPUT_SAMPLE_RATE)), + MatroskaSpec::Channels(1), + MatroskaSpec::Audio(Master::End), + MatroskaSpec::TrackEntry(Master::End), + MatroskaSpec::Tracks(Master::End), + ])?; + + Ok(Self { writer, cluster_start_ms: None }) + } + + /// Writes one Opus packet and rotates clusters before timestamp overflow. + fn write_packet(&mut self, packet: &[u8], sample_position: u64) -> Result<(), MediaError> { + let timestamp_ms = sample_position.saturating_mul(1000) / u64::from(OUTPUT_SAMPLE_RATE); + let rotate = self.cluster_start_ms.map(|start| timestamp_ms.saturating_sub(start) >= CLUSTER_DURATION_MS).unwrap_or(true); + + if rotate { + if self.cluster_start_ms.is_some() { + self.writer.write(&MatroskaSpec::Cluster(Master::End)).map_err(webm_error)?; + } + + self.writer.write(&MatroskaSpec::Cluster(Master::Start)).map_err(webm_error)?; + self.writer.write(&MatroskaSpec::Timestamp(timestamp_ms)).map_err(webm_error)?; + self.cluster_start_ms = Some(timestamp_ms); + } + + let relative = timestamp_ms.saturating_sub(self.cluster_start_ms.unwrap_or(timestamp_ms)); + if relative > i16::MAX as u64 { + return Err(MediaError::new(MediaErrorCode::WebmTimestampOverflow, "The WebM cluster timestamp exceeded its safe range.")); + } + + let block: MatroskaSpec = SimpleBlock::new_uncheked(packet, 1, relative as i16, false, None, false, true).into(); + self.writer.write(&block).map_err(webm_error) + } + + /// Closes the active cluster and finalizes the segment and output file. + fn finish(mut self) -> Result<(), MediaError> { + if self.cluster_start_ms.is_some() { + self.writer.write(&MatroskaSpec::Cluster(Master::End)).map_err(webm_error)?; + } + + self.writer.write(&MatroskaSpec::Segment(Master::End)).map_err(webm_error)?; + self.writer.into_inner().map_err(webm_error)?; + Ok(()) + } +} + +/// Writes a sequence of Matroska tags with a consistent error mapping. +fn write_tags(writer: &mut WebmWriter, tags: &[MatroskaSpec]) -> Result<(), MediaError> { + for tag in tags { + writer.write(tag).map_err(webm_error)?; + } + + Ok(()) +} + +/// Builds the mono 48-kHz output track's Opus identification header. +fn opus_head() -> Vec { + let mut data = b"OpusHead".to_vec(); + data.push(1); + data.push(1); + data.extend_from_slice(&OPUS_PRE_SKIP.to_le_bytes()); + data.extend_from_slice(&OUTPUT_SAMPLE_RATE.to_le_bytes()); + data.extend_from_slice(&0i16.to_le_bytes()); + data.push(0); + data +} + +/// Derives the operation-owned partial path adjacent to the final output. +fn partial_path(output_path: &FilePath) -> PathBuf { + let mut name = output_path.file_name().unwrap_or_default().to_os_string(); + name.push(".partial"); + output_path.with_file_name(name) +} + +/// Checks the EBML document type rather than trusting the input extension. +fn is_webm_container(path: &FilePath) -> bool { + let Ok(file) = File::open(path) else { return false; }; + WebmIterator::new(file, &[]).take(16).filter_map(Result::ok).any(|tag| { + matches!(tag, MatroskaSpec::DocType(doc_type) if doc_type.eq_ignore_ascii_case("webm")) + }) +} + +/// Converts the cooperative cancellation flag to a stable terminal error. +fn check_cancelled(job: &MediaJob) -> Result<(), MediaError> { + if job.is_cancelled() { + Err(MediaError::new(MediaErrorCode::Cancelled, "The media job was cancelled.")) + } else { + Ok(()) + } +} + +/// Maps probe failures to stable public media error categories. +fn map_probe_error(error: SymphoniaError) -> MediaError { + match error { + SymphoniaError::Unsupported(_) => MediaError::new(MediaErrorCode::UnsupportedContainer, "This media container or codec is not supported."), + _ => MediaError::new(MediaErrorCode::DamagedContainer, format!("The media container could not be read: {error}")), + } +} + +/// Maps WebM writer failures to a stable public media error category. +fn webm_error(error: impl std::fmt::Display) -> MediaError { + MediaError::new(MediaErrorCode::WebmWriteFailed, format!("The WebM output could not be written: {error}")) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Cursor; + use std::num::NonZeroU32; + use tokio::sync::broadcast::error::TryRecvError; + use symphonia::core::audio::{Channels, Position}; + use symphonia::core::codecs::audio::well_known::{CODEC_ID_AC3, CODEC_ID_PCM_S16LE}; + use symphonia::core::codecs::audio::AudioCodecParameters; + + /// Returns the checked-in, FFmpeg-free-at-test-time media fixture directory. + fn fixtures() -> PathBuf { + FilePath::new(env!("CARGO_MANIFEST_DIR")).join("tests/fixtures/media") + } + + /// Creates a temporary output and normalizes one checked-in fixture. + fn normalize_fixture(name: &str) -> Result<(MediaJobResult, PathBuf), MediaError> { + let output = std::env::temp_dir().join(format!("ai-studio-fixture-{}.webm", rand::random::())); + let result = normalize_media(&fixtures().join(name), &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &MediaJob::new())?; + Ok((result, output)) + } + + /// Verifies the generated output identification header. + #[test] + fn opus_head_describes_48_khz_mono() { + let head = opus_head(); + assert_eq!(&head[..8], b"OpusHead"); + assert_eq!(head[9], 1); + assert_eq!(u32::from_le_bytes(head[12..16].try_into().unwrap()), 48_000); + } + + /// Verifies both single-stream channel layouts accepted by the adapter. + #[test] + fn opus_adapter_accepts_single_stream_mono_and_stereo_headers() { + for channels in [1, 2] { + let mut head = opus_head(); + head[9] = channels; + let parsed = OpusHeader::parse(&head).unwrap(); + assert_eq!(parsed.channels, usize::from(channels)); + assert_eq!(parsed.pre_skip, OPUS_PRE_SKIP); + } + } + + /// Verifies unsupported multistream mappings retain their stable code. + #[test] + fn opus_adapter_rejects_multistream_mapping_with_stable_code() { + let mut head = opus_head(); + head[18] = 1; + assert_eq!(OpusHeader::parse(&head).unwrap_err().code, MediaErrorCode::UnsupportedOpusMapping); + } + + /// Verifies a decodable default track wins over an earlier fallback. + #[test] + fn track_selection_prefers_decodable_default_audio() { + let mut first_params = AudioCodecParameters::new(); + first_params.for_codec(CODEC_ID_PCM_S16LE).with_sample_rate(48_000).with_channels(Channels::Positioned(Position::FRONT_LEFT)); + let mut default_params = first_params.clone(); + default_params.for_codec(CODEC_ID_PCM_S16LE); + let mut first = Track::new(1); + first.with_codec_params(CodecParameters::Audio(first_params)); + let mut preferred = Track::new(2); + preferred.with_codec_params(CodecParameters::Audio(default_params)).with_flags(TrackFlags::DEFAULT); + assert_eq!(select_audio_track(&[first, preferred]).unwrap().id, 2); + } + + /// Verifies an undecodable default track does not mask a usable fallback. + #[test] + fn track_selection_skips_undecodable_default_audio() { + let mut unsupported = AudioCodecParameters::new(); + unsupported.for_codec(CODEC_ID_AC3).with_sample_rate(48_000).with_channels(Channels::Positioned(Position::FRONT_LEFT)); + let mut supported = AudioCodecParameters::new(); + supported.for_codec(CODEC_ID_PCM_S16LE).with_sample_rate(48_000).with_channels(Channels::Positioned(Position::FRONT_LEFT)); + let mut default = Track::new(1); + default.with_codec_params(CodecParameters::Audio(unsupported)).with_flags(TrackFlags::DEFAULT); + let mut fallback = Track::new(2); + fallback.with_codec_params(CodecParameters::Audio(supported)); + assert_eq!(select_audio_track(&[default, fallback]).unwrap().id, 2); + } + + /// Verifies long output rotates clusters before signed relative timestamps overflow. + #[test] + fn webm_writer_rotates_clusters_before_relative_timestamp_overflow() { + let path = std::env::temp_dir().join(format!("ai-studio-media-writer-{}.webm", rand::random::())); + let file = File::create(&path).unwrap(); + let mut writer = WebmOpusWriter::new(file).unwrap(); + writer.write_packet(&[0xf8, 0xff, 0xfe], 0).unwrap(); + writer.write_packet(&[0xf8, 0xff, 0xfe], 31 * 48_000).unwrap(); + writer.finish().unwrap(); + let bytes = fs::read(&path).unwrap(); + let clusters = WebmIterator::new(Cursor::new(bytes), &[]) + .filter_map(Result::ok) + .filter(|tag| matches!(tag, MatroskaSpec::Cluster(Master::Start))) + .count(); + let _ = fs::remove_file(path); + assert_eq!(clusters, 2); + } + + /// Verifies deterministic arithmetic-mean downmixing. + #[test] + fn downmix_is_bounded_and_balanced() { + assert_eq!(downmix_to_mono(&[1.0, -1.0, 0.5, 0.5], 2), vec![0.0, 0.5]); + } + + /// Verifies the configured dBFS ceiling is inclusive and a higher peak is audible. + #[test] + fn practical_silence_uses_the_configured_maximum_peak() { + let threshold = 10.0_f32.powf(SILENCE_MAX_PEAK_DBFS / 20.0); + let mut detector = AudioPeakDetector::default(); + detector.observe(&[-threshold, threshold]); + assert!(!detector.has_audible_signal()); + + detector.observe(&[threshold * 1.01]); + assert!(detector.has_audible_signal()); + } + + /// Verifies timestamp gaps become silence and overlaps do not duplicate decoded samples. + #[test] + fn timestamp_alignment_inserts_gaps_and_trims_overlaps() { + let mut discontinuities = 0; + let mut gap = vec![1.0; 960]; + append_timestamp_aligned(&mut gap, &vec![2.0; 960], 960, Some(1_920), 7, 40, &mut discontinuities); + assert_eq!(gap.len(), 2_880); + assert!(gap[960..1_920].iter().all(|sample| *sample == 0.0)); + assert!(gap[1_920..].iter().all(|sample| *sample == 2.0)); + + let mut overlap = vec![1.0; 960]; + append_timestamp_aligned(&mut overlap, &vec![2.0; 960], 960, Some(480), 7, 10, &mut discontinuities); + assert_eq!(overlap.len(), 1_440); + assert!(overlap[960..].iter().all(|sample| *sample == 2.0)); + } + + /// Verifies packet progress uses a selected-track-relative time axis. + #[test] + fn packet_timestamps_are_relative_to_the_first_audio_packet() { + let time_base = TimeBase::new(NonZeroU32::new(1).unwrap(), NonZeroU32::new(1_000).unwrap()); + assert_eq!(packet_timestamp_ms(5_250, Some(5_000), Some(time_base)), Some(250)); + assert_eq!(packet_output_start(5_250, Some(5_000), Some(time_base)), Some(12_000)); + } + + /// Verifies running updates are throttled while a phase transition remains immediate. + #[test] + fn running_progress_is_throttled_but_phase_changes_are_immediate() { + let job = MediaJob::new(); + let mut events = job.events.subscribe(); + job.publish_progress(MediaJobPhase::Transcoding, Some(0.1)); + job.publish_progress(MediaJobPhase::Transcoding, Some(0.2)); + job.publish_progress(MediaJobPhase::Probing, Some(0.0)); + + assert_eq!(events.try_recv().unwrap().progress, Some(0.1)); + assert_eq!(events.try_recv().unwrap().phase, MediaJobPhase::Probing); + assert!(matches!(events.try_recv(), Err(TryRecvError::Empty))); + } + + /// Verifies cancellation interrupts source reads rather than waiting for another packet. + #[test] + fn cancellation_aware_source_interrupts_reads() { + let path = std::env::temp_dir().join(format!("ai-studio-source-cancel-{}", rand::random::())); + fs::write(&path, vec![0u8; COPY_BLOCK_BYTES * 2]).unwrap(); + let cancelled = Arc::new(AtomicBool::new(false)); + let (mut source, _) = CancellationMediaSource::new(File::open(&path).unwrap(), Arc::clone(&cancelled)).unwrap(); + cancelled.store(true, Ordering::Relaxed); + let error = source.read(&mut [0u8; 16]).unwrap_err(); + let _ = fs::remove_file(path); + assert_eq!(error.kind(), std::io::ErrorKind::Interrupted); + } + + /// Verifies output shape and at-most-one-frame duration rounding. + #[test] + fn wav_is_normalized_to_one_mono_opus_track_with_frame_bounded_duration() { + let directory = std::env::temp_dir().join(format!("ai-studio-media-test-{}", rand::random::())); + fs::create_dir_all(&directory).unwrap(); + let input = directory.join("input.wav"); + let output = directory.join("output.webm"); + fs::write(&input, wav_silence(44_100, 4_410)).unwrap(); + let job = MediaJob::new(); + let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap(); + assert!(!result.pass_through); + assert_eq!(result.output_format, OUTPUT_FORMAT); + assert_eq!(result.output_codec, OUTPUT_CODEC); + assert!(!result.has_audible_signal); + assert!(result.duration_ms.abs_diff(100) <= 20); + + let file = File::open(&output).unwrap(); + let tags: Vec<_> = WebmIterator::new(file, &[]) + .filter_map(Result::ok) + .collect(); + assert_eq!(tags.iter().filter(|tag| matches!(tag, MatroskaSpec::TrackEntry(Master::Start))).count(), 1); + assert!(tags.iter().any(|tag| matches!(tag, MatroskaSpec::CodecID(codec) if codec == "A_OPUS"))); + assert!(tags.iter().any(|tag| matches!(tag, MatroskaSpec::Channels(1)))); + assert!(tags.iter().any(|tag| matches!(tag, MatroskaSpec::SamplingFrequency(rate) if *rate == 48_000.0))); + let _ = fs::remove_dir_all(directory); + } + + /// Verifies cancellation removes both final and partial outputs. + #[test] + fn cancellation_does_not_leave_an_output_file() { + let directory = std::env::temp_dir().join(format!("ai-studio-media-cancel-{}", rand::random::())); + fs::create_dir_all(&directory).unwrap(); + let input = directory.join("input.wav"); + let output = directory.join("output.webm"); + fs::write(&input, wav_silence(48_000, 960)).unwrap(); + let job = MediaJob::new(); + job.cancelled.store(true, Ordering::Relaxed); + let error = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap_err(); + assert_eq!(error.code, MediaErrorCode::Cancelled); + assert!(!output.exists()); + assert!(!partial_path(&output).exists()); + let _ = fs::remove_dir_all(directory); + } + + /// Verifies an exactly compliant one-track WebM is copied unchanged. + #[test] + fn suitable_audio_only_webm_opus_is_passed_through() { + let directory = std::env::temp_dir().join(format!("ai-studio-media-pass-through-{}", rand::random::())); + fs::create_dir_all(&directory).unwrap(); + let input = directory.join("input.webm"); + let output = directory.join("output.webm"); + let mut writer = WebmOpusWriter::new(File::create(&input).unwrap()).unwrap(); + writer.write_packet(&[0xf8, 0xff, 0xfe], 0).unwrap(); + writer.finish().unwrap(); + let job = MediaJob::new(); + let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap(); + assert!(result.pass_through); + assert_eq!(result.output_format, OUTPUT_FORMAT); + assert_eq!(result.output_codec, OUTPUT_CODEC); + assert_eq!(fs::read(input).unwrap(), fs::read(output).unwrap()); + assert!(!result.has_audible_signal); + let _ = fs::remove_dir_all(directory); + } + + /// Verifies an above-threshold PCM peak survives normalization classification. + #[test] + fn audible_wav_is_not_classified_as_silence() { + let directory = std::env::temp_dir().join(format!("ai-studio-media-audible-{}", rand::random::())); + fs::create_dir_all(&directory).unwrap(); + let input = directory.join("input.wav"); + let output = directory.join("output.webm"); + fs::write(&input, wav_constant(48_000, 960, 1_000)).unwrap(); + let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &MediaJob::new()).unwrap(); + assert!(result.has_audible_signal); + let _ = fs::remove_dir_all(directory); + } + + /// Exercises every checked-in supported audio container without FFmpeg at test time. + #[test] + fn checked_in_audio_fixtures_normalize_without_external_tools() { + for name in [ + "sample.m4a", + "sample.mov", + "sample.mp4", + "sample.mkv", + "sample.ogg", + "sample.mp3", + "sample.flac", + "sample.wav", + "sample.aiff", + "sample.caf", + ] { + let (result, output) = normalize_fixture(name).unwrap_or_else(|error| panic!("{name}: {error:?}")); + assert!(result.duration_ms > 0 && result.duration_ms <= 200, "{name}: {} ms", result.duration_ms); + assert!(output.is_file(), "{name}"); + let _ = fs::remove_file(output); + } + } + + /// Verifies video and subtitle tracks independently disable pass-through. + #[test] + fn pass_through_requires_exactly_one_audio_track() { + let (audio_only, audio_output) = normalize_fixture("audio-only.webm").unwrap(); + assert!(audio_only.pass_through); + let _ = fs::remove_file(audio_output); + + let (video, video_output) = normalize_fixture("video.webm").unwrap(); + assert!(!video.pass_through); + let _ = fs::remove_file(video_output); + + let (subtitle, subtitle_output) = normalize_fixture("subtitle.webm").unwrap(); + assert!(!subtitle.pass_through); + let _ = fs::remove_file(subtitle_output); + } + + /// Verifies malformed, audio-less, and unknown-codec fixtures return stable categories. + #[test] + fn fixture_errors_are_stable() { + let damaged_output = std::env::temp_dir().join(format!("ai-studio-damaged-{}.webm", rand::random::())); + let damaged = normalize_media(&fixtures().join("damaged.bin"), &damaged_output, DEFAULT_MAX_PASS_THROUGH_BYTES, &MediaJob::new()).unwrap_err(); + assert!(matches!(damaged.code, MediaErrorCode::UnknownFormat | MediaErrorCode::NotMedia | MediaErrorCode::DamagedContainer)); + + let no_audio_output = std::env::temp_dir().join(format!("ai-studio-no-audio-{}.webm", rand::random::())); + let no_audio = normalize_media(&fixtures().join("no-audio.webm"), &no_audio_output, DEFAULT_MAX_PASS_THROUGH_BYTES, &MediaJob::new()).unwrap_err(); + assert_eq!(no_audio.code, MediaErrorCode::NoAudioTrack); + + let unknown_output = std::env::temp_dir().join(format!("ai-studio-unknown-{}.webm", rand::random::())); + let unknown = normalize_media(&fixtures().join("unknown-codec.mkv"), &unknown_output, DEFAULT_MAX_PASS_THROUGH_BYTES, &MediaJob::new()).unwrap_err(); + assert_eq!(unknown.code, MediaErrorCode::UnsupportedCodec); + } + + /// Verifies long streaming input never grows the resampler's pending buffer unboundedly. + #[test] + fn long_stream_resampling_keeps_pending_input_bounded() { + let mut resampler = StreamResampler::new(44_100).unwrap(); + let chunk = vec![0.0; 441]; + let mut produced = 0usize; + for _ in 0..6_000 { + produced += resampler.push(&chunk).unwrap().len(); + if let StreamResampler::Rubato { inner, pending, .. } = &resampler { + assert!(pending.len() < inner.input_frames_next()); + } + } + produced += resampler.finish().unwrap().len(); + assert_eq!(produced, 2_880_000); + } + + /// Constructs a minimal mono 16-bit PCM WAV fixture in memory. + fn wav_silence(sample_rate: u32, samples: u32) -> Vec { + wav_constant(sample_rate, samples, 0) + } + + /// Constructs a minimal mono 16-bit PCM WAV containing one constant sample value. + fn wav_constant(sample_rate: u32, samples: u32, sample: i16) -> Vec { + let data_size = samples * 2; + let mut wav = Vec::with_capacity(44 + data_size as usize); + wav.extend_from_slice(b"RIFF"); + wav.extend_from_slice(&(36 + data_size).to_le_bytes()); + wav.extend_from_slice(b"WAVEfmt "); + wav.extend_from_slice(&16u32.to_le_bytes()); + wav.extend_from_slice(&1u16.to_le_bytes()); + wav.extend_from_slice(&1u16.to_le_bytes()); + wav.extend_from_slice(&sample_rate.to_le_bytes()); + wav.extend_from_slice(&(sample_rate * 2).to_le_bytes()); + wav.extend_from_slice(&2u16.to_le_bytes()); + wav.extend_from_slice(&16u16.to_le_bytes()); + wav.extend_from_slice(b"data"); + wav.extend_from_slice(&data_size.to_le_bytes()); + for _ in 0..samples { + wav.extend_from_slice(&sample.to_le_bytes()); + } + wav + } +} diff --git a/runtime/src/runtime_api.rs b/runtime/src/runtime_api.rs index 087c0ffd..c7de61d4 100644 --- a/runtime/src/runtime_api.rs +++ b/runtime/src/runtime_api.rs @@ -1,6 +1,6 @@ use log::info; use once_cell::sync::Lazy; -use axum::routing::{get, post}; +use axum::routing::{delete, get, post}; use axum::Router; use axum_server::tls_rustls::RustlsConfig; use std::net::SocketAddr; @@ -59,6 +59,9 @@ pub fn start_runtime_api() { .route("/system/enterprise/config/encryption_secret", get(crate::environment::read_enterprise_env_config_encryption_secret)) .route("/system/enterprise/configs", get(crate::environment::read_enterprise_configs)) .route("/retrieval/fs/extract", get(crate::file_data::extract_data)) + .route("/media/jobs", post(crate::media::create_job)) + .route("/media/jobs/{id}/events", get(crate::media::get_job_events)) + .route("/media/jobs/{id}", delete(crate::media::cancel_job)) .route("/log/paths", get(crate::log::get_log_paths)) .route("/log/event", post(crate::log::log_event)) .route("/shortcuts/register", post(crate::app_window::register_shortcut)) diff --git a/runtime/tauri.conf.json b/runtime/tauri.conf.json index 78849800..4fc34089 100644 --- a/runtime/tauri.conf.json +++ b/runtime/tauri.conf.json @@ -27,7 +27,8 @@ "../app/MindWork AI Studio/bin/dist/mindworkAIStudioServer" ], "resources": [ - "resources/libraries/*" + "resources/libraries/*", + "resources/notices/*" ], "macOS": { "exceptionDomain": "localhost" diff --git a/runtime/tests/fixtures/media/audio-only.webm b/runtime/tests/fixtures/media/audio-only.webm new file mode 100644 index 00000000..dd4b32b3 Binary files /dev/null and b/runtime/tests/fixtures/media/audio-only.webm differ diff --git a/runtime/tests/fixtures/media/damaged.bin b/runtime/tests/fixtures/media/damaged.bin new file mode 100644 index 00000000..8d44e9d0 --- /dev/null +++ b/runtime/tests/fixtures/media/damaged.bin @@ -0,0 +1 @@ +not a valid media container diff --git a/runtime/tests/fixtures/media/no-audio.webm b/runtime/tests/fixtures/media/no-audio.webm new file mode 100644 index 00000000..bdbc75e4 Binary files /dev/null and b/runtime/tests/fixtures/media/no-audio.webm differ diff --git a/runtime/tests/fixtures/media/sample.aiff b/runtime/tests/fixtures/media/sample.aiff new file mode 100644 index 00000000..402b96f9 Binary files /dev/null and b/runtime/tests/fixtures/media/sample.aiff differ diff --git a/runtime/tests/fixtures/media/sample.caf b/runtime/tests/fixtures/media/sample.caf new file mode 100644 index 00000000..a1f702a2 Binary files /dev/null and b/runtime/tests/fixtures/media/sample.caf differ diff --git a/runtime/tests/fixtures/media/sample.flac b/runtime/tests/fixtures/media/sample.flac new file mode 100644 index 00000000..c74fe7f6 Binary files /dev/null and b/runtime/tests/fixtures/media/sample.flac differ diff --git a/runtime/tests/fixtures/media/sample.m4a b/runtime/tests/fixtures/media/sample.m4a new file mode 100644 index 00000000..205fd8fa Binary files /dev/null and b/runtime/tests/fixtures/media/sample.m4a differ diff --git a/runtime/tests/fixtures/media/sample.mkv b/runtime/tests/fixtures/media/sample.mkv new file mode 100644 index 00000000..25a84e40 Binary files /dev/null and b/runtime/tests/fixtures/media/sample.mkv differ diff --git a/runtime/tests/fixtures/media/sample.mov b/runtime/tests/fixtures/media/sample.mov new file mode 100644 index 00000000..6fe861e8 Binary files /dev/null and b/runtime/tests/fixtures/media/sample.mov differ diff --git a/runtime/tests/fixtures/media/sample.mp3 b/runtime/tests/fixtures/media/sample.mp3 new file mode 100644 index 00000000..7a83a8f8 Binary files /dev/null and b/runtime/tests/fixtures/media/sample.mp3 differ diff --git a/runtime/tests/fixtures/media/sample.mp4 b/runtime/tests/fixtures/media/sample.mp4 new file mode 100644 index 00000000..508e8752 Binary files /dev/null and b/runtime/tests/fixtures/media/sample.mp4 differ diff --git a/runtime/tests/fixtures/media/sample.ogg b/runtime/tests/fixtures/media/sample.ogg new file mode 100644 index 00000000..a161fc27 Binary files /dev/null and b/runtime/tests/fixtures/media/sample.ogg differ diff --git a/runtime/tests/fixtures/media/sample.wav b/runtime/tests/fixtures/media/sample.wav new file mode 100644 index 00000000..19a3e1b3 Binary files /dev/null and b/runtime/tests/fixtures/media/sample.wav differ diff --git a/runtime/tests/fixtures/media/subtitle.vtt b/runtime/tests/fixtures/media/subtitle.vtt new file mode 100644 index 00000000..07059e48 --- /dev/null +++ b/runtime/tests/fixtures/media/subtitle.vtt @@ -0,0 +1,4 @@ +WEBVTT + +00:00.000 --> 00:00.100 +fixture diff --git a/runtime/tests/fixtures/media/subtitle.webm b/runtime/tests/fixtures/media/subtitle.webm new file mode 100644 index 00000000..2f1bf360 Binary files /dev/null and b/runtime/tests/fixtures/media/subtitle.webm differ diff --git a/runtime/tests/fixtures/media/unknown-codec.mkv b/runtime/tests/fixtures/media/unknown-codec.mkv new file mode 100644 index 00000000..a7f02a24 Binary files /dev/null and b/runtime/tests/fixtures/media/unknown-codec.mkv differ diff --git a/runtime/tests/fixtures/media/video.webm b/runtime/tests/fixtures/media/video.webm new file mode 100644 index 00000000..eca88bdb Binary files /dev/null and b/runtime/tests/fixtures/media/video.webm differ