From 117d3822b07bc56b9350cc9c6d833635ed447d82 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Tue, 14 Jul 2026 20:04:06 +0200 Subject: [PATCH] Added silence threshold check --- .../Assistants/AssistantBase.razor.cs | 6 + .../Assistants/I18N/allTexts.lua | 6 + .../Components/AssistantBlock.razor.cs | 1 + .../Components/AttachDocuments.razor.cs | 6 + .../Components/ChatComponent.razor.cs | 6 + .../Components/ReadFileContent.razor.cs | 6 + .../Components/VoiceRecorder.razor.cs | 9 +- .../Components/Workspaces.razor.cs | 2 + .../Tools/Media/MediaImportOutcome.cs | 2 + .../Tools/Media/MediaImportStatus.cs | 1 + .../Tools/Media/MediaImportWarning.cs | 4 + .../Tools/Media/MediaTranscriptionResult.cs | 9 +- .../Media/MediaTranscriptionResultStatus.cs | 3 + .../Tools/Rust/MediaJobResult.cs | 4 +- .../Services/MediaTranscriptionService.cs | 34 ++- runtime/src/media.rs | 214 +++++++++++++++++- 16 files changed, 300 insertions(+), 13 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Media/MediaImportWarning.cs diff --git a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs index ec728fb9..bbf0291d 100644 --- a/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs +++ b/app/MindWork AI Studio/Assistants/AssistantBase.razor.cs @@ -730,6 +730,12 @@ public abstract partial class AssistantBase : AssistantLowerBase wher await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.TB("The media file could not be transcribed."))); } + if (outcome.Warnings.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}")); + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message)); + } + if (outcome.Status is MediaImportStatus.CANCELLED) { await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.TB("The media transcription was canceled."))); diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index dad9ac1d..d5b95dd6 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -2305,6 +2305,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T1233815302"] = "Media tr -- Media transcription failed. Open the assistant to review it. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2177964639"] = "Media transcription failed. Open the assistant to review it." +-- Media transcription completed with a warning. Open the assistant to review it. +UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2217674098"] = "Media transcription completed with a warning. Open the assistant to review it." + -- Media is still being prepared. UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2600900617"] = "Media is still being prepared." @@ -8572,6 +8575,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T15439746 -- The selected file cannot be processed as media. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1707342767"] = "The selected file cannot be processed as media." +-- The audio track contains no audible signal, so there is nothing to transcribe. +UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1988190152"] = "The audio track contains no audible signal, so there is nothing to transcribe." + -- The media file is damaged or its format could not be identified. UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2004316549"] = "The media file is damaged or its format could not be identified." diff --git a/app/MindWork AI Studio/Components/AssistantBlock.razor.cs b/app/MindWork AI Studio/Components/AssistantBlock.razor.cs index 34aba34b..48672332 100644 --- a/app/MindWork AI Studio/Components/AssistantBlock.razor.cs +++ b/app/MindWork AI Studio/Components/AssistantBlock.razor.cs @@ -113,6 +113,7 @@ public partial class AssistantBlock : MSGComponentBase where TSetting { MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => new(Icons.Material.Filled.ChangeCircle, Color.Info, this.T("Media is still being prepared.")), MediaImportStatus.SUCCEEDED => new(Icons.Material.Filled.TaskAlt, Color.Success, this.T("The media transcript is ready.")), + MediaImportStatus.WARNING => new(Icons.Material.Filled.WarningAmber, Color.Warning, this.T("Media transcription completed with a warning. Open the assistant to review it.")), MediaImportStatus.FAILED => new(Icons.Material.Filled.Error, Color.Error, this.T("Media transcription failed. Open the assistant to review it.")), MediaImportStatus.CANCELLED => new(Icons.Material.Filled.Cancel, Color.Warning, this.T("Media transcription was canceled. Open the assistant to review it.")), diff --git a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs index c1ea22d7..c52bd115 100644 --- a/app/MindWork AI Studio/Components/AttachDocuments.razor.cs +++ b/app/MindWork AI Studio/Components/AttachDocuments.razor.cs @@ -161,6 +161,12 @@ public partial class AttachDocuments : MSGComponentBase await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed."))); } + if (outcome.Warnings.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}")); + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message)); + } + if (outcome.Status is MediaImportStatus.CANCELLED) { await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled."))); diff --git a/app/MindWork AI Studio/Components/ChatComponent.razor.cs b/app/MindWork AI Studio/Components/ChatComponent.razor.cs index 485c5961..2cee066a 100644 --- a/app/MindWork AI Studio/Components/ChatComponent.razor.cs +++ b/app/MindWork AI Studio/Components/ChatComponent.razor.cs @@ -285,6 +285,12 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed."))); } + if (outcome.Warnings.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}")); + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message)); + } + if (outcome.Status is MediaImportStatus.CANCELLED) { await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled."))); diff --git a/app/MindWork AI Studio/Components/ReadFileContent.razor.cs b/app/MindWork AI Studio/Components/ReadFileContent.razor.cs index fadb01c8..4a200f1f 100644 --- a/app/MindWork AI Studio/Components/ReadFileContent.razor.cs +++ b/app/MindWork AI Studio/Components/ReadFileContent.razor.cs @@ -125,6 +125,12 @@ public partial class ReadFileContent : MSGComponentBase await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed."))); } + if (outcome.Warnings.Count > 0) + { + var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}")); + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message)); + } + if (outcome.Status is MediaImportStatus.CANCELLED) { await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled."))); diff --git a/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs b/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs index f6ece656..8cfa1362 100644 --- a/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs +++ b/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs @@ -345,9 +345,16 @@ public partial class VoiceRecorder : MSGComponentBase var transcriptionResult = await this.MediaTranscriptionService.TranscribeVoiceAsync(this.finalRecordingPath); if (transcriptionResult.Status is not MediaTranscriptionResultStatus.SUCCEEDED) { - this.Logger.LogWarning("The transcription request failed."); if (transcriptionResult.Status is MediaTranscriptionResultStatus.CANCELLED) return; + + if (transcriptionResult.Status is MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL) + { + await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, transcriptionResult.UserMessage)); + return; + } + + this.Logger.LogWarning("The transcription request failed."); var userMessage = string.IsNullOrWhiteSpace(transcriptionResult.UserMessage) ? this.T("Unfortunately, there was an error communicating with the AI system.") : transcriptionResult.UserMessage; diff --git a/app/MindWork AI Studio/Components/Workspaces.razor.cs b/app/MindWork AI Studio/Components/Workspaces.razor.cs index 3f6b8779..8ec4165a 100644 --- a/app/MindWork AI Studio/Components/Workspaces.razor.cs +++ b/app/MindWork AI Studio/Components/Workspaces.razor.cs @@ -394,6 +394,7 @@ public partial class Workspaces : MSGComponentBase { MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => " color: var(--mud-palette-info);", MediaImportStatus.SUCCEEDED => " color: var(--mud-palette-success);", + MediaImportStatus.WARNING => " color: var(--mud-palette-warning);", MediaImportStatus.FAILED => " color: var(--mud-palette-error);", MediaImportStatus.CANCELLED => " color: var(--mud-palette-warning);", _ => string.Empty, @@ -421,6 +422,7 @@ public partial class Workspaces : MSGComponentBase MediaImportStatus.QUEUED => Icons.Material.Filled.HourglassTop, MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => Icons.Material.Filled.ChangeCircle, MediaImportStatus.SUCCEEDED => Icons.Material.Filled.TaskAlt, + MediaImportStatus.WARNING => Icons.Material.Filled.WarningAmber, MediaImportStatus.FAILED => Icons.Material.Filled.Error, MediaImportStatus.CANCELLED => Icons.Material.Filled.Cancel, diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportOutcome.cs b/app/MindWork AI Studio/Tools/Media/MediaImportOutcome.cs index 5fce8345..33eeb159 100644 --- a/app/MindWork AI Studio/Tools/Media/MediaImportOutcome.cs +++ b/app/MindWork AI Studio/Tools/Media/MediaImportOutcome.cs @@ -8,4 +8,6 @@ public sealed record MediaImportOutcome public required MediaImportStatus Status { get; init; } public IReadOnlyList Failures { get; init; } = []; + + public IReadOnlyList Warnings { get; init; } = []; } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportStatus.cs b/app/MindWork AI Studio/Tools/Media/MediaImportStatus.cs index 139315fa..7207a58d 100644 --- a/app/MindWork AI Studio/Tools/Media/MediaImportStatus.cs +++ b/app/MindWork AI Studio/Tools/Media/MediaImportStatus.cs @@ -7,6 +7,7 @@ public enum MediaImportStatus RUNNING, CANCELING, SUCCEEDED, + WARNING, FAILED, CANCELLED, } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaImportWarning.cs b/app/MindWork AI Studio/Tools/Media/MediaImportWarning.cs new file mode 100644 index 00000000..d0ac9c0f --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaImportWarning.cs @@ -0,0 +1,4 @@ +namespace AIStudio.Tools.Media; + +/// One user-visible media warning retained until its owner is displayed. +public sealed record MediaImportWarning(string FileName, string UserMessage); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs index 92e4c3b6..b486480f 100644 --- a/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs +++ b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs @@ -7,7 +7,7 @@ namespace AIStudio.Tools.Media; /// /// Terminal operation status. /// Transcript text for a successful operation. -/// Localized message suitable for display after failure. +/// Localized message suitable for display after a warning or failure. /// Optional stable runtime failure category. public sealed record MediaTranscriptionResult(MediaTranscriptionResultStatus Status, string Text, string UserMessage, MediaJobErrorCode? ErrorCode = null) { @@ -20,6 +20,13 @@ public sealed record MediaTranscriptionResult(MediaTranscriptionResultStatus Sta /// Optional runtime error category. public static MediaTranscriptionResult Failed(string userMessage, MediaJobErrorCode? errorCode = null) => new(MediaTranscriptionResultStatus.FAILED, string.Empty, userMessage, errorCode); + /// Creates a warning result for media without an audible signal. + /// Localized visible warning. + public static MediaTranscriptionResult NoAudibleSignal(string userMessage) => new( + MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL, + string.Empty, + userMessage); + /// Creates a cancelled result without relying on visible text. public static MediaTranscriptionResult Cancelled() => new(MediaTranscriptionResultStatus.CANCELLED, string.Empty, string.Empty, MediaJobErrorCode.CANCELLED); } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResultStatus.cs b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResultStatus.cs index 8abddc2d..02ff300b 100644 --- a/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResultStatus.cs +++ b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResultStatus.cs @@ -10,6 +10,9 @@ public enum MediaTranscriptionResultStatus /// The operation failed. FAILED, + + /// The media contains no signal above the practical-silence threshold. + NO_AUDIBLE_SIGNAL, /// The caller or user cancelled the operation. CANCELLED, diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs index 5fc4f70b..3552071c 100644 --- a/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs @@ -6,9 +6,11 @@ namespace AIStudio.Tools.Rust; /// Selected codec diagnostic. /// Normalized duration in milliseconds. /// Whether the source was copied unchanged. +/// Whether the normalized audio exceeds the practical-silence threshold. public sealed record MediaJobResult( string OutputPath, string DetectedFormat, string DetectedCodec, ulong DurationMs, - bool PassThrough); \ No newline at end of file + bool PassThrough, + bool HasAudibleSignal); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs index 5b4149e2..1859429c 100644 --- a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs +++ b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs @@ -192,6 +192,7 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM var status = MediaImportStatus.SUCCEEDED; List failures = []; + List warnings = []; try { var result = await this.TranscribeImportAsync(mediaPath, target, cancellation.Token); @@ -199,6 +200,11 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM this.AddCompletedText(target, result.Text); else if (result.Status is MediaTranscriptionResultStatus.CANCELLED) status = MediaImportStatus.CANCELLED; + else if (result.Status is MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL) + { + status = MediaImportStatus.WARNING; + warnings.Add(new(Path.GetFileName(mediaPath), result.UserMessage)); + } else { status = MediaImportStatus.FAILED; @@ -224,7 +230,7 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM ownedCancellation.Dispose(); } - this.CompleteImport(target, Path.GetFileName(mediaPath), status, failures); + this.CompleteImport(target, Path.GetFileName(mediaPath), status, failures, warnings); } } @@ -252,6 +258,7 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM var status = MediaImportStatus.SUCCEEDED; var currentFileName = Path.GetFileName(mediaPaths[0]); List failures = []; + List warnings = []; try { @@ -266,6 +273,15 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM break; } + if (result.Status is MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL) + { + if (status is MediaImportStatus.SUCCEEDED) + status = MediaImportStatus.WARNING; + + warnings.Add(new(currentFileName, result.UserMessage)); + continue; + } + if (result.Status is not MediaTranscriptionResultStatus.SUCCEEDED) { status = MediaImportStatus.FAILED; @@ -307,7 +323,7 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM cancellation.Dispose(); } - this.CompleteImport(target, currentFileName, status, failures); + this.CompleteImport(target, currentFileName, status, failures, warnings); } } @@ -430,6 +446,12 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM ? MediaTranscriptionResult.Failed(TB("The media pipeline ended without an output file.")) : MediaTranscriptionResult.Failed(UserMessageFor(normalized.Error.Code), normalized.Error.Code); + if (!normalized.Result.HasAudibleSignal) + { + logger.LogInformation("Skipping transcription for '{MediaPath}' because its maximum audio peak does not exceed the practical-silence threshold.", mediaPath); + return MediaTranscriptionResult.NoAudibleSignal(TB("The audio track contains no audible signal, so there is nothing to transcribe.")); + } + var providerSettings = this.ResolveProvider(); if (providerSettings is null) return MediaTranscriptionResult.Failed(TB("No usable transcription provider is configured.")); @@ -602,7 +624,12 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM } /// Publishes one retained terminal result after an entire target batch ended. - private void CompleteImport(MediaImportTarget target, string fileName, MediaImportStatus status, IReadOnlyList failures) + private void CompleteImport( + MediaImportTarget target, + string fileName, + MediaImportStatus status, + IReadOnlyList failures, + IReadOnlyList warnings) { lock (this.stateLock) { @@ -621,6 +648,7 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM Owner = target.Owner, Status = status, Failures = [.. failures], + Warnings = [.. warnings], }; } diff --git a/runtime/src/media.rs b/runtime/src/media.rs index 5dbd9a0f..30cdf9ba 100644 --- a/runtime/src/media.rs +++ b/runtime/src/media.rs @@ -68,6 +68,9 @@ const PROGRESS_EVENT_INTERVAL: StdDuration = StdDuration::from_secs(6); /// Timestamp differences above this threshold are recorded as discontinuities. const LARGE_DISCONTINUITY_MS: i64 = 1_000; +/// Maximum full-scale peak still treated as practical silence. +const SILENCE_MAX_PEAK_DBFS: f32 = -60.0; + /// Time a terminal job remains available for late SSE subscribers. const TERMINAL_JOB_RETENTION: std::time::Duration = std::time::Duration::from_secs(10 * 60); @@ -227,6 +230,9 @@ pub struct MediaJobResult { /// Whether the input was copied unchanged. pub pass_through: bool, + + /// Whether the normalized audio exceeds the practical-silence threshold. + pub has_audible_signal: bool, } /// Stable error code plus an English diagnostic intended for logs. @@ -570,6 +576,9 @@ fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_throu .ok_or_else(|| MediaError::new(MediaErrorCode::UnsupportedCodec, "None of the audio tracks uses a supported codec."))?; let track_id = selected.id; + let track_delay = selected.delay.unwrap_or(0); + let track_padding = selected.padding.unwrap_or(0); + let track_time_base = selected.time_base; let params = selected.codec_params.as_ref().and_then(CodecParameters::audio).unwrap().clone(); let detected_codec = if params.codec == CODEC_ID_OPUS { "opus".to_string() } else { format!("{}", params.codec) }; let track_duration_ms = selected.num_frames.zip(params.sample_rate) @@ -610,6 +619,18 @@ fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_throu } let result = if pass_through { + job.publish_progress(MediaJobPhase::Transcoding, Some(0.0)); + let analysis_context = SignalAnalysisContext { + track_id, + params: ¶ms, + track_delay, + track_padding, + expected_duration_ms: duration_ms, + time_base: track_time_base, + source_progress: &source_progress, + job, + }; + let signal = analyze_audio_signal(&mut *format, analysis_context)?; copy_with_cancellation(input_path, &partial_path, job)?; Ok(MediaJobResult { output_path: output_path.to_string_lossy().into_owned(), @@ -617,21 +638,22 @@ fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_throu detected_codec, duration_ms, pass_through: true, + has_audible_signal: signal.has_audible_signal(), }) } else { job.publish_progress(MediaJobPhase::Transcoding, Some(0.0)); let context = TranscodeContext { track_id, - track_delay: selected.delay.unwrap_or(0), - track_padding: selected.padding.unwrap_or(0), + track_delay, + track_padding, params, partial_path: &partial_path, output_path, detected_format, detected_codec, expected_duration_ms: duration_ms, - time_base: selected.time_base, + time_base: track_time_base, source_progress, job, }; @@ -685,6 +707,140 @@ fn is_decodable(track: &Track) -> bool { params.codec == CODEC_ID_OPUS || symphonia::default::get_codecs().make_audio_decoder(params, &AudioDecoderOptions::default()).is_ok() } +/// Streaming peak measurement over normalized full-scale floating-point samples. +#[derive(Default)] +struct AudioPeakDetector { + /// Highest absolute sample observed so far. + max_amplitude: f32, +} + +impl AudioPeakDetector { + /// Includes one bounded sample block in the maximum-peak measurement. + fn observe(&mut self, samples: &[f32]) { + for sample in samples { + let amplitude = sample.abs(); + self.max_amplitude = if amplitude.is_finite() { + self.max_amplitude.max(amplitude) + } else { + f32::INFINITY + }; + } + } + + /// Returns whether any retained sample exceeds the configured silence ceiling. + fn has_audible_signal(&self) -> bool { + self.max_amplitude > 10.0_f32.powf(SILENCE_MAX_PEAK_DBFS / 20.0) + } + + /// Returns the measured full-scale peak for diagnostics. + fn max_peak_dbfs(&self) -> f32 { + if self.max_amplitude == 0.0 { + f32::NEG_INFINITY + } else { + 20.0 * self.max_amplitude.log10() + } + } +} + +/// Inputs required to scan an otherwise pass-through-compatible audio track. +struct SignalAnalysisContext<'a> { + /// Selected track identifier. + track_id: u32, + + /// Selected track codec parameters. + params: &'a symphonia::core::codecs::audio::AudioCodecParameters, + + /// Leading decoded frames to discard. + track_delay: u32, + + /// Trailing decoded frames to discard. + track_padding: u32, + + /// Container duration used for progress reporting. + expected_duration_ms: u64, + + /// Selected track timebase used for progress reporting. + time_base: Option, + + /// Sequential byte progress fallback when the track has no duration. + source_progress: &'a SourceProgress, + + /// Cancellation and progress state for the job. + job: &'a MediaJob, +} + +/// Decodes an otherwise pass-through-compatible track solely to classify practical silence. +fn analyze_audio_signal( + format: &mut dyn symphonia::core::formats::FormatReader, + context: SignalAnalysisContext<'_>, +) -> Result { + let mut decoder = StreamDecoder::new(context.params, context.track_delay)?; + let mut detector = AudioPeakDetector::default(); + let mut decoded_tail = Vec::::new(); + let mut first_packet_pts = None::; + let mut decoded_packets = 0u64; + let mut last_progress = 0.0f64; + + loop { + check_cancelled(context.job)?; + let packet = match format.next_packet() { + Ok(Some(packet)) => packet, + Ok(None) => break, + + Err(SymphoniaError::ResetRequired) => return Err(MediaError::new(MediaErrorCode::StreamReset, "The media stream changed unexpectedly.")), + Err(SymphoniaError::IoError(error)) if error.kind() == std::io::ErrorKind::Interrupted && context.job.is_cancelled() => { + return Err(MediaError::new(MediaErrorCode::Cancelled, "The media job was cancelled.")); + } + + Err(SymphoniaError::IoError(error)) if error.kind() == std::io::ErrorKind::UnexpectedEof => break, + Err(error) => return Err(MediaError::new(MediaErrorCode::DamagedContainer, format!("The media container is damaged: {error}"))), + }; + + if packet.track_id != context.track_id { + continue; + } + + let packet_pts = packet.pts.get(); + first_packet_pts.get_or_insert(packet_pts); + let Some((mono, _)) = decoder.decode(&packet)? else { continue; }; + decoded_packets += 1; + + decoded_tail.extend_from_slice(&mono); + let emit_len = decoded_tail.len().saturating_sub(context.track_padding as usize); + if emit_len > 0 { + detector.observe(&decoded_tail[..emit_len]); + drop(decoded_tail.drain(..emit_len)); + } + + let timestamp_ms = packet_timestamp_ms(packet_pts, first_packet_pts, context.time_base); + let progress = if context.expected_duration_ms > 0 { + timestamp_ms.map(|current_ms| (current_ms as f64 / context.expected_duration_ms as f64).clamp(0.0, 0.99)) + } else { + context.source_progress.fraction() + }; + + if let Some(progress) = progress { + last_progress = last_progress.max(progress); + } + + context.job.publish_progress(MediaJobPhase::Transcoding, progress.map(|_| last_progress)); + } + + if decoded_packets == 0 { + return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The selected audio track did not yield decoded audio samples.")); + } + + log::info!( + "media signal analysis completed: track_id={}, max_peak_dbfs={}, silence_threshold_dbfs={}, has_audible_signal={}", + context.track_id, + detector.max_peak_dbfs(), + SILENCE_MAX_PEAK_DBFS, + detector.has_audible_signal(), + ); + + Ok(detector) +} + /// Immutable inputs shared across a single transcoding operation. struct TranscodeContext<'a> { /// Selected input track identifier. @@ -739,6 +895,7 @@ fn transcode( let file = File::create(context.partial_path).map_err(|error| MediaError::new(MediaErrorCode::OutputCreateFailed, error.to_string()))?; let mut writer = WebmOpusWriter::new(file)?; let mut pending = Vec::::with_capacity(OPUS_FRAME_SAMPLES * 3); + let mut signal = AudioPeakDetector::default(); let mut resampler: Option = None; let mut decoded_tail = Vec::::new(); let mut encoded = [0u8; 4_000]; @@ -805,6 +962,7 @@ fn transcode( .flatten(); let current_start = produced_samples.saturating_add(pending.len() as u64); + let previous_pending_len = pending.len(); append_timestamp_aligned( &mut pending, &resampled, @@ -814,6 +972,8 @@ fn transcode( packet_pts, &mut discontinuities, ); + + signal.observe(&pending[previous_pending_len..]); } encode_complete_frames(&mut pending, &mut opus_encoder, &mut writer, &mut encoded, &mut produced_samples, context.job)?; @@ -834,7 +994,9 @@ fn transcode( if let Some(stream_resampler) = resampler.as_mut() { // Discard the retained decoded tail (container padding), then flush the filter delay. - pending.extend_from_slice(&stream_resampler.finish()?); + let flushed = stream_resampler.finish()?; + signal.observe(&flushed); + pending.extend_from_slice(&flushed); } else { return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The selected audio track did not yield decoded audio parameters.")); } @@ -864,7 +1026,7 @@ fn transcode( } log::info!( - "media transcode completed: track_id={}, decoded_packets={}, discarded_packets={}, recoverable_decode_errors={}, first_pts={:?}, last_pts={:?}, discontinuities={}, output_bytes={}, duration_ms={}", + "media transcode completed: track_id={}, decoded_packets={}, discarded_packets={}, recoverable_decode_errors={}, first_pts={:?}, last_pts={:?}, discontinuities={}, output_bytes={}, duration_ms={}, max_peak_dbfs={}, silence_threshold_dbfs={}, has_audible_signal={}", context.track_id, decoded_packets, discarded_packets, @@ -874,6 +1036,9 @@ fn transcode( discontinuities, output_size, output_duration_ms, + signal.max_peak_dbfs(), + SILENCE_MAX_PEAK_DBFS, + signal.has_audible_signal(), ); Ok(MediaJobResult { @@ -882,6 +1047,7 @@ fn transcode( detected_codec: context.detected_codec, duration_ms: output_duration_ms, pass_through: false, + has_audible_signal: signal.has_audible_signal(), }) } @@ -1550,6 +1716,18 @@ mod tests { assert_eq!(downmix_to_mono(&[1.0, -1.0, 0.5, 0.5], 2), vec![0.0, 0.5]); } + /// Verifies the configured dBFS ceiling is inclusive and a higher peak is audible. + #[test] + fn practical_silence_uses_the_configured_maximum_peak() { + let threshold = 10.0_f32.powf(SILENCE_MAX_PEAK_DBFS / 20.0); + let mut detector = AudioPeakDetector::default(); + detector.observe(&[-threshold, threshold]); + assert!(!detector.has_audible_signal()); + + detector.observe(&[threshold * 1.01]); + assert!(detector.has_audible_signal()); + } + /// Verifies timestamp gaps become silence and overlaps do not duplicate decoded samples. #[test] fn timestamp_alignment_inserts_gaps_and_trims_overlaps() { @@ -1612,6 +1790,7 @@ mod tests { let job = MediaJob::new(); let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap(); assert!(!result.pass_through); + assert!(!result.has_audible_signal); assert!(result.duration_ms.abs_diff(100) <= 20); let file = File::open(&output).unwrap(); @@ -1656,6 +1835,20 @@ mod tests { let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap(); assert!(result.pass_through); assert_eq!(fs::read(input).unwrap(), fs::read(output).unwrap()); + assert!(!result.has_audible_signal); + let _ = fs::remove_dir_all(directory); + } + + /// Verifies an above-threshold PCM peak survives normalization classification. + #[test] + fn audible_wav_is_not_classified_as_silence() { + let directory = std::env::temp_dir().join(format!("ai-studio-media-audible-{}", rand::random::())); + fs::create_dir_all(&directory).unwrap(); + let input = directory.join("input.wav"); + let output = directory.join("output.webm"); + fs::write(&input, wav_constant(48_000, 960, 1_000)).unwrap(); + let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &MediaJob::new()).unwrap(); + assert!(result.has_audible_signal); let _ = fs::remove_dir_all(directory); } @@ -1731,6 +1924,11 @@ mod tests { /// Constructs a minimal mono 16-bit PCM WAV fixture in memory. fn wav_silence(sample_rate: u32, samples: u32) -> Vec { + wav_constant(sample_rate, samples, 0) + } + + /// Constructs a minimal mono 16-bit PCM WAV containing one constant sample value. + fn wav_constant(sample_rate: u32, samples: u32, sample: i16) -> Vec { let data_size = samples * 2; let mut wav = Vec::with_capacity(44 + data_size as usize); wav.extend_from_slice(b"RIFF"); @@ -1745,7 +1943,9 @@ mod tests { wav.extend_from_slice(&16u16.to_le_bytes()); wav.extend_from_slice(b"data"); wav.extend_from_slice(&data_size.to_le_bytes()); - wav.resize(44 + data_size as usize, 0); + for _ in 0..samples { + wav.extend_from_slice(&sample.to_le_bytes()); + } wav } -} \ No newline at end of file +}