MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-08-11 17:32:11 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Added silence threshold check

Browse Source
This commit is contained in:
Thorsten Sommer 2026-07-14 20:04:06 +02:00
parent 43a508b4de
commit 117d3822b0
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
16 changed files with 300 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
app/MindWork AI Studio/Assistants/AssistantBase.razor.cs
View File

@ -730,6 +730,12 @@ public abstract partial class AssistantBase<TSettings> : AssistantLowerBase wher
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.TB("The media file could not be transcribed.")));
}
if (outcome.Warnings.Count > 0)
{
var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}"));
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message));
}
if (outcome.Status is MediaImportStatus.CANCELLED)
{
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.TB("The media transcription was canceled.")));

6
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View File

@ -2305,6 +2305,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T1233815302"] = "Media tr
-- Media transcription failed. Open the assistant to review it.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2177964639"] = "Media transcription failed. Open the assistant to review it."
-- Media transcription completed with a warning. Open the assistant to review it.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2217674098"] = "Media transcription completed with a warning. Open the assistant to review it."
-- Media is still being prepared.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::ASSISTANTBLOCK::T2600900617"] = "Media is still being prepared."
@ -8572,6 +8575,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T15439746
-- The selected file cannot be processed as media.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1707342767"] = "The selected file cannot be processed as media."
-- The audio track contains no audible signal, so there is nothing to transcribe.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T1988190152"] = "The audio track contains no audible signal, so there is nothing to transcribe."
-- The media file is damaged or its format could not be identified.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::MEDIATRANSCRIPTIONSERVICE::T2004316549"] = "The media file is damaged or its format could not be identified."

1
app/MindWork AI Studio/Components/AssistantBlock.razor.cs
View File

@ -113,6 +113,7 @@ public partial class AssistantBlock<TSettings> : MSGComponentBase where TSetting
{
MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => new(Icons.Material.Filled.ChangeCircle, Color.Info, this.T("Media is still being prepared.")),
MediaImportStatus.SUCCEEDED => new(Icons.Material.Filled.TaskAlt, Color.Success, this.T("The media transcript is ready.")),
MediaImportStatus.WARNING => new(Icons.Material.Filled.WarningAmber, Color.Warning, this.T("Media transcription completed with a warning. Open the assistant to review it.")),
MediaImportStatus.FAILED => new(Icons.Material.Filled.Error, Color.Error, this.T("Media transcription failed. Open the assistant to review it.")),
MediaImportStatus.CANCELLED => new(Icons.Material.Filled.Cancel, Color.Warning, this.T("Media transcription was canceled. Open the assistant to review it.")),

6
app/MindWork AI Studio/Components/AttachDocuments.razor.cs
View File

@ -161,6 +161,12 @@ public partial class AttachDocuments : MSGComponentBase
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed.")));
}
if (outcome.Warnings.Count > 0)
{
var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}"));
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message));
}
if (outcome.Status is MediaImportStatus.CANCELLED)
{
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled.")));

6
app/MindWork AI Studio/Components/ChatComponent.razor.cs
View File

@ -285,6 +285,12 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed.")));
}
if (outcome.Warnings.Count > 0)
{
var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}"));
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message));
}
if (outcome.Status is MediaImportStatus.CANCELLED)
{
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled.")));

6
app/MindWork AI Studio/Components/ReadFileContent.razor.cs
View File

@ -125,6 +125,12 @@ public partial class ReadFileContent : MSGComponentBase
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The media file could not be transcribed.")));
}
if (outcome.Warnings.Count > 0)
{
var message = string.Join(Environment.NewLine, outcome.Warnings.Select(warning => $"{warning.FileName}: {warning.UserMessage}"));
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, message));
}
if (outcome.Status is MediaImportStatus.CANCELLED)
{
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, this.T("The media transcription was canceled.")));

9
app/MindWork AI Studio/Components/VoiceRecorder.razor.cs
View File

@ -345,9 +345,16 @@ public partial class VoiceRecorder : MSGComponentBase
var transcriptionResult = await this.MediaTranscriptionService.TranscribeVoiceAsync(this.finalRecordingPath);
if (transcriptionResult.Status is not MediaTranscriptionResultStatus.SUCCEEDED)
{
this.Logger.LogWarning("The transcription request failed.");
if (transcriptionResult.Status is MediaTranscriptionResultStatus.CANCELLED)
return;
if (transcriptionResult.Status is MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL)
{
await this.MessageBus.SendWarning(new(Icons.Material.Filled.VoiceChat, transcriptionResult.UserMessage));
return;
}
this.Logger.LogWarning("The transcription request failed.");
var userMessage = string.IsNullOrWhiteSpace(transcriptionResult.UserMessage)
? this.T("Unfortunately, there was an error communicating with the AI system.")
: transcriptionResult.UserMessage;

2
app/MindWork AI Studio/Components/Workspaces.razor.cs
View File

@ -394,6 +394,7 @@ public partial class Workspaces : MSGComponentBase
{
MediaImportStatus.QUEUED or MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => " color: var(--mud-palette-info);",
MediaImportStatus.SUCCEEDED => " color: var(--mud-palette-success);",
MediaImportStatus.WARNING => " color: var(--mud-palette-warning);",
MediaImportStatus.FAILED => " color: var(--mud-palette-error);",
MediaImportStatus.CANCELLED => " color: var(--mud-palette-warning);",
_ => string.Empty,
@ -421,6 +422,7 @@ public partial class Workspaces : MSGComponentBase
MediaImportStatus.QUEUED => Icons.Material.Filled.HourglassTop,
MediaImportStatus.RUNNING or MediaImportStatus.CANCELING => Icons.Material.Filled.ChangeCircle,
MediaImportStatus.SUCCEEDED => Icons.Material.Filled.TaskAlt,
MediaImportStatus.WARNING => Icons.Material.Filled.WarningAmber,
MediaImportStatus.FAILED => Icons.Material.Filled.Error,
MediaImportStatus.CANCELLED => Icons.Material.Filled.Cancel,

2
app/MindWork AI Studio/Tools/Media/MediaImportOutcome.cs
View File

@ -8,4 +8,6 @@ public sealed record MediaImportOutcome
public required MediaImportStatus Status { get; init; }
public IReadOnlyList<MediaImportFailure> Failures { get; init; } = [];
public IReadOnlyList<MediaImportWarning> Warnings { get; init; } = [];
}

1
app/MindWork AI Studio/Tools/Media/MediaImportStatus.cs
View File

@ -7,6 +7,7 @@ public enum MediaImportStatus
RUNNING,
CANCELING,
SUCCEEDED,
WARNING,
FAILED,
CANCELLED,
}

4
app/MindWork AI Studio/Tools/Media/MediaImportWarning.cs Normal file
View File

@ -0,0 +1,4 @@
namespace AIStudio.Tools.Media;
/// <summary>One user-visible media warning retained until its owner is displayed.</summary>
public sealed record MediaImportWarning(string FileName, string UserMessage);

9
app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs
View File

@ -7,7 +7,7 @@ namespace AIStudio.Tools.Media;
/// </summary>
/// <param name="Status">Terminal operation status.</param>
/// <param name="Text">Transcript text for a successful operation.</param>
/// <param name="UserMessage">Localized message suitable for display after failure.</param>
/// <param name="UserMessage">Localized message suitable for display after a warning or failure.</param>
/// <param name="ErrorCode">Optional stable runtime failure category.</param>
public sealed record MediaTranscriptionResult(MediaTranscriptionResultStatus Status, string Text, string UserMessage, MediaJobErrorCode? ErrorCode = null)
{
@ -20,6 +20,13 @@ public sealed record MediaTranscriptionResult(MediaTranscriptionResultStatus Sta
/// <param name="errorCode">Optional runtime error category.</param>
public static MediaTranscriptionResult Failed(string userMessage, MediaJobErrorCode? errorCode = null) => new(MediaTranscriptionResultStatus.FAILED, string.Empty, userMessage, errorCode);
/// <summary>Creates a warning result for media without an audible signal.</summary>
/// <param name="userMessage">Localized visible warning.</param>
public static MediaTranscriptionResult NoAudibleSignal(string userMessage) => new(
MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL,
string.Empty,
userMessage);
/// <summary>Creates a cancelled result without relying on visible text.</summary>
public static MediaTranscriptionResult Cancelled() => new(MediaTranscriptionResultStatus.CANCELLED, string.Empty, string.Empty, MediaJobErrorCode.CANCELLED);
}

3
app/MindWork AI Studio/Tools/Media/MediaTranscriptionResultStatus.cs
View File

@ -10,6 +10,9 @@ public enum MediaTranscriptionResultStatus
/// <summary>The operation failed.</summary>
FAILED,
/// <summary>The media contains no signal above the practical-silence threshold.</summary>
NO_AUDIBLE_SIGNAL,
/// <summary>The caller or user cancelled the operation.</summary>
CANCELLED,

4
app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs
View File

@ -6,9 +6,11 @@ namespace AIStudio.Tools.Rust;
/// <param name="DetectedCodec">Selected codec diagnostic.</param>
/// <param name="DurationMs">Normalized duration in milliseconds.</param>
/// <param name="PassThrough">Whether the source was copied unchanged.</param>
/// <param name="HasAudibleSignal">Whether the normalized audio exceeds the practical-silence threshold.</param>
public sealed record MediaJobResult(
string OutputPath,
string DetectedFormat,
string DetectedCodec,
ulong DurationMs,
bool PassThrough);
bool PassThrough,
bool HasAudibleSignal);

34
app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs
View File

@ -192,6 +192,7 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM
var status = MediaImportStatus.SUCCEEDED;
List<MediaImportFailure> failures = [];
List<MediaImportWarning> warnings = [];
try
{
var result = await this.TranscribeImportAsync(mediaPath, target, cancellation.Token);
@ -199,6 +200,11 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM
this.AddCompletedText(target, result.Text);
else if (result.Status is MediaTranscriptionResultStatus.CANCELLED)
status = MediaImportStatus.CANCELLED;
else if (result.Status is MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL)
{
status = MediaImportStatus.WARNING;
warnings.Add(new(Path.GetFileName(mediaPath), result.UserMessage));
}
else
{
status = MediaImportStatus.FAILED;
@ -224,7 +230,7 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM
ownedCancellation.Dispose();
}
this.CompleteImport(target, Path.GetFileName(mediaPath), status, failures);
this.CompleteImport(target, Path.GetFileName(mediaPath), status, failures, warnings);
}
}
@ -252,6 +258,7 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM
var status = MediaImportStatus.SUCCEEDED;
var currentFileName = Path.GetFileName(mediaPaths[0]);
List<MediaImportFailure> failures = [];
List<MediaImportWarning> warnings = [];
try
{
@ -266,6 +273,15 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM
break;
}
if (result.Status is MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL)
{
if (status is MediaImportStatus.SUCCEEDED)
status = MediaImportStatus.WARNING;
warnings.Add(new(currentFileName, result.UserMessage));
continue;
}
if (result.Status is not MediaTranscriptionResultStatus.SUCCEEDED)
{
status = MediaImportStatus.FAILED;
@ -307,7 +323,7 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM
cancellation.Dispose();
}
this.CompleteImport(target, currentFileName, status, failures);
this.CompleteImport(target, currentFileName, status, failures, warnings);
}
}
@ -430,6 +446,12 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM
? MediaTranscriptionResult.Failed(TB("The media pipeline ended without an output file."))
: MediaTranscriptionResult.Failed(UserMessageFor(normalized.Error.Code), normalized.Error.Code);
if (!normalized.Result.HasAudibleSignal)
{
logger.LogInformation("Skipping transcription for '{MediaPath}' because its maximum audio peak does not exceed the practical-silence threshold.", mediaPath);
return MediaTranscriptionResult.NoAudibleSignal(TB("The audio track contains no audible signal, so there is nothing to transcribe."));
}
var providerSettings = this.ResolveProvider();
if (providerSettings is null)
return MediaTranscriptionResult.Failed(TB("No usable transcription provider is configured."));
@ -602,7 +624,12 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM
}
/// <summary>Publishes one retained terminal result after an entire target batch ended.</summary>
private void CompleteImport(MediaImportTarget target, string fileName, MediaImportStatus status, IReadOnlyList<MediaImportFailure> failures)
private void CompleteImport(
MediaImportTarget target,
string fileName,
MediaImportStatus status,
IReadOnlyList<MediaImportFailure> failures,
IReadOnlyList<MediaImportWarning> warnings)
{
lock (this.stateLock)
{
@ -621,6 +648,7 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM
Owner = target.Owner,
Status = status,
Failures = [.. failures],
Warnings = [.. warnings],
};
}

214
runtime/src/media.rs
View File

@ -68,6 +68,9 @@ const PROGRESS_EVENT_INTERVAL: StdDuration = StdDuration::from_secs(6);
/// Timestamp differences above this threshold are recorded as discontinuities.
const LARGE_DISCONTINUITY_MS: i64 = 1_000;
/// Maximum full-scale peak still treated as practical silence.
const SILENCE_MAX_PEAK_DBFS: f32 = -60.0;
/// Time a terminal job remains available for late SSE subscribers.
const TERMINAL_JOB_RETENTION: std::time::Duration = std::time::Duration::from_secs(10 * 60);
@ -227,6 +230,9 @@ pub struct MediaJobResult {
/// Whether the input was copied unchanged.
pub pass_through: bool,
/// Whether the normalized audio exceeds the practical-silence threshold.
pub has_audible_signal: bool,
}
/// Stable error code plus an English diagnostic intended for logs.
@ -570,6 +576,9 @@ fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_throu
.ok_or_else(|| MediaError::new(MediaErrorCode::UnsupportedCodec, "None of the audio tracks uses a supported codec."))?;
let track_id = selected.id;
let track_delay = selected.delay.unwrap_or(0);
let track_padding = selected.padding.unwrap_or(0);
let track_time_base = selected.time_base;
let params = selected.codec_params.as_ref().and_then(CodecParameters::audio).unwrap().clone();
let detected_codec = if params.codec == CODEC_ID_OPUS { "opus".to_string() } else { format!("{}", params.codec) };
let track_duration_ms = selected.num_frames.zip(params.sample_rate)
@ -610,6 +619,18 @@ fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_throu
}
let result = if pass_through {
job.publish_progress(MediaJobPhase::Transcoding, Some(0.0));
let analysis_context = SignalAnalysisContext {
track_id,
params: &params,
track_delay,
track_padding,
expected_duration_ms: duration_ms,
time_base: track_time_base,
source_progress: &source_progress,
job,
};
let signal = analyze_audio_signal(&mut *format, analysis_context)?;
copy_with_cancellation(input_path, &partial_path, job)?;
Ok(MediaJobResult {
output_path: output_path.to_string_lossy().into_owned(),
@ -617,21 +638,22 @@ fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_throu
detected_codec,
duration_ms,
pass_through: true,
has_audible_signal: signal.has_audible_signal(),
})
} else {
job.publish_progress(MediaJobPhase::Transcoding, Some(0.0));
let context = TranscodeContext {
track_id,
track_delay: selected.delay.unwrap_or(0),
track_padding: selected.padding.unwrap_or(0),
track_delay,
track_padding,
params,
partial_path: &partial_path,
output_path,
detected_format,
detected_codec,
expected_duration_ms: duration_ms,
time_base: selected.time_base,
time_base: track_time_base,
source_progress,
job,
};
@ -685,6 +707,140 @@ fn is_decodable(track: &Track) -> bool {
params.codec == CODEC_ID_OPUS || symphonia::default::get_codecs().make_audio_decoder(params, &AudioDecoderOptions::default()).is_ok()
}
/// Streaming peak measurement over normalized full-scale floating-point samples.
#[derive(Default)]
struct AudioPeakDetector {
/// Highest absolute sample observed so far.
max_amplitude: f32,
}
impl AudioPeakDetector {
/// Includes one bounded sample block in the maximum-peak measurement.
fn observe(&mut self, samples: &[f32]) {
for sample in samples {
let amplitude = sample.abs();
self.max_amplitude = if amplitude.is_finite() {
self.max_amplitude.max(amplitude)
} else {
f32::INFINITY
};
}
}
/// Returns whether any retained sample exceeds the configured silence ceiling.
fn has_audible_signal(&self) -> bool {
self.max_amplitude > 10.0_f32.powf(SILENCE_MAX_PEAK_DBFS / 20.0)
}
/// Returns the measured full-scale peak for diagnostics.
fn max_peak_dbfs(&self) -> f32 {
if self.max_amplitude == 0.0 {
f32::NEG_INFINITY
} else {
20.0 * self.max_amplitude.log10()
}
}
}
/// Inputs required to scan an otherwise pass-through-compatible audio track.
struct SignalAnalysisContext<'a> {
/// Selected track identifier.
track_id: u32,
/// Selected track codec parameters.
params: &'a symphonia::core::codecs::audio::AudioCodecParameters,
/// Leading decoded frames to discard.
track_delay: u32,
/// Trailing decoded frames to discard.
track_padding: u32,
/// Container duration used for progress reporting.
expected_duration_ms: u64,
/// Selected track timebase used for progress reporting.
time_base: Option<TimeBase>,
/// Sequential byte progress fallback when the track has no duration.
source_progress: &'a SourceProgress,
/// Cancellation and progress state for the job.
job: &'a MediaJob,
}
/// Decodes an otherwise pass-through-compatible track solely to classify practical silence.
fn analyze_audio_signal(
format: &mut dyn symphonia::core::formats::FormatReader,
context: SignalAnalysisContext<'_>,
) -> Result<AudioPeakDetector, MediaError> {
let mut decoder = StreamDecoder::new(context.params, context.track_delay)?;
let mut detector = AudioPeakDetector::default();
let mut decoded_tail = Vec::<f32>::new();
let mut first_packet_pts = None::<i64>;
let mut decoded_packets = 0u64;
let mut last_progress = 0.0f64;
loop {
check_cancelled(context.job)?;
let packet = match format.next_packet() {
Ok(Some(packet)) => packet,
Ok(None) => break,
Err(SymphoniaError::ResetRequired) => return Err(MediaError::new(MediaErrorCode::StreamReset, "The media stream changed unexpectedly.")),
Err(SymphoniaError::IoError(error)) if error.kind() == std::io::ErrorKind::Interrupted && context.job.is_cancelled() => {
return Err(MediaError::new(MediaErrorCode::Cancelled, "The media job was cancelled."));
}
Err(SymphoniaError::IoError(error)) if error.kind() == std::io::ErrorKind::UnexpectedEof => break,
Err(error) => return Err(MediaError::new(MediaErrorCode::DamagedContainer, format!("The media container is damaged: {error}"))),
};
if packet.track_id != context.track_id {
continue;
}
let packet_pts = packet.pts.get();
first_packet_pts.get_or_insert(packet_pts);
let Some((mono, _)) = decoder.decode(&packet)? else { continue; };
decoded_packets += 1;
decoded_tail.extend_from_slice(&mono);
let emit_len = decoded_tail.len().saturating_sub(context.track_padding as usize);
if emit_len > 0 {
detector.observe(&decoded_tail[..emit_len]);
drop(decoded_tail.drain(..emit_len));
}
let timestamp_ms = packet_timestamp_ms(packet_pts, first_packet_pts, context.time_base);
let progress = if context.expected_duration_ms > 0 {
timestamp_ms.map(|current_ms| (current_ms as f64 / context.expected_duration_ms as f64).clamp(0.0, 0.99))
} else {
context.source_progress.fraction()
};
if let Some(progress) = progress {
last_progress = last_progress.max(progress);
}
context.job.publish_progress(MediaJobPhase::Transcoding, progress.map(|_| last_progress));
}
if decoded_packets == 0 {
return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The selected audio track did not yield decoded audio samples."));
}
log::info!(
"media signal analysis completed: track_id={}, max_peak_dbfs={}, silence_threshold_dbfs={}, has_audible_signal={}",
context.track_id,
detector.max_peak_dbfs(),
SILENCE_MAX_PEAK_DBFS,
detector.has_audible_signal(),
);
Ok(detector)
}
/// Immutable inputs shared across a single transcoding operation.
struct TranscodeContext<'a> {
/// Selected input track identifier.
@ -739,6 +895,7 @@ fn transcode(
let file = File::create(context.partial_path).map_err(|error| MediaError::new(MediaErrorCode::OutputCreateFailed, error.to_string()))?;
let mut writer = WebmOpusWriter::new(file)?;
let mut pending = Vec::<f32>::with_capacity(OPUS_FRAME_SAMPLES * 3);
let mut signal = AudioPeakDetector::default();
let mut resampler: Option<StreamResampler> = None;
let mut decoded_tail = Vec::<f32>::new();
let mut encoded = [0u8; 4_000];
@ -805,6 +962,7 @@ fn transcode(
.flatten();
let current_start = produced_samples.saturating_add(pending.len() as u64);
let previous_pending_len = pending.len();
append_timestamp_aligned(
&mut pending,
&resampled,
@ -814,6 +972,8 @@ fn transcode(
packet_pts,
&mut discontinuities,
);
signal.observe(&pending[previous_pending_len..]);
}
encode_complete_frames(&mut pending, &mut opus_encoder, &mut writer, &mut encoded, &mut produced_samples, context.job)?;
@ -834,7 +994,9 @@ fn transcode(
if let Some(stream_resampler) = resampler.as_mut() {
// Discard the retained decoded tail (container padding), then flush the filter delay.
pending.extend_from_slice(&stream_resampler.finish()?);
let flushed = stream_resampler.finish()?;
signal.observe(&flushed);
pending.extend_from_slice(&flushed);
} else {
return Err(MediaError::new(MediaErrorCode::InvalidAudioParameters, "The selected audio track did not yield decoded audio parameters."));
}
@ -864,7 +1026,7 @@ fn transcode(
}
log::info!(
"media transcode completed: track_id={}, decoded_packets={}, discarded_packets={}, recoverable_decode_errors={}, first_pts={:?}, last_pts={:?}, discontinuities={}, output_bytes={}, duration_ms={}",
"media transcode completed: track_id={}, decoded_packets={}, discarded_packets={}, recoverable_decode_errors={}, first_pts={:?}, last_pts={:?}, discontinuities={}, output_bytes={}, duration_ms={}, max_peak_dbfs={}, silence_threshold_dbfs={}, has_audible_signal={}",
context.track_id,
decoded_packets,
discarded_packets,
@ -874,6 +1036,9 @@ fn transcode(
discontinuities,
output_size,
output_duration_ms,
signal.max_peak_dbfs(),
SILENCE_MAX_PEAK_DBFS,
signal.has_audible_signal(),
);
Ok(MediaJobResult {
@ -882,6 +1047,7 @@ fn transcode(
detected_codec: context.detected_codec,
duration_ms: output_duration_ms,
pass_through: false,
has_audible_signal: signal.has_audible_signal(),
})
}
@ -1550,6 +1716,18 @@ mod tests {
assert_eq!(downmix_to_mono(&[1.0, -1.0, 0.5, 0.5], 2), vec![0.0, 0.5]);
}
/// Verifies the configured dBFS ceiling is inclusive and a higher peak is audible.
#[test]
fn practical_silence_uses_the_configured_maximum_peak() {
let threshold = 10.0_f32.powf(SILENCE_MAX_PEAK_DBFS / 20.0);
let mut detector = AudioPeakDetector::default();
detector.observe(&[-threshold, threshold]);
assert!(!detector.has_audible_signal());
detector.observe(&[threshold * 1.01]);
assert!(detector.has_audible_signal());
}
/// Verifies timestamp gaps become silence and overlaps do not duplicate decoded samples.
#[test]
fn timestamp_alignment_inserts_gaps_and_trims_overlaps() {
@ -1612,6 +1790,7 @@ mod tests {
let job = MediaJob::new();
let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap();
assert!(!result.pass_through);
assert!(!result.has_audible_signal);
assert!(result.duration_ms.abs_diff(100) <= 20);
let file = File::open(&output).unwrap();
@ -1656,6 +1835,20 @@ mod tests {
let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap();
assert!(result.pass_through);
assert_eq!(fs::read(input).unwrap(), fs::read(output).unwrap());
assert!(!result.has_audible_signal);
let _ = fs::remove_dir_all(directory);
}
/// Verifies an above-threshold PCM peak survives normalization classification.
#[test]
fn audible_wav_is_not_classified_as_silence() {
let directory = std::env::temp_dir().join(format!("ai-studio-media-audible-{}", rand::random::<u64>()));
fs::create_dir_all(&directory).unwrap();
let input = directory.join("input.wav");
let output = directory.join("output.webm");
fs::write(&input, wav_constant(48_000, 960, 1_000)).unwrap();
let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &MediaJob::new()).unwrap();
assert!(result.has_audible_signal);
let _ = fs::remove_dir_all(directory);
}
@ -1731,6 +1924,11 @@ mod tests {
/// Constructs a minimal mono 16-bit PCM WAV fixture in memory.
fn wav_silence(sample_rate: u32, samples: u32) -> Vec<u8> {
wav_constant(sample_rate, samples, 0)
}
/// Constructs a minimal mono 16-bit PCM WAV containing one constant sample value.
fn wav_constant(sample_rate: u32, samples: u32, sample: i16) -> Vec<u8> {
let data_size = samples * 2;
let mut wav = Vec::with_capacity(44 + data_size as usize);
wav.extend_from_slice(b"RIFF");
@ -1745,7 +1943,9 @@ mod tests {
wav.extend_from_slice(&16u16.to_le_bytes());
wav.extend_from_slice(b"data");
wav.extend_from_slice(&data_size.to_le_bytes());
wav.resize(44 + data_size as usize, 0);
for _ in 0..samples {
wav.extend_from_slice(&sample.to_le_bytes());
}
wav
}
}
}