From 169d47489244a49f6c48999654a97bff5ae6d651 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Tue, 14 Jul 2026 21:09:45 +0200 Subject: [PATCH] Switched voice recording to use PCM/WAV format --- .../Components/VoiceRecorder.razor.cs | 99 ++--- .../Provider/BaseProvider.cs | 6 +- .../Tools/AudioRecordingResult.cs | 8 - .../Tools/Rust/MediaJobResult.cs | 4 + .../Services/MediaTranscriptionService.cs | 91 ++++- .../wwwroot/audio-recorder-worklet.js | 61 +++ app/MindWork AI Studio/wwwroot/audio.js | 352 +++++++++++------- runtime/src/media.rs | 20 + 8 files changed, 437 insertions(+), 204 deletions(-) delete mode 100644 app/MindWork AI Studio/Tools/AudioRecordingResult.cs create mode 100644 app/MindWork AI Studio/wwwroot/audio-recorder-worklet.js diff --git a/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs b/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs index 8cfa1362..f754695f 100644 --- a/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs +++ b/app/MindWork AI Studio/Components/VoiceRecorder.razor.cs @@ -1,6 +1,7 @@ +using System.Buffers.Binary; + using AIStudio.Settings.DataModel; using AIStudio.Tools.Media; -using AIStudio.Tools.MIME; using AIStudio.Tools.Rust; using AIStudio.Tools.Services; @@ -10,6 +11,8 @@ namespace AIStudio.Components; public partial class VoiceRecorder : MSGComponentBase { + private const int PCM_WAV_HEADER_SIZE = 44; + [Inject] private ILogger Logger { get; init; } = null!; @@ -96,7 +99,6 @@ public partial class VoiceRecorder : MSGComponentBase private bool isTranscribing; private FileStream? currentRecordingStream; private string? currentRecordingPath; - private string? currentRecordingMimeType; private string? finalRecordingPath; private DotNetObjectReference? dotNetReference; @@ -134,18 +136,7 @@ public partial class VoiceRecorder : MSGComponentBase return; } - var mimeTypes = GetPreferredMimeTypes( - Builder.Create().UseAudio().UseSubtype(AudioSubtype.WEBM).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.OGG).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.MP4).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.AAC).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.MP3).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.AIFF).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.WAV).Build(), - Builder.Create().UseAudio().UseSubtype(AudioSubtype.FLAC).Build() - ); - - this.Logger.LogInformation("Starting audio recording with preferred MIME types: '{PreferredMimeTypes}'.", string.Join(", ", mimeTypes)); + this.Logger.LogInformation("Starting PCM/WAV audio recording."); // Create a DotNetObjectReference to pass to JavaScript: this.dotNetReference = DotNetObjectReference.Create(this); @@ -155,13 +146,8 @@ public partial class VoiceRecorder : MSGComponentBase try { - string[] mimeTypeStrings = ["audio/webm;codecs=opus", .. mimeTypes.ToStringArray()]; - var actualMimeType = await this.JsRuntime.InvokeAsync("audioRecorder.start", this.dotNetReference, mimeTypeStrings); - - // Store the MIME type for later use: - this.currentRecordingMimeType = actualMimeType; - - this.Logger.LogInformation("Audio recording started with MIME type: '{ActualMimeType}'.", actualMimeType); + await this.JsRuntime.InvokeVoidAsync("audioRecorder.start", this.dotNetReference); + this.Logger.LogInformation("PCM/WAV audio recording started."); this.isPreparing = false; this.isRecording = true; } @@ -184,9 +170,7 @@ public partial class VoiceRecorder : MSGComponentBase var recordingStoppedSuccessfully = false; try { - var result = await this.JsRuntime.InvokeAsync("audioRecorder.stop"); - if (result.ChangedMimeType) - this.Logger.LogWarning("The recorded audio MIME type was changed to '{ResultMimeType}'.", result.MimeType); + await this.JsRuntime.InvokeVoidAsync("audioRecorder.stop"); recordingStoppedSuccessfully = true; } catch (Exception e) @@ -218,24 +202,6 @@ public partial class VoiceRecorder : MSGComponentBase } } - private static MIMEType[] GetPreferredMimeTypes(params MIMEType[] mimeTypes) - { - // Default list if no parameters provided: - if (mimeTypes.Length is 0) - { - var audioBuilder = Builder.Create().UseAudio(); - return - [ - audioBuilder.UseSubtype(AudioSubtype.WEBM).Build(), - audioBuilder.UseSubtype(AudioSubtype.OGG).Build(), - audioBuilder.UseSubtype(AudioSubtype.MP4).Build(), - audioBuilder.UseSubtype(AudioSubtype.MPEG).Build(), - ]; - } - - return mimeTypes; - } - private async Task InitializeRecordingStream() { this.numReceivedChunks = 0; @@ -244,7 +210,7 @@ public partial class VoiceRecorder : MSGComponentBase if (!Directory.Exists(recordingDirectory)) Directory.CreateDirectory(recordingDirectory); - var fileName = $"recording_{DateTime.UtcNow:yyyyMMdd_HHmmss}.audio"; + var fileName = $"recording_{DateTime.UtcNow:yyyyMMdd_HHmmss}.wav"; this.currentRecordingPath = Path.Combine(recordingDirectory, fileName); this.currentRecordingStream = new FileStream(this.currentRecordingPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 8192, useAsync: true); @@ -281,6 +247,7 @@ public partial class VoiceRecorder : MSGComponentBase if (this.currentRecordingStream is not null) { await this.currentRecordingStream.FlushAsync(); + var hasPcmAudioData = await this.FinalizePcmWavHeaderAsync(this.currentRecordingStream); await this.currentRecordingStream.DisposeAsync(); this.currentRecordingStream = null; @@ -288,44 +255,48 @@ public partial class VoiceRecorder : MSGComponentBase { var fileSize = new FileInfo(this.currentRecordingPath).Length; - // Rename non-empty recordings with the correct extension based on MIME type: - if (fileSize > 0 && this.currentRecordingMimeType is not null) + if (hasPcmAudioData) { - var extension = GetFileExtension(this.currentRecordingMimeType); - var newPath = Path.ChangeExtension(this.currentRecordingPath, extension); - File.Move(this.currentRecordingPath, newPath, overwrite: true); - this.finalRecordingPath = newPath; - this.Logger.LogInformation("Finalized audio recording over {NumChunks} streamed audio chunks to the file '{RecordingPath}' with {FileSize} bytes.", this.numReceivedChunks, newPath, fileSize); + this.finalRecordingPath = this.currentRecordingPath; + this.Logger.LogInformation("Finalized audio recording over {NumChunks} streamed audio chunks to the file '{RecordingPath}' with {FileSize} bytes.", this.numReceivedChunks, this.currentRecordingPath, fileSize); } else { - this.Logger.LogWarning("Discarding an audio recording with {FileSize} bytes and MIME type '{MimeType}'.", fileSize, this.currentRecordingMimeType); + this.Logger.LogWarning("Discarding a PCM/WAV audio recording without audio data ({FileSize} bytes).", fileSize); File.Delete(this.currentRecordingPath); } } } this.currentRecordingPath = null; - this.currentRecordingMimeType = null; // Dispose the .NET reference: this.dotNetReference?.Dispose(); this.dotNetReference = null; } - private static string GetFileExtension(string mimeType) + private async Task FinalizePcmWavHeaderAsync(FileStream recordingStream) { - var baseMimeType = mimeType.Split(';')[0].Trim().ToLowerInvariant(); - return baseMimeType switch - { - "audio/webm" => ".webm", - "audio/ogg" => ".ogg", - "audio/mp4" => ".m4a", - "audio/mpeg" => ".mp3", - "audio/wav" => ".wav", - "audio/x-wav" => ".wav", - _ => ".audio" // Fallback - }; + if (recordingStream.Length <= PCM_WAV_HEADER_SIZE) + return false; + + var pcmDataSize = recordingStream.Length - PCM_WAV_HEADER_SIZE; + if (pcmDataSize > uint.MaxValue - 36) + throw new InvalidDataException("The streamed PCM recording exceeds the WAV size limit."); + + var valueBuffer = new byte[sizeof(uint)]; + BinaryPrimitives.WriteUInt32LittleEndian(valueBuffer, checked((uint)(36 + pcmDataSize))); + recordingStream.Seek(4, SeekOrigin.Begin); + await recordingStream.WriteAsync(valueBuffer); + + BinaryPrimitives.WriteUInt32LittleEndian(valueBuffer, checked((uint)pcmDataSize)); + recordingStream.Seek(40, SeekOrigin.Begin); + await recordingStream.WriteAsync(valueBuffer); + recordingStream.Seek(0, SeekOrigin.End); + await recordingStream.FlushAsync(); + + this.Logger.LogInformation("Finalized a streamed PCM/WAV header for {PcmDataSize} bytes of audio data.", pcmDataSize); + return true; } private async Task TranscribeRecordingAsync() diff --git a/app/MindWork AI Studio/Provider/BaseProvider.cs b/app/MindWork AI Studio/Provider/BaseProvider.cs index dd1ea7ae..4ad26580 100644 --- a/app/MindWork AI Studio/Provider/BaseProvider.cs +++ b/app/MindWork AI Studio/Provider/BaseProvider.cs @@ -1069,7 +1069,11 @@ public abstract class BaseProvider : IProvider, ISecretId request.Headers.Authorization = new AuthenticationHeaderValue("Bearer", await requestedSecret.Secret.Decrypt(Program.ENCRYPTION)); break; } - + + this.logger.LogInformation("Uploading transcription media '{FileName}' with content type '{ContentType}' and {FileSize} bytes.", + Path.GetFileName(audioFilePath), + mimeType.TextRepresentation, + fileStream.Length); using var response = await this.HttpClient.SendAsync(request, token); var responseBody = await response.Content.ReadAsStringAsync(token); diff --git a/app/MindWork AI Studio/Tools/AudioRecordingResult.cs b/app/MindWork AI Studio/Tools/AudioRecordingResult.cs deleted file mode 100644 index cdde82ac..00000000 --- a/app/MindWork AI Studio/Tools/AudioRecordingResult.cs +++ /dev/null @@ -1,8 +0,0 @@ -namespace AIStudio.Tools; - -public sealed class AudioRecordingResult -{ - public string MimeType { get; init; } = string.Empty; - - public bool ChangedMimeType { get; init; } -} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs b/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs index 3552071c..3dc9d551 100644 --- a/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs +++ b/app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs @@ -2,6 +2,8 @@ namespace AIStudio.Tools.Rust; /// Successful terminal result returned by Rust media normalization. /// Committed normalized output path. +/// Stable normalized container used for provider uploads. +/// Stable normalized audio codec used for provider uploads. /// Detected container diagnostic. /// Selected codec diagnostic. /// Normalized duration in milliseconds. @@ -9,6 +11,8 @@ namespace AIStudio.Tools.Rust; /// Whether the normalized audio exceeds the practical-silence threshold. public sealed record MediaJobResult( string OutputPath, + string OutputFormat, + string OutputCodec, string DetectedFormat, string DetectedCodec, ulong DurationMs, diff --git a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs index 1859429c..726bfbb9 100644 --- a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs +++ b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs @@ -12,6 +12,11 @@ namespace AIStudio.Tools.Services; /// public sealed class MediaTranscriptionService(RustService rustService, SettingsManager settingsManager, ILogger logger) : IDisposable { + private const string NORMALIZED_OUTPUT_EXTENSION = ".webm"; + private const string NORMALIZED_OUTPUT_FORMAT = "webm"; + private const string NORMALIZED_OUTPUT_CODEC = "opus"; + private static readonly byte[] WEBM_EBML_SIGNATURE = [0x1A, 0x45, 0xDF, 0xA3]; + /// Serializes attachment and file-content imports. private readonly SemaphoreSlim importQueue = new(1, 1); @@ -446,6 +451,13 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM ? MediaTranscriptionResult.Failed(TB("The media pipeline ended without an output file.")) : MediaTranscriptionResult.Failed(UserMessageFor(normalized.Error.Code), normalized.Error.Code); + var uploadContractError = await ValidateNormalizedProviderUploadAsync(normalized.Result, normalizedPath, operation.Cancellation.Token); + if (uploadContractError is not null) + { + logger.LogError("Refusing the transcription provider upload because the normalized media contract validation failed: {Diagnostic}", uploadContractError); + return MediaTranscriptionResult.Failed(TB("The media pipeline ended without an output file.")); + } + if (!normalized.Result.HasAudibleSignal) { logger.LogInformation("Skipping transcription for '{MediaPath}' because its maximum audio peak does not exceed the practical-silence threshold.", mediaPath); @@ -463,12 +475,21 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM if (provider.Provider is LLMProviders.NONE) return MediaTranscriptionResult.Failed(TB("The configured transcription provider could not be created.")); - logger.LogInformation("Transcribing normalized media '{MediaPath}' with provider '{Provider}' and model '{Model}'.", + var sourceSize = File.Exists(mediaPath) ? new FileInfo(mediaPath).Length : 0; + var normalizedSize = new FileInfo(normalizedPath).Length; + var reductionPercent = sourceSize > 0 + ? (1.0 - (double)normalizedSize / sourceSize) * 100.0 + : 0.0; + logger.LogInformation("Transcribing normalized WebM/Opus media '{NormalizedPath}' ({NormalizedSize} bytes; source '{SourcePath}' {SourceSize} bytes; size reduction {ReductionPercent:F1}%) with provider '{Provider}' and model '{Model}'.", + normalizedPath, + normalizedSize, mediaPath, + sourceSize, + reductionPercent, providerSettings.UsedLLMProvider, providerSettings.Model); - var providerResult = await provider.TranscribeAudioAsync(providerSettings.Model, normalized.Result.OutputPath, settingsManager, operation.Cancellation.Token); + var providerResult = await provider.TranscribeAudioAsync(providerSettings.Model, normalizedPath, settingsManager, operation.Cancellation.Token); operation.Cancellation.Token.ThrowIfCancellationRequested(); if (!providerResult.Success) { @@ -498,6 +519,72 @@ public sealed class MediaTranscriptionService(RustService rustService, SettingsM } } + /// Validates the fail-closed WebM/Opus contract before provider upload. + private static async Task ValidateNormalizedProviderUploadAsync(MediaJobResult result, string expectedOutputPath, CancellationToken token) + { + if (string.IsNullOrWhiteSpace(result.OutputPath)) + return "Rust returned an empty normalized output path."; + + string actualFullPath; + string expectedFullPath; + try + { + actualFullPath = Path.GetFullPath(result.OutputPath); + expectedFullPath = Path.GetFullPath(expectedOutputPath); + } + catch (Exception exception) + { + return $"The normalized output path is invalid: {exception.Message}"; + } + + var pathComparison = OperatingSystem.IsWindows() + ? StringComparison.OrdinalIgnoreCase + : StringComparison.Ordinal; + + if (!string.Equals(actualFullPath, expectedFullPath, pathComparison)) + return $"Rust returned the unexpected output path '{result.OutputPath}' instead of '{expectedOutputPath}'."; + + if (!string.Equals(Path.GetExtension(actualFullPath), NORMALIZED_OUTPUT_EXTENSION, StringComparison.OrdinalIgnoreCase)) + return $"The normalized output path '{actualFullPath}' does not use the required '{NORMALIZED_OUTPUT_EXTENSION}' extension."; + + if (!string.Equals(result.OutputFormat, NORMALIZED_OUTPUT_FORMAT, StringComparison.Ordinal)) + return $"Rust returned output format '{result.OutputFormat}' instead of '{NORMALIZED_OUTPUT_FORMAT}'."; + + if (!string.Equals(result.OutputCodec, NORMALIZED_OUTPUT_CODEC, StringComparison.Ordinal)) + return $"Rust returned output codec '{result.OutputCodec}' instead of '{NORMALIZED_OUTPUT_CODEC}'."; + + if (!File.Exists(actualFullPath)) + return $"The normalized output file '{actualFullPath}' does not exist."; + + var header = new byte[WEBM_EBML_SIGNATURE.Length]; + var bytesRead = 0; + try + { + await using var stream = File.OpenRead(actualFullPath); + while (bytesRead < header.Length) + { + var count = await stream.ReadAsync(header.AsMemory(bytesRead), token); + if (count is 0) + break; + + bytesRead += count; + } + } + catch (IOException exception) + { + return $"The normalized output file '{actualFullPath}' could not be read: {exception.Message}"; + } + catch (UnauthorizedAccessException exception) + { + return $"The normalized output file '{actualFullPath}' could not be read: {exception.Message}"; + } + + if (bytesRead != header.Length || !header.AsSpan().SequenceEqual(WEBM_EBML_SIGNATURE)) + return $"The normalized output file '{actualFullPath}' does not begin with the WebM/Matroska EBML signature."; + + return null; + } + /// Runs the Rust normalization job and drains cancellation to a terminal event. /// Source media path. /// Owned temporary output path. diff --git a/app/MindWork AI Studio/wwwroot/audio-recorder-worklet.js b/app/MindWork AI Studio/wwwroot/audio-recorder-worklet.js new file mode 100644 index 00000000..c6a10219 --- /dev/null +++ b/app/MindWork AI Studio/wwwroot/audio-recorder-worklet.js @@ -0,0 +1,61 @@ +class PCMRecorderProcessor extends AudioWorkletProcessor { + constructor(options) { + super(); + + const chunkDurationSeconds = options.processorOptions?.chunkDurationSeconds || 3; + this.chunkSamples = Math.max(128, Math.round(sampleRate * chunkDurationSeconds)); + this.samples = new Int16Array(this.chunkSamples); + this.numSamples = 0; + + this.port.onmessage = event => { + if (event.data?.type === 'flush') { + this.flush(); + this.port.postMessage({ type: 'flushed' }); + } + }; + } + + process(inputs) { + const channels = inputs[0]; + if (!channels || channels.length === 0) + return true; + + const numFrames = channels[0].length; + for (let frame = 0; frame < numFrames; frame++) { + let monoSample = 0; + for (const channel of channels) { + monoSample += channel[frame] || 0; + } + + monoSample = Math.max(-1, Math.min(1, monoSample / channels.length)); + this.samples[this.numSamples++] = monoSample < 0 + ? Math.round(monoSample * 0x8000) + : Math.round(monoSample * 0x7fff); + + if (this.numSamples === this.chunkSamples) + this.flush(); + } + + return true; + } + + flush() { + if (this.numSamples === 0) + return; + + const buffer = new ArrayBuffer(this.numSamples * 2); + const view = new DataView(buffer); + for (let index = 0; index < this.numSamples; index++) { + view.setInt16(index * 2, this.samples[index], true); + } + + this.port.postMessage({ + type: 'chunk', + buffer: buffer, + sampleCount: this.numSamples, + }, [buffer]); + this.numSamples = 0; + } +} + +registerProcessor('pcm-recorder-processor', PCMRecorderProcessor); \ No newline at end of file diff --git a/app/MindWork AI Studio/wwwroot/audio.js b/app/MindWork AI Studio/wwwroot/audio.js index 61ccc7b6..a2fd4d8f 100644 --- a/app/MindWork AI Studio/wwwroot/audio.js +++ b/app/MindWork AI Studio/wwwroot/audio.js @@ -180,25 +180,208 @@ window.playSound = async function(soundPath) { } }; -let mediaRecorder; -let actualRecordingMimeType; -let changedMimeType = false; let pendingChunkUploads = 0; let chunkUploadPromise = Promise.resolve(); let chunkUploadError = null; let recordingError = null; +let captureAudioContext = null; +let captureSourceNode = null; +let captureWorkletNode = null; +let captureSilentGainNode = null; +let pcmFlushResolve = null; +let pcmSamplesReceived = 0; // Store the media stream so we can close the microphone later: let activeMediaStream = null; // Delay in milliseconds to wait after getUserMedia() for Bluetooth profile switch (A2DP → HFP): const BLUETOOTH_PROFILE_SWITCH_DELAY_MS = 1_600; +const PCM_SAMPLE_RATE = 48_000; +const PCM_CHUNK_DURATION_SECONDS = 3; +const PCM_FLUSH_TIMEOUT_MS = 5_000; + +function queueAudioChunkUpload(upload) { + pendingChunkUploads++; + chunkUploadPromise = chunkUploadPromise + .then(upload) + .catch(error => { + chunkUploadError ??= error; + console.error('Error sending audio chunk to .NET:', error); + }) + .finally(() => pendingChunkUploads--); +} + +async function waitForAudioChunkUploads() { + let observedUploadPromise; + do { + observedUploadPromise = chunkUploadPromise; + await observedUploadPromise; + } while (pendingChunkUploads > 0 || observedUploadPromise !== chunkUploadPromise); +} + +function createPcmWavHeader(sampleRate) { + const buffer = new ArrayBuffer(44); + const view = new DataView(buffer); + + const writeAscii = (offset, value) => { + for (let index = 0; index < value.length; index++) { + view.setUint8(offset + index, value.charCodeAt(index)); + } + }; + + writeAscii(0, 'RIFF'); + view.setUint32(4, 0, true); // Finalized by .NET after all PCM data was written. + writeAscii(8, 'WAVE'); + writeAscii(12, 'fmt '); + view.setUint32(16, 16, true); + view.setUint16(20, 1, true); // PCM + view.setUint16(22, 1, true); // Mono + view.setUint32(24, sampleRate, true); + view.setUint32(28, sampleRate * 2, true); + view.setUint16(32, 2, true); + view.setUint16(34, 16, true); + writeAscii(36, 'data'); + view.setUint32(40, 0, true); // Finalized by .NET after all PCM data was written. + + return new Uint8Array(buffer); +} + +function observeAudioTrack(track) { + console.log('Audio recording - microphone track state:', { + label: track.label, + enabled: track.enabled, + muted: track.muted, + readyState: track.readyState, + settings: typeof track.getSettings === 'function' ? track.getSettings() : null, + }); + + track.addEventListener('mute', () => console.warn('Audio recording - microphone track was muted.')); + track.addEventListener('unmute', () => console.log('Audio recording - microphone track was unmuted.')); + track.addEventListener('ended', () => console.warn('Audio recording - microphone track ended.')); +} + +async function startPcmRecording(stream, dotnetRef) { + const AudioContextClass = window.AudioContext || window.webkitAudioContext; + if (!AudioContextClass || typeof AudioWorkletNode === 'undefined') { + throw new Error('PCM audio capture is unavailable because AudioWorklet is not supported.'); + } + + try { + captureAudioContext = new AudioContextClass({ + latencyHint: 'interactive', + sampleRate: PCM_SAMPLE_RATE, + }); + + if (!captureAudioContext.audioWorklet) { + throw new Error('PCM audio capture is unavailable because AudioWorklet is not supported.'); + } + + await captureAudioContext.audioWorklet.addModule('/audio-recorder-worklet.js'); + + const actualSampleRate = captureAudioContext.sampleRate; + console.log(`Audio recording - starting PCM/WAV capture at ${actualSampleRate} Hz mono.`); + + if (captureAudioContext.state === 'suspended') { + await captureAudioContext.resume(); + } + + captureSourceNode = captureAudioContext.createMediaStreamSource(stream); + captureWorkletNode = new AudioWorkletNode(captureAudioContext, 'pcm-recorder-processor', { + numberOfInputs: 1, + numberOfOutputs: 1, + outputChannelCount: [1], + processorOptions: { + chunkDurationSeconds: PCM_CHUNK_DURATION_SECONDS, + }, + }); + + captureSilentGainNode = captureAudioContext.createGain(); + captureSilentGainNode.gain.value = 0; + + captureWorkletNode.port.onmessage = event => { + if (event.data?.type === 'chunk') { + const chunkBytes = new Uint8Array(event.data.buffer); + pcmSamplesReceived += event.data.sampleCount; + console.debug(`Audio recording - received ${event.data.sampleCount} PCM samples from AudioWorklet.`); + queueAudioChunkUpload(() => dotnetRef.invokeMethodAsync('OnAudioChunkReceived', chunkBytes)); + } else if (event.data?.type === 'flushed') { + pcmFlushResolve?.(); + pcmFlushResolve = null; + } + }; + + captureWorkletNode.onprocessorerror = event => { + recordingError ??= event.error || new Error('The PCM audio processor failed.'); + console.error('Audio recording - AudioWorklet error:', recordingError); + }; + + captureSourceNode.connect(captureWorkletNode); + captureWorkletNode.connect(captureSilentGainNode); + captureSilentGainNode.connect(captureAudioContext.destination); + queueAudioChunkUpload(() => dotnetRef.invokeMethodAsync('OnAudioChunkReceived', createPcmWavHeader(actualSampleRate))); + } catch (error) { + await cleanupPcmCapture(); + throw error; + } +} + +async function flushPcmRecording() { + if (!captureWorkletNode) { + throw new Error('The PCM audio processor is unavailable.'); + } + + await new Promise((resolve, reject) => { + const timeoutId = setTimeout(() => { + pcmFlushResolve = null; + reject(new Error('Timed out while flushing PCM audio data.')); + }, PCM_FLUSH_TIMEOUT_MS); + + pcmFlushResolve = () => { + clearTimeout(timeoutId); + resolve(); + }; + captureWorkletNode.port.postMessage({ type: 'flush' }); + }); +} + +async function cleanupPcmCapture() { + captureSourceNode?.disconnect(); + captureWorkletNode?.disconnect(); + captureSilentGainNode?.disconnect(); + captureSourceNode = null; + captureWorkletNode = null; + captureSilentGainNode = null; + pcmFlushResolve = null; + + if (captureAudioContext && captureAudioContext.state !== 'closed') { + await captureAudioContext.close(); + } + captureAudioContext = null; +} window.audioRecorder = { - start: async function (dotnetRef, desiredMimeTypes = []) { - const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + start: async function (dotnetRef) { + // Reset the upload and recorder state: + pendingChunkUploads = 0; + chunkUploadPromise = Promise.resolve(); + chunkUploadError = null; + recordingError = null; + pcmSamplesReceived = 0; + + const stream = await navigator.mediaDevices.getUserMedia({ + audio: { + sampleRate: { ideal: PCM_SAMPLE_RATE }, + channelCount: { ideal: 1 }, + }, + }); activeMediaStream = stream; + const audioTracks = stream.getAudioTracks(); + if (audioTracks.length === 0) { + throw new Error('The microphone stream does not contain an audio track.'); + } + observeAudioTrack(audioTracks[0]); + // Wait for Bluetooth headsets to complete the profile switch from A2DP to HFP. // This prevents the first sound from being cut off during the switch: console.log('Audio recording - waiting for Bluetooth profile switch...'); @@ -207,144 +390,55 @@ window.audioRecorder = { // Play start recording sound effect: await window.playSound('/sounds/start_recording.ogg'); - // When only one mime type is provided as a string, convert it to an array: - if (typeof desiredMimeTypes === 'string') { - desiredMimeTypes = [desiredMimeTypes]; - } - - // Log sent mime types for debugging: - console.log('Audio recording - requested mime types: ', desiredMimeTypes); - - let mimeTypes = desiredMimeTypes.filter(type => typeof type === 'string' && type.trim() !== ''); - - // Next, we have to ensure that we have some default mime types to check as well. - // In case the provided list does not contain these, we append them: - // Use provided mime types or fallback to a default list: - const defaultMimeTypes = [ - 'audio/webm', - 'audio/ogg', - 'audio/mp4', - 'audio/mpeg', - ''// Fallback to browser default - ]; - - defaultMimeTypes.forEach(type => { - if (!mimeTypes.includes(type)) { - mimeTypes.push(type); - } - }); - - console.log('Audio recording - final mime types to check (included defaults): ', mimeTypes); - - // Find the first supported mime type: - actualRecordingMimeType = mimeTypes.find(type => - type === '' || MediaRecorder.isTypeSupported(type) - ) || ''; - - console.log('Audio recording - the browser selected the following mime type for recording: ', actualRecordingMimeType); - const options = actualRecordingMimeType ? { mimeType: actualRecordingMimeType } : {}; - mediaRecorder = new MediaRecorder(stream, options); - - // In case the browser changed the mime type: - actualRecordingMimeType = mediaRecorder.mimeType; - console.log('Audio recording - actual mime type used by the browser: ', actualRecordingMimeType); - - const actualBaseMimeType = actualRecordingMimeType.split(';')[0].trim().toLowerCase(); - - // Check the list of desired mime types against the actual one: - if (!desiredMimeTypes.some(type => type.split(';')[0].trim().toLowerCase() === actualBaseMimeType)) { - changedMimeType = true; - console.warn(`Audio recording - requested mime types ('${desiredMimeTypes.join(', ')}') do not include the actual mime type used by the browser ('${actualRecordingMimeType}').`); - } else { - changedMimeType = false; - } - - // Reset the upload and recorder state: - pendingChunkUploads = 0; - chunkUploadPromise = Promise.resolve(); - chunkUploadError = null; - recordingError = null; - - // Stream each chunk directly to .NET in recording order as it becomes available: - mediaRecorder.ondataavailable = (event) => { - if (event.data.size > 0) { - pendingChunkUploads++; - chunkUploadPromise = chunkUploadPromise - .then(async () => { - const arrayBuffer = await event.data.arrayBuffer(); - const uint8Array = new Uint8Array(arrayBuffer); - await dotnetRef.invokeMethodAsync('OnAudioChunkReceived', uint8Array); - }) - .catch(error => { - chunkUploadError ??= error; - console.error('Error sending audio chunk to .NET:', error); - }) - .finally(() => pendingChunkUploads--); - } - }; - - mediaRecorder.onerror = event => { - recordingError ??= event.error || new Error('The media recorder failed.'); - console.error('Audio recording - MediaRecorder error:', recordingError); - }; - - if (actualBaseMimeType === 'audio/mp4') { - // WebKitGTK does not reliably produce MP4 data when a timeslice is used. Let it - // finalize one complete M4A blob when stop() is called instead. - mediaRecorder.start(); - } else { - mediaRecorder.start(3000); // read the recorded data in 3-second chunks - } - - return actualRecordingMimeType; + await startPcmRecording(stream, dotnetRef); }, stop: async function () { - return new Promise((resolve, reject) => { + let stopError = null; - // Add an event listener to handle the stop event: - mediaRecorder.onstop = async () => { + try { + try { + await flushPcmRecording(); + } finally { + await cleanupPcmCapture(); + } - // Wait for all pending chunk uploads to complete before finalizing: - console.log(`Audio recording - waiting for ${pendingChunkUploads} pending uploads.`); - await chunkUploadPromise; + console.log(`Audio recording - PCM/WAV capture produced ${pcmSamplesReceived} samples.`); + if (pcmSamplesReceived === 0) { + throw new Error('The microphone did not produce any PCM audio samples.'); + } + } catch (error) { + stopError = error; + } - console.log('Audio recording - all chunks uploaded, finalizing.'); + console.log(`Audio recording - waiting for ${pendingChunkUploads} pending uploads.`); + await waitForAudioChunkUploads(); + console.log('Audio recording - all chunks uploaded, finalizing.'); - // Play stop recording sound effect: - await window.playSound('/sounds/stop_recording.ogg'); + // Play stop recording sound effect: + await window.playSound('/sounds/stop_recording.ogg'); - // - // IMPORTANT: Do NOT release the microphone here! - // Bluetooth headsets switch profiles (HFP → A2DP) when the microphone is released, - // which causes audio to be interrupted. We keep the microphone open so that the - // stop_recording and transcription_done sounds can play without interruption. - // - // Call window.audioRecorder.releaseMicrophone() after the last sound has played. - // + // + // IMPORTANT: Do NOT release the microphone here! + // Bluetooth headsets switch profiles (HFP → A2DP) when the microphone is released, + // which causes audio to be interrupted. We keep the microphone open so that the + // stop_recording and transcription_done sounds can play without interruption. + // + // Call window.audioRecorder.releaseMicrophone() after the last sound has played. + // - const error = recordingError || chunkUploadError; - if (error) { - reject(error); - return; - } - - // No need to process data here anymore, just signal completion: - resolve({ - mimeType: actualRecordingMimeType, - changedMimeType: changedMimeType, - }); - }; - - // Finally, stop the recording (which will actually trigger the onstop event): - mediaRecorder.stop(); - }); + const error = stopError || recordingError || chunkUploadError; + if (error) { + throw error; + } }, // Release the microphone after all sounds have been played. // This should be called after the transcription_done sound to allow // Bluetooth headsets to switch back to A2DP profile without interrupting audio: - releaseMicrophone: function () { + releaseMicrophone: async function () { + await cleanupPcmCapture(); + if (activeMediaStream) { console.log('Audio recording - releasing microphone (Bluetooth will switch back to A2DP)'); activeMediaStream.getTracks().forEach(track => track.stop()); diff --git a/runtime/src/media.rs b/runtime/src/media.rs index 30cdf9ba..b10d8bf6 100644 --- a/runtime/src/media.rs +++ b/runtime/src/media.rs @@ -47,6 +47,12 @@ const OPUS_FRAME_SAMPLES: usize = 960; /// Target bitrate for mono speech-oriented Opus output. const OPUS_BITRATE: u32 = 32_000; +/// Stable normalized container name returned to upload clients. +const OUTPUT_FORMAT: &str = "webm"; + +/// Stable normalized codec name returned to upload clients. +const OUTPUT_CODEC: &str = "opus"; + /// Maximum duration of a WebM cluster before rotating it. const CLUSTER_DURATION_MS: u64 = 30_000; @@ -219,6 +225,12 @@ pub struct MediaJobResult { /// Path at which the normalized output was committed. pub output_path: String, + /// Stable container produced for provider uploads. + pub output_format: String, + + /// Stable audio codec produced for provider uploads. + pub output_codec: String, + /// Human-readable detected container description for diagnostics. pub detected_format: String, @@ -634,6 +646,8 @@ fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_throu copy_with_cancellation(input_path, &partial_path, job)?; Ok(MediaJobResult { output_path: output_path.to_string_lossy().into_owned(), + output_format: OUTPUT_FORMAT.to_string(), + output_codec: OUTPUT_CODEC.to_string(), detected_format: detected_format.clone(), detected_codec, duration_ms, @@ -1043,6 +1057,8 @@ fn transcode( Ok(MediaJobResult { output_path: context.output_path.to_string_lossy().into_owned(), + output_format: OUTPUT_FORMAT.to_string(), + output_codec: OUTPUT_CODEC.to_string(), detected_format: context.detected_format, detected_codec: context.detected_codec, duration_ms: output_duration_ms, @@ -1790,6 +1806,8 @@ mod tests { let job = MediaJob::new(); let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap(); assert!(!result.pass_through); + assert_eq!(result.output_format, OUTPUT_FORMAT); + assert_eq!(result.output_codec, OUTPUT_CODEC); assert!(!result.has_audible_signal); assert!(result.duration_ms.abs_diff(100) <= 20); @@ -1834,6 +1852,8 @@ mod tests { let job = MediaJob::new(); let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap(); assert!(result.pass_through); + assert_eq!(result.output_format, OUTPUT_FORMAT); + assert_eq!(result.output_codec, OUTPUT_CODEC); assert_eq!(fs::read(input).unwrap(), fs::read(output).unwrap()); assert!(!result.has_audible_signal); let _ = fs::remove_dir_all(directory);