Added audio and video transcription for chats and assistants (#856)

app/MindWork AI Studio/wwwroot/audio-recorder-worklet.js

@@ -0,0 +1,61 @@
+class PCMRecorderProcessor extends AudioWorkletProcessor {
+    constructor(options) {
+        super();
+
+        const chunkDurationSeconds = options.processorOptions?.chunkDurationSeconds || 3;
+        this.chunkSamples = Math.max(128, Math.round(sampleRate * chunkDurationSeconds));
+        this.samples = new Int16Array(this.chunkSamples);
+        this.numSamples = 0;
+
+        this.port.onmessage = event => {
+            if (event.data?.type === 'flush') {
+                this.flush();
+                this.port.postMessage({ type: 'flushed' });
+            }
+        };
+    }
+
+    process(inputs) {
+        const channels = inputs[0];
+        if (!channels || channels.length === 0)
+            return true;
+
+        const numFrames = channels[0].length;
+        for (let frame = 0; frame < numFrames; frame++) {
+            let monoSample = 0;
+            for (const channel of channels) {
+                monoSample += channel[frame] || 0;
+            }
+
+            monoSample = Math.max(-1, Math.min(1, monoSample / channels.length));
+            this.samples[this.numSamples++] = monoSample < 0
+                ? Math.round(monoSample * 0x8000)
+                : Math.round(monoSample * 0x7fff);
+
+            if (this.numSamples === this.chunkSamples)
+                this.flush();
+        }
+
+        return true;
+    }
+
+    flush() {
+        if (this.numSamples === 0)
+            return;
+
+        const buffer = new ArrayBuffer(this.numSamples * 2);
+        const view = new DataView(buffer);
+        for (let index = 0; index < this.numSamples; index++) {
+            view.setInt16(index * 2, this.samples[index], true);
+        }
+
+        this.port.postMessage({
+            type: 'chunk',
+            buffer: buffer,
+            sampleCount: this.numSamples,
+        }, [buffer]);
+        this.numSamples = 0;
+    }
+}
+
+registerProcessor('pcm-recorder-processor', PCMRecorderProcessor);

app/MindWork AI Studio/wwwroot/audio.js

@@ -180,22 +180,208 @@ window.playSound = async function(soundPath) {
     }
 };
 
-let mediaRecorder;
-let actualRecordingMimeType;
-let changedMimeType = false;
 let pendingChunkUploads = 0;
+let chunkUploadPromise = Promise.resolve();
+let chunkUploadError = null;
+let recordingError = null;
+let captureAudioContext = null;
+let captureSourceNode = null;
+let captureWorkletNode = null;
+let captureSilentGainNode = null;
+let pcmFlushResolve = null;
+let pcmSamplesReceived = 0;
 
 // Store the media stream so we can close the microphone later:
 let activeMediaStream = null;
 
 // Delay in milliseconds to wait after getUserMedia() for Bluetooth profile switch (A2DP → HFP):
 const BLUETOOTH_PROFILE_SWITCH_DELAY_MS = 1_600;
+const PCM_SAMPLE_RATE = 48_000;
+const PCM_CHUNK_DURATION_SECONDS = 3;
+const PCM_FLUSH_TIMEOUT_MS = 5_000;
+
+function queueAudioChunkUpload(upload) {
+    pendingChunkUploads++;
+    chunkUploadPromise = chunkUploadPromise
+        .then(upload)
+        .catch(error => {
+            chunkUploadError ??= error;
+            console.error('Error sending audio chunk to .NET:', error);
+        })
+        .finally(() => pendingChunkUploads--);
+}
+
+async function waitForAudioChunkUploads() {
+    let observedUploadPromise;
+    do {
+        observedUploadPromise = chunkUploadPromise;
+        await observedUploadPromise;
+    } while (pendingChunkUploads > 0 || observedUploadPromise !== chunkUploadPromise);
+}
+
+function createPcmWavHeader(sampleRate) {
+    const buffer = new ArrayBuffer(44);
+    const view = new DataView(buffer);
+
+    const writeAscii = (offset, value) => {
+        for (let index = 0; index < value.length; index++) {
+            view.setUint8(offset + index, value.charCodeAt(index));
+        }
+    };
+
+    writeAscii(0, 'RIFF');
+    view.setUint32(4, 0, true); // Finalized by .NET after all PCM data was written.
+    writeAscii(8, 'WAVE');
+    writeAscii(12, 'fmt ');
+    view.setUint32(16, 16, true);
+    view.setUint16(20, 1, true); // PCM
+    view.setUint16(22, 1, true); // Mono
+    view.setUint32(24, sampleRate, true);
+    view.setUint32(28, sampleRate * 2, true);
+    view.setUint16(32, 2, true);
+    view.setUint16(34, 16, true);
+    writeAscii(36, 'data');
+    view.setUint32(40, 0, true); // Finalized by .NET after all PCM data was written.
+
+    return new Uint8Array(buffer);
+}
+
+function observeAudioTrack(track) {
+    console.log('Audio recording - microphone track state:', {
+        label: track.label,
+        enabled: track.enabled,
+        muted: track.muted,
+        readyState: track.readyState,
+        settings: typeof track.getSettings === 'function' ? track.getSettings() : null,
+    });
+
+    track.addEventListener('mute', () => console.warn('Audio recording - microphone track was muted.'));
+    track.addEventListener('unmute', () => console.log('Audio recording - microphone track was unmuted.'));
+    track.addEventListener('ended', () => console.warn('Audio recording - microphone track ended.'));
+}
+
+async function startPcmRecording(stream, dotnetRef) {
+    const AudioContextClass = window.AudioContext || window.webkitAudioContext;
+    if (!AudioContextClass || typeof AudioWorkletNode === 'undefined') {
+        throw new Error('PCM audio capture is unavailable because AudioWorklet is not supported.');
+    }
+
+    try {
+        captureAudioContext = new AudioContextClass({
+            latencyHint: 'interactive',
+            sampleRate: PCM_SAMPLE_RATE,
+        });
+
+        if (!captureAudioContext.audioWorklet) {
+            throw new Error('PCM audio capture is unavailable because AudioWorklet is not supported.');
+        }
+
+        await captureAudioContext.audioWorklet.addModule('/audio-recorder-worklet.js');
+
+        const actualSampleRate = captureAudioContext.sampleRate;
+        console.log(`Audio recording - starting PCM/WAV capture at ${actualSampleRate} Hz mono.`);
+
+        if (captureAudioContext.state === 'suspended') {
+            await captureAudioContext.resume();
+        }
+
+        captureSourceNode = captureAudioContext.createMediaStreamSource(stream);
+        captureWorkletNode = new AudioWorkletNode(captureAudioContext, 'pcm-recorder-processor', {
+            numberOfInputs: 1,
+            numberOfOutputs: 1,
+            outputChannelCount: [1],
+            processorOptions: {
+                chunkDurationSeconds: PCM_CHUNK_DURATION_SECONDS,
+            },
+        });
+
+        captureSilentGainNode = captureAudioContext.createGain();
+        captureSilentGainNode.gain.value = 0;
+
+        captureWorkletNode.port.onmessage = event => {
+            if (event.data?.type === 'chunk') {
+                const chunkBytes = new Uint8Array(event.data.buffer);
+                pcmSamplesReceived += event.data.sampleCount;
+                console.debug(`Audio recording - received ${event.data.sampleCount} PCM samples from AudioWorklet.`);
+                queueAudioChunkUpload(() => dotnetRef.invokeMethodAsync('OnAudioChunkReceived', chunkBytes));
+            } else if (event.data?.type === 'flushed') {
+                pcmFlushResolve?.();
+                pcmFlushResolve = null;
+            }
+        };
+
+        captureWorkletNode.onprocessorerror = event => {
+            recordingError ??= event.error || new Error('The PCM audio processor failed.');
+            console.error('Audio recording - AudioWorklet error:', recordingError);
+        };
+
+        captureSourceNode.connect(captureWorkletNode);
+        captureWorkletNode.connect(captureSilentGainNode);
+        captureSilentGainNode.connect(captureAudioContext.destination);
+        queueAudioChunkUpload(() => dotnetRef.invokeMethodAsync('OnAudioChunkReceived', createPcmWavHeader(actualSampleRate)));
+    } catch (error) {
+        await cleanupPcmCapture();
+        throw error;
+    }
+}
+
+async function flushPcmRecording() {
+    if (!captureWorkletNode) {
+        throw new Error('The PCM audio processor is unavailable.');
+    }
+
+    await new Promise((resolve, reject) => {
+        const timeoutId = setTimeout(() => {
+            pcmFlushResolve = null;
+            reject(new Error('Timed out while flushing PCM audio data.'));
+        }, PCM_FLUSH_TIMEOUT_MS);
+
+        pcmFlushResolve = () => {
+            clearTimeout(timeoutId);
+            resolve();
+        };
+        captureWorkletNode.port.postMessage({ type: 'flush' });
+    });
+}
+
+async function cleanupPcmCapture() {
+    captureSourceNode?.disconnect();
+    captureWorkletNode?.disconnect();
+    captureSilentGainNode?.disconnect();
+    captureSourceNode = null;
+    captureWorkletNode = null;
+    captureSilentGainNode = null;
+    pcmFlushResolve = null;
+
+    if (captureAudioContext && captureAudioContext.state !== 'closed') {
+        await captureAudioContext.close();
+    }
+    captureAudioContext = null;
+}
 
 window.audioRecorder = {
-    start: async function (dotnetRef, desiredMimeTypes = []) {
-        const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
+    start: async function (dotnetRef) {
+        // Reset the upload and recorder state:
+        pendingChunkUploads = 0;
+        chunkUploadPromise = Promise.resolve();
+        chunkUploadError = null;
+        recordingError = null;
+        pcmSamplesReceived = 0;
+
+        const stream = await navigator.mediaDevices.getUserMedia({
+            audio: {
+                sampleRate: { ideal: PCM_SAMPLE_RATE },
+                channelCount: { ideal: 1 },
+            },
+        });
         activeMediaStream = stream;
 
+        const audioTracks = stream.getAudioTracks();
+        if (audioTracks.length === 0) {
+            throw new Error('The microphone stream does not contain an audio track.');
+        }
+        observeAudioTrack(audioTracks[0]);
+
         // Wait for Bluetooth headsets to complete the profile switch from A2DP to HFP.
         // This prevents the first sound from being cut off during the switch:
         console.log('Audio recording - waiting for Bluetooth profile switch...');
@@ -204,121 +390,55 @@ window.audioRecorder = {
         // Play start recording sound effect:
         await window.playSound('/sounds/start_recording.ogg');
 
-        // When only one mime type is provided as a string, convert it to an array:
-        if (typeof desiredMimeTypes === 'string') {
-            desiredMimeTypes = [desiredMimeTypes];
-        }
-
-        // Log sent mime types for debugging:
-        console.log('Audio recording - requested mime types: ', desiredMimeTypes);
-
-        let mimeTypes = desiredMimeTypes.filter(type => typeof type === 'string' && type.trim() !== '');
-
-        // Next, we have to ensure that we have some default mime types to check as well.
-        // In case the provided list does not contain these, we append them:
-        // Use provided mime types or fallback to a default list:
-        const defaultMimeTypes = [
-            'audio/webm',
-            'audio/ogg',
-            'audio/mp4',
-            'audio/mpeg',
-            ''// Fallback to browser default
-        ];
-
-        defaultMimeTypes.forEach(type => {
-            if (!mimeTypes.includes(type)) {
-                mimeTypes.push(type);
-            }
-        });
-
-        console.log('Audio recording - final mime types to check (included defaults): ', mimeTypes);
-
-        // Find the first supported mime type:
-        actualRecordingMimeType = mimeTypes.find(type =>
-            type === '' || MediaRecorder.isTypeSupported(type)
-        ) || '';
-
-        console.log('Audio recording - the browser selected the following mime type for recording: ', actualRecordingMimeType);
-        const options = actualRecordingMimeType ? { mimeType: actualRecordingMimeType } : {};
-        mediaRecorder = new MediaRecorder(stream, options);
-
-        // In case the browser changed the mime type:
-        actualRecordingMimeType = mediaRecorder.mimeType;
-        console.log('Audio recording - actual mime type used by the browser: ', actualRecordingMimeType);
-
-        // Check the list of desired mime types against the actual one:
-        if (!desiredMimeTypes.includes(actualRecordingMimeType)) {
-            changedMimeType = true;
-            console.warn(`Audio recording - requested mime types ('${desiredMimeTypes.join(', ')}') do not include the actual mime type used by the browser ('${actualRecordingMimeType}').`);
-        } else {
-            changedMimeType = false;
-        }
-
-        // Reset the pending uploads counter:
-        pendingChunkUploads = 0;
-
-        // Stream each chunk directly to .NET as it becomes available:
-        mediaRecorder.ondataavailable = async (event) => {
-            if (event.data.size > 0) {
-                pendingChunkUploads++;
-                try {
-                    const arrayBuffer = await event.data.arrayBuffer();
-                    const uint8Array = new Uint8Array(arrayBuffer);
-                    await dotnetRef.invokeMethodAsync('OnAudioChunkReceived', uint8Array);
-                } catch (error) {
-                    console.error('Error sending audio chunk to .NET:', error);
-                } finally {
-                    pendingChunkUploads--;
-                }
-            }
-        };
-
-        mediaRecorder.start(3000); // read the recorded data in 3-second chunks
-        return actualRecordingMimeType;
+        await startPcmRecording(stream, dotnetRef);
     },
 
     stop: async function () {
-        return new Promise((resolve) => {
+        let stopError = null;
 
-            // Add an event listener to handle the stop event:
-            mediaRecorder.onstop = async () => {
+        try {
+            try {
+                await flushPcmRecording();
+            } finally {
+                await cleanupPcmCapture();
+            }
 
-                // Wait for all pending chunk uploads to complete before finalizing:
-                console.log(`Audio recording - waiting for ${pendingChunkUploads} pending uploads.`);
-                while (pendingChunkUploads > 0) {
-                    await new Promise(r => setTimeout(r, 10)); // wait 10 ms before checking again
-                }
+            console.log(`Audio recording - PCM/WAV capture produced ${pcmSamplesReceived} samples.`);
+            if (pcmSamplesReceived === 0) {
+                throw new Error('The microphone did not produce any PCM audio samples.');
+            }
+        } catch (error) {
+            stopError = error;
+        }
 
-                console.log('Audio recording - all chunks uploaded, finalizing.');
+        console.log(`Audio recording - waiting for ${pendingChunkUploads} pending uploads.`);
+        await waitForAudioChunkUploads();
+        console.log('Audio recording - all chunks uploaded, finalizing.');
 
-                // Play stop recording sound effect:
-                await window.playSound('/sounds/stop_recording.ogg');
+        // Play stop recording sound effect:
+        await window.playSound('/sounds/stop_recording.ogg');
 
-                //
-                // IMPORTANT: Do NOT release the microphone here!
-                // Bluetooth headsets switch profiles (HFP → A2DP) when the microphone is released,
-                // which causes audio to be interrupted. We keep the microphone open so that the
-                // stop_recording and transcription_done sounds can play without interruption.
-                //
-                // Call window.audioRecorder.releaseMicrophone() after the last sound has played.
-                //
+        //
+        // IMPORTANT: Do NOT release the microphone here!
+        // Bluetooth headsets switch profiles (HFP → A2DP) when the microphone is released,
+        // which causes audio to be interrupted. We keep the microphone open so that the
+        // stop_recording and transcription_done sounds can play without interruption.
+        //
+        // Call window.audioRecorder.releaseMicrophone() after the last sound has played.
+        //
 
-                // No need to process data here anymore, just signal completion:
-                resolve({
-                    mimeType: actualRecordingMimeType,
-                    changedMimeType: changedMimeType,
-                });
-            };
-
-            // Finally, stop the recording (which will actually trigger the onstop event):
-            mediaRecorder.stop();
-        });
+        const error = stopError || recordingError || chunkUploadError;
+        if (error) {
+            throw error;
+        }
     },
 
     // Release the microphone after all sounds have been played.
     // This should be called after the transcription_done sound to allow
     // Bluetooth headsets to switch back to A2DP profile without interrupting audio:
-    releaseMicrophone: function () {
+    releaseMicrophone: async function () {
+        await cleanupPcmCapture();
+
         if (activeMediaStream) {
             console.log('Audio recording - releasing microphone (Bluetooth will switch back to A2DP)');
             activeMediaStream.getTracks().forEach(track => track.stop());

app/MindWork AI Studio/wwwroot/changelog/v26.7.3.md

@@ -1,10 +1,12 @@
 # v26.7.3, build 245 (2026-07-xx xx:xx UTC)
 - Added support for OpenAI GPT-5.6 Sol, Terra, and Luna; Anthropic Claude Fable 5 and Mythos 5; and Google Gemini 3 Flash, Gemini 3.1 Flash-Lite, Gemini 3.1 Pro, and Gemini 3.5 Flash.
+- Added audio and video transcription for chats and assistants. AI Studio now prepares supported media locally, sends only normalized audio to the configured transcription provider, and attaches the resulting transcript instead of the original media.
 - Improved the "My Tasks Assistant": you can now provide one or more documents in addition to text or use documents alone when asking to identify tasks.
 - Improved update guidance for Flatpak installations and added an enterprise option that lets organizations manage updates entirely through their IT department.
-- Fixed an issue that could leave AI Studio unresponsive after waking the computer from sleep.
-- Fixed enterprise configuration plugins from Windows-created ZIP files not loading correctly on Linux when the ZIP contained plugin files inside a folder.
-- Fixed voice recording not starting on Linux.
+- Fixed an issue that could leave AI Studio unresponsive after waking the computer from sleep. Yes, we know this was an annoying bug, and we apologize for the inconvenience.
+- Fixed enterprise configuration plugins from Windows-created ZIP files may not load correctly on Linux when the ZIP contained plugin files inside a folder.
+- Fixed voice recording and transcription on Linux.
+- Fixed being able to switch document analysis policies while an analysis or media transcription was still in progress.
 - Upgraded Rust to v1.97.0.
 - Upgraded Tauri to v2.11.5.
 - Upgraded common dependencies.