MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-02-13 02:41:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Moved audio handling from app.js to dedicated audio.js

Browse Source
This commit is contained in:
Thorsten Sommer 2026-01-18 15:51:33 +01:00
parent 06075f2a53
commit fee00e95fe
Signed by: tsommer
GPG Key ID: 371BBA77A02C0108
3 changed files with 308 additions and 308 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
app/MindWork AI Studio/App.razor
View File

@ -27,6 +27,7 @@
<script src="system/MudBlazor.Markdown/MudBlazor.Markdown.min.js"></script>
<script src="system/CodeBeam.MudBlazor.Extensions/MudExtensions.min.js"></script>
<script src="app.js"></script>
<script src="audio.js"></script>
</body>
</html>

309
app/MindWork AI Studio/wwwroot/app.js
View File

@ -25,311 +25,4 @@ window.clearDiv = function (divName) {
window.scrollToBottom = function(element) {
element.scrollIntoView({ behavior: 'smooth', block: 'end', inline: 'nearest' });
}
// Shared the audio context for sound effects (Web Audio API does not register with Media Session):
let soundEffectContext = null;
// Cache for decoded sound effect audio buffers:
const soundEffectCache = new Map();
// Track the preload state:
let soundEffectsPreloaded = false;
// Queue system: tracks when the next sound can start playing.
// This prevents sounds from overlapping and getting "swallowed" by the audio system:
let nextAvailablePlayTime = 0;
// Minimum gap between sounds in seconds (small buffer to ensure clean transitions):
const SOUND_GAP_SECONDS = 0.55;
// List of all sound effects used in the app:
const SOUND_EFFECT_PATHS = [
'/sounds/start_recording.ogg',
'/sounds/stop_recording.ogg',
'/sounds/transcription_done.ogg'
];
// Initialize the audio context with low-latency settings.
// Should be called from a user interaction (click, keypress)
// to satisfy browser autoplay policies:
window.initSoundEffects = async function() {
if (soundEffectContext && soundEffectContext.state !== 'closed') {
// Already initialized, just ensure it's running:
if (soundEffectContext.state === 'suspended') {
await soundEffectContext.resume();
}
return;
}
try {
// Create the context with the interactive latency hint for the lowest latency:
soundEffectContext = new (window.AudioContext || window.webkitAudioContext)({
latencyHint: 'interactive'
});
// Resume immediately (needed for Safari/macOS):
if (soundEffectContext.state === 'suspended') {
await soundEffectContext.resume();
}
// Reset the queue timing:
nextAvailablePlayTime = 0;
//
// Play a very short silent buffer to "warm up" the audio pipeline.
// This helps prevent the first real sound from being cut off:
//
const silentBuffer = soundEffectContext.createBuffer(1, 1, soundEffectContext.sampleRate);
const silentSource = soundEffectContext.createBufferSource();
silentSource.buffer = silentBuffer;
silentSource.connect(soundEffectContext.destination);
silentSource.start(0);
console.log('Sound effects - AudioContext initialized with latency:', soundEffectContext.baseLatency);
// Preload all sound effects in parallel:
if (!soundEffectsPreloaded) {
await window.preloadSoundEffects();
}
} catch (error) {
console.warn('Failed to initialize sound effects:', error);
}
};
// Preload all sound effect files into the cache:
window.preloadSoundEffects = async function() {
if (soundEffectsPreloaded) {
return;
}
// Ensure that the context exists:
if (!soundEffectContext || soundEffectContext.state === 'closed') {
soundEffectContext = new (window.AudioContext || window.webkitAudioContext)({
latencyHint: 'interactive'
});
}
console.log('Sound effects - preloading', SOUND_EFFECT_PATHS.length, 'sound files...');
const preloadPromises = SOUND_EFFECT_PATHS.map(async (soundPath) => {
try {
const response = await fetch(soundPath);
const arrayBuffer = await response.arrayBuffer();
const audioBuffer = await soundEffectContext.decodeAudioData(arrayBuffer);
soundEffectCache.set(soundPath, audioBuffer);
console.log('Sound effects - preloaded:', soundPath, 'duration:', audioBuffer.duration.toFixed(2), 's');
} catch (error) {
console.warn('Sound effects - failed to preload:', soundPath, error);
}
});
await Promise.all(preloadPromises);
soundEffectsPreloaded = true;
console.log('Sound effects - all files preloaded');
};
window.playSound = async function(soundPath) {
try {
// Initialize context if needed (fallback if initSoundEffects wasn't called):
if (!soundEffectContext || soundEffectContext.state === 'closed') {
soundEffectContext = new (window.AudioContext || window.webkitAudioContext)({
latencyHint: 'interactive'
});
nextAvailablePlayTime = 0;
}
// Resume if suspended (browser autoplay policy):
if (soundEffectContext.state === 'suspended') {
await soundEffectContext.resume();
}
// Check the cache for already decoded audio:
let audioBuffer = soundEffectCache.get(soundPath);
if (!audioBuffer) {
// Fetch and decode the audio file (fallback if not preloaded):
console.log('Sound effects - loading on demand:', soundPath);
const response = await fetch(soundPath);
const arrayBuffer = await response.arrayBuffer();
audioBuffer = await soundEffectContext.decodeAudioData(arrayBuffer);
soundEffectCache.set(soundPath, audioBuffer);
}
// Calculate when this sound should start:
const currentTime = soundEffectContext.currentTime;
let startTime;
if (currentTime >= nextAvailablePlayTime) {
// No sound is playing, or the previous sound has finished; start immediately:
startTime = 0; // 0 means "now" in Web Audio API
nextAvailablePlayTime = currentTime + audioBuffer.duration + SOUND_GAP_SECONDS;
} else {
// A sound is still playing; schedule this sound to start after it:
startTime = nextAvailablePlayTime;
nextAvailablePlayTime = startTime + audioBuffer.duration + SOUND_GAP_SECONDS;
console.log('Sound effects - queued:', soundPath, 'will play in', (startTime - currentTime).toFixed(2), 's');
}
// Create a new source node and schedule playback:
const source = soundEffectContext.createBufferSource();
source.buffer = audioBuffer;
source.connect(soundEffectContext.destination);
source.start(startTime);
console.log('Sound effects - playing:', soundPath);
} catch (error) {
console.warn('Failed to play sound effect:', error);
}
};
let mediaRecorder;
let actualRecordingMimeType;
let changedMimeType = false;
let pendingChunkUploads = 0;
// Store the media stream so we can close the microphone later:
let activeMediaStream = null;
// Delay in milliseconds to wait after getUserMedia() for Bluetooth profile switch (A2DP → HFP):
const BLUETOOTH_PROFILE_SWITCH_DELAY_MS = 1_600;
window.audioRecorder = {
start: async function (dotnetRef, desiredMimeTypes = []) {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
activeMediaStream = stream;
// Wait for Bluetooth headsets to complete the profile switch from A2DP to HFP.
// This prevents the first sound from being cut off during the switch:
console.log('Audio recording - waiting for Bluetooth profile switch...');
await new Promise(r => setTimeout(r, BLUETOOTH_PROFILE_SWITCH_DELAY_MS));
// Play start recording sound effect:
await window.playSound('/sounds/start_recording.ogg');
// When only one mime type is provided as a string, convert it to an array:
if (typeof desiredMimeTypes === 'string') {
desiredMimeTypes = [desiredMimeTypes];
}
// Log sent mime types for debugging:
console.log('Audio recording - requested mime types: ', desiredMimeTypes);
let mimeTypes = desiredMimeTypes.filter(type => typeof type === 'string' && type.trim() !== '');
// Next, we have to ensure that we have some default mime types to check as well.
// In case the provided list does not contain these, we append them:
// Use provided mime types or fallback to a default list:
const defaultMimeTypes = [
'audio/webm',
'audio/ogg',
'audio/mp4',
'audio/mpeg',
''// Fallback to browser default
];
defaultMimeTypes.forEach(type => {
if (!mimeTypes.includes(type)) {
mimeTypes.push(type);
}
});
console.log('Audio recording - final mime types to check (included defaults): ', mimeTypes);
// Find the first supported mime type:
actualRecordingMimeType = mimeTypes.find(type =>
type === '' || MediaRecorder.isTypeSupported(type)
) || '';
console.log('Audio recording - the browser selected the following mime type for recording: ', actualRecordingMimeType);
const options = actualRecordingMimeType ? { mimeType: actualRecordingMimeType } : {};
mediaRecorder = new MediaRecorder(stream, options);
// In case the browser changed the mime type:
actualRecordingMimeType = mediaRecorder.mimeType;
console.log('Audio recording - actual mime type used by the browser: ', actualRecordingMimeType);
// Check the list of desired mime types against the actual one:
if (!desiredMimeTypes.includes(actualRecordingMimeType)) {
changedMimeType = true;
console.warn(`Audio recording - requested mime types ('${desiredMimeTypes.join(', ')}') do not include the actual mime type used by the browser ('${actualRecordingMimeType}').`);
} else {
changedMimeType = false;
}
// Reset the pending uploads counter:
pendingChunkUploads = 0;
// Stream each chunk directly to .NET as it becomes available:
mediaRecorder.ondataavailable = async (event) => {
if (event.data.size > 0) {
pendingChunkUploads++;
try {
const arrayBuffer = await event.data.arrayBuffer();
const uint8Array = new Uint8Array(arrayBuffer);
await dotnetRef.invokeMethodAsync('OnAudioChunkReceived', uint8Array);
} catch (error) {
console.error('Error sending audio chunk to .NET:', error);
} finally {
pendingChunkUploads--;
}
}
};
mediaRecorder.start(3000); // read the recorded data in 3-second chunks
return actualRecordingMimeType;
},
stop: async function () {
return new Promise((resolve) => {
// Add an event listener to handle the stop event:
mediaRecorder.onstop = async () => {
// Wait for all pending chunk uploads to complete before finalizing:
console.log(`Audio recording - waiting for ${pendingChunkUploads} pending uploads.`);
while (pendingChunkUploads > 0) {
await new Promise(r => setTimeout(r, 10)); // wait 10 ms before checking again
}
console.log('Audio recording - all chunks uploaded, finalizing.');
// Play stop recording sound effect:
await window.playSound('/sounds/stop_recording.ogg');
//
// IMPORTANT: Do NOT release the microphone here!
// Bluetooth headsets switch profiles (HFP → A2DP) when the microphone is released,
// which causes audio to be interrupted. We keep the microphone open so that the
// stop_recording and transcription_done sounds can play without interruption.
//
// Call window.audioRecorder.releaseMicrophone() after the last sound has played.
//
// No need to process data here anymore, just signal completion:
resolve({
mimeType: actualRecordingMimeType,
changedMimeType: changedMimeType,
});
};
// Finally, stop the recording (which will actually trigger the onstop event):
mediaRecorder.stop();
});
},
// Release the microphone after all sounds have been played.
// This should be called after the transcription_done sound to allow
// Bluetooth headsets to switch back to A2DP profile without interrupting audio:
releaseMicrophone: function () {
if (activeMediaStream) {
console.log('Audio recording - releasing microphone (Bluetooth will switch back to A2DP)');
activeMediaStream.getTracks().forEach(track => track.stop());
activeMediaStream = null;
}
}
};
}

306
app/MindWork AI Studio/wwwroot/audio.js Normal file
View File

@ -0,0 +1,306 @@
// Shared the audio context for sound effects (Web Audio API does not register with Media Session):
let soundEffectContext = null;
// Cache for decoded sound effect audio buffers:
const soundEffectCache = new Map();
// Track the preload state:
let soundEffectsPreloaded = false;
// Queue system: tracks when the next sound can start playing.
// This prevents sounds from overlapping and getting "swallowed" by the audio system:
let nextAvailablePlayTime = 0;
// Minimum gap between sounds in seconds (small buffer to ensure clean transitions):
const SOUND_GAP_SECONDS = 0.25;
// List of all sound effects used in the app:
const SOUND_EFFECT_PATHS = [
'/sounds/start_recording.ogg',
'/sounds/stop_recording.ogg',
'/sounds/transcription_done.ogg'
];
// Initialize the audio context with low-latency settings.
// Should be called from a user interaction (click, keypress)
// to satisfy browser autoplay policies:
window.initSoundEffects = async function() {
if (soundEffectContext && soundEffectContext.state !== 'closed') {
// Already initialized, just ensure it's running:
if (soundEffectContext.state === 'suspended') {
await soundEffectContext.resume();
}
return;
}
try {
// Create the context with the interactive latency hint for the lowest latency:
soundEffectContext = new (window.AudioContext || window.webkitAudioContext)({
latencyHint: 'interactive'
});
// Resume immediately (needed for Safari/macOS):
if (soundEffectContext.state === 'suspended') {
await soundEffectContext.resume();
}
// Reset the queue timing:
nextAvailablePlayTime = 0;
//
// Play a very short silent buffer to "warm up" the audio pipeline.
// This helps prevent the first real sound from being cut off:
//
const silentBuffer = soundEffectContext.createBuffer(1, 1, soundEffectContext.sampleRate);
const silentSource = soundEffectContext.createBufferSource();
silentSource.buffer = silentBuffer;
silentSource.connect(soundEffectContext.destination);
silentSource.start(0);
console.log('Sound effects - AudioContext initialized with latency:', soundEffectContext.baseLatency);
// Preload all sound effects in parallel:
if (!soundEffectsPreloaded) {
await window.preloadSoundEffects();
}
} catch (error) {
console.warn('Failed to initialize sound effects:', error);
}
};
// Preload all sound effect files into the cache:
window.preloadSoundEffects = async function() {
if (soundEffectsPreloaded) {
return;
}
// Ensure that the context exists:
if (!soundEffectContext || soundEffectContext.state === 'closed') {
soundEffectContext = new (window.AudioContext || window.webkitAudioContext)({
latencyHint: 'interactive'
});
}
console.log('Sound effects - preloading', SOUND_EFFECT_PATHS.length, 'sound files...');
const preloadPromises = SOUND_EFFECT_PATHS.map(async (soundPath) => {
try {
const response = await fetch(soundPath);
const arrayBuffer = await response.arrayBuffer();
const audioBuffer = await soundEffectContext.decodeAudioData(arrayBuffer);
soundEffectCache.set(soundPath, audioBuffer);
console.log('Sound effects - preloaded:', soundPath, 'duration:', audioBuffer.duration.toFixed(2), 's');
} catch (error) {
console.warn('Sound effects - failed to preload:', soundPath, error);
}
});
await Promise.all(preloadPromises);
soundEffectsPreloaded = true;
console.log('Sound effects - all files preloaded');
};
window.playSound = async function(soundPath) {
try {
// Initialize context if needed (fallback if initSoundEffects wasn't called):
if (!soundEffectContext || soundEffectContext.state === 'closed') {
soundEffectContext = new (window.AudioContext || window.webkitAudioContext)({
latencyHint: 'interactive'
});
nextAvailablePlayTime = 0;
}
// Resume if suspended (browser autoplay policy):
if (soundEffectContext.state === 'suspended') {
await soundEffectContext.resume();
}
// Check the cache for already decoded audio:
let audioBuffer = soundEffectCache.get(soundPath);
if (!audioBuffer) {
// Fetch and decode the audio file (fallback if not preloaded):
console.log('Sound effects - loading on demand:', soundPath);
const response = await fetch(soundPath);
const arrayBuffer = await response.arrayBuffer();
audioBuffer = await soundEffectContext.decodeAudioData(arrayBuffer);
soundEffectCache.set(soundPath, audioBuffer);
}
// Calculate when this sound should start:
const currentTime = soundEffectContext.currentTime;
let startTime;
if (currentTime >= nextAvailablePlayTime) {
// No sound is playing, or the previous sound has finished; start immediately:
startTime = 0; // 0 means "now" in Web Audio API
nextAvailablePlayTime = currentTime + audioBuffer.duration + SOUND_GAP_SECONDS;
} else {
// A sound is still playing; schedule this sound to start after it:
startTime = nextAvailablePlayTime;
nextAvailablePlayTime = startTime + audioBuffer.duration + SOUND_GAP_SECONDS;
console.log('Sound effects - queued:', soundPath, 'will play in', (startTime - currentTime).toFixed(2), 's');
}
// Create a new source node and schedule playback:
const source = soundEffectContext.createBufferSource();
source.buffer = audioBuffer;
source.connect(soundEffectContext.destination);
source.start(startTime);
console.log('Sound effects - playing:', soundPath);
} catch (error) {
console.warn('Failed to play sound effect:', error);
}
};
let mediaRecorder;
let actualRecordingMimeType;
let changedMimeType = false;
let pendingChunkUploads = 0;
// Store the media stream so we can close the microphone later:
let activeMediaStream = null;
// Delay in milliseconds to wait after getUserMedia() for Bluetooth profile switch (A2DP → HFP):
const BLUETOOTH_PROFILE_SWITCH_DELAY_MS = 1_600;
window.audioRecorder = {
start: async function (dotnetRef, desiredMimeTypes = []) {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
activeMediaStream = stream;
// Wait for Bluetooth headsets to complete the profile switch from A2DP to HFP.
// This prevents the first sound from being cut off during the switch:
console.log('Audio recording - waiting for Bluetooth profile switch...');
await new Promise(r => setTimeout(r, BLUETOOTH_PROFILE_SWITCH_DELAY_MS));
// Play start recording sound effect:
await window.playSound('/sounds/start_recording.ogg');
// When only one mime type is provided as a string, convert it to an array:
if (typeof desiredMimeTypes === 'string') {
desiredMimeTypes = [desiredMimeTypes];
}
// Log sent mime types for debugging:
console.log('Audio recording - requested mime types: ', desiredMimeTypes);
let mimeTypes = desiredMimeTypes.filter(type => typeof type === 'string' && type.trim() !== '');
// Next, we have to ensure that we have some default mime types to check as well.
// In case the provided list does not contain these, we append them:
// Use provided mime types or fallback to a default list:
const defaultMimeTypes = [
'audio/webm',
'audio/ogg',
'audio/mp4',
'audio/mpeg',
''// Fallback to browser default
];
defaultMimeTypes.forEach(type => {
if (!mimeTypes.includes(type)) {
mimeTypes.push(type);
}
});
console.log('Audio recording - final mime types to check (included defaults): ', mimeTypes);
// Find the first supported mime type:
actualRecordingMimeType = mimeTypes.find(type =>
type === '' || MediaRecorder.isTypeSupported(type)
) || '';
console.log('Audio recording - the browser selected the following mime type for recording: ', actualRecordingMimeType);
const options = actualRecordingMimeType ? { mimeType: actualRecordingMimeType } : {};
mediaRecorder = new MediaRecorder(stream, options);
// In case the browser changed the mime type:
actualRecordingMimeType = mediaRecorder.mimeType;
console.log('Audio recording - actual mime type used by the browser: ', actualRecordingMimeType);
// Check the list of desired mime types against the actual one:
if (!desiredMimeTypes.includes(actualRecordingMimeType)) {
changedMimeType = true;
console.warn(`Audio recording - requested mime types ('${desiredMimeTypes.join(', ')}') do not include the actual mime type used by the browser ('${actualRecordingMimeType}').`);
} else {
changedMimeType = false;
}
// Reset the pending uploads counter:
pendingChunkUploads = 0;
// Stream each chunk directly to .NET as it becomes available:
mediaRecorder.ondataavailable = async (event) => {
if (event.data.size > 0) {
pendingChunkUploads++;
try {
const arrayBuffer = await event.data.arrayBuffer();
const uint8Array = new Uint8Array(arrayBuffer);
await dotnetRef.invokeMethodAsync('OnAudioChunkReceived', uint8Array);
} catch (error) {
console.error('Error sending audio chunk to .NET:', error);
} finally {
pendingChunkUploads--;
}
}
};
mediaRecorder.start(3000); // read the recorded data in 3-second chunks
return actualRecordingMimeType;
},
stop: async function () {
return new Promise((resolve) => {
// Add an event listener to handle the stop event:
mediaRecorder.onstop = async () => {
// Wait for all pending chunk uploads to complete before finalizing:
console.log(`Audio recording - waiting for ${pendingChunkUploads} pending uploads.`);
while (pendingChunkUploads > 0) {
await new Promise(r => setTimeout(r, 10)); // wait 10 ms before checking again
}
console.log('Audio recording - all chunks uploaded, finalizing.');
// Play stop recording sound effect:
await window.playSound('/sounds/stop_recording.ogg');
//
// IMPORTANT: Do NOT release the microphone here!
// Bluetooth headsets switch profiles (HFP → A2DP) when the microphone is released,
// which causes audio to be interrupted. We keep the microphone open so that the
// stop_recording and transcription_done sounds can play without interruption.
//
// Call window.audioRecorder.releaseMicrophone() after the last sound has played.
//
// No need to process data here anymore, just signal completion:
resolve({
mimeType: actualRecordingMimeType,
changedMimeType: changedMimeType,
});
};
// Finally, stop the recording (which will actually trigger the onstop event):
mediaRecorder.stop();
});
},
// Release the microphone after all sounds have been played.
// This should be called after the transcription_done sound to allow
// Bluetooth headsets to switch back to A2DP profile without interrupting audio:
releaseMicrophone: function () {
if (activeMediaStream) {
console.log('Audio recording - releasing microphone (Bluetooth will switch back to A2DP)');
activeMediaStream.getTracks().forEach(track => track.stop());
activeMediaStream = null;
}
}
};