using AIStudio.Chat; using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; namespace AIStudio.Tools.Services; /// /// Coordinates serialized visible media imports and independent voice transcriptions. /// public sealed class MediaTranscriptionService(RustService rustService, SettingsManager settingsManager, ILogger logger) : IDisposable { /// Serializes attachment and file-content imports. private readonly SemaphoreSlim importQueue = new(1, 1); /// Protects operation ownership and owner-specific import state. private readonly Lock stateLock = new(); /// All operations retained so disposal can cancel voice and import work. private readonly HashSet operations = []; /// The active or queued import operation for each owner. private readonly Dictionary currentImports = []; /// The latest copied state retained for each owner across navigation. private readonly Dictionary snapshots = []; /// Owners whose complete file batches are managed by this service. private readonly HashSet activeBatches = []; /// Batch-level cancellation keeps Stop effective between two files. private readonly Dictionary batchCancellations = []; /// Prevents new work after disposal. private bool disposed; /// Raised only with the owner whose copied state changed. public event Action? StateChanged; /// Gets whether one owner has queued, running, or canceling media work. public bool IsBusy(MediaImportOwner owner) => this.GetSnapshot(owner)?.IsBusy ?? false; /// Gets the last retained state for one owner. public MediaImportSnapshot? GetSnapshot(MediaImportOwner owner) { lock (this.stateLock) return this.snapshots.GetValueOrDefault(owner); } /// Gets copied retained snapshots for navigation indicators. public IReadOnlyCollection GetSnapshots() { lock (this.stateLock) return [.. this.snapshots.Values]; } /// Starts an owner-managed attachment batch and returns without holding the UI event handler. public bool TryStartAttachmentBatch(IReadOnlyList mediaPaths, MediaImportOwner owner, ChatThread? ownerChat = null) { this.ThrowIfDisposed(); lock (this.stateLock) { if (!this.activeBatches.Add(owner)) return false; this.batchCancellations[owner] = new(); } _ = this.RunAttachmentBatchAsync(mediaPaths, owner, ownerChat); return true; } /// Starts a reattachable file-content import for one stable assistant field. public bool TryStartTextImport(string mediaPath, MediaImportTarget target) { this.ThrowIfDisposed(); lock (this.stateLock) { if (!this.activeBatches.Add(target.Owner)) return false; this.batchCancellations[target.Owner] = new(); } _ = this.RunTextImportAsync(mediaPath, target); return true; } /// Completes a field import independently of the originating Blazor component. private async Task RunTextImportAsync(string mediaPath, MediaImportTarget target) { CancellationTokenSource cancellation; lock (this.stateLock) cancellation = this.batchCancellations[target.Owner]; try { var result = await this.TranscribeImportAsync(mediaPath, target.Owner, cancellation.Token); if (result.Status is MediaTranscriptionResultStatus.SUCCEEDED) this.AddCompletedText(target, result.Text); } catch (OperationCanceledException) { var current = this.GetSnapshot(target.Owner); this.UpdateImportState(target.Owner, current?.CurrentFileName ?? string.Empty, MediaTranscriptionPhase.IDLE, null, MediaImportStatus.CANCELLED); } catch (Exception exception) { logger.LogError(exception, "Owner media text import failed for '{Owner}' and target '{TargetId}'.", target.Owner, target.TargetId); } finally { lock (this.stateLock) { this.activeBatches.Remove(target.Owner); if (this.batchCancellations.Remove(target.Owner, out var ownedCancellation)) ownedCancellation.Dispose(); } } } /// Stores a completed field transcript for reattachment after navigation. private void AddCompletedText(MediaImportTarget target, string text) { lock (this.stateLock) { var current = this.snapshots.GetValueOrDefault(target.Owner); if (current is null) return; var completed = new Dictionary(current.CompletedTextTargets, StringComparer.Ordinal) { [target.TargetId] = text, }; this.snapshots[target.Owner] = current with { CompletedTextTargets = completed }; } this.StateChanged?.Invoke(target.Owner); } /// Serially transcribes a complete owner batch while retaining every successful result. private async Task RunAttachmentBatchAsync(IReadOnlyList mediaPaths, MediaImportOwner owner, ChatThread? ownerChat) { CancellationToken batchToken; lock (this.stateLock) batchToken = this.batchCancellations[owner].Token; try { foreach (var mediaPath in mediaPaths) { batchToken.ThrowIfCancellationRequested(); var result = await this.TranscribeImportAsync(mediaPath, owner, batchToken); if (result.Status is MediaTranscriptionResultStatus.CANCELLED) break; if (result.Status is not MediaTranscriptionResultStatus.SUCCEEDED) continue; var isPersistedChat = ownerChat is not null && WorkspaceBehaviour.IsChatExisting(new LoadChat(ownerChat.WorkspaceId, ownerChat.ChatId)); var attachment = isPersistedChat ? await WorkspaceBehaviour.CreateManagedTranscriptAsync(ownerChat!, mediaPath, result.Text) : await ManagedTranscriptAttachment.CreateStagedAsync(mediaPath, result.Text); if (ownerChat is not null && attachment is { } managed && ownerChat.PendingMediaTranscripts.All(existing => existing.FilePath != managed.FilePath)) ownerChat.PendingMediaTranscripts.Add(managed); if (isPersistedChat) await WorkspaceBehaviour.StoreChatAsync(ownerChat!); this.AddCompletedAttachment(owner, attachment); } } catch (OperationCanceledException) { var current = this.GetSnapshot(owner); this.UpdateImportState(owner, current?.CurrentFileName ?? string.Empty, MediaTranscriptionPhase.IDLE, null, MediaImportStatus.CANCELLED); } catch (Exception exception) { logger.LogError(exception, "Owner media batch failed for '{Owner}'.", owner); } finally { lock (this.stateLock) { this.activeBatches.Remove(owner); if (this.batchCancellations.Remove(owner, out var cancellation)) cancellation.Dispose(); } } } /// Adds a successful partial result to the retained owner snapshot. private void AddCompletedAttachment(MediaImportOwner owner, FileAttachment attachment) { lock (this.stateLock) { var current = this.snapshots.GetValueOrDefault(owner); if (current is null) return; this.snapshots[owner] = current with { CompletedAttachments = [..current.CompletedAttachments, attachment] }; } this.StateChanged?.Invoke(owner); } /// /// Transcribes an attachment or file-content import on the serialized visible lane. /// /// Source media path. /// Media import owner. /// Caller cancellation token. /// A typed terminal result. private async Task TranscribeImportAsync(string mediaPath, MediaImportOwner owner, CancellationToken token = default) { this.ThrowIfDisposed(); var operation = this.CreateOperation(owner, token); lock (this.stateLock) { if (!this.currentImports.TryAdd(owner, operation)) throw new InvalidOperationException($"Media owner '{owner}' already has an active operation."); } this.UpdateImportState(owner, Path.GetFileName(mediaPath), MediaTranscriptionPhase.QUEUED, null, MediaImportStatus.QUEUED); try { await this.importQueue.WaitAsync(operation.Cancellation.Token); operation.HasQueueLease = true; this.UpdateImportState(owner, Path.GetFileName(mediaPath), MediaTranscriptionPhase.PROBING, 0.0, MediaImportStatus.RUNNING); var result = await this.TranscribeCoreAsync(mediaPath, operation, updateImportState: true); var status = result.Status switch { MediaTranscriptionResultStatus.SUCCEEDED => MediaImportStatus.SUCCEEDED, MediaTranscriptionResultStatus.CANCELLED => MediaImportStatus.CANCELLED, _ => MediaImportStatus.FAILED, }; this.UpdateImportState(owner, Path.GetFileName(mediaPath), MediaTranscriptionPhase.IDLE, null, status); return result; } catch (OperationCanceledException) { this.UpdateImportState(owner, Path.GetFileName(mediaPath), MediaTranscriptionPhase.IDLE, null, MediaImportStatus.CANCELLED); return MediaTranscriptionResult.Cancelled(); } finally { lock (this.stateLock) { if (this.currentImports.GetValueOrDefault(owner) == operation) this.currentImports.Remove(owner); } this.ReleaseOperation(operation); if (operation.HasQueueLease) this.importQueue.Release(); } } /// /// Transcribes a voice recording independently of the visible import lane. /// /// Voice recording path. /// Caller cancellation token. /// A typed terminal result. public async Task TranscribeVoiceAsync(string mediaPath, CancellationToken token = default) { this.ThrowIfDisposed(); var operation = this.CreateOperation(null, token); try { return await this.TranscribeCoreAsync(mediaPath, operation, updateImportState: false); } finally { this.ReleaseOperation(operation); } } /// Cancels only the queued or active operation belonging to one owner. public async Task StopAsync(MediaImportOwner owner) { MediaOperation? operation; lock (this.stateLock) { operation = this.currentImports.GetValueOrDefault(owner); this.batchCancellations.GetValueOrDefault(owner)?.Cancel(); operation?.Cancellation.Cancel(); } if (operation is not null) this.UpdateImportState(owner, this.GetSnapshot(owner)?.CurrentFileName ?? string.Empty, MediaTranscriptionPhase.CANCELING, null, MediaImportStatus.CANCELING); if (!string.IsNullOrWhiteSpace(operation?.JobId)) await rustService.CancelMediaJobAsync(operation.JobId); } /// Runs normalization, provider resolution, and upload for one owned operation. /// Source media path. /// Operation-specific cancellation and Rust-job state. /// Whether progress belongs to the visible import lane. /// A typed terminal result. private async Task TranscribeCoreAsync(string mediaPath, MediaOperation operation, bool updateImportState) { var normalizedPath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-media", $"{operation.Id:N}.webm"); Directory.CreateDirectory(Path.GetDirectoryName(normalizedPath)!); try { var normalized = await this.NormalizeAsync(mediaPath, normalizedPath, operation, updateImportState); if (normalized.Result is null) return normalized.Error is null ? MediaTranscriptionResult.Failed(TB("The media pipeline ended without an output file.")) : MediaTranscriptionResult.Failed(UserMessageFor(normalized.Error.Code), normalized.Error.Code); var providerSettings = this.ResolveProvider(); if (providerSettings is null) return MediaTranscriptionResult.Failed(TB("No usable transcription provider is configured.")); if (updateImportState) this.UpdateImportState(operation.Owner!.Value, Path.GetFileName(mediaPath), MediaTranscriptionPhase.UPLOADING, null, MediaImportStatus.RUNNING); var provider = providerSettings.CreateProvider(); if (provider.Provider is LLMProviders.NONE) return MediaTranscriptionResult.Failed(TB("The configured transcription provider could not be created.")); logger.LogInformation("Transcribing normalized media '{MediaPath}' with provider '{Provider}' and model '{Model}'.", mediaPath, providerSettings.UsedLLMProvider, providerSettings.Model); var providerResult = await provider.TranscribeAudioAsync(providerSettings.Model, normalized.Result.OutputPath, settingsManager, operation.Cancellation.Token); if (!providerResult.Success) { logger.LogWarning("The transcription provider failed for '{MediaPath}': {Diagnostic}", mediaPath, providerResult.ErrorMessage); return MediaTranscriptionResult.Failed(TB("The transcription provider could not transcribe the media file.")); } return MediaTranscriptionResult.Succeeded(providerResult.Text.Trim()); } catch (OperationCanceledException) { return MediaTranscriptionResult.Cancelled(); } catch (Exception exception) { logger.LogError(exception, "Media transcription failed for '{MediaPath}'.", mediaPath); return MediaTranscriptionResult.Failed(TB("The media file could not be transcribed.")); } finally { // NormalizeAsync does not return from cancellation until Rust has reached a terminal // phase, so deleting both paths here cannot race a still-writing worker. if (!this.RetainNormalizedMediaIfRequested(normalizedPath, operation.Id)) this.DeleteTemporaryFile(normalizedPath); this.DeleteTemporaryFile(normalizedPath + ".partial"); } } /// Runs the Rust normalization job and drains cancellation to a terminal event. /// Source media path. /// Owned temporary output path. /// Operation-specific state. /// Whether progress belongs to the import lane. /// The terminal runtime result or error. private async Task<(MediaJobResult? Result, MediaJobError? Error)> NormalizeAsync( string mediaPath, string normalizedPath, MediaOperation operation, bool updateImportState) { // The quick POST is intentionally not cancelled: losing its response could orphan a job // whose ID the client never received. Cancellation is applied immediately after ownership. var jobId = await rustService.StartMediaJobAsync(mediaPath, normalizedPath, CancellationToken.None); operation.JobId = jobId; try { operation.Cancellation.Token.ThrowIfCancellationRequested(); await foreach (var mediaEvent in rustService.StreamMediaJobEventsAsync(jobId, operation.Cancellation.Token)) { if (updateImportState && mediaEvent.Phase is MediaJobPhase.PROBING or MediaJobPhase.TRANSCODING) { var phase = mediaEvent.Phase is MediaJobPhase.PROBING ? MediaTranscriptionPhase.PROBING : MediaTranscriptionPhase.TRANSCODING; this.UpdateImportState(operation.Owner!.Value, Path.GetFileName(mediaPath), phase, mediaEvent.Progress, MediaImportStatus.RUNNING); } switch (mediaEvent.Phase) { case MediaJobPhase.COMPLETED: return (mediaEvent.Result, null); case MediaJobPhase.FAILED: if (mediaEvent.Error is not null) logger.LogWarning("Rust media normalization failed for '{MediaPath}' with {Code}: {Diagnostic}", mediaPath, mediaEvent.Error.Code, mediaEvent.Error.Message); return (null, mediaEvent.Error); case MediaJobPhase.CANCELLED: throw new OperationCanceledException(operation.Cancellation.Token); } } return (null, null); } catch (OperationCanceledException) { await rustService.CancelMediaJobAsync(jobId, CancellationToken.None); await this.DrainTerminalEventAsync(jobId); throw; } } /// Waits for Rust cleanup after cooperative cancellation. /// Owned Rust job identifier. private async Task DrainTerminalEventAsync(string jobId) { await foreach (var _ in rustService.StreamMediaJobEventsAsync(jobId, CancellationToken.None)) { // The stream itself ends immediately after the first terminal snapshot or event. } } /// Resolves the configured provider after confidence validation. private TranscriptionProvider? ResolveProvider() { var providerId = settingsManager.ConfigurationData.App.UseTranscriptionProvider; if (string.IsNullOrWhiteSpace(providerId)) return null; var providerSettings = settingsManager.ConfigurationData.TranscriptionProviders.FirstOrDefault(x => x.Id == providerId); if (providerSettings is null) return null; var minimumLevel = settingsManager.GetMinimumConfidenceLevel(Components.NONE); return providerSettings.UsedLLMProvider.GetConfidence(settingsManager).Level >= minimumLevel ? providerSettings : null; } /// Creates and registers operation-owned cancellation state. /// Media import owner. /// Caller token linked to the operation. /// The registered operation. private MediaOperation CreateOperation(MediaImportOwner? owner, CancellationToken token) { var operation = new MediaOperation(owner, token); lock (this.stateLock) this.operations.Add(operation); return operation; } /// Unregisters and disposes completed operation state. /// Completed operation. private void ReleaseOperation(MediaOperation operation) { lock (this.stateLock) this.operations.Remove(operation); operation.Dispose(); } /// Updates and publishes copied state for exactly one owner. private void UpdateImportState(MediaImportOwner owner, string fileName, MediaTranscriptionPhase phase, double? progress, MediaImportStatus status) { var snapshot = new MediaImportSnapshot { Owner = owner, CurrentFileName = fileName, Phase = phase, Progress = progress, Status = status, CompletedAttachments = this.GetSnapshot(owner)?.CompletedAttachments ?? [], CompletedTextTargets = this.GetSnapshot(owner)?.CompletedTextTargets ?? new Dictionary(), }; lock (this.stateLock) this.snapshots[owner] = snapshot; this.StateChanged?.Invoke(owner); } /// Maps runtime codes to localized user-facing fallback text. private static string UserMessageFor(MediaJobErrorCode code) => code switch { MediaJobErrorCode.FILE_NOT_FOUND => TB("The selected media file no longer exists."), MediaJobErrorCode.UNSAFE_FILE or MediaJobErrorCode.NOT_MEDIA => TB("The selected file cannot be processed as media."), MediaJobErrorCode.NO_AUDIO_TRACK => TB("The selected media file does not contain an audio track."), MediaJobErrorCode.UNSUPPORTED_CONTAINER or MediaJobErrorCode.UNSUPPORTED_CODEC or MediaJobErrorCode.UNSUPPORTED_OPUS_MAPPING => TB("This media format or audio codec is not supported."), MediaJobErrorCode.UNKNOWN_FORMAT or MediaJobErrorCode.DAMAGED_CONTAINER => TB("The media file is damaged or its format could not be identified."), _ => TB("The media file could not be prepared for transcription."), }; /// Deletes one operation-owned temporary file on a best-effort basis. private void DeleteTemporaryFile(string path) { try { if (File.Exists(path)) File.Delete(path); } catch (Exception exception) { logger.LogWarning(exception, "Could not delete operation-owned temporary media file '{Path}'.", path); } } /// Retains the exact provider upload only for opt-in debug diagnostics. private bool RetainNormalizedMediaIfRequested(string normalizedPath, Guid operationId) { #if DEBUG if (!string.Equals(Environment.GetEnvironmentVariable("MINDWORK_AI_RETAIN_NORMALIZED_MEDIA"), "true", StringComparison.OrdinalIgnoreCase) || !File.Exists(normalizedPath)) return false; try { var diagnosticDirectory = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-media", "diagnostics"); Directory.CreateDirectory(diagnosticDirectory); var diagnosticPath = Path.Combine(diagnosticDirectory, $"{operationId:N}.webm"); File.Move(normalizedPath, diagnosticPath, overwrite: true); foreach (var oldPath in new DirectoryInfo(diagnosticDirectory).EnumerateFiles("*.webm").OrderByDescending(file => file.LastWriteTimeUtc).Skip(10)) oldPath.Delete(); logger.LogInformation("Retained normalized media diagnostic '{DiagnosticPath}'.", diagnosticPath); return true; } catch (Exception exception) { logger.LogWarning(exception, "Could not retain normalized media diagnostic for operation '{OperationId}'.", operationId); } #endif return false; } /// Returns localized text while registering the US-English fallback with I18N. private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(MediaTranscriptionService).Namespace, nameof(MediaTranscriptionService)); /// Throws when a caller attempts to start work after disposal. private void ThrowIfDisposed() => ObjectDisposedException.ThrowIf(this.disposed, this); /// Cancels every active import and voice operation and releases owned resources. public void Dispose() { MediaOperation[] active; CancellationTokenSource[] batches; lock (this.stateLock) { if (this.disposed) return; this.disposed = true; active = [.. this.operations]; batches = [.. this.batchCancellations.Values]; } foreach (var operation in active) operation.Cancellation.Cancel(); foreach (var batch in batches) batch.Cancel(); // The semaphore may still be released by an operation unwinding after cancellation. } /// Cancellation and runtime-job ownership for exactly one media operation. private sealed class MediaOperation : IDisposable { /// Creates operation state linked to a caller token. /// Media import owner. /// Caller cancellation token. public MediaOperation(MediaImportOwner? owner, CancellationToken token) { this.Owner = owner; this.Cancellation = CancellationTokenSource.CreateLinkedTokenSource(token); } /// Gets the optional visible import owner; voice operations have none. public MediaImportOwner? Owner { get; } /// Gets the unique temporary-path identifier. public Guid Id { get; } = Guid.NewGuid(); /// Gets the operation-owned cancellation source. public CancellationTokenSource Cancellation { get; } /// Gets or sets the Rust job after its POST response establishes ownership. public string? JobId { get; set; } /// Gets or sets whether this operation currently owns the serialized lane. public bool HasQueueLease { get; set; } /// Disposes operation-owned cancellation state. public void Dispose() => this.Cancellation.Dispose(); } }