using System.Text; using AIStudio.Settings; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; using AIStudio.Tools.Security; using AIStudio.Tools.Services; namespace AIStudio.Tools.Media; /// /// Reads or creates the persistent Markdown transcript of an audio or video file. /// /// /// Every transcript which AI Studio keeps comes from here, so the rules which keep a paid /// transcription from being paid twice hold for every caller: an existing transcript is used as it /// is, a transcript which arrived is stored even when the caller cancelled meanwhile, and nothing is /// stored for a file which changed during its transcription or for a transcription without text. /// Where a transcript lives is up to the location, e.g. next to the media file or below the data /// directory. /// internal static class PersistentMediaTranscript { private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PersistentMediaTranscript).Namespace, nameof(PersistentMediaTranscript)); /// /// Whether a file is an audio or video file, which holds no text until it is transcribed. /// /// The path of the file. /// True for audio and video files. public static bool IsMediaPath(string path) => FileTypes.IsAllowedPath(path, FileTypes.AUDIO, FileTypes.VIDEO); /// /// Returns where the transcript of a media file is, transcribing the file only when its location holds no transcript yet. /// /// Where the transcript lives. /// The audio or video file. /// The transcription provider the caller allows, or null when no provider may hear the file. /// Transcribes the media file with the given provider, usually MediaTranscriptionService.TranscribeAsync. /// The logger of the caller. /// The cancellation token. /// Where the transcript is, or why there is none. public static async Task ReadOrCreateAsync(IMediaTranscriptLocation location, string mediaPath, TranscriptionProvider? provider, Func> transcribe, ILogger logger, CancellationToken token) { // // The stamp comes first: a location may read the whole file to find the place of its // transcript, and a change from then on must keep a transcript out of that place. // FileStamp sourceBefore; MediaTranscriptSlot slot; try { sourceBefore = FileStamp.Read(mediaPath); slot = await location.ResolveAsync(mediaPath, token); } catch (OperationCanceledException) { return PersistentMediaTranscriptResult.Cancelled(); } catch (Exception exception) { logger.LogWarning(exception, "Could not inspect the media file '{MediaFilePath}' before its transcription.", mediaPath); return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.SOURCE_UNREADABLE, null, TB("The media file could not be read.")); } try { if (await HoldsTextAsync(slot.Path, token)) { logger.LogInformation("Reusing the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath); return PersistentMediaTranscriptResult.Ready(slot, reused: true); } } catch (OperationCanceledException) { return PersistentMediaTranscriptResult.Cancelled(); } catch (Exception exception) { // // Transcribing again would mean paying for a transcript which exists, and storing it // would most likely fail for the same reason: // logger.LogWarning(exception, "Could not read the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath); return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.TRANSCRIPT_UNREADABLE, slot, string.Format(TB("Was not able to read the existing transcript: {0}"), exception.Message)); } if (provider is null) return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.NO_PROVIDER, slot, TB("No usable transcription provider is configured.")); MediaTranscriptionResult transcription; try { transcription = await transcribe(mediaPath, provider, token); } catch (OperationCanceledException) { return PersistentMediaTranscriptResult.Cancelled(); } switch (transcription.Status) { case MediaTranscriptionResultStatus.CANCELLED: return PersistentMediaTranscriptResult.Cancelled(); case MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL: return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.NO_AUDIBLE_SIGNAL, slot, transcription.UserMessage); case MediaTranscriptionResultStatus.FAILED when transcription.FileRejected: return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.REJECTED, slot, transcription.UserMessage); case MediaTranscriptionResultStatus.FAILED: return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.TRANSCRIPTION_FAILED, slot, transcription.UserMessage, transcription.ErrorCode); } if (string.IsNullOrWhiteSpace(transcription.Text)) return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.EMPTY, slot, TB("The transcription provider returned an empty transcript.")); if (!sourceBefore.StillMatches(mediaPath)) return SourceChanged(slot); var temporaryPath = $"{slot.Path}.{Guid.NewGuid():N}.tmp"; try { var directory = Path.GetDirectoryName(slot.Path); if (!string.IsNullOrWhiteSpace(directory)) Directory.CreateDirectory(directory); // // The transcription is paid for at this point. Storing it is quick, so it is completed // even when the caller cancelled meanwhile, and the next attempt can use it: // await File.WriteAllTextAsync(temporaryPath, transcription.Text, new UTF8Encoding(false), CancellationToken.None); if (!sourceBefore.StillMatches(mediaPath)) return SourceChanged(slot); File.Move(temporaryPath, slot.Path, true); logger.LogInformation("Stored the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath); return PersistentMediaTranscriptResult.Ready(slot, reused: false); } catch (Exception exception) { logger.LogWarning(exception, "Could not store the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath); return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.STORE_FAILED, slot, string.Format(TB("Was not able to store the transcript: {0}"), exception.Message)); } finally { try { if (File.Exists(temporaryPath)) File.Delete(temporaryPath); } catch (Exception exception) { logger.LogWarning(exception, "Could not remove the temporary transcript '{TempFilePath}'.", temporaryPath); } } } /// /// Reads a transcript for an AI, through the runtime like every document. /// /// /// The runtime filters prompt injections while it reads, so a transcript must never reach an AI /// by any other way, such as reading the file directly. What the filter removed is reported under /// the recording: the transcript is a file AI Studio made, while the recording is the file the /// user chose. /// /// The runtime service. /// A transcript with the status READY. /// The audio or video file the transcript belongs to. /// The action the transcript is read for, or null when it is read for none. /// The cancellation token. /// The filtered text of the transcript, or why it could not be read. public static Task ReadTranscriptAsync(RustService rustService, PersistentMediaTranscriptResult transcript, string mediaPath, PromptInjectionReportingScope? reportingScope, CancellationToken token) { if (transcript.Status is not PersistentMediaTranscriptStatus.READY) throw new ArgumentException($"Only a ready transcript can be read, but its status is {transcript.Status}.", nameof(transcript)); return rustService.ReadArbitraryFileData(transcript.TranscriptPath, int.MaxValue, reportAs: PromptInjectionSource.FileContent(mediaPath), reportingScope: reportingScope, token: token); } private static PersistentMediaTranscriptResult SourceChanged(MediaTranscriptSlot slot) => PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.SOURCE_CHANGED, slot, TB("The media file changed while it was being transcribed, so its transcript was not kept.")); /// /// Whether a transcript exists and holds more than white space. /// /// /// Reads only as far as the first character which is not white space, so a large file is /// never loaded as a whole. /// private static async Task HoldsTextAsync(string path, CancellationToken token) { if (!File.Exists(path)) return false; await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan); using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true); var buffer = new char[1024]; int read; while ((read = await reader.ReadAsync(buffer.AsMemory(), token)) > 0) { for (var index = 0; index < read; index++) if (!char.IsWhiteSpace(buffer[index])) return true; } return false; } /// /// The size and write time of a media file, to notice a change during its transcription. /// private readonly record struct FileStamp(long Length, DateTime LastWriteTimeUtc) { public static FileStamp Read(string path) { var file = new FileInfo(path); file.Refresh(); if (!file.Exists) throw new FileNotFoundException("The media file does not exist.", path); return new(file.Length, file.LastWriteTimeUtc); } public bool StillMatches(string path) { try { return Read(path) == this; } catch { return false; } } } }