using System.Text;
using AIStudio.Settings;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services;
namespace AIStudio.Tools.Media;
///
/// Reads or creates the persistent Markdown transcript of an audio or video file.
///
///
/// Every transcript which AI Studio keeps comes from here, so the rules which keep a paid
/// transcription from being paid twice hold for every caller: an existing transcript is used as it
/// is, a transcript which arrived is stored even when the caller cancelled meanwhile, and nothing is
/// stored for a file which changed during its transcription or for a transcription without text.
/// Where a transcript lives is up to the location, e.g. next to the media file or below the data
/// directory.
///
internal static class PersistentMediaTranscript
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PersistentMediaTranscript).Namespace, nameof(PersistentMediaTranscript));
///
/// Whether a file is an audio or video file, which holds no text until it is transcribed.
///
/// The path of the file.
/// True for audio and video files.
public static bool IsMediaPath(string path) => FileTypes.IsAllowedPath(path, FileTypes.AUDIO, FileTypes.VIDEO);
///
/// Returns where the transcript of a media file is, transcribing the file only when its location holds no transcript yet.
///
/// Where the transcript lives.
/// The audio or video file.
/// The transcription provider the caller allows, or null when no provider may hear the file.
/// Transcribes the media file with the given provider, usually MediaTranscriptionService.TranscribeAsync.
/// The logger of the caller.
/// The cancellation token.
/// Where the transcript is, or why there is none.
public static async Task ReadOrCreateAsync(IMediaTranscriptLocation location, string mediaPath, TranscriptionProvider? provider, Func> transcribe, ILogger logger, CancellationToken token)
{
//
// The stamp comes first: a location may read the whole file to find the place of its
// transcript, and a change from then on must keep a transcript out of that place.
//
FileStamp sourceBefore;
MediaTranscriptSlot slot;
try
{
sourceBefore = FileStamp.Read(mediaPath);
slot = await location.ResolveAsync(mediaPath, token);
}
catch (OperationCanceledException)
{
return PersistentMediaTranscriptResult.Cancelled();
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not inspect the media file '{MediaFilePath}' before its transcription.", mediaPath);
return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.SOURCE_UNREADABLE, null, TB("The media file could not be read."));
}
try
{
if (await HoldsTextAsync(slot.Path, token))
{
logger.LogInformation("Reusing the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath);
return PersistentMediaTranscriptResult.Ready(slot, reused: true);
}
}
catch (OperationCanceledException)
{
return PersistentMediaTranscriptResult.Cancelled();
}
catch (Exception exception)
{
//
// Transcribing again would mean paying for a transcript which exists, and storing it
// would most likely fail for the same reason:
//
logger.LogWarning(exception, "Could not read the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath);
return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.TRANSCRIPT_UNREADABLE, slot, string.Format(TB("Was not able to read the existing transcript: {0}"), exception.Message));
}
if (provider is null)
return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.NO_PROVIDER, slot, TB("No usable transcription provider is configured."));
MediaTranscriptionResult transcription;
try
{
transcription = await transcribe(mediaPath, provider, token);
}
catch (OperationCanceledException)
{
return PersistentMediaTranscriptResult.Cancelled();
}
switch (transcription.Status)
{
case MediaTranscriptionResultStatus.CANCELLED:
return PersistentMediaTranscriptResult.Cancelled();
case MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL:
return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.NO_AUDIBLE_SIGNAL, slot, transcription.UserMessage);
case MediaTranscriptionResultStatus.FAILED when transcription.FileRejected:
return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.REJECTED, slot, transcription.UserMessage);
case MediaTranscriptionResultStatus.FAILED:
return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.TRANSCRIPTION_FAILED, slot, transcription.UserMessage, transcription.ErrorCode);
}
if (string.IsNullOrWhiteSpace(transcription.Text))
return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.EMPTY, slot, TB("The transcription provider returned an empty transcript."));
if (!sourceBefore.StillMatches(mediaPath))
return SourceChanged(slot);
var temporaryPath = $"{slot.Path}.{Guid.NewGuid():N}.tmp";
try
{
var directory = Path.GetDirectoryName(slot.Path);
if (!string.IsNullOrWhiteSpace(directory))
Directory.CreateDirectory(directory);
//
// The transcription is paid for at this point. Storing it is quick, so it is completed
// even when the caller cancelled meanwhile, and the next attempt can use it:
//
await File.WriteAllTextAsync(temporaryPath, transcription.Text, new UTF8Encoding(false), CancellationToken.None);
if (!sourceBefore.StillMatches(mediaPath))
return SourceChanged(slot);
File.Move(temporaryPath, slot.Path, true);
logger.LogInformation("Stored the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath);
return PersistentMediaTranscriptResult.Ready(slot, reused: false);
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not store the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath);
return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.STORE_FAILED, slot, string.Format(TB("Was not able to store the transcript: {0}"), exception.Message));
}
finally
{
try
{
if (File.Exists(temporaryPath))
File.Delete(temporaryPath);
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not remove the temporary transcript '{TempFilePath}'.", temporaryPath);
}
}
}
///
/// Reads a transcript for an AI, through the runtime like every document.
///
///
/// The runtime filters prompt injections while it reads, so a transcript must never reach an AI
/// by any other way, such as reading the file directly. What the filter removed is reported under
/// the recording: the transcript is a file AI Studio made, while the recording is the file the
/// user chose.
///
/// The runtime service.
/// A transcript with the status READY.
/// The audio or video file the transcript belongs to.
/// The action the transcript is read for, or null when it is read for none.
/// The cancellation token.
/// The filtered text of the transcript, or why it could not be read.
public static Task ReadTranscriptAsync(RustService rustService, PersistentMediaTranscriptResult transcript, string mediaPath, PromptInjectionReportingScope? reportingScope, CancellationToken token)
{
if (transcript.Status is not PersistentMediaTranscriptStatus.READY)
throw new ArgumentException($"Only a ready transcript can be read, but its status is {transcript.Status}.", nameof(transcript));
return rustService.ReadArbitraryFileData(transcript.TranscriptPath, int.MaxValue, reportAs: PromptInjectionSource.FileContent(mediaPath), reportingScope: reportingScope, token: token);
}
private static PersistentMediaTranscriptResult SourceChanged(MediaTranscriptSlot slot) => PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.SOURCE_CHANGED, slot, TB("The media file changed while it was being transcribed, so its transcript was not kept."));
///
/// Whether a transcript exists and holds more than white space.
///
///
/// Reads only as far as the first character which is not white space, so a large file is
/// never loaded as a whole.
///
private static async Task HoldsTextAsync(string path, CancellationToken token)
{
if (!File.Exists(path))
return false;
await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan);
using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true);
var buffer = new char[1024];
int read;
while ((read = await reader.ReadAsync(buffer.AsMemory(), token)) > 0)
{
for (var index = 0; index < read; index++)
if (!char.IsWhiteSpace(buffer[index]))
return true;
}
return false;
}
///
/// The size and write time of a media file, to notice a change during its transcription.
///
private readonly record struct FileStamp(long Length, DateTime LastWriteTimeUtc)
{
public static FileStamp Read(string path)
{
var file = new FileInfo(path);
file.Refresh();
if (!file.Exists)
throw new FileNotFoundException("The media file does not exist.", path);
return new(file.Length, file.LastWriteTimeUtc);
}
public bool StillMatches(string path)
{
try
{
return Read(path) == this;
}
catch
{
return false;
}
}
}
}