From 752f7d828a7926d1ce0c770e307dd025a3ef3572 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Sun, 18 Oct 2026 12:06:15 +0200 Subject: [PATCH] Added transcription of audio and video files to local data sources (#1081) Co-authored-by: Paul Koudelka <106623909+PaulKoudelka@users.noreply.github.com> --- .../AssistantBatchProcessing.razor.Content.cs | 102 +----- ...sistantBatchProcessing.razor.Validation.cs | 4 +- .../AssistantBatchProcessing.razor.cs | 1 - .../Assistants/I18N/allTexts.lua | 100 ++++-- .../VisualBriefingBuildOrchestrator.Inputs.cs | 2 +- .../VisualBriefing/VisualBriefingHashing.cs | 19 -- .../VisualBriefingSourcePreparationService.cs | 2 +- .../Components/DataFolders.razor.cs | 1 + .../DataSourceCloudEmbeddingWarning.razor | 17 +- .../DataSourceCloudEmbeddingWarning.razor.cs | 31 ++ .../DataSourceLocalDirectoryDialog.razor | 32 +- .../DataSourceLocalDirectoryDialog.razor.cs | 42 ++- .../Dialogs/DataSourceLocalFileDialog.razor | 28 +- .../DataSourceLocalFileDialog.razor.cs | 43 ++- app/MindWork AI Studio/Pages/Embeddings.razor | 13 +- .../Pages/Embeddings.razor.cs | 43 ++- .../plugin.lua | 104 ++++-- .../plugin.lua | 100 ++++-- .../Provider/BaseProvider.cs | 60 +++- .../Provider/TranscriptionResult.cs | 11 +- .../DataSourceSecurityTrustExtensions.cs | 8 + .../Tools/DataBackup/DataBackupContents.cs | 1 + .../Tools/DataBackup/DataBackupEntry.cs | 15 +- .../Tools/DataBackup/DataBackupEntryKind.cs | 5 + .../Tools/DataBackup/DataBackupManifest.cs | 2 +- .../Tools/DataBackup/DataBackupWriter.cs | 2 +- .../Tools/DataBackup/DataRestorePreparer.cs | 7 +- .../IndexStore/EmbeddingStateFile.cs | 3 +- .../IndexStore/EmbeddingStateFileEntity.cs | 2 + .../IndexStore/IndexStoreDbContext.cs | 1 + .../IndexStore/IndexStoreSchemaMigrator.cs | 1 + .../20261018000000_MediaTranscriptKey.cs | 35 ++ .../IndexStoreDbContextModelSnapshot.cs | 4 + .../SqliteIndexStoreClientImplementation.cs | 4 +- .../Tools/FileContentHash.cs | 28 ++ .../Tools/FileExtractionErrorCode.cs | 6 + .../FileExtractionErrorCodeExtensions.cs | 3 + .../Tools/FileNamePatterns.cs | 27 ++ .../Tools/Media/DataSourceTranscriptStore.cs | 209 ++++++++++++ .../Tools/Media/IMediaTranscriptLocation.cs | 19 ++ .../Tools/Media/MediaSidecarTranscripts.cs | 107 ++++++ .../Tools/Media/MediaTranscriptSlot.cs | 8 + .../Tools/Media/MediaTranscriptionResult.cs | 7 +- .../Tools/Media/PersistentMediaTranscript.cs | 213 ++++++++++++ .../Media/PersistentMediaTranscriptResult.cs | 26 ++ .../Media/PersistentMediaTranscriptStatus.cs | 63 ++++ .../Tools/Services/DataBackupService.cs | 14 +- .../DataSourceEmbeddingIgnoreReason.cs | 20 +- .../DataSourceEmbeddingIgnoredFile.cs | 3 +- .../Services/DataSourceEmbeddingService.cs | 46 ++- .../Services/DataSourceEmbeddingStatus.cs | 10 +- .../Tools/Services/EmbeddedFileRecord.cs | 2 +- .../Services/Indexing/DocumentRunProgress.cs | 37 +- .../Indexing/FileSourceIndexer.Watchers.cs | 9 + .../Services/Indexing/FileSourceIndexer.cs | 317 ++++++++++++++++-- .../MediaTranscriptionAvailability.cs | 28 ++ .../Services/MediaTranscriptionService.cs | 60 +++- .../Tools/TerminalLogger.cs | 30 +- .../Tools/Validation/CloudProcessing.cs | 23 ++ .../Validation/CloudProcessingExtensions.cs | 17 + .../Tools/Validation/DataSourceValidation.cs | 9 +- .../wwwroot/changelog/v26.10.1.md | 1 + app/Tests/Provider/MediaFileRejectionTests.cs | 78 +++++ .../Settings/TranscriptionConfidenceTests.cs | 74 ++++ .../Tools/DataBackup/DataBackupWriterTests.cs | 7 + .../Tools/DataSourceEmbeddingHealthTests.cs | 25 +- .../Databases/IndexStoreManifestTests.cs | 21 ++ app/Tests/Tools/DocumentRunProgressTests.cs | 34 +- app/Tests/Tools/FileSizeLimitTests.cs | 2 +- .../Tools/FileSourceIndexerWatcherTests.cs | 73 ++++ app/Tests/Tools/FileTypesTests.cs | 4 +- .../Media/DataSourceTranscriptStoreTests.cs | 233 +++++++++++++ .../Media/MediaSidecarTranscriptsTests.cs | 57 ++++ .../Media/PersistentMediaTranscriptTests.cs | 252 ++++++++++++++ .../Tools/Media/TranscriptCleanupTests.cs | 110 ++++++ app/Tests/Tools/MediaIndexingTests.cs | 258 ++++++++++++++ app/Tests/Tools/QueuedStatusTests.cs | 2 +- app/Tests/Tools/TerminalLoggerTests.cs | 26 ++ .../Tools/ToolCalling/MailDraftSyncTests.cs | 2 +- app/Tests/Tools/UnreachableDataSourceTests.cs | 2 +- .../Tools/Validation/CloudProcessingTests.cs | 68 ++++ runtime/src/data_restore.rs | 18 +- 82 files changed, 3231 insertions(+), 302 deletions(-) create mode 100644 app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20261018000000_MediaTranscriptKey.cs create mode 100644 app/MindWork AI Studio/Tools/FileContentHash.cs create mode 100644 app/MindWork AI Studio/Tools/Media/DataSourceTranscriptStore.cs create mode 100644 app/MindWork AI Studio/Tools/Media/IMediaTranscriptLocation.cs create mode 100644 app/MindWork AI Studio/Tools/Media/MediaSidecarTranscripts.cs create mode 100644 app/MindWork AI Studio/Tools/Media/MediaTranscriptSlot.cs create mode 100644 app/MindWork AI Studio/Tools/Media/PersistentMediaTranscript.cs create mode 100644 app/MindWork AI Studio/Tools/Media/PersistentMediaTranscriptResult.cs create mode 100644 app/MindWork AI Studio/Tools/Media/PersistentMediaTranscriptStatus.cs create mode 100644 app/MindWork AI Studio/Tools/Services/Indexing/MediaTranscriptionAvailability.cs create mode 100644 app/MindWork AI Studio/Tools/Validation/CloudProcessing.cs create mode 100644 app/MindWork AI Studio/Tools/Validation/CloudProcessingExtensions.cs create mode 100644 app/Tests/Provider/MediaFileRejectionTests.cs create mode 100644 app/Tests/Settings/TranscriptionConfidenceTests.cs create mode 100644 app/Tests/Tools/Media/DataSourceTranscriptStoreTests.cs create mode 100644 app/Tests/Tools/Media/MediaSidecarTranscriptsTests.cs create mode 100644 app/Tests/Tools/Media/PersistentMediaTranscriptTests.cs create mode 100644 app/Tests/Tools/Media/TranscriptCleanupTests.cs create mode 100644 app/Tests/Tools/MediaIndexingTests.cs create mode 100644 app/Tests/Tools/Validation/CloudProcessingTests.cs diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs index b9dbee816..3d77c7acb 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Content.cs @@ -1,7 +1,4 @@ -using System.Text; - using AIStudio.Tools.Media; -using AIStudio.Tools.Rust; namespace AIStudio.Assistants.BatchProcessing; @@ -69,108 +66,37 @@ public partial class AssistantBatchProcessing private async Task LoadMediaTranscriptAsync(BatchProcessingFileResult fileResult, CancellationToken token) { - var transcriptFilePath = GetTranscriptFilePath(fileResult.FilePath); - if (File.Exists(transcriptFilePath)) - { - try - { - var existingTranscript = await File.ReadAllTextAsync(transcriptFilePath, token); - if (!string.IsNullOrWhiteSpace(existingTranscript)) - { - this.Logger.LogInformation("Reusing the existing batch transcript '{TranscriptFilePath}' for media file '{MediaFilePath}'.", transcriptFilePath, fileResult.FilePath); - return existingTranscript; - } - - this.Logger.LogWarning("The existing batch transcript '{TranscriptFilePath}' for media file '{MediaFilePath}' is empty and will be replaced.", transcriptFilePath, fileResult.FilePath); - } - catch (OperationCanceledException) - { - this.FinishFileResult(fileResult, BatchProcessingFileStatus.CANCELED, T("The batch run was canceled.")); - return null; - } - catch (Exception e) - { - this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("Was not able to read the existing transcript: {0}"), e.Message), e); - return null; - } - } - - if (!this.MediaTranscriptionService.HasUsableTranscriptionProvider) - { - this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("No usable transcription provider is configured.")); - return null; - } - - var transcription = await this.MediaTranscriptionService.TranscribeAsync(fileResult.FilePath, token); - if (transcription.Status is MediaTranscriptionResultStatus.CANCELLED) + var provider = this.MediaTranscriptionService.GetUsableProvider(); + var transcript = await PersistentMediaTranscript.ReadOrCreateAsync(MediaSidecarTranscripts.INSTANCE, fileResult.FilePath, provider, this.MediaTranscriptionService.TranscribeAsync, this.Logger, token); + if (transcript.Status is PersistentMediaTranscriptStatus.CANCELLED) { this.FinishFileResult(fileResult, BatchProcessingFileStatus.CANCELED, T("The batch run was canceled.")); return null; } - if (transcription.Status is not MediaTranscriptionResultStatus.SUCCEEDED) + if (transcript.Status is not PersistentMediaTranscriptStatus.READY) { - this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, transcription.UserMessage); + this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, transcript.UserMessage); return null; } - if (string.IsNullOrWhiteSpace(transcription.Text)) - { - this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("The transcription provider returned an empty transcript.")); - return null; - } - - return await this.StoreMediaTranscriptAsync(fileResult, transcriptFilePath, transcription.Text); - } - - private async Task StoreMediaTranscriptAsync(BatchProcessingFileResult fileResult, string transcriptFilePath, string transcript) - { - var tempFilePath = transcriptFilePath + ".tmp"; try { - // Complete the small persistence step even if cancellation arrived - // after transcription, so the expensive provider result can be - // reused when the interrupted batch is continued. - await File.WriteAllTextAsync(tempFilePath, transcript, new UTF8Encoding(false), CancellationToken.None); - File.Move(tempFilePath, transcriptFilePath, true); - this.Logger.LogInformation("Stored the batch transcript '{TranscriptFilePath}' next to media file '{MediaFilePath}'.", transcriptFilePath, fileResult.FilePath); - return transcript; + return await File.ReadAllTextAsync(transcript.TranscriptPath, token); + } + catch (OperationCanceledException) + { + this.FinishFileResult(fileResult, BatchProcessingFileStatus.CANCELED, T("The batch run was canceled.")); + return null; } catch (Exception e) { - this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("Was not able to store the transcript next to the media file: {0}"), e.Message), e); + this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("Was not able to read the existing transcript: {0}"), e.Message), e); return null; } - finally - { - try - { - if (File.Exists(tempFilePath)) - File.Delete(tempFilePath); - } - catch (Exception e) - { - this.Logger.LogWarning(e, "Was not able to remove the temporary batch transcript '{TempFilePath}'.", tempFilePath); - } - } } - private static bool IsTranscribableMedia(string filePath) => FileTypes.IsAllowedPath(filePath, FileTypes.AUDIO, FileTypes.VIDEO); + private static bool IsTranscribableMedia(string filePath) => PersistentMediaTranscript.IsMediaPath(filePath); - private static string GetTranscriptFilePath(string mediaFilePath) => mediaFilePath + TRANSCRIPT_FILE_SUFFIX; - - private static bool HasReusableTranscript(string mediaFilePath) - { - var transcriptFilePath = GetTranscriptFilePath(mediaFilePath); - try - { - return File.Exists(transcriptFilePath) && new FileInfo(transcriptFilePath).Length > 0; - } - catch - { - // The concrete read error is reported when the affected file is - // processed. Here we only decide whether a provider is required. - return File.Exists(transcriptFilePath); - } - } + private static bool HasReusableTranscript(string mediaFilePath) => MediaSidecarTranscripts.HasReusableTranscript(mediaFilePath); } \ No newline at end of file diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs index 905c4f19c..10060f85d 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.Validation.cs @@ -1,3 +1,4 @@ +using AIStudio.Tools.Media; namespace AIStudio.Assistants.BatchProcessing; @@ -160,8 +161,7 @@ public partial class AssistantBatchProcessing /// private static bool IsTranscriptArtifact(string filePath) { - var fileName = Path.GetFileName(filePath); - return fileName.EndsWith(TRANSCRIPT_FILE_SUFFIX, StringComparison.OrdinalIgnoreCase) || fileName.EndsWith(TRANSCRIPT_FILE_SUFFIX + ".tmp", StringComparison.OrdinalIgnoreCase); + return MediaSidecarTranscripts.IsArtifactPath(filePath); } /// diff --git a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs index 0f41497b4..5cedad6e2 100644 --- a/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs +++ b/app/MindWork AI Studio/Assistants/BatchProcessing/AssistantBatchProcessing.razor.cs @@ -19,7 +19,6 @@ public partial class AssistantBatchProcessing : AssistantBaseCore /// /// An element the backup holds nothing for stays as it is, since nothing tells whether it was empty - /// or whether the version of AI Studio which made the backup knew it at all. The indexes are - /// replaced only when the backup holds both of them, cf. DataBackupManifest.IncludesDataSourceIndexes. + /// or whether the version of AI Studio which made the backup knew it at all. The indexes and the + /// media transcripts are replaced only when the backup holds the indexes, cf. + /// DataBackupManifest.IncludesDataSourceIndexes and DataBackupEntry.ComesFromIndexCopies. /// private static List DetermineElements(DataBackupManifest manifest, DataRestoreTarget target) { var elements = new List(); foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded)) { - if (entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX or DataBackupEntryKind.VECTOR_STORES && !manifest.IncludesDataSourceIndexes) + if (entry.ComesFromIndexCopies && !manifest.IncludesDataSourceIndexes) continue; var entryName = DataBackupFormat.EntryName(entry.Location, entry.RelativePath); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs index 8089ea184..dc594fa4c 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs @@ -11,4 +11,5 @@ public sealed record EmbeddingStateFile( DateTimeOffset CreationUtc, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, - int ChunkCount); + int ChunkCount, + string? TranscriptKey = null); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs index 1f33c2e8c..35a3e772f 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs @@ -26,6 +26,8 @@ internal sealed class EmbeddingStateFileEntity public int ChunkCount { get; set; } + public string? TranscriptKey { get; set; } + public EmbeddingStateDataSourceEntity? DataSource { get; set; } public List Chunks { get; set; } = []; diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs index 405b41d04..98d20b8a8 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs @@ -80,6 +80,7 @@ internal sealed class IndexStoreDbContext(DbContextOptions entity.Property(file => file.LastWriteUtc).HasColumnName("last_write_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired(); entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired(); entity.Property(file => file.ChunkCount).HasColumnName("chunk_count"); + entity.Property(file => file.TranscriptKey).HasColumnName("transcript_key"); entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source"); entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path"); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs index 4742fd458..0552367f6 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs @@ -11,6 +11,7 @@ internal static class IndexStoreSchemaMigrator [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropFileConfidenceLevel))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropDataSourceName))] [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.MailboxIndex))] + [DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.MediaTranscriptKey))] public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token) { await context.Database.MigrateAsync(token); diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20261018000000_MediaTranscriptKey.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20261018000000_MediaTranscriptKey.cs new file mode 100644 index 000000000..802a6e9cc --- /dev/null +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20261018000000_MediaTranscriptKey.cs @@ -0,0 +1,35 @@ +#nullable disable + +using Microsoft.EntityFrameworkCore.Infrastructure; +using Microsoft.EntityFrameworkCore.Migrations; + +namespace AIStudio.Tools.Databases.IndexStore.Migrations; + +/// +/// Adds the key of the transcript to the indexed files, for audio and video files which a data source transcribed. +/// +/// +/// The transcripts lie below the data directory under the hash of the content of their media file. +/// The index has to know which transcript belongs to which file, so that AI Studio can remove a +/// transcript once its file is gone or changed. Every other file keeps NULL. +/// +[DbContext(typeof(IndexStoreDbContext))] +[Migration("20261018000000_MediaTranscriptKey")] +public partial class MediaTranscriptKey : Migration +{ + /// + /// Both directions go through raw SQL, like DropDataSourceName: the SQLite provider answers + /// DropColumn by rebuilding the table, and dropping the old embedded_files table would let the + /// cascade of the foreign key in embedding_chunks take every chunk with it. Adding a nullable + /// column is native in SQLite and leaves the rows alone. + /// + protected override void Up(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("ALTER TABLE embedded_files ADD COLUMN transcript_key TEXT NULL;"); + } + + protected override void Down(MigrationBuilder migrationBuilder) + { + migrationBuilder.Sql("ALTER TABLE embedded_files DROP COLUMN transcript_key;"); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs index 460f63c38..e7957e6e0 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs @@ -119,6 +119,10 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot .HasColumnType("TEXT") .HasColumnName("relative_path"); + entity.Property("TranscriptKey") + .HasColumnType("TEXT") + .HasColumnName("transcript_key"); + entity.HasKey("ParentFileId"); entity.HasIndex("AbsolutePath") diff --git a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs index dd68c14c3..2e689fc2c 100644 --- a/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs @@ -148,7 +148,8 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st file.FileSize, file.LastWriteUtc, file.EmbeddedAtUtc, - file.ChunkCount); + file.ChunkCount, + file.TranscriptKey); } var permanentFailures = await context.PermanentIndexingFailures @@ -690,6 +691,7 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st fileEntity.LastWriteUtc = file.LastWriteUtc; fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc; fileEntity.ChunkCount = file.ChunkCount; + fileEntity.TranscriptKey = file.TranscriptKey; } private static void ApplyPermanentFailure(IndexingFailureEntity failureEntity, string dataSourceId, PermanentIndexingFailure failure) diff --git a/app/MindWork AI Studio/Tools/FileContentHash.cs b/app/MindWork AI Studio/Tools/FileContentHash.cs new file mode 100644 index 000000000..23113df2a --- /dev/null +++ b/app/MindWork AI Studio/Tools/FileContentHash.cs @@ -0,0 +1,28 @@ +using System.Security.Cryptography; + +namespace AIStudio.Tools; + +/// +/// Computes the hash of the content of a file. +/// +internal static class FileContentHash +{ + /// + /// Computes a lowercase SHA-256 hash for a file without loading it fully into memory. + /// + /// The file path. + /// The cancellation token. + /// The lowercase hexadecimal hash. + internal static async Task ComputeAsync(string path, CancellationToken token) + { + await using var stream = new FileStream( + path, + FileMode.Open, + FileAccess.Read, + FileShare.Read, + 65_536, + FileOptions.Asynchronous | FileOptions.SequentialScan); + + return Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, token)); + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/FileExtractionErrorCode.cs b/app/MindWork AI Studio/Tools/FileExtractionErrorCode.cs index 6b69d8a20..0b652b9ae 100644 --- a/app/MindWork AI Studio/Tools/FileExtractionErrorCode.cs +++ b/app/MindWork AI Studio/Tools/FileExtractionErrorCode.cs @@ -91,6 +91,12 @@ public enum FileExtractionErrorCode /// NO_CONTENT, + /// + /// A selected audio or video file could not be prepared for transcription for a reason which + /// lies in the file itself, such as a damaged container or an unsupported audio codec. + /// + MEDIA_PROCESSING_FAILED, + /// /// The caller no longer needs the content, e.g. because the user closed the dialog which /// asked for it. This is not a failure: nobody has to be told about it, which is why there diff --git a/app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs b/app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs index ee07c00ce..56f92e1aa 100644 --- a/app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs +++ b/app/MindWork AI Studio/Tools/FileExtractionErrorCodeExtensions.cs @@ -37,6 +37,7 @@ internal static class FileExtractionErrorCodeExtensions FileExtractionErrorCode.FORMAT_DETECTION_FAILED => true, FileExtractionErrorCode.EXECUTABLE_REJECTED => true, FileExtractionErrorCode.UNSUPPORTED => true, + FileExtractionErrorCode.MEDIA_PROCESSING_FAILED => true, // Pages holding nothing but images are one of the recurring cases here, and that is a // property of the document, not of the environment: @@ -71,6 +72,7 @@ internal static class FileExtractionErrorCodeExtensions FileExtractionErrorCode.FORMAT_DETECTION_FAILED => TB("Unknown file type"), FileExtractionErrorCode.EXECUTABLE_REJECTED => TB("Executable program"), FileExtractionErrorCode.UNSUPPORTED => TB("Unsupported file type"), + FileExtractionErrorCode.MEDIA_PROCESSING_FAILED => TB("Media file could not be processed"), FileExtractionErrorCode.PAGE_EXTRACTION_FAILED => TB("Pages without readable text"), FileExtractionErrorCode.FILE_NOT_FOUND => TB("File does not exist anymore"), @@ -122,6 +124,7 @@ internal static class FileExtractionErrorCodeExtensions FileExtractionErrorCode.FORMAT_DETECTION_FAILED => TB("The file type of '{0}' could not be determined, so the file was not indexed. AI Studio reads it again as soon as the file changes."), FileExtractionErrorCode.EXECUTABLE_REJECTED => TB("The file '{0}' is an executable program and was not indexed, regardless of its file extension."), FileExtractionErrorCode.UNSUPPORTED => TB("The file type of '{0}' is not supported, so the file was not indexed. AI Studio reads it again as soon as the file changes."), + FileExtractionErrorCode.MEDIA_PROCESSING_FAILED => TB("The media file '{0}' could not be processed, so it was not indexed. AI Studio reads it again as soon as the file changes."), FileExtractionErrorCode.PAGE_EXTRACTION_FAILED => TB("Pages of the file '{0}' could not be read, so it was not indexed. They might contain images only. AI Studio reads it again as soon as the file changes."), // diff --git a/app/MindWork AI Studio/Tools/FileNamePatterns.cs b/app/MindWork AI Studio/Tools/FileNamePatterns.cs index 1a620bbbb..827f8d1d2 100644 --- a/app/MindWork AI Studio/Tools/FileNamePatterns.cs +++ b/app/MindWork AI Studio/Tools/FileNamePatterns.cs @@ -80,6 +80,33 @@ internal static class FileNamePatterns && FileSystemName.MatchesSimpleExpression(pattern, fileName, ignoreCase: true)); } + /// + /// Whether the patterns may select audio or video files, which a data source sends to the transcription provider. + /// + /// + /// Deliberately cautious, since it answers without looking at a single file: a pattern which + /// may match any extension, such as * or meeting*, counts. The selectors never select media. + /// + /// The parsed patterns. + /// True when audio or video files may match. + public static bool MaySelectMedia(IReadOnlyList patterns) => patterns.Any(MaySelectMedia); + + private static bool MaySelectMedia(string pattern) + { + if (pattern.StartsWith(SELECTOR_DELIMITER) && pattern.EndsWith(SELECTOR_DELIMITER)) + return false; + + var extensionStart = pattern.LastIndexOf('.'); + if (extensionStart < 0) + return pattern.IndexOfAny(['*', '?']) >= 0; + + var extension = pattern[(extensionStart + 1)..]; + if (extension.IndexOfAny(['*', '?']) >= 0) + return true; + + return FileTypes.IsAllowedPath($"media.{extension}", FileTypes.AUDIO, FileTypes.VIDEO); + } + public static bool AreEquivalent(IReadOnlyCollection? left, IReadOnlyCollection? right) { if (left is null || right is null) diff --git a/app/MindWork AI Studio/Tools/Media/DataSourceTranscriptStore.cs b/app/MindWork AI Studio/Tools/Media/DataSourceTranscriptStore.cs new file mode 100644 index 000000000..729a014e3 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/DataSourceTranscriptStore.cs @@ -0,0 +1,209 @@ +using System.Text.RegularExpressions; + +using AIStudio.Settings; + +namespace AIStudio.Tools.Media; + +/// +/// Keeps the transcripts of the audio and video files of one data source below the data directory. +/// +/// +/// A data source never writes into the folders of the user: they may be read-only, a network share, +/// or the read-only home folder of the Flatpak, and a transcript which cannot be stored would be paid +/// for again with every run. The transcripts live in 'mediaTranscripts/<data source id>/<key>.md' +/// instead, where the key is the SHA-256 hash of the content of the media file. Renaming or moving a +/// recording, or a sync tool which only touches its write time, finds the transcript again. The path +/// follows from the id and the key, so nothing stores it. +/// +/// The data directory of AI Studio. +/// The id of the data source, a GUID. +internal sealed partial class DataSourceTranscriptStore(string dataDirectory, string dataSourceId) : IMediaTranscriptLocation +{ + /// + /// The folder below the data directory which holds the transcripts of all data sources. + /// + public const string FOLDER_NAME = "mediaTranscripts"; + + private const string FILE_EXTENSION = ".md"; + + private readonly string directoryPath = Path.Join(dataDirectory, FOLDER_NAME, NormalizeId(dataSourceId)); + + /// + /// The store of a data source in the data directory of the running app. + /// + /// The id of the data source. + /// The store. + public static DataSourceTranscriptStore ForDataSource(string dataSourceId) => new(SettingsManager.DataDirectory ?? throw new InvalidOperationException("The data directory is not known yet."), dataSourceId); + + /// + /// The store of a data source in the data directory of the running app, or null when there is none to clean up. + /// + /// The id of the data source. + /// The store, or null while the data directory is not known or for an id which is no GUID. + public static DataSourceTranscriptStore? TryForDataSource(string dataSourceId) => string.IsNullOrWhiteSpace(SettingsManager.DataDirectory) || !Guid.TryParse(dataSourceId, out _) + ? null + : new(SettingsManager.DataDirectory, dataSourceId); + + /// + /// + /// Reads the whole media file to hash it, in pieces. Callers ask only for files which are about + /// to be read anyway because they are new or changed. + /// + public async Task ResolveAsync(string mediaPath, CancellationToken token) + { + var key = await FileContentHash.ComputeAsync(mediaPath, token); + return new(this.GetPath(key), key); + } + + /// + /// Whether a text is a key of this store, i.e. a lowercase SHA-256 hash. + /// + /// The text to check. + /// True for a key. + public static bool IsKey(string key) => KeyPattern().IsMatch(key); + + /// + /// The path of the transcript with the given key. + /// + /// The key of the transcript. + /// The path, whether or not the transcript exists. + public string GetPath(string key) + { + if (!IsKey(key)) + throw new ArgumentException("The key of a media transcript has to be a lowercase SHA-256 hash.", nameof(key)); + + return Path.Join(this.directoryPath, key + FILE_EXTENSION); + } + + /// + /// The keys of all transcripts of this data source. + /// + /// The keys; empty when the data source has no transcripts. + public IReadOnlyList ListKeys() + { + return this.EnumerateOwnFiles() + .Select(file => Path.GetFileName(file.FullName)) + .Where(fileName => TranscriptFileNamePattern().IsMatch(fileName)) + .Select(Path.GetFileNameWithoutExtension) + .OfType() + .ToList(); + } + + /// + /// Deletes the transcript with the given key. + /// + /// The key of the transcript. + /// True when a transcript was deleted. + public bool Delete(string key) + { + var file = new FileInfo(this.GetPath(key)); + if (!file.Exists || file.Attributes.HasFlag(FileAttributes.ReparsePoint)) + return false; + + file.Delete(); + return true; + } + + /// + /// Deletes the temporary files which an interrupted write left behind. + /// + /// + /// Call it only while nothing writes a transcript of this data source, e.g. at the end of a run. + /// + /// The number of deleted files. + public int DeleteTemporaryFiles() + { + var deleted = 0; + foreach (var file in this.EnumerateOwnFiles().Where(file => TemporaryFileNamePattern().IsMatch(file.Name))) + { + file.Delete(); + deleted++; + } + + return deleted; + } + + /// + /// Deletes every transcript of this data source, together with temporary files a crash left behind. + /// + /// + /// Removes only what this store wrote. Anything else in the folder stays, and the folder with it. + /// + public void DeleteAll() + { + foreach (var file in this.EnumerateOwnFiles()) + file.Delete(); + + var directory = new DirectoryInfo(this.directoryPath); + if (directory.Exists && !directory.Attributes.HasFlag(FileAttributes.ReparsePoint) && !directory.EnumerateFileSystemInfos().Any()) + directory.Delete(); + } + + /// + /// Copies the transcripts of all data sources, e.g. for a backup. + /// + /// + /// Copies only transcripts, file by file and in pieces. Temporary files of an interrupted write, + /// links, and anything else in the folders stay behind. + /// + /// The data directory to copy from. + /// The folder which receives the transcripts below 'mediaTranscripts/', in the same layout. + /// The cancellation token. + /// The number of copied transcripts. + public static async Task CopyAllAsync(string dataDirectory, string targetDirectory, CancellationToken token) + { + var root = new DirectoryInfo(Path.Join(dataDirectory, FOLDER_NAME)); + if (!root.Exists || root.Attributes.HasFlag(FileAttributes.ReparsePoint)) + return 0; + + var copied = 0; + foreach (var dataSourceFolder in root.EnumerateDirectories("*", new EnumerationOptions { RecurseSubdirectories = false, AttributesToSkip = FileAttributes.ReparsePoint, IgnoreInaccessible = false })) + { + if (!Guid.TryParse(dataSourceFolder.Name, out _)) + continue; + + var store = new DataSourceTranscriptStore(dataDirectory, dataSourceFolder.Name); + foreach (var file in store.EnumerateOwnFiles().Where(file => TranscriptFileNamePattern().IsMatch(file.Name))) + { + token.ThrowIfCancellationRequested(); + var targetPath = Path.Join(targetDirectory, FOLDER_NAME, dataSourceFolder.Name, file.Name); + Directory.CreateDirectory(Path.GetDirectoryName(targetPath)!); + + await using var source = new FileStream(file.FullName, FileMode.Open, FileAccess.Read, FileShare.Read, 65_536, FileOptions.Asynchronous | FileOptions.SequentialScan); + await using var target = new FileStream(targetPath, FileMode.CreateNew, FileAccess.Write, FileShare.None, 65_536, FileOptions.Asynchronous); + await source.CopyToAsync(target, token); + copied++; + } + } + + return copied; + } + + /// + /// The files this store wrote: transcripts and the temporary files of interrupted writes. Links are left out. + /// + private IEnumerable EnumerateOwnFiles() + { + var directory = new DirectoryInfo(this.directoryPath); + if (!directory.Exists || directory.Attributes.HasFlag(FileAttributes.ReparsePoint)) + return []; + + return directory + .EnumerateFiles("*", new EnumerationOptions { RecurseSubdirectories = false, AttributesToSkip = FileAttributes.ReparsePoint, IgnoreInaccessible = true }) + .Where(file => TranscriptFileNamePattern().IsMatch(file.Name) || TemporaryFileNamePattern().IsMatch(file.Name)) + .ToList(); + } + + private static string NormalizeId(string dataSourceId) => Guid.TryParse(dataSourceId, out var id) + ? id.ToString("D") + : throw new ArgumentException("The id of a data source with media transcripts has to be a GUID.", nameof(dataSourceId)); + + [GeneratedRegex("^[0-9a-f]{64}$")] + private static partial Regex KeyPattern(); + + [GeneratedRegex(@"^[0-9a-f]{64}\.md$")] + private static partial Regex TranscriptFileNamePattern(); + + [GeneratedRegex(@"^[0-9a-f]{64}\.md\.[0-9a-f]{32}\.tmp$")] + private static partial Regex TemporaryFileNamePattern(); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/IMediaTranscriptLocation.cs b/app/MindWork AI Studio/Tools/Media/IMediaTranscriptLocation.cs new file mode 100644 index 000000000..083191fde --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/IMediaTranscriptLocation.cs @@ -0,0 +1,19 @@ +namespace AIStudio.Tools.Media; + +/// +/// Where the persistent transcript of an audio or video file lives, cf. PersistentMediaTranscript. +/// +/// +/// A location only names the place. Reading and writing the transcript is left to +/// PersistentMediaTranscript, so every location keeps the same rules against paying twice. +/// +internal interface IMediaTranscriptLocation +{ + /// + /// Finds the place of the transcript of a media file. + /// + /// The audio or video file. + /// The cancellation token. + /// The place of the transcript, whether or not a transcript is there yet. + Task ResolveAsync(string mediaPath, CancellationToken token); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaSidecarTranscripts.cs b/app/MindWork AI Studio/Tools/Media/MediaSidecarTranscripts.cs new file mode 100644 index 000000000..869067d15 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaSidecarTranscripts.cs @@ -0,0 +1,107 @@ +namespace AIStudio.Tools.Media; + +/// +/// Keeps the transcript of a media file next to it, as '<media file>.transcript.md'. +/// +/// +/// The Batch Processing assistant writes its results into the folder the user chose, and its +/// transcripts belong there as well, where people find and reuse them. Data sources must not write +/// into the folders of the user and keep their transcripts below the data directory instead. +/// +internal sealed class MediaSidecarTranscripts : IMediaTranscriptLocation +{ + public const string FILE_SUFFIX = ".transcript.md"; + + public static readonly MediaSidecarTranscripts INSTANCE = new(); + + /// + public Task ResolveAsync(string mediaPath, CancellationToken token) + { + var path = GetFilePath(mediaPath); + return Task.FromResult(new MediaTranscriptSlot(path, path)); + } + + /// + /// The path of the transcript next to a media file. + /// + /// The audio or video file. + /// The path of its transcript. + public static string GetFilePath(string mediaPath) => mediaPath + FILE_SUFFIX; + + /// + /// Whether a file is a transcript next to a media file, or a temporary file left while one was stored. + /// + /// The path of the file. + /// True for transcripts and their temporary files. + public static bool IsArtifactPath(string path) + { + var fileName = Path.GetFileName(path); + if (fileName.EndsWith(FILE_SUFFIX, StringComparison.OrdinalIgnoreCase)) + return true; + + var suffixIndex = fileName.LastIndexOf(FILE_SUFFIX + ".", StringComparison.OrdinalIgnoreCase); + return suffixIndex >= 0 && fileName.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase); + } + + /// + /// Whether a file is the transcript of a media file which the selection of a folder data source selects as well. + /// + /// + /// Such a transcript holds the same text the data source gets from transcribing the media file, + /// so indexing it as a document of its own would only duplicate that text. A transcript whose + /// media file is gone, or not selected, stays a document like every other Markdown file. + /// + /// The path of the file. + /// The files the folder data source selects. + /// True when the data source indexes the media file of this transcript. + public static bool IsCoveredArtifact(string path, FolderFileSelection selection) + { + if (!TryGetMediaPath(path, out var mediaPath)) + return false; + + return PersistentMediaTranscript.IsMediaPath(mediaPath) && File.Exists(mediaPath) && selection.Selects(mediaPath); + } + + /// + /// Whether a media file has a transcript next to it, so no transcription provider is needed for it. + /// + /// The audio or video file. + /// True when a transcript which is not empty exists. + public static bool HasReusableTranscript(string mediaPath) + { + var transcriptFilePath = GetFilePath(mediaPath); + try + { + return File.Exists(transcriptFilePath) && new FileInfo(transcriptFilePath).Length > 0; + } + catch + { + // The concrete read error is reported when the affected file is + // processed. Here we only decide whether a provider is required. + return File.Exists(transcriptFilePath); + } + } + + /// + /// The media file a transcript or one of its temporary files belongs to. + /// + private static bool TryGetMediaPath(string path, out string mediaPath) + { + var suffixIndex = path.LastIndexOf(FILE_SUFFIX, StringComparison.OrdinalIgnoreCase); + if (suffixIndex <= 0) + { + mediaPath = string.Empty; + return false; + } + + var remainder = path[(suffixIndex + FILE_SUFFIX.Length)..]; + if (remainder.Length > 0 && !(remainder[0] is '.' && remainder.EndsWith(".tmp", StringComparison.OrdinalIgnoreCase))) + { + mediaPath = string.Empty; + return false; + } + + mediaPath = path[..suffixIndex]; + return true; + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaTranscriptSlot.cs b/app/MindWork AI Studio/Tools/Media/MediaTranscriptSlot.cs new file mode 100644 index 000000000..a0ee96341 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/MediaTranscriptSlot.cs @@ -0,0 +1,8 @@ +namespace AIStudio.Tools.Media; + +/// +/// The place of the persistent transcript of one audio or video file. +/// +/// The absolute path of the Markdown transcript. +/// What identifies the transcript at its location, e.g. the hash of the media content. +internal sealed record MediaTranscriptSlot(string Path, string Key); \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs index b486480fb..553aa66f1 100644 --- a/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs +++ b/app/MindWork AI Studio/Tools/Media/MediaTranscriptionResult.cs @@ -9,7 +9,8 @@ namespace AIStudio.Tools.Media; /// Transcript text for a successful operation. /// Localized message suitable for display after a warning or failure. /// Optional stable runtime failure category. -public sealed record MediaTranscriptionResult(MediaTranscriptionResultStatus Status, string Text, string UserMessage, MediaJobErrorCode? ErrorCode = null) +/// Whether the transcription provider refused the media file itself, so that it would refuse the unchanged file again. +public sealed record MediaTranscriptionResult(MediaTranscriptionResultStatus Status, string Text, string UserMessage, MediaJobErrorCode? ErrorCode = null, bool FileRejected = false) { /// Creates a successful result. /// Provider transcript. @@ -20,6 +21,10 @@ public sealed record MediaTranscriptionResult(MediaTranscriptionResultStatus Sta /// Optional runtime error category. public static MediaTranscriptionResult Failed(string userMessage, MediaJobErrorCode? errorCode = null) => new(MediaTranscriptionResultStatus.FAILED, string.Empty, userMessage, errorCode); + /// Creates a failed result for a media file which the transcription provider refused. + /// Localized visible message. + public static MediaTranscriptionResult Rejected(string userMessage) => new(MediaTranscriptionResultStatus.FAILED, string.Empty, userMessage, FileRejected: true); + /// Creates a warning result for media without an audible signal. /// Localized visible warning. public static MediaTranscriptionResult NoAudibleSignal(string userMessage) => new( diff --git a/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscript.cs b/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscript.cs new file mode 100644 index 000000000..240ae4a05 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscript.cs @@ -0,0 +1,213 @@ +using System.Text; + +using AIStudio.Settings; +using AIStudio.Tools.PluginSystem; +using AIStudio.Tools.Rust; + +namespace AIStudio.Tools.Media; + +/// +/// Reads or creates the persistent Markdown transcript of an audio or video file. +/// +/// +/// Every transcript which AI Studio keeps comes from here, so the rules which keep a paid +/// transcription from being paid twice hold for every caller: an existing transcript is used as it +/// is, a transcript which arrived is stored even when the caller cancelled meanwhile, and nothing is +/// stored for a file which changed during its transcription or for a transcription without text. +/// Where a transcript lives is up to the location, e.g. next to the media file or below the data +/// directory. +/// +internal static class PersistentMediaTranscript +{ + private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(PersistentMediaTranscript).Namespace, nameof(PersistentMediaTranscript)); + + /// + /// Whether a file is an audio or video file, which holds no text until it is transcribed. + /// + /// The path of the file. + /// True for audio and video files. + public static bool IsMediaPath(string path) => FileTypes.IsAllowedPath(path, FileTypes.AUDIO, FileTypes.VIDEO); + + /// + /// Returns where the transcript of a media file is, transcribing the file only when its location holds no transcript yet. + /// + /// Where the transcript lives. + /// The audio or video file. + /// The transcription provider the caller allows, or null when no provider may hear the file. + /// Transcribes the media file with the given provider, usually MediaTranscriptionService.TranscribeAsync. + /// The logger of the caller. + /// The cancellation token. + /// Where the transcript is, or why there is none. + public static async Task ReadOrCreateAsync(IMediaTranscriptLocation location, string mediaPath, TranscriptionProvider? provider, Func> transcribe, ILogger logger, CancellationToken token) + { + // + // The stamp comes first: a location may read the whole file to find the place of its + // transcript, and a change from then on must keep a transcript out of that place. + // + FileStamp sourceBefore; + MediaTranscriptSlot slot; + try + { + sourceBefore = FileStamp.Read(mediaPath); + slot = await location.ResolveAsync(mediaPath, token); + } + catch (OperationCanceledException) + { + return PersistentMediaTranscriptResult.Cancelled(); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not inspect the media file '{MediaFilePath}' before its transcription.", mediaPath); + return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.SOURCE_UNREADABLE, null, TB("The media file could not be read.")); + } + + try + { + if (await HoldsTextAsync(slot.Path, token)) + { + logger.LogInformation("Reusing the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath); + return PersistentMediaTranscriptResult.Ready(slot, reused: true); + } + } + catch (OperationCanceledException) + { + return PersistentMediaTranscriptResult.Cancelled(); + } + catch (Exception exception) + { + // + // Transcribing again would mean paying for a transcript which exists, and storing it + // would most likely fail for the same reason: + // + logger.LogWarning(exception, "Could not read the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath); + return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.TRANSCRIPT_UNREADABLE, slot, string.Format(TB("Was not able to read the existing transcript: {0}"), exception.Message)); + } + + if (provider is null) + return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.NO_PROVIDER, slot, TB("No usable transcription provider is configured.")); + + MediaTranscriptionResult transcription; + try + { + transcription = await transcribe(mediaPath, provider, token); + } + catch (OperationCanceledException) + { + return PersistentMediaTranscriptResult.Cancelled(); + } + + switch (transcription.Status) + { + case MediaTranscriptionResultStatus.CANCELLED: + return PersistentMediaTranscriptResult.Cancelled(); + + case MediaTranscriptionResultStatus.NO_AUDIBLE_SIGNAL: + return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.NO_AUDIBLE_SIGNAL, slot, transcription.UserMessage); + + case MediaTranscriptionResultStatus.FAILED when transcription.FileRejected: + return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.REJECTED, slot, transcription.UserMessage); + + case MediaTranscriptionResultStatus.FAILED: + return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.TRANSCRIPTION_FAILED, slot, transcription.UserMessage, transcription.ErrorCode); + } + + if (string.IsNullOrWhiteSpace(transcription.Text)) + return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.EMPTY, slot, TB("The transcription provider returned an empty transcript.")); + + if (!sourceBefore.StillMatches(mediaPath)) + return SourceChanged(slot); + + var temporaryPath = $"{slot.Path}.{Guid.NewGuid():N}.tmp"; + try + { + var directory = Path.GetDirectoryName(slot.Path); + if (!string.IsNullOrWhiteSpace(directory)) + Directory.CreateDirectory(directory); + + // + // The transcription is paid for at this point. Storing it is quick, so it is completed + // even when the caller cancelled meanwhile, and the next attempt can use it: + // + await File.WriteAllTextAsync(temporaryPath, transcription.Text, new UTF8Encoding(false), CancellationToken.None); + if (!sourceBefore.StillMatches(mediaPath)) + return SourceChanged(slot); + + File.Move(temporaryPath, slot.Path, true); + logger.LogInformation("Stored the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath); + return PersistentMediaTranscriptResult.Ready(slot, reused: false); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not store the transcript '{TranscriptFilePath}' of the media file '{MediaFilePath}'.", slot.Path, mediaPath); + return PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.STORE_FAILED, slot, string.Format(TB("Was not able to store the transcript: {0}"), exception.Message)); + } + finally + { + try + { + if (File.Exists(temporaryPath)) + File.Delete(temporaryPath); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not remove the temporary transcript '{TempFilePath}'.", temporaryPath); + } + } + } + + private static PersistentMediaTranscriptResult SourceChanged(MediaTranscriptSlot slot) => PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.SOURCE_CHANGED, slot, TB("The media file changed while it was being transcribed, so its transcript was not kept.")); + + /// + /// Whether a transcript exists and holds more than white space. + /// + /// + /// Reads only as far as the first character which is not white space, so a large file is + /// never loaded as a whole. + /// + private static async Task HoldsTextAsync(string path, CancellationToken token) + { + if (!File.Exists(path)) + return false; + + await using var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, FileOptions.Asynchronous | FileOptions.SequentialScan); + using var reader = new StreamReader(stream, Encoding.UTF8, detectEncodingFromByteOrderMarks: true); + var buffer = new char[1024]; + int read; + while ((read = await reader.ReadAsync(buffer.AsMemory(), token)) > 0) + { + for (var index = 0; index < read; index++) + if (!char.IsWhiteSpace(buffer[index])) + return true; + } + + return false; + } + + /// + /// The size and write time of a media file, to notice a change during its transcription. + /// + private readonly record struct FileStamp(long Length, DateTime LastWriteTimeUtc) + { + public static FileStamp Read(string path) + { + var file = new FileInfo(path); + file.Refresh(); + if (!file.Exists) + throw new FileNotFoundException("The media file does not exist.", path); + + return new(file.Length, file.LastWriteTimeUtc); + } + + public bool StillMatches(string path) + { + try + { + return Read(path) == this; + } + catch + { + return false; + } + } + } +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscriptResult.cs b/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscriptResult.cs new file mode 100644 index 000000000..553286f97 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscriptResult.cs @@ -0,0 +1,26 @@ +using AIStudio.Tools.Rust; + +namespace AIStudio.Tools.Media; + +/// +/// The outcome of a request for a persistent transcript, cf. PersistentMediaTranscript. +/// +/// +/// The transcript itself is not part of it: whoever needs the text reads the file at +/// TranscriptPath, so a transcript reaches the AI by the same path no matter whether it was just +/// made or stored long ago. +/// +/// How the request ended. +/// The path of the transcript; empty when the place could not be determined. +/// What identifies the transcript at its location; empty when the place could not be determined. +/// Whether an existing transcript was used instead of a new transcription. +/// A localized explanation for every status but READY and CANCELLED. +/// Why the media pipeline failed, for TRANSCRIPTION_FAILED, when it is known. +internal sealed record PersistentMediaTranscriptResult(PersistentMediaTranscriptStatus Status, string TranscriptPath, string Key, bool Reused, string UserMessage, MediaJobErrorCode? ErrorCode = null) +{ + public static PersistentMediaTranscriptResult Ready(MediaTranscriptSlot slot, bool reused) => new(PersistentMediaTranscriptStatus.READY, slot.Path, slot.Key, reused, string.Empty); + + public static PersistentMediaTranscriptResult Cancelled() => new(PersistentMediaTranscriptStatus.CANCELLED, string.Empty, string.Empty, false, string.Empty); + + public static PersistentMediaTranscriptResult Unavailable(PersistentMediaTranscriptStatus status, MediaTranscriptSlot? slot, string userMessage, MediaJobErrorCode? errorCode = null) => new(status, slot?.Path ?? string.Empty, slot?.Key ?? string.Empty, false, userMessage, errorCode); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscriptStatus.cs b/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscriptStatus.cs new file mode 100644 index 000000000..cbc2df9a0 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Media/PersistentMediaTranscriptStatus.cs @@ -0,0 +1,63 @@ +namespace AIStudio.Tools.Media; + +/// +/// How the request for a persistent transcript ended, cf. PersistentMediaTranscript. +/// +public enum PersistentMediaTranscriptStatus +{ + /// + /// The transcript is stored at its place and holds text. + /// + READY, + + /// + /// There is no transcript yet, and the caller allows no transcription provider. + /// + NO_PROVIDER, + + /// + /// The media file could not be read before it was transcribed. + /// + SOURCE_UNREADABLE, + + /// + /// A transcript exists, but it could not be read. + /// + TRANSCRIPT_UNREADABLE, + + /// + /// The transcription failed. The error code of the media pipeline tells why, when it is known. + /// + TRANSCRIPTION_FAILED, + + /// + /// The transcription provider refused the media file itself, e.g., because it is too large + /// or damaged. It would refuse the unchanged file again. + /// + REJECTED, + + /// + /// The media file holds no audible signal, so there was nothing to transcribe. + /// + NO_AUDIBLE_SIGNAL, + + /// + /// The transcription provider returned no text. Nothing was stored. + /// + EMPTY, + + /// + /// The media file changed while it was transcribed. Nothing was stored. + /// + SOURCE_CHANGED, + + /// + /// The transcript could not be stored at its place. + /// + STORE_FAILED, + + /// + /// The caller cancelled before a transcript was available. + /// + CANCELLED, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataBackupService.cs b/app/MindWork AI Studio/Tools/Services/DataBackupService.cs index f937533aa..4ad492e85 100644 --- a/app/MindWork AI Studio/Tools/Services/DataBackupService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataBackupService.cs @@ -170,7 +170,7 @@ public sealed class DataBackupService(ILogger logger, Setting var settingsJson = JsonSerializer.Serialize(settings, SettingsManager.JSON_OPTIONS); var organizationPluginFolders = DataBackupContents.FindOrganizationPluginsAmongUserPlugins(PluginFactory.AvailablePlugins.OfType(), dataDirectory); - indexCopiesDirectory = await this.CopyIndexesAsync(token); + indexCopiesDirectory = await this.CopyIndexesAsync(dataDirectory, token); var sources = new DataBackupSources(dataDirectory, configDirectory, settingsJson, indexCopiesDirectory, organizationPluginFolders); var manifest = new DataBackupManifest( @@ -207,7 +207,7 @@ public sealed class DataBackupService(ILogger logger, Setting } /// - /// Copies both indexes of the data sources, while the indexing pauses. + /// Copies both indexes of the data sources and the transcripts of their media files, while the indexing pauses. /// /// /// Both copies are made during one pause: the vector stores have to fit to the index, or a @@ -217,8 +217,10 @@ public sealed class DataBackupService(ILogger logger, Setting /// /// Without either database, the backup leaves both indexes out, and its manifest says so. /// + /// The data folder, which holds the transcripts. + /// The cancellation token. /// The directory of the copies, in the layout of the data folder, or an empty string when the backup leaves the indexes out. - private async Task CopyIndexesAsync(CancellationToken token) + private async Task CopyIndexesAsync(string dataDirectory, CancellationToken token) { var indexStore = await databaseClientProvider.GetIndexStoreAsync(token); var vectorStore = await databaseClientProvider.GetVectorStoreAsync(token); @@ -241,6 +243,12 @@ public sealed class DataBackupService(ILogger logger, Setting await vectorStore.CopyStoreForBackupAsync(preparation.BackupId, preparation.Stores[index].StoreName, token); } + // + // The index knows which transcript belongs to which media file, so the transcripts are + // copied while the indexing pauses as well, and the backup holds both as they fit together: + // + await DataSourceTranscriptStore.CopyAllAsync(dataDirectory, preparation.StagingPath, token); + return preparation.StagingPath; } catch diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoreReason.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoreReason.cs index ac5b5027b..1d9e399b0 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoreReason.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoreReason.cs @@ -5,6 +5,24 @@ namespace AIStudio.Tools.Services; /// public enum DataSourceEmbeddingIgnoreReason { - MEDIA_NOT_SUPPORTED, + /// + /// An image, which cannot be added to the index yet. + /// + IMAGE_NOT_SUPPORTED, + + /// + /// An audio or video file without a transcript, while no usable transcription provider is configured. + /// + TRANSCRIPTION_PROVIDER_MISSING, + + /// + /// An audio or video file without a transcript, while the transcription provider falls short of + /// the confidence level the data source asks for. + /// + TRANSCRIPTION_PROVIDER_NOT_ALLOWED, + + /// + /// A file larger than the limit of the data source. + /// TOO_LARGE, } \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoredFile.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoredFile.cs index 4a10bd112..1c15bcc8f 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoredFile.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingIgnoredFile.cs @@ -6,7 +6,8 @@ namespace AIStudio.Tools.Services; /// /// /// Files below folders the user chose not to scan never reach enumeration and therefore do not -/// appear here. Temporary and system files are implementation details and stay hidden as well. +/// appear here. Generated transcript artifacts and temporary/system files are implementation +/// details and stay hidden as well. /// /// The absolute path of the ignored file. /// Why the file was ignored. diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index f5d738b68..7ebcc4d40 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -7,6 +7,7 @@ using AIStudio.Settings.DataModel; using AIStudio.Tools.Databases; using AIStudio.Tools.Databases.IndexStore; using AIStudio.Tools.Databases.VectorStore; +using AIStudio.Tools.Media; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Security; using AIStudio.Tools.Services.Indexing; @@ -16,7 +17,7 @@ using static AIStudio.Tools.Services.Indexing.IndexingLogFormat; namespace AIStudio.Tools.Services; public sealed partial class DataSourceEmbeddingService(SettingsManager settingsManager, RustService rustService, DatabaseClientProvider databaseClientProvider, - PromptInjectionGuardService guardService, ILogger logger) : BackgroundService, IMessageBusReceiver + PromptInjectionGuardService guardService, MediaTranscriptionService mediaTranscriptionService, ILogger logger) : BackgroundService, IMessageBusReceiver { /// /// How long the re-index check waits for the index database before it gives up. @@ -40,7 +41,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM /// /// One indexer per kind of data source this service indexes. /// - private readonly IReadOnlyList indexers = CreateIndexers(settingsManager, rustService, guardService, logger); + private readonly IReadOnlyList indexers = CreateIndexers(settingsManager, rustService, guardService, mediaTranscriptionService, logger); private readonly DataSourceRunQueue runQueue = new(); @@ -93,12 +94,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM /// /// Creates one indexer per kind of data source, all of them cutting their text with the same chunker. /// - private static IReadOnlyList CreateIndexers(SettingsManager settingsManager, RustService rustService, PromptInjectionGuardService guardService, ILogger logger) + private static IReadOnlyList CreateIndexers(SettingsManager settingsManager, RustService rustService, PromptInjectionGuardService guardService, MediaTranscriptionService mediaTranscriptionService, ILogger logger) { var textChunker = new TextChunker(rustService, logger); return [ - new FileSourceIndexer(settingsManager, rustService, textChunker, logger), + new FileSourceIndexer(settingsManager, rustService, mediaTranscriptionService, textChunker, logger), new MailboxIndexer(settingsManager, rustService, guardService, textChunker, logger), ]; } @@ -518,10 +519,35 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM this.statuses.TryRemove(dataSource.Id, out _); await this.ResetPersistedStateAsync(dataSource.Id, null, null, CancellationToken.None); + this.DeleteMediaTranscripts(dataSource); this.statuses.TryRemove(dataSource.Id, out _); PublishStatusChanged(); } + /// + /// Deletes the transcripts of the audio and video files of a data source which the user removed. + /// + /// + /// Only the removal of a data source does this. A reset for new embedding settings keeps them, + /// cf. ResetPersistedStateAsync: they are still valid, and the new index would otherwise pay + /// for every recording once more. + /// + /// The removed data source. + private void DeleteMediaTranscripts(IDataSourceBase dataSource) + { + if (dataSource is not (DataSourceLocalDirectory or DataSourceLocalFile)) + return; + + try + { + DataSourceTranscriptStore.TryForDataSource(dataSource.Id)?.DeleteAll(); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not remove the transcripts of the deleted data source '{DataSourceName}' ({DataSourceId}).", dataSource.Name, dataSource.Id); + } + } + protected override async Task ExecuteAsync(CancellationToken stoppingToken) { // Changes to the embedding settings which do not pass through the queue, cf. ProcessMessage: @@ -1086,6 +1112,18 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM ? embeddingProviderConfidence.AllowsMailboxConfidenceLevel(dataSource.ConfidenceLevel) : embeddingProviderConfidence.AllowsDataSourceConfidenceLevel(dataSource.ConfidenceLevel); + /// + /// Whether a transcription provider of the given confidence level may hear the audio and video files of a data source. + /// + /// + /// The transcription provider receives the content of the data source just like the embedding + /// provider does, so the same rule applies to both, cf. AllowsEmbedding. + /// + /// The data source to index. + /// The confidence level of the transcription provider. + /// True when the provider may transcribe the media files of the data source. + internal static bool AllowsTranscription(IIndexedDataSource dataSource, ConfidenceLevel transcriptionProviderConfidence) => AllowsEmbedding(dataSource, transcriptionProviderConfidence); + /// /// The configured data sources this service indexes, from every list which holds some. /// diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs index b1aaac71f..3780dbddf 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingStatus.cs @@ -68,6 +68,13 @@ public sealed record DataSourceEmbeddingStatus( /// public int SkippedDocuments => this.PermanentlySkippedDocuments + this.IgnoredFiles.Count; + /// + /// Whether selected audio or video files wait for a transcription provider which this data + /// source allows, so they cannot be indexed until the settings change. + /// + public bool NeedsTranscriptionProvider => this.IgnoredFiles.Any(file => + file.Reason is DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING or DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_NOT_ALLOWED); + /// /// The settled health of this data source. Activity is represented by State /// instead, so queued and running sources can keep their activity indicator. @@ -84,7 +91,8 @@ public sealed record DataSourceEmbeddingStatus( if (this.State is DataSourceEmbeddingState.FAILED || this.FailedDocuments > 0) return DataSourceEmbeddingHealth.ERROR; - if (this.State is DataSourceEmbeddingState.COMPLETED && this.IndexedDocuments == 0 && this.SkippedDocuments > 0) + if (this.NeedsTranscriptionProvider || + this.State is DataSourceEmbeddingState.COMPLETED && this.IndexedDocuments == 0 && this.SkippedDocuments > 0) return DataSourceEmbeddingHealth.WARNING; return DataSourceEmbeddingHealth.HEALTHY; diff --git a/app/MindWork AI Studio/Tools/Services/EmbeddedFileRecord.cs b/app/MindWork AI Studio/Tools/Services/EmbeddedFileRecord.cs index f46901513..d94808c39 100644 --- a/app/MindWork AI Studio/Tools/Services/EmbeddedFileRecord.cs +++ b/app/MindWork AI Studio/Tools/Services/EmbeddedFileRecord.cs @@ -1,3 +1,3 @@ namespace AIStudio.Tools.Services; -public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount); +public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount, string? TranscriptKey = null); diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs b/app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs index 1ca7c21e9..db2c71eab 100644 --- a/app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs +++ b/app/MindWork AI Studio/Tools/Services/Indexing/DocumentRunProgress.cs @@ -226,7 +226,7 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo var embeddedAtUtc = DateTimeOffset.UtcNow; var state = document.State with { ChunkCount = chunkCount, EmbeddedAtUtc = embeddedAtUtc }; await context.IndexStore.UpsertFileAsync(context.DataSource.Id, state, token); - context.Manifest.Files[document.Key] = new EmbeddedFileRecord(state.Fingerprint, state.FileSize, state.LastWriteUtc, embeddedAtUtc, chunkCount); + context.Manifest.Files[document.Key] = new EmbeddedFileRecord(state.Fingerprint, state.FileSize, state.LastWriteUtc, embeddedAtUtc, chunkCount, state.TranscriptKey); await context.ForgetPermanentFailureAsync(document.Key, token); context.Pass.Forget(document.Key); @@ -300,7 +300,7 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo // this.PermanentlySkippedDocuments++; var occurredAtUtc = DateTimeOffset.UtcNow; - var indexingMessage = this.GetFailureMessage(extractionFailure.Code, document); + var indexingMessage = this.GetFailureMessage(extractionFailure.Code, document, extractionFailure.Message); this.failures.Add(new DataSourceEmbeddingFailure(document.Key, indexingMessage, occurredAtUtc, ExtractionCode: extractionFailure.Code, IsPermanent: true, DisplayName: document.DisplayName)); context.Manifest.Files.Remove(document.Key); context.Pass.Forget(document.Key); @@ -533,10 +533,37 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo /// /// Why reading the document failed, NONE when it was not about reading it. /// The document. + /// A localized explanation supplied by a specialized reader, when available. /// The message, ready to show. - private string GetFailureMessage(FileExtractionErrorCode code, EmbeddingDocument document) => context.DataSource is DataSourceMailbox - ? string.Format(TB("The mail '{0}' could not be indexed. AI Studio tries again during the next sync."), document.DisplayName) - : code.ToIndexingUserMessage(document.DisplayName); + private string GetFailureMessage(FileExtractionErrorCode code, EmbeddingDocument document, string? details = null) + { + if (context.DataSource is DataSourceMailbox) + return string.Format(TB("The mail '{0}' could not be indexed. AI Studio tries again during the next sync."), document.DisplayName); + + if (code is FileExtractionErrorCode.MEDIA_PROCESSING_FAILED && !string.IsNullOrWhiteSpace(details)) + { + return string.Format( + TB("The media file '{0}' could not be processed, so it was not indexed. Reason: {1} AI Studio reads it again as soon as the file changes."), + document.DisplayName, + AsSentence(details)); + } + + return code.ToIndexingUserMessage(document.DisplayName); + } + + /// + /// Ends a reason with a full stop, so the sentence after it does not run into it. + /// + /// + /// Providers write their error messages as they like, often without a full stop at the end. + /// + /// The reason. + /// The reason, ending with a punctuation mark. + internal static string AsSentence(string reason) + { + var sentence = reason.TrimEnd(); + return sentence.Length is 0 || sentence[^1] is '.' or '!' or '?' or '…' ? sentence : $"{sentence}."; + } private DataSourceEmbeddingStatus CreateStatus(DataSourceEmbeddingState state, string currentDocument, string lastError, int? currentBlock, int? currentPage, DataSourceAttention attention = DataSourceAttention.NONE, int? pendingRemovalCount = null) => new( context.DataSource.Id, diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.Watchers.cs b/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.Watchers.cs index 6801f8320..d61a3b12c 100644 --- a/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.Watchers.cs +++ b/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.Watchers.cs @@ -2,6 +2,7 @@ using System.Collections.Concurrent; using AIStudio.Settings; using AIStudio.Settings.DataModel; +using AIStudio.Tools.Media; namespace AIStudio.Tools.Services.Indexing; @@ -268,6 +269,14 @@ internal sealed partial class FileSourceIndexer if (Directory.Exists(path)) return true; + // + // The Batch Processing assistant writes the transcript of a media file next to it. When + // the data source selects the media file itself, the transcript is no document of its + // own, so writing it must not start a run either: + // + if (MediaSidecarTranscripts.IsCoveredArtifact(path, selection)) + return false; + if (selection.Selects(path)) return true; diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs b/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs index 97afbc763..fe1f6307b 100644 --- a/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs +++ b/app/MindWork AI Studio/Tools/Services/Indexing/FileSourceIndexer.cs @@ -5,6 +5,7 @@ using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.Databases.IndexStore; using AIStudio.Tools.Databases.VectorStore; +using AIStudio.Tools.Media; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; using AIStudio.Tools.Security; @@ -32,9 +33,10 @@ namespace AIStudio.Tools.Services.Indexing; /// /// The settings, read for whether local data sources refresh on their own. /// The runtime, which extracts the text of the files. +/// Transcribes selected audio and video files. /// Cuts the text of a file into chunks. /// The logger of the embedding service, so the log reads the same whoever writes it. -internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, RustService rustService, TextChunker textChunker, ILogger logger) : IIndexedSourceIndexer +internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, RustService rustService, MediaTranscriptionService mediaTranscriptionService, TextChunker textChunker, ILogger logger) : IIndexedSourceIndexer { private const bool IMAGE_EMBEDDING_ENABLED = false; private const int MAX_LOGGED_EXCLUDED_FILE_EXAMPLES = 3; @@ -45,12 +47,22 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, { INDEXABLE, EXCLUDED, - MEDIA_NOT_SUPPORTED, + IMAGE_NOT_SUPPORTED, + TRANSCRIPTION_PROVIDER_MISSING, + TRANSCRIPTION_PROVIDER_NOT_ALLOWED, TOO_LARGE, } private sealed record DataSourceMetadataSnapshot(string SourceHash, IReadOnlyDictionary FileHashes); + /// + /// What decides about the files of a data source while they are listed. + /// + /// The files of a folder data source, or null for a single file, which has no size limit. + /// What the index stores about the data source, or null when it does not matter. + /// Which transcription provider may hear audio and video files, asked for only once such a file turns up. + private sealed record InputFileRules(FolderFileSelection? Selection, DataSourceEmbeddingManifest? Manifest, Lazy MediaTranscription); + /// public bool Supports(IDataSourceBase dataSource) => dataSource is DataSourceLocalDirectory or DataSourceLocalFile; @@ -66,7 +78,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, var manifest = context.Manifest; var enumerationStartedUtc = DateTimeOffset.UtcNow; - var inputFiles = this.GetInputFiles(dataSource); + var inputFiles = this.GetInputFiles(dataSource, manifest); var enumerationDuration = DateTimeOffset.UtcNow - enumerationStartedUtc; if (inputFiles.UnreachableRootReason is { } unreachableRootReason) return this.StopWithoutRoot(context, dataSource, unreachableRootReason); @@ -133,6 +145,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, progress.RecordStillUnreadable(filePath, permanentFailure); await progress.CompleteRunAsync(metadataSnapshot.SourceHash, "data source finished after removing missing files", token); + this.CleanUpTranscripts(context, progress); return IndexedRunOutcome.DONE; } @@ -181,7 +194,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, var fingerprint = metadataSnapshot.FileHashes[file.FullName]; var existingRecord = manifest.Files.GetValueOrDefault(file.FullName); - var document = this.CreateFileDocument(context, dataSource, file, fingerprint); + var document = this.CreateFileDocument(context, dataSource, file, fingerprint, null); var reportBlockProgress = progress.BeginDocument(document); var startedAtUtc = DateTimeOffset.UtcNow; @@ -196,6 +209,14 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, ShortHash(fingerprint), progress.DoneDocuments + 1, totalFiles); + + // + // Audio and video hold no text until they are transcribed. The document then reads + // its transcript instead, and its row knows which transcript that is: + // + if (PersistentMediaTranscript.IsMediaPath(file.FullName)) + document = this.CreateFileDocument(context, dataSource, file, fingerprint, await this.ReadOrCreateTranscriptAsync(dataSource, file.FullName, token)); + var chunkCount = await context.IndexDocumentAsync(document, reportBlockProgress, token); token.ThrowIfCancellationRequested(); var fingerprintAfterEmbedding = BuildFileMetadataHash(file); @@ -203,6 +224,11 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, throw new IOException(string.Format(TB("The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run."), file.FullName)); await progress.RecordDocumentIndexedAsync(document, chunkCount, existingRecord is null, token); + + // The recording changed, so its old transcript is no longer what it says: + if (existingRecord?.TranscriptKey is { } previousTranscriptKey && !string.Equals(previousTranscriptKey, document.State.TranscriptKey, StringComparison.Ordinal)) + ForgetTranscript(DataSourceTranscriptStore.TryForDataSource(dataSource.Id), manifest, previousTranscriptKey, logger); + logger.LogInformation( "Embedded file '{FilePath}' for data source '{DataSourceName}' ({DataSourceId}) successfully. Chunks={ChunkCount}, DurationMs={DurationMs}.", file.FullName, @@ -246,6 +272,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, } await progress.CompleteRunAsync(metadataSnapshot.SourceHash, "data source embedding run finished", token); + this.CleanUpTranscripts(context, progress); logger.LogInformation( "Finished background embeddings for data source '{DataSourceName}' ({DataSourceId}). RefreshMode={RefreshMode}, Embedded={EmbeddedFiles}, New={NewFiles}, Changed={ChangedFiles}, Skipped={SkippedFiles}, PermanentlySkipped={PermanentlySkippedFiles}, RemovedMissing={RemovedMissingFiles}, Failed={FailedFiles}, Total={TotalFiles}, SourceHashPrefix={SourceHashPrefix}.", dataSource.Name, @@ -355,6 +382,79 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, return IndexedRunOutcome.DONE; } + /// + /// Deletes the transcripts which no file of the data source uses anymore, once a run got through all of it. + /// + /// + /// Only a run without a failure knows every transcript which is still needed: a file which + /// failed may have its transcript already, and its next attempt must not pay for it again. + /// The transcript of a recording which is gone has to go as well, since it holds what was said. + /// + /// The run. + /// How the run went. + private void CleanUpTranscripts(IndexedRunContext context, DocumentRunProgress progress) + { + if (progress.FailedDocuments > 0) + return; + + DeleteUnusedTranscripts(DataSourceTranscriptStore.TryForDataSource(context.DataSource.Id), context.Manifest, logger); + } + + /// + /// Deletes every transcript of a data source which no indexed file uses, and the temporary files of interrupted writes. + /// + /// The transcripts of the data source, or null when there are none to clean up. + /// What the index stores about the data source. + /// The logger. + internal static void DeleteUnusedTranscripts(DataSourceTranscriptStore? store, DataSourceEmbeddingManifest manifest, ILogger logger) + { + if (store is null) + return; + + try + { + var usedKeys = manifest.Files.Values + .Select(record => record.TranscriptKey) + .OfType() + .ToHashSet(StringComparer.Ordinal); + + var deleted = store.ListKeys().Count(key => !usedKeys.Contains(key) && store.Delete(key)); + deleted += store.DeleteTemporaryFiles(); + if (deleted > 0) + logger.LogInformation("Removed {DeletedTranscripts} transcript file(s) which no file of the data source uses anymore.", deleted); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not remove the transcripts which no file of the data source uses anymore. The next complete run tries again."); + } + } + + /// + /// Deletes the transcript with the given key, unless another file of the data source still uses it. + /// + /// + /// Two recordings with the same content share one transcript, cf. DataSourceTranscriptStore. + /// + /// The transcripts of the data source, or null when there are none to clean up. + /// What the index stores about the data source, without the file the transcript belonged to. + /// The key of the transcript, or null when the file had none. + /// The logger. + internal static void ForgetTranscript(DataSourceTranscriptStore? store, DataSourceEmbeddingManifest manifest, string? key, ILogger logger) + { + if (store is null || string.IsNullOrWhiteSpace(key) || manifest.Files.Values.Any(record => string.Equals(record.TranscriptKey, key, StringComparison.Ordinal))) + return; + + try + { + if (store.Delete(key)) + logger.LogInformation("Removed the transcript '{TranscriptKey}', since no file of the data source uses it anymore.", key); + } + catch (Exception exception) + { + logger.LogWarning(exception, "Could not remove the transcript '{TranscriptKey}'. The next complete run tries again.", key); + } + } + private async Task RemoveMissingFileEmbeddingsAsync(IndexedRunContext context, FileEnumerationResult inputFiles, CancellationToken token) { var manifest = context.Manifest; @@ -363,7 +463,9 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, { await context.DeleteDocumentPointsAsync(removedFilePath, token); await context.IndexStore.DeleteFileAsync(context.DataSource.Id, removedFilePath, token); + var removedTranscriptKey = manifest.Files.GetValueOrDefault(removedFilePath)?.TranscriptKey; manifest.Files.Remove(removedFilePath); + ForgetTranscript(DataSourceTranscriptStore.TryForDataSource(context.DataSource.Id), manifest, removedTranscriptKey, logger); removedFiles++; logger.LogInformation( "Removed stale embeddings for deleted file '{FilePath}' from data source '{DataSourceName}' ({DataSourceId}).", @@ -487,19 +589,123 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, : string.Join("; ", reasons); } - private async IAsyncEnumerable StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, PromptInjectionReportingScope reportingScope, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) + private async IAsyncEnumerable StreamEmbeddingChunksAsync(string filePath, MediaTranscriptSlot? transcript, IDataSource dataSource, EmbeddingProvider embeddingProvider, PromptInjectionReportingScope reportingScope, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token) { var options = DataSourceEmbeddingService.GetChunkingOptions(dataSource, embeddingProvider); - var strategy = TextChunker.GetStrategyForFile(filePath); - var content = await ExtractedFileText.ReadAsync(rustService, filePath, embeddingProvider, null, reportingScope, token); + + // + // An audio or video file is read through its transcript. The runtime reads the transcript + // like every other document, so it passes the same prompt injection filter, and a notice + // names the media file rather than the transcript below the data directory: + // + var contentPath = transcript?.Path ?? filePath; + PromptInjectionSource? reportAs = transcript is null ? null : PromptInjectionSource.FileContent(filePath); + var strategy = TextChunker.GetStrategyForFile(contentPath); + var content = await ExtractedFileText.ReadAsync(rustService, contentPath, embeddingProvider, reportAs, reportingScope, token); await foreach (var chunk in textChunker.SplitAsync(content, strategy, options, embeddingProvider, token)) yield return chunk; } - internal FileEnumerationResult GetInputFiles(IDataSource dataSource) + /// + /// Returns the transcript of an audio or video file, transcribing the file only when the data source holds no transcript of its content yet. + /// + /// + /// The provider is checked against the data source once more, right before the upload: the + /// settings may have changed since the files were listed. A transcript which exists is used + /// without any provider, since nothing leaves the computer for it.

+ /// + /// A transcription which cannot succeed for this file, such as one without text, is recorded + /// for good, so the file is not paid for again until it changes. + /// + /// The data source. + /// The audio or video file. + /// The cancellation token. + /// Where the transcript is. + private async Task ReadOrCreateTranscriptAsync(IDataSource dataSource, string mediaPath, CancellationToken token) + { + var provider = this.GetMediaTranscriptionAvailability(dataSource).Provider; + var transcript = await PersistentMediaTranscript.ReadOrCreateAsync(DataSourceTranscriptStore.ForDataSource(dataSource.Id), mediaPath, provider, mediaTranscriptionService.TranscribeAsync, logger, token); + return GetTranscriptOrThrow(transcript, token); + } + + /// + /// Returns the transcript, or throws what tells the run how to treat the media file without one. + /// + /// + /// A FileExtractionException records the file for good, until it changes; any other exception + /// lets the next pass try again. A transcription without text, a file which the media pipeline + /// cannot read, and a file which the transcription provider refused would fail the same way + /// every time, and every attempt would cost money or another upload. + /// + /// The outcome of the request for the transcript. + /// The cancellation token of the run. + /// Where the transcript is. + internal static MediaTranscriptSlot GetTranscriptOrThrow(PersistentMediaTranscriptResult transcript, CancellationToken token) => transcript.Status switch + { + PersistentMediaTranscriptStatus.READY => new(transcript.TranscriptPath, transcript.Key), + PersistentMediaTranscriptStatus.CANCELLED => throw new OperationCanceledException(token), + PersistentMediaTranscriptStatus.EMPTY or PersistentMediaTranscriptStatus.NO_AUDIBLE_SIGNAL => throw new FileExtractionException(FileExtractionErrorCode.NO_CONTENT, transcript.UserMessage), + PersistentMediaTranscriptStatus.TRANSCRIPTION_FAILED when IsPermanentMediaFailure(transcript.ErrorCode) => throw new FileExtractionException(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED, transcript.UserMessage), + PersistentMediaTranscriptStatus.REJECTED => throw new FileExtractionException(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED, transcript.UserMessage), + _ => throw new IOException(transcript.UserMessage), + }; + + /// + /// Which transcription provider may hear the audio and video files of a data source right now. + /// + /// + /// The transcription provider receives the content of the data source just like the embedding + /// provider, so it has to meet the confidence level of the data source as well, cf. + /// DataSourceEmbeddingService.AllowsTranscription. A data source without a confidence level + /// of its own is never sent anywhere. + /// + /// The data source. + /// The provider, or why there is none. + private MediaTranscriptionAvailability GetMediaTranscriptionAvailability(IDataSource dataSource) + { + var provider = mediaTranscriptionService.GetUsableProvider(); + if (provider is null) + return MediaTranscriptionAvailability.MISSING; + + var providerConfidence = provider.GetConfidenceLevel(settingsManager); + if (dataSource is IIndexedDataSource indexedDataSource && DataSourceEmbeddingService.AllowsTranscription(indexedDataSource, providerConfidence)) + return MediaTranscriptionAvailability.Allowed(provider); + + logger.LogInformation( + "The transcription provider '{ProviderName}' may not hear the audio and video files of data source '{DataSourceName}' ({DataSourceId}): the data source asks for the confidence level {RequiredConfidence}, while the provider has {ProviderConfidence}.", + provider.Name, + dataSource.Name, + dataSource.Id, + (dataSource as IIndexedDataSource)?.ConfidenceLevel, + providerConfidence); + + return MediaTranscriptionAvailability.NOT_ALLOWED; + } + + private static bool IsPermanentMediaFailure(MediaJobErrorCode? code) => code is + MediaJobErrorCode.UNKNOWN_FORMAT + or MediaJobErrorCode.UNSAFE_FILE + or MediaJobErrorCode.NOT_MEDIA + or MediaJobErrorCode.UNSUPPORTED_CONTAINER + or MediaJobErrorCode.NO_AUDIO_TRACK + or MediaJobErrorCode.UNSUPPORTED_CODEC + or MediaJobErrorCode.INVALID_AUDIO_PARAMETERS + or MediaJobErrorCode.INVALID_OPUS_HEADER + or MediaJobErrorCode.UNSUPPORTED_OPUS_MAPPING + or MediaJobErrorCode.DAMAGED_CONTAINER; + + /// + /// Lists the files of a data source and decides what a run does with each of them. + /// + /// The data source. + /// What the index stores about the data source, or null when it does not matter. + /// Which transcription provider may hear its audio and video files, or null to ask the settings once a media file turns up. + /// The files, and those which were left out. + internal FileEnumerationResult GetInputFiles(IDataSource dataSource, DataSourceEmbeddingManifest? manifest = null, MediaTranscriptionAvailability? mediaTranscription = null) { var result = new FileEnumerationResult(); + var mediaTranscriptionAvailability = new Lazy(() => mediaTranscription ?? this.GetMediaTranscriptionAvailability(dataSource)); switch (dataSource) { @@ -507,13 +713,13 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, var file = new FileInfo(localFile.FilePath); // No size limit: the user chose this very file, cf. FolderFileSelection.IsTooLarge. - if (this.AddInputFile(file, null, result) is RagFileIndexingDecision.EXCLUDED) + if (this.AddInputFile(file, new InputFileRules(null, manifest, mediaTranscriptionAvailability), result) is RagFileIndexingDecision.EXCLUDED) logger.LogDebug("Skipping excluded file '{FilePath}' while indexing.", file.FullName); return result; case DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path): - this.EnumerateAccessibleFiles(localDirectory, result); + this.EnumerateAccessibleFiles(localDirectory, manifest, mediaTranscriptionAvailability, result); if (result.UnreadableDirectories.Contains(localDirectory.Path)) result.UnreachableRootReason = string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."), localDirectory.Path); @@ -537,9 +743,10 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, return result; } - private void EnumerateAccessibleFiles(DataSourceLocalDirectory dataSource, FileEnumerationResult result) + private void EnumerateAccessibleFiles(DataSourceLocalDirectory dataSource, DataSourceEmbeddingManifest? manifest, Lazy mediaTranscription, FileEnumerationResult result) { var selection = FolderFileSelection.Of(dataSource); + var rules = new InputFileRules(selection, manifest, mediaTranscription); // // Every log line travels to the runtime as a request of its own. A folder holding many @@ -580,7 +787,17 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, continue; } - if (this.AddInputFile(fileInfo, selection, result) is not RagFileIndexingDecision.EXCLUDED) + // + // The transcript which the Batch Processing assistant wrote next to a selected media + // file holds the text the data source gets from the media file itself: + // + if (MediaSidecarTranscripts.IsCoveredArtifact(fileInfo.FullName, selection)) + { + logger.LogDebug("Skipping the transcript '{FilePath}', because its media file is indexed as the document.", fileInfo.FullName); + continue; + } + + if (this.AddInputFile(fileInfo, rules, result) is not RagFileIndexingDecision.EXCLUDED) continue; excludedFiles++; @@ -593,11 +810,11 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, } /// The file. - /// The files of a folder data source, or null for a single file, which has no size limit. + /// What decides about the files of the data source. /// Where the file goes, if anywhere. - private RagFileIndexingDecision AddInputFile(FileInfo file, FolderFileSelection? selection, FileEnumerationResult result) + private RagFileIndexingDecision AddInputFile(FileInfo file, InputFileRules rules, FileEnumerationResult result) { - var decision = this.GetRagFileIndexingDecision(file, selection); + var decision = this.GetRagFileIndexingDecision(file, rules); switch (decision) { case RagFileIndexingDecision.INDEXABLE: @@ -607,8 +824,16 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, case RagFileIndexingDecision.EXCLUDED: break; - case RagFileIndexingDecision.MEDIA_NOT_SUPPORTED: - result.AddIgnoredFile(file.FullName, DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED); + case RagFileIndexingDecision.IMAGE_NOT_SUPPORTED: + result.AddIgnoredFile(file.FullName, DataSourceEmbeddingIgnoreReason.IMAGE_NOT_SUPPORTED); + break; + + case RagFileIndexingDecision.TRANSCRIPTION_PROVIDER_MISSING: + result.AddIgnoredFile(file.FullName, DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING); + break; + + case RagFileIndexingDecision.TRANSCRIPTION_PROVIDER_NOT_ALLOWED: + result.AddIgnoredFile(file.FullName, DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_NOT_ALLOWED); break; case RagFileIndexingDecision.TOO_LARGE: @@ -630,24 +855,24 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, return FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE); } - private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file, FolderFileSelection? selection) + private RagFileIndexingDecision GetRagFileIndexingDecision(FileInfo file, InputFileRules rules) { if (this.IsSkippedRagFile(file)) return RagFileIndexingDecision.EXCLUDED; if (!IMAGE_EMBEDDING_ENABLED && this.IsImageFilePath(file.FullName)) - return RagFileIndexingDecision.MEDIA_NOT_SUPPORTED; + return RagFileIndexingDecision.IMAGE_NOT_SUPPORTED; // - // Audio and video hold no text until they are transcribed, and data sources do not - // transcribe them yet: + // Before any transcription: a video of several gigabytes is left out like every other + // file above the limit, rather than sent to the transcription provider. // - if (FileTypes.IsAllowedPath(file.FullName, FileTypes.AUDIO, FileTypes.VIDEO)) - return RagFileIndexingDecision.MEDIA_NOT_SUPPORTED; - - if (selection?.IsTooLarge(file) is true) + if (rules.Selection?.IsTooLarge(file) is true) return RagFileIndexingDecision.TOO_LARGE; + if (PersistentMediaTranscript.IsMediaPath(file.FullName)) + return GetMediaIndexingDecision(file, rules); + // A directory source reaches this point only after its user-configured pattern selected // the file. A single-file source was validated before it was stored. Extraction is the // authoritative capability check and records an unreadable selected file through the @@ -655,6 +880,39 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, return RagFileIndexingDecision.INDEXABLE; } + /// + /// Decides about an audio or video file, which holds no text until it is transcribed. + /// + /// + /// A media file which the index holds unchanged needs no transcript in this run, nor does one + /// which failed for good before: neither is read. A missing transcription provider, or one which + /// the data source no longer allows, therefore takes nothing out of the index. Every other media + /// file needs a provider which may hear it, unless the data source has its transcript already; + /// that is only known once the file is read, so such a file waits for the provider as well. + /// + /// The audio or video file. + /// What decides about the files of the data source. + /// INDEXABLE, or why the file is left out. + private static RagFileIndexingDecision GetMediaIndexingDecision(FileInfo file, InputFileRules rules) + { + if (rules.Manifest is { } manifest) + { + var fingerprint = BuildFileMetadataHash(file); + if (manifest.Files.TryGetValue(file.FullName, out var existingRecord) && string.Equals(existingRecord.Fingerprint, fingerprint, StringComparison.Ordinal)) + return RagFileIndexingDecision.INDEXABLE; + + if (manifest.PermanentFailures.TryGetValue(file.FullName, out var permanentFailure) && string.Equals(permanentFailure.Fingerprint, fingerprint, StringComparison.Ordinal)) + return RagFileIndexingDecision.INDEXABLE; + } + + return rules.MediaTranscription.Value.IgnoreReason switch + { + null => RagFileIndexingDecision.INDEXABLE, + DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_NOT_ALLOWED => RagFileIndexingDecision.TRANSCRIPTION_PROVIDER_NOT_ALLOWED, + _ => RagFileIndexingDecision.TRANSCRIPTION_PROVIDER_MISSING, + }; + } + private bool IsSkippedRagFile(FileInfo file) { if (FileExclusions.IsAlwaysExcludedFile(file.Name)) @@ -731,7 +989,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, return BuildMetadataHash(parts); } - private static string BuildFileMetadataHash(FileInfo file) + internal static string BuildFileMetadataHash(FileInfo file) { file.Refresh(); if (!file.Exists) @@ -774,12 +1032,13 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager, /// The data source the file belongs to. /// The file. /// The fingerprint of the file as it is about to be read. + /// The transcript an audio or video file is read through, or null for every other file. /// The document. - private EmbeddingDocument CreateFileDocument(IndexedRunContext context, IDataSource dataSource, FileInfo file, string fingerprint) => new( + private EmbeddingDocument CreateFileDocument(IndexedRunContext context, IDataSource dataSource, FileInfo file, string fingerprint, MediaTranscriptSlot? transcript) => new( file.FullName, - this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow), + this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow) with { TranscriptKey = transcript?.Key }, file.Name, - token => this.StreamEmbeddingChunksAsync(file.FullName, dataSource, context.EmbeddingProvider, context.PromptInjectionScope, token)); + token => this.StreamEmbeddingChunksAsync(file.FullName, transcript, dataSource, context.EmbeddingProvider, context.PromptInjectionScope, token)); private EmbeddingStateFile CreateEmbeddingStateFile(IDataSource dataSource, FileInfo file, string fingerprint, int chunkCount, DateTimeOffset embeddedAtUtc) { diff --git a/app/MindWork AI Studio/Tools/Services/Indexing/MediaTranscriptionAvailability.cs b/app/MindWork AI Studio/Tools/Services/Indexing/MediaTranscriptionAvailability.cs new file mode 100644 index 000000000..2db64d8a0 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Services/Indexing/MediaTranscriptionAvailability.cs @@ -0,0 +1,28 @@ +using AIStudio.Settings; + +namespace AIStudio.Tools.Services.Indexing; + +/// +/// Which transcription provider may hear the audio and video files of a data source. +/// +/// The provider, or null when none may. +/// Why audio and video files without a transcript are left out when no provider may hear them; null when there is a provider. +internal sealed record MediaTranscriptionAvailability(TranscriptionProvider? Provider, DataSourceEmbeddingIgnoreReason? IgnoreReason) +{ + /// + /// No usable transcription provider is configured. + /// + public static readonly MediaTranscriptionAvailability MISSING = new(null, DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING); + + /// + /// The configured transcription provider falls short of the confidence level of the data source. + /// + public static readonly MediaTranscriptionAvailability NOT_ALLOWED = new(null, DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_NOT_ALLOWED); + + /// + /// The given provider may hear the media files. + /// + /// The provider. + /// The availability. + public static MediaTranscriptionAvailability Allowed(TranscriptionProvider provider) => new(provider, null); +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs index f644553c1..48f43c720 100644 --- a/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs +++ b/app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs @@ -63,7 +63,7 @@ public sealed class MediaTranscriptionService( } /// Gets whether the currently configured transcription provider can be used. - public bool HasUsableTranscriptionProvider => this.ResolveProvider() is not null; + public bool HasUsableTranscriptionProvider => this.GetUsableProvider() is not null; /// Gets the last retained state for one owner. public MediaImportSnapshot? GetSnapshot(MediaImportOwner owner) @@ -411,14 +411,31 @@ public sealed class MediaTranscriptionService( /// Audio or video file path. /// Caller cancellation token. /// A typed terminal result. - public async Task TranscribeAsync(string mediaPath, CancellationToken token = default) + public Task TranscribeAsync(string mediaPath, CancellationToken token = default) => this.TranscribeWithoutImportAsync(mediaPath, null, token); + + /// + /// Transcribes an audio or video file with exactly the given provider, without starting a visible import operation. + /// + /// + /// For callers which checked the provider against rules of their own, such as the confidence + /// level of a data source. The provider is used as given and not resolved from the settings + /// again, so a change of the settings between that check and the upload cannot send the media + /// to another provider. + /// + /// Audio or video file path. + /// The transcription provider the caller checked. + /// Caller cancellation token. + /// A typed terminal result. + public Task TranscribeAsync(string mediaPath, TranscriptionProvider provider, CancellationToken token = default) => this.TranscribeWithoutImportAsync(mediaPath, provider, token); + + private async Task TranscribeWithoutImportAsync(string mediaPath, TranscriptionProvider? provider, CancellationToken token) { this.ThrowIfDisposed(); var operation = this.CreateOperation(null, token); - + try { - return await this.TranscribeCoreAsync(mediaPath, operation, updateImportState: false); + return await this.TranscribeCoreAsync(mediaPath, operation, updateImportState: false, provider); } finally { @@ -458,8 +475,9 @@ public sealed class MediaTranscriptionService( /// Source media path. /// Operation-specific cancellation and Rust-job state. /// Whether progress belongs to the visible import lane. + /// The provider a caller checked itself, or null for the configured one. /// A typed terminal result. - private async Task TranscribeCoreAsync(string mediaPath, MediaOperation operation, bool updateImportState) + private async Task TranscribeCoreAsync(string mediaPath, MediaOperation operation, bool updateImportState, TranscriptionProvider? checkedProvider = null) { var normalizedPath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-media", $"{operation.Id:N}.webm"); Directory.CreateDirectory(Path.GetDirectoryName(normalizedPath)!); @@ -497,7 +515,7 @@ public sealed class MediaTranscriptionService( return MediaTranscriptionResult.NoAudibleSignal(TB("The audio track contains no audible signal, so there is nothing to transcribe.")); } - var providerSettings = this.ResolveProvider(); + var providerSettings = checkedProvider ?? this.GetUsableProvider(); if (providerSettings is null) return MediaTranscriptionResult.Failed(TB("No usable transcription provider is configured.")); @@ -530,16 +548,20 @@ public sealed class MediaTranscriptionService( "The transcription provider failed for '{FileName}' (operation {OperationId}): {Diagnostic}", fileName, operation.Id, - providerResult.ErrorMessage); + string.IsNullOrWhiteSpace(providerResult.ErrorMessage) ? "" : providerResult.ErrorMessage); // // When the provider told us why it failed, the user gets to read it. Only that // message says whether to wait, to check the API key, or to ask the provider for // a format it can read: // - return MediaTranscriptionResult.Failed(string.IsNullOrWhiteSpace(providerResult.ErrorMessage) + var failureMessage = string.IsNullOrWhiteSpace(providerResult.ErrorMessage) ? TB("The transcription provider could not transcribe the media file.") - : providerResult.ErrorMessage); + : providerResult.ErrorMessage; + + return providerResult.FileRejected + ? MediaTranscriptionResult.Rejected(failureMessage) + : MediaTranscriptionResult.Failed(failureMessage); } return MediaTranscriptionResult.Succeeded(providerResult.Text.Trim()); @@ -719,7 +741,12 @@ public sealed class MediaTranscriptionService( } /// Resolves the configured provider after confidence validation. - private TranscriptionProvider? ResolveProvider() + /// + /// Checks the global minimum confidence only. A caller with rules of its own, such as the + /// confidence level of a data source, checks the provider against them as well. + /// + /// The configured provider, or null when none is configured or it falls short of the global minimum. + public TranscriptionProvider? GetUsableProvider() { var providerId = settingsManager.ConfigurationData.App.UseTranscriptionProvider; if (string.IsNullOrWhiteSpace(providerId)) @@ -735,6 +762,19 @@ public sealed class MediaTranscriptionService( : null; } + /// + /// The configured provider, when it may also hear the audio and video files of the given data source. + /// + /// The data source, with the confidence level it asks for. + /// The provider, or null when none is configured or it falls short of the global minimum or of the data source. + public TranscriptionProvider? GetUsableProvider(IIndexedDataSource dataSource) + { + var provider = this.GetUsableProvider(); + return provider is not null && DataSourceEmbeddingService.AllowsTranscription(dataSource, provider.GetConfidenceLevel(settingsManager)) + ? provider + : null; + } + /// Creates and registers operation-owned cancellation state. /// Optional visible media import target. /// Caller token linked to the operation. diff --git a/app/MindWork AI Studio/Tools/TerminalLogger.cs b/app/MindWork AI Studio/Tools/TerminalLogger.cs index 2eb8c84c4..27a6ce4f8 100644 --- a/app/MindWork AI Studio/Tools/TerminalLogger.cs +++ b/app/MindWork AI Studio/Tools/TerminalLogger.cs @@ -65,7 +65,7 @@ public sealed class TerminalLogger() : ConsoleFormatter(FORMATTER_NAME) var transportTimestamp = FormatTransportTimestamp(now); var logLevel = logEntry.LogLevel.ToString(); var category = logEntry.Category; - var exceptionMessage = logEntry.Exception?.Message; + var exceptionMessage = DescribeException(logEntry.Exception); var stackTrace = logEntry.Exception?.StackTrace; var colorCode = GetColorForLogLevel(logEntry.LogLevel); @@ -108,6 +108,34 @@ public sealed class TerminalLogger() : ConsoleFormatter(FORMATTER_NAME) /// The time in the round-trip format. public static string FormatTransportTimestamp(DateTimeOffset timestamp) => timestamp.ToString("O", CultureInfo.InvariantCulture); + /// + /// Describes an exception by its message and the messages of its inner exceptions. + /// + /// + /// The outer message alone often hides the cause. HttpClient, for example, reports a broken + /// upload as "Error while copying content to a stream.", and only an inner exception says + /// whether the other side closed the connection. + /// + /// The exception, or null when the log event has none. + /// The messages, from the outer exception inwards, or null without an exception. + public static string? DescribeException(Exception? exception) + { + if (exception is null) + return null; + + var description = exception.Message; + for (var innerException = exception.InnerException; innerException is not null; innerException = innerException.InnerException) + { + // An AggregateException names the messages of its inner exceptions already: + if (description.Contains(innerException.Message, StringComparison.Ordinal)) + continue; + + description = $"{description} ---> {innerException.Message}"; + } + + return description; + } + private static string GetColorForLogLevel(LogLevel logLevel) => logLevel switch { LogLevel.Trace => ANSI_GRAY, diff --git a/app/MindWork AI Studio/Tools/Validation/CloudProcessing.cs b/app/MindWork AI Studio/Tools/Validation/CloudProcessing.cs new file mode 100644 index 000000000..4edd2fa50 --- /dev/null +++ b/app/MindWork AI Studio/Tools/Validation/CloudProcessing.cs @@ -0,0 +1,23 @@ +namespace AIStudio.Tools.Validation; + +/// +/// What a data source sends to a provider which is not self-hosted. +/// +[Flags] +public enum CloudProcessing +{ + /// + /// Nothing leaves the computer or the organization. + /// + NONE = 0, + + /// + /// The content goes to an embedding provider in the cloud. + /// + EMBEDDING = 1, + + /// + /// Audio and video files go to a transcription provider in the cloud. + /// + TRANSCRIPTION = 2, +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Validation/CloudProcessingExtensions.cs b/app/MindWork AI Studio/Tools/Validation/CloudProcessingExtensions.cs new file mode 100644 index 000000000..0b91a6e6e --- /dev/null +++ b/app/MindWork AI Studio/Tools/Validation/CloudProcessingExtensions.cs @@ -0,0 +1,17 @@ +namespace AIStudio.Tools.Validation; + +public static class CloudProcessingExtensions +{ + /// + /// Whether what the user acknowledged covers what the data source sends to the cloud now. + /// + /// + /// An acknowledgment holds only for the providers it was given for. When the data source starts + /// to send its recordings to a transcription provider in the cloud, say since a new pattern + /// selects them, the user has to acknowledge that once more. + /// + /// What the user acknowledged. + /// What the data source sends to the cloud with its current settings. + /// True when nothing is sent that the user did not acknowledge. + public static bool Covers(this CloudProcessing acknowledged, CloudProcessing current) => (current & ~acknowledged) is CloudProcessing.NONE; +} \ No newline at end of file diff --git a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs index 8a5aec403..6518d8beb 100644 --- a/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs +++ b/app/MindWork AI Studio/Tools/Validation/DataSourceValidation.cs @@ -31,6 +31,8 @@ public sealed class DataSourceValidation public Func GetSelectedCloudEmbedding { get; init; } = () => false; + public Func GetSelectedCloudTranscription { get; init; } = () => false; + public Func GetSelectedEmbeddingProvider { get; init; } = () => null; public Func GetConfidenceLevel { get; init; } = () => ConfidenceLevel.NONE; @@ -158,8 +160,8 @@ public sealed class DataSourceValidation if(!File.Exists(filePath)) return TB("The file does not exist. Please select a valid file."); - if(!FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT)) - return TB("The selected file is not a supported document or readable text file."); + if(!FileTypes.IsAllowedPath(filePath, FileTypes.DOCUMENT, FileTypes.AUDIO, FileTypes.VIDEO)) + return TB("The selected file is not a supported document, readable text file, audio file, or video file."); return null; } @@ -250,6 +252,9 @@ public sealed class DataSourceValidation public string? ValidateUserAcknowledgedCloudEmbedding(bool value) { + if(this.GetSelectedCloudTranscription() && !value) + return TB("Please acknowledge that you are aware of the cloud transcription implications."); + if(this.GetSelectedCloudEmbedding() && !value) return TB("Please acknowledge that you are aware of the cloud embedding implications."); diff --git a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md index 0be12e8fb..646c7559a 100644 --- a/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md +++ b/app/MindWork AI Studio/wwwroot/changelog/v26.10.1.md @@ -10,6 +10,7 @@ - Added Requesty as a new LLM provider for chats, embeddings, and transcriptions. Requesty is an AI gateway from the United Kingdom that gives you models from many providers with a single API key. Because where your data is processed depends on the model you choose, you set its trust level yourself. Thanks, Thibault Jaigu (`Thibaultjaigu`), for this first contribution. - Added [llmman](https://github.com/llmmanorg/llmman) as a new option for the models you host yourself. llmman runs open models on your own computer or server, and AI Studio can use them for chats, embeddings, and transcriptions. Thanks, Eric Curtin (`ericcurtin`), for this first contribution. - Added support for Mistral Large 4. +- Added transcription of audio and video files in local data sources. When you select such files, for example with the pattern `*.mp3` or `*`, AI Studio sends them to your transcription provider and adds the transcripts to the index. It keeps each transcript, so an unchanged recording is transcribed only once. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution. - Added backups of all your data. Under Data Backup & Restore, on the information page and in the settings, you save your chats, settings, plugins, and the index of your data sources into a single file, for example, to take everything to a new computer. Restoring such a backup replaces your data when AI Studio restarts, or you add only its chats to the ones you have. API keys and passwords never go into a backup, and after a restore, AI Studio tells you where to enter them again. Thanks, Jens Erler (`j-erler`), for this contribution. - Added organization-wide management for backups. IT departments can prevent users from backing up their data or from restoring a backup. A backup leaves out what the configurations of your organization bring along, such as their providers and plugins, so nothing appears twice after a restore. - Added an overview of the data folders of AI Studio to the information page, for when you back up your computer with a tool of your own. It shows where the folders are and which of their contents belong into a backup. AI Studio now stores the locations of its own files, such as transcripts, relative to its data folder, so such a copy also works on a computer with another user name. diff --git a/app/Tests/Provider/MediaFileRejectionTests.cs b/app/Tests/Provider/MediaFileRejectionTests.cs new file mode 100644 index 000000000..1f2b0459f --- /dev/null +++ b/app/Tests/Provider/MediaFileRejectionTests.cs @@ -0,0 +1,78 @@ +using System.Net; + +using AIStudio.Provider; + +namespace AIStudio.Tests.Provider; + +/// +/// Checks which answers of a transcription provider refuse the media file itself. +/// +/// +/// A data source records such a file for good, until it changes. An answer counted here by +/// mistake keeps a file out of the index; one missed here uploads the file again with every run. +/// +[TestFixture] +public sealed class MediaFileRejectionTests +{ + private const string OPENAI_DAMAGED_FILE = """ + { + "error": { + "message": "Audio file might be corrupted or unsupported", + "type": "invalid_request_error", + "param": "file", + "code": "invalid_value" + } + } + """; + + private const string OPENAI_UNKNOWN_MODEL = """ + { + "error": { + "message": "Invalid model.", + "type": "invalid_request_error", + "param": "model", + "code": "invalid_value" + } + } + """; + + private const string UNDECODABLE_FORMAT = """ + { + "error": { + "message": "Could not decode the audio file.", + "param": "file" + } + } + """; + + [Test] + public void AFileTheProviderCannotReadIsRefused() + { + Assert.That(BaseProvider.IsMediaFileRejection(HttpStatusCode.BadRequest, OPENAI_DAMAGED_FILE), Is.True); + } + + [TestCase(HttpStatusCode.RequestEntityTooLarge)] + [TestCase(HttpStatusCode.UnsupportedMediaType)] + public void AFileTooLargeOrOfAnUnacceptedTypeIsRefused(HttpStatusCode statusCode) + { + Assert.That(BaseProvider.IsMediaFileRejection(statusCode, string.Empty), Is.True); + } + + [Test] + public void AProviderWhichDecodesNoWebMRefusesNoSingleFile() + { + Assert.That(BaseProvider.IsMediaFileRejection(HttpStatusCode.BadRequest, UNDECODABLE_FORMAT), Is.False, "Every file fails the same way, and only another provider helps."); + } + + [TestCase(HttpStatusCode.BadRequest, OPENAI_UNKNOWN_MODEL)] + [TestCase(HttpStatusCode.BadRequest, "Bad request")] + [TestCase(HttpStatusCode.BadRequest, "")] + [TestCase(HttpStatusCode.TooManyRequests, OPENAI_DAMAGED_FILE)] + [TestCase(HttpStatusCode.Unauthorized, "")] + [TestCase(HttpStatusCode.InternalServerError, "")] + [TestCase(HttpStatusCode.ServiceUnavailable, "")] + public void AFailureWhichMayPassRefusesNoFile(HttpStatusCode statusCode, string responseBody) + { + Assert.That(BaseProvider.IsMediaFileRejection(statusCode, responseBody), Is.False); + } +} \ No newline at end of file diff --git a/app/Tests/Settings/TranscriptionConfidenceTests.cs b/app/Tests/Settings/TranscriptionConfidenceTests.cs new file mode 100644 index 000000000..18e5b4efc --- /dev/null +++ b/app/Tests/Settings/TranscriptionConfidenceTests.cs @@ -0,0 +1,74 @@ +using AIStudio.Provider; +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools; +using AIStudio.Tools.Services; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Settings; + +/// +/// Checks which transcription providers may hear the audio and video files of a data source. +/// +/// +/// The transcription provider receives the content of a data source just like the embedding +/// provider. A data source which asks for a high confidence level must not send its recordings to a +/// provider it would never let embed its documents. +/// +[TestFixture] +public sealed class TranscriptionConfidenceTests +{ + private static IEnumerable DataSources() + { + foreach (var level in Enum.GetValues()) + { + yield return new DataSourceLocalDirectory { ConfidenceLevel = level }; + yield return new DataSourceLocalFile { ConfidenceLevel = level }; + yield return new DataSourceMailbox { ConfidenceLevel = level }; + } + } + + [TestCaseSource(nameof(DataSources))] + public void TranscriptionFollowsTheRuleOfTheEmbedding(IIndexedDataSource dataSource) + { + foreach (var providerLevel in Enum.GetValues()) + Assert.That(DataSourceEmbeddingService.AllowsTranscription(dataSource, providerLevel), Is.EqualTo(DataSourceEmbeddingService.AllowsEmbedding(dataSource, providerLevel)), $"A provider with the level {providerLevel} is treated differently for {dataSource.GetType().Name} with the level {dataSource.ConfidenceLevel}."); + } + + [Test] + public void AFolderWhichAsksForAHighLevelKeepsItsRecordingsFromAMediumProvider() + { + var directory = new DataSourceLocalDirectory { ConfidenceLevel = ConfidenceLevel.HIGH }; + Assert.Multiple(() => + { + Assert.That(DataSourceEmbeddingService.AllowsTranscription(directory, ConfidenceLevel.MEDIUM), Is.False); + Assert.That(DataSourceEmbeddingService.AllowsTranscription(directory, ConfidenceLevel.HIGH), Is.True); + }); + } + + [Test] + public void AFolderWithoutALevelStaysOpenToEveryTranscriptionProvider() + { + var directory = new DataSourceLocalDirectory { ConfidenceLevel = ConfidenceLevel.NONE }; + Assert.That(DataSourceEmbeddingService.AllowsTranscription(directory, ConfidenceLevel.UNTRUSTED), Is.True); + } + + [Test] + public void ATranscriptionProviderHasTheLevelOfItsCompany() + { + var settingsManager = new SettingsManager(NullLogger.Instance, null!); + settingsManager.ConfigurationData.Confidence.ConfidenceScheme = ConfidenceSchemes.CUSTOM; + settingsManager.ConfigurationData.Confidence.CustomConfidenceScheme[LLMProviders.OPEN_AI] = ConfidenceLevel.LOW; + + var provider = new TranscriptionProvider { Id = Guid.NewGuid().ToString(), UsedLLMProvider = LLMProviders.OPEN_AI }; + Assert.That(provider.GetConfidenceLevel(settingsManager), Is.EqualTo(ConfidenceLevel.LOW)); + } + + [Test] + public void NoTranscriptionProviderHasNoConfidence() + { + var settingsManager = new SettingsManager(NullLogger.Instance, null!); + Assert.That(TranscriptionProvider.NONE.GetConfidenceLevel(settingsManager), Is.EqualTo(ConfidenceLevel.NONE)); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/DataBackup/DataBackupWriterTests.cs b/app/Tests/Tools/DataBackup/DataBackupWriterTests.cs index 2a7c157f5..9e63970e0 100644 --- a/app/Tests/Tools/DataBackup/DataBackupWriterTests.cs +++ b/app/Tests/Tools/DataBackup/DataBackupWriterTests.cs @@ -25,6 +25,8 @@ public sealed class DataBackupWriterTests private static readonly Guid CHAT_ID = Guid.Parse("8a1c4e7f-2b5d-4f9a-b3c6-0e2d4f6a8b1c"); + private const string MEDIA_TRANSCRIPT = "mediaTranscripts/5c9e2a7b-1d3f-4b6e-8a0c-2e4f6b8d0a1c/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef.md"; + private string testDirectory = string.Empty; private string DataDirectory => Path.Combine(this.testDirectory, "data"); @@ -53,10 +55,12 @@ public sealed class DataBackupWriterTests this.WriteDataFile("media-staging/operation/transcript.md", "temporary"); this.WriteDataFile("databases/sqlite/rag-index.sqlite3", "the index while AI Studio writes to it"); this.WriteDataFile("databases/vector_database/stores/store_1/segments/segment", "the store while AI Studio writes to it"); + this.WriteDataFile(MEDIA_TRANSCRIPT, "the transcript while AI Studio writes it"); WriteFile(Path.Combine(this.ConfigDirectory, "settings.json"), "the settings on the disk, with the objects of the organization"); WriteFile(Path.Combine(this.IndexCopiesDirectory, "databases", "sqlite", "rag-index.sqlite3"), "the copy of the index"); WriteFile(Path.Combine(this.IndexCopiesDirectory, "databases", "vector_database", "stores", "store_1", "segments", "segment"), "the copy of the store"); + WriteFile(Path.Combine(this.IndexCopiesDirectory, MEDIA_TRANSCRIPT), "the copy of the transcript"); } [TearDown] @@ -84,6 +88,7 @@ public sealed class DataBackupWriterTests "data/visualBriefings/briefing/manifest.json", "data/databases/sqlite/rag-index.sqlite3", "data/databases/vector_database/stores/store_1/segments/segment", + $"data/{MEDIA_TRANSCRIPT}", DataBackupFormat.MANIFEST_ENTRY_NAME, })); @@ -92,6 +97,7 @@ public sealed class DataBackupWriterTests Assert.That(archive["config/settings.json"], Is.EqualTo(SETTINGS_JSON), "The settings come from memory, without the objects of configuration plugins."); Assert.That(archive["data/databases/sqlite/rag-index.sqlite3"], Is.EqualTo("the copy of the index")); Assert.That(archive["data/databases/vector_database/stores/store_1/segments/segment"], Is.EqualTo("the copy of the store")); + Assert.That(archive[$"data/{MEDIA_TRANSCRIPT}"], Is.EqualTo("the copy of the transcript"), "The transcripts are copied together with the index they belong to."); }); } @@ -101,6 +107,7 @@ public sealed class DataBackupWriterTests var archive = await this.WriteBackupAsync(string.Empty); Assert.That(archive.Keys.Where(name => name.StartsWith("data/databases/", StringComparison.Ordinal)), Is.Empty, "The indexes in the data folder change while AI Studio runs."); + Assert.That(archive.Keys.Where(name => name.StartsWith("data/mediaTranscripts/", StringComparison.Ordinal)), Is.Empty, "The transcripts belong to the indexes and come only together with them."); } [Test] diff --git a/app/Tests/Tools/DataSourceEmbeddingHealthTests.cs b/app/Tests/Tools/DataSourceEmbeddingHealthTests.cs index b40a33068..1cef4ed63 100644 --- a/app/Tests/Tools/DataSourceEmbeddingHealthTests.cs +++ b/app/Tests/Tools/DataSourceEmbeddingHealthTests.cs @@ -25,14 +25,33 @@ public sealed class DataSourceEmbeddingHealthTests Assert.That(status.Health, Is.EqualTo(DataSourceEmbeddingHealth.WARNING)); } - [Test] - public void AnIgnoredMediaFileKeepsAUsableDataSourceHealthy() + [TestCase(DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING)] + [TestCase(DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_NOT_ALLOWED)] + public void ATranscriptionProviderWarningRemainsActionableWhenOtherFilesWereIndexed(DataSourceEmbeddingIgnoreReason reason) { var status = Status(indexedDocuments: 1, totalDocuments: 2) with { IgnoredFiles = [ - new DataSourceEmbeddingIgnoredFile("/data/interview.mp3", DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED), + new DataSourceEmbeddingIgnoredFile("/data/interview.mp3", reason), + ], + }; + + Assert.Multiple(() => + { + Assert.That(status.Health, Is.EqualTo(DataSourceEmbeddingHealth.WARNING)); + Assert.That(status.NeedsTranscriptionProvider, Is.True, "The embedding page offers the way to the settings only then."); + }); + } + + [Test] + public void AnIgnoredImageKeepsAUsableDataSourceHealthy() + { + var status = Status(indexedDocuments: 1, totalDocuments: 2) with + { + IgnoredFiles = + [ + new DataSourceEmbeddingIgnoredFile("/data/photo.png", DataSourceEmbeddingIgnoreReason.IMAGE_NOT_SUPPORTED), ], }; diff --git a/app/Tests/Tools/Databases/IndexStoreManifestTests.cs b/app/Tests/Tools/Databases/IndexStoreManifestTests.cs index 5bbf1c305..a801bf47e 100644 --- a/app/Tests/Tools/Databases/IndexStoreManifestTests.cs +++ b/app/Tests/Tools/Databases/IndexStoreManifestTests.cs @@ -40,6 +40,27 @@ public sealed class IndexStoreManifestTests }); } + [Test] + public async Task TheManifestKnowsWhichTranscriptBelongsToAMediaFile() + { + await using var store = await TemporaryIndexStore.CreateAsync(); + await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "LOCAL_DIRECTORY", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", string.Empty, 3, TOKEN); + + var recording = Path.Combine(Path.GetTempPath(), "test-data", "meeting.mp3"); + var document = Path.Combine(Path.GetTempPath(), "test-data", "report.pdf"); + var transcriptKey = new string('f', 64); + await store.Client.UpsertFileAsync(DATA_SOURCE_ID, Row(recording, 2) with { TranscriptKey = transcriptKey }, TOKEN); + await store.Client.UpsertFileAsync(DATA_SOURCE_ID, Row(document, 4), TOKEN); + + var manifest = await store.Client.GetManifestAsync(DATA_SOURCE_ID, TOKEN); + + Assert.Multiple(() => + { + Assert.That(manifest.Files[recording].TranscriptKey, Is.EqualTo(transcriptKey), "Without the key, AI Studio cannot remove the transcript once the recording is gone."); + Assert.That(manifest.Files[document].TranscriptKey, Is.Null); + }); + } + private static EmbeddingStateFile Row(string filePath, int chunkCount) { var now = DateTimeOffset.UtcNow; diff --git a/app/Tests/Tools/DocumentRunProgressTests.cs b/app/Tests/Tools/DocumentRunProgressTests.cs index 1d553db8c..c81e36e66 100644 --- a/app/Tests/Tools/DocumentRunProgressTests.cs +++ b/app/Tests/Tools/DocumentRunProgressTests.cs @@ -64,6 +64,27 @@ public sealed class DocumentRunProgressTests }); } + [Test] + public async Task APermanentMediaFailureKeepsItsExplanation() + { + var (progress, statuses, manifest) = CreateProgress(totalDocuments: 1); + var document = Document(Path.Combine(Path.GetTempPath(), "test-data", "recording.webm")); + + await progress.RecordDocumentFailureAsync( + document, + new FileExtractionException(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED, "The media file is damaged or its format could not be identified."), + CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(statuses[^1].Failures, Has.Count.EqualTo(1)); + Assert.That(statuses[^1].Failures[0].ExtractionCode, Is.EqualTo(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED)); + Assert.That(statuses[^1].Failures[0].Reason, Does.Contain("The media file is damaged or its format could not be identified.")); + Assert.That(manifest.PermanentFailures[document.State.AbsolutePath].Code, Is.EqualTo(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED)); + Assert.That(manifest.PermanentFailures[document.State.AbsolutePath].Message, Is.EqualTo(statuses[^1].Failures[0].Reason)); + }); + } + [Test] public async Task ARunWhichOnlySkippedDocumentsForGoodIsCompleted() { @@ -379,8 +400,8 @@ public sealed class DocumentRunProgressTests var testDataPath = Path.Combine(Path.GetTempPath(), "test-data"); var ignoredFiles = new[] { - new DataSourceEmbeddingIgnoredFile(Path.Combine(testDataPath, "recording.mp3"), DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED), - new DataSourceEmbeddingIgnoredFile(Path.Combine(testDataPath, "diagram.png"), DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED), + new DataSourceEmbeddingIgnoredFile(Path.Combine(testDataPath, "recording.mp3"), DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING), + new DataSourceEmbeddingIgnoredFile(Path.Combine(testDataPath, "diagram.png"), DataSourceEmbeddingIgnoreReason.IMAGE_NOT_SUPPORTED), }; var filePatternExclusions = new[] { new DataSourceEmbeddingFilePatternGroup(".pdf", 42) }; var excludedFolders = new[] { new DataSourceEmbeddingExcludedFolderGroup("node_modules", 3) }; @@ -432,6 +453,15 @@ public sealed class DocumentRunProgressTests Assert.That(statuses[^1].Failures[0].DisplayName, Is.EqualTo("report.pdf")); } + [TestCase("Audio file might be corrupted or unsupported", "Audio file might be corrupted or unsupported.")] + [TestCase("The container is damaged. ", "The container is damaged.")] + [TestCase("Is it damaged?", "Is it damaged?")] + [TestCase("", "")] + public void AReasonEndsAsASentence(string reason, string sentence) + { + Assert.That(DocumentRunProgress.AsSentence(reason), Is.EqualTo(sentence)); + } + private static IndexingPass NewPass() => new(new DataSourceRunSignature("TEST-EMBEDDING-SIGNATURE", string.Empty)); /// diff --git a/app/Tests/Tools/FileSizeLimitTests.cs b/app/Tests/Tools/FileSizeLimitTests.cs index 25425fa54..33c6cac92 100644 --- a/app/Tests/Tools/FileSizeLimitTests.cs +++ b/app/Tests/Tools/FileSizeLimitTests.cs @@ -140,5 +140,5 @@ public sealed class FileSizeLimitTests } // Finding the files reads none of them, so it needs neither the runtime nor the chunker: - private static FileSourceIndexer CreateIndexer() => new(null!, null!, null!, NullLogger.Instance); + private static FileSourceIndexer CreateIndexer() => new(null!, null!, null!, null!, NullLogger.Instance); } \ No newline at end of file diff --git a/app/Tests/Tools/FileSourceIndexerWatcherTests.cs b/app/Tests/Tools/FileSourceIndexerWatcherTests.cs index 7307f0cd9..2ab1f86bb 100644 --- a/app/Tests/Tools/FileSourceIndexerWatcherTests.cs +++ b/app/Tests/Tools/FileSourceIndexerWatcherTests.cs @@ -1,4 +1,5 @@ using AIStudio.Tools; +using AIStudio.Tools.Media; using AIStudio.Tools.Services.Indexing; namespace AIStudio.Tests.Tools; @@ -261,6 +262,78 @@ public sealed class FileSourceIndexerWatcherTests Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.False); } + [Test] + public async Task DirectoryWatcherIgnoresTheBatchTranscriptWhenTheMediaIsSelected() + { + var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-transcript-watcher-{Guid.NewGuid():N}"); + var mediaPath = Path.Combine(directoryPath, "recording.mp3"); + Directory.CreateDirectory(directoryPath); + + try + { + await File.WriteAllBytesAsync(mediaPath, [1, 2, 3]); + var transcriptPath = MediaSidecarTranscripts.GetFilePath(mediaPath); + var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration( + directoryPath, + true, + FilePatterns: FileNamePatterns.Parse("*")); + var unrelatedPatterns = configuration with { FilePatterns = FileNamePatterns.Parse("*.tf") }; + + Assert.Multiple(() => + { + Assert.That(FileSourceIndexer.IsRelevantWatcherPath(configuration, mediaPath, WatcherChangeTypes.Changed), Is.True); + Assert.That(FileSourceIndexer.IsRelevantWatcherPath(configuration, transcriptPath, WatcherChangeTypes.Created), Is.False); + Assert.That(FileSourceIndexer.IsRelevantWatcherPath(unrelatedPatterns, transcriptPath, WatcherChangeTypes.Created), Is.False); + }); + } + finally + { + Directory.Delete(directoryPath, true); + } + } + + [Test] + public async Task DirectoryWatcherKeepsTranscriptAsADocumentWhenTheMediaIsNotSelected() + { + var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-transcript-document-{Guid.NewGuid():N}"); + var mediaPath = Path.Combine(directoryPath, "recording.mp3"); + Directory.CreateDirectory(directoryPath); + + try + { + await File.WriteAllBytesAsync(mediaPath, [1, 2, 3]); + var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration( + directoryPath, + true, + FilePatterns: FileNamePatterns.Parse("*.md")); + + Assert.That( + FileSourceIndexer.IsRelevantWatcherPath(configuration, MediaSidecarTranscripts.GetFilePath(mediaPath), WatcherChangeTypes.Changed), + Is.True); + } + finally + { + Directory.Delete(directoryPath, true); + } + } + + [TestCase("recording.mp3")] + [TestCase("meeting.mp4")] + [TestCase("voice.webm")] + public void MediaStartsARunOnlyWhenAPatternSelectsIt(string fileName) + { + var rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-media-watcher-{Guid.NewGuid():N}"); + var path = Path.Combine(rootPath, fileName); + var documents = DefaultConfiguration(rootPath); + var everything = documents with { FilePatterns = FileNamePatterns.Parse("*") }; + + Assert.Multiple(() => + { + Assert.That(FileSourceIndexer.IsRelevantWatcherPath(documents, path, WatcherChangeTypes.Created), Is.False, "The documents selector never selects audio or video."); + Assert.That(FileSourceIndexer.IsRelevantWatcherPath(everything, path, WatcherChangeTypes.Created), Is.True); + }); + } + private static FileSourceIndexer.DataSourceWatcherConfiguration DefaultConfiguration(string rootPath) => new( rootPath, true, diff --git a/app/Tests/Tools/FileTypesTests.cs b/app/Tests/Tools/FileTypesTests.cs index 954562bb0..81310044c 100644 --- a/app/Tests/Tools/FileTypesTests.cs +++ b/app/Tests/Tools/FileTypesTests.cs @@ -190,10 +190,10 @@ public class FileTypesTests [TestCase("recording.mp3")] [TestCase("meeting.mp4")] [TestCase("voice-recording.webm")] - public void LocalFileDataSourcesRejectMedia(string fileName) + public void LocalFileDataSourcesAcceptTranscribableMedia(string fileName) { var validation = new DataSourceValidation(); - WithFile([1, 2, 3], fileName, path => Assert.That(validation.ValidateFilePath(path), Is.Not.Null)); + WithFile([1, 2, 3], fileName, path => Assert.That(validation.ValidateFilePath(path), Is.Null)); } private static void AssertPath(string content, string fileName, Encoding encoding, bool expected, FileTypeFilter? type = null) diff --git a/app/Tests/Tools/Media/DataSourceTranscriptStoreTests.cs b/app/Tests/Tools/Media/DataSourceTranscriptStoreTests.cs new file mode 100644 index 000000000..b53298bd6 --- /dev/null +++ b/app/Tests/Tools/Media/DataSourceTranscriptStoreTests.cs @@ -0,0 +1,233 @@ +using AIStudio.Settings; +using AIStudio.Tools.Media; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools.Media; + +/// +/// Checks where data sources keep the transcripts of their audio and video files. +/// +[TestFixture] +public sealed class DataSourceTranscriptStoreTests +{ + private static readonly string DATA_SOURCE_ID = Guid.NewGuid().ToString(); + + private string rootPath = string.Empty; + + private string dataDirectory = string.Empty; + + private string userFolder = string.Empty; + + [SetUp] + public void CreateFolders() + { + this.rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-transcript-store-{Guid.NewGuid():N}"); + this.dataDirectory = Path.Combine(this.rootPath, "data"); + this.userFolder = Path.Combine(this.rootPath, "user"); + Directory.CreateDirectory(this.dataDirectory); + Directory.CreateDirectory(this.userFolder); + } + + [TearDown] + public void DeleteFolders() + { + if (Directory.Exists(this.rootPath)) + Directory.Delete(this.rootPath, true); + } + + [Test] + public async Task TheTranscriptLiesBelowTheDataDirectoryOfItsDataSource() + { + var mediaPath = await this.CreateMediaAsync("recording.mp3", [1, 2, 3]); + var slot = await this.Store().ResolveAsync(mediaPath, CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(DataSourceTranscriptStore.IsKey(slot.Key), Is.True); + Assert.That(slot.Path, Is.EqualTo(Path.Join(this.dataDirectory, DataSourceTranscriptStore.FOLDER_NAME, DATA_SOURCE_ID, slot.Key + ".md"))); + }); + } + + [Test] + public async Task TheSameContentFindsTheSameTranscriptWhereverTheFileLies() + { + var original = await this.CreateMediaAsync("recording.mp3", [1, 2, 3]); + var copy = await this.CreateMediaAsync(Path.Combine("moved", "renamed.mp3"), [1, 2, 3]); + File.SetLastWriteTimeUtc(copy, DateTime.UtcNow.AddDays(-3)); + var other = await this.CreateMediaAsync("other.mp3", [1, 2, 3, 4]); + + var store = this.Store(); + var originalSlot = await store.ResolveAsync(original, CancellationToken.None); + var copySlot = await store.ResolveAsync(copy, CancellationToken.None); + var otherSlot = await store.ResolveAsync(other, CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(copySlot, Is.EqualTo(originalSlot)); + Assert.That(otherSlot.Key, Is.Not.EqualTo(originalSlot.Key)); + }); + } + + [Test] + public async Task ARenamedRecordingIsPaidForOnlyOnce() + { + var mediaPath = await this.CreateMediaAsync("recording.mp3", [1, 2, 3]); + var provider = new TranscriptionProvider { Id = "test-transcription-provider" }; + var calls = 0; + Task Transcribe(string path, TranscriptionProvider _, CancellationToken __) + { + calls++; + return Task.FromResult(MediaTranscriptionResult.Succeeded("Transcript")); + } + + var first = await PersistentMediaTranscript.ReadOrCreateAsync(this.Store(), mediaPath, provider, Transcribe, NullLogger.Instance, CancellationToken.None); + + var renamedPath = Path.Combine(this.userFolder, "renamed.mp3"); + File.Move(mediaPath, renamedPath); + File.SetLastWriteTimeUtc(renamedPath, DateTime.UtcNow.AddMinutes(5)); + var second = await PersistentMediaTranscript.ReadOrCreateAsync(this.Store(), renamedPath, provider, Transcribe, NullLogger.Instance, CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(first.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY)); + Assert.That(second.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY)); + Assert.That(second.Reused, Is.True); + Assert.That(calls, Is.EqualTo(1)); + Assert.That(Directory.GetFiles(this.userFolder), Has.Length.EqualTo(1), "The data source wrote into the folder of the user."); + }); + } + + [Test] + public async Task OnlyTranscriptsAreListedAndDeleted() + { + var store = this.Store(); + var key = new string('a', 64); + var transcriptPath = store.GetPath(key); + var folder = Path.GetDirectoryName(transcriptPath)!; + Directory.CreateDirectory(folder); + await File.WriteAllTextAsync(transcriptPath, "Transcript"); + await File.WriteAllTextAsync($"{transcriptPath}.{Guid.NewGuid():N}.tmp", "Interrupted"); + await File.WriteAllTextAsync(Path.Combine(folder, "notes.md"), "Not ours"); + + Assert.That(store.ListKeys(), Is.EqualTo(new[] { key })); + + store.DeleteAll(); + + Assert.Multiple(() => + { + Assert.That(File.Exists(transcriptPath), Is.False); + Assert.That(Directory.GetFiles(folder, "*.tmp"), Is.Empty); + Assert.That(File.Exists(Path.Combine(folder, "notes.md")), Is.True, "A file which the store did not write was deleted."); + }); + } + + [Test] + public async Task DeletingAllTranscriptsRemovesTheFolderOfTheDataSource() + { + var store = this.Store(); + var transcriptPath = store.GetPath(new string('b', 64)); + Directory.CreateDirectory(Path.GetDirectoryName(transcriptPath)!); + await File.WriteAllTextAsync(transcriptPath, "Transcript"); + + store.DeleteAll(); + + Assert.That(Directory.Exists(Path.GetDirectoryName(transcriptPath)), Is.False); + } + + [Test] + public void DeletingADataSourceWithoutTranscriptsWorks() + { + Assert.DoesNotThrow(() => this.Store().DeleteAll()); + } + + [Test] + public async Task ALinkInTheFolderIsNeitherFollowedNorDeleted() + { + var store = this.Store(); + var key = new string('c', 64); + var linkPath = store.GetPath(key); + Directory.CreateDirectory(Path.GetDirectoryName(linkPath)!); + var targetPath = Path.Combine(this.userFolder, "precious.md"); + await File.WriteAllTextAsync(targetPath, "Precious"); + + try + { + File.CreateSymbolicLink(linkPath, targetPath); + } + catch (Exception exception) when (exception is UnauthorizedAccessException or IOException) + { + Assert.Ignore("Creating symbolic links needs a privilege this system does not grant."); + } + + var deleted = store.Delete(key); + store.DeleteAll(); + + Assert.Multiple(() => + { + Assert.That(deleted, Is.False); + Assert.That(File.ReadAllText(targetPath), Is.EqualTo("Precious")); + }); + } + + [Test] + public async Task ABackupCopiesOnlyTheTranscriptsOfAllDataSources() + { + var secondId = Guid.NewGuid().ToString(); + var firstPath = this.Store().GetPath(new string('d', 64)); + var secondPath = new DataSourceTranscriptStore(this.dataDirectory, secondId).GetPath(new string('e', 64)); + foreach (var path in new[] { firstPath, secondPath }) + { + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + await File.WriteAllTextAsync(path, "Transcript"); + } + + await File.WriteAllTextAsync($"{firstPath}.{Guid.NewGuid():N}.tmp", "Interrupted"); + var targetDirectory = Path.Combine(this.rootPath, "copies"); + + var copied = await DataSourceTranscriptStore.CopyAllAsync(this.dataDirectory, targetDirectory, CancellationToken.None); + + var copiedFiles = Directory.GetFiles(targetDirectory, "*", SearchOption.AllDirectories).Select(path => Path.GetRelativePath(targetDirectory, path).Replace('\\', '/')); + Assert.Multiple(() => + { + Assert.That(copied, Is.EqualTo(2)); + Assert.That(copiedFiles, Is.EquivalentTo(new[] + { + $"{DataSourceTranscriptStore.FOLDER_NAME}/{DATA_SOURCE_ID}/{new string('d', 64)}.md", + $"{DataSourceTranscriptStore.FOLDER_NAME}/{secondId}/{new string('e', 64)}.md", + })); + }); + } + + [Test] + public async Task ABackupWithoutTranscriptsCopiesNothing() + { + Assert.That(await DataSourceTranscriptStore.CopyAllAsync(this.dataDirectory, Path.Combine(this.rootPath, "copies"), CancellationToken.None), Is.Zero); + } + + [TestCase("")] + [TestCase("ABCDEF")] + [TestCase("../outside")] + public void ADataSourceIdWhichIsNoGuidIsRejected(string dataSourceId) + { + Assert.Throws(() => _ = new DataSourceTranscriptStore(this.dataDirectory, dataSourceId)); + } + + [TestCase("")] + [TestCase("../../settings")] + [TestCase("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")] + public void AKeyWhichIsNoHashIsRejected(string key) + { + Assert.Throws(() => this.Store().GetPath(key)); + } + + private DataSourceTranscriptStore Store() => new(this.dataDirectory, DATA_SOURCE_ID); + + private async Task CreateMediaAsync(string relativePath, byte[] content) + { + var path = Path.Combine(this.userFolder, relativePath); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + await File.WriteAllBytesAsync(path, content); + return path; + } +} \ No newline at end of file diff --git a/app/Tests/Tools/Media/MediaSidecarTranscriptsTests.cs b/app/Tests/Tools/Media/MediaSidecarTranscriptsTests.cs new file mode 100644 index 000000000..fd4a6ebf5 --- /dev/null +++ b/app/Tests/Tools/Media/MediaSidecarTranscriptsTests.cs @@ -0,0 +1,57 @@ +using AIStudio.Tools.Media; + +namespace AIStudio.Tests.Tools.Media; + +/// +/// Checks the transcripts which the Batch Processing assistant keeps next to its media files. +/// +[TestFixture] +public sealed class MediaSidecarTranscriptsTests +{ + [TestCase("recording.mp3.transcript.md")] + [TestCase("recording.mp3.transcript.md.0123456789abcdef.tmp")] + [TestCase("RECORDING.MP3.TRANSCRIPT.MD")] + public void GeneratedTranscriptFilesAreRecognized(string fileName) + { + Assert.That(MediaSidecarTranscripts.IsArtifactPath(fileName), Is.True); + } + + [TestCase("notes.md")] + [TestCase("recording.mp3")] + [TestCase("recording.mp3.transcript.md.backup")] + public void OtherFilesAreNoTranscripts(string fileName) + { + Assert.That(MediaSidecarTranscripts.IsArtifactPath(fileName), Is.False); + } + + [Test] + public async Task TheTranscriptLiesNextToItsMediaFile() + { + var mediaPath = Path.Combine(Path.GetTempPath(), "recording.mp3"); + var slot = await MediaSidecarTranscripts.INSTANCE.ResolveAsync(mediaPath, CancellationToken.None); + Assert.That(slot.Path, Is.EqualTo(mediaPath + MediaSidecarTranscripts.FILE_SUFFIX)); + } + + [Test] + public async Task OnlyATranscriptWithContentCanBeReused() + { + var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-sidecar-transcript-{Guid.NewGuid():N}"); + var mediaPath = Path.Combine(directoryPath, "recording.mp3"); + Directory.CreateDirectory(directoryPath); + + try + { + Assert.That(MediaSidecarTranscripts.HasReusableTranscript(mediaPath), Is.False, "A missing transcript counted as reusable."); + + await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(mediaPath), string.Empty); + Assert.That(MediaSidecarTranscripts.HasReusableTranscript(mediaPath), Is.False, "An empty transcript counted as reusable."); + + await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(mediaPath), "Transcript"); + Assert.That(MediaSidecarTranscripts.HasReusableTranscript(mediaPath), Is.True); + } + finally + { + Directory.Delete(directoryPath, true); + } + } +} \ No newline at end of file diff --git a/app/Tests/Tools/Media/PersistentMediaTranscriptTests.cs b/app/Tests/Tools/Media/PersistentMediaTranscriptTests.cs new file mode 100644 index 000000000..8d95cda91 --- /dev/null +++ b/app/Tests/Tools/Media/PersistentMediaTranscriptTests.cs @@ -0,0 +1,252 @@ +using AIStudio.Settings; +using AIStudio.Tools.Media; +using AIStudio.Tools.Rust; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools.Media; + +/// +/// Checks the rules which keep a paid transcription from being paid twice. +/// +[TestFixture] +public sealed class PersistentMediaTranscriptTests +{ + private TranscriptionProvider? provider; + + private string directoryPath = string.Empty; + + private string mediaPath = string.Empty; + + [SetUp] + public async Task CreateMediaFile() + { + this.directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-persistent-transcript-{Guid.NewGuid():N}"); + Directory.CreateDirectory(this.directoryPath); + this.mediaPath = Path.Combine(this.directoryPath, "recording.mp3"); + this.provider = new TranscriptionProvider { Id = "test-transcription-provider" }; + await File.WriteAllBytesAsync(this.mediaPath, [1, 2, 3]); + } + + [TearDown] + public void DeleteMediaFile() + { + if (Directory.Exists(this.directoryPath)) + Directory.Delete(this.directoryPath, true); + } + + [Test] + public async Task AnExistingTranscriptIsUsedWithoutAskingTheProvider() + { + await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(this.mediaPath), "Transcript"); + var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("New transcript")); + + var result = await this.ReadOrCreateAsync(transcriber, this.provider); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY)); + Assert.That(result.Reused, Is.True); + Assert.That(transcriber.Calls, Is.Zero); + }); + } + + [Test] + public async Task AnExistingTranscriptIsUsedEvenWithoutAProvider() + { + await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(this.mediaPath), "Transcript"); + var result = await this.ReadOrCreateAsync(new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("New transcript")), null); + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY)); + } + + [Test] + public async Task WithoutATranscriptAndAProviderNothingIsTranscribed() + { + var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Transcript")); + + var result = await this.ReadOrCreateAsync(transcriber, null); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.NO_PROVIDER)); + Assert.That(transcriber.Calls, Is.Zero); + }); + } + + [Test] + public async Task ANewTranscriptIsStoredAndPaidForOnlyOnce() + { + var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Transcript")); + + var first = await this.ReadOrCreateAsync(transcriber, this.provider); + var second = await this.ReadOrCreateAsync(transcriber, this.provider); + + Assert.Multiple(() => + { + Assert.That(first.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY)); + Assert.That(first.Reused, Is.False); + Assert.That(File.ReadAllText(first.TranscriptPath), Is.EqualTo("Transcript")); + Assert.That(second.Reused, Is.True); + Assert.That(transcriber.Calls, Is.EqualTo(1)); + Assert.That(Directory.GetFiles(this.directoryPath, "*.tmp"), Is.Empty); + }); + } + + [Test] + public async Task AnEmptyTranscriptIsNeverStored() + { + var result = await this.ReadOrCreateAsync(new CountingTranscriber(() => MediaTranscriptionResult.Succeeded(" \n ")), this.provider); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.EMPTY)); + Assert.That(File.Exists(MediaSidecarTranscripts.GetFilePath(this.mediaPath)), Is.False); + }); + } + + [Test] + public async Task ATranscriptHoldingOnlyWhiteSpaceIsReplaced() + { + await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(this.mediaPath), " \n "); + var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Transcript")); + + var result = await this.ReadOrCreateAsync(transcriber, this.provider); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY)); + Assert.That(transcriber.Calls, Is.EqualTo(1)); + Assert.That(File.ReadAllText(result.TranscriptPath), Is.EqualTo("Transcript")); + }); + } + + [Test] + public async Task AFileWhichChangesDuringItsTranscriptionKeepsNoTranscript() + { + var transcriber = new CountingTranscriber(() => + { + File.WriteAllBytes(this.mediaPath, [1, 2, 3, 4]); + return MediaTranscriptionResult.Succeeded("Transcript of the old file"); + }); + + var result = await this.ReadOrCreateAsync(transcriber, this.provider); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.SOURCE_CHANGED)); + Assert.That(File.Exists(MediaSidecarTranscripts.GetFilePath(this.mediaPath)), Is.False); + }); + } + + [Test] + public async Task ATranscriptWhichArrivesAfterACancellationIsStillStored() + { + using var cancellation = new CancellationTokenSource(); + var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Paid transcript"), cancellation.Cancel); + + var result = await PersistentMediaTranscript.ReadOrCreateAsync(MediaSidecarTranscripts.INSTANCE, this.mediaPath, this.provider, transcriber.TranscribeAsync, NullLogger.Instance, cancellation.Token); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY)); + Assert.That(File.ReadAllText(MediaSidecarTranscripts.GetFilePath(this.mediaPath)), Is.EqualTo("Paid transcript")); + }); + } + + [Test] + public async Task ACancelledTranscriptionStoresNothing() + { + var result = await this.ReadOrCreateAsync(new CountingTranscriber(MediaTranscriptionResult.Cancelled), this.provider); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.CANCELLED)); + Assert.That(File.Exists(MediaSidecarTranscripts.GetFilePath(this.mediaPath)), Is.False); + }); + } + + [Test] + public async Task AFailedTranscriptionKeepsTheReasonOfTheMediaPipeline() + { + var result = await this.ReadOrCreateAsync(new CountingTranscriber(() => MediaTranscriptionResult.Failed("The container is damaged.", MediaJobErrorCode.DAMAGED_CONTAINER)), this.provider); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.TRANSCRIPTION_FAILED)); + Assert.That(result.ErrorCode, Is.EqualTo(MediaJobErrorCode.DAMAGED_CONTAINER)); + Assert.That(result.UserMessage, Is.EqualTo("The container is damaged.")); + }); + } + + [Test] + public async Task AFileTheProviderRefusedIsReportedAsSuch() + { + var result = await this.ReadOrCreateAsync(new CountingTranscriber(() => MediaTranscriptionResult.Rejected("The file is too large.")), this.provider); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.REJECTED)); + Assert.That(result.UserMessage, Is.EqualTo("The file is too large.")); + Assert.That(File.Exists(MediaSidecarTranscripts.GetFilePath(this.mediaPath)), Is.False); + }); + } + + [Test] + public async Task ATranscriptWhichCannotBeStoredIsReportedAndLeavesNoTemporaryFile() + { + // + // A file where the folder of the transcript should be: writing fails for every user, root + // included, unlike a folder without write permission. + // + var blockingFilePath = Path.Combine(this.directoryPath, "not-a-folder"); + await File.WriteAllTextAsync(blockingFilePath, "occupied"); + var location = new FixedLocation(Path.Combine(blockingFilePath, "transcript.md")); + + var result = await PersistentMediaTranscript.ReadOrCreateAsync(location, this.mediaPath, this.provider, new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Transcript")).TranscribeAsync, NullLogger.Instance, CancellationToken.None); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.STORE_FAILED)); + Assert.That(Directory.GetFiles(this.directoryPath, "*.tmp", SearchOption.AllDirectories), Is.Empty); + }); + } + + [Test] + public async Task AMissingMediaFileIsReportedWithoutAskingTheProvider() + { + File.Delete(this.mediaPath); + var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Transcript")); + + var result = await this.ReadOrCreateAsync(transcriber, this.provider); + + Assert.Multiple(() => + { + Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.SOURCE_UNREADABLE)); + Assert.That(transcriber.Calls, Is.Zero); + }); + } + + private Task ReadOrCreateAsync(CountingTranscriber transcriber, TranscriptionProvider? transcriptionProvider) => PersistentMediaTranscript.ReadOrCreateAsync(MediaSidecarTranscripts.INSTANCE, this.mediaPath, transcriptionProvider, transcriber.TranscribeAsync, NullLogger.Instance, CancellationToken.None); + + /// + /// Stands in for the provider and counts how often it was asked. + /// + /// Creates the answer of the provider. + /// Runs while the provider is answering, e.g., to cancel the run in the middle of an upload. + private sealed class CountingTranscriber(Func answer, Action? whileAnswering = null) + { + public int Calls { get; private set; } + + public Task TranscribeAsync(string mediaPath, TranscriptionProvider provider, CancellationToken token) + { + this.Calls++; + whileAnswering?.Invoke(); + return Task.FromResult(answer()); + } + } + + private sealed class FixedLocation(string transcriptPath) : IMediaTranscriptLocation + { + public Task ResolveAsync(string mediaPath, CancellationToken token) => Task.FromResult(new MediaTranscriptSlot(transcriptPath, "fixed")); + } +} \ No newline at end of file diff --git a/app/Tests/Tools/Media/TranscriptCleanupTests.cs b/app/Tests/Tools/Media/TranscriptCleanupTests.cs new file mode 100644 index 000000000..cce86c806 --- /dev/null +++ b/app/Tests/Tools/Media/TranscriptCleanupTests.cs @@ -0,0 +1,110 @@ +using AIStudio.Tools.Media; +using AIStudio.Tools.Services; +using AIStudio.Tools.Services.Indexing; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools.Media; + +/// +/// Checks that the transcript of a recording leaves the data source together with the recording. +/// +/// +/// A transcript holds what was said. Once its recording is gone from the data source, keeping it +/// below the data directory would keep that text, unseen by the user. +/// +[TestFixture] +public sealed class TranscriptCleanupTests +{ + private static readonly string KEY_A = new('a', 64); + + private static readonly string KEY_B = new('b', 64); + + private static readonly string KEY_C = new('c', 64); + + private string dataDirectory = string.Empty; + + private DataSourceTranscriptStore store = null!; + + [SetUp] + public async Task CreateTranscripts() + { + this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-transcript-cleanup-{Guid.NewGuid():N}"); + this.store = new DataSourceTranscriptStore(this.dataDirectory, Guid.NewGuid().ToString()); + foreach (var key in new[] { KEY_A, KEY_B, KEY_C }) + { + var path = this.store.GetPath(key); + Directory.CreateDirectory(Path.GetDirectoryName(path)!); + await File.WriteAllTextAsync(path, $"Transcript {key[0]}"); + } + } + + [TearDown] + public void DeleteTranscripts() + { + if (Directory.Exists(this.dataDirectory)) + Directory.Delete(this.dataDirectory, true); + } + + [Test] + public void TheTranscriptOfARemovedRecordingIsDeleted() + { + var manifest = Manifest(("/data/kept.mp3", KEY_A)); + + FileSourceIndexer.ForgetTranscript(this.store, manifest, KEY_B, NullLogger.Instance); + + Assert.That(this.store.ListKeys(), Is.EquivalentTo(new[] { KEY_A, KEY_C })); + } + + [Test] + public void ATranscriptWhichAnotherRecordingSharesIsKept() + { + var manifest = Manifest(("/data/copy-of-the-removed-recording.mp3", KEY_B)); + + FileSourceIndexer.ForgetTranscript(this.store, manifest, KEY_B, NullLogger.Instance); + + Assert.That(this.store.ListKeys(), Does.Contain(KEY_B), "Two recordings with the same content share one transcript."); + } + + [Test] + public void AFileWithoutTranscriptDeletesNothing() + { + FileSourceIndexer.ForgetTranscript(this.store, new DataSourceEmbeddingManifest(), null, NullLogger.Instance); + Assert.That(this.store.ListKeys(), Has.Count.EqualTo(3)); + } + + [Test] + public async Task ACompleteRunDeletesWhatNoFileUses() + { + var temporaryPath = $"{this.store.GetPath(KEY_A)}.{Guid.NewGuid():N}.tmp"; + await File.WriteAllTextAsync(temporaryPath, "Interrupted"); + var manifest = Manifest(("/data/meeting.mp3", KEY_A), ("/data/report.pdf", null)); + + FileSourceIndexer.DeleteUnusedTranscripts(this.store, manifest, NullLogger.Instance); + + Assert.Multiple(() => + { + Assert.That(this.store.ListKeys(), Is.EqualTo(new[] { KEY_A })); + Assert.That(File.Exists(temporaryPath), Is.False, "A temporary file of an interrupted write is no transcript of anything."); + }); + } + + [Test] + public void WithoutAStoreNothingHappens() + { + Assert.DoesNotThrow(() => + { + FileSourceIndexer.DeleteUnusedTranscripts(null, new DataSourceEmbeddingManifest(), NullLogger.Instance); + FileSourceIndexer.ForgetTranscript(null, new DataSourceEmbeddingManifest(), KEY_A, NullLogger.Instance); + }); + } + + private static DataSourceEmbeddingManifest Manifest(params (string Path, string? TranscriptKey)[] files) + { + var manifest = new DataSourceEmbeddingManifest(); + foreach (var (path, transcriptKey) in files) + manifest.Files[path] = new EmbeddedFileRecord("fingerprint", 16, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, 1, transcriptKey); + + return manifest; + } +} \ No newline at end of file diff --git a/app/Tests/Tools/MediaIndexingTests.cs b/app/Tests/Tools/MediaIndexingTests.cs new file mode 100644 index 000000000..b64e89621 --- /dev/null +++ b/app/Tests/Tools/MediaIndexingTests.cs @@ -0,0 +1,258 @@ +using AIStudio.Settings; +using AIStudio.Settings.DataModel; +using AIStudio.Tools; +using AIStudio.Tools.Media; +using AIStudio.Tools.Rust; +using AIStudio.Tools.Services; +using AIStudio.Tools.Services.Indexing; + +using Microsoft.Extensions.Logging.Abstractions; + +namespace AIStudio.Tests.Tools; + +/// +/// Checks which audio and video files a local data source transcribes, and how it treats those it cannot. +/// +/// +/// Transcribing costs money with every cloud provider. A media file is therefore only sent when a +/// provider exists which the data source allows, and a file which cannot be transcribed for a +/// reason of its own is not sent again until it changes. +/// +public sealed class MediaIndexingTests +{ + private string testRoot = string.Empty; + + [SetUp] + public void CreateTestRoot() + { + this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-media-indexing-{Guid.NewGuid():N}"); + Directory.CreateDirectory(this.testRoot); + } + + [TearDown] + public void DeleteTestRoot() + { + if (Directory.Exists(this.testRoot)) + Directory.Delete(this.testRoot, true); + } + + [Test] + public void AnAllowedProviderTranscribesASelectedRecording() + { + var recording = this.CreateFile("meeting.mp3", 16); + + var result = CreateIndexer().GetInputFiles(this.Folder("*"), new DataSourceEmbeddingManifest(), Allowed()); + + Assert.Multiple(() => + { + Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording })); + Assert.That(result.IgnoredFiles, Is.Empty); + }); + } + + [Test] + public void WithoutAProviderANewRecordingWaitsForOne() + { + var recording = this.CreateFile("meeting.mp3", 16); + + var result = CreateIndexer().GetInputFiles(this.Folder("*"), new DataSourceEmbeddingManifest(), MediaTranscriptionAvailability.MISSING); + + Assert.Multiple(() => + { + Assert.That(result.Files, Is.Empty); + Assert.That(result.IgnoredFiles, Is.EqualTo(new[] { new DataSourceEmbeddingIgnoredFile(recording, DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING) })); + }); + } + + [Test] + public void AProviderBelowTheConfidenceLevelHearsNothing() + { + var recording = this.CreateFile("meeting.mp4", 16); + + var result = CreateIndexer().GetInputFiles(this.Folder("*"), new DataSourceEmbeddingManifest(), MediaTranscriptionAvailability.NOT_ALLOWED); + + Assert.Multiple(() => + { + Assert.That(result.Files, Is.Empty, "The recording would have gone to a provider the data source does not allow."); + Assert.That(result.IgnoredFiles, Is.EqualTo(new[] { new DataSourceEmbeddingIgnoredFile(recording, DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_NOT_ALLOWED) })); + }); + } + + [Test] + public void AnIndexedRecordingStaysWhenTheProviderIsGone() + { + var recording = this.CreateFile("meeting.mp3", 16); + var manifest = new DataSourceEmbeddingManifest(); + manifest.Files[recording] = new EmbeddedFileRecord(FileSourceIndexer.BuildFileMetadataHash(new FileInfo(recording)), 16, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, 3, new string('a', 64)); + + var result = CreateIndexer().GetInputFiles(this.Folder("*"), manifest, MediaTranscriptionAvailability.MISSING); + + Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording }), "An unchanged recording is not read, so it needs no provider, and leaving it out would remove it from the index."); + } + + [Test] + public void ARecordingWhichFailedForGoodStaysWithItsReasonWhenTheProviderIsGone() + { + var recording = this.CreateFile("silence.wav", 16); + var manifest = new DataSourceEmbeddingManifest(); + manifest.PermanentFailures[recording] = new PermanentIndexingFailureRecord(FileSourceIndexer.BuildFileMetadataHash(new FileInfo(recording)), FileExtractionErrorCode.NO_CONTENT, "No content", DateTimeOffset.UtcNow); + + var result = CreateIndexer().GetInputFiles(this.Folder("*"), manifest, MediaTranscriptionAvailability.NOT_ALLOWED); + + Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording }), "The run lists it with its stored reason instead of reading it."); + } + + [Test] + public void AChangedRecordingNeedsAProviderAgain() + { + var recording = this.CreateFile("meeting.mp3", 16); + var manifest = new DataSourceEmbeddingManifest(); + manifest.Files[recording] = new EmbeddedFileRecord("THE-FINGERPRINT-BEFORE-THE-CHANGE", 8, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, 3, new string('a', 64)); + + var result = CreateIndexer().GetInputFiles(this.Folder("*"), manifest, MediaTranscriptionAvailability.MISSING); + + Assert.That(result.IgnoredFiles, Is.EqualTo(new[] { new DataSourceEmbeddingIgnoredFile(recording, DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING) })); + } + + [Test] + public void ARecordingAboveTheLimitIsNeverTranscribed() + { + var recording = this.CreateFile("conference.mkv", FolderFileSelection.MegabytesToBytes(1) + 1); + var dataSource = this.Folder("*") with { MaxFileSizeMegabytes = 1 }; + + var result = CreateIndexer().GetInputFiles(dataSource, new DataSourceEmbeddingManifest(), Allowed()); + + Assert.That(result.IgnoredFiles, Is.EqualTo(new[] { new DataSourceEmbeddingIgnoredFile(recording, DataSourceEmbeddingIgnoreReason.TOO_LARGE) })); + } + + [Test] + public void ImagesAreStillLeftOut() + { + var image = this.CreateFile("photo.png", 16); + + var result = CreateIndexer().GetInputFiles(this.Folder("*"), new DataSourceEmbeddingManifest(), Allowed()); + + Assert.That(result.IgnoredFiles, Is.EqualTo(new[] { new DataSourceEmbeddingIgnoredFile(image, DataSourceEmbeddingIgnoreReason.IMAGE_NOT_SUPPORTED) })); + } + + [TestCase("meeting.mp3")] + [TestCase("meeting.mp4")] + [TestCase("meeting.wav")] + [TestCase("meeting.webm")] + public void TheDefaultPatternsSelectNoRecording(string fileName) + { + var recording = this.CreateFile(fileName, 16); + + var result = CreateIndexer().GetInputFiles(this.Folder(DataSourceLocalDirectory.DEFAULT_FILE_PATTERNS), new DataSourceEmbeddingManifest(), Allowed()); + + Assert.Multiple(() => + { + Assert.That(result.Files, Is.Empty, "Existing data sources must not start transcribing, and paying for it, after an update."); + Assert.That(FileNamePatterns.Matches(recording, FileNamePatterns.Parse(":text:")), Is.False); + }); + } + + [TestCase("*")] + [TestCase("*.mp3")] + [TestCase("*.pdf;*.mp3")] + public void APatternWhichNamesRecordingsSelectsThem(string patterns) + { + var recording = this.CreateFile("meeting.mp3", 16); + + var result = CreateIndexer().GetInputFiles(this.Folder(patterns), new DataSourceEmbeddingManifest(), Allowed()); + + Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording })); + } + + [Test] + public void TheBatchTranscriptOfASelectedRecordingIsNoDocumentOfItsOwn() + { + var recording = this.CreateFile("meeting.mp3", 16); + var transcript = this.CreateFile(Path.GetFileName(MediaSidecarTranscripts.GetFilePath(recording)), 16); + + var everything = CreateIndexer().GetInputFiles(this.Folder("*"), new DataSourceEmbeddingManifest(), Allowed()); + var markdownOnly = CreateIndexer().GetInputFiles(this.Folder("*.md"), new DataSourceEmbeddingManifest(), Allowed()); + + Assert.Multiple(() => + { + Assert.That(everything.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording }), "The transcript would duplicate the text of the recording."); + Assert.That(markdownOnly.Files.Select(file => file.FullName), Is.EqualTo(new[] { transcript }), "Without the recording, the transcript is a document like every other."); + }); + } + + [Test] + public void ASingleRecordingCanBeADataSource() + { + var recording = this.CreateFile("interview.m4a", 16); + var dataSource = new DataSourceLocalFile { Type = DataSourceType.LOCAL_FILE, FilePath = recording }; + + var result = CreateIndexer().GetInputFiles(dataSource, new DataSourceEmbeddingManifest(), Allowed()); + + Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording })); + } + + [Test] + public void AReadyTranscriptIsRead() + { + var transcript = PersistentMediaTranscriptResult.Ready(new MediaTranscriptSlot("/data/mediaTranscripts/source/key.md", "key"), reused: true); + Assert.That(FileSourceIndexer.GetTranscriptOrThrow(transcript, CancellationToken.None), Is.EqualTo(new MediaTranscriptSlot("/data/mediaTranscripts/source/key.md", "key"))); + } + + [TestCase(PersistentMediaTranscriptStatus.EMPTY)] + [TestCase(PersistentMediaTranscriptStatus.NO_AUDIBLE_SIGNAL)] + public void ARecordingWithoutTextIsRecordedForGood(PersistentMediaTranscriptStatus status) + { + var transcript = PersistentMediaTranscriptResult.Unavailable(status, null, "Nothing to read."); + + var exception = Assert.Throws(() => FileSourceIndexer.GetTranscriptOrThrow(transcript, CancellationToken.None)); + Assert.That(exception!.Code, Is.EqualTo(FileExtractionErrorCode.NO_CONTENT), "Asked again with every run, the provider would be paid again with every run."); + } + + [Test] + public void ARecordingTheMediaPipelineCannotReadIsRecordedForGood() + { + var transcript = PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.TRANSCRIPTION_FAILED, null, "The container is damaged.", MediaJobErrorCode.DAMAGED_CONTAINER); + + var exception = Assert.Throws(() => FileSourceIndexer.GetTranscriptOrThrow(transcript, CancellationToken.None)); + Assert.That(exception!.Code, Is.EqualTo(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED)); + } + + [Test] + public void ARecordingTheProviderRefusedIsRecordedForGood() + { + var transcript = PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.REJECTED, null, "The provider 'Cloud' reported an error: Audio file might be corrupted or unsupported"); + + var exception = Assert.Throws(() => FileSourceIndexer.GetTranscriptOrThrow(transcript, CancellationToken.None)); + Assert.That(exception!.Code, Is.EqualTo(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED), "Tried again with every run, the same file would be uploaded again with every run."); + } + + [TestCase(PersistentMediaTranscriptStatus.TRANSCRIPTION_FAILED)] + [TestCase(PersistentMediaTranscriptStatus.STORE_FAILED)] + [TestCase(PersistentMediaTranscriptStatus.SOURCE_CHANGED)] + [TestCase(PersistentMediaTranscriptStatus.NO_PROVIDER)] + public void AFailureWhichMayPassIsTriedAgain(PersistentMediaTranscriptStatus status) + { + var transcript = PersistentMediaTranscriptResult.Unavailable(status, null, "Try again later."); + Assert.Throws(() => FileSourceIndexer.GetTranscriptOrThrow(transcript, CancellationToken.None)); + } + + private DataSourceLocalDirectory Folder(string patterns) => new() + { + Type = DataSourceType.LOCAL_DIRECTORY, + Path = this.testRoot, + FilePatterns = patterns, + }; + + private string CreateFile(string fileName, long length) + { + var filePath = Path.Combine(this.testRoot, fileName); + using var stream = File.Create(filePath); + stream.SetLength(length); + return filePath; + } + + private static MediaTranscriptionAvailability Allowed() => MediaTranscriptionAvailability.Allowed(new TranscriptionProvider { Id = "test-transcription-provider" }); + + // Finding the files reads none of them, so it needs neither the runtime nor a transcription provider: + private static FileSourceIndexer CreateIndexer() => new(null!, null!, null!, null!, NullLogger.Instance); +} \ No newline at end of file diff --git a/app/Tests/Tools/QueuedStatusTests.cs b/app/Tests/Tools/QueuedStatusTests.cs index 40300691f..24e036967 100644 --- a/app/Tests/Tools/QueuedStatusTests.cs +++ b/app/Tests/Tools/QueuedStatusTests.cs @@ -82,7 +82,7 @@ public sealed class QueuedStatusTests PendingRemovalCount: 42, LastSyncUtc: NOW) { - IgnoredFiles = [new("/tmp/test-data/interview.mp3", DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED)], + IgnoredFiles = [new("/tmp/test-data/interview.mp3", DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING)], FilePatternExclusions = [new(".png", 5)], }; diff --git a/app/Tests/Tools/TerminalLoggerTests.cs b/app/Tests/Tools/TerminalLoggerTests.cs index 4882252fb..1d75a0cee 100644 --- a/app/Tests/Tools/TerminalLoggerTests.cs +++ b/app/Tests/Tools/TerminalLoggerTests.cs @@ -45,4 +45,30 @@ public sealed class TerminalLoggerTests CultureInfo.CurrentCulture = previousCulture; } } + + [Test] + public void AnExceptionIsDescribedTogetherWithItsCause() + { + var exception = new HttpRequestException("Error while copying content to a stream.", new IOException("Unable to write data to the transport connection.", new InvalidOperationException("Connection reset by peer."))); + + Assert.That(TerminalLogger.DescribeException(exception), Is.EqualTo("Error while copying content to a stream. ---> Unable to write data to the transport connection. ---> Connection reset by peer.")); + } + + [Test] + public void AnExceptionWithoutCauseKeepsItsMessage() + { + Assert.Multiple(() => + { + Assert.That(TerminalLogger.DescribeException(new InvalidOperationException("Something failed.")), Is.EqualTo("Something failed.")); + Assert.That(TerminalLogger.DescribeException(null), Is.Null); + }); + } + + [Test] + public void TheCausesOfAnAggregateExceptionAreNotRepeated() + { + var exception = new AggregateException(new InvalidOperationException("First failure."), new InvalidOperationException("Second failure.")); + + Assert.That(TerminalLogger.DescribeException(exception), Is.EqualTo(exception.Message)); + } } \ No newline at end of file diff --git a/app/Tests/Tools/ToolCalling/MailDraftSyncTests.cs b/app/Tests/Tools/ToolCalling/MailDraftSyncTests.cs index 2046e17a9..9782c409b 100644 --- a/app/Tests/Tools/ToolCalling/MailDraftSyncTests.cs +++ b/app/Tests/Tools/ToolCalling/MailDraftSyncTests.cs @@ -117,5 +117,5 @@ public sealed class MailDraftSyncTests : ToolRegistryTestBase } // Queueing a run needs none of the services which carry it out: - private DataSourceEmbeddingService CreateService() => new(this.SettingsManager, null!, null!, null!, NullLogger.Instance); + private DataSourceEmbeddingService CreateService() => new(this.SettingsManager, null!, null!, null!, null!, NullLogger.Instance); } \ No newline at end of file diff --git a/app/Tests/Tools/UnreachableDataSourceTests.cs b/app/Tests/Tools/UnreachableDataSourceTests.cs index 4ce359c7e..7672cdea3 100644 --- a/app/Tests/Tools/UnreachableDataSourceTests.cs +++ b/app/Tests/Tools/UnreachableDataSourceTests.cs @@ -205,7 +205,7 @@ public sealed class UnreachableDataSourceTests NullLogger.Instance); // A run which reaches no file reads none, so it needs neither the runtime nor the chunker: - var indexer = new FileSourceIndexer(settingsManager, null!, null!, NullLogger.Instance); + var indexer = new FileSourceIndexer(settingsManager, null!, null!, null!, NullLogger.Instance); var outcome = await indexer.ProcessAsync(context, DataSourceEmbeddingRefreshMode.HASH_CHECK, CancellationToken.None); return new(outcome, statuses, manifest); diff --git a/app/Tests/Tools/Validation/CloudProcessingTests.cs b/app/Tests/Tools/Validation/CloudProcessingTests.cs new file mode 100644 index 000000000..adec7184f --- /dev/null +++ b/app/Tests/Tools/Validation/CloudProcessingTests.cs @@ -0,0 +1,68 @@ +using AIStudio.Tools; +using AIStudio.Tools.Validation; + +namespace AIStudio.Tests.Tools.Validation; + +/// +/// Checks what the dialogs of local data sources tell about the cloud, and when they ask again. +/// +/// +/// A data source can send its text to an embedding provider and its recordings to a transcription +/// provider. The user acknowledges what leaves the computer, and an acknowledgment for the one must +/// not count for the other. +/// +[TestFixture] +public sealed class CloudProcessingTests +{ + [TestCase(CloudProcessing.EMBEDDING, CloudProcessing.EMBEDDING, true)] + [TestCase(CloudProcessing.EMBEDDING, CloudProcessing.EMBEDDING | CloudProcessing.TRANSCRIPTION, false)] + [TestCase(CloudProcessing.EMBEDDING, CloudProcessing.TRANSCRIPTION, false)] + [TestCase(CloudProcessing.EMBEDDING | CloudProcessing.TRANSCRIPTION, CloudProcessing.TRANSCRIPTION, true)] + [TestCase(CloudProcessing.NONE, CloudProcessing.EMBEDDING, false)] + [TestCase(CloudProcessing.NONE, CloudProcessing.NONE, true)] + public void AnAcknowledgmentCoversOnlyWhatItWasGivenFor(CloudProcessing acknowledged, CloudProcessing current, bool expected) + { + Assert.That(acknowledged.Covers(current), Is.EqualTo(expected)); + } + + [Test] + public void ARecordingSentToTheCloudNeedsAnAcknowledgment() + { + var validation = new DataSourceValidation { GetSelectedCloudTranscription = () => true }; + + Assert.Multiple(() => + { + Assert.That(validation.ValidateUserAcknowledgedCloudEmbedding(false), Is.Not.Null); + Assert.That(validation.ValidateUserAcknowledgedCloudEmbedding(true), Is.Null); + }); + } + + [Test] + public void NothingSentToTheCloudNeedsNoAcknowledgment() + { + Assert.That(new DataSourceValidation().ValidateUserAcknowledgedCloudEmbedding(false), Is.Null); + } + + [TestCase("*")] + [TestCase("*.mp3")] + [TestCase("*.MP4")] + [TestCase("meeting*")] + [TestCase("*.*")] + [TestCase("*.m?a")] + [TestCase(":documents:;*.webm")] + public void APatternWhichMaySelectRecordingsCounts(string patterns) + { + Assert.That(FileNamePatterns.MaySelectMedia(FileNamePatterns.Parse(patterns)), Is.True); + } + + [TestCase(":documents:")] + [TestCase(":text:")] + [TestCase("*.pdf;*.docx")] + [TestCase(".gitignore")] + [TestCase("Makefile")] + [TestCase("report_*.txt")] + public void APatternWhichSelectsNoRecordingDoesNotCount(string patterns) + { + Assert.That(FileNamePatterns.MaySelectMedia(FileNamePatterns.Parse(patterns)), Is.False, "The dialog would warn about a transcription which never happens."); + } +} \ No newline at end of file diff --git a/runtime/src/data_restore.rs b/runtime/src/data_restore.rs index c84f5abfa..ec5ca7884 100644 --- a/runtime/src/data_restore.rs +++ b/runtime/src/data_restore.rs @@ -52,7 +52,7 @@ const SETTINGS_FILE_NAME: &str = "settings.json"; /// (`DataBackupContents`) includes. The runtime refuses a plan which names anything else, so that /// whatever goes wrong in the app, a restore never replaces the logs, the downloaded tools, or the /// data folder as a whole. -const REPLACEABLE_ELEMENTS: [(Location, &str); 8] = [ +const REPLACEABLE_ELEMENTS: [(Location, &str); 9] = [ (Location::ConfigDirectory, SETTINGS_FILE_NAME), (Location::DataDirectory, "workspaces/"), (Location::DataDirectory, "tempChats/"), @@ -61,6 +61,7 @@ const REPLACEABLE_ELEMENTS: [(Location, &str); 8] = [ (Location::DataDirectory, "visualBriefings/"), (Location::DataDirectory, "databases/sqlite/"), (Location::DataDirectory, "databases/vector_database/stores/"), + (Location::DataDirectory, "mediaTranscripts/"), ]; /// Paths which a restore always keeps, whatever the plan says: the plugins which come with AI Studio @@ -794,6 +795,21 @@ mod tests { assert_current_data_untouched(&test); } + #[test] + fn a_restore_replaces_the_media_transcripts_of_the_data_sources() { + let test = TestFolders::new(); + test.write_current_data(); + test.write(Location::DataDirectory, "mediaTranscripts/current/transcript.md", "current transcript"); + test.prepare_restore(r#"{ "location": "DATA_DIRECTORY", "relative_path": "mediaTranscripts/", "keep": [] }"#); + test.write(Location::DataDirectory, ".restore-pending/mediaTranscripts/restored/transcript.md", "restored transcript"); + + let outcome = apply_pending_restore_in(&test.folders(), &[]); + + assert_eq!(outcome, Some(RestoreOutcome::Applied)); + assert_eq!(test.read(Location::DataDirectory, "mediaTranscripts/restored/transcript.md").as_deref(), Some("restored transcript")); + assert!(!test.exists(Location::DataDirectory, "mediaTranscripts/current")); + } + #[test] fn a_plan_which_names_another_element_is_refused() { for relative_path in ["", "events.log", "databases/", "../data/", "pandoc/"] {