Added transcription of audio and video files to local data sources (#1081)

Co-authored-by: Paul Koudelka <106623909+PaulKoudelka@users.noreply.github.com>

app/Tests/Tools/DataBackup/DataBackupWriterTests.cs

@@ -25,6 +25,8 @@ public sealed class DataBackupWriterTests
 
     private static readonly Guid CHAT_ID = Guid.Parse("8a1c4e7f-2b5d-4f9a-b3c6-0e2d4f6a8b1c");
 
+    private const string MEDIA_TRANSCRIPT = "mediaTranscripts/5c9e2a7b-1d3f-4b6e-8a0c-2e4f6b8d0a1c/0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef.md";
+
     private string testDirectory = string.Empty;
 
     private string DataDirectory => Path.Combine(this.testDirectory, "data");
@@ -53,10 +55,12 @@ public sealed class DataBackupWriterTests
         this.WriteDataFile("media-staging/operation/transcript.md", "temporary");
         this.WriteDataFile("databases/sqlite/rag-index.sqlite3", "the index while AI Studio writes to it");
         this.WriteDataFile("databases/vector_database/stores/store_1/segments/segment", "the store while AI Studio writes to it");
+        this.WriteDataFile(MEDIA_TRANSCRIPT, "the transcript while AI Studio writes it");
 
         WriteFile(Path.Combine(this.ConfigDirectory, "settings.json"), "the settings on the disk, with the objects of the organization");
         WriteFile(Path.Combine(this.IndexCopiesDirectory, "databases", "sqlite", "rag-index.sqlite3"), "the copy of the index");
         WriteFile(Path.Combine(this.IndexCopiesDirectory, "databases", "vector_database", "stores", "store_1", "segments", "segment"), "the copy of the store");
+        WriteFile(Path.Combine(this.IndexCopiesDirectory, MEDIA_TRANSCRIPT), "the copy of the transcript");
     }
 
     [TearDown]
@@ -84,6 +88,7 @@ public sealed class DataBackupWriterTests
             "data/visualBriefings/briefing/manifest.json",
             "data/databases/sqlite/rag-index.sqlite3",
             "data/databases/vector_database/stores/store_1/segments/segment",
+            $"data/{MEDIA_TRANSCRIPT}",
             DataBackupFormat.MANIFEST_ENTRY_NAME,
         }));
 
@@ -92,6 +97,7 @@ public sealed class DataBackupWriterTests
             Assert.That(archive["config/settings.json"], Is.EqualTo(SETTINGS_JSON), "The settings come from memory, without the objects of configuration plugins.");
             Assert.That(archive["data/databases/sqlite/rag-index.sqlite3"], Is.EqualTo("the copy of the index"));
             Assert.That(archive["data/databases/vector_database/stores/store_1/segments/segment"], Is.EqualTo("the copy of the store"));
+            Assert.That(archive[$"data/{MEDIA_TRANSCRIPT}"], Is.EqualTo("the copy of the transcript"), "The transcripts are copied together with the index they belong to.");
         });
     }
 
@@ -101,6 +107,7 @@ public sealed class DataBackupWriterTests
         var archive = await this.WriteBackupAsync(string.Empty);
 
         Assert.That(archive.Keys.Where(name => name.StartsWith("data/databases/", StringComparison.Ordinal)), Is.Empty, "The indexes in the data folder change while AI Studio runs.");
+        Assert.That(archive.Keys.Where(name => name.StartsWith("data/mediaTranscripts/", StringComparison.Ordinal)), Is.Empty, "The transcripts belong to the indexes and come only together with them.");
     }
 
     [Test]

app/Tests/Tools/DataSourceEmbeddingHealthTests.cs

@@ -25,14 +25,33 @@ public sealed class DataSourceEmbeddingHealthTests
         Assert.That(status.Health, Is.EqualTo(DataSourceEmbeddingHealth.WARNING));
     }
 
-    [Test]
-    public void AnIgnoredMediaFileKeepsAUsableDataSourceHealthy()
+    [TestCase(DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING)]
+    [TestCase(DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_NOT_ALLOWED)]
+    public void ATranscriptionProviderWarningRemainsActionableWhenOtherFilesWereIndexed(DataSourceEmbeddingIgnoreReason reason)
     {
         var status = Status(indexedDocuments: 1, totalDocuments: 2) with
         {
             IgnoredFiles =
             [
-                new DataSourceEmbeddingIgnoredFile("/data/interview.mp3", DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED),
+                new DataSourceEmbeddingIgnoredFile("/data/interview.mp3", reason),
+            ],
+        };
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(status.Health, Is.EqualTo(DataSourceEmbeddingHealth.WARNING));
+            Assert.That(status.NeedsTranscriptionProvider, Is.True, "The embedding page offers the way to the settings only then.");
+        });
+    }
+
+    [Test]
+    public void AnIgnoredImageKeepsAUsableDataSourceHealthy()
+    {
+        var status = Status(indexedDocuments: 1, totalDocuments: 2) with
+        {
+            IgnoredFiles =
+            [
+                new DataSourceEmbeddingIgnoredFile("/data/photo.png", DataSourceEmbeddingIgnoreReason.IMAGE_NOT_SUPPORTED),
             ],
         };
 

app/Tests/Tools/Databases/IndexStoreManifestTests.cs

@@ -40,6 +40,27 @@ public sealed class IndexStoreManifestTests
         });
     }
 
+    [Test]
+    public async Task TheManifestKnowsWhichTranscriptBelongsToAMediaFile()
+    {
+        await using var store = await TemporaryIndexStore.CreateAsync();
+        await store.Client.UpsertDataSourceAsync(DATA_SOURCE_ID, "LOCAL_DIRECTORY", "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01", "signature", string.Empty, 3, TOKEN);
+
+        var recording = Path.Combine(Path.GetTempPath(), "test-data", "meeting.mp3");
+        var document = Path.Combine(Path.GetTempPath(), "test-data", "report.pdf");
+        var transcriptKey = new string('f', 64);
+        await store.Client.UpsertFileAsync(DATA_SOURCE_ID, Row(recording, 2) with { TranscriptKey = transcriptKey }, TOKEN);
+        await store.Client.UpsertFileAsync(DATA_SOURCE_ID, Row(document, 4), TOKEN);
+
+        var manifest = await store.Client.GetManifestAsync(DATA_SOURCE_ID, TOKEN);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(manifest.Files[recording].TranscriptKey, Is.EqualTo(transcriptKey), "Without the key, AI Studio cannot remove the transcript once the recording is gone.");
+            Assert.That(manifest.Files[document].TranscriptKey, Is.Null);
+        });
+    }
+
     private static EmbeddingStateFile Row(string filePath, int chunkCount)
     {
         var now = DateTimeOffset.UtcNow;

app/Tests/Tools/DocumentRunProgressTests.cs

@@ -64,6 +64,27 @@ public sealed class DocumentRunProgressTests
         });
     }
 
+    [Test]
+    public async Task APermanentMediaFailureKeepsItsExplanation()
+    {
+        var (progress, statuses, manifest) = CreateProgress(totalDocuments: 1);
+        var document = Document(Path.Combine(Path.GetTempPath(), "test-data", "recording.webm"));
+
+        await progress.RecordDocumentFailureAsync(
+            document,
+            new FileExtractionException(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED, "The media file is damaged or its format could not be identified."),
+            CancellationToken.None);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(statuses[^1].Failures, Has.Count.EqualTo(1));
+            Assert.That(statuses[^1].Failures[0].ExtractionCode, Is.EqualTo(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED));
+            Assert.That(statuses[^1].Failures[0].Reason, Does.Contain("The media file is damaged or its format could not be identified."));
+            Assert.That(manifest.PermanentFailures[document.State.AbsolutePath].Code, Is.EqualTo(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED));
+            Assert.That(manifest.PermanentFailures[document.State.AbsolutePath].Message, Is.EqualTo(statuses[^1].Failures[0].Reason));
+        });
+    }
+
     [Test]
     public async Task ARunWhichOnlySkippedDocumentsForGoodIsCompleted()
     {
@@ -379,8 +400,8 @@ public sealed class DocumentRunProgressTests
         var testDataPath = Path.Combine(Path.GetTempPath(), "test-data");
         var ignoredFiles = new[]
         {
-            new DataSourceEmbeddingIgnoredFile(Path.Combine(testDataPath, "recording.mp3"), DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED),
-            new DataSourceEmbeddingIgnoredFile(Path.Combine(testDataPath, "diagram.png"), DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED),
+            new DataSourceEmbeddingIgnoredFile(Path.Combine(testDataPath, "recording.mp3"), DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING),
+            new DataSourceEmbeddingIgnoredFile(Path.Combine(testDataPath, "diagram.png"), DataSourceEmbeddingIgnoreReason.IMAGE_NOT_SUPPORTED),
         };
         var filePatternExclusions = new[] { new DataSourceEmbeddingFilePatternGroup(".pdf", 42) };
         var excludedFolders = new[] { new DataSourceEmbeddingExcludedFolderGroup("node_modules", 3) };
@@ -432,6 +453,15 @@ public sealed class DocumentRunProgressTests
         Assert.That(statuses[^1].Failures[0].DisplayName, Is.EqualTo("report.pdf"));
     }
 
+    [TestCase("Audio file might be corrupted or unsupported", "Audio file might be corrupted or unsupported.")]
+    [TestCase("The container is damaged. ", "The container is damaged.")]
+    [TestCase("Is it damaged?", "Is it damaged?")]
+    [TestCase("", "")]
+    public void AReasonEndsAsASentence(string reason, string sentence)
+    {
+        Assert.That(DocumentRunProgress.AsSentence(reason), Is.EqualTo(sentence));
+    }
+
     private static IndexingPass NewPass() => new(new DataSourceRunSignature("TEST-EMBEDDING-SIGNATURE", string.Empty));
 
     /// <summary>

app/Tests/Tools/FileSizeLimitTests.cs

@@ -140,5 +140,5 @@ public sealed class FileSizeLimitTests
     }
 
     // Finding the files reads none of them, so it needs neither the runtime nor the chunker:
-    private static FileSourceIndexer CreateIndexer() => new(null!, null!, null!, NullLogger.Instance);
+    private static FileSourceIndexer CreateIndexer() => new(null!, null!, null!, null!, NullLogger.Instance);
 }

app/Tests/Tools/FileSourceIndexerWatcherTests.cs

@@ -1,4 +1,5 @@
 using AIStudio.Tools;
+using AIStudio.Tools.Media;
 using AIStudio.Tools.Services.Indexing;
 
 namespace AIStudio.Tests.Tools;
@@ -261,6 +262,78 @@ public sealed class FileSourceIndexerWatcherTests
         Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.False);
     }
 
+    [Test]
+    public async Task DirectoryWatcherIgnoresTheBatchTranscriptWhenTheMediaIsSelected()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-transcript-watcher-{Guid.NewGuid():N}");
+        var mediaPath = Path.Combine(directoryPath, "recording.mp3");
+        Directory.CreateDirectory(directoryPath);
+
+        try
+        {
+            await File.WriteAllBytesAsync(mediaPath, [1, 2, 3]);
+            var transcriptPath = MediaSidecarTranscripts.GetFilePath(mediaPath);
+            var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(
+                directoryPath,
+                true,
+                FilePatterns: FileNamePatterns.Parse("*"));
+            var unrelatedPatterns = configuration with { FilePatterns = FileNamePatterns.Parse("*.tf") };
+
+            Assert.Multiple(() =>
+            {
+                Assert.That(FileSourceIndexer.IsRelevantWatcherPath(configuration, mediaPath, WatcherChangeTypes.Changed), Is.True);
+                Assert.That(FileSourceIndexer.IsRelevantWatcherPath(configuration, transcriptPath, WatcherChangeTypes.Created), Is.False);
+                Assert.That(FileSourceIndexer.IsRelevantWatcherPath(unrelatedPatterns, transcriptPath, WatcherChangeTypes.Created), Is.False);
+            });
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [Test]
+    public async Task DirectoryWatcherKeepsTranscriptAsADocumentWhenTheMediaIsNotSelected()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-transcript-document-{Guid.NewGuid():N}");
+        var mediaPath = Path.Combine(directoryPath, "recording.mp3");
+        Directory.CreateDirectory(directoryPath);
+
+        try
+        {
+            await File.WriteAllBytesAsync(mediaPath, [1, 2, 3]);
+            var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(
+                directoryPath,
+                true,
+                FilePatterns: FileNamePatterns.Parse("*.md"));
+
+            Assert.That(
+                FileSourceIndexer.IsRelevantWatcherPath(configuration, MediaSidecarTranscripts.GetFilePath(mediaPath), WatcherChangeTypes.Changed),
+                Is.True);
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+
+    [TestCase("recording.mp3")]
+    [TestCase("meeting.mp4")]
+    [TestCase("voice.webm")]
+    public void MediaStartsARunOnlyWhenAPatternSelectsIt(string fileName)
+    {
+        var rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-media-watcher-{Guid.NewGuid():N}");
+        var path = Path.Combine(rootPath, fileName);
+        var documents = DefaultConfiguration(rootPath);
+        var everything = documents with { FilePatterns = FileNamePatterns.Parse("*") };
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(FileSourceIndexer.IsRelevantWatcherPath(documents, path, WatcherChangeTypes.Created), Is.False, "The documents selector never selects audio or video.");
+            Assert.That(FileSourceIndexer.IsRelevantWatcherPath(everything, path, WatcherChangeTypes.Created), Is.True);
+        });
+    }
+
     private static FileSourceIndexer.DataSourceWatcherConfiguration DefaultConfiguration(string rootPath) => new(
         rootPath,
         true,

app/Tests/Tools/FileTypesTests.cs

@@ -190,10 +190,10 @@ public class FileTypesTests
     [TestCase("recording.mp3")]
     [TestCase("meeting.mp4")]
     [TestCase("voice-recording.webm")]
-    public void LocalFileDataSourcesRejectMedia(string fileName)
+    public void LocalFileDataSourcesAcceptTranscribableMedia(string fileName)
     {
         var validation = new DataSourceValidation();
-        WithFile([1, 2, 3], fileName, path => Assert.That(validation.ValidateFilePath(path), Is.Not.Null));
+        WithFile([1, 2, 3], fileName, path => Assert.That(validation.ValidateFilePath(path), Is.Null));
     }
 
     private static void AssertPath(string content, string fileName, Encoding encoding, bool expected, FileTypeFilter? type = null)

app/Tests/Tools/Media/DataSourceTranscriptStoreTests.cs

@@ -0,0 +1,233 @@
+using AIStudio.Settings;
+using AIStudio.Tools.Media;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools.Media;
+
+/// <summary>
+/// Checks where data sources keep the transcripts of their audio and video files.
+/// </summary>
+[TestFixture]
+public sealed class DataSourceTranscriptStoreTests
+{
+    private static readonly string DATA_SOURCE_ID = Guid.NewGuid().ToString();
+
+    private string rootPath = string.Empty;
+
+    private string dataDirectory = string.Empty;
+
+    private string userFolder = string.Empty;
+
+    [SetUp]
+    public void CreateFolders()
+    {
+        this.rootPath = Path.Combine(Path.GetTempPath(), $"ai-studio-transcript-store-{Guid.NewGuid():N}");
+        this.dataDirectory = Path.Combine(this.rootPath, "data");
+        this.userFolder = Path.Combine(this.rootPath, "user");
+        Directory.CreateDirectory(this.dataDirectory);
+        Directory.CreateDirectory(this.userFolder);
+    }
+
+    [TearDown]
+    public void DeleteFolders()
+    {
+        if (Directory.Exists(this.rootPath))
+            Directory.Delete(this.rootPath, true);
+    }
+
+    [Test]
+    public async Task TheTranscriptLiesBelowTheDataDirectoryOfItsDataSource()
+    {
+        var mediaPath = await this.CreateMediaAsync("recording.mp3", [1, 2, 3]);
+        var slot = await this.Store().ResolveAsync(mediaPath, CancellationToken.None);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(DataSourceTranscriptStore.IsKey(slot.Key), Is.True);
+            Assert.That(slot.Path, Is.EqualTo(Path.Join(this.dataDirectory, DataSourceTranscriptStore.FOLDER_NAME, DATA_SOURCE_ID, slot.Key + ".md")));
+        });
+    }
+
+    [Test]
+    public async Task TheSameContentFindsTheSameTranscriptWhereverTheFileLies()
+    {
+        var original = await this.CreateMediaAsync("recording.mp3", [1, 2, 3]);
+        var copy = await this.CreateMediaAsync(Path.Combine("moved", "renamed.mp3"), [1, 2, 3]);
+        File.SetLastWriteTimeUtc(copy, DateTime.UtcNow.AddDays(-3));
+        var other = await this.CreateMediaAsync("other.mp3", [1, 2, 3, 4]);
+
+        var store = this.Store();
+        var originalSlot = await store.ResolveAsync(original, CancellationToken.None);
+        var copySlot = await store.ResolveAsync(copy, CancellationToken.None);
+        var otherSlot = await store.ResolveAsync(other, CancellationToken.None);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(copySlot, Is.EqualTo(originalSlot));
+            Assert.That(otherSlot.Key, Is.Not.EqualTo(originalSlot.Key));
+        });
+    }
+
+    [Test]
+    public async Task ARenamedRecordingIsPaidForOnlyOnce()
+    {
+        var mediaPath = await this.CreateMediaAsync("recording.mp3", [1, 2, 3]);
+        var provider = new TranscriptionProvider { Id = "test-transcription-provider" };
+        var calls = 0;
+        Task<MediaTranscriptionResult> Transcribe(string path, TranscriptionProvider _, CancellationToken __)
+        {
+            calls++;
+            return Task.FromResult(MediaTranscriptionResult.Succeeded("Transcript"));
+        }
+
+        var first = await PersistentMediaTranscript.ReadOrCreateAsync(this.Store(), mediaPath, provider, Transcribe, NullLogger.Instance, CancellationToken.None);
+
+        var renamedPath = Path.Combine(this.userFolder, "renamed.mp3");
+        File.Move(mediaPath, renamedPath);
+        File.SetLastWriteTimeUtc(renamedPath, DateTime.UtcNow.AddMinutes(5));
+        var second = await PersistentMediaTranscript.ReadOrCreateAsync(this.Store(), renamedPath, provider, Transcribe, NullLogger.Instance, CancellationToken.None);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(first.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY));
+            Assert.That(second.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY));
+            Assert.That(second.Reused, Is.True);
+            Assert.That(calls, Is.EqualTo(1));
+            Assert.That(Directory.GetFiles(this.userFolder), Has.Length.EqualTo(1), "The data source wrote into the folder of the user.");
+        });
+    }
+
+    [Test]
+    public async Task OnlyTranscriptsAreListedAndDeleted()
+    {
+        var store = this.Store();
+        var key = new string('a', 64);
+        var transcriptPath = store.GetPath(key);
+        var folder = Path.GetDirectoryName(transcriptPath)!;
+        Directory.CreateDirectory(folder);
+        await File.WriteAllTextAsync(transcriptPath, "Transcript");
+        await File.WriteAllTextAsync($"{transcriptPath}.{Guid.NewGuid():N}.tmp", "Interrupted");
+        await File.WriteAllTextAsync(Path.Combine(folder, "notes.md"), "Not ours");
+
+        Assert.That(store.ListKeys(), Is.EqualTo(new[] { key }));
+
+        store.DeleteAll();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(File.Exists(transcriptPath), Is.False);
+            Assert.That(Directory.GetFiles(folder, "*.tmp"), Is.Empty);
+            Assert.That(File.Exists(Path.Combine(folder, "notes.md")), Is.True, "A file which the store did not write was deleted.");
+        });
+    }
+
+    [Test]
+    public async Task DeletingAllTranscriptsRemovesTheFolderOfTheDataSource()
+    {
+        var store = this.Store();
+        var transcriptPath = store.GetPath(new string('b', 64));
+        Directory.CreateDirectory(Path.GetDirectoryName(transcriptPath)!);
+        await File.WriteAllTextAsync(transcriptPath, "Transcript");
+
+        store.DeleteAll();
+
+        Assert.That(Directory.Exists(Path.GetDirectoryName(transcriptPath)), Is.False);
+    }
+
+    [Test]
+    public void DeletingADataSourceWithoutTranscriptsWorks()
+    {
+        Assert.DoesNotThrow(() => this.Store().DeleteAll());
+    }
+
+    [Test]
+    public async Task ALinkInTheFolderIsNeitherFollowedNorDeleted()
+    {
+        var store = this.Store();
+        var key = new string('c', 64);
+        var linkPath = store.GetPath(key);
+        Directory.CreateDirectory(Path.GetDirectoryName(linkPath)!);
+        var targetPath = Path.Combine(this.userFolder, "precious.md");
+        await File.WriteAllTextAsync(targetPath, "Precious");
+
+        try
+        {
+            File.CreateSymbolicLink(linkPath, targetPath);
+        }
+        catch (Exception exception) when (exception is UnauthorizedAccessException or IOException)
+        {
+            Assert.Ignore("Creating symbolic links needs a privilege this system does not grant.");
+        }
+
+        var deleted = store.Delete(key);
+        store.DeleteAll();
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(deleted, Is.False);
+            Assert.That(File.ReadAllText(targetPath), Is.EqualTo("Precious"));
+        });
+    }
+
+    [Test]
+    public async Task ABackupCopiesOnlyTheTranscriptsOfAllDataSources()
+    {
+        var secondId = Guid.NewGuid().ToString();
+        var firstPath = this.Store().GetPath(new string('d', 64));
+        var secondPath = new DataSourceTranscriptStore(this.dataDirectory, secondId).GetPath(new string('e', 64));
+        foreach (var path in new[] { firstPath, secondPath })
+        {
+            Directory.CreateDirectory(Path.GetDirectoryName(path)!);
+            await File.WriteAllTextAsync(path, "Transcript");
+        }
+
+        await File.WriteAllTextAsync($"{firstPath}.{Guid.NewGuid():N}.tmp", "Interrupted");
+        var targetDirectory = Path.Combine(this.rootPath, "copies");
+
+        var copied = await DataSourceTranscriptStore.CopyAllAsync(this.dataDirectory, targetDirectory, CancellationToken.None);
+
+        var copiedFiles = Directory.GetFiles(targetDirectory, "*", SearchOption.AllDirectories).Select(path => Path.GetRelativePath(targetDirectory, path).Replace('\\', '/'));
+        Assert.Multiple(() =>
+        {
+            Assert.That(copied, Is.EqualTo(2));
+            Assert.That(copiedFiles, Is.EquivalentTo(new[]
+            {
+                $"{DataSourceTranscriptStore.FOLDER_NAME}/{DATA_SOURCE_ID}/{new string('d', 64)}.md",
+                $"{DataSourceTranscriptStore.FOLDER_NAME}/{secondId}/{new string('e', 64)}.md",
+            }));
+        });
+    }
+
+    [Test]
+    public async Task ABackupWithoutTranscriptsCopiesNothing()
+    {
+        Assert.That(await DataSourceTranscriptStore.CopyAllAsync(this.dataDirectory, Path.Combine(this.rootPath, "copies"), CancellationToken.None), Is.Zero);
+    }
+
+    [TestCase("")]
+    [TestCase("ABCDEF")]
+    [TestCase("../outside")]
+    public void ADataSourceIdWhichIsNoGuidIsRejected(string dataSourceId)
+    {
+        Assert.Throws<ArgumentException>(() => _ = new DataSourceTranscriptStore(this.dataDirectory, dataSourceId));
+    }
+
+    [TestCase("")]
+    [TestCase("../../settings")]
+    [TestCase("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA")]
+    public void AKeyWhichIsNoHashIsRejected(string key)
+    {
+        Assert.Throws<ArgumentException>(() => this.Store().GetPath(key));
+    }
+
+    private DataSourceTranscriptStore Store() => new(this.dataDirectory, DATA_SOURCE_ID);
+
+    private async Task<string> CreateMediaAsync(string relativePath, byte[] content)
+    {
+        var path = Path.Combine(this.userFolder, relativePath);
+        Directory.CreateDirectory(Path.GetDirectoryName(path)!);
+        await File.WriteAllBytesAsync(path, content);
+        return path;
+    }
+}

app/Tests/Tools/Media/MediaSidecarTranscriptsTests.cs

@@ -0,0 +1,57 @@
+using AIStudio.Tools.Media;
+
+namespace AIStudio.Tests.Tools.Media;
+
+/// <summary>
+/// Checks the transcripts which the Batch Processing assistant keeps next to its media files.
+/// </summary>
+[TestFixture]
+public sealed class MediaSidecarTranscriptsTests
+{
+    [TestCase("recording.mp3.transcript.md")]
+    [TestCase("recording.mp3.transcript.md.0123456789abcdef.tmp")]
+    [TestCase("RECORDING.MP3.TRANSCRIPT.MD")]
+    public void GeneratedTranscriptFilesAreRecognized(string fileName)
+    {
+        Assert.That(MediaSidecarTranscripts.IsArtifactPath(fileName), Is.True);
+    }
+
+    [TestCase("notes.md")]
+    [TestCase("recording.mp3")]
+    [TestCase("recording.mp3.transcript.md.backup")]
+    public void OtherFilesAreNoTranscripts(string fileName)
+    {
+        Assert.That(MediaSidecarTranscripts.IsArtifactPath(fileName), Is.False);
+    }
+
+    [Test]
+    public async Task TheTranscriptLiesNextToItsMediaFile()
+    {
+        var mediaPath = Path.Combine(Path.GetTempPath(), "recording.mp3");
+        var slot = await MediaSidecarTranscripts.INSTANCE.ResolveAsync(mediaPath, CancellationToken.None);
+        Assert.That(slot.Path, Is.EqualTo(mediaPath + MediaSidecarTranscripts.FILE_SUFFIX));
+    }
+
+    [Test]
+    public async Task OnlyATranscriptWithContentCanBeReused()
+    {
+        var directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-sidecar-transcript-{Guid.NewGuid():N}");
+        var mediaPath = Path.Combine(directoryPath, "recording.mp3");
+        Directory.CreateDirectory(directoryPath);
+
+        try
+        {
+            Assert.That(MediaSidecarTranscripts.HasReusableTranscript(mediaPath), Is.False, "A missing transcript counted as reusable.");
+
+            await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(mediaPath), string.Empty);
+            Assert.That(MediaSidecarTranscripts.HasReusableTranscript(mediaPath), Is.False, "An empty transcript counted as reusable.");
+
+            await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(mediaPath), "Transcript");
+            Assert.That(MediaSidecarTranscripts.HasReusableTranscript(mediaPath), Is.True);
+        }
+        finally
+        {
+            Directory.Delete(directoryPath, true);
+        }
+    }
+}

app/Tests/Tools/Media/PersistentMediaTranscriptTests.cs

@@ -0,0 +1,252 @@
+using AIStudio.Settings;
+using AIStudio.Tools.Media;
+using AIStudio.Tools.Rust;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools.Media;
+
+/// <summary>
+/// Checks the rules which keep a paid transcription from being paid twice.
+/// </summary>
+[TestFixture]
+public sealed class PersistentMediaTranscriptTests
+{
+    private TranscriptionProvider? provider;
+
+    private string directoryPath = string.Empty;
+
+    private string mediaPath = string.Empty;
+
+    [SetUp]
+    public async Task CreateMediaFile()
+    {
+        this.directoryPath = Path.Combine(Path.GetTempPath(), $"ai-studio-persistent-transcript-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(this.directoryPath);
+        this.mediaPath = Path.Combine(this.directoryPath, "recording.mp3");
+        this.provider = new TranscriptionProvider { Id = "test-transcription-provider" };
+        await File.WriteAllBytesAsync(this.mediaPath, [1, 2, 3]);
+    }
+
+    [TearDown]
+    public void DeleteMediaFile()
+    {
+        if (Directory.Exists(this.directoryPath))
+            Directory.Delete(this.directoryPath, true);
+    }
+
+    [Test]
+    public async Task AnExistingTranscriptIsUsedWithoutAskingTheProvider()
+    {
+        await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(this.mediaPath), "Transcript");
+        var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("New transcript"));
+
+        var result = await this.ReadOrCreateAsync(transcriber, this.provider);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY));
+            Assert.That(result.Reused, Is.True);
+            Assert.That(transcriber.Calls, Is.Zero);
+        });
+    }
+
+    [Test]
+    public async Task AnExistingTranscriptIsUsedEvenWithoutAProvider()
+    {
+        await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(this.mediaPath), "Transcript");
+        var result = await this.ReadOrCreateAsync(new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("New transcript")), null);
+        Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY));
+    }
+
+    [Test]
+    public async Task WithoutATranscriptAndAProviderNothingIsTranscribed()
+    {
+        var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Transcript"));
+
+        var result = await this.ReadOrCreateAsync(transcriber, null);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.NO_PROVIDER));
+            Assert.That(transcriber.Calls, Is.Zero);
+        });
+    }
+
+    [Test]
+    public async Task ANewTranscriptIsStoredAndPaidForOnlyOnce()
+    {
+        var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Transcript"));
+
+        var first = await this.ReadOrCreateAsync(transcriber, this.provider);
+        var second = await this.ReadOrCreateAsync(transcriber, this.provider);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(first.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY));
+            Assert.That(first.Reused, Is.False);
+            Assert.That(File.ReadAllText(first.TranscriptPath), Is.EqualTo("Transcript"));
+            Assert.That(second.Reused, Is.True);
+            Assert.That(transcriber.Calls, Is.EqualTo(1));
+            Assert.That(Directory.GetFiles(this.directoryPath, "*.tmp"), Is.Empty);
+        });
+    }
+
+    [Test]
+    public async Task AnEmptyTranscriptIsNeverStored()
+    {
+        var result = await this.ReadOrCreateAsync(new CountingTranscriber(() => MediaTranscriptionResult.Succeeded(" \n ")), this.provider);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.EMPTY));
+            Assert.That(File.Exists(MediaSidecarTranscripts.GetFilePath(this.mediaPath)), Is.False);
+        });
+    }
+
+    [Test]
+    public async Task ATranscriptHoldingOnlyWhiteSpaceIsReplaced()
+    {
+        await File.WriteAllTextAsync(MediaSidecarTranscripts.GetFilePath(this.mediaPath), "  \n ");
+        var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Transcript"));
+
+        var result = await this.ReadOrCreateAsync(transcriber, this.provider);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY));
+            Assert.That(transcriber.Calls, Is.EqualTo(1));
+            Assert.That(File.ReadAllText(result.TranscriptPath), Is.EqualTo("Transcript"));
+        });
+    }
+
+    [Test]
+    public async Task AFileWhichChangesDuringItsTranscriptionKeepsNoTranscript()
+    {
+        var transcriber = new CountingTranscriber(() =>
+        {
+            File.WriteAllBytes(this.mediaPath, [1, 2, 3, 4]);
+            return MediaTranscriptionResult.Succeeded("Transcript of the old file");
+        });
+
+        var result = await this.ReadOrCreateAsync(transcriber, this.provider);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.SOURCE_CHANGED));
+            Assert.That(File.Exists(MediaSidecarTranscripts.GetFilePath(this.mediaPath)), Is.False);
+        });
+    }
+
+    [Test]
+    public async Task ATranscriptWhichArrivesAfterACancellationIsStillStored()
+    {
+        using var cancellation = new CancellationTokenSource();
+        var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Paid transcript"), cancellation.Cancel);
+
+        var result = await PersistentMediaTranscript.ReadOrCreateAsync(MediaSidecarTranscripts.INSTANCE, this.mediaPath, this.provider, transcriber.TranscribeAsync, NullLogger.Instance, cancellation.Token);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.READY));
+            Assert.That(File.ReadAllText(MediaSidecarTranscripts.GetFilePath(this.mediaPath)), Is.EqualTo("Paid transcript"));
+        });
+    }
+
+    [Test]
+    public async Task ACancelledTranscriptionStoresNothing()
+    {
+        var result = await this.ReadOrCreateAsync(new CountingTranscriber(MediaTranscriptionResult.Cancelled), this.provider);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.CANCELLED));
+            Assert.That(File.Exists(MediaSidecarTranscripts.GetFilePath(this.mediaPath)), Is.False);
+        });
+    }
+
+    [Test]
+    public async Task AFailedTranscriptionKeepsTheReasonOfTheMediaPipeline()
+    {
+        var result = await this.ReadOrCreateAsync(new CountingTranscriber(() => MediaTranscriptionResult.Failed("The container is damaged.", MediaJobErrorCode.DAMAGED_CONTAINER)), this.provider);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.TRANSCRIPTION_FAILED));
+            Assert.That(result.ErrorCode, Is.EqualTo(MediaJobErrorCode.DAMAGED_CONTAINER));
+            Assert.That(result.UserMessage, Is.EqualTo("The container is damaged."));
+        });
+    }
+
+    [Test]
+    public async Task AFileTheProviderRefusedIsReportedAsSuch()
+    {
+        var result = await this.ReadOrCreateAsync(new CountingTranscriber(() => MediaTranscriptionResult.Rejected("The file is too large.")), this.provider);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.REJECTED));
+            Assert.That(result.UserMessage, Is.EqualTo("The file is too large."));
+            Assert.That(File.Exists(MediaSidecarTranscripts.GetFilePath(this.mediaPath)), Is.False);
+        });
+    }
+
+    [Test]
+    public async Task ATranscriptWhichCannotBeStoredIsReportedAndLeavesNoTemporaryFile()
+    {
+        //
+        // A file where the folder of the transcript should be: writing fails for every user, root
+        // included, unlike a folder without write permission.
+        //
+        var blockingFilePath = Path.Combine(this.directoryPath, "not-a-folder");
+        await File.WriteAllTextAsync(blockingFilePath, "occupied");
+        var location = new FixedLocation(Path.Combine(blockingFilePath, "transcript.md"));
+
+        var result = await PersistentMediaTranscript.ReadOrCreateAsync(location, this.mediaPath, this.provider, new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Transcript")).TranscribeAsync, NullLogger.Instance, CancellationToken.None);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.STORE_FAILED));
+            Assert.That(Directory.GetFiles(this.directoryPath, "*.tmp", SearchOption.AllDirectories), Is.Empty);
+        });
+    }
+
+    [Test]
+    public async Task AMissingMediaFileIsReportedWithoutAskingTheProvider()
+    {
+        File.Delete(this.mediaPath);
+        var transcriber = new CountingTranscriber(() => MediaTranscriptionResult.Succeeded("Transcript"));
+
+        var result = await this.ReadOrCreateAsync(transcriber, this.provider);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Status, Is.EqualTo(PersistentMediaTranscriptStatus.SOURCE_UNREADABLE));
+            Assert.That(transcriber.Calls, Is.Zero);
+        });
+    }
+
+    private Task<PersistentMediaTranscriptResult> ReadOrCreateAsync(CountingTranscriber transcriber, TranscriptionProvider? transcriptionProvider) => PersistentMediaTranscript.ReadOrCreateAsync(MediaSidecarTranscripts.INSTANCE, this.mediaPath, transcriptionProvider, transcriber.TranscribeAsync, NullLogger.Instance, CancellationToken.None);
+
+    /// <summary>
+    /// Stands in for the provider and counts how often it was asked.
+    /// </summary>
+    /// <param name="answer">Creates the answer of the provider.</param>
+    /// <param name="whileAnswering">Runs while the provider is answering, e.g., to cancel the run in the middle of an upload.</param>
+    private sealed class CountingTranscriber(Func<MediaTranscriptionResult> answer, Action? whileAnswering = null)
+    {
+        public int Calls { get; private set; }
+
+        public Task<MediaTranscriptionResult> TranscribeAsync(string mediaPath, TranscriptionProvider provider, CancellationToken token)
+        {
+            this.Calls++;
+            whileAnswering?.Invoke();
+            return Task.FromResult(answer());
+        }
+    }
+
+    private sealed class FixedLocation(string transcriptPath) : IMediaTranscriptLocation
+    {
+        public Task<MediaTranscriptSlot> ResolveAsync(string mediaPath, CancellationToken token) => Task.FromResult(new MediaTranscriptSlot(transcriptPath, "fixed"));
+    }
+}

app/Tests/Tools/Media/TranscriptCleanupTests.cs

@@ -0,0 +1,110 @@
+using AIStudio.Tools.Media;
+using AIStudio.Tools.Services;
+using AIStudio.Tools.Services.Indexing;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools.Media;
+
+/// <summary>
+/// Checks that the transcript of a recording leaves the data source together with the recording.
+/// </summary>
+/// <remarks>
+/// A transcript holds what was said. Once its recording is gone from the data source, keeping it
+/// below the data directory would keep that text, unseen by the user.
+/// </remarks>
+[TestFixture]
+public sealed class TranscriptCleanupTests
+{
+    private static readonly string KEY_A = new('a', 64);
+
+    private static readonly string KEY_B = new('b', 64);
+
+    private static readonly string KEY_C = new('c', 64);
+
+    private string dataDirectory = string.Empty;
+
+    private DataSourceTranscriptStore store = null!;
+
+    [SetUp]
+    public async Task CreateTranscripts()
+    {
+        this.dataDirectory = Path.Combine(Path.GetTempPath(), $"ai-studio-transcript-cleanup-{Guid.NewGuid():N}");
+        this.store = new DataSourceTranscriptStore(this.dataDirectory, Guid.NewGuid().ToString());
+        foreach (var key in new[] { KEY_A, KEY_B, KEY_C })
+        {
+            var path = this.store.GetPath(key);
+            Directory.CreateDirectory(Path.GetDirectoryName(path)!);
+            await File.WriteAllTextAsync(path, $"Transcript {key[0]}");
+        }
+    }
+
+    [TearDown]
+    public void DeleteTranscripts()
+    {
+        if (Directory.Exists(this.dataDirectory))
+            Directory.Delete(this.dataDirectory, true);
+    }
+
+    [Test]
+    public void TheTranscriptOfARemovedRecordingIsDeleted()
+    {
+        var manifest = Manifest(("/data/kept.mp3", KEY_A));
+
+        FileSourceIndexer.ForgetTranscript(this.store, manifest, KEY_B, NullLogger.Instance);
+
+        Assert.That(this.store.ListKeys(), Is.EquivalentTo(new[] { KEY_A, KEY_C }));
+    }
+
+    [Test]
+    public void ATranscriptWhichAnotherRecordingSharesIsKept()
+    {
+        var manifest = Manifest(("/data/copy-of-the-removed-recording.mp3", KEY_B));
+
+        FileSourceIndexer.ForgetTranscript(this.store, manifest, KEY_B, NullLogger.Instance);
+
+        Assert.That(this.store.ListKeys(), Does.Contain(KEY_B), "Two recordings with the same content share one transcript.");
+    }
+
+    [Test]
+    public void AFileWithoutTranscriptDeletesNothing()
+    {
+        FileSourceIndexer.ForgetTranscript(this.store, new DataSourceEmbeddingManifest(), null, NullLogger.Instance);
+        Assert.That(this.store.ListKeys(), Has.Count.EqualTo(3));
+    }
+
+    [Test]
+    public async Task ACompleteRunDeletesWhatNoFileUses()
+    {
+        var temporaryPath = $"{this.store.GetPath(KEY_A)}.{Guid.NewGuid():N}.tmp";
+        await File.WriteAllTextAsync(temporaryPath, "Interrupted");
+        var manifest = Manifest(("/data/meeting.mp3", KEY_A), ("/data/report.pdf", null));
+
+        FileSourceIndexer.DeleteUnusedTranscripts(this.store, manifest, NullLogger.Instance);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(this.store.ListKeys(), Is.EqualTo(new[] { KEY_A }));
+            Assert.That(File.Exists(temporaryPath), Is.False, "A temporary file of an interrupted write is no transcript of anything.");
+        });
+    }
+
+    [Test]
+    public void WithoutAStoreNothingHappens()
+    {
+        Assert.DoesNotThrow(() =>
+        {
+            FileSourceIndexer.DeleteUnusedTranscripts(null, new DataSourceEmbeddingManifest(), NullLogger.Instance);
+            FileSourceIndexer.ForgetTranscript(null, new DataSourceEmbeddingManifest(), KEY_A, NullLogger.Instance);
+        });
+    }
+
+    private static DataSourceEmbeddingManifest Manifest(params (string Path, string? TranscriptKey)[] files)
+    {
+        var manifest = new DataSourceEmbeddingManifest();
+        foreach (var (path, transcriptKey) in files)
+            manifest.Files[path] = new EmbeddedFileRecord("fingerprint", 16, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, 1, transcriptKey);
+
+        return manifest;
+    }
+}

app/Tests/Tools/MediaIndexingTests.cs

@@ -0,0 +1,258 @@
+using AIStudio.Settings;
+using AIStudio.Settings.DataModel;
+using AIStudio.Tools;
+using AIStudio.Tools.Media;
+using AIStudio.Tools.Rust;
+using AIStudio.Tools.Services;
+using AIStudio.Tools.Services.Indexing;
+
+using Microsoft.Extensions.Logging.Abstractions;
+
+namespace AIStudio.Tests.Tools;
+
+/// <summary>
+/// Checks which audio and video files a local data source transcribes, and how it treats those it cannot.
+/// </summary>
+/// <remarks>
+/// Transcribing costs money with every cloud provider. A media file is therefore only sent when a
+/// provider exists which the data source allows, and a file which cannot be transcribed for a
+/// reason of its own is not sent again until it changes.
+/// </remarks>
+public sealed class MediaIndexingTests
+{
+    private string testRoot = string.Empty;
+
+    [SetUp]
+    public void CreateTestRoot()
+    {
+        this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-media-indexing-{Guid.NewGuid():N}");
+        Directory.CreateDirectory(this.testRoot);
+    }
+
+    [TearDown]
+    public void DeleteTestRoot()
+    {
+        if (Directory.Exists(this.testRoot))
+            Directory.Delete(this.testRoot, true);
+    }
+
+    [Test]
+    public void AnAllowedProviderTranscribesASelectedRecording()
+    {
+        var recording = this.CreateFile("meeting.mp3", 16);
+
+        var result = CreateIndexer().GetInputFiles(this.Folder("*"), new DataSourceEmbeddingManifest(), Allowed());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording }));
+            Assert.That(result.IgnoredFiles, Is.Empty);
+        });
+    }
+
+    [Test]
+    public void WithoutAProviderANewRecordingWaitsForOne()
+    {
+        var recording = this.CreateFile("meeting.mp3", 16);
+
+        var result = CreateIndexer().GetInputFiles(this.Folder("*"), new DataSourceEmbeddingManifest(), MediaTranscriptionAvailability.MISSING);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Files, Is.Empty);
+            Assert.That(result.IgnoredFiles, Is.EqualTo(new[] { new DataSourceEmbeddingIgnoredFile(recording, DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING) }));
+        });
+    }
+
+    [Test]
+    public void AProviderBelowTheConfidenceLevelHearsNothing()
+    {
+        var recording = this.CreateFile("meeting.mp4", 16);
+
+        var result = CreateIndexer().GetInputFiles(this.Folder("*"), new DataSourceEmbeddingManifest(), MediaTranscriptionAvailability.NOT_ALLOWED);
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Files, Is.Empty, "The recording would have gone to a provider the data source does not allow.");
+            Assert.That(result.IgnoredFiles, Is.EqualTo(new[] { new DataSourceEmbeddingIgnoredFile(recording, DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_NOT_ALLOWED) }));
+        });
+    }
+
+    [Test]
+    public void AnIndexedRecordingStaysWhenTheProviderIsGone()
+    {
+        var recording = this.CreateFile("meeting.mp3", 16);
+        var manifest = new DataSourceEmbeddingManifest();
+        manifest.Files[recording] = new EmbeddedFileRecord(FileSourceIndexer.BuildFileMetadataHash(new FileInfo(recording)), 16, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, 3, new string('a', 64));
+
+        var result = CreateIndexer().GetInputFiles(this.Folder("*"), manifest, MediaTranscriptionAvailability.MISSING);
+
+        Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording }), "An unchanged recording is not read, so it needs no provider, and leaving it out would remove it from the index.");
+    }
+
+    [Test]
+    public void ARecordingWhichFailedForGoodStaysWithItsReasonWhenTheProviderIsGone()
+    {
+        var recording = this.CreateFile("silence.wav", 16);
+        var manifest = new DataSourceEmbeddingManifest();
+        manifest.PermanentFailures[recording] = new PermanentIndexingFailureRecord(FileSourceIndexer.BuildFileMetadataHash(new FileInfo(recording)), FileExtractionErrorCode.NO_CONTENT, "No content", DateTimeOffset.UtcNow);
+
+        var result = CreateIndexer().GetInputFiles(this.Folder("*"), manifest, MediaTranscriptionAvailability.NOT_ALLOWED);
+
+        Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording }), "The run lists it with its stored reason instead of reading it.");
+    }
+
+    [Test]
+    public void AChangedRecordingNeedsAProviderAgain()
+    {
+        var recording = this.CreateFile("meeting.mp3", 16);
+        var manifest = new DataSourceEmbeddingManifest();
+        manifest.Files[recording] = new EmbeddedFileRecord("THE-FINGERPRINT-BEFORE-THE-CHANGE", 8, DateTimeOffset.UtcNow, DateTimeOffset.UtcNow, 3, new string('a', 64));
+
+        var result = CreateIndexer().GetInputFiles(this.Folder("*"), manifest, MediaTranscriptionAvailability.MISSING);
+
+        Assert.That(result.IgnoredFiles, Is.EqualTo(new[] { new DataSourceEmbeddingIgnoredFile(recording, DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING) }));
+    }
+
+    [Test]
+    public void ARecordingAboveTheLimitIsNeverTranscribed()
+    {
+        var recording = this.CreateFile("conference.mkv", FolderFileSelection.MegabytesToBytes(1) + 1);
+        var dataSource = this.Folder("*") with { MaxFileSizeMegabytes = 1 };
+
+        var result = CreateIndexer().GetInputFiles(dataSource, new DataSourceEmbeddingManifest(), Allowed());
+
+        Assert.That(result.IgnoredFiles, Is.EqualTo(new[] { new DataSourceEmbeddingIgnoredFile(recording, DataSourceEmbeddingIgnoreReason.TOO_LARGE) }));
+    }
+
+    [Test]
+    public void ImagesAreStillLeftOut()
+    {
+        var image = this.CreateFile("photo.png", 16);
+
+        var result = CreateIndexer().GetInputFiles(this.Folder("*"), new DataSourceEmbeddingManifest(), Allowed());
+
+        Assert.That(result.IgnoredFiles, Is.EqualTo(new[] { new DataSourceEmbeddingIgnoredFile(image, DataSourceEmbeddingIgnoreReason.IMAGE_NOT_SUPPORTED) }));
+    }
+
+    [TestCase("meeting.mp3")]
+    [TestCase("meeting.mp4")]
+    [TestCase("meeting.wav")]
+    [TestCase("meeting.webm")]
+    public void TheDefaultPatternsSelectNoRecording(string fileName)
+    {
+        var recording = this.CreateFile(fileName, 16);
+
+        var result = CreateIndexer().GetInputFiles(this.Folder(DataSourceLocalDirectory.DEFAULT_FILE_PATTERNS), new DataSourceEmbeddingManifest(), Allowed());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(result.Files, Is.Empty, "Existing data sources must not start transcribing, and paying for it, after an update.");
+            Assert.That(FileNamePatterns.Matches(recording, FileNamePatterns.Parse(":text:")), Is.False);
+        });
+    }
+
+    [TestCase("*")]
+    [TestCase("*.mp3")]
+    [TestCase("*.pdf;*.mp3")]
+    public void APatternWhichNamesRecordingsSelectsThem(string patterns)
+    {
+        var recording = this.CreateFile("meeting.mp3", 16);
+
+        var result = CreateIndexer().GetInputFiles(this.Folder(patterns), new DataSourceEmbeddingManifest(), Allowed());
+
+        Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording }));
+    }
+
+    [Test]
+    public void TheBatchTranscriptOfASelectedRecordingIsNoDocumentOfItsOwn()
+    {
+        var recording = this.CreateFile("meeting.mp3", 16);
+        var transcript = this.CreateFile(Path.GetFileName(MediaSidecarTranscripts.GetFilePath(recording)), 16);
+
+        var everything = CreateIndexer().GetInputFiles(this.Folder("*"), new DataSourceEmbeddingManifest(), Allowed());
+        var markdownOnly = CreateIndexer().GetInputFiles(this.Folder("*.md"), new DataSourceEmbeddingManifest(), Allowed());
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(everything.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording }), "The transcript would duplicate the text of the recording.");
+            Assert.That(markdownOnly.Files.Select(file => file.FullName), Is.EqualTo(new[] { transcript }), "Without the recording, the transcript is a document like every other.");
+        });
+    }
+
+    [Test]
+    public void ASingleRecordingCanBeADataSource()
+    {
+        var recording = this.CreateFile("interview.m4a", 16);
+        var dataSource = new DataSourceLocalFile { Type = DataSourceType.LOCAL_FILE, FilePath = recording };
+
+        var result = CreateIndexer().GetInputFiles(dataSource, new DataSourceEmbeddingManifest(), Allowed());
+
+        Assert.That(result.Files.Select(file => file.FullName), Is.EqualTo(new[] { recording }));
+    }
+
+    [Test]
+    public void AReadyTranscriptIsRead()
+    {
+        var transcript = PersistentMediaTranscriptResult.Ready(new MediaTranscriptSlot("/data/mediaTranscripts/source/key.md", "key"), reused: true);
+        Assert.That(FileSourceIndexer.GetTranscriptOrThrow(transcript, CancellationToken.None), Is.EqualTo(new MediaTranscriptSlot("/data/mediaTranscripts/source/key.md", "key")));
+    }
+
+    [TestCase(PersistentMediaTranscriptStatus.EMPTY)]
+    [TestCase(PersistentMediaTranscriptStatus.NO_AUDIBLE_SIGNAL)]
+    public void ARecordingWithoutTextIsRecordedForGood(PersistentMediaTranscriptStatus status)
+    {
+        var transcript = PersistentMediaTranscriptResult.Unavailable(status, null, "Nothing to read.");
+
+        var exception = Assert.Throws<FileExtractionException>(() => FileSourceIndexer.GetTranscriptOrThrow(transcript, CancellationToken.None));
+        Assert.That(exception!.Code, Is.EqualTo(FileExtractionErrorCode.NO_CONTENT), "Asked again with every run, the provider would be paid again with every run.");
+    }
+
+    [Test]
+    public void ARecordingTheMediaPipelineCannotReadIsRecordedForGood()
+    {
+        var transcript = PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.TRANSCRIPTION_FAILED, null, "The container is damaged.", MediaJobErrorCode.DAMAGED_CONTAINER);
+
+        var exception = Assert.Throws<FileExtractionException>(() => FileSourceIndexer.GetTranscriptOrThrow(transcript, CancellationToken.None));
+        Assert.That(exception!.Code, Is.EqualTo(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED));
+    }
+
+    [Test]
+    public void ARecordingTheProviderRefusedIsRecordedForGood()
+    {
+        var transcript = PersistentMediaTranscriptResult.Unavailable(PersistentMediaTranscriptStatus.REJECTED, null, "The provider 'Cloud' reported an error: Audio file might be corrupted or unsupported");
+
+        var exception = Assert.Throws<FileExtractionException>(() => FileSourceIndexer.GetTranscriptOrThrow(transcript, CancellationToken.None));
+        Assert.That(exception!.Code, Is.EqualTo(FileExtractionErrorCode.MEDIA_PROCESSING_FAILED), "Tried again with every run, the same file would be uploaded again with every run.");
+    }
+
+    [TestCase(PersistentMediaTranscriptStatus.TRANSCRIPTION_FAILED)]
+    [TestCase(PersistentMediaTranscriptStatus.STORE_FAILED)]
+    [TestCase(PersistentMediaTranscriptStatus.SOURCE_CHANGED)]
+    [TestCase(PersistentMediaTranscriptStatus.NO_PROVIDER)]
+    public void AFailureWhichMayPassIsTriedAgain(PersistentMediaTranscriptStatus status)
+    {
+        var transcript = PersistentMediaTranscriptResult.Unavailable(status, null, "Try again later.");
+        Assert.Throws<IOException>(() => FileSourceIndexer.GetTranscriptOrThrow(transcript, CancellationToken.None));
+    }
+
+    private DataSourceLocalDirectory Folder(string patterns) => new()
+    {
+        Type = DataSourceType.LOCAL_DIRECTORY,
+        Path = this.testRoot,
+        FilePatterns = patterns,
+    };
+
+    private string CreateFile(string fileName, long length)
+    {
+        var filePath = Path.Combine(this.testRoot, fileName);
+        using var stream = File.Create(filePath);
+        stream.SetLength(length);
+        return filePath;
+    }
+
+    private static MediaTranscriptionAvailability Allowed() => MediaTranscriptionAvailability.Allowed(new TranscriptionProvider { Id = "test-transcription-provider" });
+
+    // Finding the files reads none of them, so it needs neither the runtime nor a transcription provider:
+    private static FileSourceIndexer CreateIndexer() => new(null!, null!, null!, null!, NullLogger.Instance);
+}

app/Tests/Tools/QueuedStatusTests.cs

@@ -82,7 +82,7 @@ public sealed class QueuedStatusTests
         PendingRemovalCount: 42,
         LastSyncUtc: NOW)
     {
-        IgnoredFiles = [new("/tmp/test-data/interview.mp3", DataSourceEmbeddingIgnoreReason.MEDIA_NOT_SUPPORTED)],
+        IgnoredFiles = [new("/tmp/test-data/interview.mp3", DataSourceEmbeddingIgnoreReason.TRANSCRIPTION_PROVIDER_MISSING)],
         FilePatternExclusions = [new(".png", 5)],
     };
 

app/Tests/Tools/TerminalLoggerTests.cs

@@ -45,4 +45,30 @@ public sealed class TerminalLoggerTests
             CultureInfo.CurrentCulture = previousCulture;
         }
     }
+
+    [Test]
+    public void AnExceptionIsDescribedTogetherWithItsCause()
+    {
+        var exception = new HttpRequestException("Error while copying content to a stream.", new IOException("Unable to write data to the transport connection.", new InvalidOperationException("Connection reset by peer.")));
+
+        Assert.That(TerminalLogger.DescribeException(exception), Is.EqualTo("Error while copying content to a stream. ---> Unable to write data to the transport connection. ---> Connection reset by peer."));
+    }
+
+    [Test]
+    public void AnExceptionWithoutCauseKeepsItsMessage()
+    {
+        Assert.Multiple(() =>
+        {
+            Assert.That(TerminalLogger.DescribeException(new InvalidOperationException("Something failed.")), Is.EqualTo("Something failed."));
+            Assert.That(TerminalLogger.DescribeException(null), Is.Null);
+        });
+    }
+
+    [Test]
+    public void TheCausesOfAnAggregateExceptionAreNotRepeated()
+    {
+        var exception = new AggregateException(new InvalidOperationException("First failure."), new InvalidOperationException("Second failure."));
+
+        Assert.That(TerminalLogger.DescribeException(exception), Is.EqualTo(exception.Message));
+    }
 }

app/Tests/Tools/ToolCalling/MailDraftSyncTests.cs

@@ -117,5 +117,5 @@ public sealed class MailDraftSyncTests : ToolRegistryTestBase
     }
 
     // Queueing a run needs none of the services which carry it out:
-    private DataSourceEmbeddingService CreateService() => new(this.SettingsManager, null!, null!, null!, NullLogger<DataSourceEmbeddingService>.Instance);
+    private DataSourceEmbeddingService CreateService() => new(this.SettingsManager, null!, null!, null!, null!, NullLogger<DataSourceEmbeddingService>.Instance);
 }

app/Tests/Tools/UnreachableDataSourceTests.cs

@@ -205,7 +205,7 @@ public sealed class UnreachableDataSourceTests
             NullLogger.Instance);
 
         // A run which reaches no file reads none, so it needs neither the runtime nor the chunker:
-        var indexer = new FileSourceIndexer(settingsManager, null!, null!, NullLogger.Instance);
+        var indexer = new FileSourceIndexer(settingsManager, null!, null!, null!, NullLogger.Instance);
         var outcome = await indexer.ProcessAsync(context, DataSourceEmbeddingRefreshMode.HASH_CHECK, CancellationToken.None);
 
         return new(outcome, statuses, manifest);

app/Tests/Tools/Validation/CloudProcessingTests.cs

@@ -0,0 +1,68 @@
+using AIStudio.Tools;
+using AIStudio.Tools.Validation;
+
+namespace AIStudio.Tests.Tools.Validation;
+
+/// <summary>
+/// Checks what the dialogs of local data sources tell about the cloud, and when they ask again.
+/// </summary>
+/// <remarks>
+/// A data source can send its text to an embedding provider and its recordings to a transcription
+/// provider. The user acknowledges what leaves the computer, and an acknowledgment for the one must
+/// not count for the other.
+/// </remarks>
+[TestFixture]
+public sealed class CloudProcessingTests
+{
+    [TestCase(CloudProcessing.EMBEDDING, CloudProcessing.EMBEDDING, true)]
+    [TestCase(CloudProcessing.EMBEDDING, CloudProcessing.EMBEDDING | CloudProcessing.TRANSCRIPTION, false)]
+    [TestCase(CloudProcessing.EMBEDDING, CloudProcessing.TRANSCRIPTION, false)]
+    [TestCase(CloudProcessing.EMBEDDING | CloudProcessing.TRANSCRIPTION, CloudProcessing.TRANSCRIPTION, true)]
+    [TestCase(CloudProcessing.NONE, CloudProcessing.EMBEDDING, false)]
+    [TestCase(CloudProcessing.NONE, CloudProcessing.NONE, true)]
+    public void AnAcknowledgmentCoversOnlyWhatItWasGivenFor(CloudProcessing acknowledged, CloudProcessing current, bool expected)
+    {
+        Assert.That(acknowledged.Covers(current), Is.EqualTo(expected));
+    }
+
+    [Test]
+    public void ARecordingSentToTheCloudNeedsAnAcknowledgment()
+    {
+        var validation = new DataSourceValidation { GetSelectedCloudTranscription = () => true };
+
+        Assert.Multiple(() =>
+        {
+            Assert.That(validation.ValidateUserAcknowledgedCloudEmbedding(false), Is.Not.Null);
+            Assert.That(validation.ValidateUserAcknowledgedCloudEmbedding(true), Is.Null);
+        });
+    }
+
+    [Test]
+    public void NothingSentToTheCloudNeedsNoAcknowledgment()
+    {
+        Assert.That(new DataSourceValidation().ValidateUserAcknowledgedCloudEmbedding(false), Is.Null);
+    }
+
+    [TestCase("*")]
+    [TestCase("*.mp3")]
+    [TestCase("*.MP4")]
+    [TestCase("meeting*")]
+    [TestCase("*.*")]
+    [TestCase("*.m?a")]
+    [TestCase(":documents:;*.webm")]
+    public void APatternWhichMaySelectRecordingsCounts(string patterns)
+    {
+        Assert.That(FileNamePatterns.MaySelectMedia(FileNamePatterns.Parse(patterns)), Is.True);
+    }
+
+    [TestCase(":documents:")]
+    [TestCase(":text:")]
+    [TestCase("*.pdf;*.docx")]
+    [TestCase(".gitignore")]
+    [TestCase("Makefile")]
+    [TestCase("report_*.txt")]
+    public void APatternWhichSelectsNoRecordingDoesNotCount(string patterns)
+    {
+        Assert.That(FileNamePatterns.MaySelectMedia(FileNamePatterns.Parse(patterns)), Is.False, "The dialog would warn about a transcription which never happens.");
+    }
+}