Added transcription of audio and video files to local data sources (#1081)

Co-authored-by: Paul Koudelka <106623909+PaulKoudelka@users.noreply.github.com>

app/MindWork AI Studio/Tools/DataBackup/DataBackupContents.cs

@@ -46,6 +46,7 @@ public static class DataBackupContents
         new(DataBackupLocation.DATA_DIRECTORY, "visualBriefings/", DataBackupEntryKind.VISUAL_BRIEFINGS),
         new(DataBackupLocation.DATA_DIRECTORY, "databases/sqlite/", DataBackupEntryKind.DATA_SOURCE_INDEX),
         new(DataBackupLocation.DATA_DIRECTORY, "databases/vector_database/stores/", DataBackupEntryKind.VECTOR_STORES),
+        new(DataBackupLocation.DATA_DIRECTORY, "mediaTranscripts/", DataBackupEntryKind.MEDIA_TRANSCRIPTS),
 
         new(DataBackupLocation.CONFIG_DIRECTORY, "settings.v*.json", DataBackupEntryKind.SETTINGS_OF_OTHER_VERSIONS),
         new(DataBackupLocation.DATA_DIRECTORY, "plugins/.internal/", DataBackupEntryKind.BUILT_IN_PLUGINS),

app/MindWork AI Studio/Tools/DataBackup/DataBackupEntry.cs

@@ -18,7 +18,20 @@ public sealed record DataBackupEntry(DataBackupLocation Location, string Relativ
         or DataBackupEntryKind.TOKENIZERS
         or DataBackupEntryKind.VISUAL_BRIEFINGS
         or DataBackupEntryKind.DATA_SOURCE_INDEX
-        or DataBackupEntryKind.VECTOR_STORES;
+        or DataBackupEntryKind.VECTOR_STORES
+        or DataBackupEntryKind.MEDIA_TRANSCRIPTS;
+
+    /// <summary>
+    /// Whether a backup takes the entry from the copies it makes while the indexing pauses, rather than from the data folder.
+    /// </summary>
+    /// <remarks>
+    /// The indexing writes to these entries, and they refer to each other: the index knows which
+    /// transcript belongs to which file. A backup therefore copies them together while nothing
+    /// writes, cf. DataBackupService.CopyIndexesAsync, and a restore replaces them only together.
+    /// </remarks>
+    public bool ComesFromIndexCopies => this.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX
+        or DataBackupEntryKind.VECTOR_STORES
+        or DataBackupEntryKind.MEDIA_TRANSCRIPTS;
 
     /// <summary>
     /// Whether the entry is a folder rather than a single file.

app/MindWork AI Studio/Tools/DataBackup/DataBackupEntryKind.cs

@@ -45,6 +45,11 @@ public enum DataBackupEntryKind
     /// </summary>
     VECTOR_STORES,
 
+    /// <summary>
+    /// The transcripts of the audio and video files of the data sources. Making them again costs money.
+    /// </summary>
+    MEDIA_TRANSCRIPTS,
+
     /// <summary>
     /// The copies of the settings which every version of AI Studio keeps for itself.
     /// </summary>

app/MindWork AI Studio/Tools/DataBackup/DataBackupManifest.cs

@@ -9,7 +9,7 @@ namespace AIStudio.Tools.DataBackup;
 /// <param name="CreatedAtUtc">When the backup was written.</param>
 /// <param name="SourceDataDirectory">The data folder the backup was made from, for whoever wonders where it came from.</param>
 /// <param name="SourceConfigDirectory">The configuration folder the backup was made from.</param>
-/// <param name="IncludesDataSourceIndexes">Whether the backup holds the indexes of the data sources. Without them, a restore keeps the indexes there are.</param>
+/// <param name="IncludesDataSourceIndexes">Whether the backup holds the indexes of the data sources and the transcripts of their media files. Without them, a restore keeps the indexes and transcripts there are.</param>
 /// <param name="Entries">Every file of the backup, the manifest aside.</param>
 public sealed record DataBackupManifest(
     int FormatVersion,

app/MindWork AI Studio/Tools/DataBackup/DataBackupWriter.cs

@@ -101,7 +101,7 @@ public static class DataBackupWriter
         var files = new List<BackupFile>();
         foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY))
         {
-            var root = entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX or DataBackupEntryKind.VECTOR_STORES
+            var root = entry.ComesFromIndexCopies
                 ? sources.IndexCopiesDirectory
                 : sources.DataDirectory;
 

app/MindWork AI Studio/Tools/DataBackup/DataRestorePreparer.cs

@@ -147,15 +147,16 @@ public static class DataRestorePreparer
     /// </summary>
     /// <remarks>
     /// An element the backup holds nothing for stays as it is, since nothing tells whether it was empty
-    /// or whether the version of AI Studio which made the backup knew it at all. The indexes are
-    /// replaced only when the backup holds both of them, cf. DataBackupManifest.IncludesDataSourceIndexes.
+    /// or whether the version of AI Studio which made the backup knew it at all. The indexes and the
+    /// media transcripts are replaced only when the backup holds the indexes, cf.
+    /// DataBackupManifest.IncludesDataSourceIndexes and DataBackupEntry.ComesFromIndexCopies.
     /// </remarks>
     private static List<DataRestoreElement> DetermineElements(DataBackupManifest manifest, DataRestoreTarget target)
     {
         var elements = new List<DataRestoreElement>();
         foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded))
         {
-            if (entry.Kind is DataBackupEntryKind.DATA_SOURCE_INDEX or DataBackupEntryKind.VECTOR_STORES && !manifest.IncludesDataSourceIndexes)
+            if (entry.ComesFromIndexCopies && !manifest.IncludesDataSourceIndexes)
                 continue;
 
             var entryName = DataBackupFormat.EntryName(entry.Location, entry.RelativePath);