MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-08-11 20:12:12 +00:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

Added audio and video transcription for chats and assistants

Browse Source
This commit is contained in:
Thorsten Sommer 2026-07-13 21:59:40 +02:00
parent 5dc6282908
commit 0c02cf427d
Signed by untrusted user who does not match committer: tsommer
GPG Key ID: 371BBA77A02C0108
36 changed files with 2066 additions and 143 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
app/MindWork AI Studio/Assistants/AssistantBase.razor
View File

@ -31,6 +31,7 @@
@if (this.Body is not null)
{
<MediaTranscriptionStatus/>
<CascadingValue Value="@this">
<CascadingValue Value="@this.Component">
@this.Body
@ -38,7 +39,7 @@
</CascadingValue>
<MudStack Row="true" AlignItems="AlignItems.Center" StretchItems="StretchItems.Start" Class="mb-3">
<MudButton Disabled="@(this.SubmitDisabled || this.IsProcessing)" Variant="Variant.Filled" OnClick="@(async () => await this.Start())" Style="@this.SubmitButtonStyle">
<MudButton Disabled="@(this.SubmitDisabled || this.IsProcessing || this.MediaTranscriptionService.IsBusy)" Variant="Variant.Filled" OnClick="@(async () => await this.Start())" Style="@this.SubmitButtonStyle">
@this.SubmitText
</MudButton>
@if (this.IsProcessing)

3
app/MindWork AI Studio/Assistants/AssistantBase.razor.cs
View File

@ -47,6 +47,9 @@ public abstract partial class AssistantBase<TSettings> : AssistantLowerBase wher
/// </summary>
[Inject]
protected AIJobService AIJobService { get; init; } = null!;
[Inject]
protected MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
protected abstract string Title { get; }

19
app/MindWork AI Studio/Chat/ChatThread.cs
View File

@ -24,6 +24,11 @@ public sealed record ChatThread
/// </summary>
public Guid WorkspaceId { get; set; }
/// <summary>
/// The monotonically increasing number used for managed media transcript filenames.
/// </summary>
public ulong LastMediaTranscriptNumber { get; set; }
/// <summary>
/// Specifies the provider selected for the chat thread.
/// </summary>
@ -240,14 +245,28 @@ public sealed record ChatThread
{
var previousBlock = sortedBlocks[index - 1];
if (previousBlock.Role is ChatRole.USER && previousBlock.HideFromUser)
{
DeleteManagedAttachments(previousBlock);
this.Blocks.Remove(previousBlock);
}
}
}
DeleteManagedAttachments(block);
// Remove the block from the chat thread:
this.Blocks.Remove(block);
}
private static void DeleteManagedAttachments(ContentBlock block)
{
if (block.Content is not ContentText textContent)
return;
foreach (var attachment in textContent.FileAttachments)
ManagedTranscriptAttachment.TryDeleteOwnedFile(attachment);
}
/// <summary>
/// Transforms this chat thread to an ERI chat thread.
/// </summary>

3
app/MindWork AI Studio/Chat/FileAttachment.cs
View File

@ -14,6 +14,7 @@ namespace AIStudio.Chat;
[JsonPolymorphic(TypeDiscriminatorPropertyName = "$type")]
[JsonDerivedType(typeof(FileAttachment), typeDiscriminator: "file")]
[JsonDerivedType(typeof(FileAttachmentImage), typeDiscriminator: "image")]
[JsonDerivedType(typeof(ManagedTranscriptAttachment), typeDiscriminator: "managed_transcript")]
public record FileAttachment(FileAttachmentType Type, string FileName, string FilePath, long FileSizeBytes)
{
/// <summary>
@ -56,7 +57,7 @@ public record FileAttachment(FileAttachmentType Type, string FileName, string Fi
/// <summary>
/// Rebuilds the attachment from its current file path so file type detection uses the latest rules.
/// </summary>
public FileAttachment Normalize() => FromPath(this.FilePath);
public virtual FileAttachment Normalize() => FromPath(this.FilePath);
/// <summary>
/// Creates a FileAttachment from a file path by automatically determining the type,

93
app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs Normal file
View File

@ -0,0 +1,93 @@
using System.Text;
using AIStudio.Settings;
namespace AIStudio.Chat;
public sealed record ManagedTranscriptAttachment(string FileName, string FilePath, long FileSizeBytes, string OriginalFileName, bool IsStaged)
: FileAttachment(FileAttachmentType.DOCUMENT, FileName, FilePath, FileSizeBytes)
{
public override FileAttachment Normalize()
{
var size = File.Exists(this.FilePath) ? new FileInfo(this.FilePath).Length : 0;
return this with { FileName = Path.GetFileName(this.FilePath), FileSizeBytes = size };
}
public static async Task<ManagedTranscriptAttachment> CreateStagedAsync(string originalPath, string transcript)
{
var operationDirectory = Path.Combine(SettingsManager.DataDirectory!, "media-staging", Guid.NewGuid().ToString("N"));
Directory.CreateDirectory(operationDirectory);
var originalFileName = Path.GetFileName(originalPath);
var stagingPath = Path.Combine(operationDirectory, $"{Guid.NewGuid():N}.md");
var markdown = $"""
# Transcription of {originalFileName}
{transcript.Trim()}
""";
await File.WriteAllTextAsync(stagingPath, markdown, new UTF8Encoding(false));
return new ManagedTranscriptAttachment(
Path.GetFileName(stagingPath),
stagingPath,
new FileInfo(stagingPath).Length,
originalFileName,
true);
}
public static bool TryDeleteOwnedFile(FileAttachment attachment)
{
if (attachment is not ManagedTranscriptAttachment managed || !File.Exists(managed.FilePath))
return false;
var fileInfo = new FileInfo(managed.FilePath);
var parentInfo = fileInfo.Directory!;
var canonicalParent = parentInfo.ResolveLinkTarget(true)?.FullName ?? parentInfo.FullName;
var fullPath = fileInfo.ResolveLinkTarget(true)?.FullName ?? Path.Combine(canonicalParent, fileInfo.Name);
var dataDirectory = new DirectoryInfo(SettingsManager.DataDirectory!);
var canonicalDataRoot = EnsureTrailingSeparator(dataDirectory.ResolveLinkTarget(true)?.FullName ?? dataDirectory.FullName);
var stagingDirectory = new DirectoryInfo(Path.Combine(SettingsManager.DataDirectory!, "media-staging"));
var stagingRoot = EnsureTrailingSeparator(stagingDirectory.ResolveLinkTarget(true)?.FullName ?? stagingDirectory.FullName);
var transcriptsSegment = $"{Path.DirectorySeparatorChar}attachments{Path.DirectorySeparatorChar}transcripts{Path.DirectorySeparatorChar}";
var comparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
var owned = fullPath.StartsWith(stagingRoot, comparison)
|| (fullPath.StartsWith(canonicalDataRoot, comparison)
&& fullPath.Contains(transcriptsSegment, comparison));
if (!owned)
return false;
File.Delete(fullPath);
var parent = Path.GetDirectoryName(fullPath);
if (managed.IsStaged && parent is not null && Directory.Exists(parent) && !Directory.EnumerateFileSystemEntries(parent).Any())
Directory.Delete(parent);
return true;
}
internal static string NormalizeOriginalStem(string originalFileName)
{
var stem = Path.GetFileNameWithoutExtension(originalFileName).Normalize(NormalizationForm.FormC);
var invalid = Path.GetInvalidFileNameChars().ToHashSet();
var normalized = new StringBuilder();
foreach (var character in stem)
{
normalized.Append(char.IsControl(character)
|| invalid.Contains(character)
|| character is '/' or '\\'
? '-'
: character);
}
var result = normalized.ToString().Trim(' ', '.', '-');
if (string.IsNullOrWhiteSpace(result))
result = "media";
return result.Length <= 80 ? result : result[..80];
}
private static string EnsureTrailingSeparator(string path) => path.EndsWith(Path.DirectorySeparatorChar) ? path : path + Path.DirectorySeparatorChar;
}

6
app/MindWork AI Studio/Components/AttachDocuments.razor
View File

@ -52,7 +52,7 @@
}
else
{
@if (!this.Disabled)
@if (!this.IsUnavailable)
{
<MudStack Row="true" AlignItems="AlignItems.Center" StretchItems="StretchItems.None" Wrap="Wrap.Wrap">
<MudText Typo="Typo.body1" Inline="true">
@ -73,7 +73,7 @@ else
<MudPaper Height="20em" Outlined="true" Class="@this.dragClass" Style="overflow-y: auto;">
@foreach (var fileAttachment in this.DocumentPaths)
{
@if (this.Disabled)
@if (this.IsUnavailable)
{
<MudChip T="string" Color="Color.Dark" Text="@fileAttachment.FileName" tabindex="-1" Icon="@Icons.Material.Filled.Search" OnClick="@(() => this.InvestigateFile(fileAttachment))"/>
}
@ -84,7 +84,7 @@ else
}
</MudPaper>
</div>
@if (!this.Disabled)
@if (!this.IsUnavailable)
{
<MudButton OnClick="@(async () => await this.ClearAllFiles())" Variant="Variant.Filled" Color="Color.Info" Class="mt-2" StartIcon="@Icons.Material.Filled.Delete">
@T("Clear file list")

153
app/MindWork AI Studio/Components/AttachDocuments.razor.cs
View File

@ -75,12 +75,16 @@ public partial class AttachDocuments : MSGComponentBase
[Inject]
private PandocAvailabilityService PandocAvailabilityService { get; init; } = null!;
[Inject]
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top;
private static readonly string DROP_FILES_HERE_TEXT = TB("Drop files here to attach them.");
private uint numDropAreasAboveThis;
private bool isComponentHovered;
private bool isDraggingOver;
private bool IsUnavailable => this.Disabled || this.MediaTranscriptionService.IsBusy;
#region Overrides of MSGComponentBase
@ -95,7 +99,7 @@ public partial class AttachDocuments : MSGComponentBase
protected override async Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
{
if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
return;
switch (triggeredEvent)
@ -168,29 +172,7 @@ public partial class AttachDocuments : MSGComponentBase
return;
}
// Ensure that Pandoc is installed and ready:
var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync(
showSuccessMessage: false,
showDialog: true);
// If Pandoc is not available (user cancelled installation), abort file drop:
if (!pandocState.IsAvailable)
{
this.Logger.LogWarning("The user cancelled the Pandoc installation or Pandoc is not available. Aborting file drop.");
this.isDraggingOver = false;
this.ClearDragClass();
this.StateHasChanged();
return;
}
foreach (var path in paths)
{
if(!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, path, this.ValidateMediaFileTypes, this.Provider))
continue;
this.DocumentPaths.Add(FileAttachment.FromPath(path));
}
await this.AddFileBatchAsync(paths);
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
await this.OnChange(this.DocumentPaths);
this.isDraggingOver = false;
@ -208,53 +190,36 @@ public partial class AttachDocuments : MSGComponentBase
private async Task AddFilesManually()
{
if (this.Disabled)
if (this.IsUnavailable)
return;
// Ensure that Pandoc is installed and ready:
var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync(
showSuccessMessage: false,
showDialog: true);
// If Pandoc is not available (user cancelled installation), abort file selection:
if (!pandocState.IsAvailable)
{
this.Logger.LogWarning("The user cancelled the Pandoc installation or Pandoc is not available. Aborting file selection.");
return;
}
var selectFiles = await this.RustService.SelectFiles(T("Select files to attach"));
if (selectFiles.UserCancelled)
return;
foreach (var selectedFilePath in selectFiles.SelectedFilePaths)
{
if (!File.Exists(selectedFilePath))
continue;
if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.ATTACHING_CONTENT, selectedFilePath, this.ValidateMediaFileTypes, this.Provider))
continue;
this.DocumentPaths.Add(FileAttachment.FromPath(selectedFilePath));
}
await this.AddFileBatchAsync(selectFiles.SelectedFilePaths);
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
await this.OnChange(this.DocumentPaths);
}
private async Task OpenAttachmentsDialog()
{
if (this.Disabled)
if (this.IsUnavailable)
return;
var previousAttachments = this.DocumentPaths.ToHashSet();
this.DocumentPaths = await ReviewAttachmentsDialog.OpenDialogAsync(this.DialogService, this.DocumentPaths);
foreach (var removedAttachment in previousAttachments.Except(this.DocumentPaths))
ManagedTranscriptAttachment.TryDeleteOwnedFile(removedAttachment);
}
private async Task ClearAllFiles()
{
if (this.Disabled)
if (this.IsUnavailable)
return;
foreach (var attachment in this.DocumentPaths)
ManagedTranscriptAttachment.TryDeleteOwnedFile(attachment);
this.DocumentPaths.Clear();
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
await this.OnChange(this.DocumentPaths);
@ -266,7 +231,7 @@ public partial class AttachDocuments : MSGComponentBase
private void OnMouseEnter(EventArgs _)
{
if(this.Disabled || this.PauseCatchingDrops)
if(this.IsUnavailable || this.PauseCatchingDrops)
return;
this.Logger.LogDebug("Attach documents component '{Name}' is hovered.", this.Name);
@ -277,7 +242,7 @@ public partial class AttachDocuments : MSGComponentBase
private void OnMouseLeave(EventArgs _)
{
if(this.Disabled || this.PauseCatchingDrops)
if(this.IsUnavailable || this.PauseCatchingDrops)
return;
this.Logger.LogDebug("Attach documents component '{Name}' is no longer hovered.", this.Name);
@ -288,15 +253,97 @@ public partial class AttachDocuments : MSGComponentBase
private async Task RemoveDocument(FileAttachment fileAttachment)
{
if (this.Disabled)
if (this.IsUnavailable)
return;
this.DocumentPaths.Remove(fileAttachment);
ManagedTranscriptAttachment.TryDeleteOwnedFile(fileAttachment);
await this.DocumentPathsChanged.InvokeAsync(this.DocumentPaths);
await this.OnChange(this.DocumentPaths);
}
private async Task AddFileBatchAsync(IEnumerable<string> paths)
{
var existingPaths = paths.Where(File.Exists).ToList();
var mediaPaths = existingPaths.Where(IsTranscribableMedia).ToList();
var regularPaths = existingPaths.Except(mediaPaths).ToList();
var canAddRegularFiles = true;
if (regularPaths.Count > 0)
{
var pandocState = await this.PandocAvailabilityService.EnsureAvailabilityAsync(
showSuccessMessage: false,
showDialog: true);
canAddRegularFiles = pandocState.IsAvailable;
}
foreach (var path in regularPaths)
{
if (!canAddRegularFiles)
break;
if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(
FileExtensionValidation.UseCase.ATTACHING_CONTENT,
path,
this.ValidateMediaFileTypes,
this.Provider))
continue;
this.DocumentPaths.Add(FileAttachment.FromPath(path));
}
if (mediaPaths.Count is 0)
return;
if (string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider))
{
await this.MessageBus.SendWarning(new(
Icons.Material.Filled.VoiceChat,
this.T("Media files require a configured transcription provider. Configure one in the transcription settings.")));
return;
}
var names = string.Join(Environment.NewLine, mediaPaths.Select(path => $"• {Path.GetFileName(path)}"));
var dialogParameters = new DialogParameters<ConfirmDialog>
{
{
x => x.Message,
$"{this.T("The selected audio and video files will be prepared locally. Their audio will then be uploaded to the configured transcription provider.")}{Environment.NewLine}{Environment.NewLine}{names}"
},
};
var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(
this.T("Transcribe media files"),
dialogParameters,
DialogOptions.FULLSCREEN);
var dialogResult = await dialogReference.Result;
if (dialogResult is null || dialogResult.Canceled)
return;
var failures = new List<string>();
foreach (var mediaPath in mediaPaths)
{
var result = await this.MediaTranscriptionService.TranscribeAsync(mediaPath);
if (!result.Success)
{
if (result.ErrorMessage == "The media transcription was cancelled.")
break;
failures.Add($"{Path.GetFileName(mediaPath)}: {result.ErrorMessage}");
continue;
}
var attachment = await ManagedTranscriptAttachment.CreateStagedAsync(mediaPath, result.Text);
this.DocumentPaths.Add(attachment);
}
if (failures.Count > 0)
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, string.Join(Environment.NewLine, failures)));
}
private static bool IsTranscribableMedia(string path) => FileTypes.IsAllowedPath(path, FileTypes.AUDIO) || FileTypes.IsAllowedPath(path, FileTypes.VIDEO);
/// <summary>
/// The user might want to check what we actually extract from his file and therefore give the LLM as an input.
/// </summary>

3
app/MindWork AI Studio/Components/ChatComponent.razor
View File

@ -33,6 +33,7 @@
}
</ChildContent>
<FooterContent>
<MediaTranscriptionStatus/>
<MudElement Style="flex: 0 0 auto;">
<MudTextField
T="string"
@ -100,7 +101,7 @@
</MudTooltip>
}
<AttachDocuments Name="File Attachments" Layer="@DropLayers.PAGES" DocumentPaths="@this.ComposerState.FileAttachments" DocumentPathsChanged="@this.ComposerAttachmentsChanged" CatchAllDocuments="true" UseSmallForm="true" Provider="@this.Provider"/>
<AttachDocuments Name="File Attachments" Layer="@DropLayers.PAGES" DocumentPaths="@this.ComposerState.FileAttachments" DocumentPathsChanged="@this.ComposerAttachmentsChanged" CatchAllDocuments="true" UseSmallForm="true" Provider="@this.Provider" Disabled="@this.MediaTranscriptionService.IsBusy"/>
<MudDivider Vertical="true" Style="height: 24px; align-self: center;"/>

13
app/MindWork AI Studio/Components/ChatComponent.razor.cs
View File

@ -4,6 +4,7 @@ using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.AIJobs;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
using Microsoft.AspNetCore.Components.Web;
@ -54,6 +55,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
[Inject]
private AIJobService AIJobService { get; init; } = null!;
[Inject]
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top;
private static readonly Dictionary<string, object?> USER_INPUT_ATTRIBUTES = new();
@ -683,6 +687,9 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
private async Task SendMessage(bool reuseLastUserPrompt = false)
{
if (this.MediaTranscriptionService.IsBusy)
return;
if (!this.IsProviderSelected)
return;
@ -986,12 +993,8 @@ public partial class ChatComponent : MSGComponentBase, IAsyncDisposable
if (workspaceId == Guid.Empty)
return;
// Delete the chat from the current workspace or the temporary storage:
await WorkspaceBehaviour.DeleteChatAsync(this.DialogService, this.ChatThread!.WorkspaceId, this.ChatThread.ChatId, askForConfirmation: false);
this.ChatThread!.WorkspaceId = workspaceId;
await WorkspaceBehaviour.MoveChatAsync(this.ChatThread!, workspaceId);
this.MarkCurrentChatAsLoadedParameter();
await this.SaveThread();
await this.SyncWorkspaceHeaderWithChatThreadAsync();
}

19
app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor Normal file
View File

@ -0,0 +1,19 @@
@inherits MSGComponentBase
@inject MediaTranscriptionService MediaTranscriptionService
@using AIStudio.Tools.Services
@if (this.MediaTranscriptionService.IsBusy)
{
<MudPaper Outlined="true" Class="pa-2 mb-2">
<MudStack Row="true" AlignItems="AlignItems.Center">
<MudProgressCircular Size="Size.Small" Indeterminate="@(this.MediaTranscriptionService.Progress is null)" Value="@((this.MediaTranscriptionService.Progress ?? 0) * 100)"/>
<MudText Typo="Typo.body2">
@this.StatusText
</MudText>
<MudSpacer/>
<MudTooltip Text="@T("Stop media transcription")">
<MudIconButton Icon="@Icons.Material.Filled.Stop" Color="Color.Error" OnClick="@this.MediaTranscriptionService.StopAsync"/>
</MudTooltip>
</MudStack>
</MudPaper>
}

28
app/MindWork AI Studio/Components/MediaTranscriptionStatus.razor.cs Normal file
View File

@ -0,0 +1,28 @@
using AIStudio.Tools.Services;
namespace AIStudio.Components;
public partial class MediaTranscriptionStatus
{
private string StatusText => this.MediaTranscriptionService.Phase switch
{
MediaTranscriptionPhase.PROBING => $"{this.T("Inspecting media")}: {this.MediaTranscriptionService.CurrentFileName}",
MediaTranscriptionPhase.TRANSCODING => $"{this.T("Preparing audio")}: {this.MediaTranscriptionService.CurrentFileName}",
MediaTranscriptionPhase.UPLOADING => $"{this.T("Transcribing")}: {this.MediaTranscriptionService.CurrentFileName}",
_ => this.MediaTranscriptionService.CurrentFileName,
};
protected override async Task OnInitializedAsync()
{
this.MediaTranscriptionService.StateChanged += this.OnStateChanged;
await base.OnInitializedAsync();
}
private void OnStateChanged() => _ = this.InvokeAsync(this.StateHasChanged);
protected override void DisposeResources()
{
this.MediaTranscriptionService.StateChanged -= this.OnStateChanged;
}
}

6
app/MindWork AI Studio/Components/ReadFileContent.razor
View File

@ -1,11 +1,13 @@
@inherits MSGComponentBase
<MediaTranscriptionStatus/>
@if (this.EnableDragDrop)
{
<div @onmouseenter="@this.OnMouseEnter" @onmouseleave="@this.OnMouseLeave">
<MudPaper Outlined="true" Class="@this.dragClass">
<MudStack Row="true" AlignItems="AlignItems.Center" StretchItems="StretchItems.None" Wrap="Wrap.Wrap">
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Disabled="@this.Disabled">
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Disabled="@(this.Disabled || this.MediaTranscriptionService.IsBusy)">
@this.ButtonText
</MudButton>
<MudText Typo="Typo.body2">
@ -17,7 +19,7 @@
}
else
{
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Class="mb-3" Disabled="@this.Disabled">
<MudButton StartIcon="@Icons.Material.Filled.Description" OnClick="@(async () => await this.SelectFile())" Variant="Variant.Filled" Class="mb-3" Disabled="@(this.Disabled || this.MediaTranscriptionService.IsBusy)">
@this.ButtonText
</MudButton>
}

62
app/MindWork AI Studio/Components/ReadFileContent.razor.cs
View File

@ -1,6 +1,7 @@
using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
using AIStudio.Tools.Validation;
using AIStudio.Dialogs;
using Microsoft.AspNetCore.Components;
@ -47,12 +48,16 @@ public partial class ReadFileContent : MSGComponentBase
[Inject]
private PandocAvailabilityService PandocAvailabilityService { get; init; } = null!;
[Inject]
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
private const string DEFAULT_DRAG_CLASS = "relative rounded-lg border-2 border-dashed pa-3 mb-3 mud-width-full";
private string ButtonText => string.IsNullOrWhiteSpace(this.Text) ? T("Use file content as input") : this.Text;
private string dragClass = DEFAULT_DRAG_CLASS;
private uint numDropAreasAboveThis;
private bool isComponentHovered;
private bool IsUnavailable => this.Disabled || this.MediaTranscriptionService.IsBusy;
#region Overrides of MSGComponentBase
@ -72,7 +77,7 @@ public partial class ReadFileContent : MSGComponentBase
if (!this.EnableDragDrop)
return;
if (this.Disabled && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
if (this.IsUnavailable && triggeredEvent == Event.TAURI_EVENT_RECEIVED)
return;
switch (triggeredEvent)
@ -126,10 +131,7 @@ public partial class ReadFileContent : MSGComponentBase
private async Task SelectFile()
{
if (this.Disabled)
return;
if (!await this.EnsurePandocAvailability())
if (this.IsUnavailable)
return;
var selectedFile = await this.RustService.SelectFile(T("Select file to read its content"));
@ -161,9 +163,6 @@ public partial class ReadFileContent : MSGComponentBase
private async Task LoadFirstValidFile(List<string> paths)
{
if (!await this.EnsurePandocAvailability())
return;
foreach (var path in paths)
{
if (await this.LoadFileIfValid(path))
@ -179,6 +178,12 @@ public partial class ReadFileContent : MSGComponentBase
return false;
}
if (FileTypes.IsAllowedPath(filePath, FileTypes.AUDIO) || FileTypes.IsAllowedPath(filePath, FileTypes.VIDEO))
return await this.LoadMediaTranscriptAsync(filePath);
if (!await this.EnsurePandocAvailability())
return false;
if (!await FileExtensionValidation.IsExtensionValidWithNotifyAsync(FileExtensionValidation.UseCase.DIRECTLY_LOADING_CONTENT, filePath))
{
this.Logger.LogWarning("User attempted to load unsupported file: {FilePath}", filePath);
@ -200,6 +205,43 @@ public partial class ReadFileContent : MSGComponentBase
}
}
private async Task<bool> LoadMediaTranscriptAsync(string filePath)
{
if (string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider))
{
await this.MessageBus.SendWarning(new(
Icons.Material.Filled.VoiceChat,
this.T("Media files require a configured transcription provider. Configure one in the transcription settings.")));
return false;
}
var dialogParameters = new DialogParameters<ConfirmDialog>
{
{
x => x.Message,
$"{this.T("The selected media file will be prepared locally. Its audio will then be uploaded to the configured transcription provider.")}{Environment.NewLine}{Environment.NewLine}{Path.GetFileName(filePath)}"
},
};
var dialogReference = await this.DialogService.ShowAsync<ConfirmDialog>(
this.T("Transcribe media file"),
dialogParameters,
Dialogs.DialogOptions.FULLSCREEN);
var dialogResult = await dialogReference.Result;
if (dialogResult is null || dialogResult.Canceled)
return false;
var result = await this.MediaTranscriptionService.TranscribeAsync(filePath);
if (!result.Success)
{
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, result.ErrorMessage));
return false;
}
await this.FileContentChanged.InvokeAsync(result.Text);
return true;
}
private bool CanCatchDroppedFile() => this.numDropAreasAboveThis is 0 && (this.isComponentHovered || this.CatchAllDocuments);
private void SetDragClass() => this.dragClass = $"{DEFAULT_DRAG_CLASS} mud-border-primary border-2";
@ -208,7 +250,7 @@ public partial class ReadFileContent : MSGComponentBase
private void OnMouseEnter(EventArgs _)
{
if(this.Disabled || this.numDropAreasAboveThis > 0)
if(this.IsUnavailable || this.numDropAreasAboveThis > 0)
return;
this.Logger.LogDebug("Read file content component is hovered.");
@ -219,7 +261,7 @@ public partial class ReadFileContent : MSGComponentBase
private void OnMouseLeave(EventArgs _)
{
if(this.Disabled)
if(this.IsUnavailable)
return;
this.Logger.LogDebug("Read file content component is no longer hovered.");

78
app/MindWork AI Studio/Components/VoiceRecorder.razor.cs
View File

@ -1,4 +1,3 @@
using AIStudio.Provider;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.MIME;
using AIStudio.Tools.Rust;
@ -25,6 +24,9 @@ public partial class VoiceRecorder : MSGComponentBase
[Inject]
private VoiceRecordingAvailabilityService VoiceRecordingAvailabilityService { get; init; } = null!;
[Inject]
private MediaTranscriptionService MediaTranscriptionService { get; init; } = null!;
#region Overrides of MSGComponentBase
protected override async Task OnInitializedAsync()
@ -101,7 +103,8 @@ public partial class VoiceRecorder : MSGComponentBase
&& !string.IsNullOrWhiteSpace(this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider);
private bool IsVoiceRecordingAvailable => this.ShouldRenderVoiceRecording
&& this.VoiceRecordingAvailabilityService.IsAvailable;
&& this.VoiceRecordingAvailabilityService.IsAvailable
&& !this.MediaTranscriptionService.IsBusy;
private string Tooltip => !this.VoiceRecordingAvailabilityService.IsAvailable
? T("Voice recording is unavailable because the client could not initialize audio playback.")
@ -151,7 +154,7 @@ public partial class VoiceRecorder : MSGComponentBase
try
{
var mimeTypeStrings = mimeTypes.ToStringArray();
string[] mimeTypeStrings = ["audio/webm;codecs=opus", .. mimeTypes.ToStringArray()];
var actualMimeType = await this.JsRuntime.InvokeAsync<string>("audioRecorder.start", this.dotNetReference, mimeTypeStrings);
// Store the MIME type for later use:
@ -317,52 +320,7 @@ public partial class VoiceRecorder : MSGComponentBase
try
{
// Get the configured transcription provider ID:
var transcriptionProviderId = this.SettingsManager.ConfigurationData.App.UseTranscriptionProvider;
if (string.IsNullOrWhiteSpace(transcriptionProviderId))
{
this.Logger.LogWarning("No transcription provider is configured.");
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("No transcription provider is configured.")));
return;
}
// Find the transcription provider in the list of configured providers:
var transcriptionProviderSettings = this.SettingsManager.ConfigurationData.TranscriptionProviders
.FirstOrDefault(x => x.Id == transcriptionProviderId);
if (transcriptionProviderSettings is null)
{
this.Logger.LogWarning("The configured transcription provider with ID '{ProviderId}' was not found.", transcriptionProviderId);
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The configured transcription provider was not found.")));
return;
}
// Check the confidence level:
var minimumLevel = this.SettingsManager.GetMinimumConfidenceLevel(Tools.Components.NONE);
var providerConfidence = transcriptionProviderSettings.UsedLLMProvider.GetConfidence(this.SettingsManager);
if (providerConfidence.Level < minimumLevel)
{
this.Logger.LogWarning(
"The configured transcription provider '{ProviderName}' has a confidence level of '{ProviderLevel}', which is below the minimum required level of '{MinimumLevel}'.",
transcriptionProviderSettings.UsedLLMProvider,
providerConfidence.Level,
minimumLevel);
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("The configured transcription provider does not meet the minimum confidence level.")));
return;
}
// Create the provider instance:
var provider = transcriptionProviderSettings.CreateProvider();
if (provider.Provider is LLMProviders.NONE)
{
this.Logger.LogError("Failed to create the transcription provider instance.");
await this.MessageBus.SendError(new(Icons.Material.Filled.VoiceChat, this.T("Failed to create the transcription provider.")));
return;
}
// Call the transcription API:
this.Logger.LogInformation("Starting transcription with provider '{ProviderName}' and model '{ModelName}'.", transcriptionProviderSettings.UsedLLMProvider, transcriptionProviderSettings.Model.ToString());
var transcriptionResult = await provider.TranscribeAudioAsync(transcriptionProviderSettings.Model, this.finalRecordingPath, this.SettingsManager);
var transcriptionResult = await this.MediaTranscriptionService.TranscribeAsync(this.finalRecordingPath);
if (!transcriptionResult.Success)
{
this.Logger.LogWarning("The transcription request failed.");
@ -406,19 +364,6 @@ public partial class VoiceRecorder : MSGComponentBase
// Copy the transcribed text to the clipboard:
await this.RustService.CopyText2Clipboard(this.Snackbar, transcribedText);
// Delete the recording file:
try
{
if (File.Exists(this.finalRecordingPath))
{
File.Delete(this.finalRecordingPath);
this.Logger.LogInformation("Deleted the recording file '{RecordingPath}'.", this.finalRecordingPath);
}
}
catch (Exception ex)
{
this.Logger.LogError(ex, "Failed to delete the recording file '{RecordingPath}'.", this.finalRecordingPath);
}
}
catch (Exception ex)
{
@ -429,6 +374,15 @@ public partial class VoiceRecorder : MSGComponentBase
{
await this.ReleaseMicrophoneAsync();
try
{
if (File.Exists(this.finalRecordingPath))
File.Delete(this.finalRecordingPath);
}
catch (Exception ex)
{
this.Logger.LogError(ex, "Failed to delete the recording file '{RecordingPath}'.", this.finalRecordingPath);
}
this.finalRecordingPath = null;
this.isTranscribing = false;
this.StateHasChanged();

6
app/MindWork AI Studio/Pages/Information.razor
View File

@ -308,7 +308,11 @@
<ThirdPartyComponent Name="Rust Crypto" Developer="Artyom Pavlov, Tony Arcieri, Brian Warner, Arthur Gautier, Vlad Filippov, Friedel Ziegelmayer, Nicolas Stalder & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/RustCrypto/traits/blob/master/cipher/LICENSE-MIT" RepositoryUrl="https://github.com/RustCrypto" UseCase="@T("When transferring sensitive data between Rust runtime and .NET app, we encrypt the data. We use some libraries from the Rust Crypto project for this purpose: cipher, aes, cbc, pbkdf2, hmac, and sha2. We are thankful for the great work of the Rust Crypto project.")"/>
<ThirdPartyComponent Name="rcgen" Developer="RustTLS developers, est31 & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/rustls/rcgen/blob/main/LICENSE" RepositoryUrl="https://github.com/rustls/rcgen" UseCase="@T("For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose.")"/>
<ThirdPartyComponent Name="windows-registry" Developer="Microsoft, Kenny Kerr, Ryan Levick, Rafael Rivera, sivadeilra, Marijn Suijten & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/microsoft/windows-rs/blob/master/license-mit" RepositoryUrl="https://github.com/microsoft/windows-rs" UseCase="@T("This library is used to access the Windows registry. We use this for Windows enterprise environments to read the desired configuration.")"/>
<ThirdPartyComponent Name="file-format" Developer="Mickaël Malécot & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/mmalecot/file-format/blob/main/LICENSE-MIT" RepositoryUrl="https://github.com/mmalecot/file-format" UseCase="@T("This library is used to determine the file type of a file. This is necessary, e.g., when we want to stream a file.")"/>
<ThirdPartyComponent Name="file-format" Developer="Mickaël Malécot & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/mmalecot/file-format/blob/main/LICENSE-MIT" RepositoryUrl="https://github.com/mmalecot/file-format" UseCase="@T("This library identifies files by their content. It is used for document streaming and as the first safety and media classification step before local audio processing.")"/>
<ThirdPartyComponent Name="Symphonia" Developer="Philip Deljanov & Open Source Community" LicenseName="MPL-2.0" LicenseUrl="https://github.com/pdeljanov/Symphonia/blob/v0.6.0/LICENSE" RepositoryUrl="https://github.com/pdeljanov/Symphonia" UseCase="@T("Symphonia is used for media container demuxing and audio decoding. The exact MPL-covered source is available from the repository linked and is identified in the offline notices bundled with AI Studio.")"/>
<ThirdPartyComponent Name="Ropus" Developer="0x4D44, Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo, Mozilla, Amazon & Open Source Community" LicenseName="BSD-3-Clause" LicenseUrl="https://github.com/0x4D44/ropus/blob/main/LICENSE" RepositoryUrl="https://github.com/0x4d44/ropus" UseCase="@T("Ropus provides the Opus encoder and decoder used by the media pipeline.")"/>
<ThirdPartyComponent Name="Rubato" Developer="Henrik Enquist & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/HEnquist/rubato/blob/master/LICENSE-MIT" RepositoryUrl="https://github.com/HEnquist/rubato" UseCase="@T("We use Rubato to resample the decoded audio to 48 kHz before the Opus encoding.")"/>
<ThirdPartyComponent Name="webm-iterable" Developer="Austin Blake & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/austinleroy/webm-iterable/blob/main/LICENSE" RepositoryUrl="https://github.com/austinleroy/webm-iterable" UseCase="@T("webm-iterable provides the EBML and WebM writing path for normalized audio.")"/>
<ThirdPartyComponent Name="calamine" Developer="Johann Tuffe, Joel Natividad, Eric Jolibois, Dmitriy & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/tafia/calamine/blob/master/LICENSE-MIT.md" RepositoryUrl="https://github.com/tafia/calamine" UseCase="@T("This library is used to read Excel and OpenDocument spreadsheet files. This is necessary, e.g., for using spreadsheets as a data source for a chat.")"/>
<ThirdPartyComponent Name="PDFium" Developer="Lei Zhang, Tom Sepez, Dan Sinclair, and Foxit, Google, Chromium, Collabora, Ada, DocsCorp, Dropbox, Microsoft, and PSPDFKit Teams & Open Source Community" LicenseName="Apache-2.0" LicenseUrl="https://pdfium.googlesource.com/pdfium/+/refs/heads/main/LICENSE" RepositoryUrl="https://pdfium.googlesource.com/pdfium" UseCase="@T("This library is used to read PDF files. This is necessary, e.g., for using PDFs as a data source for a chat.")"/>
<ThirdPartyComponent Name="pdfium-render" Developer="Alastair Carey, Dorian Rudolph & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/ajrcarey/pdfium-render/blob/master/LICENSE.md" RepositoryUrl="https://github.com/ajrcarey/pdfium-render" UseCase="@T("This library is used to read PDF files. This is necessary, e.g., for using PDFs as a data source for a chat.")"/>

2
app/MindWork AI Studio/Program.cs
View File

@ -136,6 +136,7 @@ internal sealed class Program
builder.Services.AddSingleton<AIJobService>();
builder.Services.AddSingleton<AssistantSessionService>();
builder.Services.AddSingleton<VoiceRecordingAvailabilityService>();
builder.Services.AddSingleton<MediaTranscriptionService>();
builder.Services.AddSingleton<AssistantPluginInstallService>();
builder.Services.AddSingleton<UpdatePolicy>();
builder.Services.AddSingleton<DataSourceService>();
@ -148,6 +149,7 @@ internal sealed class Program
builder.Services.AddTransient<AssistantPluginAuditService>();
builder.Services.AddHostedService<UpdateService>();
builder.Services.AddHostedService<TemporaryChatService>();
builder.Services.AddHostedService<TranscriptStagingCleanupService>();
builder.Services.AddHostedService<EnterpriseEnvironmentService>();
builder.Services.AddSingleton<DatabaseClientProvider>();
builder.Services.AddHostedService<GlobalShortcutService>();

3
app/MindWork AI Studio/Tools/Rust/CreateMediaJobRequest.cs Normal file
View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Rust;
public sealed record CreateMediaJobRequest(string InputPath, string OutputPath, ulong? MaxPassThroughBytes = null);

3
app/MindWork AI Studio/Tools/Rust/CreateMediaJobResponse.cs Normal file
View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Rust;
public sealed record CreateMediaJobResponse(string JobId);

2
app/MindWork AI Studio/Tools/Rust/FileTypes.cs
View File

@ -61,7 +61,7 @@ public static class FileTypes
public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"),
"jpg", "jpeg", "png", "gif", "bmp", "tiff", "svg", "webp", "heic");
public static readonly FileTypeFilter AUDIO = FileTypeFilter.Leaf(TB("Audio"),
"mp3", "wav", "wave", "aac", "flac", "ogg", "m4a", "wma", "alac", "aiff", "m4b");
"mp3", "wav", "wave", "aac", "flac", "ogg", "opus", "m4a", "m4b", "wma", "alac", "aif", "aiff", "caf");
public static readonly FileTypeFilter VIDEO = FileTypeFilter.Leaf(TB("Video"),
"mp4", "m4v", "avi", "mkv", "mov", "wmv", "flv", "webm");

3
app/MindWork AI Studio/Tools/Rust/MediaJobError.cs Normal file
View File

@ -0,0 +1,3 @@
namespace AIStudio.Tools.Rust;
public sealed record MediaJobError(string Code, string Message);

7
app/MindWork AI Studio/Tools/Rust/MediaJobEvent.cs Normal file
View File

@ -0,0 +1,7 @@
namespace AIStudio.Tools.Rust;
public sealed record MediaJobEvent(
MediaJobPhase Phase,
double? Progress,
MediaJobResult? Result,
MediaJobError? Error);

11
app/MindWork AI Studio/Tools/Rust/MediaJobPhase.cs Normal file
View File

@ -0,0 +1,11 @@
namespace AIStudio.Tools.Rust;
public enum MediaJobPhase
{
UNKNOWN,
PROBING,
TRANSCODING,
COMPLETED,
FAILED,
CANCELLED,
}

8
app/MindWork AI Studio/Tools/Rust/MediaJobResult.cs Normal file
View File

@ -0,0 +1,8 @@
namespace AIStudio.Tools.Rust;
public sealed record MediaJobResult(
string OutputPath,
string DetectedFormat,
string DetectedCodec,
ulong DurationMs,
bool PassThrough);

9
app/MindWork AI Studio/Tools/Services/MediaTranscriptionPhase.cs Normal file
View File

@ -0,0 +1,9 @@
namespace AIStudio.Tools.Services;
public enum MediaTranscriptionPhase
{
IDLE,
PROBING,
TRANSCODING,
UPLOADING,
}

173
app/MindWork AI Studio/Tools/Services/MediaTranscriptionService.cs Normal file
View File

@ -0,0 +1,173 @@
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools.Rust;
namespace AIStudio.Tools.Services;
public sealed class MediaTranscriptionService(RustService rustService, SettingsManager settingsManager, ILogger<MediaTranscriptionService> logger) : IDisposable
{
private readonly SemaphoreSlim queue = new(1, 1);
private readonly Lock stateLock = new();
private CancellationTokenSource? currentCancellation;
private string? currentJobId;
public event Action? StateChanged;
public bool IsBusy { get; private set; }
public string CurrentFileName { get; private set; } = string.Empty;
public MediaTranscriptionPhase Phase { get; private set; } = MediaTranscriptionPhase.IDLE;
public double? Progress { get; private set; }
public async Task<TranscriptionResult> TranscribeAsync(string mediaPath, CancellationToken token = default)
{
await this.queue.WaitAsync(token);
var normalizedPath = Path.Combine(
Path.GetTempPath(),
"mindwork-ai-studio-media",
$"{Guid.NewGuid():N}.webm");
Directory.CreateDirectory(Path.GetDirectoryName(normalizedPath)!);
using var operationCancellation = CancellationTokenSource.CreateLinkedTokenSource(token);
lock (this.stateLock)
this.currentCancellation = operationCancellation;
try
{
this.UpdateState(true, Path.GetFileName(mediaPath), MediaTranscriptionPhase.PROBING, 0.0);
var jobId = await rustService.StartMediaJobAsync(mediaPath, normalizedPath, operationCancellation.Token);
lock (this.stateLock)
this.currentJobId = jobId;
MediaJobResult? normalized = null;
await foreach (var mediaEvent in rustService.StreamMediaJobEventsAsync(jobId, operationCancellation.Token))
{
switch (mediaEvent.Phase)
{
case MediaJobPhase.PROBING:
this.UpdateState(true, this.CurrentFileName, MediaTranscriptionPhase.PROBING, mediaEvent.Progress);
break;
case MediaJobPhase.TRANSCODING:
this.UpdateState(true, this.CurrentFileName, MediaTranscriptionPhase.TRANSCODING, mediaEvent.Progress);
break;
case MediaJobPhase.COMPLETED:
normalized = mediaEvent.Result;
break;
case MediaJobPhase.CANCELLED:
throw new OperationCanceledException(operationCancellation.Token);
case MediaJobPhase.FAILED:
return TranscriptionResult.Failure(mediaEvent.Error?.Message ?? "The media file could not be prepared.");
}
}
if (normalized is null)
return TranscriptionResult.Failure("The media pipeline ended without an output file.");
var providerSettings = this.ResolveProvider();
if (providerSettings is null)
return TranscriptionResult.Failure("No usable transcription provider is configured.");
this.UpdateState(true, this.CurrentFileName, MediaTranscriptionPhase.UPLOADING, null);
var provider = providerSettings.CreateProvider();
if (provider.Provider is LLMProviders.NONE)
return TranscriptionResult.Failure("The configured transcription provider could not be created.");
logger.LogInformation(
"Transcribing normalized media '{MediaPath}' with provider '{Provider}' and model '{Model}'.",
mediaPath,
providerSettings.UsedLLMProvider,
providerSettings.Model);
var result = await provider.TranscribeAudioAsync(
providerSettings.Model,
normalized.OutputPath,
settingsManager,
operationCancellation.Token);
return result.Success
? TranscriptionResult.FromText(result.Text.Trim())
: result;
}
catch (OperationCanceledException)
{
return TranscriptionResult.Failure("The media transcription was cancelled.");
}
catch (Exception exception)
{
logger.LogError(exception, "Media transcription failed for '{MediaPath}'.", mediaPath);
return TranscriptionResult.Failure("The media file could not be transcribed.");
}
finally
{
lock (this.stateLock)
{
this.currentJobId = null;
this.currentCancellation = null;
}
try
{
if (File.Exists(normalizedPath))
File.Delete(normalizedPath);
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not delete normalized media file '{NormalizedPath}'.", normalizedPath);
}
this.UpdateState(false, string.Empty, MediaTranscriptionPhase.IDLE, null);
this.queue.Release();
}
}
public async Task StopAsync()
{
string? jobId;
lock (this.stateLock)
{
this.currentCancellation?.Cancel();
jobId = this.currentJobId;
}
if (!string.IsNullOrWhiteSpace(jobId))
await rustService.CancelMediaJobAsync(jobId);
}
private TranscriptionProvider? ResolveProvider()
{
var providerId = settingsManager.ConfigurationData.App.UseTranscriptionProvider;
if (string.IsNullOrWhiteSpace(providerId))
return null;
var providerSettings = settingsManager.ConfigurationData.TranscriptionProviders.FirstOrDefault(x => x.Id == providerId);
if (providerSettings is null)
return null;
var minimumLevel = settingsManager.GetMinimumConfidenceLevel(Components.NONE);
return providerSettings.UsedLLMProvider.GetConfidence(settingsManager).Level >= minimumLevel
? providerSettings
: null;
}
private void UpdateState(bool isBusy, string fileName, MediaTranscriptionPhase phase, double? progress)
{
this.IsBusy = isBusy;
this.CurrentFileName = fileName;
this.Phase = phase;
this.Progress = progress;
this.StateChanged?.Invoke();
}
public void Dispose()
{
this.currentCancellation?.Cancel();
this.currentCancellation?.Dispose();
this.queue.Dispose();
}
}

54
app/MindWork AI Studio/Tools/Services/RustService.Media.cs Normal file
View File

@ -0,0 +1,54 @@
using System.Runtime.CompilerServices;
using System.Text.Json;
using AIStudio.Tools.Rust;
namespace AIStudio.Tools.Services;
public partial class RustService
{
public async Task<string> StartMediaJobAsync(string inputPath, string outputPath, CancellationToken token = default)
{
using var response = await this.http.PostAsJsonAsync(
"/media/jobs",
new CreateMediaJobRequest(inputPath, outputPath),
this.jsonRustSerializerOptions,
token);
response.EnsureSuccessStatusCode();
var result = await response.Content.ReadFromJsonAsync<CreateMediaJobResponse>(this.jsonRustSerializerOptions, token);
return result?.JobId ?? throw new InvalidOperationException("The Rust runtime did not return a media job ID.");
}
public async IAsyncEnumerable<MediaJobEvent> StreamMediaJobEventsAsync(string jobId, [EnumeratorCancellation] CancellationToken token = default)
{
using var request = new HttpRequestMessage(HttpMethod.Get, $"/media/jobs/{Uri.EscapeDataString(jobId)}/events");
using var response = await this.http.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, token);
response.EnsureSuccessStatusCode();
await using var stream = await response.Content.ReadAsStreamAsync(token);
using var reader = new StreamReader(stream);
while (!reader.EndOfStream && !token.IsCancellationRequested)
{
var line = await reader.ReadLineAsync(token);
if (line is null || !line.StartsWith("data:", StringComparison.Ordinal))
continue;
var json = line["data:".Length..].Trim();
var mediaEvent = JsonSerializer.Deserialize<MediaJobEvent>(json, this.jsonRustSerializerOptions);
if (mediaEvent is not null)
yield return mediaEvent;
if (mediaEvent?.Phase is MediaJobPhase.COMPLETED or MediaJobPhase.FAILED or MediaJobPhase.CANCELLED)
yield break;
}
}
public async Task CancelMediaJobAsync(string jobId, CancellationToken token = default)
{
using var response = await this.http.DeleteAsync($"/media/jobs/{Uri.EscapeDataString(jobId)}", token);
if (response is { IsSuccessStatusCode: false, StatusCode: not System.Net.HttpStatusCode.NotFound })
response.EnsureSuccessStatusCode();
}
}

31
app/MindWork AI Studio/Tools/Services/TranscriptStagingCleanupService.cs Normal file
View File

@ -0,0 +1,31 @@
using AIStudio.Settings;
namespace AIStudio.Tools.Services;
public sealed class TranscriptStagingCleanupService(ILogger<TranscriptStagingCleanupService> logger) : BackgroundService
{
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
while (string.IsNullOrWhiteSpace(SettingsManager.DataDirectory) && !stoppingToken.IsCancellationRequested)
await Task.Delay(TimeSpan.FromSeconds(1), stoppingToken);
if (stoppingToken.IsCancellationRequested)
return;
var stagingRoot = Path.Combine(SettingsManager.DataDirectory!, "media-staging");
if (!Directory.Exists(stagingRoot))
return;
foreach (var directory in Directory.EnumerateDirectories(stagingRoot))
{
try
{
Directory.Delete(directory, true);
}
catch (Exception exception)
{
logger.LogWarning(exception, "Could not remove orphaned media staging directory '{Directory}'.", directory);
}
}
}
}

94
app/MindWork AI Studio/Tools/WorkspaceBehaviour.cs
View File

@ -754,6 +754,7 @@ public static class WorkspaceBehaviour
Directory.CreateDirectory(chatDirectory);
await FinalizeStagedTranscriptsAsync(chat, chatDirectory);
var chatNamePath = Path.Join(chatDirectory, "name");
await File.WriteAllTextAsync(chatNamePath, chat.Name);
@ -769,6 +770,99 @@ public static class WorkspaceBehaviour
}
}
public static async Task MoveChatAsync(ChatThread chat, Guid targetWorkspaceId)
{
if (chat.WorkspaceId == targetWorkspaceId)
return;
var sourceDirectory = chat.WorkspaceId == Guid.Empty
? Path.Join(SettingsManager.DataDirectory, "tempChats", chat.ChatId.ToString())
: Path.Join(SettingsManager.DataDirectory, "workspaces", chat.WorkspaceId.ToString(), chat.ChatId.ToString());
var targetDirectory = targetWorkspaceId == Guid.Empty
? Path.Join(SettingsManager.DataDirectory, "tempChats", chat.ChatId.ToString())
: Path.Join(SettingsManager.DataDirectory, "workspaces", targetWorkspaceId.ToString(), chat.ChatId.ToString());
if (Directory.Exists(sourceDirectory))
{
Directory.CreateDirectory(Path.GetDirectoryName(targetDirectory)!);
if (Directory.Exists(targetDirectory))
throw new IOException($"The target chat directory already exists: '{targetDirectory}'.");
Directory.Move(sourceDirectory, targetDirectory);
UpdateAttachmentPathsAfterMove(chat, sourceDirectory, targetDirectory);
}
chat.WorkspaceId = targetWorkspaceId;
await StoreChatAsync(chat);
InvalidateWorkspaceTreeCache();
}
private static void UpdateAttachmentPathsAfterMove(ChatThread chat, string sourceDirectory, string targetDirectory)
{
var sourcePrefix = sourceDirectory.EndsWith(Path.DirectorySeparatorChar)
? sourceDirectory
: sourceDirectory + Path.DirectorySeparatorChar;
var pathComparison = OperatingSystem.IsWindows() ? StringComparison.OrdinalIgnoreCase : StringComparison.Ordinal;
foreach (var content in chat.Blocks.Select(block => block.Content).OfType<ContentText>())
{
for (var index = 0; index < content.FileAttachments.Count; index++)
{
var attachment = content.FileAttachments[index];
if (!Path.GetFullPath(attachment.FilePath).StartsWith(sourcePrefix, pathComparison))
continue;
var relativePath = Path.GetRelativePath(sourceDirectory, attachment.FilePath);
var movedPath = Path.Combine(targetDirectory, relativePath);
content.FileAttachments[index] = attachment switch
{
ManagedTranscriptAttachment managed => managed with { FilePath = movedPath },
FileAttachmentImage image => image with { FilePath = movedPath },
_ => attachment with { FilePath = movedPath },
};
}
}
}
private static async Task FinalizeStagedTranscriptsAsync(ChatThread chat, string chatDirectory)
{
var transcriptDirectory = Path.Combine(chatDirectory, "attachments", "transcripts");
foreach (var content in chat.Blocks.Select(block => block.Content).OfType<ContentText>())
{
for (var index = 0; index < content.FileAttachments.Count; index++)
{
if (content.FileAttachments[index] is not ManagedTranscriptAttachment { IsStaged: true } staged
|| !File.Exists(staged.FilePath))
continue;
Directory.CreateDirectory(transcriptDirectory);
string targetPath;
do
{
chat.LastMediaTranscriptNumber++;
var stem = ManagedTranscriptAttachment.NormalizeOriginalStem(staged.OriginalFileName);
targetPath = Path.Combine(transcriptDirectory, $"{stem}-transcript-{chat.LastMediaTranscriptNumber:D4}.md");
} while (File.Exists(targetPath));
File.Move(staged.FilePath, targetPath);
var sourceDirectory = Path.GetDirectoryName(staged.FilePath);
if (sourceDirectory is not null && Directory.Exists(sourceDirectory) && !Directory.EnumerateFileSystemEntries(sourceDirectory).Any())
Directory.Delete(sourceDirectory);
content.FileAttachments[index] = new ManagedTranscriptAttachment(
Path.GetFileName(targetPath),
targetPath,
new FileInfo(targetPath).Length,
staged.OriginalFileName,
false);
}
}
await Task.CompletedTask;
}
public static async Task<ChatThread?> LoadChatAsync(LoadChat loadChat)
{
var (acquired, semaphore) = await TryAcquireChatSemaphoreAsync(loadChat.WorkspaceId, loadChat.ChatId, nameof(LoadChatAsync));

1
app/MindWork AI Studio/wwwroot/changelog/v26.7.3.md
View File

@ -1,5 +1,6 @@
# v26.7.3, build 245 (2026-07-xx xx:xx UTC)
- Added support for OpenAI GPT-5.6 Sol, Terra, and Luna; Anthropic Claude Fable 5 and Mythos 5; and Google Gemini 3 Flash, Gemini 3.1 Flash-Lite, Gemini 3.1 Pro, and Gemini 3.5 Flash.
- Added audio and video transcription for chats and assistants. AI Studio now prepares supported media locally, sends only normalized audio to the configured transcription provider, and attaches the resulting transcript instead of the original media.
- Improved the "My Tasks Assistant": you can now provide one or more documents in addition to text or use documents alone when asking to identify tasks.
- Improved update guidance for Flatpak installations and added an enterprise option that lets organizations manage updates entirely through their IT department.
- Fixed an issue that could leave AI Studio unresponsive after waking the computer from sleep.

391
runtime/Cargo.lock generated
View File

@ -490,6 +490,43 @@ dependencies = [
"windows-sys 0.52.0",
]
[[package]]
name = "audio-core"
version = "0.2.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f93ebbf82d06013f4c41fe71303feb980cddd78496d904d06be627972de51a24"
[[package]]
name = "audioadapter"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c75c3943c6c7279bb25a449a8d1727480730ab2efd7b6fd5d6ca51927096e6e4"
dependencies = [
"audio-core",
"num-traits",
]
[[package]]
name = "audioadapter-buffers"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ece3390b6eb40379094843a1da5aaccc34bc0d85a8cbf68d09fe092fee6de29e"
dependencies = [
"audioadapter",
"audioadapter-sample",
"num-traits",
]
[[package]]
name = "audioadapter-sample"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1592f90413568e259413c21a41a3d571feb1774255c209e7966d98f9db708c90"
dependencies = [
"audio-core",
"num-traits",
]
[[package]]
name = "autocfg"
version = "1.3.0"
@ -1923,6 +1960,35 @@ dependencies = [
"num-traits",
]
[[package]]
name = "ebml-iterable"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5173ac3752f08b526a6991509615e1a345b221ec3c58c7633433e8c9582312"
dependencies = [
"ebml-iterable-specification",
"ebml-iterable-specification-derive",
"futures",
]
[[package]]
name = "ebml-iterable-specification"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f56467af159a98735d44231f53eaa505e919e6003266f103b99649a93f106784"
[[package]]
name = "ebml-iterable-specification-derive"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b066b81018300fdce40f71c4db355a102699324af96fad28f25ab1b5f87de066"
dependencies = [
"ebml-iterable-specification",
"proc-macro2",
"quote",
"syn 1.0.109",
]
[[package]]
name = "ecow"
version = "0.3.0"
@ -2112,6 +2178,12 @@ dependencies = [
"zune-inflate",
]
[[package]]
name = "extended"
version = "0.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "af9673d8203fcb076b19dfd17e38b3d4ae9f44959416ea532ce72415a6020365"
[[package]]
name = "fast-float2"
version = "0.2.3"
@ -4023,11 +4095,14 @@ dependencies = [
"rand 0.10.2",
"rand_chacha 0.10.0",
"rcgen",
"ropus",
"rubato",
"rustls",
"serde",
"serde_json",
"sha2 0.11.0",
"strum_macros",
"symphonia",
"sys-locale",
"sysinfo 0.39.6",
"tauri",
@ -4043,6 +4118,7 @@ dependencies = [
"tokio",
"tokio-stream",
"webkit2gtk",
"webm-iterable",
"whoami",
"windows-native-keyring-store",
"windows-registry",
@ -5126,6 +5202,15 @@ dependencies = [
"syn 2.0.117",
]
[[package]]
name = "primal-check"
version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc0d895b311e3af9902528fbb8f928688abbd95872819320517cc24ca6b2bd08"
dependencies = [
"num-integer",
]
[[package]]
name = "proc-macro-crate"
version = "1.3.1"
@ -5561,6 +5646,15 @@ dependencies = [
"yasna",
]
[[package]]
name = "realfft"
version = "3.5.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f821338fddb99d089116342c46e9f1fbf3828dba077674613e734e01d6ea8677"
dependencies = [
"rustfft",
]
[[package]]
name = "redox_syscall"
version = "0.4.1"
@ -5613,6 +5707,12 @@ dependencies = [
"regex-syntax",
]
[[package]]
name = "regex-lite"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cab834c73d247e67f4fae452806d17d3c7501756d98c8808d7c9c7aa7d18f973"
[[package]]
name = "regex-syntax"
version = "0.8.10"
@ -5740,6 +5840,16 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4e27ee8bb91ca0adcf0ecb116293afa12d393f9c2b9b9cd54d33e8078fe19839"
[[package]]
name = "ropus"
version = "0.12.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "80804dadbfa2851c95fe45ff9ae8f4328d6371fdc0d51b14740d49a8b41d3758"
dependencies = [
"cc",
"wide",
]
[[package]]
name = "roxmltree"
version = "0.20.0"
@ -5757,6 +5867,22 @@ dependencies = [
"smallvec",
]
[[package]]
name = "rubato"
version = "4.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f57c655d11e929f05a8663b323ff553f8d9773be05dfdc087795955bedeb8d92"
dependencies = [
"audioadapter",
"audioadapter-buffers",
"num-complex",
"num-integer",
"num-traits",
"realfft",
"visibility",
"windowfunctions",
]
[[package]]
name = "rustc-demangle"
version = "0.1.27"
@ -5778,6 +5904,20 @@ dependencies = [
"semver",
]
[[package]]
name = "rustfft"
version = "6.4.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "21db5f9893e91f41798c88680037dba611ca6674703c1a18601b01a72c8adb89"
dependencies = [
"num-complex",
"num-integer",
"num-traits",
"primal-check",
"strength_reduce",
"transpose",
]
[[package]]
name = "rusticata-macros"
version = "4.1.0"
@ -5902,6 +6042,15 @@ version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
[[package]]
name = "safe_arch"
version = "0.7.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "96b02de82ddbe1b636e6170c21be622223aea188ef2e139be0a5b219ec215323"
dependencies = [
"bytemuck",
]
[[package]]
name = "same-file"
version = "1.0.6"
@ -6532,6 +6681,12 @@ version = "1.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3"
[[package]]
name = "strength_reduce"
version = "0.2.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fe895eb47f22e2ddd4dabc02bce419d2e643c8e3b585c78158b349195bc24d82"
[[package]]
name = "string_cache"
version = "0.9.0"
@ -6606,6 +6761,192 @@ dependencies = [
"serde_json",
]
[[package]]
name = "symphonia"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1758d6c853020a7244de03cc3e0185eaea3f58715122422dd3cc7452e6d4c16a"
dependencies = [
"lazy_static",
"symphonia-bundle-flac",
"symphonia-bundle-mp3",
"symphonia-codec-aac",
"symphonia-codec-alac",
"symphonia-codec-pcm",
"symphonia-codec-vorbis",
"symphonia-core",
"symphonia-format-caf",
"symphonia-format-isomp4",
"symphonia-format-mkv",
"symphonia-format-ogg",
"symphonia-format-riff",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-flac"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee69ad01236a67260b82fd1ff9790dd75ead29f2f46af145e63b7e72273e0e03"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-bundle-mp3"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "350f1f2f2e19ad4dd315db94304d1eb361b29af070681f94e51b8fdaad769546"
dependencies = [
"lazy_static",
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-aac"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1979c515a76371b186aad2feff5f23e21cbec775bf95de08bf1e3af92a2ad76"
dependencies = [
"lazy_static",
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-alac"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a149cbfc7fb5c405d123a273227d31de17138419552112bf1aa7b73e65827b8"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-pcm"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "50baee168f0e9dcf6ba7fc06e8b57eb62072a4490cc7cf13af77e72baae5d328"
dependencies = [
"log",
"symphonia-core",
]
[[package]]
name = "symphonia-codec-vorbis"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45b07b4423cd8e0fc472575909a5554b12c2f58e3c190b38c24f042e732fd8de"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-common"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8257891ffa7f05e02b58f4761e2abf7e5278c8744fd59e981559e050f86eef55"
dependencies = [
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-core"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "95ec293b5f288383b72a7bffcade6b2860b642cf66f28b3bd5967349a49938b1"
dependencies = [
"bitflags 2.11.1",
"bytemuck",
"lazy_static",
"log",
"num-complex",
"smallvec",
]
[[package]]
name = "symphonia-format-caf"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cde3ca76633d3400ab57195456c09f8a58d775ff5452329f3f212b6efc8622f5"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-format-isomp4"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2d179a01305b3505940135a9f0180d6ef4b487912748fe97554756f120fbd05e"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-format-mkv"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fb17713e134f5ad316c2690fa3104590ccc85842cdbcf82c3cd1a845cb08aa74"
dependencies = [
"lazy_static",
"log",
"symphonia-common",
"symphonia-core",
]
[[package]]
name = "symphonia-format-ogg"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b05a67e02b1e4fca1a261ba4fe06910a9357489ad8c36aafdd2960e9c6559433"
dependencies = [
"log",
"symphonia-common",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-format-riff"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "17424452a777666d3eaf09a5c651029b15b6a333812fcc5b5474f2a3f0cff3f0"
dependencies = [
"extended",
"log",
"symphonia-core",
"symphonia-metadata",
]
[[package]]
name = "symphonia-metadata"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a31acf5cd623398a6208e2225d18f4b20f761c55098a796a5247ad516a4a8681"
dependencies = [
"lazy_static",
"log",
"regex-lite",
"smallvec",
"symphonia-core",
]
[[package]]
name = "syn"
version = "1.0.109"
@ -7356,6 +7697,7 @@ dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
"tokio-util",
]
[[package]]
@ -7603,6 +7945,16 @@ dependencies = [
"once_cell",
]
[[package]]
name = "transpose"
version = "0.2.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1ad61aed86bc3faea4300c7aee358b4c6d0c8d6ccc36524c96e4c92ccf26e77e"
dependencies = [
"num-integer",
"strength_reduce",
]
[[package]]
name = "tray-icon"
version = "0.24.1"
@ -7896,6 +8248,17 @@ version = "0.0.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "051eb1abcf10076295e815102942cc58f9d5e3b4560e46e53c21e8ff6f3af7b1"
[[package]]
name = "visibility"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d674d135b4a8c1d7e813e2f8d1c9a58308aee4a680323066025e53132218bd91"
dependencies = [
"proc-macro2",
"quote",
"syn 2.0.117",
]
[[package]]
name = "vswhom"
version = "0.1.0"
@ -8174,6 +8537,15 @@ dependencies = [
"system-deps",
]
[[package]]
name = "webm-iterable"
version = "0.6.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd9fbf173b4b38f2f8bbb0082a0d4cb21f263a70811f5fccb1663c421c66d9f9"
dependencies = [
"ebml-iterable",
]
[[package]]
name = "webpki-root-certs"
version = "1.0.4"
@ -8248,6 +8620,16 @@ dependencies = [
"web-sys",
]
[[package]]
name = "wide"
version = "0.7.33"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ce5da8ecb62bcd8ec8b7ea19f69a51275e91299be594ea5cc6ef7819e16cd03"
dependencies = [
"bytemuck",
"safe_arch",
]
[[package]]
name = "winapi"
version = "0.3.9"
@ -8294,6 +8676,15 @@ dependencies = [
"windows-version",
]
[[package]]
name = "windowfunctions"
version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90628d739333b7c5d2ee0b70210b97b8cddc38440c682c96fd9e2c24c2db5f3a"
dependencies = [
"num-traits",
]
[[package]]
name = "windows"
version = "0.61.3"

6
runtime/Cargo.toml
View File

@ -20,7 +20,7 @@ serde_json = "1.0.150"
keyring-core = "1.0.0"
arboard = "3.6.1"
tokio = { version = "1.52.3", features = ["rt", "rt-multi-thread", "macros", "process"] }
tokio-stream = "0.1.18"
tokio-stream = { version = "0.1.18", features = ["sync"] }
futures = "0.3.32"
async-stream = "0.3.6"
flexi_logger = "0.31.9"
@ -39,6 +39,10 @@ hmac = "0.13.0"
sha2 = "0.11.0"
rcgen = { version = "0.14.8", features = ["pem"] }
file-format = "0.29.0"
symphonia = { version = "0.6", default-features = false, features = ["aac", "aiff", "alac", "caf", "flac", "isomp4", "mkv", "mp1", "mp2", "mp3", "ogg", "pcm", "vorbis", "wav"] }
ropus = "=0.12.18"
rubato = { version = "4", default-features = false, features = ["fft_resampler"] }
webm-iterable = "0.6.4"
calamine = "0.36.0"
pdfium-render = "0.9.1"
sys-locale = "0.3.2"

162
runtime/resources/notices/THIRD_PARTY_MEDIA_NOTICES.md Normal file
View File

@ -0,0 +1,162 @@
# Media pipeline third-party notices
These notices are bundled offline with MindWork AI Studio.
## Symphonia 0.6.0
Copyright (c) 2019-2026 The Project Symphonia Developers.
MindWork AI Studio uses the unmodified Symphonia 0.6.0 crates. The exact corresponding source is:
- https://github.com/pdeljanov/Symphonia/tree/v0.6.0
- https://crates.io/api/v1/crates/symphonia/0.6.0/download
If a future AI Studio release modifies MPL-covered Symphonia files, those modifications must be identified and made available separately under MPL-2.0. No such modifications are present in this release.
Mozilla Public License Version 2.0
1. Definitions
1.1. “Contributor” means each individual or legal entity that creates, contributes to the creation of, or owns Covered Software.
1.2. “Contributor Version” means the combination of the Contributions of others (if any) used by a Contributor and that particular Contributor’s Contribution.
1.3. “Contribution” means Covered Software of a particular Contributor.
1.4. “Covered Software” means Source Code Form to which the initial Contributor has attached the notice in Exhibit A, the Executable Form of such Source Code Form, and Modifications of such Source Code Form, in each case including portions thereof.
1.5. “Incompatible With Secondary Licenses” means that the initial Contributor has attached the notice described in Exhibit B to the Covered Software; or that the Covered Software was made available under the terms of version 1.1 or earlier of the License, but not also under the terms of a Secondary License.
1.6. “Executable Form” means any form of the work other than Source Code Form.
1.7. “Larger Work” means a work that combines Covered Software with other material, in a separate file or files, that is not Covered Software.
1.8. “License” means this document.
1.9. “Licensable” means having the right to grant, to the maximum extent possible, whether at the time of the initial grant or subsequently, any and all of the rights conveyed by this License.
1.10. “Modifications” means any of the following: any file in Source Code Form that results from an addition to, deletion from, or modification of the contents of Covered Software; or any new file in Source Code Form that contains any Covered Software.
1.11. “Patent Claims” of a Contributor means any patent claim(s), including without limitation, method, process, and apparatus claims, in any patent Licensable by such Contributor that would be infringed, but for the grant of the License, by the making, using, selling, offering for sale, having made, import, or transfer of either its Contributions or its Contributor Version.
1.12. “Secondary License” means either the GNU General Public License, Version 2.0, the GNU Lesser General Public License, Version 2.1, the GNU Affero General Public License, Version 3.0, or any later versions of those licenses.
1.13. “Source Code Form” means the form of the work preferred for making modifications.
1.14. “You” means an individual or a legal entity exercising rights under this License. For legal entities, “You” includes any entity that controls, is controlled by, or is under common control with You. “Control” means ownership of more than fifty percent of the outstanding shares or beneficial ownership of such entity.
2. License Grants and Conditions
2.1. Grants. Each Contributor hereby grants You a world-wide, royalty-free, non-exclusive license under intellectual property rights (other than patent or trademark) Licensable by such Contributor to use, reproduce, make available, modify, display, perform, distribute, and otherwise exploit its Contributions, either on an unmodified basis, with Modifications, or as part of a Larger Work; and under Patent Claims of such Contributor to make, use, sell, offer for sale, have made, import, and otherwise transfer either its Contributions or its Contributor Version.
2.2. Effective Date. The licenses granted in Section 2.1 with respect to any Contribution become effective for each Contribution on the date the Contributor first distributes such Contribution.
2.3. Limitations on Grant Scope. No license is granted in the trademarks, service marks, or logos of any Contributor. Except as otherwise provided in this License, no Contributor grants additional rights by implication, estoppel, or otherwise.
2.4. Subsequent Licenses. No Contributor makes additional grants as a result of Your choice to distribute the Covered Software under a subsequent version of this License or the terms of a Secondary License.
2.5. Representation. Each Contributor represents that the Contributor believes its Contributions are its original creation(s) or it has sufficient rights to grant the rights to its Contributions conveyed by this License.
2.6. Fair Use. This License is not intended to limit any rights You have under applicable copyright doctrines of fair use, fair dealing, or other equivalents.
2.7. Conditions. Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in Section 2.1.
3. Responsibilities
3.1. Distribution of Source Form. All distribution of Covered Software in Source Code Form, including any Modifications that You create or to which You contribute, must be under the terms of this License. You must inform recipients that the Source Code Form of the Covered Software is governed by the terms of this License, and how they can obtain a copy of this License. You may not attempt to alter or restrict the recipients’ rights in the Source Code Form.
3.2. Distribution of Executable Form. If You distribute Covered Software in Executable Form then such Covered Software must also be made available in Source Code Form, as described in Section 3.1, and You must inform recipients how they can obtain a copy of such Source Code Form by reasonable means in a timely manner, at a charge no more than the cost of distribution to the recipient; and You may distribute such Executable Form under the terms of this License, or sublicense it under different terms, provided that the license for the Executable Form does not attempt to limit or alter the recipients’ rights in the Source Code Form under this License.
3.3. Distribution of a Larger Work. You may create and distribute a Larger Work under terms of Your choice, provided that You also comply with the requirements of this License for the Covered Software. If the Larger Work is a combination of Covered Software with a work governed by one or more Secondary Licenses, and the Covered Software is not Incompatible With Secondary Licenses, this License permits You to additionally distribute such Covered Software under the terms of such Secondary License(s), so that the recipient of the Larger Work may, at their option, further distribute the Covered Software under the terms of either this License or such Secondary License(s).
3.4. Notices. You may not remove or alter the substance of any license notices contained within the Source Code Form of the Covered Software, except that You may alter any license notices to the extent required to remedy known factual inaccuracies. You must include a copy of this License with every copy of the Covered Software You distribute. You may add additional accurate notices of copyright ownership.
3.5. Application of Additional Terms. You may choose to offer, and to charge a fee for, warranty, support, indemnity or liability obligations to one or more recipients of Covered Software. However, You may do so only on Your own behalf, and not on behalf of any Contributor.
4. Inability to Comply Due to Statute or Regulation
If it is impossible for You to comply with any of the terms of this License with respect to some or all of the Covered Software due to statute, judicial order, or regulation then You must comply with the terms of this License to the maximum extent possible and describe the limitations and the code they affect. Such description must be placed in a text file included with all distributions of the Covered Software under this License.
5. Termination
5.1. The rights granted under this License will terminate automatically if You fail to comply with any of its terms. However, if You become compliant, then the rights granted under this License from a particular Contributor are reinstated provisionally, unless and until such Contributor explicitly and finally terminates Your grants, and on an ongoing basis, if such Contributor fails to notify You of the non-compliance by some reasonable means prior to 60 days after You have come back into compliance. Moreover, Your grants from a particular Contributor are reinstated on an ongoing basis if such Contributor notifies You of the non-compliance by some reasonable means, this is the first time You have received notice of non-compliance with this License from such Contributor, and You become compliant prior to 30 days after Your receipt of the notice.
5.2. If You initiate litigation against any entity by asserting a patent infringement claim alleging that a Contributor Version directly or indirectly infringes any patent, then the rights granted to You by any and all Contributors for the Covered Software under Section 2.1 will terminate.
5.3. In the event of termination under Sections 5.1 or 5.2, all end user license agreements (excluding distributors and resellers) which have been validly granted by You or Your distributors under this License prior to termination shall survive termination.
6. Disclaimer of Warranty
Covered Software is provided under this License on an “as is” basis, without warranty of any kind, either expressed, implied, or statutory, including, without limitation, warranties that the Covered Software is free of defects, merchantable, fit for a particular purpose or non-infringing. The entire risk as to the quality and performance of the Covered Software is with You.
7. Limitation of Liability
Under no circumstances and under no legal theory, whether tort, contract, or otherwise, shall any Contributor, or anyone who distributes Covered Software as permitted above, be liable to You for any direct, indirect, special, incidental, or consequential damages of any character including, without limitation, damages for lost profits, loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses, even if such party shall have been informed of the possibility of such damages.
8. Litigation
Any litigation relating to this License may be brought only in the courts of a jurisdiction where the defendant maintains its principal place of business and such litigation shall be governed by laws of that jurisdiction, without reference to its conflict-of-law provisions. Nothing in this Section shall prevent a party’s ability to bring cross-claims or counter-claims.
9. Miscellaneous
This License represents the complete agreement concerning the subject matter hereof. If any provision is held to be unenforceable, such provision shall be reformed only to the extent necessary to make it enforceable. Any law or regulation which provides that the language of a contract shall be construed against the drafter shall not be used to construe this License against a Contributor.
10. Versions of the License
10.1. New Versions. Mozilla Foundation is the license steward. Except as provided in Section 10.3, no one other than the license steward has the right to modify or publish new versions of this License.
10.2. Effect of New Versions. You may distribute the Covered Software under the terms of the version of the License under which You originally received the Covered Software, or under the terms of any subsequent version published by the license steward.
10.3. Modified Versions. If you create software not governed by this License, and you want to create a new license for such software, you may create and use a modified version of this License if you rename the license and remove any references to the name of the license steward (except to note that such modified license differs from this License).
10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses. If You choose to distribute Source Code Form that is Incompatible With Secondary Licenses under the terms of this version of the License, the notice described in Exhibit B of this License must be attached.
Exhibit A - Source Code Form License Notice
This Source Code Form is subject to the terms of the Mozilla Public License, v. 2.0. If a copy of the MPL was not distributed with this file, You can obtain one at https://mozilla.org/MPL/2.0/.
Exhibit B - “Incompatible With Secondary Licenses” Notice
This Source Code Form is “Incompatible With Secondary Licenses”, as defined by the Mozilla Public License, v. 2.0.
## Ropus 0.12.18
Copyright 2001-2023 Xiph.Org, Skype Limited, Octasic, Jean-Marc Valin, Timothy B. Terriberry, CSIRO, Gregory Maxwell, Mark Borgerding, Erik de Castro Lopo, Mozilla, Amazon
Copyright (c) 2026 Martin Davidson (Rust port additions)
Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
- Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
- Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
- Neither the name of Internet Society, IETF or IETF Trust, nor the names of specific contributors, may be used to endorse or promote products derived from this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
Opus is subject to the royalty-free patent licenses specified at:
- Xiph.Org Foundation: https://datatracker.ietf.org/ipr/1524/
- Microsoft Corporation: https://datatracker.ietf.org/ipr/1914/
- Broadcom Corporation: https://datatracker.ietf.org/ipr/1526/
## Rubato 4.0.0 (MIT option)
Copyright (c) 2020 Henrik Enquist
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
## webm-iterable 0.6.4
MIT License
Copyright (c) 2021 Austin Blake
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

1
runtime/src/lib.rs
View File

@ -11,6 +11,7 @@ pub mod runtime_api;
pub mod runtime_certificate;
pub mod file_data;
pub mod metadata;
pub mod media;
pub mod pdfium;
pub mod pandoc;
pub mod qdrant_edge_database;

745
runtime/src/media.rs Normal file
View File

@ -0,0 +1,745 @@
use std::collections::HashMap;
use std::convert::Infallible;
use std::fs::{self, File};
use std::path::{Path as FilePath, PathBuf};
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::{Arc, Mutex, RwLock};
use axum::extract::Path;
use axum::http::StatusCode;
use axum::response::sse::{Event, KeepAlive, Sse};
use axum::response::IntoResponse;
use axum::Json;
use file_format::{FileFormat, Kind};
use futures::Stream;
use once_cell::sync::Lazy;
use ropus::{Application, Bitrate, Channels as OpusChannels, DecodeMode, Decoder as OpusDecoder, Encoder as OpusEncoder};
use rubato::audioadapter::Adapter;
use rubato::audioadapter_buffers::direct::SequentialSliceOfVecs;
use rubato::{Fft, FixedSync, Resampler};
use serde::{Deserialize, Serialize};
use symphonia::core::audio::sample::Sample;
use symphonia::core::codecs::audio::{well_known::CODEC_ID_OPUS, AudioDecoder, AudioDecoderOptions};
use symphonia::core::codecs::CodecParameters;
use symphonia::core::errors::Error as SymphoniaError;
use symphonia::core::formats::{FormatOptions, Track, TrackFlags, TrackType};
use symphonia::core::io::MediaSourceStream;
use symphonia::core::meta::MetadataOptions;
use symphonia::core::formats::probe::Hint;
use symphonia::core::units::Timestamp;
use tokio::sync::broadcast;
use tokio_stream::wrappers::BroadcastStream;
use tokio_stream::StreamExt;
use webm_iterable::matroska_spec::{Master, MatroskaSpec, SimpleBlock};
use webm_iterable::{WebmIterator, WebmWriter, WriteOptions};
use crate::api_token::APIToken;
const OUTPUT_SAMPLE_RATE: u32 = 48_000;
const OPUS_FRAME_SAMPLES: usize = 960;
const OPUS_BITRATE: u32 = 32_000;
const CLUSTER_DURATION_MS: u64 = 30_000;
const OPUS_PRE_SKIP: u16 = 312;
const DEFAULT_MAX_PASS_THROUGH_BYTES: u64 = 25 * 1024 * 1024;
static JOBS: Lazy<RwLock<HashMap<String, Arc<MediaJob>>>> = Lazy::new(|| RwLock::new(HashMap::new()));
#[derive(Debug, Deserialize)]
pub struct CreateMediaJobRequest {
pub input_path: String,
pub output_path: Option<String>,
pub max_pass_through_bytes: Option<u64>,
}
#[derive(Debug, Serialize)]
pub struct CreateMediaJobResponse {
pub job_id: String,
}
#[derive(Clone, Debug, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum MediaJobPhase {
Probing,
Transcoding,
Completed,
Failed,
Cancelled,
}
#[derive(Clone, Debug, Serialize)]
pub struct MediaJobEvent {
pub phase: MediaJobPhase,
pub progress: Option<f64>,
pub result: Option<MediaJobResult>,
pub error: Option<MediaError>,
}
#[derive(Clone, Debug, Serialize)]
pub struct MediaJobResult {
pub output_path: String,
pub detected_format: String,
pub detected_codec: String,
pub duration_ms: u64,
pub pass_through: bool,
}
#[derive(Clone, Debug, Serialize)]
pub struct MediaError {
pub code: String,
pub message: String,
}
impl MediaError {
fn new(code: &str, message: impl Into<String>) -> Self {
Self { code: code.to_string(), message: message.into() }
}
}
struct MediaJob {
cancelled: AtomicBool,
current: Mutex<MediaJobEvent>,
events: broadcast::Sender<MediaJobEvent>,
}
impl MediaJob {
fn new() -> Self {
let initial = MediaJobEvent {
phase: MediaJobPhase::Probing,
progress: Some(0.0),
result: None,
error: None,
};
let (events, _) = broadcast::channel(32);
Self { cancelled: AtomicBool::new(false), current: Mutex::new(initial), events }
}
fn publish(&self, event: MediaJobEvent) {
*self.current.lock().unwrap() = event.clone();
let _ = self.events.send(event);
}
fn is_cancelled(&self) -> bool {
self.cancelled.load(Ordering::Relaxed)
}
}
pub async fn create_job(
_token: APIToken,
Json(request): Json<CreateMediaJobRequest>,
) -> Result<Json<CreateMediaJobResponse>, (StatusCode, Json<MediaError>)> {
let input_path = PathBuf::from(&request.input_path);
if !input_path.is_file() {
return Err((StatusCode::BAD_REQUEST, Json(MediaError::new("file_not_found", "The selected media file does not exist."))));
}
let output_path = request.output_path.map(PathBuf::from).unwrap_or_else(|| {
let parent = input_path.parent().unwrap_or_else(|| FilePath::new("."));
let stem = input_path.file_stem().and_then(|value| value.to_str()).unwrap_or("media");
parent.join(format!("{stem}-normalized.webm"))
});
let job_id = format!("{}-{}", std::process::id(), rand::random::<u64>());
let job = Arc::new(MediaJob::new());
JOBS.write().unwrap().insert(job_id.clone(), Arc::clone(&job));
let completed_job_id = job_id.clone();
tauri::async_runtime::spawn(async move {
let max_pass_through_bytes = request.max_pass_through_bytes.unwrap_or(DEFAULT_MAX_PASS_THROUGH_BYTES);
let task_job = Arc::clone(&job);
let result = tokio::task::spawn_blocking(move || normalize_media(&input_path, &output_path, max_pass_through_bytes, &task_job)).await;
match result {
Ok(Ok(result)) => job.publish(MediaJobEvent {
phase: MediaJobPhase::Completed,
progress: Some(1.0),
result: Some(result),
error: None,
}),
Ok(Err(error)) if error.code == "cancelled" => job.publish(MediaJobEvent {
phase: MediaJobPhase::Cancelled,
progress: None,
result: None,
error: None,
}),
Ok(Err(error)) => job.publish(MediaJobEvent {
phase: MediaJobPhase::Failed,
progress: None,
result: None,
error: Some(error),
}),
Err(error) => job.publish(MediaJobEvent {
phase: MediaJobPhase::Failed,
progress: None,
result: None,
error: Some(MediaError::new("internal_error", format!("The media worker failed: {error}"))),
}),
}
tokio::time::sleep(std::time::Duration::from_secs(600)).await;
JOBS.write().unwrap().remove(&completed_job_id);
});
Ok(Json(CreateMediaJobResponse { job_id }))
}
pub async fn get_job_events(
_token: APIToken,
Path(job_id): Path<String>,
) -> Result<Sse<impl Stream<Item = Result<Event, Infallible>>>, StatusCode> {
let job = JOBS.read().unwrap().get(&job_id).cloned().ok_or(StatusCode::NOT_FOUND)?;
let current = job.current.lock().unwrap().clone();
let initial = tokio_stream::once(current);
let updates = BroadcastStream::new(job.events.subscribe()).filter_map(|event| event.ok());
let stream = initial.chain(updates).map(|event| {
let data = serde_json::to_string(&event).unwrap_or_else(|_| "{}".to_string());
Ok(Event::default().event(phase_name(&event.phase)).data(data))
});
Ok(Sse::new(stream).keep_alive(KeepAlive::default()))
}
pub async fn cancel_job(_token: APIToken, Path(job_id): Path<String>) -> impl IntoResponse {
match JOBS.read().unwrap().get(&job_id) {
Some(job) => {
job.cancelled.store(true, Ordering::Relaxed);
StatusCode::NO_CONTENT
}
None => StatusCode::NOT_FOUND,
}
}
fn phase_name(phase: &MediaJobPhase) -> &'static str {
match phase {
MediaJobPhase::Probing => "probing",
MediaJobPhase::Transcoding => "transcoding",
MediaJobPhase::Completed => "completed",
MediaJobPhase::Failed => "failed",
MediaJobPhase::Cancelled => "cancelled",
}
}
fn normalize_media(input_path: &FilePath, output_path: &FilePath, max_pass_through_bytes: u64, job: &MediaJob) -> Result<MediaJobResult, MediaError> {
check_cancelled(job)?;
let detected = FileFormat::from_file(input_path)
.map_err(|error| MediaError::new("unknown_format", format!("The file type could not be identified: {error}")))?;
if detected.kind() == Kind::Executable {
return Err(MediaError::new("unsafe_file", "The selected file contains executable data and cannot be processed as media."));
}
if !matches!(detected.kind(), Kind::Audio | Kind::Video)
&& !matches!(detected, FileFormat::ExtensibleBinaryMetaLanguage)
{
return Err(MediaError::new("not_media", format!("The selected file is not supported media (detected as {detected:?}).")));
}
let file = File::open(input_path).map_err(|error| MediaError::new("file_open_failed", error.to_string()))?;
let mss = MediaSourceStream::new(Box::new(file), Default::default());
let mut hint = Hint::new();
if let Some(extension) = input_path.extension().and_then(|value| value.to_str()) {
hint.with_extension(extension);
}
let mut format = symphonia::default::get_probe()
.probe(&hint, mss, FormatOptions::default(), MetadataOptions::default())
.map_err(map_probe_error)?;
let detected_format = format!("{detected:?} / {}", format.format_info().long_name);
let tracks = format.tracks();
let mut audio_tracks: Vec<&Track> = tracks.iter().filter(|track| {
matches!(track.codec_params, Some(CodecParameters::Audio(_)))
}).collect();
if audio_tracks.is_empty() {
return Err(MediaError::new("no_audio_track", "The selected media file does not contain an audio track."));
}
audio_tracks.sort_by_key(|track| !track.flags.contains(TrackFlags::DEFAULT));
let selected = audio_tracks.into_iter().find(|track| is_decodable(track))
.ok_or_else(|| MediaError::new("unsupported_codec", "None of the audio tracks uses a supported codec."))?;
let track_id = selected.id;
let params = selected.codec_params.as_ref().and_then(CodecParameters::audio).unwrap().clone();
let detected_codec = if params.codec == CODEC_ID_OPUS { "opus".to_string() } else { format!("{}", params.codec) };
let duration_ms = selected.time_base.zip(selected.duration).and_then(|(time_base, duration)| {
let timestamp = Timestamp::new(i64::try_from(duration.get()).ok()?);
let time = time_base.calc_time(timestamp)?;
Some((time.as_secs_f64() * 1000.0).max(0.0).round() as u64)
}).unwrap_or(0);
let audio_only = tracks.iter().all(|track| track.track_type() != Some(TrackType::Video));
let channels = params.channels.as_ref().map(|value| value.count()).unwrap_or(0);
let pass_through = is_webm_container(input_path)
&& audio_only
&& tracks.iter().filter(|track| track.track_type() == Some(TrackType::Audio)).count() == 1
&& params.codec == CODEC_ID_OPUS
&& params.sample_rate == Some(OUTPUT_SAMPLE_RATE)
&& channels == 1
&& input_path.metadata().map(|metadata| metadata.len() <= max_pass_through_bytes).unwrap_or(false);
let partial_path = partial_path(output_path);
if let Some(parent) = partial_path.parent() {
fs::create_dir_all(parent).map_err(|error| MediaError::new("output_create_failed", error.to_string()))?;
}
let result = if pass_through {
check_cancelled(job)?;
fs::copy(input_path, &partial_path).map_err(|error| MediaError::new("output_write_failed", error.to_string()))?;
check_cancelled(job)?;
Ok(MediaJobResult {
output_path: output_path.to_string_lossy().into_owned(),
detected_format: detected_format.clone(),
detected_codec,
duration_ms,
pass_through: true,
})
} else {
job.publish(MediaJobEvent {
phase: MediaJobPhase::Transcoding,
progress: Some(0.0),
result: None,
error: None,
});
transcode(&mut *format, track_id, &params, &partial_path, output_path, detected_format, detected_codec, duration_ms, job)
};
match result {
Ok(result) => {
if let Err(error) = fs::rename(&partial_path, output_path) {
let _ = fs::remove_file(&partial_path);
return Err(MediaError::new("output_commit_failed", error.to_string()));
}
Ok(result)
}
Err(error) => {
let _ = fs::remove_file(&partial_path);
Err(error)
}
}
}
fn is_decodable(track: &Track) -> bool {
let Some(params) = track.codec_params.as_ref().and_then(CodecParameters::audio) else { return false; };
params.codec == CODEC_ID_OPUS || symphonia::default::get_codecs().make_audio_decoder(params, &AudioDecoderOptions::default()).is_ok()
}
fn transcode(
format: &mut dyn symphonia::core::formats::FormatReader,
track_id: u32,
params: &symphonia::core::codecs::audio::AudioCodecParameters,
partial_path: &FilePath,
output_path: &FilePath,
detected_format: String,
detected_codec: String,
expected_duration_ms: u64,
job: &MediaJob,
) -> Result<MediaJobResult, MediaError> {
let input_rate = params.sample_rate.unwrap_or(OUTPUT_SAMPLE_RATE);
let input_channels = params.channels.as_ref().map(|value| value.count()).unwrap_or(1).max(1);
let mut symphonia_decoder: Option<Box<dyn AudioDecoder>> = if params.codec == CODEC_ID_OPUS {
None
} else {
Some(symphonia::default::get_codecs().make_audio_decoder(params, &AudioDecoderOptions::default())
.map_err(|_| MediaError::new("unsupported_codec", "The selected audio codec is not supported."))?)
};
let mut opus_decoder = if params.codec == CODEC_ID_OPUS {
let channels = if input_channels == 1 { OpusChannels::Mono } else { OpusChannels::Stereo };
Some(OpusDecoder::new(OUTPUT_SAMPLE_RATE, channels)
.map_err(|error| MediaError::new("decoder_init_failed", error.to_string()))?)
} else { None };
let mut opus_encoder = OpusEncoder::builder(OUTPUT_SAMPLE_RATE, OpusChannels::Mono, Application::Audio)
.bitrate(Bitrate::Bits(OPUS_BITRATE))
.vbr(true)
.build()
.map_err(|error| MediaError::new("encoder_init_failed", error.to_string()))?;
let file = File::create(partial_path).map_err(|error| MediaError::new("output_create_failed", error.to_string()))?;
let mut writer = WebmOpusWriter::new(file)?;
let mut pending = Vec::<f32>::with_capacity(OPUS_FRAME_SAMPLES * 3);
let mut interleaved = Vec::<f32>::new();
let mut opus_pcm = vec![0i16; 5_760 * input_channels];
let mut encoded = [0u8; 4_000];
let mut produced_samples = 0u64;
let mut input_samples = 0u64;
let mut resampled_samples = 0u64;
loop {
check_cancelled(job)?;
let packet = match format.next_packet() {
Ok(Some(packet)) => packet,
Ok(None) => break,
Err(SymphoniaError::ResetRequired) => return Err(MediaError::new("stream_reset", "The media stream changed unexpectedly.")),
Err(SymphoniaError::IoError(error)) if error.kind() == std::io::ErrorKind::UnexpectedEof => break,
Err(error) => return Err(MediaError::new("damaged_container", format!("The media container is damaged: {error}"))),
};
if packet.track_id != track_id {
continue;
}
let mono = if let Some(decoder) = symphonia_decoder.as_mut() {
let decoded = match decoder.decode(&packet) {
Ok(decoded) => decoded,
Err(SymphoniaError::DecodeError(_)) => continue,
Err(error) => return Err(MediaError::new("decode_failed", format!("Audio decoding failed: {error}"))),
};
interleaved.resize(decoded.samples_interleaved(), f32::MID);
decoded.copy_to_slice_interleaved(&mut interleaved);
downmix_to_mono(&interleaved, decoded.num_planes())
} else {
let decoder = opus_decoder.as_mut().unwrap();
let frames = decoder.decode(&packet.data, &mut opus_pcm, DecodeMode::Normal)
.map_err(|error| MediaError::new("decode_failed", error.to_string()))?;
let samples = &opus_pcm[..frames * input_channels];
if input_channels == 1 {
samples.iter().map(|sample| f32::from(*sample) / 32768.0).collect()
} else {
samples.chunks_exact(input_channels).map(|frame| {
frame.iter().map(|sample| f32::from(*sample) / 32768.0).sum::<f32>() / input_channels as f32
}).collect()
}
};
input_samples += mono.len() as u64;
let expected_resampled_samples = input_samples.saturating_mul(u64::from(OUTPUT_SAMPLE_RATE)) / u64::from(input_rate);
let expected_chunk_samples = expected_resampled_samples.saturating_sub(resampled_samples) as usize;
let mut resampled = resample_chunk(&mono, input_rate)?;
resampled.resize(expected_chunk_samples, 0.0);
resampled.truncate(expected_chunk_samples);
resampled_samples += resampled.len() as u64;
pending.extend_from_slice(&resampled);
let mut consumed = 0usize;
while pending.len() - consumed >= OPUS_FRAME_SAMPLES {
check_cancelled(job)?;
let frame = &pending[consumed..consumed + OPUS_FRAME_SAMPLES];
let length = opus_encoder.encode_float(frame, &mut encoded)
.map_err(|error| MediaError::new("encode_failed", error.to_string()))?;
writer.write_packet(&encoded[..length], produced_samples)?;
produced_samples += OPUS_FRAME_SAMPLES as u64;
consumed += OPUS_FRAME_SAMPLES;
}
if consumed > 0 {
pending.drain(..consumed);
}
if expected_duration_ms > 0 {
let current_ms = produced_samples.saturating_mul(1000) / u64::from(OUTPUT_SAMPLE_RATE);
job.publish(MediaJobEvent {
phase: MediaJobPhase::Transcoding,
progress: Some((current_ms as f64 / expected_duration_ms as f64).clamp(0.0, 0.99)),
result: None,
error: None,
});
}
}
if !pending.is_empty() {
pending.resize(OPUS_FRAME_SAMPLES, 0.0);
let length = opus_encoder.encode_float(&pending, &mut encoded)
.map_err(|error| MediaError::new("encode_failed", error.to_string()))?;
writer.write_packet(&encoded[..length], produced_samples)?;
produced_samples += OPUS_FRAME_SAMPLES as u64;
}
writer.finish()?;
check_cancelled(job)?;
Ok(MediaJobResult {
output_path: output_path.to_string_lossy().into_owned(),
detected_format,
detected_codec,
duration_ms: produced_samples.saturating_mul(1000) / u64::from(OUTPUT_SAMPLE_RATE),
pass_through: false,
})
}
fn downmix_to_mono(samples: &[f32], channels: usize) -> Vec<f32> {
if channels <= 1 {
return samples.to_vec();
}
samples.chunks_exact(channels).map(|frame| frame.iter().copied().sum::<f32>() / channels as f32).collect()
}
fn resample_chunk(samples: &[f32], input_rate: u32) -> Result<Vec<f32>, MediaError> {
if input_rate == OUTPUT_SAMPLE_RATE || samples.is_empty() {
return Ok(samples.to_vec());
}
let input_data = vec![samples.to_vec()];
let input = SequentialSliceOfVecs::new(&input_data, 1, samples.len())
.map_err(|error| MediaError::new("resample_failed", error.to_string()))?;
let mut resampler = Fft::<f32>::new(input_rate as usize, OUTPUT_SAMPLE_RATE as usize, samples.len().max(256), 1, FixedSync::Input)
.map_err(|error| MediaError::new("resample_failed", error.to_string()))?;
let output = resampler.process_all(&input, samples.len(), None)
.map_err(|error| MediaError::new("resample_failed", error.to_string()))?;
let mut result = Vec::with_capacity(output.frames());
for frame in 0..output.frames() {
result.push(output.read_sample(0, frame).unwrap_or(0.0));
}
Ok(result)
}
struct WebmOpusWriter {
writer: WebmWriter<File>,
cluster_start_ms: Option<u64>,
}
impl WebmOpusWriter {
fn new(file: File) -> Result<Self, MediaError> {
let mut writer = WebmWriter::new(file);
write_tags(&mut writer, &[
MatroskaSpec::Ebml(Master::Start),
MatroskaSpec::EbmlVersion(1),
MatroskaSpec::EbmlReadVersion(1),
MatroskaSpec::EbmlMaxIdLength(4),
MatroskaSpec::EbmlMaxSizeLength(8),
MatroskaSpec::DocType("webm".to_string()),
MatroskaSpec::DocTypeVersion(4),
MatroskaSpec::DocTypeReadVersion(2),
MatroskaSpec::Ebml(Master::End),
])?;
writer.write_advanced(&MatroskaSpec::Segment(Master::Start), WriteOptions::is_unknown_sized_element()).map_err(webm_error)?;
write_tags(&mut writer, &[
MatroskaSpec::Info(Master::Start),
MatroskaSpec::TimestampScale(1_000_000),
MatroskaSpec::MuxingApp("MindWork AI Studio".to_string()),
MatroskaSpec::WritingApp("MindWork AI Studio".to_string()),
MatroskaSpec::Info(Master::End),
MatroskaSpec::Tracks(Master::Start),
MatroskaSpec::TrackEntry(Master::Start),
MatroskaSpec::TrackNumber(1),
MatroskaSpec::TrackUID(1),
MatroskaSpec::TrackType(2),
MatroskaSpec::FlagDefault(1),
MatroskaSpec::CodecID("A_OPUS".to_string()),
MatroskaSpec::CodecPrivate(opus_head()),
MatroskaSpec::CodecDelay(u64::from(OPUS_PRE_SKIP) * 1_000_000_000 / u64::from(OUTPUT_SAMPLE_RATE)),
MatroskaSpec::SeekPreRoll(80_000_000),
MatroskaSpec::Audio(Master::Start),
MatroskaSpec::SamplingFrequency(f64::from(OUTPUT_SAMPLE_RATE)),
MatroskaSpec::Channels(1),
MatroskaSpec::Audio(Master::End),
MatroskaSpec::TrackEntry(Master::End),
MatroskaSpec::Tracks(Master::End),
])?;
Ok(Self { writer, cluster_start_ms: None })
}
fn write_packet(&mut self, packet: &[u8], sample_position: u64) -> Result<(), MediaError> {
let timestamp_ms = sample_position.saturating_mul(1000) / u64::from(OUTPUT_SAMPLE_RATE);
let rotate = self.cluster_start_ms.map(|start| timestamp_ms.saturating_sub(start) >= CLUSTER_DURATION_MS).unwrap_or(true);
if rotate {
if self.cluster_start_ms.is_some() {
self.writer.write(&MatroskaSpec::Cluster(Master::End)).map_err(webm_error)?;
}
self.writer.write(&MatroskaSpec::Cluster(Master::Start)).map_err(webm_error)?;
self.writer.write(&MatroskaSpec::Timestamp(timestamp_ms)).map_err(webm_error)?;
self.cluster_start_ms = Some(timestamp_ms);
}
let relative = timestamp_ms.saturating_sub(self.cluster_start_ms.unwrap_or(timestamp_ms));
if relative > i16::MAX as u64 {
return Err(MediaError::new("webm_timestamp_overflow", "The WebM cluster timestamp exceeded its safe range."));
}
let block: MatroskaSpec = SimpleBlock::new_uncheked(packet, 1, relative as i16, false, None, false, true).into();
self.writer.write(&block).map_err(webm_error)
}
fn finish(mut self) -> Result<(), MediaError> {
if self.cluster_start_ms.is_some() {
self.writer.write(&MatroskaSpec::Cluster(Master::End)).map_err(webm_error)?;
}
self.writer.write(&MatroskaSpec::Segment(Master::End)).map_err(webm_error)?;
self.writer.into_inner().map_err(webm_error)?;
Ok(())
}
}
fn write_tags(writer: &mut WebmWriter<File>, tags: &[MatroskaSpec]) -> Result<(), MediaError> {
for tag in tags {
writer.write(tag).map_err(webm_error)?;
}
Ok(())
}
fn opus_head() -> Vec<u8> {
let mut data = b"OpusHead".to_vec();
data.push(1);
data.push(1);
data.extend_from_slice(&OPUS_PRE_SKIP.to_le_bytes());
data.extend_from_slice(&OUTPUT_SAMPLE_RATE.to_le_bytes());
data.extend_from_slice(&0i16.to_le_bytes());
data.push(0);
data
}
fn partial_path(output_path: &FilePath) -> PathBuf {
let mut name = output_path.file_name().unwrap_or_default().to_os_string();
name.push(".partial");
output_path.with_file_name(name)
}
fn is_webm_container(path: &FilePath) -> bool {
let Ok(file) = File::open(path) else { return false; };
WebmIterator::new(file, &[]).take(16).filter_map(Result::ok).any(|tag| {
matches!(tag, MatroskaSpec::DocType(doc_type) if doc_type.eq_ignore_ascii_case("webm"))
})
}
fn check_cancelled(job: &MediaJob) -> Result<(), MediaError> {
if job.is_cancelled() {
Err(MediaError::new("cancelled", "The media job was cancelled."))
} else {
Ok(())
}
}
fn map_probe_error(error: SymphoniaError) -> MediaError {
match error {
SymphoniaError::Unsupported(_) => MediaError::new("unsupported_container", "This media container or codec is not supported."),
_ => MediaError::new("damaged_container", format!("The media container could not be read: {error}")),
}
}
fn webm_error(error: impl std::fmt::Display) -> MediaError {
MediaError::new("webm_write_failed", format!("The WebM output could not be written: {error}"))
}
#[cfg(test)]
mod tests {
use super::*;
use std::io::Cursor;
#[test]
fn opus_head_describes_48_khz_mono() {
let head = opus_head();
assert_eq!(&head[..8], b"OpusHead");
assert_eq!(head[9], 1);
assert_eq!(u32::from_le_bytes(head[12..16].try_into().unwrap()), 48_000);
}
#[test]
fn webm_writer_rotates_clusters_before_relative_timestamp_overflow() {
let path = std::env::temp_dir().join(format!("ai-studio-media-writer-{}.webm", rand::random::<u64>()));
let file = File::create(&path).unwrap();
let mut writer = WebmOpusWriter::new(file).unwrap();
writer.write_packet(&[0xf8, 0xff, 0xfe], 0).unwrap();
writer.write_packet(&[0xf8, 0xff, 0xfe], 31 * 48_000).unwrap();
writer.finish().unwrap();
let bytes = fs::read(&path).unwrap();
let clusters = WebmIterator::new(Cursor::new(bytes), &[])
.filter_map(Result::ok)
.filter(|tag| matches!(tag, MatroskaSpec::Cluster(Master::Start)))
.count();
let _ = fs::remove_file(path);
assert_eq!(clusters, 2);
}
#[test]
fn downmix_is_bounded_and_balanced() {
assert_eq!(downmix_to_mono(&[1.0, -1.0, 0.5, 0.5], 2), vec![0.0, 0.5]);
}
#[test]
fn wav_is_normalized_to_one_mono_opus_track_with_frame_bounded_duration() {
let directory = std::env::temp_dir().join(format!("ai-studio-media-test-{}", rand::random::<u64>()));
fs::create_dir_all(&directory).unwrap();
let input = directory.join("input.wav");
let output = directory.join("output.webm");
fs::write(&input, wav_silence(44_100, 4_410)).unwrap();
let job = MediaJob::new();
let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap();
assert!(!result.pass_through);
assert!(result.duration_ms.abs_diff(100) <= 20);
let file = File::open(&output).unwrap();
let tags: Vec<_> = WebmIterator::new(file, &[])
.filter_map(Result::ok)
.collect();
assert_eq!(tags.iter().filter(|tag| matches!(tag, MatroskaSpec::TrackEntry(Master::Start))).count(), 1);
assert!(tags.iter().any(|tag| matches!(tag, MatroskaSpec::CodecID(codec) if codec == "A_OPUS")));
assert!(tags.iter().any(|tag| matches!(tag, MatroskaSpec::Channels(1))));
assert!(tags.iter().any(|tag| matches!(tag, MatroskaSpec::SamplingFrequency(rate) if *rate == 48_000.0)));
let _ = fs::remove_dir_all(directory);
}
#[test]
fn cancellation_does_not_leave_an_output_file() {
let directory = std::env::temp_dir().join(format!("ai-studio-media-cancel-{}", rand::random::<u64>()));
fs::create_dir_all(&directory).unwrap();
let input = directory.join("input.wav");
let output = directory.join("output.webm");
fs::write(&input, wav_silence(48_000, 960)).unwrap();
let job = MediaJob::new();
job.cancelled.store(true, Ordering::Relaxed);
let error = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap_err();
assert_eq!(error.code, "cancelled");
assert!(!output.exists());
assert!(!partial_path(&output).exists());
let _ = fs::remove_dir_all(directory);
}
#[test]
fn suitable_audio_only_webm_opus_is_passed_through() {
let directory = std::env::temp_dir().join(format!("ai-studio-media-pass-through-{}", rand::random::<u64>()));
fs::create_dir_all(&directory).unwrap();
let input = directory.join("input.webm");
let output = directory.join("output.webm");
let mut writer = WebmOpusWriter::new(File::create(&input).unwrap()).unwrap();
writer.write_packet(&[0xf8, 0xff, 0xfe], 0).unwrap();
writer.finish().unwrap();
let job = MediaJob::new();
let result = normalize_media(&input, &output, DEFAULT_MAX_PASS_THROUGH_BYTES, &job).unwrap();
assert!(result.pass_through);
assert_eq!(fs::read(input).unwrap(), fs::read(output).unwrap());
let _ = fs::remove_dir_all(directory);
}
fn wav_silence(sample_rate: u32, samples: u32) -> Vec<u8> {
let data_size = samples * 2;
let mut wav = Vec::with_capacity(44 + data_size as usize);
wav.extend_from_slice(b"RIFF");
wav.extend_from_slice(&(36 + data_size).to_le_bytes());
wav.extend_from_slice(b"WAVEfmt ");
wav.extend_from_slice(&16u32.to_le_bytes());
wav.extend_from_slice(&1u16.to_le_bytes());
wav.extend_from_slice(&1u16.to_le_bytes());
wav.extend_from_slice(&sample_rate.to_le_bytes());
wav.extend_from_slice(&(sample_rate * 2).to_le_bytes());
wav.extend_from_slice(&2u16.to_le_bytes());
wav.extend_from_slice(&16u16.to_le_bytes());
wav.extend_from_slice(b"data");
wav.extend_from_slice(&data_size.to_le_bytes());
wav.resize(44 + data_size as usize, 0);
wav
}
}

5
runtime/src/runtime_api.rs
View File

@ -1,6 +1,6 @@
use log::info;
use once_cell::sync::Lazy;
use axum::routing::{get, post};
use axum::routing::{delete, get, post};
use axum::Router;
use axum_server::tls_rustls::RustlsConfig;
use std::net::SocketAddr;
@ -59,6 +59,9 @@ pub fn start_runtime_api() {
.route("/system/enterprise/config/encryption_secret", get(crate::environment::read_enterprise_env_config_encryption_secret))
.route("/system/enterprise/configs", get(crate::environment::read_enterprise_configs))
.route("/retrieval/fs/extract", get(crate::file_data::extract_data))
.route("/media/jobs", post(crate::media::create_job))
.route("/media/jobs/{id}/events", get(crate::media::get_job_events))
.route("/media/jobs/{id}", delete(crate::media::cancel_job))
.route("/log/paths", get(crate::log::get_log_paths))
.route("/log/event", post(crate::log::log_event))
.route("/shortcuts/register", post(crate::app_window::register_shortcut))

3
runtime/tauri.conf.json
View File

@ -27,7 +27,8 @@
"../app/MindWork AI Studio/bin/dist/mindworkAIStudioServer"
],
"resources": [
"resources/libraries/*"
"resources/libraries/*",
"resources/notices/*"
],
"macOS": {
"exceptionDomain": "localhost"