mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-04 02:07:21 +00:00
Added transcription of audio and video files to local data sources (#1081)
Co-authored-by: Paul Koudelka <106623909+PaulKoudelka@users.noreply.github.com>
This commit is contained in:
Thorsten SommerandPaul Koudelka authored and 
GitHub committed
1 parent
8642ce2d81
commit
752f7d828a
82 files changed
+3231
-302
No files matched your search
+14
-88
@@ -1,7 +1,4 @@
|
||||
using System.Text;
|
||||
|
||||
using AIStudio.Tools.Media;
|
||||
using AIStudio.Tools.Rust;
|
||||
|
||||
namespace AIStudio.Assistants.BatchProcessing;
|
||||
|
||||
@@ -69,108 +66,37 @@ public partial class AssistantBatchProcessing
|
||||
|
||||
private async Task<string?> LoadMediaTranscriptAsync(BatchProcessingFileResult fileResult, CancellationToken token)
|
||||
{
|
||||
var transcriptFilePath = GetTranscriptFilePath(fileResult.FilePath);
|
||||
if (File.Exists(transcriptFilePath))
|
||||
{
|
||||
try
|
||||
{
|
||||
var existingTranscript = await File.ReadAllTextAsync(transcriptFilePath, token);
|
||||
if (!string.IsNullOrWhiteSpace(existingTranscript))
|
||||
{
|
||||
this.Logger.LogInformation("Reusing the existing batch transcript '{TranscriptFilePath}' for media file '{MediaFilePath}'.", transcriptFilePath, fileResult.FilePath);
|
||||
return existingTranscript;
|
||||
}
|
||||
|
||||
this.Logger.LogWarning("The existing batch transcript '{TranscriptFilePath}' for media file '{MediaFilePath}' is empty and will be replaced.", transcriptFilePath, fileResult.FilePath);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.CANCELED, T("The batch run was canceled."));
|
||||
return null;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("Was not able to read the existing transcript: {0}"), e.Message), e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
if (!this.MediaTranscriptionService.HasUsableTranscriptionProvider)
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("No usable transcription provider is configured."));
|
||||
return null;
|
||||
}
|
||||
|
||||
var transcription = await this.MediaTranscriptionService.TranscribeAsync(fileResult.FilePath, token);
|
||||
if (transcription.Status is MediaTranscriptionResultStatus.CANCELLED)
|
||||
var provider = this.MediaTranscriptionService.GetUsableProvider();
|
||||
var transcript = await PersistentMediaTranscript.ReadOrCreateAsync(MediaSidecarTranscripts.INSTANCE, fileResult.FilePath, provider, this.MediaTranscriptionService.TranscribeAsync, this.Logger, token);
|
||||
if (transcript.Status is PersistentMediaTranscriptStatus.CANCELLED)
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.CANCELED, T("The batch run was canceled."));
|
||||
return null;
|
||||
}
|
||||
|
||||
if (transcription.Status is not MediaTranscriptionResultStatus.SUCCEEDED)
|
||||
if (transcript.Status is not PersistentMediaTranscriptStatus.READY)
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, transcription.UserMessage);
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, transcript.UserMessage);
|
||||
return null;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(transcription.Text))
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("The transcription provider returned an empty transcript."));
|
||||
return null;
|
||||
}
|
||||
|
||||
return await this.StoreMediaTranscriptAsync(fileResult, transcriptFilePath, transcription.Text);
|
||||
}
|
||||
|
||||
private async Task<string?> StoreMediaTranscriptAsync(BatchProcessingFileResult fileResult, string transcriptFilePath, string transcript)
|
||||
{
|
||||
var tempFilePath = transcriptFilePath + ".tmp";
|
||||
try
|
||||
{
|
||||
// Complete the small persistence step even if cancellation arrived
|
||||
// after transcription, so the expensive provider result can be
|
||||
// reused when the interrupted batch is continued.
|
||||
await File.WriteAllTextAsync(tempFilePath, transcript, new UTF8Encoding(false), CancellationToken.None);
|
||||
File.Move(tempFilePath, transcriptFilePath, true);
|
||||
this.Logger.LogInformation("Stored the batch transcript '{TranscriptFilePath}' next to media file '{MediaFilePath}'.", transcriptFilePath, fileResult.FilePath);
|
||||
return transcript;
|
||||
return await File.ReadAllTextAsync(transcript.TranscriptPath, token);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.CANCELED, T("The batch run was canceled."));
|
||||
return null;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("Was not able to store the transcript next to the media file: {0}"), e.Message), e);
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("Was not able to read the existing transcript: {0}"), e.Message), e);
|
||||
return null;
|
||||
}
|
||||
finally
|
||||
{
|
||||
try
|
||||
{
|
||||
if (File.Exists(tempFilePath))
|
||||
File.Delete(tempFilePath);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.Logger.LogWarning(e, "Was not able to remove the temporary batch transcript '{TempFilePath}'.", tempFilePath);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private static bool IsTranscribableMedia(string filePath) => FileTypes.IsAllowedPath(filePath, FileTypes.AUDIO, FileTypes.VIDEO);
|
||||
private static bool IsTranscribableMedia(string filePath) => PersistentMediaTranscript.IsMediaPath(filePath);
|
||||
|
||||
private static string GetTranscriptFilePath(string mediaFilePath) => mediaFilePath + TRANSCRIPT_FILE_SUFFIX;
|
||||
|
||||
private static bool HasReusableTranscript(string mediaFilePath)
|
||||
{
|
||||
var transcriptFilePath = GetTranscriptFilePath(mediaFilePath);
|
||||
try
|
||||
{
|
||||
return File.Exists(transcriptFilePath) && new FileInfo(transcriptFilePath).Length > 0;
|
||||
}
|
||||
catch
|
||||
{
|
||||
// The concrete read error is reported when the affected file is
|
||||
// processed. Here we only decide whether a provider is required.
|
||||
return File.Exists(transcriptFilePath);
|
||||
}
|
||||
}
|
||||
private static bool HasReusableTranscript(string mediaFilePath) => MediaSidecarTranscripts.HasReusableTranscript(mediaFilePath);
|
||||
}
|
||||
+2
-2
@@ -1,3 +1,4 @@
|
||||
using AIStudio.Tools.Media;
|
||||
|
||||
namespace AIStudio.Assistants.BatchProcessing;
|
||||
|
||||
@@ -160,8 +161,7 @@ public partial class AssistantBatchProcessing
|
||||
/// </summary>
|
||||
private static bool IsTranscriptArtifact(string filePath)
|
||||
{
|
||||
var fileName = Path.GetFileName(filePath);
|
||||
return fileName.EndsWith(TRANSCRIPT_FILE_SUFFIX, StringComparison.OrdinalIgnoreCase) || fileName.EndsWith(TRANSCRIPT_FILE_SUFFIX + ".tmp", StringComparison.OrdinalIgnoreCase);
|
||||
return MediaSidecarTranscripts.IsArtifactPath(filePath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
||||
@@ -19,7 +19,6 @@ public partial class AssistantBatchProcessing : AssistantBaseCore<SettingsDialog
|
||||
private const string DEFAULT_RESULTS_FILENAME = "batch-results.csv";
|
||||
private const string CSV_EXTENSION = ".csv";
|
||||
private const string RESULT_FILE_SUFFIX = "_result";
|
||||
private const string TRANSCRIPT_FILE_SUFFIX = ".transcript.md";
|
||||
private const string TIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
|
||||
private const char LOG_SEPARATOR = ';';
|
||||
|
||||
|
||||
@@ -343,9 +343,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T4242312602"] = "Send to .
|
||||
-- Copy result
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::ASSISTANTBASE::T83711157"] = "Copy result"
|
||||
|
||||
-- The transcription provider returned an empty transcript.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1080540822"] = "The transcription provider returned an empty transcript."
|
||||
|
||||
-- Hide Expert Settings
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1108876344"] = "Hide Expert Settings"
|
||||
|
||||
@@ -460,9 +457,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Tools for this batch run
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2247412388"] = "Tools for this batch run"
|
||||
|
||||
-- No usable transcription provider is configured.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2282521655"] = "No usable transcription provider is configured."
|
||||
|
||||
-- Was not able to create the output folder: {0}
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2290092642"] = "Was not able to create the output folder: {0}"
|
||||
|
||||
@@ -580,9 +574,6 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- {0} of {1} files processed
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3648144402"] = "{0} of {1} files processed"
|
||||
|
||||
-- Was not able to store the transcript next to the media file: {0}
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3691287653"] = "Was not able to store the transcript next to the media file: {0}"
|
||||
|
||||
-- Time
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3756319748"] = "Time"
|
||||
|
||||
@@ -3958,6 +3949,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2373663376"] = "Good to kno
|
||||
-- Data folder
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2422278827"] = "Data folder"
|
||||
|
||||
-- The transcripts of the audio and video files in your local data sources.
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T260303721"] = "The transcripts of the audio and video files in your local data sources."
|
||||
|
||||
-- Explanation
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T2840582448"] = "Explanation"
|
||||
|
||||
@@ -4036,9 +4030,18 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T1159107
|
||||
-- No, I will choose another embedding
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T1246976418"] = "No, I will choose another embedding"
|
||||
|
||||
-- No, I will choose other providers or files
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T1811383874"] = "No, I will choose other providers or files"
|
||||
|
||||
-- Warning: Your transcription provider '{0}' is not self-hosted. Audio and video files this data source selects are sent to it to be transcribed, which can cost money. AI Studio keeps every transcript, so an unchanged recording is sent only once. MindWork AI Studio has no control over what that third party does with the recordings after they are sent.
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T200667320"] = "Warning: Your transcription provider '{0}' is not self-hosted. Audio and video files this data source selects are sent to it to be transcribed, which can cost money. AI Studio keeps every transcript, so an unchanged recording is sent only once. MindWork AI Studio has no control over what that third party does with the recordings after they are sent."
|
||||
|
||||
-- Every mail of this mailbox in the selected folder and period, together with the text of its attachments,
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T2046044636"] = "Every mail of this mailbox in the selected folder and period, together with the text of its attachments,"
|
||||
|
||||
-- Yes, please send my data to the external providers
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T2239851837"] = "Yes, please send my data to the external providers"
|
||||
|
||||
-- The data source '{0}'
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCECLOUDEMBEDDINGWARNING::T2503488371"] = "The data source '{0}'"
|
||||
|
||||
@@ -6775,6 +6778,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T286958948"]
|
||||
-- Token limit
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2961294165"] = "Token limit"
|
||||
|
||||
-- Audio and video files which this data source selects are skipped, because your transcription provider does not meet the confidence level this data source requires.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3017942214"] = "Audio and video files which this data source selects are skipped, because your transcription provider does not meet the confidence level this data source requires."
|
||||
|
||||
-- Audio and video files which this data source selects are skipped until you choose a transcription provider in the settings.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3119708878"] = "Audio and video files which this data source selects are skipped until you choose a transcription provider in the settings."
|
||||
|
||||
-- Please enter 0 or a positive overlap length.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
|
||||
|
||||
@@ -6799,12 +6808,12 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T529871858"]
|
||||
-- Select the base directory
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory"
|
||||
|
||||
-- Which files should be indexed? Separate multiple patterns with a semicolon, e.g., *.txt;*.pdf;*.docx;.gitignore. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file. Audio and video files, e.g., *.mp3 or *.mp4, are transcribed by your transcription provider.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T599406584"] = "Which files should be indexed? Separate multiple patterns with a semicolon, e.g., *.txt;*.pdf;*.docx;.gitignore. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file. Audio and video files, e.g., *.mp3 or *.mp4, are transcribed by your transcription provider."
|
||||
|
||||
-- Yes, index hidden files and folders as well
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T607153623"] = "Yes, index hidden files and folders as well"
|
||||
|
||||
-- Which files should be indexed? Separate multiple patterns with a semicolon, e.g., *.txt;*.pdf;*.docx;.gitignore. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T761052435"] = "Which files should be indexed? Separate multiple patterns with a semicolon, e.g., *.txt;*.pdf;*.docx;.gitignore. Use :documents: for every document and readable text file, :text: for readable text files only, or * for every file."
|
||||
|
||||
-- The data source token limit must not be larger than the embedding provider token limit ({0}).
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T787118522"] = "The data source token limit must not be larger than the embedding provider token limit ({0})."
|
||||
|
||||
@@ -6940,9 +6949,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1108876344"] = "
|
||||
-- Optional expert settings for how this data source is split before embedding.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1133561850"] = "Optional expert settings for how this data source is split before embedding."
|
||||
|
||||
-- Select a file for this data source. The content of this file will be processed for the data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1190880267"] = "Select a file for this data source. The content of this file will be processed for the data source."
|
||||
|
||||
-- Default tokenizer
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1220918127"] = "Default tokenizer"
|
||||
|
||||
@@ -6976,6 +6982,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T236253137"] = "R
|
||||
-- Please enter a token limit of at least 1.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2406580478"] = "Please enter a token limit of at least 1."
|
||||
|
||||
-- This audio or video file is skipped, because your transcription provider does not meet the confidence level this data source requires.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2468760910"] = "This audio or video file is skipped, because your transcription provider does not meet the confidence level this data source requires."
|
||||
|
||||
-- Add
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add"
|
||||
|
||||
@@ -7003,6 +7012,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "
|
||||
-- Select the file
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file"
|
||||
|
||||
-- This audio or video file is skipped until you choose a transcription provider in the settings.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4026150982"] = "This audio or video file is skipped until you choose a transcription provider in the settings."
|
||||
|
||||
-- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding provider.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T529871858"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding provider."
|
||||
|
||||
@@ -7021,6 +7033,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T900713019"] = "C
|
||||
-- Selected file path for this data source
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T939749563"] = "Selected file path for this data source"
|
||||
|
||||
-- Select a file for this data source. The content of this file will be processed for the data source. An audio or video file is transcribed by your transcription provider first.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T966073786"] = "Select a file for this data source. The content of this file will be processed for the data source. An audio or video file is transcribed by your transcription provider first."
|
||||
|
||||
-- Token overlap
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T981382809"] = "Token overlap"
|
||||
|
||||
@@ -10153,15 +10168,15 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1143368054"] = "Other cause"
|
||||
-- Current file: {0}
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1166856644"] = "Current file: {0}"
|
||||
|
||||
-- Transcription provider not allowed
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1283933457"] = "Transcription provider not allowed"
|
||||
|
||||
-- Mailboxes are synced every 16 minutes while the automatic refresh is on. Each mail is embedded only once: when it moves to another folder, AI Studio merely notes where it lies now.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1297391911"] = "Mailboxes are synced every 16 minutes while the automatic refresh is on. Each mail is embedded only once: when it moves to another folder, AI Studio merely notes where it lies now."
|
||||
|
||||
-- File {0} of {1} is being indexed: block {2}, page {3}.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1298290372"] = "File {0} of {1} is being indexed: block {2}, page {3}."
|
||||
|
||||
-- These image, audio, or video files match the data source patterns, but they cannot currently be added to the index.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1302021733"] = "These image, audio, or video files match the data source patterns, but they cannot currently be added to the index."
|
||||
|
||||
-- Mail
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1399880782"] = "Mail"
|
||||
|
||||
@@ -10192,6 +10207,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1616414701"] = "File {0} of {1} i
|
||||
-- Skipped: {0}
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1775451753"] = "Skipped: {0}"
|
||||
|
||||
-- These audio or video files match the data source patterns, but your transcription provider does not meet the confidence level this data source requires. Choose another transcription provider, or change the confidence level of the data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1811387825"] = "These audio or video files match the data source patterns, but your transcription provider does not meet the confidence level this data source requires. Choose another transcription provider, or change the confidence level of the data source."
|
||||
|
||||
-- Tried again during the next run
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T1946414905"] = "Tried again during the next run"
|
||||
|
||||
@@ -10264,6 +10282,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3628102488"] = "{0} files ({1})"
|
||||
-- No local data source has been queued for embedding yet.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3774205531"] = "No local data source has been queued for embedding yet."
|
||||
|
||||
-- These audio or video files match the data source patterns, but no transcription provider is configured that AI Studio may use. Once you choose one in the settings, AI Studio transcribes the files and adds their text to the index.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3775894071"] = "These audio or video files match the data source patterns, but no transcription provider is configured that AI Studio may use. Once you choose one in the settings, AI Studio transcribes the files and adds their text to the index."
|
||||
|
||||
-- Change the settings of the mailbox
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T3788376715"] = "Change the settings of the mailbox"
|
||||
|
||||
@@ -10282,9 +10303,15 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T419343388"] = "Remove them from t
|
||||
-- Not synced completely yet. AI Studio works through the mailbox piece by piece, the newest mails first, and older mails are still missing.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T4205079539"] = "Not synced completely yet. AI Studio works through the mailbox piece by piece, the newest mails first, and older mails are still missing."
|
||||
|
||||
-- These image files match the data source patterns, but images cannot currently be added to the index.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T422032787"] = "These image files match the data source patterns, but images cannot currently be added to the index."
|
||||
|
||||
-- Current mail: {0}
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T4242653147"] = "Current mail: {0}"
|
||||
|
||||
-- No transcription provider
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T4253595107"] = "No transcription provider"
|
||||
|
||||
-- Skipped files: {0}. Open a group below to see why and what you can do.
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T4287145754"] = "Skipped files: {0}. Open a group below to see why and what you can do."
|
||||
|
||||
@@ -10303,6 +10330,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T819555436"] = "Failed mails: {0}"
|
||||
-- Skipped
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T852993367"] = "Skipped"
|
||||
|
||||
-- Open the transcription settings
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T905182399"] = "Open the transcription settings"
|
||||
|
||||
-- Indexed: {0}
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T933775466"] = "Indexed: {0}"
|
||||
|
||||
@@ -12328,6 +12358,12 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXPORTFORMATEXTENSIONS::T677355172"] = "Un
|
||||
-- Not a readable spreadsheet
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1175970425"] = "Not a readable spreadsheet"
|
||||
|
||||
-- The media file '{0}' could not be processed, so it was not indexed. AI Studio reads it again as soon as the file changes.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1193172614"] = "The media file '{0}' could not be processed, so it was not indexed. AI Studio reads it again as soon as the file changes."
|
||||
|
||||
-- Media file could not be processed
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1302965532"] = "Media file could not be processed"
|
||||
|
||||
-- Not a text file
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::FILEEXTRACTIONERRORCODEEXTENSIONS::T1465212038"] = "Not a text file"
|
||||
|
||||
@@ -12562,6 +12598,24 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::MAIL::MAILBOXPROVIDERTEMPLATEEXTENSIONS::T4026
|
||||
-- Edit Mailbox
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::MAILBOXEDITING::T3250445637"] = "Edit Mailbox"
|
||||
|
||||
-- The transcription provider returned an empty transcript.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::MEDIA::PERSISTENTMEDIATRANSCRIPT::T1080540822"] = "The transcription provider returned an empty transcript."
|
||||
|
||||
-- No usable transcription provider is configured.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::MEDIA::PERSISTENTMEDIATRANSCRIPT::T2282521655"] = "No usable transcription provider is configured."
|
||||
|
||||
-- Was not able to read the existing transcript: {0}
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::MEDIA::PERSISTENTMEDIATRANSCRIPT::T2397111152"] = "Was not able to read the existing transcript: {0}"
|
||||
|
||||
-- The media file changed while it was being transcribed, so its transcript was not kept.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::MEDIA::PERSISTENTMEDIATRANSCRIPT::T2455890169"] = "The media file changed while it was being transcribed, so its transcript was not kept."
|
||||
|
||||
-- The media file could not be read.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::MEDIA::PERSISTENTMEDIATRANSCRIPT::T3826117761"] = "The media file could not be read."
|
||||
|
||||
-- Was not able to store the transcript: {0}
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::MEDIA::PERSISTENTMEDIATRANSCRIPT::T920159834"] = "Was not able to store the transcript: {0}"
|
||||
|
||||
-- AI Studio couldn't install Pandoc because the archive was not found.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::PANDOC::T1059477764"] = "AI Studio couldn't install Pandoc because the archive was not found."
|
||||
|
||||
@@ -13633,6 +13687,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T1225
|
||||
-- Some mails could not be indexed. The list below says which ones and why.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T2360744026"] = "Some mails could not be indexed. The list below says which ones and why."
|
||||
|
||||
-- The media file '{0}' could not be processed, so it was not indexed. Reason: {1} AI Studio reads it again as soon as the file changes.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T2967004571"] = "The media file '{0}' could not be processed, so it was not indexed. Reason: {1} AI Studio reads it again as soon as the file changes."
|
||||
|
||||
-- The mail '{0}' could not be indexed. AI Studio tries again during the next sync.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T804280374"] = "The mail '{0}' could not be indexed. AI Studio tries again during the next sync."
|
||||
|
||||
@@ -14443,8 +14500,8 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T2222598306"
|
||||
-- Please select your security policy.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T2250909198"] = "Please select your security policy."
|
||||
|
||||
-- The selected file is not a supported document or readable text file.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T2728355006"] = "The selected file is not a supported document or readable text file."
|
||||
-- Please acknowledge that you are aware of the cloud transcription implications.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T2818048408"] = "Please acknowledge that you are aware of the cloud transcription implications."
|
||||
|
||||
-- Please test the connection before saving.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T285470497"] = "Please test the connection before saving."
|
||||
@@ -14491,6 +14548,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T490875633"]
|
||||
-- The file path must not be empty. Please select a file.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T878007824"] = "The file path must not be empty. Please select a file."
|
||||
|
||||
-- The selected file is not a supported document, readable text file, audio file, or video file.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T90753429"] = "The selected file is not a supported document, readable text file, audio file, or video file."
|
||||
|
||||
-- Please enter the secret necessary for authentication.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::VALIDATION::DATASOURCEVALIDATION::T968385876"] = "Please enter the secret necessary for authentication."
|
||||
|
||||
|
||||
+1
-1
@@ -147,7 +147,7 @@ internal sealed partial class VisualBriefingBuildOrchestrator
|
||||
VisualBriefingBuildStage.SOURCE_PREPARATION,
|
||||
"A briefing source is no longer reachable.",
|
||||
$"Source {source.SourceId:D} failed the reachability check.");
|
||||
var sourceHash = await VisualBriefingHashing.ComputeFileAsync(source.Path, token);
|
||||
var sourceHash = await FileContentHash.ComputeAsync(source.Path, token);
|
||||
var transcriptHash = string.Empty;
|
||||
if (source.IsMedia)
|
||||
{
|
||||
|
||||
@@ -23,25 +23,6 @@ internal static class VisualBriefingHashing
|
||||
/// <returns>The lowercase hexadecimal hash.</returns>
|
||||
internal static string ComputeSections(params string?[] values) => Compute(string.Join('\u001e', values.Select(value => value ?? string.Empty)));
|
||||
|
||||
/// <summary>
|
||||
/// Computes a lowercase SHA-256 hash for a file without loading it fully into memory.
|
||||
/// </summary>
|
||||
/// <param name="path">The file path.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The lowercase hexadecimal hash.</returns>
|
||||
internal static async Task<string> ComputeFileAsync(string path, CancellationToken token)
|
||||
{
|
||||
await using var stream = new FileStream(
|
||||
path,
|
||||
FileMode.Open,
|
||||
FileAccess.Read,
|
||||
FileShare.Read,
|
||||
65_536,
|
||||
FileOptions.Asynchronous | FileOptions.SequentialScan);
|
||||
|
||||
return Convert.ToHexStringLower(await SHA256.HashDataAsync(stream, token));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns canonical JSON for one value, with ordinally sorted object properties.
|
||||
/// </summary>
|
||||
|
||||
+1
-1
@@ -41,7 +41,7 @@ internal sealed class VisualBriefingSourcePreparationService(VisualBriefingStore
|
||||
var info = new FileInfo(source.Path);
|
||||
totalBytes += info.Length;
|
||||
|
||||
var sourceHash = await VisualBriefingHashing.ComputeFileAsync(source.Path, token);
|
||||
var sourceHash = await FileContentHash.ComputeAsync(source.Path, token);
|
||||
var transcriptHash = string.Empty;
|
||||
|
||||
if (source.IsMedia)
|
||||
|
||||
Reference in new issue
Block a user