mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-21 18:58:00 +00:00
Compare commits
4
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
1c8632bbe0 | ||
|
|
7fbb894d5b | ||
|
|
e019c24ea2 | ||
|
|
5d6e48ef93 |
No files matched your search
@@ -25,6 +25,10 @@
|
||||
@T("Supported audio and video files are transcribed automatically without an additional dialog. Each transcript is stored next to its media file as '<media-file>.transcript.md' and reused when an interrupted run is continued.")
|
||||
</MudJustifiedText>
|
||||
|
||||
<MudJustifiedText Typo="Typo.body2" Class="mb-3">
|
||||
@T("Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.")
|
||||
</MudJustifiedText>
|
||||
|
||||
<MudStack Class="mb-3">
|
||||
<MudButton OnClick="@this.ToggleExpertSettings" Variant="Variant.Text" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Tune">
|
||||
@(this.showExpertSettings ? T("Hide Expert Settings") : T("Show Expert Settings"))
|
||||
|
||||
+77
-6
@@ -1,18 +1,87 @@
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools.Media;
|
||||
using AIStudio.Tools.Rust;
|
||||
|
||||
namespace AIStudio.Assistants.BatchProcessing;
|
||||
|
||||
public partial class AssistantBatchProcessing
|
||||
{
|
||||
/// <summary>
|
||||
/// Loads a document through the Rust content stream or resolves a persistent
|
||||
/// transcript for an audio or video file.
|
||||
/// What one file of the batch gives to the model: either text or an image.
|
||||
/// </summary>
|
||||
private Task<string?> LoadInputContentAsync(BatchProcessingFileResult fileResult, CancellationToken token)
|
||||
private readonly record struct BatchInput(string Text, FileAttachmentImage? Image);
|
||||
|
||||
/// <summary>
|
||||
/// Loads a document through the Rust content stream, resolves a persistent
|
||||
/// transcript for an audio or video file, or prepares an image for the model.
|
||||
/// </summary>
|
||||
/// <returns>The input for the model, or <c>null</c> when the file already got its final status.</returns>
|
||||
private async Task<BatchInput?> LoadInputContentAsync(BatchProcessingFileResult fileResult, CancellationToken token)
|
||||
{
|
||||
return IsTranscribableMedia(fileResult.FilePath)
|
||||
? this.LoadMediaTranscriptAsync(fileResult, token)
|
||||
: this.LoadDocumentContentAsync(fileResult, token);
|
||||
if (IsImage(fileResult.FilePath))
|
||||
return this.LoadImage(fileResult) is { } image ? new BatchInput(string.Empty, image) : null;
|
||||
|
||||
var text = IsTranscribableMedia(fileResult.FilePath)
|
||||
? await this.LoadMediaTranscriptAsync(fileResult, token)
|
||||
: await this.LoadDocumentContentAsync(fileResult, token);
|
||||
|
||||
return text is null ? null : new BatchInput(text, null);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks an image before it goes to the model.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The request itself would only skip an image it cannot send, with a message, and ask the model
|
||||
/// anyway. The model would then answer about nothing, and the file would count as done, so that
|
||||
/// continuing the run would skip it as well. A failed image is processed again when the run is
|
||||
/// continued, e.g., with a model that can read images.
|
||||
///
|
||||
/// How many images a model accepts needs no check of its own: every file gets a thread of its own,
|
||||
/// see CallAIAsync, so each request carries exactly one image, in every round with the model. A
|
||||
/// limit of one image is therefore always met, and SupportsImageInput already refuses a limit of zero.
|
||||
///
|
||||
/// Unlike the text of documents and transcripts, an image does not pass the prompt injection
|
||||
/// filter of the runtime, which reads text only. Images in a chat do not either. Instructions
|
||||
/// within an image therefore meet only the safeguards behind the filter: the system prompt tells
|
||||
/// the model to ignore them, a thread which read internal content, such as a mailbox, may no
|
||||
/// longer send data out, and without free address choice, its default, the Read Web Page tool
|
||||
/// refuses an address which appears only in the image.
|
||||
/// </remarks>
|
||||
private FileAttachmentImage? LoadImage(BatchProcessingFileResult fileResult)
|
||||
{
|
||||
if (!this.ProviderSettings.SupportsImageInput())
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("The selected model cannot read images. Select a model with image support to process this image."));
|
||||
return null;
|
||||
}
|
||||
|
||||
if (!FileTypes.IsAllowedPath(fileResult.FilePath, FileTypes.MODEL_IMAGE))
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF."));
|
||||
return null;
|
||||
}
|
||||
|
||||
long fileSize;
|
||||
try
|
||||
{
|
||||
fileSize = new FileInfo(fileResult.FilePath).Length;
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("Was not able to read the file: {0}"), e.Message), e);
|
||||
return null;
|
||||
}
|
||||
|
||||
var maxImageSizeBytes = IImageSourceExtensions.MaxImageSizeBytesFor(this.ProviderSettings.UsedLLMProvider);
|
||||
if (fileSize > maxImageSizeBytes)
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, string.Format(T("The image is larger than {0} MB, which is the most the selected provider accepts for one image."), (maxImageSizeBytes / 1_000_000d).ToString("0.#", Tools.PluginSystem.I18N.I.Culture)));
|
||||
return null;
|
||||
}
|
||||
|
||||
return new FileAttachmentImage(fileResult.FileName, fileResult.FilePath, fileSize);
|
||||
}
|
||||
|
||||
private async Task<string?> LoadDocumentContentAsync(BatchProcessingFileResult fileResult, CancellationToken token)
|
||||
@@ -109,5 +178,7 @@ public partial class AssistantBatchProcessing
|
||||
|
||||
private static bool IsTranscribableMedia(string filePath) => PersistentMediaTranscript.IsMediaPath(filePath);
|
||||
|
||||
private static bool IsImage(string filePath) => FileTypes.IsAllowedPath(filePath, FileTypes.IMAGE);
|
||||
|
||||
private static bool HasReusableTranscript(string mediaFilePath) => MediaSidecarTranscripts.HasReusableTranscript(mediaFilePath);
|
||||
}
|
||||
+16
-6
@@ -56,6 +56,8 @@ public partial class AssistantBatchProcessing
|
||||
# Task description
|
||||
You are a batch document processing agent. Each request contains exactly one DOCUMENT.
|
||||
Your task is to process this DOCUMENT strictly according to the instructions below.
|
||||
A DOCUMENT may also be an image attached to the request. Then, the content of the
|
||||
DOCUMENT is what the image shows, including any text within it.
|
||||
|
||||
# Scope and precedence
|
||||
Use only information explicitly contained in the DOCUMENT and the instructions.
|
||||
@@ -88,11 +90,20 @@ public partial class AssistantBatchProcessing
|
||||
""";
|
||||
}
|
||||
|
||||
private static string BuildImageUserPrompt(string fileName)
|
||||
{
|
||||
return $"""
|
||||
# DOCUMENT
|
||||
File name: {fileName}
|
||||
Content: The DOCUMENT is the image attached to this request.
|
||||
""";
|
||||
}
|
||||
|
||||
/// <param name="fileName">The name of the document being processed.</param>
|
||||
/// <param name="fileContent">The content handed to the model.</param>
|
||||
/// <param name="input">The text or the image handed to the model.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>The answer of the model, and which tools it used to get there.</returns>
|
||||
private async Task<(string Answer, string UsedTools)> CallAIAsync(string fileName, string fileContent, CancellationToken token)
|
||||
private async Task<(string Answer, string UsedTools)> CallAIAsync(string fileName, BatchInput input, CancellationToken token)
|
||||
{
|
||||
//
|
||||
// Every file of the batch gets the tools the user picked for the job. The batch builds its
|
||||
@@ -120,10 +131,9 @@ public partial class AssistantBatchProcessing
|
||||
};
|
||||
|
||||
this.AssignRuntimeIdentity(chatThread);
|
||||
var userPrompt = new ContentText
|
||||
{
|
||||
Text = BuildUserPrompt(fileName, fileContent),
|
||||
};
|
||||
var userPrompt = input.Image is null
|
||||
? new ContentText { Text = BuildUserPrompt(fileName, input.Text) }
|
||||
: new ContentText { Text = BuildImageUserPrompt(fileName), FileAttachments = [input.Image] };
|
||||
|
||||
chatThread.Blocks.Add(new ContentBlock
|
||||
{
|
||||
|
||||
+38
-3
@@ -1,6 +1,8 @@
|
||||
using System.Diagnostics;
|
||||
using System.Globalization;
|
||||
|
||||
using AIStudio.Settings;
|
||||
|
||||
namespace AIStudio.Assistants.BatchProcessing;
|
||||
|
||||
public partial class AssistantBatchProcessing
|
||||
@@ -42,6 +44,7 @@ public partial class AssistantBatchProcessing
|
||||
}
|
||||
|
||||
this.PrepareFileResults(resolvedOutputDirectory, files, previousLog, previousResults);
|
||||
await this.WarnAboutUnreadableImagesAsync();
|
||||
await this.CheckpointAssistantSession();
|
||||
await this.RunBatchAsync(resolvedOutputDirectory);
|
||||
}
|
||||
@@ -88,6 +91,27 @@ public partial class AssistantBatchProcessing
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Tells the user before the first file when the images of the run will fail, because the
|
||||
/// selected model cannot read images.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Counts only the images which this run still processes. Images done by an earlier run of a
|
||||
/// continued batch keep their results and do not fail.
|
||||
/// </remarks>
|
||||
private async Task WarnAboutUnreadableImagesAsync()
|
||||
{
|
||||
var numImages = this.fileResults.Count(fileResult => fileResult.Status is not BatchProcessingFileStatus.DONE && IsImage(fileResult.FilePath));
|
||||
if (numImages == 0 || this.ProviderSettings.SupportsImageInput())
|
||||
return;
|
||||
|
||||
var warning = numImages == 1
|
||||
? T("The selected model cannot read images, so the image among the selected files will fail. All other files are processed as usual.")
|
||||
: string.Format(T("The selected model cannot read images, so the {0} images among the selected files will fail. All other files are processed as usual."), numImages);
|
||||
|
||||
await this.MessageBus.SendWarning(new(Icons.Material.Filled.ImageNotSupported, warning));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Processes all documents which are not restored from a previous run.
|
||||
/// </summary>
|
||||
@@ -188,14 +212,14 @@ public partial class AssistantBatchProcessing
|
||||
/// </remarks>
|
||||
private async Task ProcessOneFileAsync(BatchProcessingFileResult fileResult, string resolvedOutputDirectory, CancellationToken token)
|
||||
{
|
||||
var fileContent = await this.LoadInputContentAsync(fileResult, token);
|
||||
if (fileContent is null)
|
||||
var input = await this.LoadInputContentAsync(fileResult, token);
|
||||
if (input is null)
|
||||
return;
|
||||
|
||||
string aiAnswer;
|
||||
try
|
||||
{
|
||||
(aiAnswer, fileResult.UsedTools) = await this.CallAIAsync(fileResult.FileName, fileContent, token);
|
||||
(aiAnswer, fileResult.UsedTools) = await this.CallAIAsync(fileResult.FileName, input.Value, token);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
@@ -217,6 +241,17 @@ public partial class AssistantBatchProcessing
|
||||
return;
|
||||
}
|
||||
|
||||
//
|
||||
// The request reads the image anew in every round with the model, e.g., after each tool
|
||||
// call, and quietly leaves out an image which is gone by then. The answer would be about
|
||||
// nothing and must not count as a result, so that continuing the run processes the image again:
|
||||
//
|
||||
if (input.Value.Image is { Exists: false })
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("The image was no longer available while the model was asked about it."));
|
||||
return;
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(aiAnswer))
|
||||
{
|
||||
this.FinishFileResult(fileResult, BatchProcessingFileStatus.FAILED, T("The AI answer was empty."));
|
||||
|
||||
+15
-43
@@ -1,3 +1,4 @@
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Tools.Media;
|
||||
|
||||
namespace AIStudio.Assistants.BatchProcessing;
|
||||
@@ -82,39 +83,18 @@ public partial class AssistantBatchProcessing
|
||||
private IReadOnlyList<string> FindInputFiles(string resolvedOutputDirectory)
|
||||
{
|
||||
var selection = new FolderFileSelection(FileNamePatterns.Parse(this.filePatterns), FileExclusions.Parse(this.excludePatterns), this.includeSubdirectories, this.includeHiddenFilesAndFolders, this.maxFileSizeMegabytes);
|
||||
var outputFolder = new BatchProcessingOutputFolder(this.inputDirectory, resolvedOutputDirectory, this.ResolveResultsFileName());
|
||||
var files = new SortedSet<string>(StringComparer.OrdinalIgnoreCase);
|
||||
this.numTooLargeFiles = 0;
|
||||
|
||||
var normalizedInputDirectory = TrimDirectorySeparator(Path.GetFullPath(this.inputDirectory));
|
||||
var normalizedOutputDirectory = TrimDirectorySeparator(Path.GetFullPath(resolvedOutputDirectory));
|
||||
|
||||
// When the output folder is a folder of its own, we skip everything
|
||||
// inside it. When it is the input folder itself, we must not skip the
|
||||
// whole folder: we would not find any document at all. We then skip
|
||||
// our own output artifacts instead.
|
||||
var isOutputSeparateFolder = !string.Equals(normalizedInputDirectory, normalizedOutputDirectory, StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
// The separator is essential: without it, an output folder named 'out'
|
||||
// would also exclude a document named 'output-notes.md':
|
||||
var outputDirectoryPrefix = normalizedOutputDirectory + Path.DirectorySeparatorChar;
|
||||
|
||||
foreach (var file in selection.EnumerateCandidates(
|
||||
this.inputDirectory,
|
||||
includeDirectory: directory => !isOutputSeparateFolder
|
||||
|| !string.Equals(TrimDirectorySeparator(Path.GetFullPath(directory)), normalizedOutputDirectory, StringComparison.OrdinalIgnoreCase),
|
||||
includeDirectory: outputFolder.Enters,
|
||||
reportInaccessibleDirectory: (directory, exception) =>
|
||||
throw new IOException($"Could not access directory '{directory}'.", exception)))
|
||||
{
|
||||
var normalizedFile = Path.GetFullPath(file);
|
||||
if (!selection.Selects(normalizedFile) || IsTranscriptArtifact(normalizedFile))
|
||||
continue;
|
||||
|
||||
if (isOutputSeparateFolder)
|
||||
{
|
||||
if (normalizedFile.StartsWith(outputDirectoryPrefix, StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
}
|
||||
else if (this.IsOwnOutputArtifact(normalizedFile))
|
||||
if (!selection.Selects(normalizedFile) || IsTranscriptArtifact(normalizedFile) || outputFolder.IsOutputOfRun(normalizedFile))
|
||||
continue;
|
||||
|
||||
//
|
||||
@@ -136,25 +116,6 @@ public partial class AssistantBatchProcessing
|
||||
return [.. files];
|
||||
}
|
||||
|
||||
private static string TrimDirectorySeparator(string path) => path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
|
||||
|
||||
/// <summary>
|
||||
/// Checks whether a file is an output artifact of this assistant. We need
|
||||
/// this when the output folder is the input folder: without it, the results
|
||||
/// of a previous run would be processed as documents.
|
||||
/// </summary>
|
||||
private bool IsOwnOutputArtifact(string filePath)
|
||||
{
|
||||
var fileName = Path.GetFileName(filePath);
|
||||
if (string.Equals(fileName, LOG_FILENAME, StringComparison.OrdinalIgnoreCase))
|
||||
return true;
|
||||
|
||||
if (string.Equals(fileName, this.ResolveResultsFileName(), StringComparison.OrdinalIgnoreCase))
|
||||
return true;
|
||||
|
||||
return fileName.EndsWith(RESULT_FILE_SUFFIX, StringComparison.OrdinalIgnoreCase);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks for persistent or interrupted media transcript artifacts. They
|
||||
/// always live beside their source file, independently of the output folder.
|
||||
@@ -217,6 +178,17 @@ public partial class AssistantBatchProcessing
|
||||
return null;
|
||||
}
|
||||
|
||||
//
|
||||
// A model that cannot read images would get the text of the request alone, without the
|
||||
// image, and answer about nothing. Such images fail one by one, see LoadImage, while the
|
||||
// documents next to them are processed. A run of nothing but images does not start at all:
|
||||
//
|
||||
if (files.All(IsImage) && !this.ProviderSettings.SupportsImageInput())
|
||||
{
|
||||
this.AddInputIssue(T("All selected files are images, but the selected model cannot read images. Select a model with image support."));
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
Directory.CreateDirectory(resolvedOutputDirectory);
|
||||
|
||||
@@ -18,7 +18,7 @@ public partial class AssistantBatchProcessing : AssistantBaseCore<SettingsDialog
|
||||
private const string DEFAULT_OUTPUT_DIRECTORY_NAME = "ai-results";
|
||||
private const string DEFAULT_RESULTS_FILENAME = "batch-results.csv";
|
||||
private const string CSV_EXTENSION = ".csv";
|
||||
private const string RESULT_FILE_SUFFIX = "_result";
|
||||
internal const string RESULT_FILE_SUFFIX = "_result";
|
||||
private const string TIME_FORMAT = "yyyy-MM-dd HH:mm:ss";
|
||||
private const char LOG_SEPARATOR = ';';
|
||||
|
||||
@@ -26,7 +26,7 @@ public partial class AssistantBatchProcessing : AssistantBaseCore<SettingsDialog
|
||||
/// The name of the log file. It is fixed, so that a later batch run finds
|
||||
/// the log of a previous run and can continue it.
|
||||
/// </summary>
|
||||
private const string LOG_FILENAME = "log.csv";
|
||||
internal const string LOG_FILENAME = "log.csv";
|
||||
|
||||
protected override Tools.Components Component => Tools.Components.BATCH_PROCESSING_ASSISTANT;
|
||||
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
namespace AIStudio.Assistants.BatchProcessing;
|
||||
|
||||
/// <summary>
|
||||
/// Tells the search for documents which of the files it finds a batch run wrote itself.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A run writes its result files, its log, and its results table directly into the output folder.
|
||||
/// What the search must leave out therefore depends on where this folder lies. Below the input
|
||||
/// folder, the search leaves out all of it. When it is the input folder itself, leaving out the
|
||||
/// whole folder would leave no document at all, so the search leaves out what a run writes, by
|
||||
/// name. Anywhere else, above the input folder included, the search never comes across the output
|
||||
/// folder and leaves out nothing.
|
||||
/// </remarks>
|
||||
/// <param name="inputDirectory">The folder the run reads its documents from.</param>
|
||||
/// <param name="outputDirectory">The folder the run writes into.</param>
|
||||
/// <param name="resultsFileName">The file name of the results table.</param>
|
||||
internal sealed class BatchProcessingOutputFolder(string inputDirectory, string outputDirectory, string resultsFileName)
|
||||
{
|
||||
private readonly string fullOutputDirectory = Path.GetFullPath(outputDirectory);
|
||||
|
||||
private readonly bool isInputFolder = IsSameFolder(inputDirectory, outputDirectory);
|
||||
|
||||
//
|
||||
// The same folder with a separator at its end would count as lying below itself:
|
||||
//
|
||||
private readonly bool liesBelowInputFolder = !IsSameFolder(inputDirectory, outputDirectory) && PathTools.IsInsideDirectory(inputDirectory, outputDirectory);
|
||||
|
||||
/// <summary>
|
||||
/// Whether the search enters a subfolder of the input folder: every one but the output folder.
|
||||
/// </summary>
|
||||
/// <param name="directory">The subfolder.</param>
|
||||
public bool Enters(string directory) => !this.liesBelowInputFolder || !IsSameFolder(directory, this.fullOutputDirectory);
|
||||
|
||||
/// <summary>
|
||||
/// Whether a file the search found was written by a run instead of being a document.
|
||||
/// </summary>
|
||||
/// <param name="filePath">The full path of the file.</param>
|
||||
public bool IsOutputOfRun(string filePath)
|
||||
{
|
||||
//
|
||||
// Checked with the separator after the name of the folder: otherwise, an output folder
|
||||
// named 'out' would also leave out a document named 'output-notes.md':
|
||||
//
|
||||
if (this.liesBelowInputFolder)
|
||||
return PathTools.IsInsideDirectory(this.fullOutputDirectory, filePath);
|
||||
|
||||
//
|
||||
// A run writes into the output folder itself, never into one of its subfolders:
|
||||
//
|
||||
return this.isInputFolder
|
||||
&& Path.GetDirectoryName(Path.GetFullPath(filePath)) is { } fileDirectory
|
||||
&& IsSameFolder(fileDirectory, this.fullOutputDirectory)
|
||||
&& this.IsOwnOutputArtifact(filePath);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks whether a file is an output artifact of the assistant. We need
|
||||
/// this when the output folder is the input folder: without it, the results
|
||||
/// of a previous run would be processed as documents.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A result file is recognized by its name without the extension, e.g., report_result.md or
|
||||
/// report_result_2.docx, cf. AssistantBatchProcessing.CreateResultFileName. Its extension does
|
||||
/// not count, because the user may have chosen another file format since a previous run.
|
||||
/// </remarks>
|
||||
private bool IsOwnOutputArtifact(string filePath)
|
||||
{
|
||||
var fileName = Path.GetFileName(filePath);
|
||||
if (string.Equals(fileName, AssistantBatchProcessing.LOG_FILENAME, StringComparison.OrdinalIgnoreCase)
|
||||
|| string.Equals(fileName, resultsFileName, StringComparison.OrdinalIgnoreCase))
|
||||
return true;
|
||||
|
||||
var stem = Path.GetFileNameWithoutExtension(fileName);
|
||||
var suffixIndex = stem.LastIndexOf(AssistantBatchProcessing.RESULT_FILE_SUFFIX, StringComparison.OrdinalIgnoreCase);
|
||||
if (suffixIndex < 0)
|
||||
return false;
|
||||
|
||||
//
|
||||
// A second result of the same name gets a counter, e.g., report_result_2.md:
|
||||
//
|
||||
var counter = stem[(suffixIndex + AssistantBatchProcessing.RESULT_FILE_SUFFIX.Length)..];
|
||||
return counter.Length is 0 || (counter.Length > 1 && counter[0] is '_' && counter[1..].All(char.IsAsciiDigit));
|
||||
}
|
||||
|
||||
private static bool IsSameFolder(string firstDirectory, string secondDirectory) => string.Equals(TrimDirectorySeparator(Path.GetFullPath(firstDirectory)), TrimDirectorySeparator(Path.GetFullPath(secondDirectory)), PathTools.COMPARISON);
|
||||
|
||||
private static string TrimDirectorySeparator(string path) => path.TrimEnd(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar);
|
||||
}
|
||||
@@ -424,6 +424,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- The selected folder does not exist.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T17705912"] = "The selected folder does not exist."
|
||||
|
||||
-- The image was no longer available while the model was asked about it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1805791865"] = "The image was no longer available while the model was asked about it."
|
||||
|
||||
-- Minimum pause between files
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1829787634"] = "Minimum pause between files"
|
||||
|
||||
@@ -529,9 +532,15 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Was not able to write the result file: {0}
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2991295581"] = "Was not able to write the result file: {0}"
|
||||
|
||||
-- Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3000909441"] = "Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start."
|
||||
|
||||
-- The batch run finished, but one file could not be processed. See the progress table and log for details.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3201532790"] = "The batch run finished, but one file could not be processed. See the progress table and log for details."
|
||||
|
||||
-- The selected model cannot read images, so the {0} images among the selected files will fail. All other files are processed as usual.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T324865195"] = "The selected model cannot read images, so the {0} images among the selected files will fail. All other files are processed as usual."
|
||||
|
||||
-- Maximum pause between files
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3250003796"] = "Maximum pause between files"
|
||||
|
||||
@@ -550,6 +559,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Show Expert Settings
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3361153305"] = "Show Expert Settings"
|
||||
|
||||
-- All selected files are images, but the selected model cannot read images. Select a model with image support.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3365102967"] = "All selected files are images, but the selected model cannot read images. Select a model with image support."
|
||||
|
||||
-- Header of the result column (optional)
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T340994102"] = "Header of the result column (optional)"
|
||||
|
||||
@@ -562,6 +574,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- The file name of the CSV results table. We add the ending .csv when it is missing. When left empty, we use 'batch-results.csv'.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3439247329"] = "The file name of the CSV results table. We add the ending .csv when it is missing. When left empty, we use 'batch-results.csv'."
|
||||
|
||||
-- The selected model cannot read images. Select a model with image support to process this image.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T346986514"] = "The selected model cannot read images. Select a model with image support to process this image."
|
||||
|
||||
-- Document analysis policy
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3510564924"] = "Document analysis policy"
|
||||
|
||||
@@ -646,6 +661,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Start batch processing
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T50133258"] = "Start batch processing"
|
||||
|
||||
-- This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T532496054"] = "This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF."
|
||||
|
||||
-- Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T544244392"] = "Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again."
|
||||
|
||||
@@ -670,12 +688,18 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Include hidden files and folders?
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T85084423"] = "Include hidden files and folders?"
|
||||
|
||||
-- The image is larger than {0} MB, which is the most the selected provider accepts for one image.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T87450623"] = "The image is larger than {0} MB, which is the most the selected provider accepts for one image."
|
||||
|
||||
-- Restore default
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T913168756"] = "Restore default"
|
||||
|
||||
-- The AI may use these tools while working on each document. Every tool has to meet the confidence requirements of the selected provider, so a tool may stay unavailable even when it is selected here.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T967206794"] = "The AI may use these tools while working on each document. Every tool has to meet the confidence requirements of the selected provider, so a tool may stay unavailable even when it is selected here."
|
||||
|
||||
-- The selected model cannot read images, so the image among the selected files will fail. All other files are processed as usual.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T9738678"] = "The selected model cannot read images, so the image among the selected files will fail. All other files are processed as usual."
|
||||
|
||||
-- Comma (,)
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::BATCHPROCESSINGCSVSEPARATOREXTENSIONS::T1676507543"] = "Comma (,)"
|
||||
|
||||
@@ -4120,9 +4144,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "Yo
|
||||
-- Automatic local data source refresh
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1208397349"] = "Automatic local data source refresh"
|
||||
|
||||
-- Edit Local Directory Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1215599168"] = "Edit Local Directory Data Source"
|
||||
|
||||
-- Refresh
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T135637716"] = "Refresh"
|
||||
|
||||
@@ -4150,12 +4171,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2067030738"] = "Ar
|
||||
-- Local Directory Data Source Information
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2146756020"] = "Local Directory Data Source Information"
|
||||
|
||||
-- Edit ERI v1 Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T221059217"] = "Edit ERI v1 Data Source"
|
||||
|
||||
-- Edit Local File Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2453292893"] = "Edit Local File Data Source"
|
||||
|
||||
-- Local data sources refresh when files change, mailboxes every 16 minutes.
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2511157475"] = "Local data sources refresh when files change, mailboxes every 16 minutes."
|
||||
|
||||
@@ -6805,9 +6820,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2460883298"
|
||||
-- Add
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add"
|
||||
|
||||
-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2686150329"] = "The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed."
|
||||
|
||||
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
|
||||
|
||||
@@ -7066,9 +7078,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "T
|
||||
-- Data Source Name
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name"
|
||||
|
||||
-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T890383764"] = "The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed."
|
||||
|
||||
-- Cancel
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T900713019"] = "Cancel"
|
||||
|
||||
@@ -8581,6 +8590,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T7117
|
||||
-- Default minimum pause between files
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1008440099"] = "Default minimum pause between files"
|
||||
|
||||
-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1038539663"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format."
|
||||
|
||||
-- Instructions
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1221801316"] = "Instructions"
|
||||
|
||||
@@ -8635,9 +8647,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T24
|
||||
-- When enabled, new batch runs start with the defaults configured below.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2592677194"] = "When enabled, new batch runs start with the defaults configured below."
|
||||
|
||||
-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2594325620"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats."
|
||||
|
||||
-- Subfolders are included
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2607092632"] = "Subfolders are included"
|
||||
|
||||
@@ -10372,6 +10381,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T819555436"] = "Failed mails: {0}"
|
||||
-- Skipped
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T852993367"] = "Skipped"
|
||||
|
||||
-- Edit this data source
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T878700886"] = "Edit this data source"
|
||||
|
||||
-- Open the transcription settings
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T905182399"] = "Open the transcription settings"
|
||||
|
||||
@@ -12208,18 +12220,39 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
|
||||
-- Qdrant Edge is not available.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available."
|
||||
|
||||
-- Edit Local Directory Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T1215599168"] = "Edit Local Directory Data Source"
|
||||
|
||||
-- Edit ERI v1 Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T221059217"] = "Edit ERI v1 Data Source"
|
||||
|
||||
-- Edit Local File Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T2453292893"] = "Edit Local File Data Source"
|
||||
|
||||
-- Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T1555760974"] = "Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are."
|
||||
|
||||
-- AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2205693550"] = "AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in."
|
||||
|
||||
-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one."
|
||||
|
||||
-- and {0} more.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "and {0} more."
|
||||
|
||||
-- Your embedding provider runs in the cloud, so preparing files again costs money.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2635623258"] = "Your embedding provider runs in the cloud, so preparing files again costs money."
|
||||
|
||||
-- This change makes the prepared documents of the following data sources unusable ({0}):
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "This change makes the prepared documents of the following data sources unusable ({0}):"
|
||||
|
||||
-- Do you want to apply this change anyway?
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Do you want to apply this change anyway?"
|
||||
|
||||
-- New Location for This Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3462449568"] = "New Location for This Data Source"
|
||||
|
||||
-- Documents Will Be Prepared Again
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Documents Will Be Prepared Again"
|
||||
|
||||
@@ -13402,6 +13435,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source like p
|
||||
-- Document
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document"
|
||||
|
||||
-- Image for AI models
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T791951937"] = "Image for AI models"
|
||||
|
||||
-- Plugin archive
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T927001356"] = "Plugin archive"
|
||||
|
||||
@@ -13759,18 +13795,21 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T8042
|
||||
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1214987337"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."
|
||||
|
||||
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1393648320"] = "The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source."
|
||||
|
||||
-- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1935191670"] = "The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run."
|
||||
|
||||
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2268993048"] = "The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source."
|
||||
|
||||
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2692299965"] = "The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source."
|
||||
|
||||
-- The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2999277901"] = "The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source."
|
||||
|
||||
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3230000698"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it."
|
||||
|
||||
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3722338131"] = "The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source."
|
||||
|
||||
-- The file '{0}' could not be read. Please check whether you are allowed to read it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3924882233"] = "The file '{0}' could not be read. Please check whether you are allowed to read it."
|
||||
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Tools.MIME;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
|
||||
@@ -6,7 +7,23 @@ namespace AIStudio.Chat;
|
||||
public static class IImageSourceExtensions
|
||||
{
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(IImageSourceExtensions).Namespace, nameof(IImageSourceExtensions));
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// The largest image we send to a model. A larger image is skipped by TryAsBase64.
|
||||
/// </summary>
|
||||
public const long MAX_IMAGE_SIZE_BYTES = 10_000_000;
|
||||
|
||||
/// <summary>
|
||||
/// The largest image file, in bytes, which we send through the given provider.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The limit of the provider where it states one, but never more than MAX_IMAGE_SIZE_BYTES:
|
||||
/// TryAsBase64 skips every larger image, whatever the provider would accept.
|
||||
/// </remarks>
|
||||
/// <param name="llmProvider">The provider the image is sent through.</param>
|
||||
/// <returns>The size limit of one image file.</returns>
|
||||
public static long MaxImageSizeBytesFor(LLMProviders llmProvider) => Math.Min(llmProvider.GetMaxImageSizeBytes() ?? MAX_IMAGE_SIZE_BYTES, MAX_IMAGE_SIZE_BYTES);
|
||||
|
||||
public static MIMEType DetermineMimeType(this IImageSource image)
|
||||
{
|
||||
switch (image.SourceType)
|
||||
@@ -97,7 +114,7 @@ public static class IImageSourceExtensions
|
||||
{
|
||||
// Read the length of the content:
|
||||
var lengthBytes = response.Content.Headers.ContentLength;
|
||||
if(lengthBytes > 10_000_000)
|
||||
if(lengthBytes > MAX_IMAGE_SIZE_BYTES)
|
||||
{
|
||||
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The image at the URL is too large (>10 MB). Skipping the image.")));
|
||||
return (success: false, string.Empty);
|
||||
@@ -116,7 +133,7 @@ public static class IImageSourceExtensions
|
||||
{
|
||||
// Read the content length:
|
||||
var length = new FileInfo(image.Source).Length;
|
||||
if(length > 10_000_000)
|
||||
if(length > MAX_IMAGE_SIZE_BYTES)
|
||||
{
|
||||
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.ImageNotSupported, TB("The local image file is too large (>10 MB). Skipping the image.")));
|
||||
return (success: false, string.Empty);
|
||||
|
||||
@@ -270,70 +270,8 @@ public partial class DataSourceManagement : MSGComponentBase
|
||||
|
||||
private async Task EditDataSource(IDataSource dataSource)
|
||||
{
|
||||
if (dataSource.IsEnterpriseConfiguration)
|
||||
return;
|
||||
|
||||
IDataSource? editedDataSource = null;
|
||||
var lockDataSourceOrigin = dataSource is IInternalDataSource
|
||||
&& await this.DataSourceEmbeddingService.ShouldLockDataSourceOriginAsync(dataSource.Id);
|
||||
switch (dataSource)
|
||||
{
|
||||
case DataSourceLocalFile localFile:
|
||||
var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, localFile },
|
||||
{ x => x.LockSource, lockDataSourceOrigin },
|
||||
};
|
||||
|
||||
var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Edit Local File Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localFileDialogResult = await localFileDialogReference.Result;
|
||||
if (localFileDialogResult is null || localFileDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
editedDataSource = (DataSourceLocalFile)localFileDialogResult.Data!;
|
||||
break;
|
||||
|
||||
case DataSourceLocalDirectory localDirectory:
|
||||
var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, localDirectory },
|
||||
{ x => x.LockSource, lockDataSourceOrigin },
|
||||
};
|
||||
|
||||
var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Edit Local Directory Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
|
||||
if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
editedDataSource = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
|
||||
break;
|
||||
|
||||
case DataSourceERI_V1 eriDataSource:
|
||||
var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, eriDataSource },
|
||||
};
|
||||
|
||||
var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>(T("Edit ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var eriDialogResult = await eriDialogReference.Result;
|
||||
if (eriDialogResult is null || eriDialogResult.Canceled)
|
||||
return;
|
||||
|
||||
editedDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
|
||||
break;
|
||||
}
|
||||
|
||||
if(editedDataSource is null)
|
||||
return;
|
||||
|
||||
this.SettingsManager.ConfigurationData.DataSources[this.SettingsManager.ConfigurationData.DataSources.IndexOf(dataSource)] = editedDataSource;
|
||||
|
||||
await this.SettingsManager.StoreSettings();
|
||||
await this.DataSourceEmbeddingService.QueueDataSourceAsync(editedDataSource);
|
||||
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
|
||||
if (await DataSourceEditing.EditAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, dataSource.Id))
|
||||
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
|
||||
}
|
||||
|
||||
private async Task DeleteDataSource(IDataSource dataSource)
|
||||
|
||||
@@ -47,31 +47,7 @@
|
||||
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
|
||||
@T("Select a root directory for this data source.")
|
||||
</MudJustifiedText>
|
||||
@if (!this.CanChangeSource)
|
||||
{
|
||||
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
|
||||
@T("The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed.")
|
||||
</MudAlert>
|
||||
}
|
||||
@if (this.CanChangeSource)
|
||||
{
|
||||
<SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" EnableDragDrop="true" CatchAllDocuments="true" />
|
||||
}
|
||||
else
|
||||
{
|
||||
<MudTextField
|
||||
T="string"
|
||||
Text="@this.dataPath"
|
||||
Label="@T("Selected base directory for this data source")"
|
||||
Class="mb-3"
|
||||
ReadOnly="@true"
|
||||
Validation="@this.dataSourceValidation.ValidatePath"
|
||||
Adornment="Adornment.Start"
|
||||
AdornmentIcon="@Icons.Material.Filled.Folder"
|
||||
UserAttributes="@SPELLCHECK_ATTRIBUTES"
|
||||
Variant="Variant.Outlined"
|
||||
/>
|
||||
}
|
||||
<SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" EnableDragDrop="true" CatchAllDocuments="true" />
|
||||
|
||||
<MudTextSwitch
|
||||
Label="@T("Include subfolders?")"
|
||||
|
||||
@@ -23,16 +23,6 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
|
||||
[Parameter]
|
||||
public DataSourceLocalDirectory DataSource { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the folder this data source reads must stay as it is.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Set once the index holds something for this data source. The embedding is not locked along
|
||||
/// with it: it can be changed, and DataSourceReindexWarning asks what that costs.
|
||||
/// </remarks>
|
||||
[Parameter]
|
||||
public bool LockSource { get; set; }
|
||||
|
||||
[Inject]
|
||||
private IDialogService DialogService { get; init; } = null!;
|
||||
|
||||
@@ -215,8 +205,6 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
|
||||
? T("Audio and video files which this data source selects are skipped, because your transcription provider does not meet the confidence level this data source requires.")
|
||||
: null;
|
||||
|
||||
private bool CanChangeSource => !this.IsEditing || !this.LockSource;
|
||||
|
||||
private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
|
||||
|
||||
private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
|
||||
@@ -244,9 +232,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
|
||||
Type = DataSourceType.LOCAL_DIRECTORY,
|
||||
EmbeddingId = this.dataEmbeddingId,
|
||||
|
||||
// Kept out of reach of the form while the source is locked, so a stale field cannot point an
|
||||
// indexed data source somewhere else:
|
||||
Path = this.CanChangeSource ? this.dataPath : this.DataSource.Path,
|
||||
Path = this.dataPath,
|
||||
FilePatterns = this.NormalizeFilePatterns(),
|
||||
|
||||
// Kept as typed, since the line breaks group the entries for the eye:
|
||||
@@ -268,6 +254,14 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
|
||||
if (!this.dataIsValid)
|
||||
return;
|
||||
|
||||
await this.StoreValidatedDataSourceAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Continues Save after form validation, asking before the draft is returned to its caller.
|
||||
/// </summary>
|
||||
internal async Task StoreValidatedDataSourceAsync()
|
||||
{
|
||||
var addedDataSource = this.CreateDataSource();
|
||||
|
||||
//
|
||||
|
||||
@@ -47,31 +47,7 @@
|
||||
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
|
||||
@T("Select a file for this data source. The content of this file will be processed for the data source. An audio or video file is transcribed by your transcription provider first.")
|
||||
</MudJustifiedText>
|
||||
@if (!this.CanChangeSource)
|
||||
{
|
||||
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
|
||||
@T("The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed.")
|
||||
</MudAlert>
|
||||
}
|
||||
@if (this.CanChangeSource)
|
||||
{
|
||||
<SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" EnableDragDrop="true" CatchAllDocuments="true" />
|
||||
}
|
||||
else
|
||||
{
|
||||
<MudTextField
|
||||
T="string"
|
||||
Text="@this.dataFilePath"
|
||||
Label="@T("Selected file path for this data source")"
|
||||
Class="mb-3"
|
||||
ReadOnly="@true"
|
||||
Validation="@this.dataSourceValidation.ValidateFilePath"
|
||||
Adornment="Adornment.Start"
|
||||
AdornmentIcon="@Icons.Material.Filled.AttachFile"
|
||||
UserAttributes="@SPELLCHECK_ATTRIBUTES"
|
||||
Variant="Variant.Outlined"
|
||||
/>
|
||||
}
|
||||
<SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" EnableDragDrop="true" CatchAllDocuments="true" />
|
||||
|
||||
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
|
||||
@T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding provider.")
|
||||
|
||||
@@ -24,16 +24,6 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
|
||||
[Parameter]
|
||||
public DataSourceLocalFile DataSource { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Whether the file this data source reads must stay as it is.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Set once the index holds something for this data source. The embedding is not locked along
|
||||
/// with it: it can be changed, and DataSourceReindexWarning asks what that costs.
|
||||
/// </remarks>
|
||||
[Parameter]
|
||||
public bool LockSource { get; set; }
|
||||
|
||||
[Inject]
|
||||
private IDialogService DialogService { get; init; } = null!;
|
||||
|
||||
@@ -206,8 +196,6 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
|
||||
? T("This audio or video file is skipped, because your transcription provider does not meet the confidence level this data source requires.")
|
||||
: null;
|
||||
|
||||
private bool CanChangeSource => !this.IsEditing || !this.LockSource;
|
||||
|
||||
private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
|
||||
|
||||
private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
|
||||
@@ -235,9 +223,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
|
||||
Type = DataSourceType.LOCAL_FILE,
|
||||
EmbeddingId = this.dataEmbeddingId,
|
||||
|
||||
// Kept out of reach of the form while the source is locked, so a stale field cannot point an
|
||||
// indexed data source somewhere else:
|
||||
FilePath = this.CanChangeSource ? this.dataFilePath : this.DataSource.FilePath,
|
||||
FilePath = this.dataFilePath,
|
||||
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
|
||||
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
|
||||
ConfidenceLevel = this.dataConfidenceLevel,
|
||||
@@ -252,6 +238,14 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
|
||||
if (!this.dataIsValid)
|
||||
return;
|
||||
|
||||
await this.StoreValidatedDataSourceAsync();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Continues Save after form validation, asking before the draft is returned to its caller.
|
||||
/// </summary>
|
||||
internal async Task StoreValidatedDataSourceAsync()
|
||||
{
|
||||
var addedDataSource = this.CreateDataSource();
|
||||
|
||||
//
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
<ConfigurationDirectory OptionDescription="@T("Default input folder")" Disabled="@this.DefaultsDisabled" DirectoryDialogTitle="@T("Select the default input folder")" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.InputDirectory)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.InputDirectory = value)" OptionHelp="@T("Leave empty when an input folder should be selected for every batch run.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.InputDirectory, out var meta) && meta.IsLocked"/>
|
||||
<ConfigurationOption OptionDescription="@T("Include subfolders by default?")" Disabled="@this.DefaultsDisabled" LabelOn="@T("Subfolders are included")" LabelOff="@T("Only the selected folder is processed")" State="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeSubdirectories)" StateUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeSubdirectories = value)" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.IncludeSubdirectories, out var meta) && meta.IsLocked"/>
|
||||
<ConfigurationOption OptionDescription="@T("Include hidden files and folders by default?")" Disabled="@this.DefaultsDisabled" LabelOn="@T("Hidden files and folders are included")" LabelOff="@T("Hidden files and folders are ignored")" State="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeHiddenFilesAndFolders)" StateUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.IncludeHiddenFilesAndFolders = value)" OptionHelp="@T("Hidden are files and folders whose names start with a dot, such as .git or .env, and those hidden by the operating system. A file pattern which starts with a dot, such as .gitignore, selects such files anyway.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.IncludeHiddenFilesAndFolders, out var meta) && meta.IsLocked"/>
|
||||
<ConfigurationText OptionDescription="@T("Default file patterns")" Disabled="@this.DefaultsDisabled" Icon="@Icons.Material.Filled.FilterAlt" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns = value)" ResetValue="@(() => DataBatchProcessing.DEFAULT_FILE_PATTERNS)" ResetButtonText="@T("Restore default patterns")" OptionHelp="@T("Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.FilePatterns, out var meta) && meta.IsLocked"/>
|
||||
<ConfigurationText OptionDescription="@T("Default file patterns")" Disabled="@this.DefaultsDisabled" Icon="@Icons.Material.Filled.FilterAlt" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.FilePatterns = value)" ResetValue="@(() => DataBatchProcessing.DEFAULT_FILE_PATTERNS)" ResetButtonText="@T("Restore default patterns")" OptionHelp="@T("Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.FilePatterns, out var meta) && meta.IsLocked"/>
|
||||
<ConfigurationText OptionDescription="@T("Default excluded files and folders")" Disabled="@this.DefaultsDisabled" Icon="@Icons.Material.Filled.FilterAltOff" Text="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.ExcludePatterns)" TextUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.ExcludePatterns = value)" NumLines="@FileExclusions.DEFAULT_LINE_COUNT" MaxLines="@FileExclusions.MAX_LINE_COUNT" ResetValue="@(() => FileExclusions.DEFAULT_PATTERNS)" ResetButtonText="@T("Restore default")" OptionHelp="@T("These files and folders are left out, even when a file pattern selects them. Separate entries with a semicolon or a line break. End an entry with a slash to exclude every folder of that name, e.g., node_modules/. Other entries name files, e.g., *.min.js.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.ExcludePatterns, out var meta) && meta.IsLocked"/>
|
||||
<ConfigurationNumber T="int" OptionDescription="@T("Ignore files larger than")" Min="@FolderFileSelection.SMALLEST_MAX_FILE_SIZE_MEGABYTES" Unit="MB" Disabled="@this.DefaultsDisabled" Value="@(() => this.SettingsManager.ConfigurationData.BatchProcessing.MaxFileSizeMegabytes)" ValueUpdate="@(value => this.SettingsManager.ConfigurationData.BatchProcessing.MaxFileSizeMegabytes = value)" OptionHelp="@T("Matching files above this size are left out of a batch run, e.g., images of virtual machines.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.BatchProcessing, x => x.MaxFileSizeMegabytes, out var meta) && meta.IsLocked"/>
|
||||
|
||||
|
||||
@@ -296,7 +296,7 @@
|
||||
@*
|
||||
What the user has to decide comes with the buttons to decide it. The
|
||||
sentence above says what is at stake; reading it and then having to find
|
||||
the settings of the mailbox is where people give up.
|
||||
the settings of the mailbox or the data source is where people give up.
|
||||
*@
|
||||
@if (status.Attention is DataSourceAttention.AUTH_FAILED)
|
||||
{
|
||||
@@ -320,6 +320,14 @@
|
||||
</MudButton>
|
||||
</MudStack>
|
||||
}
|
||||
else if (status.Attention is DataSourceAttention.LOCATION_UNREACHABLE && this.CanEditDataSource(status))
|
||||
{
|
||||
<MudStack Row="@true" Wrap="Wrap.Wrap" Spacing="2">
|
||||
<MudButton Variant="Variant.Filled" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Edit" OnClick="@(async () => await this.EditDataSource(status))">
|
||||
@T("Edit this data source")
|
||||
</MudButton>
|
||||
</MudStack>
|
||||
}
|
||||
</MudStack>
|
||||
</ChildContent>
|
||||
</MudExpansionPanel>
|
||||
|
||||
@@ -509,6 +509,25 @@ public partial class Embeddings : MSGComponentBase
|
||||
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
|
||||
}
|
||||
|
||||
/// <remarks>
|
||||
/// Not offered for a data source the organization manages: its settings cannot be edited.
|
||||
/// </remarks>
|
||||
private bool CanEditDataSource(DataSourceEmbeddingStatus status) => DataSourceEditing.CanEdit(this.SettingsManager, status.DataSourceId);
|
||||
|
||||
/// <summary>
|
||||
/// Opens the settings of a data source, e.g. to select the new location of a moved folder.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Saving them indexes the data source right away, and that run reconnects what is already
|
||||
/// stored to the new location. The page reloads through CONFIGURATION_CHANGED, which it
|
||||
/// receives like everybody else.
|
||||
/// </remarks>
|
||||
private async Task EditDataSource(DataSourceEmbeddingStatus status)
|
||||
{
|
||||
if (await DataSourceEditing.EditAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, status.DataSourceId))
|
||||
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Agrees to the removal a mailbox held back, for the very number the user read.
|
||||
/// </summary>
|
||||
|
||||
@@ -650,7 +650,7 @@ CONFIG["SETTINGS"] = {}
|
||||
--
|
||||
-- Configure the default file patterns and whether subfolders as well as hidden files and folders are included.
|
||||
-- Separate multiple patterns with semicolons.
|
||||
-- CONFIG["SETTINGS"]["DataBatchProcessing.FilePatterns"] = "*.pdf;*.docx;*.pptx;*.xlsx;*.md;*.txt;*.mp3;*.wav;*.wave;*.aac;*.flac;*.ogg;*.opus;*.m4a;*.m4b;*.wma;*.alac;*.aif;*.aiff;*.caf;*.mp4;*.m4v;*.avi;*.mkv;*.mov;*.wmv;*.flv;*.webm"
|
||||
-- CONFIG["SETTINGS"]["DataBatchProcessing.FilePatterns"] = "*.pdf;*.docx;*.pptx;*.xlsx;*.md;*.txt;*.jpg;*.jpeg;*.png;*.webp;*.gif;*.mp3;*.wav;*.wave;*.aac;*.flac;*.ogg;*.opus;*.m4a;*.m4b;*.wma;*.alac;*.aif;*.aiff;*.caf;*.mp4;*.m4v;*.avi;*.mkv;*.mov;*.wmv;*.flv;*.webm"
|
||||
-- CONFIG["SETTINGS"]["DataBatchProcessing.IncludeSubdirectories"] = false
|
||||
-- CONFIG["SETTINGS"]["DataBatchProcessing.IncludeHiddenFilesAndFolders"] = false
|
||||
--
|
||||
|
||||
+63
-24
@@ -426,6 +426,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- The selected folder does not exist.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T17705912"] = "Der ausgewählte Ordner existiert nicht."
|
||||
|
||||
-- The image was no longer available while the model was asked about it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1805791865"] = "Das Bild war nicht mehr verfügbar, während das Modell dazu befragt wurde."
|
||||
|
||||
-- Minimum pause between files
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1829787634"] = "Mindestpause zwischen Dateien"
|
||||
|
||||
@@ -531,9 +534,15 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Was not able to write the result file: {0}
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2991295581"] = "Die Ergebnisdatei konnte nicht geschrieben werden: {0}"
|
||||
|
||||
-- Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3000909441"] = "Bilder im JPG-, PNG-, WebP- oder GIF-Format werden direkt an das Modell übergeben, daher benötigen Sie ein Modell, das Bilder versteht. Falls das ausgewählte Modell keine Bilder lesen kann, entfernen Sie die Dateimuster für Bilder, z. B. *.jpg oder *.png, bevor Sie starten."
|
||||
|
||||
-- The batch run finished, but one file could not be processed. See the progress table and log for details.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3201532790"] = "Die Stapelverarbeitung ist abgeschlossen, aber eine Datei konnte nicht verarbeitet werden. Weitere Informationen finden Sie in der Fortschrittstabelle und im Protokoll."
|
||||
|
||||
-- The selected model cannot read images, so the {0} images among the selected files will fail. All other files are processed as usual.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T324865195"] = "Das ausgewählte Modell kann keine Bilder lesen, daher schlägt die Verarbeitung der {0} Bilder unter den ausgewählten Dateien fehl. Alle anderen Dateien werden wie gewohnt verarbeitet."
|
||||
|
||||
-- Maximum pause between files
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3250003796"] = "Maximale Pause zwischen Dateien"
|
||||
|
||||
@@ -552,6 +561,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Show Expert Settings
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3361153305"] = "Experten-Einstellungen anzeigen"
|
||||
|
||||
-- All selected files are images, but the selected model cannot read images. Select a model with image support.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3365102967"] = "Alle ausgewählten Dateien sind Bilder, aber das ausgewählte Modell kann keine Bilder lesen. Wählen Sie ein Modell mit Bildunterstützung aus."
|
||||
|
||||
-- Header of the result column (optional)
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T340994102"] = "Überschrift der Ergebnisspalte (optional)"
|
||||
|
||||
@@ -564,6 +576,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- The file name of the CSV results table. We add the ending .csv when it is missing. When left empty, we use 'batch-results.csv'.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3439247329"] = "Der Dateiname der CSV-Ergebnistabelle. Die Endung .csv wird ergänzt, falls sie fehlt. Wenn Sie das Feld leer lassen, wird 'batch-results.csv' verwendet."
|
||||
|
||||
-- The selected model cannot read images. Select a model with image support to process this image.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T346986514"] = "Das ausgewählte Modell kann keine Bilder lesen. Wählen Sie ein Modell mit Bildunterstützung aus, um dieses Bild zu verarbeiten."
|
||||
|
||||
-- Document analysis policy
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3510564924"] = "Regelwerk für die Dokumentenanalyse"
|
||||
|
||||
@@ -648,6 +663,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Start batch processing
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T50133258"] = "Stapelverarbeitung starten"
|
||||
|
||||
-- This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T532496054"] = "Dieses Bildformat kann nicht an das Modell übergeben werden. Bitte konvertieren Sie das Bild in JPG, PNG, WebP oder GIF."
|
||||
|
||||
-- Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T544244392"] = "Die Ergebnistabelle des vorherigen Durchlaufs konnte nicht gelesen werden. Die bereits abgeschlossenen Dokumente können nicht wiederhergestellt werden und werden erneut verarbeitet."
|
||||
|
||||
@@ -672,12 +690,18 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Include hidden files and folders?
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T85084423"] = "Versteckte Dateien und Ordner einbeziehen?"
|
||||
|
||||
-- The image is larger than {0} MB, which is the most the selected provider accepts for one image.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T87450623"] = "Das Bild überschreitet die maximale Größe von {0} MB, die der ausgewählte Anbieter pro Bild akzeptiert."
|
||||
|
||||
-- Restore default
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T913168756"] = "Standard wiederherstellen"
|
||||
|
||||
-- The AI may use these tools while working on each document. Every tool has to meet the confidence requirements of the selected provider, so a tool may stay unavailable even when it is selected here.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T967206794"] = "Die KI kann diese Werkzeuge bei der Arbeit an jedem Dokument verwenden. Jedes Werkzeug muss die Vertrauensanforderungen des ausgewählten Anbieters erfüllen. Daher kann ein Werkzeug trotz Auswahl hier weiterhin nicht verfügbar sein."
|
||||
|
||||
-- The selected model cannot read images, so the image among the selected files will fail. All other files are processed as usual.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T9738678"] = "Das ausgewählte Modell kann keine Bilder lesen, daher schlägt die Verarbeitung des Bildes unter den ausgewählten Dateien fehl. Alle anderen Dateien werden wie gewohnt verarbeitet."
|
||||
|
||||
-- Comma (,)
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::BATCHPROCESSINGCSVSEPARATOREXTENSIONS::T1676507543"] = "Komma (,)"
|
||||
|
||||
@@ -4122,9 +4146,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "Si
|
||||
-- Automatic local data source refresh
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1208397349"] = "Automatische Aktualisierung lokaler Datenquellen"
|
||||
|
||||
-- Edit Local Directory Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1215599168"] = "Datenquelle bearbeiten: Lokaler Ordner"
|
||||
|
||||
-- Refresh
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T135637716"] = "Aktualisieren"
|
||||
|
||||
@@ -4152,12 +4173,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2067030738"] = "M
|
||||
-- Local Directory Data Source Information
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2146756020"] = "Informationen zur lokalen Ordner-Datenquelle"
|
||||
|
||||
-- Edit ERI v1 Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T221059217"] = "ERI v1 Datenquelle bearbeiten"
|
||||
|
||||
-- Edit Local File Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2453292893"] = "Datenquelle bearbeiten: Lokale Datei"
|
||||
|
||||
-- Local data sources refresh when files change, mailboxes every 16 minutes.
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2511157475"] = "Lokale Datenquellen werden aktualisiert, wenn sich Dateien ändern, Postfächer alle 16 Minuten."
|
||||
|
||||
@@ -6807,9 +6822,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2460883298"
|
||||
-- Add
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Hinzufügen"
|
||||
|
||||
-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2686150329"] = "Die Dokumente dieser Datenquelle sind bereits vorbereitet, daher kann ihr Ordner nicht geändert werden. Ein anderer Ordner enthält andere Dokumente und ist damit eine andere Datenquelle: Bitte fügen Sie dafür eine neue hinzu. Den Einbettungsanbieter darunter können Sie ändern."
|
||||
|
||||
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "Die von Ihnen ausgewählte Einbettung läuft lokal oder innerhalb Ihrer Organisation. Ihre Daten werden nicht in die Cloud übertragen."
|
||||
|
||||
@@ -7068,9 +7080,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "D
|
||||
-- Data Source Name
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Name der Datenquelle"
|
||||
|
||||
-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T890383764"] = "Die Dokumente dieser Datenquelle sind bereits vorbereitet, daher kann ihre Datei nicht geändert werden. Eine andere Datei enthält andere Inhalte und ist damit eine andere Datenquelle: Bitte fügen Sie dafür eine neue hinzu. Den Einbettungsanbieter darunter können Sie ändern."
|
||||
|
||||
-- Cancel
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T900713019"] = "Abbrechen"
|
||||
|
||||
@@ -8583,6 +8592,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T7117
|
||||
-- Default minimum pause between files
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1008440099"] = "Standardmäßige Mindestpause zwischen Dateien"
|
||||
|
||||
-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1038539663"] = "Trennen Sie mehrere Dateimuster durch ein Semikolon, z. B. *.pdf;*.docx. Die Standardmuster umfassen alle unterstützten Audio- und Videoformate sowie Bilder im JPG-, PNG-, WebP- und GIF-Format."
|
||||
|
||||
-- Instructions
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1221801316"] = "Anweisungen"
|
||||
|
||||
@@ -8637,9 +8649,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T24
|
||||
-- When enabled, new batch runs start with the defaults configured below.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2592677194"] = "Wenn aktiviert, werden neue Stapel-Durchläufe mit den unten konfigurierten Standardwerten gestartet."
|
||||
|
||||
-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2594325620"] = "Trennen Sie mehrere Dateimuster durch ein Semikolon, z. B. *.pdf;*.docx. Die Standardmuster umfassen alle unterstützten Audio- und Videoformate."
|
||||
|
||||
-- Subfolders are included
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2607092632"] = "Unterordner werden einbezogen"
|
||||
|
||||
@@ -10374,6 +10383,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T819555436"] = "Fehlgeschlagene E-
|
||||
-- Skipped
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T852993367"] = "Übersprungen"
|
||||
|
||||
-- Edit this data source
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T878700886"] = "Diese Datenquelle bearbeiten"
|
||||
|
||||
-- Open the transcription settings
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T905182399"] = "Transkriptionseinstellungen öffnen"
|
||||
|
||||
@@ -12210,18 +12222,39 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
|
||||
-- Qdrant Edge is not available.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge ist nicht verfügbar."
|
||||
|
||||
-- Edit Local Directory Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T1215599168"] = "Datenquelle bearbeiten: Lokaler Ordner"
|
||||
|
||||
-- Edit ERI v1 Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T221059217"] = "ERI v1 Datenquelle bearbeiten"
|
||||
|
||||
-- Edit Local File Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T2453292893"] = "Datenquelle bearbeiten: Lokale Datei"
|
||||
|
||||
-- Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T1555760974"] = "Jede andere Datei wird erneut an Ihren Einbettungsanbieter gesendet, was bei vielen Dateien eine Weile dauert. Dateien, die am neuen Speicherort fehlen, werden aus dem Index entfernt, während Ihre Dateien selbst unverändert bleiben."
|
||||
|
||||
-- AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2205693550"] = "Was AI Studio bereits vorbereitet hat, bleibt für jede Datei erhalten, die es am neuen Speicherort unverändert findet, also mit demselben Dateityp, derselben Größe und demselben Änderungsdatum. In einem Ordner muss eine solche Datei außerdem ihren Namen behalten und im selben Unterordner liegen."
|
||||
|
||||
-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "Sie beantworten weiterhin Stichwortsuchen, aber die Suche nach Bedeutung funktioniert nicht mehr, und es können keine weiteren Dokumente für sie vorbereitet werden. Bereits vorbereitete Datenquellen bleiben zudem an diesen Anbieter gebunden, sodass Sie sie nicht einfach auf einen anderen umstellen können."
|
||||
|
||||
-- and {0} more.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "und {0} weitere."
|
||||
|
||||
-- Your embedding provider runs in the cloud, so preparing files again costs money.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2635623258"] = "Ihr Einbettungsanbieter läuft in der Cloud, daher kostet es Geld, Dateien erneut vorzubereiten."
|
||||
|
||||
-- This change makes the prepared documents of the following data sources unusable ({0}):
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "Durch diese Änderung werden die vorbereiteten Dokumente der folgenden Datenquellen unbrauchbar ({0}):"
|
||||
|
||||
-- Do you want to apply this change anyway?
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Möchten Sie diese Änderung trotzdem übernehmen?"
|
||||
|
||||
-- New Location for This Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3462449568"] = "Neuer Speicherort für diese Datenquelle"
|
||||
|
||||
-- Documents Will Be Prepared Again
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Dokumente werden erneut vorbereitet"
|
||||
|
||||
@@ -13404,6 +13437,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source Code
|
||||
-- Document
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Dokument"
|
||||
|
||||
-- Image for AI models
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T791951937"] = "Bild für KI-Modelle"
|
||||
|
||||
-- Plugin archive
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T927001356"] = "Plugin-Archiv"
|
||||
|
||||
@@ -13761,18 +13797,21 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T8042
|
||||
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1214987337"] = "Der Ordner „{0}“ konnte nicht geöffnet werden. Bitte prüfen Sie, ob Sie die erforderlichen Leserechte haben. Aus dem Index wurde nichts entfernt."
|
||||
|
||||
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1393648320"] = "Die Datei „{0}“ ist nicht erreichbar. Möglicherweise ist das Laufwerk, auf dem sie sich befindet, nicht verbunden, oder die Datei wurde verschoben oder umbenannt. Aus dem Index wurde nichts entfernt. Sobald die Datei wieder verfügbar ist, aktualisieren Sie diese Datenquelle. Wenn sie dauerhaft nicht mehr vorhanden ist, entfernen Sie diese Datenquelle."
|
||||
|
||||
-- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1935191670"] = "Die Datei „{0}“ wurde während der Indexierung geändert. Die bereits indexierten Inhalte werden verworfen, und die Datei wird beim nächsten Durchlauf erneut verarbeitet."
|
||||
|
||||
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2268993048"] = "Die Datei „{0}“ ist nicht erreichbar. Möglicherweise ist das Laufwerk, auf dem sie sich befindet, nicht verbunden, oder die Datei wurde verschoben oder umbenannt. Aus dem Index wurde nichts entfernt. Wenn die Datei verschoben oder umbenannt wurde, bearbeiten Sie diese Datenquelle und wählen Sie den neuen Speicherort der Datei aus. Sobald die Datei wieder verfügbar ist, aktualisieren Sie diese Datenquelle. Wenn sie dauerhaft nicht mehr vorhanden ist, entfernen Sie diese Datenquelle."
|
||||
|
||||
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2692299965"] = "Der Ordner „{0}“ ist nicht erreichbar. Möglicherweise ist das Laufwerk, auf dem er sich befindet, nicht verbunden, oder der Ordner wurde verschoben oder umbenannt. Aus dem Index wurde nichts entfernt. Wenn der Ordner verschoben oder umbenannt wurde, bearbeiten Sie diese Datenquelle und wählen Sie den neuen Speicherort des Ordners aus. Sobald der Ordner wieder verfügbar ist, aktualisieren Sie diese Datenquelle."
|
||||
|
||||
-- The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2999277901"] = "Der Ordner „{0}“ scheint derzeit leer zu sein. AI Studio hat den Index dieser Datenquelle beibehalten, da ein Ordner auf einem nicht verbundenen Laufwerk leer erscheinen kann. Verbinden Sie das Laufwerk und aktualisieren Sie diese Datenquelle. Wenn Sie den Ordner absichtlich geleert haben, entfernen Sie diese Datenquelle."
|
||||
|
||||
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3230000698"] = "Der Ordner „{0}“ konnte nicht geöffnet werden. Bitte prüfen Sie, ob Sie ihn lesen dürfen."
|
||||
|
||||
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3722338131"] = "Der Ordner „{0}“ ist nicht erreichbar. Möglicherweise ist das Laufwerk, auf dem er sich befindet, nicht verbunden, oder der Ordner wurde verschoben oder umbenannt. Aus dem Index wurde nichts entfernt. Sobald der Ordner wieder verfügbar ist, aktualisieren Sie diese Datenquelle."
|
||||
|
||||
-- The file '{0}' could not be read. Please check whether you are allowed to read it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3924882233"] = "Die Datei „{0}“ konnte nicht gelesen werden. Bitte prüfen Sie, ob Sie berechtigt sind, sie zu lesen."
|
||||
|
||||
|
||||
+63
-24
@@ -426,6 +426,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- The selected folder does not exist.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T17705912"] = "The selected folder does not exist."
|
||||
|
||||
-- The image was no longer available while the model was asked about it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1805791865"] = "The image was no longer available while the model was asked about it."
|
||||
|
||||
-- Minimum pause between files
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T1829787634"] = "Minimum pause between files"
|
||||
|
||||
@@ -531,9 +534,15 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Was not able to write the result file: {0}
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T2991295581"] = "Was not able to write the result file: {0}"
|
||||
|
||||
-- Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3000909441"] = "Images in JPG, PNG, WebP, or GIF format are given directly to the model, so you need a model that understands images. If the selected model cannot read images, remove the image patterns, such as *.jpg or *.png, before you start."
|
||||
|
||||
-- The batch run finished, but one file could not be processed. See the progress table and log for details.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3201532790"] = "The batch run finished, but one file could not be processed. See the progress table and log for details."
|
||||
|
||||
-- The selected model cannot read images, so the {0} images among the selected files will fail. All other files are processed as usual.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T324865195"] = "The selected model cannot read images, so the {0} images among the selected files will fail. All other files are processed as usual."
|
||||
|
||||
-- Maximum pause between files
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3250003796"] = "Maximum pause between files"
|
||||
|
||||
@@ -552,6 +561,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Show Expert Settings
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3361153305"] = "Show Expert Settings"
|
||||
|
||||
-- All selected files are images, but the selected model cannot read images. Select a model with image support.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3365102967"] = "All selected files are images, but the selected model cannot read images. Select a model with image support."
|
||||
|
||||
-- Header of the result column (optional)
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T340994102"] = "Header of the result column (optional)"
|
||||
|
||||
@@ -564,6 +576,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- The file name of the CSV results table. We add the ending .csv when it is missing. When left empty, we use 'batch-results.csv'.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3439247329"] = "The file name of the CSV results table. We add the ending .csv when it is missing. When left empty, we use 'batch-results.csv'."
|
||||
|
||||
-- The selected model cannot read images. Select a model with image support to process this image.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T346986514"] = "The selected model cannot read images. Select a model with image support to process this image."
|
||||
|
||||
-- Document analysis policy
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T3510564924"] = "Document analysis policy"
|
||||
|
||||
@@ -648,6 +663,9 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Start batch processing
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T50133258"] = "Start batch processing"
|
||||
|
||||
-- This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T532496054"] = "This image format cannot be given to the model. Please convert the image to JPG, PNG, WebP, or GIF."
|
||||
|
||||
-- Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T544244392"] = "Was not able to read the results table of the previous run. Its completed documents cannot be restored and will be processed again."
|
||||
|
||||
@@ -672,12 +690,18 @@ UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING
|
||||
-- Include hidden files and folders?
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T85084423"] = "Include hidden files and folders?"
|
||||
|
||||
-- The image is larger than {0} MB, which is the most the selected provider accepts for one image.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T87450623"] = "The image is larger than {0} MB, which is the most the selected provider accepts for one image."
|
||||
|
||||
-- Restore default
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T913168756"] = "Restore default"
|
||||
|
||||
-- The AI may use these tools while working on each document. Every tool has to meet the confidence requirements of the selected provider, so a tool may stay unavailable even when it is selected here.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T967206794"] = "The AI may use these tools while working on each document. Every tool has to meet the confidence requirements of the selected provider, so a tool may stay unavailable even when it is selected here."
|
||||
|
||||
-- The selected model cannot read images, so the image among the selected files will fail. All other files are processed as usual.
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::ASSISTANTBATCHPROCESSING::T9738678"] = "The selected model cannot read images, so the image among the selected files will fail. All other files are processed as usual."
|
||||
|
||||
-- Comma (,)
|
||||
UI_TEXT_CONTENT["AISTUDIO::ASSISTANTS::BATCHPROCESSING::BATCHPROCESSINGCSVSEPARATOREXTENSIONS::T1676507543"] = "Comma (,)"
|
||||
|
||||
@@ -4122,9 +4146,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "Yo
|
||||
-- Automatic local data source refresh
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1208397349"] = "Automatic local data source refresh"
|
||||
|
||||
-- Edit Local Directory Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1215599168"] = "Edit Local Directory Data Source"
|
||||
|
||||
-- Refresh
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T135637716"] = "Refresh"
|
||||
|
||||
@@ -4152,12 +4173,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2067030738"] = "Ar
|
||||
-- Local Directory Data Source Information
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2146756020"] = "Local Directory Data Source Information"
|
||||
|
||||
-- Edit ERI v1 Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T221059217"] = "Edit ERI v1 Data Source"
|
||||
|
||||
-- Edit Local File Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2453292893"] = "Edit Local File Data Source"
|
||||
|
||||
-- Local data sources refresh when files change, mailboxes every 16 minutes.
|
||||
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2511157475"] = "Local data sources refresh when files change, mailboxes every 16 minutes."
|
||||
|
||||
@@ -6807,9 +6822,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2460883298"
|
||||
-- Add
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add"
|
||||
|
||||
-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2686150329"] = "The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed."
|
||||
|
||||
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
|
||||
|
||||
@@ -7068,9 +7080,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "T
|
||||
-- Data Source Name
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name"
|
||||
|
||||
-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T890383764"] = "The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed."
|
||||
|
||||
-- Cancel
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T900713019"] = "Cancel"
|
||||
|
||||
@@ -8583,6 +8592,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGASSISTANTBIAS::T7117
|
||||
-- Default minimum pause between files
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1008440099"] = "Default minimum pause between files"
|
||||
|
||||
-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1038539663"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats as well as images in JPG, PNG, WebP, and GIF format."
|
||||
|
||||
-- Instructions
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T1221801316"] = "Instructions"
|
||||
|
||||
@@ -8637,9 +8649,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T24
|
||||
-- When enabled, new batch runs start with the defaults configured below.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2592677194"] = "When enabled, new batch runs start with the defaults configured below."
|
||||
|
||||
-- Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats.
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2594325620"] = "Separate multiple file patterns with a semicolon, e.g., *.pdf;*.docx. The standard patterns include all supported audio and video formats."
|
||||
|
||||
-- Subfolders are included
|
||||
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T2607092632"] = "Subfolders are included"
|
||||
|
||||
@@ -10374,6 +10383,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T819555436"] = "Failed mails: {0}"
|
||||
-- Skipped
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T852993367"] = "Skipped"
|
||||
|
||||
-- Edit this data source
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T878700886"] = "Edit this data source"
|
||||
|
||||
-- Open the transcription settings
|
||||
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T905182399"] = "Open the transcription settings"
|
||||
|
||||
@@ -12210,18 +12222,39 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
|
||||
-- Qdrant Edge is not available.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available."
|
||||
|
||||
-- Edit Local Directory Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T1215599168"] = "Edit Local Directory Data Source"
|
||||
|
||||
-- Edit ERI v1 Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T221059217"] = "Edit ERI v1 Data Source"
|
||||
|
||||
-- Edit Local File Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T2453292893"] = "Edit Local File Data Source"
|
||||
|
||||
-- Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T1555760974"] = "Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are."
|
||||
|
||||
-- AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2205693550"] = "AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in."
|
||||
|
||||
-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one."
|
||||
|
||||
-- and {0} more.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "and {0} more."
|
||||
|
||||
-- Your embedding provider runs in the cloud, so preparing files again costs money.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2635623258"] = "Your embedding provider runs in the cloud, so preparing files again costs money."
|
||||
|
||||
-- This change makes the prepared documents of the following data sources unusable ({0}):
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "This change makes the prepared documents of the following data sources unusable ({0}):"
|
||||
|
||||
-- Do you want to apply this change anyway?
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Do you want to apply this change anyway?"
|
||||
|
||||
-- New Location for This Data Source
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3462449568"] = "New Location for This Data Source"
|
||||
|
||||
-- Documents Will Be Prepared Again
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Documents Will Be Prepared Again"
|
||||
|
||||
@@ -13404,6 +13437,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T378481461"] = "Source like p
|
||||
-- Document
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T4165204724"] = "Document"
|
||||
|
||||
-- Image for AI models
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T791951937"] = "Image for AI models"
|
||||
|
||||
-- Plugin archive
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T927001356"] = "Plugin archive"
|
||||
|
||||
@@ -13761,18 +13797,21 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T8042
|
||||
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1214987337"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."
|
||||
|
||||
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1393648320"] = "The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source."
|
||||
|
||||
-- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1935191670"] = "The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run."
|
||||
|
||||
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2268993048"] = "The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source."
|
||||
|
||||
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2692299965"] = "The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source."
|
||||
|
||||
-- The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2999277901"] = "The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source."
|
||||
|
||||
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3230000698"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it."
|
||||
|
||||
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3722338131"] = "The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source."
|
||||
|
||||
-- The file '{0}' could not be read. Please check whether you are allowed to read it.
|
||||
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3924882233"] = "The file '{0}' could not be read. Please check whether you are allowed to read it."
|
||||
|
||||
|
||||
@@ -288,6 +288,27 @@ public static class LLMProvidersExtensions
|
||||
_ => false,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// The largest image file, in bytes, which the provider accepts in a request, where it states one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The limit belongs to the API of the provider, not to the model: the same model may accept
|
||||
/// larger images through another provider. What we actually send is asked through
|
||||
/// IImageSourceExtensions.MaxImageSizeBytesFor, which also applies our own limit.
|
||||
/// </remarks>
|
||||
/// <param name="llmProvider">The provider to check.</param>
|
||||
/// <returns>The size limit of one image file, or null when the provider states none we know of.</returns>
|
||||
public static long? GetMaxImageSizeBytes(this LLMProviders llmProvider) => llmProvider switch
|
||||
{
|
||||
//
|
||||
// Anthropic accepts at most 5 MiB (5,242,880 bytes) of base64 per image. Base64 needs four
|
||||
// characters for every three bytes, which leaves 3,932,160 bytes for the image file itself:
|
||||
//
|
||||
LLMProviders.ANTHROPIC => 3_932_160,
|
||||
|
||||
_ => null,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Creates a new provider instance based on the provider value.
|
||||
/// </summary>
|
||||
|
||||
@@ -11,7 +11,7 @@ namespace AIStudio.Settings.DataModel;
|
||||
/// <param name="configSelection">The managed-configuration selector.</param>
|
||||
public sealed class DataBatchProcessing(Expression<Func<Data, DataBatchProcessing>>? configSelection = null)
|
||||
{
|
||||
public const string DEFAULT_FILE_PATTERNS = "*.pdf;*.docx;*.pptx;*.xlsx;*.md;*.txt;*.mp3;*.wav;*.wave;*.aac;*.flac;*.ogg;*.opus;*.m4a;*.m4b;*.wma;*.alac;*.aif;*.aiff;*.caf;*.mp4;*.m4v;*.avi;*.mkv;*.mov;*.wmv;*.flv;*.webm";
|
||||
public const string DEFAULT_FILE_PATTERNS = "*.pdf;*.docx;*.pptx;*.xlsx;*.md;*.txt;*.jpg;*.jpeg;*.png;*.webp;*.gif;*.mp3;*.wav;*.wave;*.aac;*.flac;*.ogg;*.opus;*.m4a;*.m4b;*.wma;*.alac;*.aif;*.aiff;*.caf;*.mp4;*.m4v;*.avi;*.mkv;*.mov;*.wmv;*.flv;*.webm";
|
||||
public const int MIN_DELAY_SECONDS = 6;
|
||||
public const int MAX_DELAY_SECONDS = 300;
|
||||
public const int DEFAULT_MIN_DELAY_SECONDS = 6;
|
||||
|
||||
@@ -81,10 +81,17 @@ public static partial class ProviderExtensions
|
||||
/// wants to know whether an image may be sent has to accept both, which is why the question is asked
|
||||
/// here instead of at each call site: attaching a file and validating an already attached file must
|
||||
/// never disagree about it.
|
||||
///
|
||||
/// A limit of zero images takes the capability back. Zero is a real answer, e.g., from an operator
|
||||
/// whose engine accepts no images at all, see ImageLimits; an unknown limit says nothing.
|
||||
/// </remarks>
|
||||
/// <param name="provider">The configured provider.</param>
|
||||
/// <returns><c>true</c> when the model accepts image input.</returns>
|
||||
public static bool SupportsImageInput(this Provider provider) => provider.GetModelProfile().HasAny(Capability.SINGLE_IMAGE_INPUT | Capability.MULTIPLE_IMAGE_INPUT);
|
||||
public static bool SupportsImageInput(this Provider provider)
|
||||
{
|
||||
var profile = provider.GetModelProfile();
|
||||
return profile.HasAny(Capability.SINGLE_IMAGE_INPUT | Capability.MULTIPLE_IMAGE_INPUT) && profile.Images.MaxInOneMessage is not 0;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks whether this model can be used for chatting.
|
||||
|
||||
@@ -0,0 +1,115 @@
|
||||
using AIStudio.Dialogs;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
using DialogOptions = AIStudio.Dialogs.DialogOptions;
|
||||
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Opens the settings of a data source, and stores what the user changed in them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Kept here rather than in the places which offer it -- the data source table and the background
|
||||
/// embeddings page, which leads to it when the file or the folder of a data source cannot be
|
||||
/// reached -- so that saving a data source works the same way everywhere: the settings are stored,
|
||||
/// and the data source is indexed right after. Mailboxes are kept in a list of their own and have
|
||||
/// MailboxEditing instead.
|
||||
/// </remarks>
|
||||
public static class DataSourceEditing
|
||||
{
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEditing).Namespace, nameof(DataSourceEditing));
|
||||
|
||||
/// <summary>
|
||||
/// Whether the user may edit a data source.
|
||||
/// </summary>
|
||||
/// <param name="settingsManager">The settings, which hold the data source.</param>
|
||||
/// <param name="dataSourceId">The id of the data source.</param>
|
||||
/// <returns>False when the data source is gone or managed by the organization.</returns>
|
||||
public static bool CanEdit(SettingsManager settingsManager, string dataSourceId) =>
|
||||
settingsManager.ConfigurationData.DataSources.Any(dataSource => dataSource.Id == dataSourceId && !dataSource.IsEnterpriseConfiguration);
|
||||
|
||||
/// <summary>
|
||||
/// Lets the user edit a data source, and stores and indexes it when they save.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The caller tells the rest of the app that the configuration changed, as the sender of that message.
|
||||
/// </remarks>
|
||||
/// <param name="dialogService">The dialog service to show the settings with.</param>
|
||||
/// <param name="settingsManager">The settings, which hold the data source.</param>
|
||||
/// <param name="embeddingService">The service which indexes the data source.</param>
|
||||
/// <param name="dataSourceId">The id of the data source.</param>
|
||||
/// <returns>True when the user saved the data source. False when they cancelled, or the data source is gone or managed by the organization.</returns>
|
||||
public static async Task<bool> EditAsync(IDialogService dialogService, SettingsManager settingsManager, DataSourceEmbeddingService embeddingService, string dataSourceId)
|
||||
{
|
||||
var dataSources = settingsManager.ConfigurationData.DataSources;
|
||||
var storedDataSource = dataSources.Find(dataSource => dataSource.Id == dataSourceId);
|
||||
if (storedDataSource is null || storedDataSource.IsEnterpriseConfiguration)
|
||||
return false;
|
||||
|
||||
IDataSource? editedDataSource = null;
|
||||
switch (storedDataSource)
|
||||
{
|
||||
case DataSourceLocalFile localFile:
|
||||
var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, localFile },
|
||||
};
|
||||
|
||||
var localFileDialogReference = await dialogService.ShowAsync<DataSourceLocalFileDialog>(TB("Edit Local File Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localFileDialogResult = await localFileDialogReference.Result;
|
||||
if (localFileDialogResult is null || localFileDialogResult.Canceled)
|
||||
return false;
|
||||
|
||||
editedDataSource = (DataSourceLocalFile)localFileDialogResult.Data!;
|
||||
break;
|
||||
|
||||
case DataSourceLocalDirectory localDirectory:
|
||||
var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, localDirectory },
|
||||
};
|
||||
|
||||
var localDirectoryDialogReference = await dialogService.ShowAsync<DataSourceLocalDirectoryDialog>(TB("Edit Local Directory Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
|
||||
if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
|
||||
return false;
|
||||
|
||||
editedDataSource = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
|
||||
break;
|
||||
|
||||
case DataSourceERI_V1 eriDataSource:
|
||||
var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, eriDataSource },
|
||||
};
|
||||
|
||||
var eriDialogReference = await dialogService.ShowAsync<DataSourceERI_V1Dialog>(TB("Edit ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
|
||||
var eriDialogResult = await eriDialogReference.Result;
|
||||
if (eriDialogResult is null || eriDialogResult.Canceled)
|
||||
return false;
|
||||
|
||||
editedDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
|
||||
break;
|
||||
}
|
||||
|
||||
if (editedDataSource is null)
|
||||
return false;
|
||||
|
||||
// Found anew, since the list may have changed while the dialog was open:
|
||||
var position = dataSources.IndexOf(storedDataSource);
|
||||
if (position < 0)
|
||||
return false;
|
||||
|
||||
dataSources[position] = editedDataSource;
|
||||
|
||||
await settingsManager.StoreSettings();
|
||||
await embeddingService.QueueDataSourceAsync(editedDataSource);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
@@ -2,6 +2,7 @@ using System.Text;
|
||||
|
||||
using AIStudio.Dialogs;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.PluginSystem;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
@@ -19,6 +20,10 @@ namespace AIStudio.Tools;
|
||||
/// Nothing is asked when nothing is lost. A data source only reaches the question when the edit
|
||||
/// really changes its embedding signature and when the index already holds something for it, so
|
||||
/// renaming an embedding provider or editing a data source nobody has indexed yet stays silent.
|
||||
/// A new location of a local file or folder has a question of its own, under the same condition of
|
||||
/// a stored index: the signature stays, but every file which is not found unchanged at the new
|
||||
/// location is prepared again. When the signature changes as well, only its question is asked,
|
||||
/// since everything is prepared again anyway.
|
||||
/// </remarks>
|
||||
public static class DataSourceReindexWarning
|
||||
{
|
||||
@@ -84,11 +89,60 @@ public static class DataSourceReindexWarning
|
||||
DataSourceEmbeddingProviders.TryResolve(settingsManager, before, out var resolvedBeforeProvider);
|
||||
var beforeProvider = resolvedBeforeProvider ?? EmbeddingProvider.NONE;
|
||||
|
||||
if (!EmbeddingChangeImpact.AffectsStoredIndex(before, beforeProvider, after, afterProvider))
|
||||
// Everything is prepared again, wherever the files are, so this question covers a new location as well:
|
||||
if (EmbeddingChangeImpact.AffectsStoredIndex(before, beforeProvider, after, afterProvider))
|
||||
{
|
||||
var affected = await embeddingService.GetDataSourcesWithStoredIndexAsync([after], token);
|
||||
return await ConfirmAsync(dialogService, affected, !afterProvider.IsSelfHosted);
|
||||
}
|
||||
|
||||
if (!HasLocationChanged(before, after))
|
||||
return true;
|
||||
|
||||
var affected = await embeddingService.GetDataSourcesWithStoredIndexAsync([after], token);
|
||||
return await ConfirmAsync(dialogService, affected, !afterProvider.IsSelfHosted);
|
||||
var indexed = await embeddingService.GetDataSourcesWithStoredIndexAsync([after], token);
|
||||
return indexed.Count == 0 || await ConfirmNewLocationAsync(dialogService, !afterProvider.IsSelfHosted);
|
||||
}
|
||||
|
||||
private static bool HasLocationChanged(IIndexedDataSource before, IIndexedDataSource after) => (before, after) switch
|
||||
{
|
||||
(DataSourceLocalDirectory oldFolder, DataSourceLocalDirectory newFolder) => !string.Equals(oldFolder.Path, newFolder.Path, StringComparison.Ordinal),
|
||||
(DataSourceLocalFile oldFile, DataSourceLocalFile newFile) => !string.Equals(oldFile.FilePath, newFile.FilePath, StringComparison.Ordinal),
|
||||
_ => false,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Asks before a local file or folder with a stored index is saved with a new location.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The conditions for keeping a file are the ones FileSourceIndexer.FindRelocatedFile checks.
|
||||
/// </remarks>
|
||||
/// <param name="dialogService">The dialog service to ask with.</param>
|
||||
/// <param name="usesCloudEmbedding">Whether preparing a file again costs money.</param>
|
||||
/// <returns>True when the edit may be saved.</returns>
|
||||
private static async Task<bool> ConfirmNewLocationAsync(IDialogService dialogService, bool usesCloudEmbedding)
|
||||
{
|
||||
var body = new StringBuilder();
|
||||
body.AppendLine(TB("AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in."));
|
||||
body.AppendLine();
|
||||
body.AppendLine(TB("Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are."));
|
||||
|
||||
if (usesCloudEmbedding)
|
||||
{
|
||||
body.AppendLine();
|
||||
body.AppendLine(TB("Your embedding provider runs in the cloud, so preparing files again costs money."));
|
||||
}
|
||||
|
||||
body.AppendLine();
|
||||
body.AppendLine(TB("Do you want to apply this change anyway?"));
|
||||
|
||||
var dialogParameters = new DialogParameters<ConfirmDialog>
|
||||
{
|
||||
{ x => x.MarkdownBody, body.ToString() },
|
||||
};
|
||||
|
||||
var dialogReference = await dialogService.ShowAsync<ConfirmDialog>(TB("New Location for This Data Source"), dialogParameters, Dialogs.DialogOptions.FULLSCREEN);
|
||||
var dialogResult = await dialogReference.Result;
|
||||
return dialogResult is not null && !dialogResult.Canceled;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
@@ -192,4 +246,4 @@ public static class DataSourceReindexWarning
|
||||
var dialogResult = await dialogReference.Result;
|
||||
return dialogResult is not null && !dialogResult.Canceled;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,6 +29,11 @@ public abstract partial class IndexStoreClient(string name, string path) : Datab
|
||||
|
||||
public abstract Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token);
|
||||
|
||||
/// <summary>
|
||||
/// Updates the location and fingerprint of a document, preserving its chunks and their ids.
|
||||
/// </summary>
|
||||
public abstract Task RelocateFileAsync(string dataSourceId, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token);
|
||||
|
||||
public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token);
|
||||
|
||||
public abstract Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token);
|
||||
|
||||
@@ -52,6 +52,9 @@ public sealed partial class NoIndexStoreClient(string name, string? unavailableR
|
||||
|
||||
public override Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task RelocateFileAsync(string dataSourceId, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token) =>
|
||||
Task.FromException(new InvalidOperationException(unavailableReason ?? "The index of the data sources is not available."));
|
||||
|
||||
public override Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
+16
-2
@@ -149,7 +149,9 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
|
||||
file.LastWriteUtc,
|
||||
file.EmbeddedAtUtc,
|
||||
file.ChunkCount,
|
||||
file.TranscriptKey);
|
||||
file.TranscriptKey,
|
||||
file.RelativePath,
|
||||
file.ParentFileId);
|
||||
}
|
||||
|
||||
var permanentFailures = await context.PermanentIndexingFailures
|
||||
@@ -233,6 +235,18 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
|
||||
await context.SaveChangesAsync(token);
|
||||
}
|
||||
|
||||
public override async Task RelocateFileAsync(string dataSourceId, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
var file = await context.EmbeddedFiles.SingleAsync(file => file.DataSourceId == dataSourceId && file.ParentFileId == parentFileId, token);
|
||||
// Keep the parent and chunk ids: moving a document does not change its stored vectors.
|
||||
file.AbsolutePath = absolutePath;
|
||||
file.RelativePath = relativePath;
|
||||
file.FileName = Path.GetFileName(absolutePath);
|
||||
file.Fingerprint = fingerprint;
|
||||
await context.SaveChangesAsync(token);
|
||||
}
|
||||
|
||||
public override async Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token)
|
||||
{
|
||||
await using var context = this.CreateContext();
|
||||
@@ -762,4 +776,4 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
|
||||
client.SetLogger(databaseClientLogger);
|
||||
return client;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -37,7 +37,10 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
|
||||
public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, CancellationToken token) =>
|
||||
Task.FromException<IReadOnlyList<VectorSearchResult>>(this.CreateUnavailableException());
|
||||
|
||||
public override Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token) =>
|
||||
public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token) =>
|
||||
Task.FromException(this.CreateUnavailableException());
|
||||
|
||||
public override Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token) =>
|
||||
Task.FromException(this.CreateUnavailableException());
|
||||
|
||||
public override Task OptimizeVectorStore(string storeName, CancellationToken token) =>
|
||||
|
||||
+9
-3
@@ -27,6 +27,7 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
private const string INSERT_PATH = "/system/qdrant-edge/insert";
|
||||
private const string SEARCH_PATH = "/system/qdrant-edge/search";
|
||||
private const string DELETE_FILE_PATH = "/system/qdrant-edge/delete-file";
|
||||
private const string RELOCATE_FILE_PATH = "/system/qdrant-edge/relocate-file";
|
||||
private const string OPTIMIZE_PATH = "/system/qdrant-edge/optimize";
|
||||
private const string DELETE_STORE_PATH = "/system/qdrant-edge/delete-store";
|
||||
private const string PREPARE_BACKUP_PATH = "/system/qdrant-edge/backup/prepare";
|
||||
@@ -156,8 +157,11 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
token) ?? [];
|
||||
}
|
||||
|
||||
public override Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token) =>
|
||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_FILE_PATH, new DeleteEmbeddingByFileRequest(storeName, filePath), token);
|
||||
public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token) =>
|
||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_FILE_PATH, new DeleteEmbeddingByFileRequest(storeName, filePath, parentFileId), token);
|
||||
|
||||
public override Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token) =>
|
||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, RELOCATE_FILE_PATH, new RelocateEmbeddingFileRequest(storeName, parentFileId, absolutePath, Path.GetFileName(absolutePath), relativePath, fingerprint), token);
|
||||
|
||||
public override Task OptimizeVectorStore(string storeName, CancellationToken token) =>
|
||||
rustService.ExecuteDatabaseOperation(DATABASE_NAME, OPTIMIZE_PATH, new OptimizeVectorStoreRequest(storeName), token);
|
||||
@@ -194,7 +198,9 @@ public sealed class QdrantEdgeClientImplementation(
|
||||
/// <param name="PointIds">The only points the search may return, or null for the whole store.</param>
|
||||
private sealed record SearchEmbeddingRequest(string StoreName, IReadOnlyList<float> Vector, int MaxMatches, IReadOnlyList<string>? PointIds);
|
||||
|
||||
private sealed record DeleteEmbeddingByFileRequest(string StoreName, string FilePath);
|
||||
private sealed record DeleteEmbeddingByFileRequest(string StoreName, string FilePath, string ParentFileId);
|
||||
|
||||
private sealed record RelocateEmbeddingFileRequest(string StoreName, string ParentFileId, string AbsolutePath, string FileName, string RelativePath, string Fingerprint);
|
||||
|
||||
private sealed record OptimizeVectorStoreRequest(string StoreName);
|
||||
|
||||
|
||||
@@ -27,7 +27,12 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
|
||||
/// <returns>The matches, best first, all of them inside the filter.</returns>
|
||||
public abstract Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, CancellationToken token);
|
||||
|
||||
public abstract Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token);
|
||||
public abstract Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token);
|
||||
|
||||
/// <summary>
|
||||
/// Updates the location of a document's points without changing their ids or vectors.
|
||||
/// </summary>
|
||||
public abstract Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token);
|
||||
|
||||
public abstract Task OptimizeVectorStore(string storeName, CancellationToken token);
|
||||
|
||||
|
||||
@@ -43,7 +43,7 @@ public static class MailboxEditing
|
||||
{
|
||||
{ x => x.IsEditing, true },
|
||||
{ x => x.DataSource, storedMailbox },
|
||||
{ x => x.LockSource, await embeddingService.ShouldLockDataSourceOriginAsync(storedMailbox.Id) },
|
||||
{ x => x.LockSource, await embeddingService.ShouldLockMailboxOriginAsync(storedMailbox.Id) },
|
||||
};
|
||||
|
||||
var dialogReference = await dialogService.ShowAsync<DataSourceMailboxDialog>(TB("Edit Mailbox"), dialogParameters, DialogOptions.FULLSCREEN);
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Shared rules for comparing file system paths.
|
||||
/// Shared rules for comparing and resolving file system paths.
|
||||
/// </summary>
|
||||
public static class PathTools
|
||||
{
|
||||
@@ -42,4 +42,30 @@ public static class PathTools
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Finds the place where a file dialog opens for a path which may no longer exist.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Given a file or a folder which was moved or renamed, the native dialog cannot open its place
|
||||
/// and falls back to wherever it was used last. The nearest folder which still exists is the
|
||||
/// better start: the file or the folder was moved from there, so its new place is often close by.
|
||||
/// A relative path, or a path of another operating system, e.g. from a backup, has no such folder.
|
||||
/// </remarks>
|
||||
/// <param name="path">The file or the folder the dialog should open at.</param>
|
||||
/// <returns>The path itself when it exists, else its nearest parent folder which exists, or null when there is none.</returns>
|
||||
public static string? FindNearestExistingPath(string path)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(path) || !Path.IsPathFullyQualified(path))
|
||||
return null;
|
||||
|
||||
if (File.Exists(path) || Directory.Exists(path))
|
||||
return path;
|
||||
|
||||
for (var directory = Path.GetDirectoryName(path); !string.IsNullOrEmpty(directory); directory = Path.GetDirectoryName(directory))
|
||||
if (Directory.Exists(directory))
|
||||
return directory;
|
||||
|
||||
return null;
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
using System.Text;
|
||||
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Tools.Security;
|
||||
|
||||
namespace AIStudio.Tools.RAG;
|
||||
|
||||
@@ -100,6 +100,17 @@ public static class FileTypes
|
||||
public static readonly FileTypeFilter IMAGE = FileTypeFilter.Leaf(TB("Image"),
|
||||
"jpg", "jpeg", "png", "gif", "bmp", "tiff", "svg", "webp", "heic", "avif");
|
||||
|
||||
/// <summary>
|
||||
/// The image formats we hand to a model as they are.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The providers accept these, while other formats, such as TIFF or HEIC, would end in an
|
||||
/// error of the provider which tells the user nothing. The default file patterns of the batch
|
||||
/// processing assistant list the same formats, see DataBatchProcessing.DEFAULT_FILE_PATTERNS.
|
||||
/// </remarks>
|
||||
public static readonly FileTypeFilter MODEL_IMAGE = FileTypeFilter.Leaf(TB("Image for AI models"),
|
||||
"jpg", "jpeg", "png", "webp", "gif");
|
||||
|
||||
/// <summary>
|
||||
/// Gets the prototype visual-asset image formats.
|
||||
/// </summary>
|
||||
|
||||
@@ -24,4 +24,9 @@ public enum DataSourceAttention
|
||||
/// The organization allows only its own mail servers, and the mailbox is on another one. AI Studio does not connect to it until the organization allows the server.
|
||||
/// </summary>
|
||||
SERVER_NOT_ALLOWED,
|
||||
|
||||
/// <summary>
|
||||
/// The file or the folder of a local data source cannot be reached, cannot be opened, or appears empty. AI Studio keeps the index until the file or the folder is back, the user selects its new location, or removes the data source.
|
||||
/// </summary>
|
||||
LOCATION_UNREACHABLE,
|
||||
}
|
||||
@@ -1,7 +1,27 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
/// <summary>
|
||||
/// What the index store holds for one data source, read at the start of a run.
|
||||
/// </summary>
|
||||
public sealed class DataSourceEmbeddingManifest
|
||||
{
|
||||
/// <summary>
|
||||
/// How the index compares the paths of the documents it stores, on every OS.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Ignores case, unlike PathTools.COMPARER, which follows the file system. The index store keeps
|
||||
/// the paths with the NOCASE collation and allows each path of a data source once, so two files
|
||||
/// which differ in their case alone share one row there. Whatever the indexer keys or compares by
|
||||
/// these paths has to agree with the store; telling the two files apart would let them replace
|
||||
/// each other's row on every run.
|
||||
/// </remarks>
|
||||
public static readonly StringComparer PATH_COMPARER = StringComparer.OrdinalIgnoreCase;
|
||||
|
||||
/// <summary>
|
||||
/// The comparison matching PATH_COMPARER, e.g., for prefixes.
|
||||
/// </summary>
|
||||
public const StringComparison PATH_COMPARISON = StringComparison.OrdinalIgnoreCase;
|
||||
|
||||
public string EmbeddingProviderId { get; set; } = string.Empty;
|
||||
|
||||
public string EmbeddingSignature { get; set; } = string.Empty;
|
||||
@@ -10,7 +30,7 @@ public sealed class DataSourceEmbeddingManifest
|
||||
|
||||
public int VectorSize { get; set; }
|
||||
|
||||
public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(PATH_COMPARER);
|
||||
|
||||
/// <summary>
|
||||
/// The files whose indexing began but never came to an end, by their absolute path.
|
||||
@@ -21,7 +41,7 @@ public sealed class DataSourceEmbeddingManifest
|
||||
/// meanwhile. Indexing such a file again replaces both. A file which is gone, or which the data
|
||||
/// source no longer selects, is never indexed again, so the indexer has to remove them itself.
|
||||
/// </remarks>
|
||||
public HashSet<string> UnfinishedFiles { get; init; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
public HashSet<string> UnfinishedFiles { get; init; } = new(PATH_COMPARER);
|
||||
|
||||
/// <summary>
|
||||
/// The files whose indexing failed for a reason which lies in the file itself, keyed by their
|
||||
@@ -32,5 +52,5 @@ public sealed class DataSourceEmbeddingManifest
|
||||
/// a folder holding hundreds of scanned documents without a text layer would be read again on
|
||||
/// every single run, with the outcome known in advance.
|
||||
/// </remarks>
|
||||
public Dictionary<string, PermanentIndexingFailureRecord> PermanentFailures { get; init; } = new(StringComparer.OrdinalIgnoreCase);
|
||||
public Dictionary<string, PermanentIndexingFailureRecord> PermanentFailures { get; init; } = new(PATH_COMPARER);
|
||||
}
|
||||
@@ -158,14 +158,18 @@ public sealed partial class DataSourceEmbeddingService
|
||||
/// files which no longer belong to the data source, and the ones which now do have to wait for
|
||||
/// the next run.
|
||||
///
|
||||
/// Only a local folder chooses its documents by file patterns, exclusions, switches, and a size
|
||||
/// limit. For every other kind of data source, this is empty.
|
||||
/// Local files and folders include their configured paths, so relocating one stops a run which
|
||||
/// still reads the old location. Folders also include their file selection. Neither changes the
|
||||
/// embedding signature: unchanged documents keep their vectors.
|
||||
/// </remarks>
|
||||
/// <param name="dataSource">The data source to index.</param>
|
||||
/// <returns>The description of its selection, or an empty string.</returns>
|
||||
internal static string BuildSelectionSignature(IIndexedDataSource dataSource) => dataSource is DataSourceLocalDirectory directory
|
||||
? FolderFileSelection.Of(directory).Describe()
|
||||
: string.Empty;
|
||||
internal static string BuildSelectionSignature(IIndexedDataSource dataSource) => dataSource switch
|
||||
{
|
||||
DataSourceLocalDirectory directory => $"{directory.Path.Length}:{directory.Path}|{FolderFileSelection.Of(directory).Describe()}",
|
||||
DataSourceLocalFile file => $"{file.FilePath.Length}:{file.FilePath}",
|
||||
_ => string.Empty,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Describes the indexing run a data source would get with a given embedding provider.
|
||||
|
||||
@@ -190,24 +190,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether the file or folder a data source reads must stay as it is.
|
||||
/// Whether a mailbox's origin must stay as it is.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Locked as soon as the index holds anything, because where a data source reads from is what it
|
||||
/// is: another folder is another data source, and the path reaches no signature, so swapping it
|
||||
/// would leave the stored index describing documents nobody points at any more.
|
||||
///
|
||||
/// The embedding provider used to be locked along with it and no longer is. It does reach the
|
||||
/// signature, so changing it rebuilds the index cleanly -- and DataSourceReindexWarning asks
|
||||
/// before it does. Locking it as well left a data source whose provider was deleted stuck on
|
||||
/// keyword search for good, with no way back.
|
||||
/// Locked as soon as the index holds anything: another server, account, or folder holds other
|
||||
/// mails, which makes it another data source. Local files and folders are not locked, since they
|
||||
/// may move, and the file indexer finds their documents again, cf. FileSourceIndexer.RelocateStoredFilesAsync.
|
||||
/// The embedding provider is never locked; DataSourceReindexWarning asks what changing it costs.
|
||||
///
|
||||
/// Unclear counts as locked: an unavailable index database says nothing about what is stored.
|
||||
/// </remarks>
|
||||
/// <param name="dataSourceId">The data source to ask about.</param>
|
||||
/// <param name="dataSourceId">The mailbox to ask about.</param>
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
/// <returns>True when the source must not be changed.</returns>
|
||||
public async Task<bool> ShouldLockDataSourceOriginAsync(string dataSourceId, CancellationToken token = default)
|
||||
public async Task<bool> ShouldLockMailboxOriginAsync(string dataSourceId, CancellationToken token = default)
|
||||
{
|
||||
var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
|
||||
if (!indexStore.IsAvailable)
|
||||
@@ -1504,4 +1500,4 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
{
|
||||
_ = MessageBus.INSTANCE.SendMessage(null, Event.RAG_EMBEDDING_STATUS_CHANGED, true);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,3 +1,4 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount, string? TranscriptKey = null);
|
||||
public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount, string? TranscriptKey = null,
|
||||
string? RelativePath = null, string? ParentFileId = null);
|
||||
@@ -226,7 +226,7 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
|
||||
var embeddedAtUtc = DateTimeOffset.UtcNow;
|
||||
var state = document.State with { ChunkCount = chunkCount, EmbeddedAtUtc = embeddedAtUtc };
|
||||
await context.IndexStore.UpsertFileAsync(context.DataSource.Id, state, token);
|
||||
context.Manifest.Files[document.Key] = new EmbeddedFileRecord(state.Fingerprint, state.FileSize, state.LastWriteUtc, embeddedAtUtc, chunkCount, state.TranscriptKey);
|
||||
context.Manifest.Files[document.Key] = new EmbeddedFileRecord(state.Fingerprint, state.FileSize, state.LastWriteUtc, embeddedAtUtc, chunkCount, state.TranscriptKey, state.RelativePath, state.ParentFileId);
|
||||
await context.ForgetPermanentFailureAsync(document.Key, token);
|
||||
context.Pass.Forget(document.Key);
|
||||
|
||||
|
||||
@@ -0,0 +1,91 @@
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
|
||||
namespace AIStudio.Tools.Services.Indexing;
|
||||
|
||||
internal sealed partial class FileSourceIndexer
|
||||
{
|
||||
/// <summary>
|
||||
/// Reconnects unchanged files before enumeration and missing-file cleanup. In particular, media
|
||||
/// must already be known under their new paths when enumeration asks whether they need transcription.
|
||||
/// </summary>
|
||||
private async Task<bool> RelocateStoredFilesAsync(IndexedRunContext context, IDataSource dataSource, CancellationToken token)
|
||||
{
|
||||
DocumentRunProgress? progress = null;
|
||||
foreach (var (oldPath, record) in context.Manifest.Files.ToArray())
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
var file = FindRelocatedFile(dataSource, oldPath, record);
|
||||
if (file is null || record.ParentFileId is not { } parentFileId || (!DataSourceEmbeddingManifest.PATH_COMPARER.Equals(oldPath, file.FullName) && context.Manifest.Files.ContainsKey(file.FullName)))
|
||||
continue;
|
||||
|
||||
progress ??= DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger);
|
||||
if (progress.HasDoneItsShare(DateTimeOffset.UtcNow))
|
||||
return false;
|
||||
progress.Publish();
|
||||
|
||||
var relativePath = this.TryGetRelativePath(dataSource, file);
|
||||
// Use the metadata which passed the reuse check. If the file changes during these
|
||||
// writes, enumeration must see a different fingerprint and index its new content.
|
||||
var fingerprint = BuildFileMetadataHash(file.FullName, file.Name, record.LastWriteUtc.UtcDateTime, record.FileSize);
|
||||
|
||||
// Vectors first, then SQLite. Selecting by the unchanged parent id makes this safe to
|
||||
// retry if cancellation or a crash falls between the two writes. A failure aborts the
|
||||
// run before missing-file cleanup can remove the documents we are reconnecting.
|
||||
await context.VectorStore.RelocateEmbeddingFileAsync(context.CollectionName, parentFileId, file.FullName, relativePath, fingerprint, token);
|
||||
token.ThrowIfCancellationRequested();
|
||||
await context.IndexStore.RelocateFileAsync(dataSource.Id, parentFileId, file.FullName, relativePath, fingerprint, token);
|
||||
context.Manifest.Files.Remove(oldPath);
|
||||
context.Manifest.Files[file.FullName] = record with { Fingerprint = fingerprint, RelativePath = relativePath };
|
||||
|
||||
logger.LogInformation("Reused stored embeddings for relocated file '{OldPath}' at '{NewPath}' in data source '{DataSourceId}'.", oldPath, file.FullName, dataSource.Id);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Uses the same size and modification-time check as ordinary indexing, matching a folder's
|
||||
/// documents by relative path. It never searches another folder for a missing document.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A stored path which still lies below the selected folder stays as it is: it points at its
|
||||
/// file, and a file which is gone from there is gone. Mapping it through its old relative path
|
||||
/// would move it, e.g., when the selected folder grows to its parent. A copy at the mapped place
|
||||
/// would then take over the identity of the document while the original is indexed anew.
|
||||
/// Checking this first also spares every run of an unmoved folder resolving each stored path.
|
||||
/// </remarks>
|
||||
internal static FileInfo? FindRelocatedFile(IDataSource dataSource, string oldPath, EmbeddedFileRecord record)
|
||||
{
|
||||
if (record.ParentFileId is null)
|
||||
return null;
|
||||
|
||||
string targetPath;
|
||||
switch (dataSource)
|
||||
{
|
||||
case DataSourceLocalFile localFile when RelocatedFilePaths.IsLocalAbsolute(localFile.FilePath):
|
||||
targetPath = Path.GetFullPath(localFile.FilePath);
|
||||
break;
|
||||
|
||||
case DataSourceLocalDirectory localDirectory when RelocatedFilePaths.IsBelow(localDirectory.Path, oldPath):
|
||||
return null;
|
||||
|
||||
case DataSourceLocalDirectory localDirectory:
|
||||
var resolvedPath = RelocatedFilePaths.Resolve(localDirectory.Path, oldPath, record.RelativePath);
|
||||
if (resolvedPath is null)
|
||||
return null;
|
||||
targetPath = resolvedPath;
|
||||
break;
|
||||
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
|
||||
if (string.Equals(oldPath, targetPath, StringComparison.Ordinal)
|
||||
|| !string.Equals(Path.GetExtension(RelocatedFilePaths.GetStoredFileName(oldPath)), Path.GetExtension(targetPath), StringComparison.OrdinalIgnoreCase))
|
||||
return null;
|
||||
|
||||
var file = new FileInfo(targetPath);
|
||||
return file.Exists && file.Length == record.FileSize && file.LastWriteTimeUtc == record.LastWriteUtc.UtcDateTime ? file : null;
|
||||
}
|
||||
}
|
||||
@@ -251,7 +251,7 @@ internal sealed partial class FileSourceIndexer
|
||||
return true;
|
||||
|
||||
if (configuration.TargetFileName is { } targetFileName)
|
||||
return string.Equals(fileName, targetFileName, StringComparison.OrdinalIgnoreCase);
|
||||
return string.Equals(fileName, targetFileName, PathTools.COMPARISON);
|
||||
|
||||
//
|
||||
// Git renames .git/index.lock to .git/index with almost every command, and a build or
|
||||
@@ -297,13 +297,13 @@ internal sealed partial class FileSourceIndexer
|
||||
|
||||
private static DataSourceWatcherConfiguration? GetWatchConfiguration(IDataSourceBase dataSource) => dataSource switch
|
||||
{
|
||||
DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path) => new DataSourceWatcherConfiguration(
|
||||
DataSourceLocalDirectory localDirectory when RelocatedFilePaths.IsLocalAbsolute(localDirectory.Path) && Directory.Exists(localDirectory.Path) => new DataSourceWatcherConfiguration(
|
||||
localDirectory.Path,
|
||||
localDirectory.IncludeSubdirectories,
|
||||
localDirectory.IncludeHiddenFilesAndFolders,
|
||||
FilePatterns: FileNamePatterns.Parse(localDirectory.EffectiveFilePatterns),
|
||||
ExcludePatterns: FileExclusions.Parse(localDirectory.EffectiveExcludePatterns)),
|
||||
DataSourceLocalFile localFile when File.Exists(localFile.FilePath) && !string.IsNullOrWhiteSpace(Path.GetDirectoryName(localFile.FilePath)) => new DataSourceWatcherConfiguration(
|
||||
DataSourceLocalFile localFile when RelocatedFilePaths.IsLocalAbsolute(localFile.FilePath) && File.Exists(localFile.FilePath) && !string.IsNullOrWhiteSpace(Path.GetDirectoryName(localFile.FilePath)) => new DataSourceWatcherConfiguration(
|
||||
Path.GetDirectoryName(localFile.FilePath)!,
|
||||
false,
|
||||
TargetFileName: Path.GetFileName(localFile.FilePath)),
|
||||
@@ -314,8 +314,8 @@ internal sealed partial class FileSourceIndexer
|
||||
{
|
||||
return left.IncludeSubdirectories == right.IncludeSubdirectories
|
||||
&& left.IncludeHiddenFilesAndFolders == right.IncludeHiddenFilesAndFolders
|
||||
&& string.Equals(left.RootPath, right.RootPath, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(left.TargetFileName, right.TargetFileName, StringComparison.OrdinalIgnoreCase)
|
||||
&& string.Equals(left.RootPath, right.RootPath, PathTools.COMPARISON)
|
||||
&& string.Equals(left.TargetFileName, right.TargetFileName, PathTools.COMPARISON)
|
||||
&& FileNamePatterns.AreEquivalent(left.FilePatterns, right.FilePatterns)
|
||||
&& FileNamePatterns.AreEquivalent(left.ExcludePatterns, right.ExcludePatterns);
|
||||
}
|
||||
|
||||
@@ -77,12 +77,25 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
|
||||
var manifest = context.Manifest;
|
||||
|
||||
if (!await this.RelocateStoredFilesAsync(context, dataSource, token))
|
||||
return await DocumentRunProgress.FromStoredState(context, manifest.Files.Count, logger).PauseRunAsync(null, "file relocation paused", token);
|
||||
|
||||
var enumerationStartedUtc = DateTimeOffset.UtcNow;
|
||||
var inputFiles = this.GetInputFiles(dataSource, manifest);
|
||||
var enumerationDuration = DateTimeOffset.UtcNow - enumerationStartedUtc;
|
||||
if (inputFiles.UnreachableRootReason is { } unreachableRootReason)
|
||||
return this.StopWithoutRoot(context, dataSource, unreachableRootReason);
|
||||
|
||||
// An unmounted volume can leave a readable, empty mount directory. There is no way to
|
||||
// distinguish that from intentionally removing every file, so keep the stored index.
|
||||
// Explicit exclusions still take effect: they are evidence of a nonempty folder.
|
||||
if (dataSource is DataSourceLocalDirectory directory
|
||||
&& (manifest.Files.Count > 0 || manifest.UnfinishedFiles.Count > 0 || manifest.PermanentFailures.Count > 0)
|
||||
&& inputFiles.Files.Count == 0 && inputFiles.IgnoredFiles.Count == 0
|
||||
&& inputFiles.FilePatternExclusionCount == 0 && inputFiles.ExcludedFolderCount == 0
|
||||
&& inputFiles.FailedFiles == 0 && inputFiles.UnreadableDirectories.Count == 0)
|
||||
return this.StopWithoutRoot(context, dataSource, string.Format(TB("The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source."), directory.Path));
|
||||
|
||||
var indexedFiles = inputFiles.Files;
|
||||
var totalFiles = indexedFiles.Count + inputFiles.FailedFiles + inputFiles.IgnoredFiles.Count;
|
||||
|
||||
@@ -364,7 +377,8 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
/// connected, or a network share while the VPN is down, would remove every embedding of the
|
||||
/// data source, and embedding it all again once the folder is back costs as long as it took the
|
||||
/// first time. Nothing is written: the stored hash still tells which state of the data source
|
||||
/// the index holds, so the next run picks up from there.
|
||||
/// the index holds, so the next run picks up from there. The status asks for attention, so the
|
||||
/// embeddings page offers to edit the data source, e.g. to select the new location of a moved folder.
|
||||
/// </remarks>
|
||||
/// <param name="context">The run.</param>
|
||||
/// <param name="dataSource">The data source.</param>
|
||||
@@ -378,7 +392,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
dataSource.Id,
|
||||
context.Manifest.Files.Count);
|
||||
|
||||
DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger).PublishRunFailure(reason);
|
||||
DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger).PublishRunFailure(reason, DataSourceAttention.LOCATION_UNREACHABLE);
|
||||
return IndexedRunOutcome.DONE;
|
||||
}
|
||||
|
||||
@@ -509,9 +523,10 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
/// user may not read that folder. Removing it would throw away its index, and embedding it again
|
||||
/// once the folder can be read costs as long as it took the first time.
|
||||
/// <br /><br />
|
||||
/// Paths are compared ignoring their case, as everywhere in the manifest. On Linux and macOS, that
|
||||
/// keeps a file whose path differs from an unreadable folder in its case alone, which costs no more
|
||||
/// than a stale entry until the folder can be read again.
|
||||
/// Paths are compared ignoring their case, as everywhere in the index, cf. the remarks on
|
||||
/// DataSourceEmbeddingManifest.PATH_COMPARER. On Linux and macOS, that keeps a file whose path
|
||||
/// differs from an unreadable folder in its case alone, which costs no more than a stale entry
|
||||
/// until the folder can be read again.
|
||||
/// </remarks>
|
||||
/// <param name="storedPaths">The paths of the files the index knows about.</param>
|
||||
/// <param name="files">The files this run found.</param>
|
||||
@@ -521,15 +536,16 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
{
|
||||
var existingPaths = files
|
||||
.Select(file => file.FullName)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
.ToHashSet(DataSourceEmbeddingManifest.PATH_COMPARER);
|
||||
|
||||
var unreadablePrefixes = unreadableDirectories
|
||||
.Select(directory => Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)) + Path.DirectorySeparatorChar)
|
||||
.Select(Path.GetFullPath)
|
||||
.Select(directory => Path.EndsInDirectorySeparator(directory) ? directory : directory + Path.DirectorySeparatorChar)
|
||||
.ToList();
|
||||
|
||||
return storedPaths
|
||||
.Where(path => !existingPaths.Contains(path))
|
||||
.Where(path => !unreadablePrefixes.Any(prefix => path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
|
||||
.Where(path => !unreadablePrefixes.Any(prefix => path.StartsWith(prefix, DataSourceEmbeddingManifest.PATH_COMPARISON)))
|
||||
.ToList();
|
||||
}
|
||||
|
||||
@@ -709,7 +725,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
|
||||
switch (dataSource)
|
||||
{
|
||||
case DataSourceLocalFile localFile when File.Exists(localFile.FilePath):
|
||||
case DataSourceLocalFile localFile when RelocatedFilePaths.IsLocalAbsolute(localFile.FilePath) && File.Exists(localFile.FilePath):
|
||||
var file = new FileInfo(localFile.FilePath);
|
||||
|
||||
// No size limit: the user chose this very file, cf. FolderFileSelection.IsTooLarge.
|
||||
@@ -718,9 +734,10 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
|
||||
return result;
|
||||
|
||||
case DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path):
|
||||
case DataSourceLocalDirectory localDirectory when RelocatedFilePaths.IsLocalAbsolute(localDirectory.Path) && Directory.Exists(localDirectory.Path):
|
||||
this.EnumerateAccessibleFiles(localDirectory, manifest, mediaTranscriptionAvailability, result);
|
||||
if (result.UnreadableDirectories.Contains(localDirectory.Path))
|
||||
if (result.UnreadableDirectories.Any(directory => string.Equals(Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)),
|
||||
Path.TrimEndingDirectorySeparator(Path.GetFullPath(localDirectory.Path)), PathTools.COMPARISON)))
|
||||
result.UnreachableRootReason = string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."), localDirectory.Path);
|
||||
|
||||
return result;
|
||||
@@ -732,11 +749,11 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
switch (dataSource)
|
||||
{
|
||||
case DataSourceLocalFile localFile:
|
||||
result.UnreachableRootReason = string.Format(TB("The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source."), localFile.FilePath);
|
||||
result.UnreachableRootReason = string.Format(TB("The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source."), localFile.FilePath);
|
||||
break;
|
||||
|
||||
case DataSourceLocalDirectory localDirectory:
|
||||
result.UnreachableRootReason = string.Format(TB("The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source."), localDirectory.Path);
|
||||
result.UnreachableRootReason = string.Format(TB("The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source."), localDirectory.Path);
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -935,8 +952,8 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList<FileInfo> indexedFiles)
|
||||
{
|
||||
var fileHashes = indexedFiles
|
||||
.OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase)
|
||||
.ToDictionary(file => file.FullName, BuildFileMetadataHash, StringComparer.OrdinalIgnoreCase);
|
||||
.OrderBy(file => file.FullName, DataSourceEmbeddingManifest.PATH_COMPARER)
|
||||
.ToDictionary(file => file.FullName, BuildFileMetadataHash, DataSourceEmbeddingManifest.PATH_COMPARER);
|
||||
|
||||
var sourceHash = dataSource switch
|
||||
{
|
||||
@@ -980,7 +997,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
indexedFiles.Count.ToString()
|
||||
};
|
||||
|
||||
foreach (var file in indexedFiles.OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase))
|
||||
foreach (var file in indexedFiles.OrderBy(file => file.FullName, DataSourceEmbeddingManifest.PATH_COMPARER))
|
||||
{
|
||||
parts.Add(this.TryGetRelativePath(dataSource, file));
|
||||
parts.Add(fileHashes[file.FullName]);
|
||||
@@ -1002,14 +1019,24 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
"0");
|
||||
}
|
||||
|
||||
return BuildMetadataHash(
|
||||
"file",
|
||||
file.FullName,
|
||||
file.Name,
|
||||
file.LastWriteTimeUtc.Ticks.ToString(),
|
||||
file.Length.ToString());
|
||||
return BuildFileMetadataHash(file.FullName, file.Name, file.LastWriteTimeUtc, file.Length);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The fingerprint of a file which exists, from its write time and size as they were read.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The one formula for both an indexed file and a relocated one: if the two drifted apart, every
|
||||
/// relocated file would look changed to the next run and be embedded again.
|
||||
/// </remarks>
|
||||
/// <param name="fullName">The absolute path of the file.</param>
|
||||
/// <param name="name">The name of the file.</param>
|
||||
/// <param name="lastWriteUtc">When the file was last written to.</param>
|
||||
/// <param name="length">The size of the file in bytes.</param>
|
||||
/// <returns>The fingerprint.</returns>
|
||||
private static string BuildFileMetadataHash(string fullName, string name, DateTime lastWriteUtc, long length) =>
|
||||
BuildMetadataHash("file", fullName, name, lastWriteUtc.Ticks.ToString(), length.ToString());
|
||||
|
||||
private static string BuildMetadataHash(params string[] parts)
|
||||
{
|
||||
return BuildMetadataHash((IEnumerable<string>)parts);
|
||||
@@ -1036,7 +1063,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
|
||||
/// <returns>The document.</returns>
|
||||
private EmbeddingDocument CreateFileDocument(IndexedRunContext context, IDataSource dataSource, FileInfo file, string fingerprint, MediaTranscriptSlot? transcript) => new(
|
||||
file.FullName,
|
||||
this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow) with { TranscriptKey = transcript?.Key },
|
||||
this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow) with { ParentFileId = context.GetFileParentId(file.FullName), TranscriptKey = transcript?.Key },
|
||||
file.Name,
|
||||
token => this.StreamEmbeddingChunksAsync(file.FullName, transcript, dataSource, context.EmbeddingProvider, context.PromptInjectionScope, token));
|
||||
|
||||
|
||||
@@ -7,9 +7,11 @@ namespace AIStudio.Tools.Services.Indexing;
|
||||
/// The ids under which documents and their chunks are stored.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Derived rather than random, so the same document of the same data source always ends up under
|
||||
/// the same ids, in the vector store as well as in the index store. The formats must stay as they
|
||||
/// are: every stored id was made with them.
|
||||
/// Derived rather than random, so the same document of the same data source ends up under the same
|
||||
/// ids, in the vector store as well as in the index store. Once stored, they are the identity of the
|
||||
/// document: a file which moves to another folder keeps its ids, although they were derived from its
|
||||
/// old path and its old fingerprint, cf. FileSourceIndexer.RelocateStoredFilesAsync. The formats must
|
||||
/// stay as they are: every stored id was made with them.
|
||||
/// </remarks>
|
||||
internal static class IndexedDocumentIds
|
||||
{
|
||||
@@ -17,11 +19,31 @@ internal static class IndexedDocumentIds
|
||||
/// The id of one chunk, which is also the id of its point in the vector store.
|
||||
/// </summary>
|
||||
/// <param name="dataSourceId">The data source the document belongs to.</param>
|
||||
/// <param name="fingerprint">The fingerprint of the document as it was read.</param>
|
||||
/// <param name="source">What the chunk ids of the document derive from, cf. CreateChunkIdSource.</param>
|
||||
/// <param name="chunkIndex">The position of the chunk within the document.</param>
|
||||
/// <returns>The chunk id.</returns>
|
||||
public static string CreateChunkId(string dataSourceId, string fingerprint, int chunkIndex) =>
|
||||
CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
|
||||
public static string CreateChunkId(string dataSourceId, string source, int chunkIndex) =>
|
||||
CreateStableGuid($"{dataSourceId}:chunk:{source}:{chunkIndex}");
|
||||
|
||||
/// <summary>
|
||||
/// What the chunk ids of a document derive from: nearly always its fingerprint.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A moved file keeps its parent id and its chunk ids, but its fingerprint now names the new path.
|
||||
/// Another file may later appear at the old path with exactly the fingerprint those chunk ids came
|
||||
/// from, and the fingerprint alone would hand it the same ids. The two documents would then take
|
||||
/// over each other's chunks. Hence the parent id joins whenever a document is not stored under the
|
||||
/// id its key gives, which only happens after a move. Every other document keeps the ids it has.
|
||||
/// </remarks>
|
||||
/// <param name="dataSourceId">The data source the document belongs to.</param>
|
||||
/// <param name="documentKey">The key of the document.</param>
|
||||
/// <param name="parentId">The id the document is stored under.</param>
|
||||
/// <param name="fingerprint">The fingerprint of the document as it is about to be read.</param>
|
||||
/// <returns>What to pass to CreateChunkId.</returns>
|
||||
public static string CreateChunkIdSource(string dataSourceId, string documentKey, string parentId, string fingerprint) =>
|
||||
string.Equals(parentId, CreateParentId(dataSourceId, documentKey), StringComparison.Ordinal)
|
||||
? fingerprint
|
||||
: $"{parentId}:{fingerprint}";
|
||||
|
||||
/// <summary>
|
||||
/// The id of a document's row in the index store, which its chunks point at.
|
||||
|
||||
@@ -41,6 +41,9 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
|
||||
private long storedChunksSinceLastOptimization;
|
||||
private bool hasPendingChanges;
|
||||
|
||||
private Dictionary<string, string>? fileParentIds;
|
||||
private HashSet<string>? reservedFileParentIds;
|
||||
|
||||
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(IndexedRunContext).Namespace, nameof(IndexedRunContext));
|
||||
|
||||
/// <summary>
|
||||
@@ -66,6 +69,38 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
|
||||
|
||||
public IndexingPass Pass => pass;
|
||||
|
||||
/// <summary>
|
||||
/// The id a file is stored under: the one it already has, or a new one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A relocated file keeps the id it got when it was indexed first, which came from its path back
|
||||
/// then. A new file may now appear at that very path, so the ids already stored are reserved, and
|
||||
/// a new file whose derived id is taken gets a random one instead. Its chunk ids include that id,
|
||||
/// cf. IndexedDocumentIds.CreateChunkIdSource.
|
||||
/// </remarks>
|
||||
/// <param name="documentKey">The key of the file, which is its absolute path.</param>
|
||||
/// <returns>The parent id of the file.</returns>
|
||||
public string GetFileParentId(string documentKey)
|
||||
{
|
||||
if (this.fileParentIds is null)
|
||||
{
|
||||
this.fileParentIds = manifest.Files
|
||||
.Where(file => file.Value.ParentFileId is not null)
|
||||
.ToDictionary(file => file.Key, file => file.Value.ParentFileId!, DataSourceEmbeddingManifest.PATH_COMPARER);
|
||||
this.reservedFileParentIds = this.fileParentIds.Values.ToHashSet(StringComparer.Ordinal);
|
||||
}
|
||||
|
||||
if (this.fileParentIds.TryGetValue(documentKey, out var storedId))
|
||||
return storedId;
|
||||
|
||||
var parentId = IndexedDocumentIds.CreateParentId(dataSource.Id, documentKey);
|
||||
while (!this.reservedFileParentIds!.Add(parentId))
|
||||
parentId = Guid.NewGuid().ToString();
|
||||
|
||||
this.fileParentIds[documentKey] = parentId;
|
||||
return parentId;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The run as one action of the prompt injection filter.
|
||||
/// </summary>
|
||||
@@ -124,10 +159,11 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
|
||||
var embeddingBatchSize = Math.Max(1, embeddingProvider.EffectiveEmbeddingBatchSize);
|
||||
var batch = new List<EmbeddingChunkDraft>(embeddingBatchSize);
|
||||
var totalChunkCount = 0;
|
||||
var chunkIdSource = IndexedDocumentIds.CreateChunkIdSource(dataSource.Id, document.Key, document.State.ParentFileId, document.State.Fingerprint);
|
||||
|
||||
await foreach (var chunk in document.StreamChunks(token))
|
||||
{
|
||||
batch.Add(new(IndexedDocumentIds.CreateChunkId(dataSource.Id, document.State.Fingerprint, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber));
|
||||
batch.Add(new(IndexedDocumentIds.CreateChunkId(dataSource.Id, chunkIdSource, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber));
|
||||
totalChunkCount++;
|
||||
reportBlockProgress(totalChunkCount, chunk.PageNumber);
|
||||
|
||||
@@ -163,7 +199,10 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
|
||||
/// <param name="token">The cancellation token.</param>
|
||||
public async Task DeleteDocumentPointsAsync(string documentKey, CancellationToken token)
|
||||
{
|
||||
await vectorStore.DeleteEmbeddingByFile(this.CollectionName, documentKey, token);
|
||||
// A relocated document keeps its original parent id. Delete by that identity as well as
|
||||
// by path, including when a previous relocation stopped between its two database writes.
|
||||
var parentId = manifest.Files.GetValueOrDefault(documentKey)?.ParentFileId ?? string.Empty;
|
||||
await vectorStore.DeleteEmbeddingByFile(this.CollectionName, documentKey, parentId, token);
|
||||
this.hasPendingChanges = true;
|
||||
}
|
||||
|
||||
@@ -401,4 +440,4 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
|
||||
|
||||
batch.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -44,7 +44,7 @@ internal sealed class IndexingPass(DataSourceRunSignature runSignature)
|
||||
/// <summary>
|
||||
/// The documents which failed during this pass for a reason which may pass, by their keys.
|
||||
/// </summary>
|
||||
private readonly Dictionary<string, FailedDocument> failedDocuments = new(StringComparer.OrdinalIgnoreCase);
|
||||
private readonly Dictionary<string, FailedDocument> failedDocuments = new(DataSourceEmbeddingManifest.PATH_COMPARER);
|
||||
|
||||
//
|
||||
// Which kinds of provider failure the user was already told about in this pass. A rejected API
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
namespace AIStudio.Tools.Services.Indexing;
|
||||
|
||||
/// <summary>
|
||||
/// Reads paths stored by the computer which created an index. Its path syntax may differ from
|
||||
/// that of the computer restoring the backup; never resolve an old absolute path on this machine.
|
||||
/// </summary>
|
||||
internal static class RelocatedFilePaths
|
||||
{
|
||||
internal static bool IsWindowsAbsolute(string path) => path.StartsWith(@"\\", StringComparison.Ordinal)
|
||||
|| (path.Length >= 3 && char.IsAsciiLetter(path[0]) && path[1] == ':' && path[2] is '\\' or '/');
|
||||
|
||||
internal static bool IsLocalAbsolute(string path) => !string.IsNullOrWhiteSpace(path) && Path.IsPathFullyQualified(path)
|
||||
&& (OperatingSystem.IsWindows() || !IsWindowsAbsolute(path));
|
||||
|
||||
internal static string GetStoredFileName(string path)
|
||||
{
|
||||
var separator = IsWindowsAbsolute(path) ? Math.Max(path.LastIndexOf('/'), path.LastIndexOf('\\')) : path.LastIndexOf('/');
|
||||
return path[(separator + 1)..];
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Whether a stored path still lies below a root, spelled exactly as the root is now.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Compared ordinally, unlike PathTools.IsInsideDirectory: once the name of a folder changed in its
|
||||
/// case alone, its stored paths no longer match the paths a run finds, on Windows as well.
|
||||
/// </remarks>
|
||||
internal static bool IsBelow(string root, string storedPath)
|
||||
{
|
||||
if (!IsLocalAbsolute(root))
|
||||
return false;
|
||||
|
||||
try
|
||||
{
|
||||
var prefix = Path.GetFullPath(root);
|
||||
if (!Path.EndsInDirectorySeparator(prefix))
|
||||
prefix += Path.DirectorySeparatorChar;
|
||||
|
||||
return storedPath.StartsWith(prefix, StringComparison.Ordinal);
|
||||
}
|
||||
catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Maps a stored relative path below a newly selected root, including drive roots and mounted
|
||||
/// volumes. Backslashes are separators only for a Windows index: they are legal Unix file names.
|
||||
/// </summary>
|
||||
internal static string? Resolve(string root, string oldAbsolutePath, string? relativePath)
|
||||
{
|
||||
if (!IsLocalAbsolute(root) || string.IsNullOrWhiteSpace(relativePath))
|
||||
return null;
|
||||
|
||||
var windowsSource = IsWindowsAbsolute(oldAbsolutePath);
|
||||
if (!windowsSource && !oldAbsolutePath.StartsWith('/'))
|
||||
return null;
|
||||
|
||||
var parts = relativePath.Split(windowsSource ? ['/', '\\'] : ['/']);
|
||||
foreach (var part in parts)
|
||||
{
|
||||
// Check after decoding the source separators, so a Windows '..\\' cannot evade the
|
||||
// containment check on Unix. Reject target-invalid names rather than changing them.
|
||||
if (part is "" or "." or ".." || part.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0)
|
||||
return null;
|
||||
|
||||
if (windowsSource && part.Contains(':'))
|
||||
return null;
|
||||
|
||||
if (OperatingSystem.IsWindows() && (part.EndsWith('.') || part.EndsWith(' ')))
|
||||
return null;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
var target = Path.GetFullPath(Path.Combine(root, Path.Combine(parts)));
|
||||
return PathTools.IsInsideDirectory(root, target) ? target : null;
|
||||
}
|
||||
catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -13,9 +13,10 @@ public sealed partial class RustService
|
||||
async () =>
|
||||
{
|
||||
var encodedTitle = Uri.EscapeDataString(title);
|
||||
var result = initialDirectory is null
|
||||
var startDirectory = initialDirectory is null ? null : PathTools.FindNearestExistingPath(initialDirectory);
|
||||
var result = startDirectory is null
|
||||
? await this.http.PostAsync($"/select/directory?title={encodedTitle}", null)
|
||||
: await this.http.PostAsJsonAsync($"/select/directory?title={encodedTitle}", new PreviousDirectory(initialDirectory), this.jsonRustSerializerOptions);
|
||||
: await this.http.PostAsJsonAsync($"/select/directory?title={encodedTitle}", new PreviousDirectory(startDirectory), this.jsonRustSerializerOptions);
|
||||
|
||||
if (result.IsSuccessStatusCode)
|
||||
return await result.Content.ReadFromJsonAsync<DirectorySelectionResponse>(this.jsonRustSerializerOptions);
|
||||
@@ -54,10 +55,11 @@ public sealed partial class RustService
|
||||
"select file",
|
||||
async () =>
|
||||
{
|
||||
var startPath = initialFile is null ? null : PathTools.FindNearestExistingPath(initialFile);
|
||||
var payload = new SelectFileOptions
|
||||
{
|
||||
Title = title,
|
||||
PreviousFile = initialFile is null ? null : new (initialFile),
|
||||
PreviousFile = startPath is null ? null : new (startPath),
|
||||
Filter = FileTypes.ToFileDialogFilter(filter)
|
||||
};
|
||||
|
||||
@@ -77,10 +79,11 @@ public sealed partial class RustService
|
||||
"select files",
|
||||
async () =>
|
||||
{
|
||||
var startPath = initialFile is null ? null : PathTools.FindNearestExistingPath(initialFile);
|
||||
var payload = new SelectFileOptions
|
||||
{
|
||||
Title = title,
|
||||
PreviousFile = initialFile is null ? null : new (initialFile),
|
||||
PreviousFile = startPath is null ? null : new (startPath),
|
||||
Filter = FileTypes.ToFileDialogFilter(filter)
|
||||
};
|
||||
|
||||
|
||||
@@ -38,8 +38,10 @@
|
||||
- Improved local data sources for folders. You can now choose which files AI Studio indexes, for example, only PDF files, and whether it includes subfolders as well as hidden files and folders. By default, AI Studio now also indexes text files it did not recognize before, such as program code of any kind or a LICENSE file, so a Git repository makes a useful data source. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution.
|
||||
- Improved local data sources for folders, which now leave out hidden files and folders as well as build output and downloaded dependencies, such as the folders `node_modules`, `bin`, and `obj`. You can change this in the expert settings of the data source. When an existing data source contains such files, AI Studio removes them from its index during the next update.
|
||||
- Improved the background embeddings page, which now lists the files of a local data source that AI Studio left out, for example, images or files that are too large. It also shows which file types and folders lie outside what the data source selects.
|
||||
- Improved the Batch Processing assistant, which now also processes images in JPG, PNG, WebP, and GIF format when the selected model can read images. If you used the assistant before, your saved file patterns stay as they are: click "Restore default patterns" or add patterns such as `*.jpg;*.png` yourself.
|
||||
- Improved the Batch Processing assistant, which now leaves out hidden files and folders as well as build output and downloaded dependencies, such as the folder `node_modules`. You can change both in the expert settings of the assistant.
|
||||
- Improved local data sources for folders and the Batch Processing assistant, which now leave out files larger than 1 GB, such as images of virtual machines. You can change this limit.
|
||||
- Improved local data sources for files and folders. When you move or rename the file or folder, for example, to another drive or a new computer, you can now select its new location when you edit the data source. AI Studio keeps what it has already prepared for every unchanged file, so these files do not go to your embedding provider again. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution.
|
||||
- Improved the organization-wide management of the Batch Processing assistant. IT departments can now preset whether it includes hidden files and folders, which files and folders it leaves out, and how large a file may be, either fixed or as a default which users may change.
|
||||
- Improved the organization-wide management of the app settings. IT departments can now also set whether AI Studio saves energy while the AI writes its answers, which helps on virtual desktops, for example.
|
||||
- Improved attaching files to a chat. You can now attach any file that contains readable text, for example, program code or configuration files of a kind AI Studio did not know before.
|
||||
@@ -71,7 +73,7 @@
|
||||
- Fixed AI Studio continuing to update the index of a data source with the old settings after you, or the configuration of your organization, changed its embedding provider or the expert settings of the data source, which could take hours for nothing. AI Studio now stops right away and starts over with the new settings.
|
||||
- Fixed AI Studio continuing to send the content of a data source to an embedding provider after you deleted that provider, or after a confidence level changed so that the provider may no longer read the data source. AI Studio now stops right away.
|
||||
- Fixed data sources staying unavailable after the configuration of your organization changed their embedding provider, until one of their documents changed, or you restarted AI Studio. When AI Studio updates your data sources automatically, it now rebuilds their index right away.
|
||||
- Fixed AI Studio deleting the index of a local data source when it could not reach its folder or file, for example, because the drive was not connected. Once the folder was back, AI Studio indexed everything again, which could take hours. AI Studio now keeps the index and tells you that it cannot reach the folder. The same applies to subfolders you are not allowed to open.
|
||||
- Fixed AI Studio deleting the index of a local data source when it could not reach its folder or file, for example, because the drive was not connected. Once the folder was back, AI Studio indexed everything again, which could take hours. AI Studio now keeps the index and tells you that it cannot reach the folder. The same applies to subfolders you are not allowed to open and to a folder that suddenly appears empty, which can happen when a drive is not connected.
|
||||
- Fixed AI Studio rebuilding the index of a data source from scratch without asking you when part of the index was missing on your computer. Since this sends every document to your embedding provider once more, AI Studio now tells you about the missing index and offers to repair the data source.
|
||||
- Fixed saving a local data source whose embedding provider had been deleted, without choosing a new one. The dialog now asks you to select an embedding provider, so that AI Studio can prepare the data source again.
|
||||
- Fixed justified texts being hyphenated by the rules of English even when AI Studio shows another language, such as German. Screen readers now also know which language AI Studio uses.
|
||||
@@ -107,7 +109,9 @@
|
||||
- Fixed AI Studio not checking the transcripts of audio and video files for suspicious instructions (prompt injection) in the Batch Processing assistant and when you load such a file into a field of an assistant. AI Studio now checks these transcripts like all other documents before the AI sees them, even very long ones.
|
||||
- Fixed AI Studio deleting the API keys you entered yourself for the providers of your organization when your organization changed the secret that protects its configuration. These keys now stay, so you no longer need to enter them again.
|
||||
- Fixed sharing plugins when the share dialog of Windows or macOS does not open or stops responding. AI Studio now notices this and offers to save the plugin archive to a location of your choice instead. Until you restart AI Studio, the plugin page then offers this export instead of sharing.
|
||||
- Fixed the Batch Processing assistant finding no documents when you chose an output folder that contains the input folder, such as its parent folder.
|
||||
- Fixed the Batch Processing assistant processing its own result files again as new documents when you chose the input folder as the output folder.
|
||||
- Updated the code contributions on the supporters page, which now thank everyone who has contributed code to AI Studio so far.
|
||||
- Upgraded several libraries to improve security.
|
||||
- Upgraded to Rust v1.99.0
|
||||
- Upgraded to Rust v1.99.1
|
||||
- Upgraded to .NET 9.0.21
|
||||
@@ -0,0 +1,165 @@
|
||||
using AIStudio.Assistants.BatchProcessing;
|
||||
|
||||
namespace AIStudio.Tests.Assistants;
|
||||
|
||||
/// <summary>
|
||||
/// Checks which files the search of the batch processing assistant leaves out because a batch run
|
||||
/// wrote them.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A run writes everything directly into its output folder. The search comes across any of it only
|
||||
/// when that folder lies below the input folder or is the input folder itself. An output folder
|
||||
/// above the input folder once made the search leave out every document.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class BatchProcessingOutputFolderTests
|
||||
{
|
||||
private const string RESULTS_FILE_NAME = "batch-results.csv";
|
||||
|
||||
private static readonly string DESKTOP = Path.Combine(Path.GetTempPath(), "ai-studio-batch-output-folder", "Desktop");
|
||||
private static readonly string PROJECT_FOLDER = Path.Combine(DESKTOP, "Batch");
|
||||
private static readonly string INPUT_FOLDER = Path.Combine(PROJECT_FOLDER, "Documents");
|
||||
|
||||
[Test]
|
||||
public void AnOutputFolderAboveTheInputFolderLeavesOutNoDocument()
|
||||
{
|
||||
var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, PROJECT_FOLDER, RESULTS_FILE_NAME);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.pdf")), Is.False);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "notes", "minutes.md")), Is.False);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "log.csv")), Is.False, "The log of a run lies in the output folder, so a file of this name in the input folder is a document.");
|
||||
Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "notes")), Is.True);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnOutputFolderSeveralLevelsAboveTheInputFolderLeavesOutNoDocument()
|
||||
{
|
||||
var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, DESKTOP, RESULTS_FILE_NAME);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.pdf")), Is.False);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "notes", "minutes.md")), Is.False);
|
||||
Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "notes")), Is.True);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheDefaultOutputFolderBelowTheInputFolderIsLeftOut()
|
||||
{
|
||||
var resultsFolder = Path.Combine(INPUT_FOLDER, "ai-results");
|
||||
var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, resultsFolder, RESULTS_FILE_NAME);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(resultsFolder, "report_result.md")), Is.True);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.pdf")), Is.False);
|
||||
Assert.That(outputFolder.Enters(resultsFolder), Is.False);
|
||||
Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "notes")), Is.True);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnOutputFolderDeepBelowTheInputFolderIsLeftOutAlone()
|
||||
{
|
||||
var parentFolder = Path.Combine(INPUT_FOLDER, "2026", "october");
|
||||
var resultsFolder = Path.Combine(parentFolder, "results");
|
||||
var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, resultsFolder, RESULTS_FILE_NAME);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(resultsFolder, "report_result.md")), Is.True);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(parentFolder, "report.pdf")), Is.False);
|
||||
Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "2026")), Is.True);
|
||||
Assert.That(outputFolder.Enters(parentFolder), Is.True);
|
||||
Assert.That(outputFolder.Enters(resultsFolder), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ADocumentWhoseNameBeginsLikeTheOutputFolderIsNoOutput()
|
||||
{
|
||||
var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, Path.Combine(INPUT_FOLDER, "out"), RESULTS_FILE_NAME);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "output-notes.md")), Is.False);
|
||||
Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "output")), Is.True);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnOutputFolderNextToTheInputFolderWithTheSameBeginningLeavesOutNothing()
|
||||
{
|
||||
var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, $"{INPUT_FOLDER}-results", RESULTS_FILE_NAME);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.pdf")), Is.False);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "log.csv")), Is.False);
|
||||
Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "notes")), Is.True);
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public void TheInputFolderAsOutputFolderLeavesOutTheLogAndTheResultsTable(bool withTrailingSeparator)
|
||||
{
|
||||
var outputDirectory = withTrailingSeparator ? INPUT_FOLDER + Path.DirectorySeparatorChar : INPUT_FOLDER;
|
||||
var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, outputDirectory, RESULTS_FILE_NAME);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "log.csv")), Is.True);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, RESULTS_FILE_NAME)), Is.True);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.pdf")), Is.False);
|
||||
Assert.That(outputFolder.Enters(Path.Combine(INPUT_FOLDER, "notes")), Is.True, "Leaving out the output folder would leave out the input folder, too.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheInputFolderAsOutputFolderLeavesOutTheResultFilesOfEveryFormat()
|
||||
{
|
||||
var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, INPUT_FOLDER, RESULTS_FILE_NAME);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report_result.md")), Is.True);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report_result_2.docx")), Is.True, "A second result of the same name gets a counter.");
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "Report_Result.HTML")), Is.True);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.v2_result.md")), Is.True, "Only the last extension of the document makes way for the suffix.");
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheInputFolderAsOutputFolderKeepsDocumentsWithSimilarNames()
|
||||
{
|
||||
var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, INPUT_FOLDER, RESULTS_FILE_NAME);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report.md")), Is.False);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "result.md")), Is.False);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "my_results.md")), Is.False);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report_result_draft.md")), Is.False);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(INPUT_FOLDER, "report_result_.md")), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheInputFolderAsOutputFolderKeepsFilesInSubfolders()
|
||||
{
|
||||
var outputFolder = new BatchProcessingOutputFolder(INPUT_FOLDER, INPUT_FOLDER, RESULTS_FILE_NAME);
|
||||
var subfolder = Path.Combine(INPUT_FOLDER, "notes");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(subfolder, "log.csv")), Is.False, "A run writes into the output folder itself, never into one of its subfolders.");
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(subfolder, RESULTS_FILE_NAME)), Is.False);
|
||||
Assert.That(outputFolder.IsOutputOfRun(Path.Combine(subfolder, "report_result.md")), Is.False);
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
using AIStudio.Chat;
|
||||
using AIStudio.Provider;
|
||||
|
||||
namespace AIStudio.Tests.Provider;
|
||||
|
||||
/// <summary>
|
||||
/// Checks how large an image may be when we send it through a provider.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A provider states its limit or says nothing. Either way, TryAsBase64 skips every image above our
|
||||
/// own limit, so a larger limit of a provider must never reach the checks done before sending.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ImageSizeLimitTests
|
||||
{
|
||||
[Test]
|
||||
public void AnthropicLeavesRoomForTheBase64OfTheImage()
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(LLMProviders.ANTHROPIC.GetMaxImageSizeBytes(), Is.EqualTo(3_932_160));
|
||||
Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(LLMProviders.ANTHROPIC), Is.EqualTo(3_932_160));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AProviderWithoutAStatedLimitGetsOurOwn()
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(LLMProviders.OPEN_AI.GetMaxImageSizeBytes(), Is.Null, "Nobody stated a limit, and no number stands in for that.");
|
||||
Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(LLMProviders.OPEN_AI), Is.EqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void NoProviderGetsMoreThanTryAsBase64Sends()
|
||||
{
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
foreach (var provider in Enum.GetValues<LLMProviders>())
|
||||
Assert.That(IImageSourceExtensions.MaxImageSizeBytesFor(provider), Is.LessThanOrEqualTo(IImageSourceExtensions.MAX_IMAGE_SIZE_BYTES), provider.ToString());
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.Rust;
|
||||
|
||||
namespace AIStudio.Tests.Settings;
|
||||
|
||||
public class DataBatchProcessingTests
|
||||
{
|
||||
/// <summary>
|
||||
/// The default patterns are a constant and cannot be built from FileTypes, so this keeps both
|
||||
/// lists of the images we hand to the model in step.
|
||||
/// </summary>
|
||||
[Test]
|
||||
public void DefaultPatternsHoldEveryImageTheModelCanRead()
|
||||
{
|
||||
var patterns = DataBatchProcessing.DEFAULT_FILE_PATTERNS.Split(';');
|
||||
var imagePatterns = FileTypes.MODEL_IMAGE.FlattenExtensions().Select(extension => $"*.{extension}");
|
||||
|
||||
Assert.That(patterns, Is.SupersetOf(imagePatterns));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
|
||||
namespace AIStudio.Tests.Settings;
|
||||
|
||||
/// <summary>
|
||||
/// Checks whether a configured model counts as able to read images.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Attaching a file to a chat, building a request, and the Batch Processing assistant all ask this one
|
||||
/// question. A limit of zero images is a statement about the installation, so it has to win over the
|
||||
/// capability, while a limit nobody stated must not take anything away.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class ImageInputSupportTests
|
||||
{
|
||||
private const string MACHINE = "55555555-5555-5555-5555-555555555555";
|
||||
|
||||
/// <summary>
|
||||
/// A model the rules know as reading text only, so that every image capability comes from the overrides.
|
||||
/// </summary>
|
||||
private static readonly Model MODEL = new("llama3.3:70b", null);
|
||||
|
||||
[Test]
|
||||
public void AModelWithoutImageCapabilityCannotReadImages()
|
||||
{
|
||||
Assert.That(ProviderWith(null).SupportsImageInput(), Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnImageCapabilityWithoutAStatedLimitReadsImages()
|
||||
{
|
||||
Assert.That(ProviderWith(new() { MultipleImageInput = true }).SupportsImageInput(), Is.True, "Nobody stated a limit, and an unknown limit takes nothing away.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ALimitOfOneImageStillReadsImages()
|
||||
{
|
||||
Assert.That(ProviderWith(new() { MultipleImageInput = true, MaxImagesPerMessage = 1 }).SupportsImageInput(), Is.True);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ALimitOfZeroImagesPerMessageTakesTheCapabilityBack()
|
||||
{
|
||||
Assert.That(ProviderWith(new() { MultipleImageInput = true, MaxImagesPerMessage = 0 }).SupportsImageInput(), Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ALimitOfZeroImagesPerRequestTakesTheCapabilityBack()
|
||||
{
|
||||
Assert.That(ProviderWith(new() { MultipleImageInput = true, MaxImagesPerRequest = 0 }).SupportsImageInput(), Is.False, "A message cannot carry more images than its whole request.");
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// A configured self-hosted provider, the way the settings hold one.
|
||||
/// </summary>
|
||||
/// <param name="overrides">What the person switched, or nothing when they switched nothing.</param>
|
||||
/// <returns>The configured provider.</returns>
|
||||
private static AIStudio.Settings.Provider ProviderWith(ProviderCapabilityOverrides? overrides) => new(1, MACHINE, "A machine of my own", LLMProviders.SELF_HOSTED, MODEL, IsSelfHosted: true)
|
||||
{
|
||||
CapabilityOverrides = overrides,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,752 @@
|
||||
using System.Reflection;
|
||||
using System.Security.Cryptography;
|
||||
using System.Text;
|
||||
using System.Text.Json;
|
||||
|
||||
using AIStudio.Components;
|
||||
using AIStudio.Dialogs;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tests.Tools.Databases;
|
||||
using AIStudio.Tools;
|
||||
using AIStudio.Tools.Databases;
|
||||
using AIStudio.Tools.Databases.IndexStore;
|
||||
using AIStudio.Tools.DataBackup;
|
||||
using AIStudio.Tools.Security;
|
||||
using AIStudio.Tools.Services;
|
||||
using AIStudio.Tools.Services.Indexing;
|
||||
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
using MudBlazor;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Runs the actual edit, queue, cancellation, manifest preparation, and file indexing code against
|
||||
/// a migrated SQLite store. Only the vector store and dialog rendering are replaced.
|
||||
/// </summary>
|
||||
[NonParallelizable]
|
||||
public sealed class DataSourceRelocationWorkflowTests
|
||||
{
|
||||
private const string EMBEDDING_ID = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01";
|
||||
|
||||
// Frozen from the format before relocation support. Do not generate this with the current
|
||||
// signature builder: doing that would hide an accidental rebuild of every existing index.
|
||||
private const string LEGACY_SIGNATURE = "2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100";
|
||||
|
||||
private static readonly TimeSpan TEST_TIMEOUT = TimeSpan.FromSeconds(10);
|
||||
|
||||
private string testRoot = string.Empty;
|
||||
private TemporaryIndexStore store = null!;
|
||||
private SettingsManager settings = null!;
|
||||
private DatabaseClientProvider databases = null!;
|
||||
private DataSourceEmbeddingService service = null!;
|
||||
private RelocationVectorStore vectors = null!;
|
||||
private IInternalDataSource source = null!;
|
||||
private readonly List<string> chunkIds = [];
|
||||
private readonly List<Task> runningTasks = [];
|
||||
private CancellationTokenSource lifetime = null!;
|
||||
|
||||
private string OriginalFolder => Path.Combine(this.testRoot, "original");
|
||||
|
||||
[SetUp]
|
||||
public async Task SetUp()
|
||||
{
|
||||
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-relocation-workflow-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(this.testRoot);
|
||||
this.store = await TemporaryIndexStore.CreateAsync();
|
||||
this.settings = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
|
||||
this.settings.ConfigurationData.App.EnabledPreviewFeatures.Add(PreviewFeatures.PRE_RAG_2024);
|
||||
this.settings.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh = false;
|
||||
this.settings.ConfigurationData.EmbeddingProviders.Add(new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small")));
|
||||
this.vectors = new RelocationVectorStore();
|
||||
this.databases = new DatabaseClientProvider(null!, NullLoggerFactory.Instance);
|
||||
var clients = Field<Dictionary<DatabaseRole, DatabaseClient>>(this.databases, "clients");
|
||||
clients.Add(DatabaseRole.INDEX_STORE, this.store.Client);
|
||||
clients.Add(DatabaseRole.VECTOR_STORE, this.vectors);
|
||||
var guard = new PromptInjectionGuardService(null!, this.settings, NullLogger<PromptInjectionGuardService>.Instance, NullLoggerFactory.Instance);
|
||||
this.service = new DataSourceEmbeddingService(this.settings, null!, this.databases, guard, null!, NullLogger<DataSourceEmbeddingService>.Instance);
|
||||
this.lifetime = new CancellationTokenSource();
|
||||
this.chunkIds.Clear();
|
||||
this.runningTasks.Clear();
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public async Task TearDown()
|
||||
{
|
||||
this.lifetime.Cancel();
|
||||
try
|
||||
{
|
||||
await Task.WhenAll(this.runningTasks).WaitAsync(TEST_TIMEOUT);
|
||||
}
|
||||
catch (OperationCanceledException) when (this.lifetime.IsCancellationRequested)
|
||||
{
|
||||
// A failed test may leave its deliberately blocked run for teardown to stop.
|
||||
}
|
||||
finally
|
||||
{
|
||||
this.service.Dispose();
|
||||
this.databases.Dispose();
|
||||
await this.store.DisposeAsync();
|
||||
this.lifetime.Dispose();
|
||||
Directory.Delete(this.testRoot, true);
|
||||
}
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task AnExistingIndexSurvivesTheUpgradeWithoutReembedding(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty, "An upgrade is not a relocation.");
|
||||
Assert.That(this.vectors.DeletedStores, Is.Zero);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task EditingAndCancelingDoNothingAndSavingOnlyRelocatesUnchangedFiles(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var newPath = this.MoveSource("moved");
|
||||
var original = this.source;
|
||||
var edit = this.OpenEdit(newPath);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False);
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
|
||||
Assert.That(edit.Warning.ShowCount, Is.Zero, "Typing a path must not open the save confirmation.");
|
||||
});
|
||||
|
||||
edit.Dialog.Instance.Cancel();
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False);
|
||||
|
||||
await this.SaveEditAsync(newPath);
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty, "Saving queues the operation; the dialog must not write to either index.");
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
|
||||
Assert.That(this.vectors.DeletedStores, Is.Zero);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
});
|
||||
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task SavingAnUnchangedPathDoesNotWarnOrRelocate(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var currentPath = this.source is DataSourceLocalDirectory folder ? folder.Path : ((DataSourceLocalFile)this.source).FilePath;
|
||||
var edit = this.OpenEdit(currentPath);
|
||||
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.Zero);
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
|
||||
Assert.That(this.vectors.DeletedStores, Is.Zero);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task SavingWaitsForThePathWarningBeforeChangingConfigurationOrQueueingWork(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var original = this.source;
|
||||
var edit = this.OpenEdit(this.MoveSource("moved"));
|
||||
var save = InvokeTask(edit.Component, "StoreValidatedDataSourceAsync");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
|
||||
Assert.That(save.IsCompleted, Is.False);
|
||||
Assert.That(edit.ParentTask.IsCompleted, Is.False);
|
||||
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False);
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
|
||||
});
|
||||
|
||||
edit.Warning.Complete(DialogResult.Ok(true));
|
||||
await save.WaitAsync(TEST_TIMEOUT);
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task CancelingThePathWarningKeepsTheConfiguredPathAndQueuesNothing(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var original = this.source;
|
||||
var edit = this.OpenEdit(this.MoveSource("moved"));
|
||||
edit.Warning.Complete(DialogResult.Cancel());
|
||||
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync");
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
|
||||
Assert.That(edit.ParentTask.IsCompleted, Is.False, "Rejecting the warning leaves the edit dialog open.");
|
||||
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False);
|
||||
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
|
||||
});
|
||||
edit.Dialog.Instance.Cancel();
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task ANewLocationOfADataSourceWithoutAnIndexIsSavedWithoutAQuestion(bool singleFile)
|
||||
{
|
||||
await this.AddSourceAsync(singleFile);
|
||||
var newPath = this.MoveSource("moved");
|
||||
var edit = this.OpenEdit(newPath);
|
||||
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
var saved = this.settings.ConfigurationData.DataSources.Single();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.Zero, "Nothing was prepared for the data source, so nothing can be lost.");
|
||||
Assert.That(saved is DataSourceLocalDirectory folder ? folder.Path : ((DataSourceLocalFile)saved).FilePath, Is.EqualTo(newPath));
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task ANewLocationTogetherWithANewEmbeddingAsksOnlyOnce(bool singleFile)
|
||||
{
|
||||
const string OTHER_EMBEDDING_ID = "c1b5d5e3-2a4f-4b1b-9d0e-6b7c8d9e0f12";
|
||||
this.settings.ConfigurationData.EmbeddingProviders.Add(new EmbeddingProvider(2, OTHER_EMBEDDING_ID, "Other embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-large", "text-embedding-3-large")));
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var edit = this.OpenEdit(this.MoveSource("moved"));
|
||||
SetField(edit.Component, "dataEmbeddingId", OTHER_EMBEDDING_ID);
|
||||
edit.Warning.Complete(DialogResult.Ok(true));
|
||||
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1), "Everything is prepared again anyway, so the location needs no question of its own.");
|
||||
Assert.That(edit.Warning.Title, Is.EqualTo("Documents Will Be Prepared Again"));
|
||||
});
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task ASecondSavedPathCancelsTheFirstRelocationAndRunsWithTheLatestPath(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
await this.SaveEditAsync(this.MoveSource("first"));
|
||||
this.vectors.BlockNextRelocation = true;
|
||||
var firstRun = this.RunNextAsync();
|
||||
await this.vectors.RelocationStarted.Task.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
var secondPath = this.MoveSource("second");
|
||||
await this.SaveEditAsync(secondPath);
|
||||
await firstRun.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
Assert.That(this.vectors.CanceledRelocations, Is.EqualTo(1), "The first run must be canceled without finishing its remaining documents.");
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(1));
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.RelocatedPaths.Skip(1).ToArray(), Has.Length.EqualTo(this.chunkIds.Count));
|
||||
Assert.That(this.vectors.Locations.Values.All(path => singleFile ? path == secondPath : path.StartsWith(secondPath + Path.DirectorySeparatorChar, StringComparison.Ordinal)), Is.True);
|
||||
Assert.That(this.vectors.DeletedStores, Is.Zero);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False, "The obsolete destination must not leave another run behind.");
|
||||
});
|
||||
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task DeletingTheSourceCancelsRelocationBeforeRemovingTheStoredIndex(bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
await this.SaveEditAsync(this.MoveSource("moved"));
|
||||
this.vectors.BlockNextRelocation = true;
|
||||
var run = this.RunNextAsync();
|
||||
await this.vectors.RelocationStarted.Task.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
// A second save also leaves a pending run; deletion has to discard that one as well.
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
this.settings.ConfigurationData.DataSources.Clear();
|
||||
await this.service.RemoveDataSourceAsync(this.source).WaitAsync(TEST_TIMEOUT);
|
||||
await run.WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.vectors.CanceledRelocations, Is.EqualTo(1));
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(1));
|
||||
Assert.That(this.vectors.DeletedStores, Is.EqualTo(1));
|
||||
Assert.That(this.vectors.Locations, Is.Empty);
|
||||
Assert.That(manifest.Files, Is.Empty);
|
||||
Assert.That(this.service.GetStatuses(), Is.Empty);
|
||||
Assert.That(this.Queue.TryDequeue(out _), Is.False);
|
||||
});
|
||||
}
|
||||
|
||||
private DataSourceRunQueue Queue => Field<DataSourceRunQueue>(this.service, "runQueue");
|
||||
|
||||
[TestCase(@"E:\Documents", '\\', false)]
|
||||
[TestCase(@"\\server\share\Documents", '\\', false)]
|
||||
[TestCase("/home/alice/Documents", '/', false)]
|
||||
[TestCase(@"E:\Documents", '\\', true)]
|
||||
[TestCase(@"\\server\share\Documents", '\\', true)]
|
||||
[TestCase("/Volumes/External drive/Documents", '/', true)]
|
||||
public async Task ABackupCanReconnectFilesFromAnotherOSWithoutReembedding(string oldRoot, char separator, bool singleFile)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(singleFile);
|
||||
var originalFolder = Path.Combine(this.testRoot, "original");
|
||||
await this.StoreForeignPathsAsync(oldRoot, separator);
|
||||
var restored = await this.BackupAndRestoreIndexAsync();
|
||||
|
||||
// Opening the restored index while the old drive or network share is unavailable must not
|
||||
// prune anything. No real external volume or network connection is needed by these tests.
|
||||
if (!RelocatedFilePaths.IsLocalAbsolute(oldRoot))
|
||||
{
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
await this.AssertOriginalChunksAsync();
|
||||
}
|
||||
|
||||
// User documents are not in an AI Studio backup. Copy them separately, preserving their
|
||||
// modification times, as when reconnecting an external drive after restoring a computer.
|
||||
var destination = Path.Combine(this.testRoot, "mounted volume", "Übertragene Dokumente");
|
||||
foreach (var original in Directory.EnumerateFiles(originalFolder, "*", SearchOption.AllDirectories))
|
||||
{
|
||||
var localPath = Path.Combine(destination, Path.GetRelativePath(originalFolder, original));
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(localPath)!);
|
||||
File.Copy(original, localPath);
|
||||
File.SetLastWriteTimeUtc(localPath, File.GetLastWriteTimeUtc(original));
|
||||
}
|
||||
|
||||
var newPath = singleFile ? Path.Combine(destination, "notes.md") : destination;
|
||||
await this.SaveEditAsync(newPath);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
var manifest = await restored.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
var hits = await restored.SearchChunksAsync(this.source.Id, "indexed", 10, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
|
||||
Assert.That(this.vectors.DeletedStores, Is.Zero);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
|
||||
Assert.That(hits.Select(hit => hit.AbsolutePath), Is.EquivalentTo(this.vectors.Locations.Values));
|
||||
Assert.That(hits.All(hit => PathTools.IsInsideDirectory(destination, hit.AbsolutePath)), Is.True);
|
||||
});
|
||||
|
||||
// A restart/hash check after relocation must not relocate or prepare everything a second time.
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
|
||||
await this.AssertOriginalChunksAsync();
|
||||
|
||||
// A disconnected external drive is not an empty folder. Reconnecting it must also keep
|
||||
// the embeddings, rather than paying to prepare the same documents again.
|
||||
var disconnected = destination + "-disconnected";
|
||||
Directory.Move(destination, disconnected);
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Directory.Move(disconnected, destination);
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
|
||||
});
|
||||
}
|
||||
|
||||
private async Task StoreForeignPathsAsync(string oldRoot, char separator)
|
||||
{
|
||||
await using var context = this.store.CreateContext();
|
||||
foreach (var file in await context.EmbeddedFiles.ToListAsync())
|
||||
{
|
||||
var relative = file.RelativePath.Replace(Path.DirectorySeparatorChar, separator);
|
||||
var oldPath = oldRoot + separator + relative;
|
||||
file.AbsolutePath = oldPath;
|
||||
file.RelativePath = relative;
|
||||
file.Fingerprint = LegacyFingerprint(oldPath, file.FileName, file.LastWriteUtc.UtcDateTime.Ticks, file.FileSize);
|
||||
this.vectors.Locations[file.ParentFileId] = oldPath;
|
||||
}
|
||||
await context.SaveChangesAsync();
|
||||
this.source = this.source switch
|
||||
{
|
||||
DataSourceLocalDirectory folder => folder with { Path = oldRoot },
|
||||
DataSourceLocalFile file => file with { FilePath = oldRoot + separator + "notes.md" },
|
||||
_ => throw new InvalidOperationException(),
|
||||
};
|
||||
this.settings.ConfigurationData.DataSources[0] = this.source;
|
||||
}
|
||||
|
||||
private async Task<SqliteIndexStoreClientImplementation> BackupAndRestoreIndexAsync()
|
||||
{
|
||||
var dataDirectory = Path.Combine(this.testRoot, "source-data");
|
||||
var configDirectory = Path.Combine(this.testRoot, "source-config");
|
||||
var copies = Path.Combine(this.testRoot, "index-copies");
|
||||
Directory.CreateDirectory(dataDirectory);
|
||||
Directory.CreateDirectory(configDirectory);
|
||||
await this.store.Client.BackupToDirectoryAsync(Path.Combine(copies, "databases", "sqlite"), CancellationToken.None);
|
||||
|
||||
// This fixture's vector client stores metadata only. The runtime tests separately check
|
||||
// backing up, reopening, and relocating actual Qdrant vectors without changing their values.
|
||||
var vectorDirectory = Path.Combine(copies, "databases", "vector_database", "stores", "store_test");
|
||||
Directory.CreateDirectory(vectorDirectory);
|
||||
await File.WriteAllTextAsync(Path.Combine(vectorDirectory, "locations.json"), JsonSerializer.Serialize(this.vectors.Locations));
|
||||
var archivePath = Path.Combine(this.testRoot, "backup.zip");
|
||||
var settingsJson = JsonSerializer.Serialize(this.settings.ConfigurationData, SettingsManager.JSON_OPTIONS);
|
||||
await using (var output = File.Create(archivePath))
|
||||
await DataBackupWriter.WriteAsync(output, new DataBackupSources(dataDirectory, configDirectory, settingsJson, copies, []),
|
||||
new DataBackupManifest(DataBackupFormat.FORMAT_VERSION, "26.10.1", "V6", DateTimeOffset.UtcNow, dataDirectory, configDirectory, true, []), null, CancellationToken.None);
|
||||
|
||||
var target = new DataRestoreTarget(Path.Combine(this.testRoot, "restored-data"), Path.Combine(this.testRoot, "restored-config"), "26.10.1", AIStudio.Settings.Version.V6, []);
|
||||
var result = await DataRestorePreparer.PrepareAsync(archivePath, target, null, CancellationToken.None);
|
||||
Assert.That(result.Success, Is.True);
|
||||
|
||||
// The runtime applies this prepared directory at restart. Open the exact restored bytes
|
||||
// here, so this test exercises the real backup writer and restore preparer, not a rebuilt DB.
|
||||
var pendingData = Path.Combine(target.DataDirectory, DataRestoreFormat.PENDING_DIRECTORY);
|
||||
var restoredPath = Path.Combine(pendingData, "databases", "sqlite", "rag-index.sqlite3");
|
||||
var restored = new SqliteIndexStoreClientImplementation("Restored SQLite", restoredPath, pendingData, string.Empty);
|
||||
Field<Dictionary<DatabaseRole, DatabaseClient>>(this.databases, "clients")[DatabaseRole.INDEX_STORE] = restored;
|
||||
this.vectors.Locations.Clear();
|
||||
foreach (var (id, path) in JsonSerializer.Deserialize<Dictionary<string, string>>(await File.ReadAllTextAsync(Path.Combine(pendingData, "databases", "vector_database", "stores", "store_test", "locations.json")))!)
|
||||
this.vectors.Locations.Add(id, path);
|
||||
|
||||
var restoredSettings = JsonSerializer.Deserialize<Data>(await File.ReadAllTextAsync(Path.Combine(target.ConfigDirectory, DataRestoreFormat.PENDING_DIRECTORY, "settings.json")), SettingsManager.JSON_OPTIONS)!;
|
||||
this.settings.ConfigurationData.DataSources.Clear();
|
||||
this.settings.ConfigurationData.DataSources.AddRange(restoredSettings.DataSources);
|
||||
this.source = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
|
||||
return restored;
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task AnEmptyMountDirectoryKeepsTheIndexAndReconnectingReusesIt(bool relocated)
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(false);
|
||||
var original = ((DataSourceLocalDirectory)this.source).Path;
|
||||
var disconnected = Path.Combine(this.testRoot, "disconnected-drive");
|
||||
Directory.Move(original, disconnected);
|
||||
var mount = relocated ? Path.Combine(this.testRoot, "new-mount") : original;
|
||||
Directory.CreateDirectory(mount);
|
||||
var storedHash = (await this.store.Client.GetDataSourceStateAsync(this.source.Id, CancellationToken.None))!.SourceHash;
|
||||
|
||||
if (relocated)
|
||||
await this.SaveEditAsync(mount);
|
||||
else
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
|
||||
// A second refresh must not accept the empty mount as a new baseline and prune the index.
|
||||
for (var attempt = 0; attempt < 2; attempt++)
|
||||
{
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
await this.AssertOriginalChunksAsync();
|
||||
var state = await this.store.Client.GetDataSourceStateAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
|
||||
Assert.That(this.service.GetStatuses().Single().LastError, Does.Contain("appears empty"));
|
||||
Assert.That(state!.SourceHash, Is.EqualTo(storedHash));
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
});
|
||||
if (attempt == 0)
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
}
|
||||
|
||||
Directory.Delete(mount);
|
||||
Directory.Move(disconnected, mount);
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
await this.AssertOriginalChunksAsync();
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(relocated ? this.chunkIds.Count : 0));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task RemovingSomeFilesStillCleansUpOnlyTheirIndexEntries()
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(false);
|
||||
var removed = Path.Combine(((DataSourceLocalDirectory)this.source).Path, "notes.md");
|
||||
var removedChunkId = IndexedDocumentIds.CreateChunkId(this.source.Id, LegacyFingerprint(new FileInfo(removed)), 0);
|
||||
File.Delete(removed);
|
||||
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
var chunks = await this.store.Client.SearchChunksAsync(this.source.Id, "indexed", 10, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(chunks.Select(chunk => chunk.ChunkId), Is.EquivalentTo(this.chunkIds.Except([removedChunkId])));
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.EqualTo(1));
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ExplicitlyExcludingEveryFileStillRemovesTheirIndexEntries()
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(false);
|
||||
this.source = ((DataSourceLocalDirectory)this.source) with { FilePatterns = "*.pdf" };
|
||||
this.settings.ConfigurationData.DataSources[0] = this.source;
|
||||
|
||||
await this.service.QueueDataSourceAsync(this.source);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(manifest.Files, Is.Empty);
|
||||
Assert.That(this.vectors.Locations, Is.Empty);
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.EqualTo(2));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AFolderWhoseNameChangesOnlyInCaseKeepsItsEmbeddings()
|
||||
{
|
||||
await this.SeedLegacyIndexAsync(false);
|
||||
var renamed = Path.Combine(this.testRoot, "Original");
|
||||
|
||||
// Two moves, since a volume which ignores case may refuse to rename a folder to its own name:
|
||||
var detour = Path.Combine(this.testRoot, "renaming");
|
||||
Directory.Move(((DataSourceLocalDirectory)this.source).Path, detour);
|
||||
Directory.Move(detour, renamed);
|
||||
|
||||
await this.SaveEditAsync(renamed);
|
||||
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
|
||||
|
||||
await this.AssertOriginalChunksAsync();
|
||||
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
|
||||
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
|
||||
Assert.That(this.vectors.InsertedBatches, Is.Zero);
|
||||
Assert.That(manifest.Files.Keys.All(path => path.StartsWith(renamed + Path.DirectorySeparatorChar, StringComparison.Ordinal)), Is.True, "The index must store the new spelling of the folder.");
|
||||
});
|
||||
}
|
||||
|
||||
private async Task SeedLegacyIndexAsync(bool singleFile)
|
||||
{
|
||||
var paths = await this.AddSourceAsync(singleFile);
|
||||
var id = this.source.Id;
|
||||
await this.store.Client.UpsertDataSourceAsync(id, this.source.Type.ToString(), EMBEDDING_ID, LEGACY_SIGNATURE, "legacy-source-hash", 3, CancellationToken.None);
|
||||
|
||||
foreach (var relative in paths)
|
||||
{
|
||||
var path = Path.Combine(this.OriginalFolder, relative);
|
||||
var file = new FileInfo(path);
|
||||
var fingerprint = LegacyFingerprint(file);
|
||||
var parentId = IndexedDocumentIds.CreateParentId(id, path);
|
||||
var chunkId = IndexedDocumentIds.CreateChunkId(id, fingerprint, 0);
|
||||
var embeddedAt = DateTimeOffset.UtcNow.AddDays(-10);
|
||||
await this.store.Client.UpsertFileAsync(id, new EmbeddingStateFile(parentId, path, file.Name, relative, "md", fingerprint, file.Length,
|
||||
new DateTimeOffset(file.CreationTimeUtc), new DateTimeOffset(file.LastWriteTimeUtc), embeddedAt, 1), CancellationToken.None);
|
||||
await this.store.Client.UpsertChunksAsync(id, [new EmbeddingStateChunk(chunkId, parentId, 1, 0, "Already indexed report", embeddedAt)], CancellationToken.None);
|
||||
this.vectors.Locations.Add(parentId, path);
|
||||
this.chunkIds.Add(chunkId);
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Writes the documents of a data source and configures it, without anything in the index.
|
||||
/// </summary>
|
||||
/// <returns>The paths of the documents, relative to the folder.</returns>
|
||||
private async Task<string[]> AddSourceAsync(bool singleFile)
|
||||
{
|
||||
var root = this.OriginalFolder;
|
||||
Directory.CreateDirectory(Path.Combine(root, "nested"));
|
||||
var id = Guid.NewGuid().ToString();
|
||||
var folder = new DataSourceLocalDirectory
|
||||
{
|
||||
Id = id, Name = "Existing documents", Type = DataSourceType.LOCAL_DIRECTORY, Path = root,
|
||||
EmbeddingId = EMBEDDING_ID, ConfidenceLevel = ConfidenceLevel.NONE, MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 100,
|
||||
FilePatterns = "*.md", ExcludePatterns = string.Empty,
|
||||
};
|
||||
this.source = singleFile ? new DataSourceLocalFile
|
||||
{
|
||||
Id = id, Name = folder.Name, Type = DataSourceType.LOCAL_FILE, FilePath = Path.Combine(root, "notes.md"),
|
||||
EmbeddingId = EMBEDDING_ID, ConfidenceLevel = ConfidenceLevel.NONE, MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 100,
|
||||
} : folder;
|
||||
this.settings.ConfigurationData.DataSources.Add(this.source);
|
||||
|
||||
var paths = singleFile ? new[] { "notes.md" } : new[] { "notes.md", Path.Combine("nested", "report.md") };
|
||||
foreach (var relative in paths)
|
||||
await File.WriteAllTextAsync(Path.Combine(root, relative), "Already indexed report");
|
||||
|
||||
return paths;
|
||||
}
|
||||
|
||||
private string MoveSource(string destinationName)
|
||||
{
|
||||
var current = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
|
||||
var destination = Path.Combine(this.testRoot, destinationName);
|
||||
if (current is DataSourceLocalDirectory folder)
|
||||
Directory.Move(folder.Path, destination);
|
||||
else
|
||||
{
|
||||
destination += ".md";
|
||||
File.Move(((DataSourceLocalFile)current).FilePath, destination);
|
||||
}
|
||||
return destination;
|
||||
}
|
||||
|
||||
private async Task SaveEditAsync(string newPath)
|
||||
{
|
||||
var edit = this.OpenEdit(newPath);
|
||||
edit.Warning.Complete(DialogResult.Ok(true));
|
||||
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
|
||||
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
|
||||
Assert.That(edit.Warning.ComponentType, Is.EqualTo(typeof(ConfirmDialog)));
|
||||
Assert.That(edit.Warning.Title, Is.EqualTo("New Location for This Data Source"));
|
||||
Assert.That(edit.Warning.Parameters!.Get<string>(nameof(ConfirmDialog.MarkdownBody)), Does.Contain("unchanged at the new location").And.Contain("removed from the index").And.Contain("files on disk stay as they are").And.Contain("costs money"));
|
||||
});
|
||||
this.source = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
|
||||
}
|
||||
|
||||
private OpenedEdit OpenEdit(string newPath)
|
||||
{
|
||||
var storedSource = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
|
||||
var dialog = new TestDataSourceDialog();
|
||||
var warning = new TestDataSourceDialog();
|
||||
var parent = new DataSourceManagement();
|
||||
Property(parent, "SettingsManager", this.settings);
|
||||
Property(parent, "MessageBus", MessageBus.INSTANCE);
|
||||
Property(parent, "DataSourceEmbeddingService", this.service);
|
||||
Property(parent, "DialogService", dialog.Service);
|
||||
var parentTask = InvokeTask(parent, "EditDataSource", storedSource);
|
||||
object component = storedSource is DataSourceLocalDirectory ? new DataSourceLocalDirectoryDialog() : new DataSourceLocalFileDialog();
|
||||
Property(component, "SettingsManager", this.settings);
|
||||
Property(component, "DialogService", warning.Service);
|
||||
Property(component, "DataSourceEmbeddingService", this.service);
|
||||
Property(component, "MudDialog", dialog.Instance);
|
||||
Property(component, "IsEditing", true);
|
||||
Property(component, "DataSource", storedSource);
|
||||
SetField(component, "dataId", storedSource.Id);
|
||||
SetField(component, "dataName", storedSource.Name);
|
||||
SetField(component, "dataEmbeddingId", storedSource.EmbeddingId);
|
||||
SetField(component, "dataConfidenceLevel", storedSource.ConfidenceLevel);
|
||||
SetField(component, "dataMaxChunkTokenLength", storedSource.MaxChunkTokenLength);
|
||||
SetField(component, "dataChunkOverlapTokenLength", storedSource.ChunkOverlapTokenLength);
|
||||
SetField(component, storedSource is DataSourceLocalDirectory ? "dataPath" : "dataFilePath", newPath);
|
||||
if (storedSource is DataSourceLocalDirectory folder)
|
||||
{
|
||||
SetField(component, "dataFilePatterns", folder.FilePatterns);
|
||||
SetField(component, "dataExcludePatterns", folder.ExcludePatterns);
|
||||
}
|
||||
return new(component, dialog, warning, parentTask);
|
||||
}
|
||||
|
||||
private Task RunNextAsync()
|
||||
{
|
||||
Assert.That(this.Queue.TryDequeue(out var request), Is.True, "Saving must queue a run.");
|
||||
var task = InvokeTask(this.service, "RunDataSourceAsync", request!.DataSourceId, request.RefreshMode, this.lifetime.Token);
|
||||
this.runningTasks.Add(task);
|
||||
return task;
|
||||
}
|
||||
|
||||
private async Task AssertOriginalChunksAsync()
|
||||
{
|
||||
var index = await this.databases.GetIndexStoreAsync();
|
||||
var chunks = await index.SearchChunksAsync(this.source.Id, "indexed", 100, CancellationToken.None);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(chunks.Select(chunk => chunk.ChunkId), Is.EquivalentTo(this.chunkIds));
|
||||
Assert.That(chunks.All(chunk => chunk.ChunkText == "Already indexed report"), Is.True);
|
||||
Assert.That(chunks.All(chunk => chunk.EmbeddedAtUtc < DateTimeOffset.UtcNow.AddDays(-9)), Is.True, "Updating the app or its paths must not recreate existing chunks.");
|
||||
});
|
||||
}
|
||||
|
||||
private static string LegacyFingerprint(FileInfo file)
|
||||
{
|
||||
return LegacyFingerprint(file.FullName, file.Name, file.LastWriteTimeUtc.Ticks, file.Length);
|
||||
}
|
||||
|
||||
private static string LegacyFingerprint(string path, string name, long writeTicks, long size)
|
||||
{
|
||||
var parts = new[] { "file", path, name, writeTicks.ToString(), size.ToString() };
|
||||
var serialized = string.Concat(parts.Select(part => $"{part.Length}:{part}|"));
|
||||
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(serialized)));
|
||||
}
|
||||
|
||||
// Keep reflection at the fixture boundary: exercise the production queue worker without
|
||||
// starting the unrelated application startup loop, and install the isolated database clients.
|
||||
private static T Field<T>(object instance, string name) => (T)instance.GetType().GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(instance)!;
|
||||
|
||||
private static void SetField(object instance, string name, object value) => instance.GetType().GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(instance, value);
|
||||
|
||||
private static void Property(object instance, string name, object value)
|
||||
{
|
||||
for (var type = instance.GetType(); type is not null; type = type.BaseType)
|
||||
if (type.GetProperty(name, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly) is { } property)
|
||||
{
|
||||
property.SetValue(instance, value);
|
||||
return;
|
||||
}
|
||||
throw new MissingMemberException(name);
|
||||
}
|
||||
|
||||
private static Task InvokeTask(object instance, string name, params object[] arguments) =>
|
||||
(Task)instance.GetType().GetMethod(name, BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(instance, arguments)!;
|
||||
|
||||
private sealed record OpenedEdit(object Component, TestDataSourceDialog Dialog, TestDataSourceDialog Warning, Task ParentTask);
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
using AIStudio.Tools.Databases.IndexStore;
|
||||
using AIStudio.Tools.Services.Indexing;
|
||||
|
||||
namespace AIStudio.Tests.Tools.Databases;
|
||||
|
||||
public sealed class IndexStoreRelocationTests
|
||||
{
|
||||
[Test]
|
||||
public async Task RelocationPreservesChunksAndTranscriptAndUpdatesSearchLocations()
|
||||
{
|
||||
await using var store = await TemporaryIndexStore.CreateAsync();
|
||||
var sourceId = Guid.NewGuid().ToString();
|
||||
var oldPath = Path.Combine(Path.GetTempPath(), "original", "notes.md");
|
||||
var newPath = Path.Combine(Path.GetTempPath(), "moved", "renamed.md");
|
||||
var parentId = IndexedDocumentIds.CreateParentId(sourceId, oldPath);
|
||||
var chunkId = IndexedDocumentIds.CreateChunkId(sourceId, "old-fingerprint", 0);
|
||||
var now = DateTimeOffset.UtcNow;
|
||||
var token = CancellationToken.None;
|
||||
await store.Client.UpsertDataSourceAsync(sourceId, "LOCAL_DIRECTORY", "provider", "signature", "old-hash", 3, token);
|
||||
await store.Client.UpsertFileAsync(sourceId, new EmbeddingStateFile(parentId, oldPath, "notes.md", "notes.md", "md", "old-fingerprint", 100, now, now, now, 1) { TranscriptKey = "transcript" }, token);
|
||||
await store.Client.UpsertChunksAsync(sourceId, [new EmbeddingStateChunk(chunkId, parentId, 2, 0, "report about relocation", now)], token);
|
||||
|
||||
// Repeating the metadata write after an interrupted relocation must keep the same chunks.
|
||||
await store.Client.RelocateFileAsync(sourceId, parentId, newPath, "renamed.md", "new-fingerprint", token);
|
||||
await store.Client.RelocateFileAsync(sourceId, parentId, newPath, "renamed.md", "new-fingerprint", token);
|
||||
var manifest = await store.Client.GetManifestAsync(sourceId, token);
|
||||
var hits = await store.Client.SearchChunksAsync(sourceId, "relocation", 10, token);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(manifest.Files.Keys, Is.EquivalentTo(new[] { newPath }));
|
||||
Assert.That(manifest.Files[newPath].Fingerprint, Is.EqualTo("new-fingerprint"));
|
||||
Assert.That(manifest.Files[newPath].ParentFileId, Is.EqualTo(parentId));
|
||||
Assert.That(manifest.Files[newPath].RelativePath, Is.EqualTo("renamed.md"));
|
||||
Assert.That(manifest.Files[newPath].TranscriptKey, Is.EqualTo("transcript"));
|
||||
Assert.That(hits, Has.Count.EqualTo(1));
|
||||
Assert.That(hits[0].ChunkId, Is.EqualTo(chunkId));
|
||||
Assert.That(hits[0].AbsolutePath, Is.EqualTo(newPath));
|
||||
Assert.That(hits[0].FileName, Is.EqualTo("renamed.md"));
|
||||
Assert.That(hits[0].PageNumber, Is.EqualTo(2));
|
||||
Assert.That(hits[0].ChunkText, Is.EqualTo("report about relocation"));
|
||||
});
|
||||
|
||||
await store.Client.DeleteFileAsync(sourceId, newPath, token);
|
||||
Assert.That(await store.Client.SearchChunksAsync(sourceId, "relocation", 10, token), Is.Empty, "Ordinary removal must still find a relocated document's chunks.");
|
||||
}
|
||||
}
|
||||
@@ -35,6 +35,15 @@ public sealed class FileSourceIndexerWatcherTests
|
||||
Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SingleFileWatcherIgnoresAnotherCaseDistinctFileOnUnix()
|
||||
{
|
||||
var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(Path.GetTempPath(), false, TargetFileName: "Report.md");
|
||||
|
||||
Assert.That(FileSourceIndexer.IsRelevantWatcherPath(configuration, Path.Combine(Path.GetTempPath(), "report.md"), WatcherChangeTypes.Changed),
|
||||
Is.EqualTo(OperatingSystem.IsWindows()));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void SingleFileWatcherRecognizesARenameAwayFromTheConfiguredFile()
|
||||
{
|
||||
@@ -262,6 +271,15 @@ public sealed class FileSourceIndexerWatcherTests
|
||||
Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.False);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RelocatingToADirectoryWithDifferentCasingUsesTheLocalPathRules()
|
||||
{
|
||||
var before = DefaultConfiguration(Path.Combine(Path.GetTempPath(), "Documents"));
|
||||
var after = before with { RootPath = Path.Combine(Path.GetTempPath(), "documents") };
|
||||
|
||||
Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.EqualTo(OperatingSystem.IsWindows()), "On Unix these can be different directories; their watchers must not be treated as interchangeable.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task DirectoryWatcherIgnoresTheBatchTranscriptWhenTheMediaIsSelected()
|
||||
{
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.Services;
|
||||
using AIStudio.Tools.Services.Indexing;
|
||||
|
||||
using Microsoft.Extensions.Logging.Abstractions;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
public sealed class FileSourceRelocationTests
|
||||
{
|
||||
private string testRoot = string.Empty;
|
||||
|
||||
[SetUp]
|
||||
public void SetUp()
|
||||
{
|
||||
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-relocation-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(this.testRoot);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void TearDown() => Directory.Delete(this.testRoot, true);
|
||||
|
||||
[Test]
|
||||
public void AMovedFolderFindsAnUnchangedNestedDocument()
|
||||
{
|
||||
var oldRoot = Path.Combine(this.testRoot, "original");
|
||||
var oldFile = Path.Combine(oldRoot, "nested", "notes.md");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(oldFile)!);
|
||||
File.WriteAllText(oldFile, "Some indexed text");
|
||||
var record = Record(oldFile, Path.Combine("nested", "notes.md"));
|
||||
var newRoot = Path.Combine(this.testRoot, "moved");
|
||||
Directory.Move(oldRoot, newRoot);
|
||||
|
||||
var found = FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = newRoot }, oldFile, record);
|
||||
|
||||
Assert.That(found?.FullName, Is.EqualTo(Path.Combine(newRoot, "nested", "notes.md")));
|
||||
}
|
||||
|
||||
[TestCase("size")]
|
||||
[TestCase("modification time")]
|
||||
[TestCase("missing file")]
|
||||
public void AChangedOrMissingDocumentCannotReuseItsEmbeddings(string change)
|
||||
{
|
||||
var file = Path.Combine(this.testRoot, "notes.md");
|
||||
File.WriteAllText(file, "Some indexed text");
|
||||
var record = Record(file, "notes.md");
|
||||
switch (change)
|
||||
{
|
||||
case "size":
|
||||
File.AppendAllText(file, " with more text");
|
||||
File.SetLastWriteTimeUtc(file, record.LastWriteUtc.UtcDateTime);
|
||||
break;
|
||||
case "modification time":
|
||||
File.SetLastWriteTimeUtc(file, record.LastWriteUtc.UtcDateTime.AddMinutes(1));
|
||||
break;
|
||||
case "missing file":
|
||||
File.Delete(file);
|
||||
break;
|
||||
}
|
||||
|
||||
// The old place lies outside the selected folder, so only the change can prevent the reuse:
|
||||
var oldPath = Path.Combine(Path.GetDirectoryName(this.testRoot)!, "moved-away", "notes.md");
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, oldPath, record), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AStoredPathBelowTheSelectedFolderIsNotMappedToACopy()
|
||||
{
|
||||
// The folder grows to its parent, which holds a copy with the same size and time at the
|
||||
// place the old relative path names. The original is still there and stays the document.
|
||||
var original = Path.Combine(this.testRoot, "project", "notes.md");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(original)!);
|
||||
File.WriteAllText(original, "Some indexed text");
|
||||
var copy = Path.Combine(this.testRoot, "notes.md");
|
||||
File.Copy(original, copy);
|
||||
File.SetLastWriteTimeUtc(copy, File.GetLastWriteTimeUtc(original));
|
||||
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, original, Record(original, "notes.md")), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARenamedSingleFileCanReuseItsEmbeddings()
|
||||
{
|
||||
var oldFile = Path.Combine(this.testRoot, "notes.md");
|
||||
File.WriteAllText(oldFile, "Some indexed text");
|
||||
var record = Record(oldFile, "notes.md");
|
||||
var newFile = Path.Combine(this.testRoot, "renamed.md");
|
||||
File.Move(oldFile, newFile);
|
||||
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalFile { FilePath = newFile }, oldFile, record)?.FullName, Is.EqualTo(newFile));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ARelativePathCannotEscapeTheConfiguredFolder()
|
||||
{
|
||||
var outside = Path.Combine(this.testRoot, "notes.md");
|
||||
File.WriteAllText(outside, "Some indexed text");
|
||||
var record = Record(outside, Path.Combine("..", "notes.md"));
|
||||
var folder = Path.Combine(this.testRoot, "selected");
|
||||
Directory.CreateDirectory(folder);
|
||||
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = folder }, Path.Combine(this.testRoot, "old", "notes.md"), record), Is.Null);
|
||||
}
|
||||
|
||||
private static EmbeddedFileRecord Record(string path, string relativePath)
|
||||
{
|
||||
var file = new FileInfo(path);
|
||||
return new("old-fingerprint", file.Length, new DateTimeOffset(file.LastWriteTimeUtc), DateTimeOffset.UtcNow, 1, RelativePath: relativePath, ParentFileId: Guid.NewGuid().ToString());
|
||||
}
|
||||
|
||||
[TestCase(@"E:\Documents\nested\notes.md", @"nested\notes.md")]
|
||||
[TestCase(@"\\server\share\Documents\nested\notes.md", @"nested\notes.md")]
|
||||
[TestCase(@"\\?\E:\Documents\nested\notes.md", @"nested\notes.md")]
|
||||
[TestCase("E:/Documents/nested/notes.md", "nested/notes.md")]
|
||||
[TestCase("/home/alice/Documents/nested/notes.md", "nested/notes.md")]
|
||||
[TestCase("/Volumes/External drive/Documents/nested/notes.md", "nested/notes.md")]
|
||||
public void ABackupFromAnotherPathSyntaxFindsANestedFile(string oldPath, string relativePath)
|
||||
{
|
||||
var localFile = Path.Combine(this.testRoot, "nested", "notes.md");
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(localFile)!);
|
||||
File.WriteAllText(localFile, "Some indexed text");
|
||||
|
||||
var found = FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, oldPath, Record(localFile, relativePath));
|
||||
|
||||
Assert.That(found?.FullName, Is.EqualTo(localFile));
|
||||
}
|
||||
|
||||
[TestCase(@"C:\a.folder\README")]
|
||||
[TestCase(@"\\server\a.folder\README")]
|
||||
[TestCase("/media/a.folder/README")]
|
||||
public void AForeignDirectoryNameDoesNotBecomeTheExtensionOfAnExtensionlessFile(string oldPath)
|
||||
{
|
||||
var localFile = Path.Combine(this.testRoot, "README");
|
||||
File.WriteAllText(localFile, "Some indexed text");
|
||||
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalFile { FilePath = localFile }, oldPath, Record(localFile, "README"))?.FullName, Is.EqualTo(localFile));
|
||||
}
|
||||
|
||||
[TestCase(@"..\notes.md")]
|
||||
[TestCase("../notes.md")]
|
||||
[TestCase(@"nested\..\..\notes.md")]
|
||||
[TestCase(@"C:\notes.md")]
|
||||
[TestCase(@"C:notes.md")]
|
||||
[TestCase(@"\\server\share\notes.md")]
|
||||
[TestCase("/etc/notes.md")]
|
||||
public void AStoredWindowsPathCannotEscapeTheNewRoot(string relativePath)
|
||||
{
|
||||
Assert.That(RelocatedFilePaths.Resolve(this.testRoot, @"E:\Documents\notes.md", relativePath), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnExternalDriveRootIsAValidRelocationDestination()
|
||||
{
|
||||
// No file needs to be written at the drive root: this checks resolving the destination.
|
||||
var root = Path.GetPathRoot(this.testRoot)!;
|
||||
|
||||
Assert.That(RelocatedFilePaths.Resolve(root, @"E:\notes.md", "notes.md"), Is.EqualTo(Path.Combine(root, "notes.md")));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AUnixBackslashInAFileNameIsNotReinterpretedAsADirectory()
|
||||
{
|
||||
if (OperatingSystem.IsWindows())
|
||||
{
|
||||
Assert.That(RelocatedFilePaths.Resolve(this.testRoot, @"/home/alice/notes\draft.md", @"notes\draft.md"), Is.Null, "A Windows filename cannot contain a backslash from a Unix filename.");
|
||||
return;
|
||||
}
|
||||
|
||||
var localFile = Path.Combine(this.testRoot, @"notes\draft.md");
|
||||
File.WriteAllText(localFile, "Some indexed text");
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, @"/home/alice/notes\draft.md", Record(localFile, @"notes\draft.md"))?.FullName, Is.EqualTo(localFile));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnOldForeignRootIsNotResolvedRelativeToTheWorkingDirectory()
|
||||
{
|
||||
var foreignRoot = OperatingSystem.IsWindows() ? "/home/alice/Documents" : @"E:\Documents";
|
||||
|
||||
Assert.That(RelocatedFilePaths.Resolve(foreignRoot, @"E:\Documents\notes.md", "notes.md"), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ReducedTimestampPrecisionOnAnotherDriveDoesNotSilentlyReuseOldContent()
|
||||
{
|
||||
var localFile = Path.Combine(this.testRoot, "notes.md");
|
||||
File.WriteAllText(localFile, "Some indexed text");
|
||||
var record = Record(localFile, "notes.md");
|
||||
// The old filesystem stored a finer timestamp than the copied file now reports. Without
|
||||
// a stored content hash, equal size alone is not enough to prove that it is unchanged.
|
||||
record = record with { LastWriteUtc = record.LastWriteUtc.AddTicks(1) };
|
||||
|
||||
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, @"E:\Documents\notes.md", record), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ANewFileAtAnOldLocationCannotOverwriteTheRelocatedDocument()
|
||||
{
|
||||
var source = new DataSourceLocalDirectory { Id = Guid.NewGuid().ToString(), Path = this.testRoot };
|
||||
var originalPath = Path.Combine(this.testRoot, "nested", "notes.md");
|
||||
var relocatedPath = Path.Combine(this.testRoot, "notes.md");
|
||||
var originalId = IndexedDocumentIds.CreateParentId(source.Id, originalPath);
|
||||
var manifest = new DataSourceEmbeddingManifest();
|
||||
manifest.Files[relocatedPath] = new("fingerprint", 100, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 1, RelativePath: "notes.md", ParentFileId: originalId);
|
||||
|
||||
// Only identity assignment is exercised; none of the run's services are called.
|
||||
var context = new IndexedRunContext(source, null!, null!, null!, null!, manifest,
|
||||
new RunShare(RunShare.DEFAULT_DURATION), new IndexingPass(new DataSourceRunSignature("signature", "selection")), null!, null!, _ => { }, NullLogger.Instance);
|
||||
var newId = context.GetFileParentId(originalPath);
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(context.GetFileParentId(relocatedPath), Is.EqualTo(originalId));
|
||||
Assert.That(newId, Is.Not.EqualTo(originalId));
|
||||
Assert.That(context.GetFileParentId(originalPath), Is.EqualTo(newId), "A media file may ask for its identity once before and once after transcription.");
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -31,4 +31,21 @@ public sealed class IndexedDocumentIdsTests
|
||||
{
|
||||
Assert.That(IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, "/tmp/test-data/report.pdf"), Is.EqualTo("d06c4351-2e57-2941-aa4c-d7a814bfa418"));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ADocumentStoredUnderTheIdOfItsKeyKeepsItsChunkIds()
|
||||
{
|
||||
const string KEY = "/tmp/test-data/report.pdf";
|
||||
var source = IndexedDocumentIds.CreateChunkIdSource(DATA_SOURCE_ID, KEY, IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, KEY), "3F9A0C");
|
||||
|
||||
Assert.That(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, source, 0), Is.EqualTo("7c9d2822-f9d1-a94e-8d83-30cea4f0856f"), "Files and mails which never moved must keep the ids they are stored under.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheChunkIdOfAMovedDocumentIsPinned()
|
||||
{
|
||||
var source = IndexedDocumentIds.CreateChunkIdSource(DATA_SOURCE_ID, "/tmp/moved/report.pdf", "0b5c2f7e-3d41-4a8e-9f60-2c7d1e8a4b93", "3F9A0C");
|
||||
|
||||
Assert.That(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, source, 0), Is.EqualTo("aa597d9b-ede8-5449-80d0-f209cde0ecc0"), "A file at the old place with the same fingerprint must not get the chunk ids of the moved one.");
|
||||
}
|
||||
}
|
||||
@@ -112,7 +112,7 @@ public sealed class OutdatedRunTests
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void ASingleFileHasNoSelection()
|
||||
public void RelocatingASingleFileStopsTheOldRunWithoutInvalidatingItsEmbeddings()
|
||||
{
|
||||
var dataSource = new DataSourceLocalFile
|
||||
{
|
||||
@@ -125,7 +125,26 @@ public sealed class OutdatedRunTests
|
||||
FilePath = "/tmp/test-data/report.pdf",
|
||||
};
|
||||
|
||||
Assert.That(DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM).SelectionSignature, Is.Empty, "The file is the one document of its data source, so there is nothing to choose among.");
|
||||
var before = DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
|
||||
var after = DataSourceEmbeddingService.GetRunSignature(dataSource with { FilePath = "/tmp/moved/report.pdf" }, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(DataSourceEmbeddingService.IsRunOutdated(before, after), Is.True);
|
||||
Assert.That(after.EmbeddingSignature, Is.EqualTo(before.EmbeddingSignature));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void RelocatingAFolderStopsTheOldRunWithoutInvalidatingItsEmbeddings()
|
||||
{
|
||||
var dataSource = DataSource(ConfidenceLevel.LOW);
|
||||
var before = DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
|
||||
var after = DataSourceEmbeddingService.GetRunSignature(dataSource with { Path = "/tmp/moved" }, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(DataSourceEmbeddingService.IsRunOutdated(before, after), Is.True);
|
||||
Assert.That(after.EmbeddingSignature, Is.EqualTo(before.EmbeddingSignature));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
using AIStudio.Tools;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Checks where a file dialog opens for a file or a folder which was moved or renamed.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The native dialog cannot open a place which no longer exists and falls back to wherever it was
|
||||
/// used last, which may be anywhere. The nearest folder which still exists is where the file or the
|
||||
/// folder was moved from, so its new place is often close by.
|
||||
/// </remarks>
|
||||
[TestFixture]
|
||||
public sealed class PathToolsTests
|
||||
{
|
||||
private string testRoot = string.Empty;
|
||||
|
||||
[SetUp]
|
||||
public void CreateTestRoot()
|
||||
{
|
||||
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-path-tools-{Guid.NewGuid():N}");
|
||||
Directory.CreateDirectory(this.testRoot);
|
||||
}
|
||||
|
||||
[TearDown]
|
||||
public void DeleteTestRoot()
|
||||
{
|
||||
if (Directory.Exists(this.testRoot))
|
||||
Directory.Delete(this.testRoot, true);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnExistingFolderOpensAsItIs()
|
||||
{
|
||||
var folder = Directory.CreateDirectory(Path.Combine(this.testRoot, "documents")).FullName;
|
||||
|
||||
Assert.That(PathTools.FindNearestExistingPath(folder), Is.EqualTo(folder));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnExistingFileOpensAsItIs()
|
||||
{
|
||||
var file = Path.Combine(this.testRoot, "report.pdf");
|
||||
File.WriteAllText(file, "report");
|
||||
|
||||
Assert.That(PathTools.FindNearestExistingPath(file), Is.EqualTo(file));
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AMovedFolderOpensAtItsNearestFolderWhichStillExists()
|
||||
{
|
||||
var parent = Directory.CreateDirectory(Path.Combine(this.testRoot, "projects")).FullName;
|
||||
var movedFolder = Path.Combine(parent, "2026", "documents");
|
||||
|
||||
Assert.That(PathTools.FindNearestExistingPath(movedFolder), Is.EqualTo(parent), "Every parent folder counts, not only the immediate one.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AMovedFileOpensAtItsFolder()
|
||||
{
|
||||
var file = Path.Combine(this.testRoot, "report.pdf");
|
||||
|
||||
Assert.That(PathTools.FindNearestExistingPath(file), Is.EqualTo(this.testRoot));
|
||||
}
|
||||
|
||||
[TestCase("")]
|
||||
[TestCase(" ")]
|
||||
[TestCase("documents/report.pdf")]
|
||||
public void APathWithoutAPlaceOpensWhereverTheDialogWants(string path)
|
||||
{
|
||||
Assert.That(PathTools.FindNearestExistingPath(path), Is.Null);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void APathOfAnotherOperatingSystemOpensWhereverTheDialogWants()
|
||||
{
|
||||
var path = OperatingSystem.IsWindows() ? "/Users/somebody/documents" : @"C:\Users\somebody\documents";
|
||||
|
||||
Assert.That(PathTools.FindNearestExistingPath(path), Is.Null, "A backup from another computer must not make the dialog open at a folder which happens to exist here.");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
using AIStudio.Tools.Databases.VectorStore;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Keeps the identities and locations of existing vectors. New embeddings are forbidden: these
|
||||
/// tests reconnect previously indexed files and must never contact an embedding provider.
|
||||
/// </summary>
|
||||
internal sealed class RelocationVectorStore() : VectorStoreClient("Relocation test vectors", string.Empty)
|
||||
{
|
||||
public Dictionary<string, string> Locations { get; } = new(StringComparer.Ordinal);
|
||||
|
||||
public List<string> RelocatedPaths { get; } = [];
|
||||
|
||||
public TaskCompletionSource<object?> RelocationStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public bool BlockNextRelocation { get; set; }
|
||||
|
||||
public int CanceledRelocations { get; private set; }
|
||||
|
||||
public int DeletedStores { get; private set; }
|
||||
|
||||
public int DeletedDocuments { get; private set; }
|
||||
|
||||
public int InsertedBatches { get; private set; }
|
||||
|
||||
public override async Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
Assert.That(this.Locations.ContainsKey(parentFileId), Is.True, "Relocation must address an existing vector identity.");
|
||||
this.Locations[parentFileId] = absolutePath;
|
||||
this.RelocatedPaths.Add(absolutePath);
|
||||
if (!this.BlockNextRelocation)
|
||||
return;
|
||||
|
||||
this.BlockNextRelocation = false;
|
||||
this.RelocationStarted.TrySetResult(null);
|
||||
try
|
||||
{
|
||||
await Task.Delay(Timeout.InfiniteTimeSpan, token);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
this.CanceledRelocations++;
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
||||
public override Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) => Task.FromResult(new VectorStoreEnsureResult(false));
|
||||
|
||||
public override Task<bool> VectorStoreExists(string storeName, CancellationToken token) => Task.FromResult(true);
|
||||
|
||||
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token)
|
||||
{
|
||||
this.InsertedBatches++;
|
||||
throw new InvalidOperationException("Existing unchanged files must not be embedded again.");
|
||||
}
|
||||
|
||||
public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token)
|
||||
{
|
||||
this.DeletedDocuments++;
|
||||
foreach (var id in this.Locations.Where(entry => entry.Key == parentFileId || entry.Value == filePath).Select(entry => entry.Key).ToArray())
|
||||
this.Locations.Remove(id);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public override Task DeleteVectorStore(string storeName, CancellationToken token)
|
||||
{
|
||||
this.DeletedStores++;
|
||||
this.Locations.Clear();
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public override Task OptimizeVectorStore(string storeName, CancellationToken token) => Task.CompletedTask;
|
||||
|
||||
public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, CancellationToken token) => throw new NotSupportedException();
|
||||
|
||||
public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, CancellationToken token) => throw new NotSupportedException();
|
||||
|
||||
public override Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token) => throw new NotSupportedException();
|
||||
|
||||
public override Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) => throw new NotSupportedException();
|
||||
|
||||
public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo()
|
||||
{
|
||||
await Task.CompletedTask;
|
||||
yield break;
|
||||
}
|
||||
|
||||
public override void Dispose()
|
||||
{
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,76 @@
|
||||
using System.Reflection;
|
||||
|
||||
using MudBlazor;
|
||||
|
||||
namespace AIStudio.Tests.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Controls a dialog result without rendering a Blazor circuit. The real dialog save handler and
|
||||
/// the real parent edit handler still run, with the same parameters and result as in the UI.
|
||||
/// </summary>
|
||||
internal sealed class TestDataSourceDialog
|
||||
{
|
||||
private readonly TaskCompletionSource<DialogResult?> result = new(TaskCreationOptions.RunContinuationsAsynchronously);
|
||||
|
||||
public IDialogService Service { get; }
|
||||
|
||||
public IMudDialogInstance Instance { get; }
|
||||
|
||||
public DialogParameters? Parameters { get; private set; }
|
||||
|
||||
public Type? ComponentType { get; private set; }
|
||||
|
||||
public string? Title { get; private set; }
|
||||
|
||||
public int ShowCount { get; private set; }
|
||||
|
||||
public TestDataSourceDialog()
|
||||
{
|
||||
var reference = Create<IDialogReference>((method, _) => method.Name == "get_Result"
|
||||
? this.result.Task
|
||||
: throw new NotSupportedException(method.Name));
|
||||
this.Service = Create<IDialogService>((method, arguments) =>
|
||||
{
|
||||
if (method.Name != "ShowAsync")
|
||||
throw new NotSupportedException(method.Name);
|
||||
|
||||
ArgumentNullException.ThrowIfNull(arguments);
|
||||
this.ShowCount++;
|
||||
this.ComponentType = method.GetGenericArguments().Single();
|
||||
this.Title = arguments.OfType<string>().FirstOrDefault();
|
||||
this.Parameters = arguments.OfType<DialogParameters>().Single();
|
||||
return Task.FromResult(reference);
|
||||
});
|
||||
this.Instance = Create<IMudDialogInstance>((method, arguments) =>
|
||||
{
|
||||
switch (method.Name)
|
||||
{
|
||||
case "Close":
|
||||
this.Complete(arguments!.OfType<DialogResult>().Single());
|
||||
return null;
|
||||
case "Cancel":
|
||||
this.Complete(DialogResult.Cancel());
|
||||
return null;
|
||||
default:
|
||||
throw new NotSupportedException(method.Name);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
public void Complete(DialogResult dialogResult) => this.result.TrySetResult(dialogResult);
|
||||
|
||||
private static T Create<T>(Func<MethodInfo, object?[]?, object?> invoke) where T : class
|
||||
{
|
||||
var proxy = DispatchProxy.Create<T, DialogCallProxy>();
|
||||
((DialogCallProxy)(object)proxy).InvokeCall = invoke;
|
||||
return proxy;
|
||||
}
|
||||
}
|
||||
|
||||
// DispatchProxy generates a subclass, so this type must be public and cannot be sealed.
|
||||
public class DialogCallProxy : DispatchProxy
|
||||
{
|
||||
public Func<MethodInfo, object?[]?, object?> InvokeCall { get; set; } = null!;
|
||||
|
||||
protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) => this.InvokeCall(targetMethod!, args);
|
||||
}
|
||||
@@ -92,7 +92,7 @@ public sealed class SemanticSearchToolUnreachableTests
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TheResultNamesAnUnreachableDataSourceWithoutItsID()
|
||||
public void TheResultNamesAnUnreachableDataSourceWithoutItsId()
|
||||
{
|
||||
var result = SemanticSearchTool.DescribeUnreachableResult(INTRANET);
|
||||
|
||||
|
||||
@@ -62,6 +62,49 @@ public sealed class UnreachableDataSourceTests
|
||||
AssertTheIndexWasKept(outcome, statuses, manifest, 2, folder);
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AMissingRelocationTargetKeepsTheOriginalIndex()
|
||||
{
|
||||
var folder = Path.Combine(this.testRoot, "moved-but-unavailable");
|
||||
var (outcome, statuses, manifest) = await RunAsync(Folder(folder), Path.Combine(this.testRoot, "original", "report.pdf"));
|
||||
|
||||
AssertTheIndexWasKept(outcome, statuses, manifest, 1, folder);
|
||||
}
|
||||
|
||||
[TestCase(false)]
|
||||
[TestCase(true)]
|
||||
public async Task AReadableEmptyMountDirectoryKeepsItsIndex(bool emptySubfolder)
|
||||
{
|
||||
var folder = Path.Combine(this.testRoot, "mounted-drive");
|
||||
Directory.CreateDirectory(emptySubfolder ? Path.Combine(folder, "documents") : folder);
|
||||
var unfinishedFile = Path.Combine(folder, "unfinished.pdf");
|
||||
|
||||
var (outcome, statuses, manifest) = await RunAsync(Folder(folder), [Path.Combine(folder, "report.pdf")], [unfinishedFile]);
|
||||
|
||||
AssertTheIndexWasKept(outcome, statuses, manifest, 1, folder);
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(manifest.UnfinishedFiles, Does.Contain(unfinishedFile), "An unmounted drive must not discard interrupted work either.");
|
||||
Assert.That(statuses[^1].LastError, Does.Contain("appears empty").And.Contain("remove this data source"));
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task ANewEmptyFolderCanFinishWithoutAnIndex()
|
||||
{
|
||||
var folder = Path.Combine(this.testRoot, "new-empty-folder");
|
||||
Directory.CreateDirectory(folder);
|
||||
|
||||
var (outcome, statuses, manifest) = await RunAsync(Folder(folder));
|
||||
|
||||
Assert.Multiple(() =>
|
||||
{
|
||||
Assert.That(outcome, Is.EqualTo(IndexedRunOutcome.DONE));
|
||||
Assert.That(statuses[^1].State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
|
||||
Assert.That(manifest.Files, Is.Empty);
|
||||
});
|
||||
}
|
||||
|
||||
[Test]
|
||||
public async Task AMissingFileKeepsItsIndex()
|
||||
{
|
||||
@@ -130,6 +173,15 @@ public sealed class UnreachableDataSourceTests
|
||||
Assert.That(goneFiles, Is.EquivalentTo([deleted, besideSubfolder]), "A folder whose name only begins like the one which could not be read is no part of it.");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void AnUnreadableDriveRootKeepsItsIndexedFiles()
|
||||
{
|
||||
var root = Path.GetPathRoot(Path.GetFullPath(Path.GetTempPath()))!;
|
||||
var file = Path.Combine(root, "documents", "notes.md");
|
||||
|
||||
Assert.That(FileSourceIndexer.GetGoneFiles([file], [], [root]), Is.Empty, "A drive root already ends in its separator; adding another would fail to protect its files.");
|
||||
}
|
||||
|
||||
private static DataSourceLocalDirectory Folder(string path) => new()
|
||||
{
|
||||
Num = 1,
|
||||
@@ -152,6 +204,7 @@ public sealed class UnreachableDataSourceTests
|
||||
Assert.That(statuses[^1].State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
|
||||
Assert.That(statuses[^1].IndexedDocuments, Is.EqualTo(indexedFiles), "The index still holds these files, and the page has to say so.");
|
||||
Assert.That(statuses[^1].LastError, Does.Contain(unreachablePath));
|
||||
Assert.That(statuses[^1].Attention, Is.EqualTo(DataSourceAttention.LOCATION_UNREACHABLE), "The embeddings page offers to edit the data source, e.g. to select the new location of a moved folder.");
|
||||
});
|
||||
}
|
||||
|
||||
@@ -181,12 +234,16 @@ public sealed class UnreachableDataSourceTests
|
||||
var statuses = new List<DataSourceEmbeddingStatus>();
|
||||
var manifest = new DataSourceEmbeddingManifest { SourceHash = STORED_SOURCE_HASH };
|
||||
foreach (var file in indexedFiles)
|
||||
manifest.Files[file] = new EmbeddedFileRecord("FINGERPRINT", 1024, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 4);
|
||||
manifest.Files[file] = new EmbeddedFileRecord("FINGERPRINT", 1024, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 4,
|
||||
RelativePath: Path.GetFileName(file), ParentFileId: IndexedDocumentIds.CreateParentId(dataSource.Id, file));
|
||||
|
||||
manifest.UnfinishedFiles.UnionWith(unfinishedFiles);
|
||||
|
||||
var scannedFile = Path.Combine(Path.GetDirectoryName(indexedFiles[0])!, "scan.pdf");
|
||||
manifest.PermanentFailures[scannedFile] = new PermanentIndexingFailureRecord("FINGERPRINT", FileExtractionErrorCode.NO_TEXT_EXTRACTED, "No text could be read.", DateTimeOffset.UnixEpoch);
|
||||
if (indexedFiles.Length > 0)
|
||||
{
|
||||
var scannedFile = Path.Combine(Path.GetDirectoryName(indexedFiles[0])!, "scan.pdf");
|
||||
manifest.PermanentFailures[scannedFile] = new PermanentIndexingFailureRecord("FINGERPRINT", FileExtractionErrorCode.NO_TEXT_EXTRACTED, "No text could be read.", DateTimeOffset.UnixEpoch);
|
||||
}
|
||||
|
||||
var embeddingProvider = new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"));
|
||||
var settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
|
||||
|
||||
+1
-1
@@ -3,7 +3,7 @@
|
||||
257
|
||||
9.0.122 (commit 17951ad7c6)
|
||||
9.0.21 (commit d280e4fb38)
|
||||
1.99.0 (commit b940084d7)
|
||||
1.99.1 (commit 58cf8f9c3)
|
||||
8.15.0
|
||||
2.11.5
|
||||
9f102728b59, release
|
||||
|
||||
@@ -11,9 +11,9 @@ use qdrant_edge::external::uuid::Uuid;
|
||||
use qdrant_edge::{
|
||||
Condition, Distance, EdgeConfig, EdgeOptimizersConfig, EdgeShard, EdgeVectorParams,
|
||||
FieldCondition, Filter, HasIdCondition, HnswIndexConfig, Match, MatchValue, NamedQuery,
|
||||
Payload, PointId, PointInsertOperations, PointOperations, PointStruct, QueryEnum,
|
||||
Payload, PayloadOps, PointId, PointInsertOperations, PointOperations, PointStruct, QueryEnum,
|
||||
QueryRequest, ScoredPoint, ScoringQuery, UpdateOperation, ValueVariants, VectorInternal,
|
||||
Vectors, WithPayloadInterface, WithVector,
|
||||
SetPayloadOp, Vectors, WithPayloadInterface, WithVector,
|
||||
};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tauri::Manager;
|
||||
@@ -131,6 +131,17 @@ pub struct SearchQdrantEdgeEmbeddingRequest {
|
||||
pub struct DeleteQdrantEdgeEmbeddingByFileRequest {
|
||||
pub store_name: String,
|
||||
pub file_path: String,
|
||||
pub parent_file_id: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct RelocateQdrantEdgeEmbeddingFileRequest {
|
||||
pub store_name: String,
|
||||
pub parent_file_id: String,
|
||||
pub absolute_path: String,
|
||||
pub file_name: String,
|
||||
pub relative_path: String,
|
||||
pub fingerprint: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -459,11 +470,17 @@ impl QdrantEdgeDatabase {
|
||||
.collect())
|
||||
}
|
||||
|
||||
fn delete_embedding_by_file(&mut self, store_name: &str, file_path: &str) -> QdrantEdgeResult<()> {
|
||||
fn delete_embedding_by_file(&mut self, store_name: &str, file_path: &str, parent_file_id: &str) -> QdrantEdgeResult<()> {
|
||||
let Some(shard) = self.get_existing_store(store_name)? else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
// The location may already have changed if a relocation stopped before SQLite caught up.
|
||||
if !parent_file_id.is_empty() {
|
||||
shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::DeletePointsByFilter(match_keyword_filter("parent_file_id", parent_file_id)?),
|
||||
))?;
|
||||
}
|
||||
shard.update(UpdateOperation::PointOperation(
|
||||
PointOperations::DeletePointsByFilter(match_keyword_filter("file_path", file_path)?),
|
||||
))?;
|
||||
@@ -471,6 +488,33 @@ impl QdrantEdgeDatabase {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn relocate_embedding_file(&mut self, request: RelocateQdrantEdgeEmbeddingFileRequest) -> QdrantEdgeResult<()> {
|
||||
if request.parent_file_id.is_empty() {
|
||||
return Err("Relocating a document requires its parent file id.".into());
|
||||
}
|
||||
let Some(shard) = self.get_existing_store(&request.store_name)? else {
|
||||
return Err("The vector store for the relocated document is missing.".into());
|
||||
};
|
||||
|
||||
let payload = json!({
|
||||
"file_path": request.absolute_path,
|
||||
"absolute_path": request.absolute_path,
|
||||
"file_name": request.file_name,
|
||||
"relative_path": request.relative_path,
|
||||
"fingerprint": request.fingerprint,
|
||||
});
|
||||
// Parent and chunk ids are identities, even when they were originally derived from a path.
|
||||
// Keep them and the vectors intact; only the document's location changes.
|
||||
shard.update(UpdateOperation::PayloadOperation(PayloadOps::SetPayload(SetPayloadOp {
|
||||
payload: qdrant_edge::external::serde_json::from_value(payload)?,
|
||||
points: None,
|
||||
filter: Some(match_keyword_filter("parent_file_id", &request.parent_file_id)?),
|
||||
key: None,
|
||||
})))?;
|
||||
shard.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn optimize_store(&mut self, store_name: &str) -> QdrantEdgeResult<()> {
|
||||
let Some(shard) = self.get_existing_store(store_name)? else {
|
||||
return Ok(());
|
||||
@@ -776,7 +820,13 @@ pub async fn search_qdrant_edge_embeddings(Json(request): Json<SearchQdrantEdgeE
|
||||
|
||||
pub async fn delete_qdrant_edge_embedding_by_file(Json(request): Json<DeleteQdrantEdgeEmbeddingByFileRequest>) -> Json<QdrantEdgeResponse<()>> {
|
||||
run_qdrant_edge_request(move |database| {
|
||||
database.delete_embedding_by_file(&request.store_name, &request.file_path)
|
||||
database.delete_embedding_by_file(&request.store_name, &request.file_path, &request.parent_file_id)
|
||||
}).await
|
||||
}
|
||||
|
||||
pub async fn relocate_qdrant_edge_embedding_file(Json(request): Json<RelocateQdrantEdgeEmbeddingFileRequest>) -> Json<QdrantEdgeResponse<()>> {
|
||||
run_qdrant_edge_request(move |database| {
|
||||
database.relocate_embedding_file(request)
|
||||
}).await
|
||||
}
|
||||
|
||||
@@ -1489,6 +1539,49 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn relocating_a_document_preserves_its_vectors_and_can_be_retried() {
|
||||
let test_directory = test_directory("relocation");
|
||||
let mut database = QdrantEdgeDatabase::new(test_directory.clone());
|
||||
let mut point = test_point(POINT_X, search_vector());
|
||||
point.parent_file_id = "original-parent".to_string();
|
||||
point.page_number = Some(2);
|
||||
database.insert_embedding(SEARCH_STORE, vec![point]).unwrap();
|
||||
|
||||
for _ in 0..2 {
|
||||
database.relocate_embedding_file(RelocateQdrantEdgeEmbeddingFileRequest {
|
||||
store_name: SEARCH_STORE.to_string(),
|
||||
parent_file_id: "original-parent".to_string(),
|
||||
absolute_path: "/tmp/moved/renamed.md".to_string(),
|
||||
file_name: "renamed.md".to_string(),
|
||||
relative_path: "renamed.md".to_string(),
|
||||
fingerprint: "new-fingerprint".to_string(),
|
||||
}).unwrap();
|
||||
}
|
||||
drop(database);
|
||||
|
||||
let mut database = QdrantEdgeDatabase::new(test_directory.clone());
|
||||
let found = database.search_embedding(SEARCH_STORE, search_vector(), 10, None).unwrap();
|
||||
assert_eq!(found.len(), 1);
|
||||
assert_eq!(found[0].point_id, POINT_X);
|
||||
assert_eq!(found[0].parent_file_id, "original-parent");
|
||||
assert_eq!(found[0].file_path, "/tmp/moved/renamed.md");
|
||||
assert_eq!(found[0].absolute_path, "/tmp/moved/renamed.md");
|
||||
assert_eq!(found[0].file_name, "renamed.md");
|
||||
assert_eq!(found[0].relative_path, "renamed.md");
|
||||
assert_eq!(found[0].fingerprint, "new-fingerprint");
|
||||
assert_eq!(found[0].text, "Some text.");
|
||||
assert_eq!(found[0].page_number, Some(2));
|
||||
assert!((found[0].score - 1.0).abs() < 0.0001);
|
||||
|
||||
// SQLite may still know the old path when a relocation was interrupted. Its stable parent
|
||||
// id must be enough to remove the vectors if the document changed before the next retry.
|
||||
database.delete_embedding_by_file(SEARCH_STORE, "/tmp/test-data/notes.md", "original-parent").unwrap();
|
||||
assert!(database.search_embedding(SEARCH_STORE, search_vector(), 10, None).unwrap().is_empty());
|
||||
drop(database);
|
||||
fs::remove_dir_all(test_directory).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn obsolete_wal_segments_keeps_the_newest_closed_segment() {
|
||||
let file_names = ["closed-120", "open-7", "closed-0", "tmp-open-8", "closed-40", ".wal", "first-index", "closed-abc"].map(String::from);
|
||||
@@ -1651,6 +1744,53 @@ mod tests {
|
||||
fs::remove_dir_all(data_directory).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_restored_store_can_relocate_foreign_paths_without_replacing_its_vectors() {
|
||||
for old_path in [r"E:\Documents\nested\notes.md", r"\\server\share\nested\notes.md", "/Volumes/External drive/nested/notes.md"] {
|
||||
let data_directory = test_directory("backup-relocation");
|
||||
let base_path = data_directory.join("databases").join("vector_database");
|
||||
let mut database = QdrantEdgeDatabase::new(base_path);
|
||||
let mut point = test_point(POINT_X, vec![1.0, 0.0, 0.0]);
|
||||
point.parent_file_id = "original-parent".to_string();
|
||||
point.file_path = old_path.to_string();
|
||||
point.absolute_path = old_path.to_string();
|
||||
point.embedded_at_utc = "2026-10-01T12:00:00Z".to_string();
|
||||
database.insert_embedding(SEARCH_STORE, vec![point]).unwrap();
|
||||
|
||||
let preparation = database.prepare_backup(&data_directory).unwrap();
|
||||
database.copy_store_for_backup(SEARCH_STORE, &data_directory, &preparation.backup_id).unwrap();
|
||||
let restored_base = PathBuf::from(&preparation.staging_path).join("databases").join("vector_database");
|
||||
let mut restored = QdrantEdgeDatabase::new(restored_base.clone());
|
||||
let new_path = data_directory.join("mounted drive").join("nested").join("notes.md").to_string_lossy().to_string();
|
||||
restored.relocate_embedding_file(RelocateQdrantEdgeEmbeddingFileRequest {
|
||||
store_name: SEARCH_STORE.to_string(),
|
||||
parent_file_id: "original-parent".to_string(),
|
||||
absolute_path: new_path.clone(),
|
||||
file_name: "notes.md".to_string(),
|
||||
relative_path: Path::new("nested").join("notes.md").to_string_lossy().to_string(),
|
||||
fingerprint: "relocated-fingerprint".to_string(),
|
||||
}).unwrap();
|
||||
drop(restored);
|
||||
|
||||
let mut restored = QdrantEdgeDatabase::new(restored_base);
|
||||
let found = restored.search_embedding(SEARCH_STORE, vec![1.0, 0.0, 0.0], 10, None).unwrap();
|
||||
assert_eq!(found.len(), 1);
|
||||
assert_eq!(found[0].point_id, POINT_X);
|
||||
assert_eq!(found[0].parent_file_id, "original-parent");
|
||||
assert_eq!(found[0].file_path, new_path);
|
||||
assert_eq!(found[0].absolute_path, new_path);
|
||||
assert_eq!(found[0].text, "Some text.");
|
||||
assert_eq!(found[0].embedded_at_utc, "2026-10-01T12:00:00Z");
|
||||
assert!((found[0].score - 1.0).abs() < 0.0001);
|
||||
|
||||
let original = database.search_embedding(SEARCH_STORE, vec![1.0, 0.0, 0.0], 10, None).unwrap();
|
||||
assert_eq!(original[0].file_path, old_path, "relocating a restored copy must not change the original store");
|
||||
drop(restored);
|
||||
drop(database);
|
||||
fs::remove_dir_all(data_directory).unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_backup_id_cannot_name_another_directory() {
|
||||
let data_directory = Path::new("/data");
|
||||
|
||||
@@ -42,6 +42,7 @@ fn create_router() -> Router {
|
||||
.route("/system/qdrant-edge/insert", post(crate::qdrant_edge_database::insert_qdrant_edge_embedding))
|
||||
.route("/system/qdrant-edge/search", post(crate::qdrant_edge_database::search_qdrant_edge_embeddings))
|
||||
.route("/system/qdrant-edge/delete-file", post(crate::qdrant_edge_database::delete_qdrant_edge_embedding_by_file))
|
||||
.route("/system/qdrant-edge/relocate-file", post(crate::qdrant_edge_database::relocate_qdrant_edge_embedding_file))
|
||||
.route("/system/qdrant-edge/optimize", post(crate::qdrant_edge_database::optimize_qdrant_edge_store))
|
||||
.route("/system/qdrant-edge/delete-store", post(crate::qdrant_edge_database::delete_qdrant_edge_store))
|
||||
.route("/system/qdrant-edge/backup/prepare", post(crate::qdrant_edge_database::prepare_qdrant_edge_backup))
|
||||
|
||||
Reference in new issue
Block a user