Allowed moving local data sources without embedding them again (#1091)

Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Paul KoudelkaandThorsten Sommer authored and GitHub committed 2026-10-20 18:50:09 +02:00
1 parent f6e48e95f3
commit 5d6e48ef93
49 files changed
+2280 -297

No files matched your search

@@ -4120,9 +4120,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "Yo
-- Automatic local data source refresh
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1208397349"] = "Automatic local data source refresh"
-- Edit Local Directory Data Source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1215599168"] = "Edit Local Directory Data Source"
-- Refresh
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T135637716"] = "Refresh"
@@ -4150,12 +4147,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2067030738"] = "Ar
-- Local Directory Data Source Information
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2146756020"] = "Local Directory Data Source Information"
-- Edit ERI v1 Data Source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T221059217"] = "Edit ERI v1 Data Source"
-- Edit Local File Data Source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2453292893"] = "Edit Local File Data Source"
-- Local data sources refresh when files change, mailboxes every 16 minutes.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2511157475"] = "Local data sources refresh when files change, mailboxes every 16 minutes."
@@ -6805,9 +6796,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2460883298"
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add"
-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2686150329"] = "The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed."
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
@@ -7066,9 +7054,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "T
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name"
-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T890383764"] = "The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed."
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T900713019"] = "Cancel"
@@ -10372,6 +10357,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T819555436"] = "Failed mails: {0}"
-- Skipped
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T852993367"] = "Skipped"
-- Edit this data source
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T878700886"] = "Edit this data source"
-- Open the transcription settings
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T905182399"] = "Open the transcription settings"
@@ -12208,18 +12196,39 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
-- Qdrant Edge is not available.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available."
-- Edit Local Directory Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T1215599168"] = "Edit Local Directory Data Source"
-- Edit ERI v1 Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T221059217"] = "Edit ERI v1 Data Source"
-- Edit Local File Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T2453292893"] = "Edit Local File Data Source"
-- Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T1555760974"] = "Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are."
-- AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2205693550"] = "AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in."
-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one."
-- and {0} more.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "and {0} more."
-- Your embedding provider runs in the cloud, so preparing files again costs money.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2635623258"] = "Your embedding provider runs in the cloud, so preparing files again costs money."
-- This change makes the prepared documents of the following data sources unusable ({0}):
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "This change makes the prepared documents of the following data sources unusable ({0}):"
-- Do you want to apply this change anyway?
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Do you want to apply this change anyway?"
-- New Location for This Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3462449568"] = "New Location for This Data Source"
-- Documents Will Be Prepared Again
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Documents Will Be Prepared Again"
@@ -13759,18 +13768,21 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T8042
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1214987337"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1393648320"] = "The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source."
-- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1935191670"] = "The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run."
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2268993048"] = "The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source."
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2692299965"] = "The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source."
-- The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2999277901"] = "The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source."
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3230000698"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it."
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3722338131"] = "The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source."
-- The file '{0}' could not be read. Please check whether you are allowed to read it.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3924882233"] = "The file '{0}' could not be read. Please check whether you are allowed to read it."
@@ -270,70 +270,8 @@ public partial class DataSourceManagement : MSGComponentBase
private async Task EditDataSource(IDataSource dataSource)
{
if (dataSource.IsEnterpriseConfiguration)
return;
IDataSource? editedDataSource = null;
var lockDataSourceOrigin = dataSource is IInternalDataSource
&& await this.DataSourceEmbeddingService.ShouldLockDataSourceOriginAsync(dataSource.Id);
switch (dataSource)
{
case DataSourceLocalFile localFile:
var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
{
{ x => x.IsEditing, true },
{ x => x.DataSource, localFile },
{ x => x.LockSource, lockDataSourceOrigin },
};
var localFileDialogReference = await this.DialogService.ShowAsync<DataSourceLocalFileDialog>(T("Edit Local File Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
var localFileDialogResult = await localFileDialogReference.Result;
if (localFileDialogResult is null || localFileDialogResult.Canceled)
return;
editedDataSource = (DataSourceLocalFile)localFileDialogResult.Data!;
break;
case DataSourceLocalDirectory localDirectory:
var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
{
{ x => x.IsEditing, true },
{ x => x.DataSource, localDirectory },
{ x => x.LockSource, lockDataSourceOrigin },
};
var localDirectoryDialogReference = await this.DialogService.ShowAsync<DataSourceLocalDirectoryDialog>(T("Edit Local Directory Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
return;
editedDataSource = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
break;
case DataSourceERI_V1 eriDataSource:
var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
{
{ x => x.IsEditing, true },
{ x => x.DataSource, eriDataSource },
};
var eriDialogReference = await this.DialogService.ShowAsync<DataSourceERI_V1Dialog>(T("Edit ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
var eriDialogResult = await eriDialogReference.Result;
if (eriDialogResult is null || eriDialogResult.Canceled)
return;
editedDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
break;
}
if(editedDataSource is null)
return;
this.SettingsManager.ConfigurationData.DataSources[this.SettingsManager.ConfigurationData.DataSources.IndexOf(dataSource)] = editedDataSource;
await this.SettingsManager.StoreSettings();
await this.DataSourceEmbeddingService.QueueDataSourceAsync(editedDataSource);
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
if (await DataSourceEditing.EditAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, dataSource.Id))
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
}
private async Task DeleteDataSource(IDataSource dataSource)
@@ -47,31 +47,7 @@
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("Select a root directory for this data source.")
</MudJustifiedText>
@if (!this.CanChangeSource)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
@T("The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed.")
</MudAlert>
}
@if (this.CanChangeSource)
{
<SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" EnableDragDrop="true" CatchAllDocuments="true" />
}
else
{
<MudTextField
T="string"
Text="@this.dataPath"
Label="@T("Selected base directory for this data source")"
Class="mb-3"
ReadOnly="@true"
Validation="@this.dataSourceValidation.ValidatePath"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.Folder"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
Variant="Variant.Outlined"
/>
}
<SelectDirectory @bind-Directory="@this.dataPath" Label="@T("Selected base directory for this data source")" DirectoryDialogTitle="@T("Select the base directory")" Validation="@this.dataSourceValidation.ValidatePath" EnableDragDrop="true" CatchAllDocuments="true" />
<MudTextSwitch
Label="@T("Include subfolders?")"
@@ -23,16 +23,6 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
[Parameter]
public DataSourceLocalDirectory DataSource { get; set; }
/// <summary>
/// Whether the folder this data source reads must stay as it is.
/// </summary>
/// <remarks>
/// Set once the index holds something for this data source. The embedding is not locked along
/// with it: it can be changed, and DataSourceReindexWarning asks what that costs.
/// </remarks>
[Parameter]
public bool LockSource { get; set; }
[Inject]
private IDialogService DialogService { get; init; } = null!;
@@ -215,8 +205,6 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
? T("Audio and video files which this data source selects are skipped, because your transcription provider does not meet the confidence level this data source requires.")
: null;
private bool CanChangeSource => !this.IsEditing || !this.LockSource;
private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
@@ -244,9 +232,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
Type = DataSourceType.LOCAL_DIRECTORY,
EmbeddingId = this.dataEmbeddingId,
// Kept out of reach of the form while the source is locked, so a stale field cannot point an
// indexed data source somewhere else:
Path = this.CanChangeSource ? this.dataPath : this.DataSource.Path,
Path = this.dataPath,
FilePatterns = this.NormalizeFilePatterns(),
// Kept as typed, since the line breaks group the entries for the eye:
@@ -268,6 +254,14 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
if (!this.dataIsValid)
return;
await this.StoreValidatedDataSourceAsync();
}
/// <summary>
/// Continues Save after form validation, asking before the draft is returned to its caller.
/// </summary>
internal async Task StoreValidatedDataSourceAsync()
{
var addedDataSource = this.CreateDataSource();
//
@@ -47,31 +47,7 @@
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("Select a file for this data source. The content of this file will be processed for the data source. An audio or video file is transcribed by your transcription provider first.")
</MudJustifiedText>
@if (!this.CanChangeSource)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-3">
@T("The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed.")
</MudAlert>
}
@if (this.CanChangeSource)
{
<SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" EnableDragDrop="true" CatchAllDocuments="true" />
}
else
{
<MudTextField
T="string"
Text="@this.dataFilePath"
Label="@T("Selected file path for this data source")"
Class="mb-3"
ReadOnly="@true"
Validation="@this.dataSourceValidation.ValidateFilePath"
Adornment="Adornment.Start"
AdornmentIcon="@Icons.Material.Filled.AttachFile"
UserAttributes="@SPELLCHECK_ATTRIBUTES"
Variant="Variant.Outlined"
/>
}
<SelectFile @bind-File="@this.dataFilePath" Label="@T("Selected file path for this data source")" FileDialogTitle="@T("Select the file")" Validation="@this.dataSourceValidation.ValidateFilePath" EnableDragDrop="true" CatchAllDocuments="true" />
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding provider.")
@@ -24,16 +24,6 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
[Parameter]
public DataSourceLocalFile DataSource { get; set; }
/// <summary>
/// Whether the file this data source reads must stay as it is.
/// </summary>
/// <remarks>
/// Set once the index holds something for this data source. The embedding is not locked along
/// with it: it can be changed, and DataSourceReindexWarning asks what that costs.
/// </remarks>
[Parameter]
public bool LockSource { get; set; }
[Inject]
private IDialogService DialogService { get; init; } = null!;
@@ -206,8 +196,6 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
? T("This audio or video file is skipped, because your transcription provider does not meet the confidence level this data source requires.")
: null;
private bool CanChangeSource => !this.IsEditing || !this.LockSource;
private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ConfidenceLevels => ConfigurationSelectDataFactory.GetDataSourceConfidenceLevelsData();
private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
@@ -235,9 +223,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
Type = DataSourceType.LOCAL_FILE,
EmbeddingId = this.dataEmbeddingId,
// Kept out of reach of the form while the source is locked, so a stale field cannot point an
// indexed data source somewhere else:
FilePath = this.CanChangeSource ? this.dataFilePath : this.DataSource.FilePath,
FilePath = this.dataFilePath,
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
ConfidenceLevel = this.dataConfidenceLevel,
@@ -252,6 +238,14 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
if (!this.dataIsValid)
return;
await this.StoreValidatedDataSourceAsync();
}
/// <summary>
/// Continues Save after form validation, asking before the draft is returned to its caller.
/// </summary>
internal async Task StoreValidatedDataSourceAsync()
{
var addedDataSource = this.CreateDataSource();
//
@@ -296,7 +296,7 @@
@*
What the user has to decide comes with the buttons to decide it. The
sentence above says what is at stake; reading it and then having to find
the settings of the mailbox is where people give up.
the settings of the mailbox or the data source is where people give up.
*@
@if (status.Attention is DataSourceAttention.AUTH_FAILED)
{
@@ -320,6 +320,14 @@
</MudButton>
</MudStack>
}
else if (status.Attention is DataSourceAttention.LOCATION_UNREACHABLE && this.CanEditDataSource(status))
{
<MudStack Row="@true" Wrap="Wrap.Wrap" Spacing="2">
<MudButton Variant="Variant.Filled" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Edit" OnClick="@(async () => await this.EditDataSource(status))">
@T("Edit this data source")
</MudButton>
</MudStack>
}
</MudStack>
</ChildContent>
</MudExpansionPanel>
@@ -509,6 +509,25 @@ public partial class Embeddings : MSGComponentBase
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
}
/// <remarks>
/// Not offered for a data source the organization manages: its settings cannot be edited.
/// </remarks>
private bool CanEditDataSource(DataSourceEmbeddingStatus status) => DataSourceEditing.CanEdit(this.SettingsManager, status.DataSourceId);
/// <summary>
/// Opens the settings of a data source, e.g. to select the new location of a moved folder.
/// </summary>
/// <remarks>
/// Saving them indexes the data source right away, and that run reconnects what is already
/// stored to the new location. The page reloads through CONFIGURATION_CHANGED, which it
/// receives like everybody else.
/// </remarks>
private async Task EditDataSource(DataSourceEmbeddingStatus status)
{
if (await DataSourceEditing.EditAsync(this.DialogService, this.SettingsManager, this.DataSourceEmbeddingService, status.DataSourceId))
await this.MessageBus.SendMessage<bool>(this, Event.CONFIGURATION_CHANGED);
}
/// <summary>
/// Agrees to the removal a mailbox held back, for the very number the user read.
/// </summary>
@@ -4122,9 +4122,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "Si
-- Automatic local data source refresh
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1208397349"] = "Automatische Aktualisierung lokaler Datenquellen"
-- Edit Local Directory Data Source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1215599168"] = "Datenquelle bearbeiten: Lokaler Ordner"
-- Refresh
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T135637716"] = "Aktualisieren"
@@ -4152,12 +4149,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2067030738"] = "M
-- Local Directory Data Source Information
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2146756020"] = "Informationen zur lokalen Ordner-Datenquelle"
-- Edit ERI v1 Data Source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T221059217"] = "ERI v1 Datenquelle bearbeiten"
-- Edit Local File Data Source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2453292893"] = "Datenquelle bearbeiten: Lokale Datei"
-- Local data sources refresh when files change, mailboxes every 16 minutes.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2511157475"] = "Lokale Datenquellen werden aktualisiert, wenn sich Dateien ändern, Postfächer alle 16 Minuten."
@@ -6807,9 +6798,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2460883298"
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Hinzufügen"
-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2686150329"] = "Die Dokumente dieser Datenquelle sind bereits vorbereitet, daher kann ihr Ordner nicht geändert werden. Ein anderer Ordner enthält andere Dokumente und ist damit eine andere Datenquelle: Bitte fügen Sie dafür eine neue hinzu. Den Einbettungsanbieter darunter können Sie ändern."
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "Die von Ihnen ausgewählte Einbettung läuft lokal oder innerhalb Ihrer Organisation. Ihre Daten werden nicht in die Cloud übertragen."
@@ -7068,9 +7056,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "D
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Name der Datenquelle"
-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T890383764"] = "Die Dokumente dieser Datenquelle sind bereits vorbereitet, daher kann ihre Datei nicht geändert werden. Eine andere Datei enthält andere Inhalte und ist damit eine andere Datenquelle: Bitte fügen Sie dafür eine neue hinzu. Den Einbettungsanbieter darunter können Sie ändern."
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T900713019"] = "Abbrechen"
@@ -10374,6 +10359,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T819555436"] = "Fehlgeschlagene E-
-- Skipped
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T852993367"] = "Übersprungen"
-- Edit this data source
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T878700886"] = "Diese Datenquelle bearbeiten"
-- Open the transcription settings
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T905182399"] = "Transkriptionseinstellungen öffnen"
@@ -12210,18 +12198,39 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
-- Qdrant Edge is not available.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge ist nicht verfügbar."
-- Edit Local Directory Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T1215599168"] = "Datenquelle bearbeiten: Lokaler Ordner"
-- Edit ERI v1 Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T221059217"] = "ERI v1 Datenquelle bearbeiten"
-- Edit Local File Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T2453292893"] = "Datenquelle bearbeiten: Lokale Datei"
-- Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T1555760974"] = "Jede andere Datei wird erneut an Ihren Einbettungsanbieter gesendet, was bei vielen Dateien eine Weile dauert. Dateien, die am neuen Speicherort fehlen, werden aus dem Index entfernt, während Ihre Dateien selbst unverändert bleiben."
-- AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2205693550"] = "Was AI Studio bereits vorbereitet hat, bleibt für jede Datei erhalten, die es am neuen Speicherort unverändert findet, also mit demselben Dateityp, derselben Größe und demselben Änderungsdatum. In einem Ordner muss eine solche Datei außerdem ihren Namen behalten und im selben Unterordner liegen."
-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "Sie beantworten weiterhin Stichwortsuchen, aber die Suche nach Bedeutung funktioniert nicht mehr, und es können keine weiteren Dokumente für sie vorbereitet werden. Bereits vorbereitete Datenquellen bleiben zudem an diesen Anbieter gebunden, sodass Sie sie nicht einfach auf einen anderen umstellen können."
-- and {0} more.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "und {0} weitere."
-- Your embedding provider runs in the cloud, so preparing files again costs money.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2635623258"] = "Ihr Einbettungsanbieter läuft in der Cloud, daher kostet es Geld, Dateien erneut vorzubereiten."
-- This change makes the prepared documents of the following data sources unusable ({0}):
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "Durch diese Änderung werden die vorbereiteten Dokumente der folgenden Datenquellen unbrauchbar ({0}):"
-- Do you want to apply this change anyway?
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Möchten Sie diese Änderung trotzdem übernehmen?"
-- New Location for This Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3462449568"] = "Neuer Speicherort für diese Datenquelle"
-- Documents Will Be Prepared Again
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Dokumente werden erneut vorbereitet"
@@ -13761,18 +13770,21 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T8042
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1214987337"] = "Der Ordner „{0}“ konnte nicht geöffnet werden. Bitte prüfen Sie, ob Sie die erforderlichen Leserechte haben. Aus dem Index wurde nichts entfernt."
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1393648320"] = "Die Datei „{0}“ ist nicht erreichbar. Möglicherweise ist das Laufwerk, auf dem sie sich befindet, nicht verbunden, oder die Datei wurde verschoben oder umbenannt. Aus dem Index wurde nichts entfernt. Sobald die Datei wieder verfügbar ist, aktualisieren Sie diese Datenquelle. Wenn sie dauerhaft nicht mehr vorhanden ist, entfernen Sie diese Datenquelle."
-- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1935191670"] = "Die Datei „{0}“ wurde während der Indexierung geändert. Die bereits indexierten Inhalte werden verworfen, und die Datei wird beim nächsten Durchlauf erneut verarbeitet."
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2268993048"] = "Die Datei „{0}“ ist nicht erreichbar. Möglicherweise ist das Laufwerk, auf dem sie sich befindet, nicht verbunden, oder die Datei wurde verschoben oder umbenannt. Aus dem Index wurde nichts entfernt. Wenn die Datei verschoben oder umbenannt wurde, bearbeiten Sie diese Datenquelle und wählen Sie den neuen Speicherort der Datei aus. Sobald die Datei wieder verfügbar ist, aktualisieren Sie diese Datenquelle. Wenn sie dauerhaft nicht mehr vorhanden ist, entfernen Sie diese Datenquelle."
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2692299965"] = "Der Ordner „{0}“ ist nicht erreichbar. Möglicherweise ist das Laufwerk, auf dem er sich befindet, nicht verbunden, oder der Ordner wurde verschoben oder umbenannt. Aus dem Index wurde nichts entfernt. Wenn der Ordner verschoben oder umbenannt wurde, bearbeiten Sie diese Datenquelle und wählen Sie den neuen Speicherort des Ordners aus. Sobald der Ordner wieder verfügbar ist, aktualisieren Sie diese Datenquelle."
-- The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2999277901"] = "Der Ordner „{0}“ scheint derzeit leer zu sein. AI Studio hat den Index dieser Datenquelle beibehalten, da ein Ordner auf einem nicht verbundenen Laufwerk leer erscheinen kann. Verbinden Sie das Laufwerk und aktualisieren Sie diese Datenquelle. Wenn Sie den Ordner absichtlich geleert haben, entfernen Sie diese Datenquelle."
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3230000698"] = "Der Ordner „{0}“ konnte nicht geöffnet werden. Bitte prüfen Sie, ob Sie ihn lesen dürfen."
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3722338131"] = "Der Ordner „{0}“ ist nicht erreichbar. Möglicherweise ist das Laufwerk, auf dem er sich befindet, nicht verbunden, oder der Ordner wurde verschoben oder umbenannt. Aus dem Index wurde nichts entfernt. Sobald der Ordner wieder verfügbar ist, aktualisieren Sie diese Datenquelle."
-- The file '{0}' could not be read. Please check whether you are allowed to read it.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3924882233"] = "Die Datei „{0}“ konnte nicht gelesen werden. Bitte prüfen Sie, ob Sie berechtigt sind, sie zu lesen."
@@ -4122,9 +4122,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1084943026"] = "Yo
-- Automatic local data source refresh
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1208397349"] = "Automatic local data source refresh"
-- Edit Local Directory Data Source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T1215599168"] = "Edit Local Directory Data Source"
-- Refresh
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T135637716"] = "Refresh"
@@ -4152,12 +4149,6 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2067030738"] = "Ar
-- Local Directory Data Source Information
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2146756020"] = "Local Directory Data Source Information"
-- Edit ERI v1 Data Source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T221059217"] = "Edit ERI v1 Data Source"
-- Edit Local File Data Source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2453292893"] = "Edit Local File Data Source"
-- Local data sources refresh when files change, mailboxes every 16 minutes.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATASOURCEMANAGEMENT::T2511157475"] = "Local data sources refresh when files change, mailboxes every 16 minutes."
@@ -6807,9 +6798,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2460883298"
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add"
-- The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2686150329"] = "The documents of this data source are already prepared, so its folder cannot be changed. Another folder holds other documents, which makes it another data source: please add one for it. The embedding provider below can be changed."
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
@@ -7068,9 +7056,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T787118522"] = "T
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name"
-- The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T890383764"] = "The documents of this data source are already prepared, so its file cannot be changed. Another file holds other content, which makes it another data source: please add one for it. The embedding provider below can be changed."
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T900713019"] = "Cancel"
@@ -10374,6 +10359,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T819555436"] = "Failed mails: {0}"
-- Skipped
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T852993367"] = "Skipped"
-- Edit this data source
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T878700886"] = "Edit this data source"
-- Open the transcription settings
UI_TEXT_CONTENT["AISTUDIO::PAGES::EMBEDDINGS::T905182399"] = "Open the transcription settings"
@@ -12210,18 +12198,39 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEM
-- Qdrant Edge is not available.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::QDRANTEDGECLIENTIMPLEMENTATION::T744445696"] = "Qdrant Edge is not available."
-- Edit Local Directory Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T1215599168"] = "Edit Local Directory Data Source"
-- Edit ERI v1 Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T221059217"] = "Edit ERI v1 Data Source"
-- Edit Local File Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEEDITING::T2453292893"] = "Edit Local File Data Source"
-- Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T1555760974"] = "Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are."
-- AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2205693550"] = "AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in."
-- They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2343773457"] = "They keep answering keyword searches, but searching them by meaning stops working, and no further documents can be prepared for them. The ones which are already prepared stay tied to this provider as well, so you cannot simply move them to another one."
-- and {0} more.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2519847121"] = "and {0} more."
-- Your embedding provider runs in the cloud, so preparing files again costs money.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T2635623258"] = "Your embedding provider runs in the cloud, so preparing files again costs money."
-- This change makes the prepared documents of the following data sources unusable ({0}):
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3337378891"] = "This change makes the prepared documents of the following data sources unusable ({0}):"
-- Do you want to apply this change anyway?
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3419411838"] = "Do you want to apply this change anyway?"
-- New Location for This Data Source
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T3462449568"] = "New Location for This Data Source"
-- Documents Will Be Prepared Again
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATASOURCEREINDEXWARNING::T737291513"] = "Documents Will Be Prepared Again"
@@ -13761,18 +13770,21 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::DOCUMENTRUNPROGRESS::T8042
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1214987337"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1393648320"] = "The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source."
-- The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T1935191670"] = "The file '{0}' changed while it was being indexed. What was indexed of it is discarded, and the file is tried again during the next run."
-- The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2268993048"] = "The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source."
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2692299965"] = "The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source."
-- The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T2999277901"] = "The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source."
-- The folder '{0}' could not be opened. Please check whether you are allowed to read it.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3230000698"] = "The folder '{0}' could not be opened. Please check whether you are allowed to read it."
-- The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3722338131"] = "The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source."
-- The file '{0}' could not be read. Please check whether you are allowed to read it.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::INDEXING::FILESOURCEINDEXER::T3924882233"] = "The file '{0}' could not be read. Please check whether you are allowed to read it."
@@ -0,0 +1,115 @@
using AIStudio.Dialogs;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Services;
using DialogOptions = AIStudio.Dialogs.DialogOptions;
namespace AIStudio.Tools;
/// <summary>
/// Opens the settings of a data source, and stores what the user changed in them.
/// </summary>
/// <remarks>
/// Kept here rather than in the places which offer it -- the data source table and the background
/// embeddings page, which leads to it when the file or the folder of a data source cannot be
/// reached -- so that saving a data source works the same way everywhere: the settings are stored,
/// and the data source is indexed right after. Mailboxes are kept in a list of their own and have
/// MailboxEditing instead.
/// </remarks>
public static class DataSourceEditing
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(DataSourceEditing).Namespace, nameof(DataSourceEditing));
/// <summary>
/// Whether the user may edit a data source.
/// </summary>
/// <param name="settingsManager">The settings, which hold the data source.</param>
/// <param name="dataSourceId">The id of the data source.</param>
/// <returns>False when the data source is gone or managed by the organization.</returns>
public static bool CanEdit(SettingsManager settingsManager, string dataSourceId) =>
settingsManager.ConfigurationData.DataSources.Any(dataSource => dataSource.Id == dataSourceId && !dataSource.IsEnterpriseConfiguration);
/// <summary>
/// Lets the user edit a data source, and stores and indexes it when they save.
/// </summary>
/// <remarks>
/// The caller tells the rest of the app that the configuration changed, as the sender of that message.
/// </remarks>
/// <param name="dialogService">The dialog service to show the settings with.</param>
/// <param name="settingsManager">The settings, which hold the data source.</param>
/// <param name="embeddingService">The service which indexes the data source.</param>
/// <param name="dataSourceId">The id of the data source.</param>
/// <returns>True when the user saved the data source. False when they cancelled, or the data source is gone or managed by the organization.</returns>
public static async Task<bool> EditAsync(IDialogService dialogService, SettingsManager settingsManager, DataSourceEmbeddingService embeddingService, string dataSourceId)
{
var dataSources = settingsManager.ConfigurationData.DataSources;
var storedDataSource = dataSources.Find(dataSource => dataSource.Id == dataSourceId);
if (storedDataSource is null || storedDataSource.IsEnterpriseConfiguration)
return false;
IDataSource? editedDataSource = null;
switch (storedDataSource)
{
case DataSourceLocalFile localFile:
var localFileDialogParameters = new DialogParameters<DataSourceLocalFileDialog>
{
{ x => x.IsEditing, true },
{ x => x.DataSource, localFile },
};
var localFileDialogReference = await dialogService.ShowAsync<DataSourceLocalFileDialog>(TB("Edit Local File Data Source"), localFileDialogParameters, DialogOptions.FULLSCREEN);
var localFileDialogResult = await localFileDialogReference.Result;
if (localFileDialogResult is null || localFileDialogResult.Canceled)
return false;
editedDataSource = (DataSourceLocalFile)localFileDialogResult.Data!;
break;
case DataSourceLocalDirectory localDirectory:
var localDirectoryDialogParameters = new DialogParameters<DataSourceLocalDirectoryDialog>
{
{ x => x.IsEditing, true },
{ x => x.DataSource, localDirectory },
};
var localDirectoryDialogReference = await dialogService.ShowAsync<DataSourceLocalDirectoryDialog>(TB("Edit Local Directory Data Source"), localDirectoryDialogParameters, DialogOptions.FULLSCREEN);
var localDirectoryDialogResult = await localDirectoryDialogReference.Result;
if (localDirectoryDialogResult is null || localDirectoryDialogResult.Canceled)
return false;
editedDataSource = (DataSourceLocalDirectory)localDirectoryDialogResult.Data!;
break;
case DataSourceERI_V1 eriDataSource:
var eriDialogParameters = new DialogParameters<DataSourceERI_V1Dialog>
{
{ x => x.IsEditing, true },
{ x => x.DataSource, eriDataSource },
};
var eriDialogReference = await dialogService.ShowAsync<DataSourceERI_V1Dialog>(TB("Edit ERI v1 Data Source"), eriDialogParameters, DialogOptions.FULLSCREEN);
var eriDialogResult = await eriDialogReference.Result;
if (eriDialogResult is null || eriDialogResult.Canceled)
return false;
editedDataSource = (DataSourceERI_V1)eriDialogResult.Data!;
break;
}
if (editedDataSource is null)
return false;
// Found anew, since the list may have changed while the dialog was open:
var position = dataSources.IndexOf(storedDataSource);
if (position < 0)
return false;
dataSources[position] = editedDataSource;
await settingsManager.StoreSettings();
await embeddingService.QueueDataSourceAsync(editedDataSource);
return true;
}
}
@@ -2,6 +2,7 @@ using System.Text;
using AIStudio.Dialogs;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Services;
@@ -19,6 +20,10 @@ namespace AIStudio.Tools;
/// Nothing is asked when nothing is lost. A data source only reaches the question when the edit
/// really changes its embedding signature and when the index already holds something for it, so
/// renaming an embedding provider or editing a data source nobody has indexed yet stays silent.
/// A new location of a local file or folder has a question of its own, under the same condition of
/// a stored index: the signature stays, but every file which is not found unchanged at the new
/// location is prepared again. When the signature changes as well, only its question is asked,
/// since everything is prepared again anyway.
/// </remarks>
public static class DataSourceReindexWarning
{
@@ -84,11 +89,60 @@ public static class DataSourceReindexWarning
DataSourceEmbeddingProviders.TryResolve(settingsManager, before, out var resolvedBeforeProvider);
var beforeProvider = resolvedBeforeProvider ?? EmbeddingProvider.NONE;
if (!EmbeddingChangeImpact.AffectsStoredIndex(before, beforeProvider, after, afterProvider))
// Everything is prepared again, wherever the files are, so this question covers a new location as well:
if (EmbeddingChangeImpact.AffectsStoredIndex(before, beforeProvider, after, afterProvider))
{
var affected = await embeddingService.GetDataSourcesWithStoredIndexAsync([after], token);
return await ConfirmAsync(dialogService, affected, !afterProvider.IsSelfHosted);
}
if (!HasLocationChanged(before, after))
return true;
var affected = await embeddingService.GetDataSourcesWithStoredIndexAsync([after], token);
return await ConfirmAsync(dialogService, affected, !afterProvider.IsSelfHosted);
var indexed = await embeddingService.GetDataSourcesWithStoredIndexAsync([after], token);
return indexed.Count == 0 || await ConfirmNewLocationAsync(dialogService, !afterProvider.IsSelfHosted);
}
private static bool HasLocationChanged(IIndexedDataSource before, IIndexedDataSource after) => (before, after) switch
{
(DataSourceLocalDirectory oldFolder, DataSourceLocalDirectory newFolder) => !string.Equals(oldFolder.Path, newFolder.Path, StringComparison.Ordinal),
(DataSourceLocalFile oldFile, DataSourceLocalFile newFile) => !string.Equals(oldFile.FilePath, newFile.FilePath, StringComparison.Ordinal),
_ => false,
};
/// <summary>
/// Asks before a local file or folder with a stored index is saved with a new location.
/// </summary>
/// <remarks>
/// The conditions for keeping a file are the ones FileSourceIndexer.FindRelocatedFile checks.
/// </remarks>
/// <param name="dialogService">The dialog service to ask with.</param>
/// <param name="usesCloudEmbedding">Whether preparing a file again costs money.</param>
/// <returns>True when the edit may be saved.</returns>
private static async Task<bool> ConfirmNewLocationAsync(IDialogService dialogService, bool usesCloudEmbedding)
{
var body = new StringBuilder();
body.AppendLine(TB("AI Studio keeps what it has already prepared for every file it finds unchanged at the new location, that is, of the same type and with the same size and modification time. In a folder, such a file also has to keep its name and the subfolder it is in."));
body.AppendLine();
body.AppendLine(TB("Every other file goes to your embedding provider once more, which takes a while with many files. Files missing at the new location are removed from the index, while your files on disk stay as they are."));
if (usesCloudEmbedding)
{
body.AppendLine();
body.AppendLine(TB("Your embedding provider runs in the cloud, so preparing files again costs money."));
}
body.AppendLine();
body.AppendLine(TB("Do you want to apply this change anyway?"));
var dialogParameters = new DialogParameters<ConfirmDialog>
{
{ x => x.MarkdownBody, body.ToString() },
};
var dialogReference = await dialogService.ShowAsync<ConfirmDialog>(TB("New Location for This Data Source"), dialogParameters, Dialogs.DialogOptions.FULLSCREEN);
var dialogResult = await dialogReference.Result;
return dialogResult is not null && !dialogResult.Canceled;
}
/// <summary>
@@ -192,4 +246,4 @@ public static class DataSourceReindexWarning
var dialogResult = await dialogReference.Result;
return dialogResult is not null && !dialogResult.Canceled;
}
}
}
@@ -29,6 +29,11 @@ public abstract partial class IndexStoreClient(string name, string path) : Datab
public abstract Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token);
/// <summary>
/// Updates the location and fingerprint of a document, preserving its chunks and their ids.
/// </summary>
public abstract Task RelocateFileAsync(string dataSourceId, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token);
public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token);
public abstract Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token);
@@ -52,6 +52,9 @@ public sealed partial class NoIndexStoreClient(string name, string? unavailableR
public override Task UpsertFileAsync(string dataSourceId, EmbeddingStateFile file, CancellationToken token) => Task.CompletedTask;
public override Task RelocateFileAsync(string dataSourceId, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token) =>
Task.FromException(new InvalidOperationException(unavailableReason ?? "The index of the data sources is not available."));
public override Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask;
public override Task UpsertPermanentFailureAsync(string dataSourceId, PermanentIndexingFailure failure, CancellationToken token) => Task.CompletedTask;
@@ -149,7 +149,9 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
file.LastWriteUtc,
file.EmbeddedAtUtc,
file.ChunkCount,
file.TranscriptKey);
file.TranscriptKey,
file.RelativePath,
file.ParentFileId);
}
var permanentFailures = await context.PermanentIndexingFailures
@@ -233,6 +235,18 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
await context.SaveChangesAsync(token);
}
public override async Task RelocateFileAsync(string dataSourceId, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token)
{
await using var context = this.CreateContext();
var file = await context.EmbeddedFiles.SingleAsync(file => file.DataSourceId == dataSourceId && file.ParentFileId == parentFileId, token);
// Keep the parent and chunk ids: moving a document does not change its stored vectors.
file.AbsolutePath = absolutePath;
file.RelativePath = relativePath;
file.FileName = Path.GetFileName(absolutePath);
file.Fingerprint = fingerprint;
await context.SaveChangesAsync(token);
}
public override async Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token)
{
await using var context = this.CreateContext();
@@ -762,4 +776,4 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
client.SetLogger(databaseClientLogger);
return client;
}
}
}
@@ -37,7 +37,10 @@ public sealed class NoVectorStoreClient(string name, string? unavailableReason,
public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, CancellationToken token) =>
Task.FromException<IReadOnlyList<VectorSearchResult>>(this.CreateUnavailableException());
public override Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token) =>
public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token) =>
Task.FromException(this.CreateUnavailableException());
public override Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token) =>
Task.FromException(this.CreateUnavailableException());
public override Task OptimizeVectorStore(string storeName, CancellationToken token) =>
@@ -27,6 +27,7 @@ public sealed class QdrantEdgeClientImplementation(
private const string INSERT_PATH = "/system/qdrant-edge/insert";
private const string SEARCH_PATH = "/system/qdrant-edge/search";
private const string DELETE_FILE_PATH = "/system/qdrant-edge/delete-file";
private const string RELOCATE_FILE_PATH = "/system/qdrant-edge/relocate-file";
private const string OPTIMIZE_PATH = "/system/qdrant-edge/optimize";
private const string DELETE_STORE_PATH = "/system/qdrant-edge/delete-store";
private const string PREPARE_BACKUP_PATH = "/system/qdrant-edge/backup/prepare";
@@ -156,8 +157,11 @@ public sealed class QdrantEdgeClientImplementation(
token) ?? [];
}
public override Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token) =>
rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_FILE_PATH, new DeleteEmbeddingByFileRequest(storeName, filePath), token);
public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token) =>
rustService.ExecuteDatabaseOperation(DATABASE_NAME, DELETE_FILE_PATH, new DeleteEmbeddingByFileRequest(storeName, filePath, parentFileId), token);
public override Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token) =>
rustService.ExecuteDatabaseOperation(DATABASE_NAME, RELOCATE_FILE_PATH, new RelocateEmbeddingFileRequest(storeName, parentFileId, absolutePath, Path.GetFileName(absolutePath), relativePath, fingerprint), token);
public override Task OptimizeVectorStore(string storeName, CancellationToken token) =>
rustService.ExecuteDatabaseOperation(DATABASE_NAME, OPTIMIZE_PATH, new OptimizeVectorStoreRequest(storeName), token);
@@ -194,7 +198,9 @@ public sealed class QdrantEdgeClientImplementation(
/// <param name="PointIds">The only points the search may return, or null for the whole store.</param>
private sealed record SearchEmbeddingRequest(string StoreName, IReadOnlyList<float> Vector, int MaxMatches, IReadOnlyList<string>? PointIds);
private sealed record DeleteEmbeddingByFileRequest(string StoreName, string FilePath);
private sealed record DeleteEmbeddingByFileRequest(string StoreName, string FilePath, string ParentFileId);
private sealed record RelocateEmbeddingFileRequest(string StoreName, string ParentFileId, string AbsolutePath, string FileName, string RelativePath, string Fingerprint);
private sealed record OptimizeVectorStoreRequest(string StoreName);
@@ -27,7 +27,12 @@ public abstract class VectorStoreClient(string name, string path): DatabaseClien
/// <returns>The matches, best first, all of them inside the filter.</returns>
public abstract Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, CancellationToken token);
public abstract Task DeleteEmbeddingByFile(string storeName, string filePath, CancellationToken token);
public abstract Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token);
/// <summary>
/// Updates the location of a document's points without changing their ids or vectors.
/// </summary>
public abstract Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token);
public abstract Task OptimizeVectorStore(string storeName, CancellationToken token);
@@ -43,7 +43,7 @@ public static class MailboxEditing
{
{ x => x.IsEditing, true },
{ x => x.DataSource, storedMailbox },
{ x => x.LockSource, await embeddingService.ShouldLockDataSourceOriginAsync(storedMailbox.Id) },
{ x => x.LockSource, await embeddingService.ShouldLockMailboxOriginAsync(storedMailbox.Id) },
};
var dialogReference = await dialogService.ShowAsync<DataSourceMailboxDialog>(TB("Edit Mailbox"), dialogParameters, DialogOptions.FULLSCREEN);
+27 -1
View File
@@ -1,7 +1,7 @@
namespace AIStudio.Tools;
/// <summary>
/// Shared rules for comparing file system paths.
/// Shared rules for comparing and resolving file system paths.
/// </summary>
public static class PathTools
{
@@ -42,4 +42,30 @@ public static class PathTools
return false;
}
}
/// <summary>
/// Finds the place where a file dialog opens for a path which may no longer exist.
/// </summary>
/// <remarks>
/// Given a file or a folder which was moved or renamed, the native dialog cannot open its place
/// and falls back to wherever it was used last. The nearest folder which still exists is the
/// better start: the file or the folder was moved from there, so its new place is often close by.
/// A relative path, or a path of another operating system, e.g. from a backup, has no such folder.
/// </remarks>
/// <param name="path">The file or the folder the dialog should open at.</param>
/// <returns>The path itself when it exists, else its nearest parent folder which exists, or null when there is none.</returns>
public static string? FindNearestExistingPath(string path)
{
if (string.IsNullOrWhiteSpace(path) || !Path.IsPathFullyQualified(path))
return null;
if (File.Exists(path) || Directory.Exists(path))
return path;
for (var directory = Path.GetDirectoryName(path); !string.IsNullOrEmpty(directory); directory = Path.GetDirectoryName(directory))
if (Directory.Exists(directory))
return directory;
return null;
}
}
@@ -1,6 +1,5 @@
using System.Text;
using AIStudio.Chat;
using AIStudio.Tools.Security;
namespace AIStudio.Tools.RAG;
@@ -24,4 +24,9 @@ public enum DataSourceAttention
/// The organization allows only its own mail servers, and the mailbox is on another one. AI Studio does not connect to it until the organization allows the server.
/// </summary>
SERVER_NOT_ALLOWED,
/// <summary>
/// The file or the folder of a local data source cannot be reached, cannot be opened, or appears empty. AI Studio keeps the index until the file or the folder is back, the user selects its new location, or removes the data source.
/// </summary>
LOCATION_UNREACHABLE,
}
@@ -1,7 +1,27 @@
namespace AIStudio.Tools.Services;
/// <summary>
/// What the index store holds for one data source, read at the start of a run.
/// </summary>
public sealed class DataSourceEmbeddingManifest
{
/// <summary>
/// How the index compares the paths of the documents it stores, on every OS.
/// </summary>
/// <remarks>
/// Ignores case, unlike PathTools.COMPARER, which follows the file system. The index store keeps
/// the paths with the NOCASE collation and allows each path of a data source once, so two files
/// which differ in their case alone share one row there. Whatever the indexer keys or compares by
/// these paths has to agree with the store; telling the two files apart would let them replace
/// each other's row on every run.
/// </remarks>
public static readonly StringComparer PATH_COMPARER = StringComparer.OrdinalIgnoreCase;
/// <summary>
/// The comparison matching PATH_COMPARER, e.g., for prefixes.
/// </summary>
public const StringComparison PATH_COMPARISON = StringComparison.OrdinalIgnoreCase;
public string EmbeddingProviderId { get; set; } = string.Empty;
public string EmbeddingSignature { get; set; } = string.Empty;
@@ -10,7 +30,7 @@ public sealed class DataSourceEmbeddingManifest
public int VectorSize { get; set; }
public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(StringComparer.OrdinalIgnoreCase);
public Dictionary<string, EmbeddedFileRecord> Files { get; init; } = new(PATH_COMPARER);
/// <summary>
/// The files whose indexing began but never came to an end, by their absolute path.
@@ -21,7 +41,7 @@ public sealed class DataSourceEmbeddingManifest
/// meanwhile. Indexing such a file again replaces both. A file which is gone, or which the data
/// source no longer selects, is never indexed again, so the indexer has to remove them itself.
/// </remarks>
public HashSet<string> UnfinishedFiles { get; init; } = new(StringComparer.OrdinalIgnoreCase);
public HashSet<string> UnfinishedFiles { get; init; } = new(PATH_COMPARER);
/// <summary>
/// The files whose indexing failed for a reason which lies in the file itself, keyed by their
@@ -32,5 +52,5 @@ public sealed class DataSourceEmbeddingManifest
/// a folder holding hundreds of scanned documents without a text layer would be read again on
/// every single run, with the outcome known in advance.
/// </remarks>
public Dictionary<string, PermanentIndexingFailureRecord> PermanentFailures { get; init; } = new(StringComparer.OrdinalIgnoreCase);
public Dictionary<string, PermanentIndexingFailureRecord> PermanentFailures { get; init; } = new(PATH_COMPARER);
}
@@ -158,14 +158,18 @@ public sealed partial class DataSourceEmbeddingService
/// files which no longer belong to the data source, and the ones which now do have to wait for
/// the next run.
///
/// Only a local folder chooses its documents by file patterns, exclusions, switches, and a size
/// limit. For every other kind of data source, this is empty.
/// Local files and folders include their configured paths, so relocating one stops a run which
/// still reads the old location. Folders also include their file selection. Neither changes the
/// embedding signature: unchanged documents keep their vectors.
/// </remarks>
/// <param name="dataSource">The data source to index.</param>
/// <returns>The description of its selection, or an empty string.</returns>
internal static string BuildSelectionSignature(IIndexedDataSource dataSource) => dataSource is DataSourceLocalDirectory directory
? FolderFileSelection.Of(directory).Describe()
: string.Empty;
internal static string BuildSelectionSignature(IIndexedDataSource dataSource) => dataSource switch
{
DataSourceLocalDirectory directory => $"{directory.Path.Length}:{directory.Path}|{FolderFileSelection.Of(directory).Describe()}",
DataSourceLocalFile file => $"{file.FilePath.Length}:{file.FilePath}",
_ => string.Empty,
};
/// <summary>
/// Describes the indexing run a data source would get with a given embedding provider.
@@ -190,24 +190,20 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
}
/// <summary>
/// Whether the file or folder a data source reads must stay as it is.
/// Whether a mailbox's origin must stay as it is.
/// </summary>
/// <remarks>
/// Locked as soon as the index holds anything, because where a data source reads from is what it
/// is: another folder is another data source, and the path reaches no signature, so swapping it
/// would leave the stored index describing documents nobody points at any more.
///
/// The embedding provider used to be locked along with it and no longer is. It does reach the
/// signature, so changing it rebuilds the index cleanly -- and DataSourceReindexWarning asks
/// before it does. Locking it as well left a data source whose provider was deleted stuck on
/// keyword search for good, with no way back.
/// Locked as soon as the index holds anything: another server, account, or folder holds other
/// mails, which makes it another data source. Local files and folders are not locked, since they
/// may move, and the file indexer finds their documents again, cf. FileSourceIndexer.RelocateStoredFilesAsync.
/// The embedding provider is never locked; DataSourceReindexWarning asks what changing it costs.
///
/// Unclear counts as locked: an unavailable index database says nothing about what is stored.
/// </remarks>
/// <param name="dataSourceId">The data source to ask about.</param>
/// <param name="dataSourceId">The mailbox to ask about.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>True when the source must not be changed.</returns>
public async Task<bool> ShouldLockDataSourceOriginAsync(string dataSourceId, CancellationToken token = default)
public async Task<bool> ShouldLockMailboxOriginAsync(string dataSourceId, CancellationToken token = default)
{
var indexStore = await databaseClientProvider.GetIndexStoreAsync(token);
if (!indexStore.IsAvailable)
@@ -1504,4 +1500,4 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
{
_ = MessageBus.INSTANCE.SendMessage(null, Event.RAG_EMBEDDING_STATUS_CHANGED, true);
}
}
}
@@ -1,3 +1,4 @@
namespace AIStudio.Tools.Services;
public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount, string? TranscriptKey = null);
public sealed record EmbeddedFileRecord(string Fingerprint, long FileSize, DateTimeOffset LastWriteUtc, DateTimeOffset EmbeddedAtUtc, int ChunkCount, string? TranscriptKey = null,
string? RelativePath = null, string? ParentFileId = null);
@@ -226,7 +226,7 @@ internal sealed class DocumentRunProgress(IndexedRunContext context, int totalDo
var embeddedAtUtc = DateTimeOffset.UtcNow;
var state = document.State with { ChunkCount = chunkCount, EmbeddedAtUtc = embeddedAtUtc };
await context.IndexStore.UpsertFileAsync(context.DataSource.Id, state, token);
context.Manifest.Files[document.Key] = new EmbeddedFileRecord(state.Fingerprint, state.FileSize, state.LastWriteUtc, embeddedAtUtc, chunkCount, state.TranscriptKey);
context.Manifest.Files[document.Key] = new EmbeddedFileRecord(state.Fingerprint, state.FileSize, state.LastWriteUtc, embeddedAtUtc, chunkCount, state.TranscriptKey, state.RelativePath, state.ParentFileId);
await context.ForgetPermanentFailureAsync(document.Key, token);
context.Pass.Forget(document.Key);
@@ -0,0 +1,91 @@
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
namespace AIStudio.Tools.Services.Indexing;
internal sealed partial class FileSourceIndexer
{
/// <summary>
/// Reconnects unchanged files before enumeration and missing-file cleanup. In particular, media
/// must already be known under their new paths when enumeration asks whether they need transcription.
/// </summary>
private async Task<bool> RelocateStoredFilesAsync(IndexedRunContext context, IDataSource dataSource, CancellationToken token)
{
DocumentRunProgress? progress = null;
foreach (var (oldPath, record) in context.Manifest.Files.ToArray())
{
token.ThrowIfCancellationRequested();
var file = FindRelocatedFile(dataSource, oldPath, record);
if (file is null || record.ParentFileId is not { } parentFileId || (!DataSourceEmbeddingManifest.PATH_COMPARER.Equals(oldPath, file.FullName) && context.Manifest.Files.ContainsKey(file.FullName)))
continue;
progress ??= DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger);
if (progress.HasDoneItsShare(DateTimeOffset.UtcNow))
return false;
progress.Publish();
var relativePath = this.TryGetRelativePath(dataSource, file);
// Use the metadata which passed the reuse check. If the file changes during these
// writes, enumeration must see a different fingerprint and index its new content.
var fingerprint = BuildFileMetadataHash(file.FullName, file.Name, record.LastWriteUtc.UtcDateTime, record.FileSize);
// Vectors first, then SQLite. Selecting by the unchanged parent id makes this safe to
// retry if cancellation or a crash falls between the two writes. A failure aborts the
// run before missing-file cleanup can remove the documents we are reconnecting.
await context.VectorStore.RelocateEmbeddingFileAsync(context.CollectionName, parentFileId, file.FullName, relativePath, fingerprint, token);
token.ThrowIfCancellationRequested();
await context.IndexStore.RelocateFileAsync(dataSource.Id, parentFileId, file.FullName, relativePath, fingerprint, token);
context.Manifest.Files.Remove(oldPath);
context.Manifest.Files[file.FullName] = record with { Fingerprint = fingerprint, RelativePath = relativePath };
logger.LogInformation("Reused stored embeddings for relocated file '{OldPath}' at '{NewPath}' in data source '{DataSourceId}'.", oldPath, file.FullName, dataSource.Id);
}
return true;
}
/// <summary>
/// Uses the same size and modification-time check as ordinary indexing, matching a folder's
/// documents by relative path. It never searches another folder for a missing document.
/// </summary>
/// <remarks>
/// A stored path which still lies below the selected folder stays as it is: it points at its
/// file, and a file which is gone from there is gone. Mapping it through its old relative path
/// would move it, e.g., when the selected folder grows to its parent. A copy at the mapped place
/// would then take over the identity of the document while the original is indexed anew.
/// Checking this first also spares every run of an unmoved folder resolving each stored path.
/// </remarks>
internal static FileInfo? FindRelocatedFile(IDataSource dataSource, string oldPath, EmbeddedFileRecord record)
{
if (record.ParentFileId is null)
return null;
string targetPath;
switch (dataSource)
{
case DataSourceLocalFile localFile when RelocatedFilePaths.IsLocalAbsolute(localFile.FilePath):
targetPath = Path.GetFullPath(localFile.FilePath);
break;
case DataSourceLocalDirectory localDirectory when RelocatedFilePaths.IsBelow(localDirectory.Path, oldPath):
return null;
case DataSourceLocalDirectory localDirectory:
var resolvedPath = RelocatedFilePaths.Resolve(localDirectory.Path, oldPath, record.RelativePath);
if (resolvedPath is null)
return null;
targetPath = resolvedPath;
break;
default:
return null;
}
if (string.Equals(oldPath, targetPath, StringComparison.Ordinal)
|| !string.Equals(Path.GetExtension(RelocatedFilePaths.GetStoredFileName(oldPath)), Path.GetExtension(targetPath), StringComparison.OrdinalIgnoreCase))
return null;
var file = new FileInfo(targetPath);
return file.Exists && file.Length == record.FileSize && file.LastWriteTimeUtc == record.LastWriteUtc.UtcDateTime ? file : null;
}
}
@@ -251,7 +251,7 @@ internal sealed partial class FileSourceIndexer
return true;
if (configuration.TargetFileName is { } targetFileName)
return string.Equals(fileName, targetFileName, StringComparison.OrdinalIgnoreCase);
return string.Equals(fileName, targetFileName, PathTools.COMPARISON);
//
// Git renames .git/index.lock to .git/index with almost every command, and a build or
@@ -297,13 +297,13 @@ internal sealed partial class FileSourceIndexer
private static DataSourceWatcherConfiguration? GetWatchConfiguration(IDataSourceBase dataSource) => dataSource switch
{
DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path) => new DataSourceWatcherConfiguration(
DataSourceLocalDirectory localDirectory when RelocatedFilePaths.IsLocalAbsolute(localDirectory.Path) && Directory.Exists(localDirectory.Path) => new DataSourceWatcherConfiguration(
localDirectory.Path,
localDirectory.IncludeSubdirectories,
localDirectory.IncludeHiddenFilesAndFolders,
FilePatterns: FileNamePatterns.Parse(localDirectory.EffectiveFilePatterns),
ExcludePatterns: FileExclusions.Parse(localDirectory.EffectiveExcludePatterns)),
DataSourceLocalFile localFile when File.Exists(localFile.FilePath) && !string.IsNullOrWhiteSpace(Path.GetDirectoryName(localFile.FilePath)) => new DataSourceWatcherConfiguration(
DataSourceLocalFile localFile when RelocatedFilePaths.IsLocalAbsolute(localFile.FilePath) && File.Exists(localFile.FilePath) && !string.IsNullOrWhiteSpace(Path.GetDirectoryName(localFile.FilePath)) => new DataSourceWatcherConfiguration(
Path.GetDirectoryName(localFile.FilePath)!,
false,
TargetFileName: Path.GetFileName(localFile.FilePath)),
@@ -314,8 +314,8 @@ internal sealed partial class FileSourceIndexer
{
return left.IncludeSubdirectories == right.IncludeSubdirectories
&& left.IncludeHiddenFilesAndFolders == right.IncludeHiddenFilesAndFolders
&& string.Equals(left.RootPath, right.RootPath, StringComparison.OrdinalIgnoreCase)
&& string.Equals(left.TargetFileName, right.TargetFileName, StringComparison.OrdinalIgnoreCase)
&& string.Equals(left.RootPath, right.RootPath, PathTools.COMPARISON)
&& string.Equals(left.TargetFileName, right.TargetFileName, PathTools.COMPARISON)
&& FileNamePatterns.AreEquivalent(left.FilePatterns, right.FilePatterns)
&& FileNamePatterns.AreEquivalent(left.ExcludePatterns, right.ExcludePatterns);
}
@@ -77,12 +77,25 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
var manifest = context.Manifest;
if (!await this.RelocateStoredFilesAsync(context, dataSource, token))
return await DocumentRunProgress.FromStoredState(context, manifest.Files.Count, logger).PauseRunAsync(null, "file relocation paused", token);
var enumerationStartedUtc = DateTimeOffset.UtcNow;
var inputFiles = this.GetInputFiles(dataSource, manifest);
var enumerationDuration = DateTimeOffset.UtcNow - enumerationStartedUtc;
if (inputFiles.UnreachableRootReason is { } unreachableRootReason)
return this.StopWithoutRoot(context, dataSource, unreachableRootReason);
// An unmounted volume can leave a readable, empty mount directory. There is no way to
// distinguish that from intentionally removing every file, so keep the stored index.
// Explicit exclusions still take effect: they are evidence of a nonempty folder.
if (dataSource is DataSourceLocalDirectory directory
&& (manifest.Files.Count > 0 || manifest.UnfinishedFiles.Count > 0 || manifest.PermanentFailures.Count > 0)
&& inputFiles.Files.Count == 0 && inputFiles.IgnoredFiles.Count == 0
&& inputFiles.FilePatternExclusionCount == 0 && inputFiles.ExcludedFolderCount == 0
&& inputFiles.FailedFiles == 0 && inputFiles.UnreadableDirectories.Count == 0)
return this.StopWithoutRoot(context, dataSource, string.Format(TB("The folder '{0}' currently appears empty. AI Studio kept the index of this data source, because the folder of a drive that is not connected can look empty. Connect the drive and refresh this data source. If you emptied the folder on purpose, remove this data source."), directory.Path));
var indexedFiles = inputFiles.Files;
var totalFiles = indexedFiles.Count + inputFiles.FailedFiles + inputFiles.IgnoredFiles.Count;
@@ -364,7 +377,8 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
/// connected, or a network share while the VPN is down, would remove every embedding of the
/// data source, and embedding it all again once the folder is back costs as long as it took the
/// first time. Nothing is written: the stored hash still tells which state of the data source
/// the index holds, so the next run picks up from there.
/// the index holds, so the next run picks up from there. The status asks for attention, so the
/// embeddings page offers to edit the data source, e.g. to select the new location of a moved folder.
/// </remarks>
/// <param name="context">The run.</param>
/// <param name="dataSource">The data source.</param>
@@ -378,7 +392,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
dataSource.Id,
context.Manifest.Files.Count);
DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger).PublishRunFailure(reason);
DocumentRunProgress.FromStoredState(context, context.Manifest.Files.Count, logger).PublishRunFailure(reason, DataSourceAttention.LOCATION_UNREACHABLE);
return IndexedRunOutcome.DONE;
}
@@ -509,9 +523,10 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
/// user may not read that folder. Removing it would throw away its index, and embedding it again
/// once the folder can be read costs as long as it took the first time.
/// <br /><br />
/// Paths are compared ignoring their case, as everywhere in the manifest. On Linux and macOS, that
/// keeps a file whose path differs from an unreadable folder in its case alone, which costs no more
/// than a stale entry until the folder can be read again.
/// Paths are compared ignoring their case, as everywhere in the index, cf. the remarks on
/// DataSourceEmbeddingManifest.PATH_COMPARER. On Linux and macOS, that keeps a file whose path
/// differs from an unreadable folder in its case alone, which costs no more than a stale entry
/// until the folder can be read again.
/// </remarks>
/// <param name="storedPaths">The paths of the files the index knows about.</param>
/// <param name="files">The files this run found.</param>
@@ -521,15 +536,16 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
{
var existingPaths = files
.Select(file => file.FullName)
.ToHashSet(StringComparer.OrdinalIgnoreCase);
.ToHashSet(DataSourceEmbeddingManifest.PATH_COMPARER);
var unreadablePrefixes = unreadableDirectories
.Select(directory => Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)) + Path.DirectorySeparatorChar)
.Select(Path.GetFullPath)
.Select(directory => Path.EndsInDirectorySeparator(directory) ? directory : directory + Path.DirectorySeparatorChar)
.ToList();
return storedPaths
.Where(path => !existingPaths.Contains(path))
.Where(path => !unreadablePrefixes.Any(prefix => path.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)))
.Where(path => !unreadablePrefixes.Any(prefix => path.StartsWith(prefix, DataSourceEmbeddingManifest.PATH_COMPARISON)))
.ToList();
}
@@ -709,7 +725,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
switch (dataSource)
{
case DataSourceLocalFile localFile when File.Exists(localFile.FilePath):
case DataSourceLocalFile localFile when RelocatedFilePaths.IsLocalAbsolute(localFile.FilePath) && File.Exists(localFile.FilePath):
var file = new FileInfo(localFile.FilePath);
// No size limit: the user chose this very file, cf. FolderFileSelection.IsTooLarge.
@@ -718,9 +734,10 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
return result;
case DataSourceLocalDirectory localDirectory when Directory.Exists(localDirectory.Path):
case DataSourceLocalDirectory localDirectory when RelocatedFilePaths.IsLocalAbsolute(localDirectory.Path) && Directory.Exists(localDirectory.Path):
this.EnumerateAccessibleFiles(localDirectory, manifest, mediaTranscriptionAvailability, result);
if (result.UnreadableDirectories.Contains(localDirectory.Path))
if (result.UnreadableDirectories.Any(directory => string.Equals(Path.TrimEndingDirectorySeparator(Path.GetFullPath(directory)),
Path.TrimEndingDirectorySeparator(Path.GetFullPath(localDirectory.Path)), PathTools.COMPARISON)))
result.UnreachableRootReason = string.Format(TB("The folder '{0}' could not be opened. Please check whether you are allowed to read it. Nothing was removed from the index."), localDirectory.Path);
return result;
@@ -732,11 +749,11 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
switch (dataSource)
{
case DataSourceLocalFile localFile:
result.UnreachableRootReason = string.Format(TB("The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. Once the file is back, refresh this data source. When it is gone for good, remove this data source."), localFile.FilePath);
result.UnreachableRootReason = string.Format(TB("The file '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the file was moved or renamed. Nothing was removed from the index. If the file was moved or renamed, edit this data source to select its new location. Once the file is back, refresh this data source. When it is gone for good, remove this data source."), localFile.FilePath);
break;
case DataSourceLocalDirectory localDirectory:
result.UnreachableRootReason = string.Format(TB("The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. Once the folder is back, refresh this data source."), localDirectory.Path);
result.UnreachableRootReason = string.Format(TB("The folder '{0}' cannot be reached. Perhaps the drive it is on is not connected, or the folder was moved or renamed. Nothing was removed from the index. If the folder was moved or renamed, edit this data source to select its new location. Once the folder is back, refresh this data source."), localDirectory.Path);
break;
}
@@ -935,8 +952,8 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
private DataSourceMetadataSnapshot BuildDataSourceMetadataSnapshot(IDataSource dataSource, IReadOnlyList<FileInfo> indexedFiles)
{
var fileHashes = indexedFiles
.OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase)
.ToDictionary(file => file.FullName, BuildFileMetadataHash, StringComparer.OrdinalIgnoreCase);
.OrderBy(file => file.FullName, DataSourceEmbeddingManifest.PATH_COMPARER)
.ToDictionary(file => file.FullName, BuildFileMetadataHash, DataSourceEmbeddingManifest.PATH_COMPARER);
var sourceHash = dataSource switch
{
@@ -980,7 +997,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
indexedFiles.Count.ToString()
};
foreach (var file in indexedFiles.OrderBy(file => file.FullName, StringComparer.OrdinalIgnoreCase))
foreach (var file in indexedFiles.OrderBy(file => file.FullName, DataSourceEmbeddingManifest.PATH_COMPARER))
{
parts.Add(this.TryGetRelativePath(dataSource, file));
parts.Add(fileHashes[file.FullName]);
@@ -1002,14 +1019,24 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
"0");
}
return BuildMetadataHash(
"file",
file.FullName,
file.Name,
file.LastWriteTimeUtc.Ticks.ToString(),
file.Length.ToString());
return BuildFileMetadataHash(file.FullName, file.Name, file.LastWriteTimeUtc, file.Length);
}
/// <summary>
/// The fingerprint of a file which exists, from its write time and size as they were read.
/// </summary>
/// <remarks>
/// The one formula for both an indexed file and a relocated one: if the two drifted apart, every
/// relocated file would look changed to the next run and be embedded again.
/// </remarks>
/// <param name="fullName">The absolute path of the file.</param>
/// <param name="name">The name of the file.</param>
/// <param name="lastWriteUtc">When the file was last written to.</param>
/// <param name="length">The size of the file in bytes.</param>
/// <returns>The fingerprint.</returns>
private static string BuildFileMetadataHash(string fullName, string name, DateTime lastWriteUtc, long length) =>
BuildMetadataHash("file", fullName, name, lastWriteUtc.Ticks.ToString(), length.ToString());
private static string BuildMetadataHash(params string[] parts)
{
return BuildMetadataHash((IEnumerable<string>)parts);
@@ -1036,7 +1063,7 @@ internal sealed partial class FileSourceIndexer(SettingsManager settingsManager,
/// <returns>The document.</returns>
private EmbeddingDocument CreateFileDocument(IndexedRunContext context, IDataSource dataSource, FileInfo file, string fingerprint, MediaTranscriptSlot? transcript) => new(
file.FullName,
this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow) with { TranscriptKey = transcript?.Key },
this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTimeOffset.UtcNow) with { ParentFileId = context.GetFileParentId(file.FullName), TranscriptKey = transcript?.Key },
file.Name,
token => this.StreamEmbeddingChunksAsync(file.FullName, transcript, dataSource, context.EmbeddingProvider, context.PromptInjectionScope, token));
@@ -7,9 +7,11 @@ namespace AIStudio.Tools.Services.Indexing;
/// The ids under which documents and their chunks are stored.
/// </summary>
/// <remarks>
/// Derived rather than random, so the same document of the same data source always ends up under
/// the same ids, in the vector store as well as in the index store. The formats must stay as they
/// are: every stored id was made with them.
/// Derived rather than random, so the same document of the same data source ends up under the same
/// ids, in the vector store as well as in the index store. Once stored, they are the identity of the
/// document: a file which moves to another folder keeps its ids, although they were derived from its
/// old path and its old fingerprint, cf. FileSourceIndexer.RelocateStoredFilesAsync. The formats must
/// stay as they are: every stored id was made with them.
/// </remarks>
internal static class IndexedDocumentIds
{
@@ -17,11 +19,31 @@ internal static class IndexedDocumentIds
/// The id of one chunk, which is also the id of its point in the vector store.
/// </summary>
/// <param name="dataSourceId">The data source the document belongs to.</param>
/// <param name="fingerprint">The fingerprint of the document as it was read.</param>
/// <param name="source">What the chunk ids of the document derive from, cf. CreateChunkIdSource.</param>
/// <param name="chunkIndex">The position of the chunk within the document.</param>
/// <returns>The chunk id.</returns>
public static string CreateChunkId(string dataSourceId, string fingerprint, int chunkIndex) =>
CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
public static string CreateChunkId(string dataSourceId, string source, int chunkIndex) =>
CreateStableGuid($"{dataSourceId}:chunk:{source}:{chunkIndex}");
/// <summary>
/// What the chunk ids of a document derive from: nearly always its fingerprint.
/// </summary>
/// <remarks>
/// A moved file keeps its parent id and its chunk ids, but its fingerprint now names the new path.
/// Another file may later appear at the old path with exactly the fingerprint those chunk ids came
/// from, and the fingerprint alone would hand it the same ids. The two documents would then take
/// over each other's chunks. Hence the parent id joins whenever a document is not stored under the
/// id its key gives, which only happens after a move. Every other document keeps the ids it has.
/// </remarks>
/// <param name="dataSourceId">The data source the document belongs to.</param>
/// <param name="documentKey">The key of the document.</param>
/// <param name="parentId">The id the document is stored under.</param>
/// <param name="fingerprint">The fingerprint of the document as it is about to be read.</param>
/// <returns>What to pass to CreateChunkId.</returns>
public static string CreateChunkIdSource(string dataSourceId, string documentKey, string parentId, string fingerprint) =>
string.Equals(parentId, CreateParentId(dataSourceId, documentKey), StringComparison.Ordinal)
? fingerprint
: $"{parentId}:{fingerprint}";
/// <summary>
/// The id of a document's row in the index store, which its chunks point at.
@@ -41,6 +41,9 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
private long storedChunksSinceLastOptimization;
private bool hasPendingChanges;
private Dictionary<string, string>? fileParentIds;
private HashSet<string>? reservedFileParentIds;
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(IndexedRunContext).Namespace, nameof(IndexedRunContext));
/// <summary>
@@ -66,6 +69,38 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
public IndexingPass Pass => pass;
/// <summary>
/// The id a file is stored under: the one it already has, or a new one.
/// </summary>
/// <remarks>
/// A relocated file keeps the id it got when it was indexed first, which came from its path back
/// then. A new file may now appear at that very path, so the ids already stored are reserved, and
/// a new file whose derived id is taken gets a random one instead. Its chunk ids include that id,
/// cf. IndexedDocumentIds.CreateChunkIdSource.
/// </remarks>
/// <param name="documentKey">The key of the file, which is its absolute path.</param>
/// <returns>The parent id of the file.</returns>
public string GetFileParentId(string documentKey)
{
if (this.fileParentIds is null)
{
this.fileParentIds = manifest.Files
.Where(file => file.Value.ParentFileId is not null)
.ToDictionary(file => file.Key, file => file.Value.ParentFileId!, DataSourceEmbeddingManifest.PATH_COMPARER);
this.reservedFileParentIds = this.fileParentIds.Values.ToHashSet(StringComparer.Ordinal);
}
if (this.fileParentIds.TryGetValue(documentKey, out var storedId))
return storedId;
var parentId = IndexedDocumentIds.CreateParentId(dataSource.Id, documentKey);
while (!this.reservedFileParentIds!.Add(parentId))
parentId = Guid.NewGuid().ToString();
this.fileParentIds[documentKey] = parentId;
return parentId;
}
/// <summary>
/// The run as one action of the prompt injection filter.
/// </summary>
@@ -124,10 +159,11 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
var embeddingBatchSize = Math.Max(1, embeddingProvider.EffectiveEmbeddingBatchSize);
var batch = new List<EmbeddingChunkDraft>(embeddingBatchSize);
var totalChunkCount = 0;
var chunkIdSource = IndexedDocumentIds.CreateChunkIdSource(dataSource.Id, document.Key, document.State.ParentFileId, document.State.Fingerprint);
await foreach (var chunk in document.StreamChunks(token))
{
batch.Add(new(IndexedDocumentIds.CreateChunkId(dataSource.Id, document.State.Fingerprint, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber));
batch.Add(new(IndexedDocumentIds.CreateChunkId(dataSource.Id, chunkIdSource, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber));
totalChunkCount++;
reportBlockProgress(totalChunkCount, chunk.PageNumber);
@@ -163,7 +199,10 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
/// <param name="token">The cancellation token.</param>
public async Task DeleteDocumentPointsAsync(string documentKey, CancellationToken token)
{
await vectorStore.DeleteEmbeddingByFile(this.CollectionName, documentKey, token);
// A relocated document keeps its original parent id. Delete by that identity as well as
// by path, including when a previous relocation stopped between its two database writes.
var parentId = manifest.Files.GetValueOrDefault(documentKey)?.ParentFileId ?? string.Empty;
await vectorStore.DeleteEmbeddingByFile(this.CollectionName, documentKey, parentId, token);
this.hasPendingChanges = true;
}
@@ -401,4 +440,4 @@ internal sealed class IndexedRunContext(IIndexedDataSource dataSource, Embedding
batch.Clear();
}
}
}
@@ -44,7 +44,7 @@ internal sealed class IndexingPass(DataSourceRunSignature runSignature)
/// <summary>
/// The documents which failed during this pass for a reason which may pass, by their keys.
/// </summary>
private readonly Dictionary<string, FailedDocument> failedDocuments = new(StringComparer.OrdinalIgnoreCase);
private readonly Dictionary<string, FailedDocument> failedDocuments = new(DataSourceEmbeddingManifest.PATH_COMPARER);
//
// Which kinds of provider failure the user was already told about in this pass. A rejected API
@@ -0,0 +1,85 @@
namespace AIStudio.Tools.Services.Indexing;
/// <summary>
/// Reads paths stored by the computer which created an index. Its path syntax may differ from
/// that of the computer restoring the backup; never resolve an old absolute path on this machine.
/// </summary>
internal static class RelocatedFilePaths
{
internal static bool IsWindowsAbsolute(string path) => path.StartsWith(@"\\", StringComparison.Ordinal)
|| (path.Length >= 3 && char.IsAsciiLetter(path[0]) && path[1] == ':' && path[2] is '\\' or '/');
internal static bool IsLocalAbsolute(string path) => !string.IsNullOrWhiteSpace(path) && Path.IsPathFullyQualified(path)
&& (OperatingSystem.IsWindows() || !IsWindowsAbsolute(path));
internal static string GetStoredFileName(string path)
{
var separator = IsWindowsAbsolute(path) ? Math.Max(path.LastIndexOf('/'), path.LastIndexOf('\\')) : path.LastIndexOf('/');
return path[(separator + 1)..];
}
/// <summary>
/// Whether a stored path still lies below a root, spelled exactly as the root is now.
/// </summary>
/// <remarks>
/// Compared ordinally, unlike PathTools.IsInsideDirectory: once the name of a folder changed in its
/// case alone, its stored paths no longer match the paths a run finds, on Windows as well.
/// </remarks>
internal static bool IsBelow(string root, string storedPath)
{
if (!IsLocalAbsolute(root))
return false;
try
{
var prefix = Path.GetFullPath(root);
if (!Path.EndsInDirectorySeparator(prefix))
prefix += Path.DirectorySeparatorChar;
return storedPath.StartsWith(prefix, StringComparison.Ordinal);
}
catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
{
return false;
}
}
/// <summary>
/// Maps a stored relative path below a newly selected root, including drive roots and mounted
/// volumes. Backslashes are separators only for a Windows index: they are legal Unix file names.
/// </summary>
internal static string? Resolve(string root, string oldAbsolutePath, string? relativePath)
{
if (!IsLocalAbsolute(root) || string.IsNullOrWhiteSpace(relativePath))
return null;
var windowsSource = IsWindowsAbsolute(oldAbsolutePath);
if (!windowsSource && !oldAbsolutePath.StartsWith('/'))
return null;
var parts = relativePath.Split(windowsSource ? ['/', '\\'] : ['/']);
foreach (var part in parts)
{
// Check after decoding the source separators, so a Windows '..\\' cannot evade the
// containment check on Unix. Reject target-invalid names rather than changing them.
if (part is "" or "." or ".." || part.IndexOfAny(Path.GetInvalidFileNameChars()) >= 0)
return null;
if (windowsSource && part.Contains(':'))
return null;
if (OperatingSystem.IsWindows() && (part.EndsWith('.') || part.EndsWith(' ')))
return null;
}
try
{
var target = Path.GetFullPath(Path.Combine(root, Path.Combine(parts)));
return PathTools.IsInsideDirectory(root, target) ? target : null;
}
catch (Exception exception) when (exception is ArgumentException or NotSupportedException or PathTooLongException)
{
return null;
}
}
}
@@ -13,9 +13,10 @@ public sealed partial class RustService
async () =>
{
var encodedTitle = Uri.EscapeDataString(title);
var result = initialDirectory is null
var startDirectory = initialDirectory is null ? null : PathTools.FindNearestExistingPath(initialDirectory);
var result = startDirectory is null
? await this.http.PostAsync($"/select/directory?title={encodedTitle}", null)
: await this.http.PostAsJsonAsync($"/select/directory?title={encodedTitle}", new PreviousDirectory(initialDirectory), this.jsonRustSerializerOptions);
: await this.http.PostAsJsonAsync($"/select/directory?title={encodedTitle}", new PreviousDirectory(startDirectory), this.jsonRustSerializerOptions);
if (result.IsSuccessStatusCode)
return await result.Content.ReadFromJsonAsync<DirectorySelectionResponse>(this.jsonRustSerializerOptions);
@@ -54,10 +55,11 @@ public sealed partial class RustService
"select file",
async () =>
{
var startPath = initialFile is null ? null : PathTools.FindNearestExistingPath(initialFile);
var payload = new SelectFileOptions
{
Title = title,
PreviousFile = initialFile is null ? null : new (initialFile),
PreviousFile = startPath is null ? null : new (startPath),
Filter = FileTypes.ToFileDialogFilter(filter)
};
@@ -77,10 +79,11 @@ public sealed partial class RustService
"select files",
async () =>
{
var startPath = initialFile is null ? null : PathTools.FindNearestExistingPath(initialFile);
var payload = new SelectFileOptions
{
Title = title,
PreviousFile = initialFile is null ? null : new (initialFile),
PreviousFile = startPath is null ? null : new (startPath),
Filter = FileTypes.ToFileDialogFilter(filter)
};
@@ -40,6 +40,7 @@
- Improved the background embeddings page, which now lists the files of a local data source that AI Studio left out, for example, images or files that are too large. It also shows which file types and folders lie outside what the data source selects.
- Improved the Batch Processing assistant, which now leaves out hidden files and folders as well as build output and downloaded dependencies, such as the folder `node_modules`. You can change both in the expert settings of the assistant.
- Improved local data sources for folders and the Batch Processing assistant, which now leave out files larger than 1 GB, such as images of virtual machines. You can change this limit.
- Improved local data sources for files and folders. When you move or rename the file or folder, for example, to another drive or a new computer, you can now select its new location when you edit the data source. AI Studio keeps what it has already prepared for every unchanged file, so these files do not go to your embedding provider again. Thanks, Paul Koudelka (`PaulKoudelka`), for this contribution.
- Improved the organization-wide management of the Batch Processing assistant. IT departments can now preset whether it includes hidden files and folders, which files and folders it leaves out, and how large a file may be, either fixed or as a default which users may change.
- Improved the organization-wide management of the app settings. IT departments can now also set whether AI Studio saves energy while the AI writes its answers, which helps on virtual desktops, for example.
- Improved attaching files to a chat. You can now attach any file that contains readable text, for example, program code or configuration files of a kind AI Studio did not know before.
@@ -71,7 +72,7 @@
- Fixed AI Studio continuing to update the index of a data source with the old settings after you, or the configuration of your organization, changed its embedding provider or the expert settings of the data source, which could take hours for nothing. AI Studio now stops right away and starts over with the new settings.
- Fixed AI Studio continuing to send the content of a data source to an embedding provider after you deleted that provider, or after a confidence level changed so that the provider may no longer read the data source. AI Studio now stops right away.
- Fixed data sources staying unavailable after the configuration of your organization changed their embedding provider, until one of their documents changed, or you restarted AI Studio. When AI Studio updates your data sources automatically, it now rebuilds their index right away.
- Fixed AI Studio deleting the index of a local data source when it could not reach its folder or file, for example, because the drive was not connected. Once the folder was back, AI Studio indexed everything again, which could take hours. AI Studio now keeps the index and tells you that it cannot reach the folder. The same applies to subfolders you are not allowed to open.
- Fixed AI Studio deleting the index of a local data source when it could not reach its folder or file, for example, because the drive was not connected. Once the folder was back, AI Studio indexed everything again, which could take hours. AI Studio now keeps the index and tells you that it cannot reach the folder. The same applies to subfolders you are not allowed to open and to a folder that suddenly appears empty, which can happen when a drive is not connected.
- Fixed AI Studio rebuilding the index of a data source from scratch without asking you when part of the index was missing on your computer. Since this sends every document to your embedding provider once more, AI Studio now tells you about the missing index and offers to repair the data source.
- Fixed saving a local data source whose embedding provider had been deleted, without choosing a new one. The dialog now asks you to select an embedding provider, so that AI Studio can prepare the data source again.
- Fixed justified texts being hyphenated by the rules of English even when AI Studio shows another language, such as German. Screen readers now also know which language AI Studio uses.
@@ -0,0 +1,752 @@
using System.Reflection;
using System.Security.Cryptography;
using System.Text;
using System.Text.Json;
using AIStudio.Components;
using AIStudio.Dialogs;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tests.Tools.Databases;
using AIStudio.Tools;
using AIStudio.Tools.Databases;
using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.DataBackup;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services;
using AIStudio.Tools.Services.Indexing;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging.Abstractions;
using MudBlazor;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Runs the actual edit, queue, cancellation, manifest preparation, and file indexing code against
/// a migrated SQLite store. Only the vector store and dialog rendering are replaced.
/// </summary>
[NonParallelizable]
public sealed class DataSourceRelocationWorkflowTests
{
private const string EMBEDDING_ID = "b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01";
// Frozen from the format before relocation support. Do not generate this with the current
// signature builder: doing that would hide an accidental rebuild of every existing index.
private const string LEGACY_SIGNATURE = "2|b0a4c4d2-1f3e-4f0a-8c9d-5a6b7c8d9e01|OPEN_AI|text-embedding-3-small|NONE|http://localhost:1234|NONE||8192|512|100";
private static readonly TimeSpan TEST_TIMEOUT = TimeSpan.FromSeconds(10);
private string testRoot = string.Empty;
private TemporaryIndexStore store = null!;
private SettingsManager settings = null!;
private DatabaseClientProvider databases = null!;
private DataSourceEmbeddingService service = null!;
private RelocationVectorStore vectors = null!;
private IInternalDataSource source = null!;
private readonly List<string> chunkIds = [];
private readonly List<Task> runningTasks = [];
private CancellationTokenSource lifetime = null!;
private string OriginalFolder => Path.Combine(this.testRoot, "original");
[SetUp]
public async Task SetUp()
{
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-relocation-workflow-{Guid.NewGuid():N}");
Directory.CreateDirectory(this.testRoot);
this.store = await TemporaryIndexStore.CreateAsync();
this.settings = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
this.settings.ConfigurationData.App.EnabledPreviewFeatures.Add(PreviewFeatures.PRE_RAG_2024);
this.settings.ConfigurationData.App.DataSourceIndexing.AutomaticRefresh = false;
this.settings.ConfigurationData.EmbeddingProviders.Add(new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small")));
this.vectors = new RelocationVectorStore();
this.databases = new DatabaseClientProvider(null!, NullLoggerFactory.Instance);
var clients = Field<Dictionary<DatabaseRole, DatabaseClient>>(this.databases, "clients");
clients.Add(DatabaseRole.INDEX_STORE, this.store.Client);
clients.Add(DatabaseRole.VECTOR_STORE, this.vectors);
var guard = new PromptInjectionGuardService(null!, this.settings, NullLogger<PromptInjectionGuardService>.Instance, NullLoggerFactory.Instance);
this.service = new DataSourceEmbeddingService(this.settings, null!, this.databases, guard, null!, NullLogger<DataSourceEmbeddingService>.Instance);
this.lifetime = new CancellationTokenSource();
this.chunkIds.Clear();
this.runningTasks.Clear();
}
[TearDown]
public async Task TearDown()
{
this.lifetime.Cancel();
try
{
await Task.WhenAll(this.runningTasks).WaitAsync(TEST_TIMEOUT);
}
catch (OperationCanceledException) when (this.lifetime.IsCancellationRequested)
{
// A failed test may leave its deliberately blocked run for teardown to stop.
}
finally
{
this.service.Dispose();
this.databases.Dispose();
await this.store.DisposeAsync();
this.lifetime.Dispose();
Directory.Delete(this.testRoot, true);
}
}
[TestCase(false)]
[TestCase(true)]
public async Task AnExistingIndexSurvivesTheUpgradeWithoutReembedding(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.RelocatedPaths, Is.Empty, "An upgrade is not a relocation.");
Assert.That(this.vectors.DeletedStores, Is.Zero);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
});
}
[TestCase(false)]
[TestCase(true)]
public async Task EditingAndCancelingDoNothingAndSavingOnlyRelocatesUnchangedFiles(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
var newPath = this.MoveSource("moved");
var original = this.source;
var edit = this.OpenEdit(newPath);
Assert.Multiple(() =>
{
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
Assert.That(this.Queue.TryDequeue(out _), Is.False);
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
Assert.That(edit.Warning.ShowCount, Is.Zero, "Typing a path must not open the save confirmation.");
});
edit.Dialog.Instance.Cancel();
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
Assert.That(this.Queue.TryDequeue(out _), Is.False);
await this.SaveEditAsync(newPath);
Assert.That(this.vectors.RelocatedPaths, Is.Empty, "Saving queues the operation; the dialog must not write to either index.");
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
Assert.That(this.vectors.DeletedStores, Is.Zero);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
});
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
}
[TestCase(false)]
[TestCase(true)]
public async Task SavingAnUnchangedPathDoesNotWarnOrRelocate(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
var currentPath = this.source is DataSourceLocalDirectory folder ? folder.Path : ((DataSourceLocalFile)this.source).FilePath;
var edit = this.OpenEdit(currentPath);
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.Zero);
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
Assert.That(this.vectors.DeletedStores, Is.Zero);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
});
}
[TestCase(false)]
[TestCase(true)]
public async Task SavingWaitsForThePathWarningBeforeChangingConfigurationOrQueueingWork(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
var original = this.source;
var edit = this.OpenEdit(this.MoveSource("moved"));
var save = InvokeTask(edit.Component, "StoreValidatedDataSourceAsync");
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
Assert.That(save.IsCompleted, Is.False);
Assert.That(edit.ParentTask.IsCompleted, Is.False);
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
Assert.That(this.Queue.TryDequeue(out _), Is.False);
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
});
edit.Warning.Complete(DialogResult.Ok(true));
await save.WaitAsync(TEST_TIMEOUT);
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
}
[TestCase(false)]
[TestCase(true)]
public async Task CancelingThePathWarningKeepsTheConfiguredPathAndQueuesNothing(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
var original = this.source;
var edit = this.OpenEdit(this.MoveSource("moved"));
edit.Warning.Complete(DialogResult.Cancel());
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync");
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
Assert.That(edit.ParentTask.IsCompleted, Is.False, "Rejecting the warning leaves the edit dialog open.");
Assert.That(this.settings.ConfigurationData.DataSources.Single(), Is.EqualTo(original));
Assert.That(this.Queue.TryDequeue(out _), Is.False);
Assert.That(this.vectors.RelocatedPaths, Is.Empty);
});
edit.Dialog.Instance.Cancel();
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
}
[TestCase(false)]
[TestCase(true)]
public async Task ANewLocationOfADataSourceWithoutAnIndexIsSavedWithoutAQuestion(bool singleFile)
{
await this.AddSourceAsync(singleFile);
var newPath = this.MoveSource("moved");
var edit = this.OpenEdit(newPath);
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
var saved = this.settings.ConfigurationData.DataSources.Single();
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.Zero, "Nothing was prepared for the data source, so nothing can be lost.");
Assert.That(saved is DataSourceLocalDirectory folder ? folder.Path : ((DataSourceLocalFile)saved).FilePath, Is.EqualTo(newPath));
});
}
[TestCase(false)]
[TestCase(true)]
public async Task ANewLocationTogetherWithANewEmbeddingAsksOnlyOnce(bool singleFile)
{
const string OTHER_EMBEDDING_ID = "c1b5d5e3-2a4f-4b1b-9d0e-6b7c8d9e0f12";
this.settings.ConfigurationData.EmbeddingProviders.Add(new EmbeddingProvider(2, OTHER_EMBEDDING_ID, "Other embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-large", "text-embedding-3-large")));
await this.SeedLegacyIndexAsync(singleFile);
var edit = this.OpenEdit(this.MoveSource("moved"));
SetField(edit.Component, "dataEmbeddingId", OTHER_EMBEDDING_ID);
edit.Warning.Complete(DialogResult.Ok(true));
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1), "Everything is prepared again anyway, so the location needs no question of its own.");
Assert.That(edit.Warning.Title, Is.EqualTo("Documents Will Be Prepared Again"));
});
}
[TestCase(false)]
[TestCase(true)]
public async Task ASecondSavedPathCancelsTheFirstRelocationAndRunsWithTheLatestPath(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
await this.SaveEditAsync(this.MoveSource("first"));
this.vectors.BlockNextRelocation = true;
var firstRun = this.RunNextAsync();
await this.vectors.RelocationStarted.Task.WaitAsync(TEST_TIMEOUT);
var secondPath = this.MoveSource("second");
await this.SaveEditAsync(secondPath);
await firstRun.WaitAsync(TEST_TIMEOUT);
Assert.That(this.vectors.CanceledRelocations, Is.EqualTo(1), "The first run must be canceled without finishing its remaining documents.");
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(1));
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.RelocatedPaths.Skip(1).ToArray(), Has.Length.EqualTo(this.chunkIds.Count));
Assert.That(this.vectors.Locations.Values.All(path => singleFile ? path == secondPath : path.StartsWith(secondPath + Path.DirectorySeparatorChar, StringComparison.Ordinal)), Is.True);
Assert.That(this.vectors.DeletedStores, Is.Zero);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.Queue.TryDequeue(out _), Is.False, "The obsolete destination must not leave another run behind.");
});
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
}
[TestCase(false)]
[TestCase(true)]
public async Task DeletingTheSourceCancelsRelocationBeforeRemovingTheStoredIndex(bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
await this.SaveEditAsync(this.MoveSource("moved"));
this.vectors.BlockNextRelocation = true;
var run = this.RunNextAsync();
await this.vectors.RelocationStarted.Task.WaitAsync(TEST_TIMEOUT);
// A second save also leaves a pending run; deletion has to discard that one as well.
await this.service.QueueDataSourceAsync(this.source);
this.settings.ConfigurationData.DataSources.Clear();
await this.service.RemoveDataSourceAsync(this.source).WaitAsync(TEST_TIMEOUT);
await run.WaitAsync(TEST_TIMEOUT);
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.vectors.CanceledRelocations, Is.EqualTo(1));
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(1));
Assert.That(this.vectors.DeletedStores, Is.EqualTo(1));
Assert.That(this.vectors.Locations, Is.Empty);
Assert.That(manifest.Files, Is.Empty);
Assert.That(this.service.GetStatuses(), Is.Empty);
Assert.That(this.Queue.TryDequeue(out _), Is.False);
});
}
private DataSourceRunQueue Queue => Field<DataSourceRunQueue>(this.service, "runQueue");
[TestCase(@"E:\Documents", '\\', false)]
[TestCase(@"\\server\share\Documents", '\\', false)]
[TestCase("/home/alice/Documents", '/', false)]
[TestCase(@"E:\Documents", '\\', true)]
[TestCase(@"\\server\share\Documents", '\\', true)]
[TestCase("/Volumes/External drive/Documents", '/', true)]
public async Task ABackupCanReconnectFilesFromAnotherOSWithoutReembedding(string oldRoot, char separator, bool singleFile)
{
await this.SeedLegacyIndexAsync(singleFile);
var originalFolder = Path.Combine(this.testRoot, "original");
await this.StoreForeignPathsAsync(oldRoot, separator);
var restored = await this.BackupAndRestoreIndexAsync();
// Opening the restored index while the old drive or network share is unavailable must not
// prune anything. No real external volume or network connection is needed by these tests.
if (!RelocatedFilePaths.IsLocalAbsolute(oldRoot))
{
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
await this.AssertOriginalChunksAsync();
}
// User documents are not in an AI Studio backup. Copy them separately, preserving their
// modification times, as when reconnecting an external drive after restoring a computer.
var destination = Path.Combine(this.testRoot, "mounted volume", "Übertragene Dokumente");
foreach (var original in Directory.EnumerateFiles(originalFolder, "*", SearchOption.AllDirectories))
{
var localPath = Path.Combine(destination, Path.GetRelativePath(originalFolder, original));
Directory.CreateDirectory(Path.GetDirectoryName(localPath)!);
File.Copy(original, localPath);
File.SetLastWriteTimeUtc(localPath, File.GetLastWriteTimeUtc(original));
}
var newPath = singleFile ? Path.Combine(destination, "notes.md") : destination;
await this.SaveEditAsync(newPath);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
var manifest = await restored.GetManifestAsync(this.source.Id, CancellationToken.None);
var hits = await restored.SearchChunksAsync(this.source.Id, "indexed", 10, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
Assert.That(this.vectors.DeletedStores, Is.Zero);
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
Assert.That(manifest.Files.Keys, Is.EquivalentTo(this.vectors.Locations.Values));
Assert.That(hits.Select(hit => hit.AbsolutePath), Is.EquivalentTo(this.vectors.Locations.Values));
Assert.That(hits.All(hit => PathTools.IsInsideDirectory(destination, hit.AbsolutePath)), Is.True);
});
// A restart/hash check after relocation must not relocate or prepare everything a second time.
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
await this.AssertOriginalChunksAsync();
// A disconnected external drive is not an empty folder. Reconnecting it must also keep
// the embeddings, rather than paying to prepare the same documents again.
var disconnected = destination + "-disconnected";
Directory.Move(destination, disconnected);
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
await this.AssertOriginalChunksAsync();
Directory.Move(disconnected, destination);
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
});
}
private async Task StoreForeignPathsAsync(string oldRoot, char separator)
{
await using var context = this.store.CreateContext();
foreach (var file in await context.EmbeddedFiles.ToListAsync())
{
var relative = file.RelativePath.Replace(Path.DirectorySeparatorChar, separator);
var oldPath = oldRoot + separator + relative;
file.AbsolutePath = oldPath;
file.RelativePath = relative;
file.Fingerprint = LegacyFingerprint(oldPath, file.FileName, file.LastWriteUtc.UtcDateTime.Ticks, file.FileSize);
this.vectors.Locations[file.ParentFileId] = oldPath;
}
await context.SaveChangesAsync();
this.source = this.source switch
{
DataSourceLocalDirectory folder => folder with { Path = oldRoot },
DataSourceLocalFile file => file with { FilePath = oldRoot + separator + "notes.md" },
_ => throw new InvalidOperationException(),
};
this.settings.ConfigurationData.DataSources[0] = this.source;
}
private async Task<SqliteIndexStoreClientImplementation> BackupAndRestoreIndexAsync()
{
var dataDirectory = Path.Combine(this.testRoot, "source-data");
var configDirectory = Path.Combine(this.testRoot, "source-config");
var copies = Path.Combine(this.testRoot, "index-copies");
Directory.CreateDirectory(dataDirectory);
Directory.CreateDirectory(configDirectory);
await this.store.Client.BackupToDirectoryAsync(Path.Combine(copies, "databases", "sqlite"), CancellationToken.None);
// This fixture's vector client stores metadata only. The runtime tests separately check
// backing up, reopening, and relocating actual Qdrant vectors without changing their values.
var vectorDirectory = Path.Combine(copies, "databases", "vector_database", "stores", "store_test");
Directory.CreateDirectory(vectorDirectory);
await File.WriteAllTextAsync(Path.Combine(vectorDirectory, "locations.json"), JsonSerializer.Serialize(this.vectors.Locations));
var archivePath = Path.Combine(this.testRoot, "backup.zip");
var settingsJson = JsonSerializer.Serialize(this.settings.ConfigurationData, SettingsManager.JSON_OPTIONS);
await using (var output = File.Create(archivePath))
await DataBackupWriter.WriteAsync(output, new DataBackupSources(dataDirectory, configDirectory, settingsJson, copies, []),
new DataBackupManifest(DataBackupFormat.FORMAT_VERSION, "26.10.1", "V6", DateTimeOffset.UtcNow, dataDirectory, configDirectory, true, []), null, CancellationToken.None);
var target = new DataRestoreTarget(Path.Combine(this.testRoot, "restored-data"), Path.Combine(this.testRoot, "restored-config"), "26.10.1", AIStudio.Settings.Version.V6, []);
var result = await DataRestorePreparer.PrepareAsync(archivePath, target, null, CancellationToken.None);
Assert.That(result.Success, Is.True);
// The runtime applies this prepared directory at restart. Open the exact restored bytes
// here, so this test exercises the real backup writer and restore preparer, not a rebuilt DB.
var pendingData = Path.Combine(target.DataDirectory, DataRestoreFormat.PENDING_DIRECTORY);
var restoredPath = Path.Combine(pendingData, "databases", "sqlite", "rag-index.sqlite3");
var restored = new SqliteIndexStoreClientImplementation("Restored SQLite", restoredPath, pendingData, string.Empty);
Field<Dictionary<DatabaseRole, DatabaseClient>>(this.databases, "clients")[DatabaseRole.INDEX_STORE] = restored;
this.vectors.Locations.Clear();
foreach (var (id, path) in JsonSerializer.Deserialize<Dictionary<string, string>>(await File.ReadAllTextAsync(Path.Combine(pendingData, "databases", "vector_database", "stores", "store_test", "locations.json")))!)
this.vectors.Locations.Add(id, path);
var restoredSettings = JsonSerializer.Deserialize<Data>(await File.ReadAllTextAsync(Path.Combine(target.ConfigDirectory, DataRestoreFormat.PENDING_DIRECTORY, "settings.json")), SettingsManager.JSON_OPTIONS)!;
this.settings.ConfigurationData.DataSources.Clear();
this.settings.ConfigurationData.DataSources.AddRange(restoredSettings.DataSources);
this.source = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
return restored;
}
[TestCase(false)]
[TestCase(true)]
public async Task AnEmptyMountDirectoryKeepsTheIndexAndReconnectingReusesIt(bool relocated)
{
await this.SeedLegacyIndexAsync(false);
var original = ((DataSourceLocalDirectory)this.source).Path;
var disconnected = Path.Combine(this.testRoot, "disconnected-drive");
Directory.Move(original, disconnected);
var mount = relocated ? Path.Combine(this.testRoot, "new-mount") : original;
Directory.CreateDirectory(mount);
var storedHash = (await this.store.Client.GetDataSourceStateAsync(this.source.Id, CancellationToken.None))!.SourceHash;
if (relocated)
await this.SaveEditAsync(mount);
else
await this.service.QueueDataSourceAsync(this.source);
// A second refresh must not accept the empty mount as a new baseline and prune the index.
for (var attempt = 0; attempt < 2; attempt++)
{
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
var state = await this.store.Client.GetDataSourceStateAsync(this.source.Id, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
Assert.That(this.service.GetStatuses().Single().LastError, Does.Contain("appears empty"));
Assert.That(state!.SourceHash, Is.EqualTo(storedHash));
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
});
if (attempt == 0)
await this.service.QueueDataSourceAsync(this.source);
}
Directory.Delete(mount);
Directory.Move(disconnected, mount);
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(relocated ? this.chunkIds.Count : 0));
});
}
[Test]
public async Task RemovingSomeFilesStillCleansUpOnlyTheirIndexEntries()
{
await this.SeedLegacyIndexAsync(false);
var removed = Path.Combine(((DataSourceLocalDirectory)this.source).Path, "notes.md");
var removedChunkId = IndexedDocumentIds.CreateChunkId(this.source.Id, LegacyFingerprint(new FileInfo(removed)), 0);
File.Delete(removed);
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
var chunks = await this.store.Client.SearchChunksAsync(this.source.Id, "indexed", 10, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(chunks.Select(chunk => chunk.ChunkId), Is.EquivalentTo(this.chunkIds.Except([removedChunkId])));
Assert.That(this.vectors.DeletedDocuments, Is.EqualTo(1));
Assert.That(this.vectors.InsertedBatches, Is.Zero);
});
}
[Test]
public async Task ExplicitlyExcludingEveryFileStillRemovesTheirIndexEntries()
{
await this.SeedLegacyIndexAsync(false);
this.source = ((DataSourceLocalDirectory)this.source) with { FilePatterns = "*.pdf" };
this.settings.ConfigurationData.DataSources[0] = this.source;
await this.service.QueueDataSourceAsync(this.source);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(manifest.Files, Is.Empty);
Assert.That(this.vectors.Locations, Is.Empty);
Assert.That(this.vectors.DeletedDocuments, Is.EqualTo(2));
});
}
[Test]
public async Task AFolderWhoseNameChangesOnlyInCaseKeepsItsEmbeddings()
{
await this.SeedLegacyIndexAsync(false);
var renamed = Path.Combine(this.testRoot, "Original");
// Two moves, since a volume which ignores case may refuse to rename a folder to its own name:
var detour = Path.Combine(this.testRoot, "renaming");
Directory.Move(((DataSourceLocalDirectory)this.source).Path, detour);
Directory.Move(detour, renamed);
await this.SaveEditAsync(renamed);
await this.RunNextAsync().WaitAsync(TEST_TIMEOUT);
await this.AssertOriginalChunksAsync();
var manifest = await this.store.Client.GetManifestAsync(this.source.Id, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(this.service.GetStatuses().Single().State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(this.vectors.RelocatedPaths, Has.Count.EqualTo(this.chunkIds.Count));
Assert.That(this.vectors.DeletedDocuments, Is.Zero);
Assert.That(this.vectors.InsertedBatches, Is.Zero);
Assert.That(manifest.Files.Keys.All(path => path.StartsWith(renamed + Path.DirectorySeparatorChar, StringComparison.Ordinal)), Is.True, "The index must store the new spelling of the folder.");
});
}
private async Task SeedLegacyIndexAsync(bool singleFile)
{
var paths = await this.AddSourceAsync(singleFile);
var id = this.source.Id;
await this.store.Client.UpsertDataSourceAsync(id, this.source.Type.ToString(), EMBEDDING_ID, LEGACY_SIGNATURE, "legacy-source-hash", 3, CancellationToken.None);
foreach (var relative in paths)
{
var path = Path.Combine(this.OriginalFolder, relative);
var file = new FileInfo(path);
var fingerprint = LegacyFingerprint(file);
var parentId = IndexedDocumentIds.CreateParentId(id, path);
var chunkId = IndexedDocumentIds.CreateChunkId(id, fingerprint, 0);
var embeddedAt = DateTimeOffset.UtcNow.AddDays(-10);
await this.store.Client.UpsertFileAsync(id, new EmbeddingStateFile(parentId, path, file.Name, relative, "md", fingerprint, file.Length,
new DateTimeOffset(file.CreationTimeUtc), new DateTimeOffset(file.LastWriteTimeUtc), embeddedAt, 1), CancellationToken.None);
await this.store.Client.UpsertChunksAsync(id, [new EmbeddingStateChunk(chunkId, parentId, 1, 0, "Already indexed report", embeddedAt)], CancellationToken.None);
this.vectors.Locations.Add(parentId, path);
this.chunkIds.Add(chunkId);
}
}
/// <summary>
/// Writes the documents of a data source and configures it, without anything in the index.
/// </summary>
/// <returns>The paths of the documents, relative to the folder.</returns>
private async Task<string[]> AddSourceAsync(bool singleFile)
{
var root = this.OriginalFolder;
Directory.CreateDirectory(Path.Combine(root, "nested"));
var id = Guid.NewGuid().ToString();
var folder = new DataSourceLocalDirectory
{
Id = id, Name = "Existing documents", Type = DataSourceType.LOCAL_DIRECTORY, Path = root,
EmbeddingId = EMBEDDING_ID, ConfidenceLevel = ConfidenceLevel.NONE, MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 100,
FilePatterns = "*.md", ExcludePatterns = string.Empty,
};
this.source = singleFile ? new DataSourceLocalFile
{
Id = id, Name = folder.Name, Type = DataSourceType.LOCAL_FILE, FilePath = Path.Combine(root, "notes.md"),
EmbeddingId = EMBEDDING_ID, ConfidenceLevel = ConfidenceLevel.NONE, MaxChunkTokenLength = 512, ChunkOverlapTokenLength = 100,
} : folder;
this.settings.ConfigurationData.DataSources.Add(this.source);
var paths = singleFile ? new[] { "notes.md" } : new[] { "notes.md", Path.Combine("nested", "report.md") };
foreach (var relative in paths)
await File.WriteAllTextAsync(Path.Combine(root, relative), "Already indexed report");
return paths;
}
private string MoveSource(string destinationName)
{
var current = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
var destination = Path.Combine(this.testRoot, destinationName);
if (current is DataSourceLocalDirectory folder)
Directory.Move(folder.Path, destination);
else
{
destination += ".md";
File.Move(((DataSourceLocalFile)current).FilePath, destination);
}
return destination;
}
private async Task SaveEditAsync(string newPath)
{
var edit = this.OpenEdit(newPath);
edit.Warning.Complete(DialogResult.Ok(true));
await InvokeTask(edit.Component, "StoreValidatedDataSourceAsync").WaitAsync(TEST_TIMEOUT);
await edit.ParentTask.WaitAsync(TEST_TIMEOUT);
Assert.Multiple(() =>
{
Assert.That(edit.Warning.ShowCount, Is.EqualTo(1));
Assert.That(edit.Warning.ComponentType, Is.EqualTo(typeof(ConfirmDialog)));
Assert.That(edit.Warning.Title, Is.EqualTo("New Location for This Data Source"));
Assert.That(edit.Warning.Parameters!.Get<string>(nameof(ConfirmDialog.MarkdownBody)), Does.Contain("unchanged at the new location").And.Contain("removed from the index").And.Contain("files on disk stay as they are").And.Contain("costs money"));
});
this.source = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
}
private OpenedEdit OpenEdit(string newPath)
{
var storedSource = (IInternalDataSource)this.settings.ConfigurationData.DataSources.Single();
var dialog = new TestDataSourceDialog();
var warning = new TestDataSourceDialog();
var parent = new DataSourceManagement();
Property(parent, "SettingsManager", this.settings);
Property(parent, "MessageBus", MessageBus.INSTANCE);
Property(parent, "DataSourceEmbeddingService", this.service);
Property(parent, "DialogService", dialog.Service);
var parentTask = InvokeTask(parent, "EditDataSource", storedSource);
object component = storedSource is DataSourceLocalDirectory ? new DataSourceLocalDirectoryDialog() : new DataSourceLocalFileDialog();
Property(component, "SettingsManager", this.settings);
Property(component, "DialogService", warning.Service);
Property(component, "DataSourceEmbeddingService", this.service);
Property(component, "MudDialog", dialog.Instance);
Property(component, "IsEditing", true);
Property(component, "DataSource", storedSource);
SetField(component, "dataId", storedSource.Id);
SetField(component, "dataName", storedSource.Name);
SetField(component, "dataEmbeddingId", storedSource.EmbeddingId);
SetField(component, "dataConfidenceLevel", storedSource.ConfidenceLevel);
SetField(component, "dataMaxChunkTokenLength", storedSource.MaxChunkTokenLength);
SetField(component, "dataChunkOverlapTokenLength", storedSource.ChunkOverlapTokenLength);
SetField(component, storedSource is DataSourceLocalDirectory ? "dataPath" : "dataFilePath", newPath);
if (storedSource is DataSourceLocalDirectory folder)
{
SetField(component, "dataFilePatterns", folder.FilePatterns);
SetField(component, "dataExcludePatterns", folder.ExcludePatterns);
}
return new(component, dialog, warning, parentTask);
}
private Task RunNextAsync()
{
Assert.That(this.Queue.TryDequeue(out var request), Is.True, "Saving must queue a run.");
var task = InvokeTask(this.service, "RunDataSourceAsync", request!.DataSourceId, request.RefreshMode, this.lifetime.Token);
this.runningTasks.Add(task);
return task;
}
private async Task AssertOriginalChunksAsync()
{
var index = await this.databases.GetIndexStoreAsync();
var chunks = await index.SearchChunksAsync(this.source.Id, "indexed", 100, CancellationToken.None);
Assert.Multiple(() =>
{
Assert.That(chunks.Select(chunk => chunk.ChunkId), Is.EquivalentTo(this.chunkIds));
Assert.That(chunks.All(chunk => chunk.ChunkText == "Already indexed report"), Is.True);
Assert.That(chunks.All(chunk => chunk.EmbeddedAtUtc < DateTimeOffset.UtcNow.AddDays(-9)), Is.True, "Updating the app or its paths must not recreate existing chunks.");
});
}
private static string LegacyFingerprint(FileInfo file)
{
return LegacyFingerprint(file.FullName, file.Name, file.LastWriteTimeUtc.Ticks, file.Length);
}
private static string LegacyFingerprint(string path, string name, long writeTicks, long size)
{
var parts = new[] { "file", path, name, writeTicks.ToString(), size.ToString() };
var serialized = string.Concat(parts.Select(part => $"{part.Length}:{part}|"));
return Convert.ToHexString(SHA256.HashData(Encoding.UTF8.GetBytes(serialized)));
}
// Keep reflection at the fixture boundary: exercise the production queue worker without
// starting the unrelated application startup loop, and install the isolated database clients.
private static T Field<T>(object instance, string name) => (T)instance.GetType().GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.GetValue(instance)!;
private static void SetField(object instance, string name, object value) => instance.GetType().GetField(name, BindingFlags.Instance | BindingFlags.NonPublic)!.SetValue(instance, value);
private static void Property(object instance, string name, object value)
{
for (var type = instance.GetType(); type is not null; type = type.BaseType)
if (type.GetProperty(name, BindingFlags.Instance | BindingFlags.Public | BindingFlags.NonPublic | BindingFlags.DeclaredOnly) is { } property)
{
property.SetValue(instance, value);
return;
}
throw new MissingMemberException(name);
}
private static Task InvokeTask(object instance, string name, params object[] arguments) =>
(Task)instance.GetType().GetMethod(name, BindingFlags.Instance | BindingFlags.NonPublic)!.Invoke(instance, arguments)!;
private sealed record OpenedEdit(object Component, TestDataSourceDialog Dialog, TestDataSourceDialog Warning, Task ParentTask);
}
@@ -0,0 +1,47 @@
using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.Services.Indexing;
namespace AIStudio.Tests.Tools.Databases;
public sealed class IndexStoreRelocationTests
{
[Test]
public async Task RelocationPreservesChunksAndTranscriptAndUpdatesSearchLocations()
{
await using var store = await TemporaryIndexStore.CreateAsync();
var sourceId = Guid.NewGuid().ToString();
var oldPath = Path.Combine(Path.GetTempPath(), "original", "notes.md");
var newPath = Path.Combine(Path.GetTempPath(), "moved", "renamed.md");
var parentId = IndexedDocumentIds.CreateParentId(sourceId, oldPath);
var chunkId = IndexedDocumentIds.CreateChunkId(sourceId, "old-fingerprint", 0);
var now = DateTimeOffset.UtcNow;
var token = CancellationToken.None;
await store.Client.UpsertDataSourceAsync(sourceId, "LOCAL_DIRECTORY", "provider", "signature", "old-hash", 3, token);
await store.Client.UpsertFileAsync(sourceId, new EmbeddingStateFile(parentId, oldPath, "notes.md", "notes.md", "md", "old-fingerprint", 100, now, now, now, 1) { TranscriptKey = "transcript" }, token);
await store.Client.UpsertChunksAsync(sourceId, [new EmbeddingStateChunk(chunkId, parentId, 2, 0, "report about relocation", now)], token);
// Repeating the metadata write after an interrupted relocation must keep the same chunks.
await store.Client.RelocateFileAsync(sourceId, parentId, newPath, "renamed.md", "new-fingerprint", token);
await store.Client.RelocateFileAsync(sourceId, parentId, newPath, "renamed.md", "new-fingerprint", token);
var manifest = await store.Client.GetManifestAsync(sourceId, token);
var hits = await store.Client.SearchChunksAsync(sourceId, "relocation", 10, token);
Assert.Multiple(() =>
{
Assert.That(manifest.Files.Keys, Is.EquivalentTo(new[] { newPath }));
Assert.That(manifest.Files[newPath].Fingerprint, Is.EqualTo("new-fingerprint"));
Assert.That(manifest.Files[newPath].ParentFileId, Is.EqualTo(parentId));
Assert.That(manifest.Files[newPath].RelativePath, Is.EqualTo("renamed.md"));
Assert.That(manifest.Files[newPath].TranscriptKey, Is.EqualTo("transcript"));
Assert.That(hits, Has.Count.EqualTo(1));
Assert.That(hits[0].ChunkId, Is.EqualTo(chunkId));
Assert.That(hits[0].AbsolutePath, Is.EqualTo(newPath));
Assert.That(hits[0].FileName, Is.EqualTo("renamed.md"));
Assert.That(hits[0].PageNumber, Is.EqualTo(2));
Assert.That(hits[0].ChunkText, Is.EqualTo("report about relocation"));
});
await store.Client.DeleteFileAsync(sourceId, newPath, token);
Assert.That(await store.Client.SearchChunksAsync(sourceId, "relocation", 10, token), Is.Empty, "Ordinary removal must still find a relocated document's chunks.");
}
}
@@ -35,6 +35,15 @@ public sealed class FileSourceIndexerWatcherTests
Is.False);
}
[Test]
public void SingleFileWatcherIgnoresAnotherCaseDistinctFileOnUnix()
{
var configuration = new FileSourceIndexer.DataSourceWatcherConfiguration(Path.GetTempPath(), false, TargetFileName: "Report.md");
Assert.That(FileSourceIndexer.IsRelevantWatcherPath(configuration, Path.Combine(Path.GetTempPath(), "report.md"), WatcherChangeTypes.Changed),
Is.EqualTo(OperatingSystem.IsWindows()));
}
[Test]
public void SingleFileWatcherRecognizesARenameAwayFromTheConfiguredFile()
{
@@ -262,6 +271,15 @@ public sealed class FileSourceIndexerWatcherTests
Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.False);
}
[Test]
public void RelocatingToADirectoryWithDifferentCasingUsesTheLocalPathRules()
{
var before = DefaultConfiguration(Path.Combine(Path.GetTempPath(), "Documents"));
var after = before with { RootPath = Path.Combine(Path.GetTempPath(), "documents") };
Assert.That(FileSourceIndexer.IsSameWatchConfiguration(before, after), Is.EqualTo(OperatingSystem.IsWindows()), "On Unix these can be different directories; their watchers must not be treated as interchangeable.");
}
[Test]
public async Task DirectoryWatcherIgnoresTheBatchTranscriptWhenTheMediaIsSelected()
{
@@ -0,0 +1,217 @@
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Services;
using AIStudio.Tools.Services.Indexing;
using Microsoft.Extensions.Logging.Abstractions;
namespace AIStudio.Tests.Tools;
public sealed class FileSourceRelocationTests
{
private string testRoot = string.Empty;
[SetUp]
public void SetUp()
{
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-relocation-{Guid.NewGuid():N}");
Directory.CreateDirectory(this.testRoot);
}
[TearDown]
public void TearDown() => Directory.Delete(this.testRoot, true);
[Test]
public void AMovedFolderFindsAnUnchangedNestedDocument()
{
var oldRoot = Path.Combine(this.testRoot, "original");
var oldFile = Path.Combine(oldRoot, "nested", "notes.md");
Directory.CreateDirectory(Path.GetDirectoryName(oldFile)!);
File.WriteAllText(oldFile, "Some indexed text");
var record = Record(oldFile, Path.Combine("nested", "notes.md"));
var newRoot = Path.Combine(this.testRoot, "moved");
Directory.Move(oldRoot, newRoot);
var found = FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = newRoot }, oldFile, record);
Assert.That(found?.FullName, Is.EqualTo(Path.Combine(newRoot, "nested", "notes.md")));
}
[TestCase("size")]
[TestCase("modification time")]
[TestCase("missing file")]
public void AChangedOrMissingDocumentCannotReuseItsEmbeddings(string change)
{
var file = Path.Combine(this.testRoot, "notes.md");
File.WriteAllText(file, "Some indexed text");
var record = Record(file, "notes.md");
switch (change)
{
case "size":
File.AppendAllText(file, " with more text");
File.SetLastWriteTimeUtc(file, record.LastWriteUtc.UtcDateTime);
break;
case "modification time":
File.SetLastWriteTimeUtc(file, record.LastWriteUtc.UtcDateTime.AddMinutes(1));
break;
case "missing file":
File.Delete(file);
break;
}
// The old place lies outside the selected folder, so only the change can prevent the reuse:
var oldPath = Path.Combine(Path.GetDirectoryName(this.testRoot)!, "moved-away", "notes.md");
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, oldPath, record), Is.Null);
}
[Test]
public void AStoredPathBelowTheSelectedFolderIsNotMappedToACopy()
{
// The folder grows to its parent, which holds a copy with the same size and time at the
// place the old relative path names. The original is still there and stays the document.
var original = Path.Combine(this.testRoot, "project", "notes.md");
Directory.CreateDirectory(Path.GetDirectoryName(original)!);
File.WriteAllText(original, "Some indexed text");
var copy = Path.Combine(this.testRoot, "notes.md");
File.Copy(original, copy);
File.SetLastWriteTimeUtc(copy, File.GetLastWriteTimeUtc(original));
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, original, Record(original, "notes.md")), Is.Null);
}
[Test]
public void ARenamedSingleFileCanReuseItsEmbeddings()
{
var oldFile = Path.Combine(this.testRoot, "notes.md");
File.WriteAllText(oldFile, "Some indexed text");
var record = Record(oldFile, "notes.md");
var newFile = Path.Combine(this.testRoot, "renamed.md");
File.Move(oldFile, newFile);
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalFile { FilePath = newFile }, oldFile, record)?.FullName, Is.EqualTo(newFile));
}
[Test]
public void ARelativePathCannotEscapeTheConfiguredFolder()
{
var outside = Path.Combine(this.testRoot, "notes.md");
File.WriteAllText(outside, "Some indexed text");
var record = Record(outside, Path.Combine("..", "notes.md"));
var folder = Path.Combine(this.testRoot, "selected");
Directory.CreateDirectory(folder);
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = folder }, Path.Combine(this.testRoot, "old", "notes.md"), record), Is.Null);
}
private static EmbeddedFileRecord Record(string path, string relativePath)
{
var file = new FileInfo(path);
return new("old-fingerprint", file.Length, new DateTimeOffset(file.LastWriteTimeUtc), DateTimeOffset.UtcNow, 1, RelativePath: relativePath, ParentFileId: Guid.NewGuid().ToString());
}
[TestCase(@"E:\Documents\nested\notes.md", @"nested\notes.md")]
[TestCase(@"\\server\share\Documents\nested\notes.md", @"nested\notes.md")]
[TestCase(@"\\?\E:\Documents\nested\notes.md", @"nested\notes.md")]
[TestCase("E:/Documents/nested/notes.md", "nested/notes.md")]
[TestCase("/home/alice/Documents/nested/notes.md", "nested/notes.md")]
[TestCase("/Volumes/External drive/Documents/nested/notes.md", "nested/notes.md")]
public void ABackupFromAnotherPathSyntaxFindsANestedFile(string oldPath, string relativePath)
{
var localFile = Path.Combine(this.testRoot, "nested", "notes.md");
Directory.CreateDirectory(Path.GetDirectoryName(localFile)!);
File.WriteAllText(localFile, "Some indexed text");
var found = FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, oldPath, Record(localFile, relativePath));
Assert.That(found?.FullName, Is.EqualTo(localFile));
}
[TestCase(@"C:\a.folder\README")]
[TestCase(@"\\server\a.folder\README")]
[TestCase("/media/a.folder/README")]
public void AForeignDirectoryNameDoesNotBecomeTheExtensionOfAnExtensionlessFile(string oldPath)
{
var localFile = Path.Combine(this.testRoot, "README");
File.WriteAllText(localFile, "Some indexed text");
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalFile { FilePath = localFile }, oldPath, Record(localFile, "README"))?.FullName, Is.EqualTo(localFile));
}
[TestCase(@"..\notes.md")]
[TestCase("../notes.md")]
[TestCase(@"nested\..\..\notes.md")]
[TestCase(@"C:\notes.md")]
[TestCase(@"C:notes.md")]
[TestCase(@"\\server\share\notes.md")]
[TestCase("/etc/notes.md")]
public void AStoredWindowsPathCannotEscapeTheNewRoot(string relativePath)
{
Assert.That(RelocatedFilePaths.Resolve(this.testRoot, @"E:\Documents\notes.md", relativePath), Is.Null);
}
[Test]
public void AnExternalDriveRootIsAValidRelocationDestination()
{
// No file needs to be written at the drive root: this checks resolving the destination.
var root = Path.GetPathRoot(this.testRoot)!;
Assert.That(RelocatedFilePaths.Resolve(root, @"E:\notes.md", "notes.md"), Is.EqualTo(Path.Combine(root, "notes.md")));
}
[Test]
public void AUnixBackslashInAFileNameIsNotReinterpretedAsADirectory()
{
if (OperatingSystem.IsWindows())
{
Assert.That(RelocatedFilePaths.Resolve(this.testRoot, @"/home/alice/notes\draft.md", @"notes\draft.md"), Is.Null, "A Windows filename cannot contain a backslash from a Unix filename.");
return;
}
var localFile = Path.Combine(this.testRoot, @"notes\draft.md");
File.WriteAllText(localFile, "Some indexed text");
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, @"/home/alice/notes\draft.md", Record(localFile, @"notes\draft.md"))?.FullName, Is.EqualTo(localFile));
}
[Test]
public void AnOldForeignRootIsNotResolvedRelativeToTheWorkingDirectory()
{
var foreignRoot = OperatingSystem.IsWindows() ? "/home/alice/Documents" : @"E:\Documents";
Assert.That(RelocatedFilePaths.Resolve(foreignRoot, @"E:\Documents\notes.md", "notes.md"), Is.Null);
}
[Test]
public void ReducedTimestampPrecisionOnAnotherDriveDoesNotSilentlyReuseOldContent()
{
var localFile = Path.Combine(this.testRoot, "notes.md");
File.WriteAllText(localFile, "Some indexed text");
var record = Record(localFile, "notes.md");
// The old filesystem stored a finer timestamp than the copied file now reports. Without
// a stored content hash, equal size alone is not enough to prove that it is unchanged.
record = record with { LastWriteUtc = record.LastWriteUtc.AddTicks(1) };
Assert.That(FileSourceIndexer.FindRelocatedFile(new DataSourceLocalDirectory { Path = this.testRoot }, @"E:\Documents\notes.md", record), Is.Null);
}
[Test]
public void ANewFileAtAnOldLocationCannotOverwriteTheRelocatedDocument()
{
var source = new DataSourceLocalDirectory { Id = Guid.NewGuid().ToString(), Path = this.testRoot };
var originalPath = Path.Combine(this.testRoot, "nested", "notes.md");
var relocatedPath = Path.Combine(this.testRoot, "notes.md");
var originalId = IndexedDocumentIds.CreateParentId(source.Id, originalPath);
var manifest = new DataSourceEmbeddingManifest();
manifest.Files[relocatedPath] = new("fingerprint", 100, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 1, RelativePath: "notes.md", ParentFileId: originalId);
// Only identity assignment is exercised; none of the run's services are called.
var context = new IndexedRunContext(source, null!, null!, null!, null!, manifest,
new RunShare(RunShare.DEFAULT_DURATION), new IndexingPass(new DataSourceRunSignature("signature", "selection")), null!, null!, _ => { }, NullLogger.Instance);
var newId = context.GetFileParentId(originalPath);
Assert.Multiple(() =>
{
Assert.That(context.GetFileParentId(relocatedPath), Is.EqualTo(originalId));
Assert.That(newId, Is.Not.EqualTo(originalId));
Assert.That(context.GetFileParentId(originalPath), Is.EqualTo(newId), "A media file may ask for its identity once before and once after transcription.");
});
}
}
@@ -31,4 +31,21 @@ public sealed class IndexedDocumentIdsTests
{
Assert.That(IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, "/tmp/test-data/report.pdf"), Is.EqualTo("d06c4351-2e57-2941-aa4c-d7a814bfa418"));
}
[Test]
public void ADocumentStoredUnderTheIdOfItsKeyKeepsItsChunkIds()
{
const string KEY = "/tmp/test-data/report.pdf";
var source = IndexedDocumentIds.CreateChunkIdSource(DATA_SOURCE_ID, KEY, IndexedDocumentIds.CreateParentId(DATA_SOURCE_ID, KEY), "3F9A0C");
Assert.That(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, source, 0), Is.EqualTo("7c9d2822-f9d1-a94e-8d83-30cea4f0856f"), "Files and mails which never moved must keep the ids they are stored under.");
}
[Test]
public void TheChunkIdOfAMovedDocumentIsPinned()
{
var source = IndexedDocumentIds.CreateChunkIdSource(DATA_SOURCE_ID, "/tmp/moved/report.pdf", "0b5c2f7e-3d41-4a8e-9f60-2c7d1e8a4b93", "3F9A0C");
Assert.That(IndexedDocumentIds.CreateChunkId(DATA_SOURCE_ID, source, 0), Is.EqualTo("aa597d9b-ede8-5449-80d0-f209cde0ecc0"), "A file at the old place with the same fingerprint must not get the chunk ids of the moved one.");
}
}
+21 -2
View File
@@ -112,7 +112,7 @@ public sealed class OutdatedRunTests
}
[Test]
public void ASingleFileHasNoSelection()
public void RelocatingASingleFileStopsTheOldRunWithoutInvalidatingItsEmbeddings()
{
var dataSource = new DataSourceLocalFile
{
@@ -125,7 +125,26 @@ public sealed class OutdatedRunTests
FilePath = "/tmp/test-data/report.pdf",
};
Assert.That(DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM).SelectionSignature, Is.Empty, "The file is the one document of its data source, so there is nothing to choose among.");
var before = DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
var after = DataSourceEmbeddingService.GetRunSignature(dataSource with { FilePath = "/tmp/moved/report.pdf" }, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
Assert.Multiple(() =>
{
Assert.That(DataSourceEmbeddingService.IsRunOutdated(before, after), Is.True);
Assert.That(after.EmbeddingSignature, Is.EqualTo(before.EmbeddingSignature));
});
}
[Test]
public void RelocatingAFolderStopsTheOldRunWithoutInvalidatingItsEmbeddings()
{
var dataSource = DataSource(ConfidenceLevel.LOW);
var before = DataSourceEmbeddingService.GetRunSignature(dataSource, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
var after = DataSourceEmbeddingService.GetRunSignature(dataSource with { Path = "/tmp/moved" }, TestEmbeddingProvider(), ConfidenceLevel.MEDIUM);
Assert.Multiple(() =>
{
Assert.That(DataSourceEmbeddingService.IsRunOutdated(before, after), Is.True);
Assert.That(after.EmbeddingSignature, Is.EqualTo(before.EmbeddingSignature));
});
}
[Test]
+81
View File
@@ -0,0 +1,81 @@
using AIStudio.Tools;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Checks where a file dialog opens for a file or a folder which was moved or renamed.
/// </summary>
/// <remarks>
/// The native dialog cannot open a place which no longer exists and falls back to wherever it was
/// used last, which may be anywhere. The nearest folder which still exists is where the file or the
/// folder was moved from, so its new place is often close by.
/// </remarks>
[TestFixture]
public sealed class PathToolsTests
{
private string testRoot = string.Empty;
[SetUp]
public void CreateTestRoot()
{
this.testRoot = Path.Combine(Path.GetTempPath(), $"ai-studio-path-tools-{Guid.NewGuid():N}");
Directory.CreateDirectory(this.testRoot);
}
[TearDown]
public void DeleteTestRoot()
{
if (Directory.Exists(this.testRoot))
Directory.Delete(this.testRoot, true);
}
[Test]
public void AnExistingFolderOpensAsItIs()
{
var folder = Directory.CreateDirectory(Path.Combine(this.testRoot, "documents")).FullName;
Assert.That(PathTools.FindNearestExistingPath(folder), Is.EqualTo(folder));
}
[Test]
public void AnExistingFileOpensAsItIs()
{
var file = Path.Combine(this.testRoot, "report.pdf");
File.WriteAllText(file, "report");
Assert.That(PathTools.FindNearestExistingPath(file), Is.EqualTo(file));
}
[Test]
public void AMovedFolderOpensAtItsNearestFolderWhichStillExists()
{
var parent = Directory.CreateDirectory(Path.Combine(this.testRoot, "projects")).FullName;
var movedFolder = Path.Combine(parent, "2026", "documents");
Assert.That(PathTools.FindNearestExistingPath(movedFolder), Is.EqualTo(parent), "Every parent folder counts, not only the immediate one.");
}
[Test]
public void AMovedFileOpensAtItsFolder()
{
var file = Path.Combine(this.testRoot, "report.pdf");
Assert.That(PathTools.FindNearestExistingPath(file), Is.EqualTo(this.testRoot));
}
[TestCase("")]
[TestCase(" ")]
[TestCase("documents/report.pdf")]
public void APathWithoutAPlaceOpensWhereverTheDialogWants(string path)
{
Assert.That(PathTools.FindNearestExistingPath(path), Is.Null);
}
[Test]
public void APathOfAnotherOperatingSystemOpensWhereverTheDialogWants()
{
var path = OperatingSystem.IsWindows() ? "/Users/somebody/documents" : @"C:\Users\somebody\documents";
Assert.That(PathTools.FindNearestExistingPath(path), Is.Null, "A backup from another computer must not make the dialog open at a folder which happens to exist here.");
}
}
+93
View File
@@ -0,0 +1,93 @@
using AIStudio.Tools.Databases.VectorStore;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Keeps the identities and locations of existing vectors. New embeddings are forbidden: these
/// tests reconnect previously indexed files and must never contact an embedding provider.
/// </summary>
internal sealed class RelocationVectorStore() : VectorStoreClient("Relocation test vectors", string.Empty)
{
public Dictionary<string, string> Locations { get; } = new(StringComparer.Ordinal);
public List<string> RelocatedPaths { get; } = [];
public TaskCompletionSource<object?> RelocationStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously);
public bool BlockNextRelocation { get; set; }
public int CanceledRelocations { get; private set; }
public int DeletedStores { get; private set; }
public int DeletedDocuments { get; private set; }
public int InsertedBatches { get; private set; }
public override async Task RelocateEmbeddingFileAsync(string storeName, string parentFileId, string absolutePath, string relativePath, string fingerprint, CancellationToken token)
{
token.ThrowIfCancellationRequested();
Assert.That(this.Locations.ContainsKey(parentFileId), Is.True, "Relocation must address an existing vector identity.");
this.Locations[parentFileId] = absolutePath;
this.RelocatedPaths.Add(absolutePath);
if (!this.BlockNextRelocation)
return;
this.BlockNextRelocation = false;
this.RelocationStarted.TrySetResult(null);
try
{
await Task.Delay(Timeout.InfiniteTimeSpan, token);
}
catch (OperationCanceledException)
{
this.CanceledRelocations++;
throw;
}
}
public override Task<VectorStoreEnsureResult> EnsureVectorStoreExists(string storeName, string dataSourceName, int vectorSize, CancellationToken token) => Task.FromResult(new VectorStoreEnsureResult(false));
public override Task<bool> VectorStoreExists(string storeName, CancellationToken token) => Task.FromResult(true);
public override Task InsertEmbedding(string storeName, IReadOnlyList<VectorStoragePoint> points, CancellationToken token)
{
this.InsertedBatches++;
throw new InvalidOperationException("Existing unchanged files must not be embedded again.");
}
public override Task DeleteEmbeddingByFile(string storeName, string filePath, string parentFileId, CancellationToken token)
{
this.DeletedDocuments++;
foreach (var id in this.Locations.Where(entry => entry.Key == parentFileId || entry.Value == filePath).Select(entry => entry.Key).ToArray())
this.Locations.Remove(id);
return Task.CompletedTask;
}
public override Task DeleteVectorStore(string storeName, CancellationToken token)
{
this.DeletedStores++;
this.Locations.Clear();
return Task.CompletedTask;
}
public override Task OptimizeVectorStore(string storeName, CancellationToken token) => Task.CompletedTask;
public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, CancellationToken token) => throw new NotSupportedException();
public override Task<IReadOnlyList<VectorSearchResult>> SearchEmbeddingAsync(string storeName, IReadOnlyList<float> vector, int maxMatches, VectorSearchFilter filter, CancellationToken token) => throw new NotSupportedException();
public override Task<VectorStoreBackupPreparation> PrepareBackupAsync(CancellationToken token) => throw new NotSupportedException();
public override Task<long> CopyStoreForBackupAsync(string backupId, string storeName, CancellationToken token) => throw new NotSupportedException();
public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo()
{
await Task.CompletedTask;
yield break;
}
public override void Dispose()
{
}
}
+76
View File
@@ -0,0 +1,76 @@
using System.Reflection;
using MudBlazor;
namespace AIStudio.Tests.Tools;
/// <summary>
/// Controls a dialog result without rendering a Blazor circuit. The real dialog save handler and
/// the real parent edit handler still run, with the same parameters and result as in the UI.
/// </summary>
internal sealed class TestDataSourceDialog
{
private readonly TaskCompletionSource<DialogResult?> result = new(TaskCreationOptions.RunContinuationsAsynchronously);
public IDialogService Service { get; }
public IMudDialogInstance Instance { get; }
public DialogParameters? Parameters { get; private set; }
public Type? ComponentType { get; private set; }
public string? Title { get; private set; }
public int ShowCount { get; private set; }
public TestDataSourceDialog()
{
var reference = Create<IDialogReference>((method, _) => method.Name == "get_Result"
? this.result.Task
: throw new NotSupportedException(method.Name));
this.Service = Create<IDialogService>((method, arguments) =>
{
if (method.Name != "ShowAsync")
throw new NotSupportedException(method.Name);
ArgumentNullException.ThrowIfNull(arguments);
this.ShowCount++;
this.ComponentType = method.GetGenericArguments().Single();
this.Title = arguments.OfType<string>().FirstOrDefault();
this.Parameters = arguments.OfType<DialogParameters>().Single();
return Task.FromResult(reference);
});
this.Instance = Create<IMudDialogInstance>((method, arguments) =>
{
switch (method.Name)
{
case "Close":
this.Complete(arguments!.OfType<DialogResult>().Single());
return null;
case "Cancel":
this.Complete(DialogResult.Cancel());
return null;
default:
throw new NotSupportedException(method.Name);
}
});
}
public void Complete(DialogResult dialogResult) => this.result.TrySetResult(dialogResult);
private static T Create<T>(Func<MethodInfo, object?[]?, object?> invoke) where T : class
{
var proxy = DispatchProxy.Create<T, DialogCallProxy>();
((DialogCallProxy)(object)proxy).InvokeCall = invoke;
return proxy;
}
}
// DispatchProxy generates a subclass, so this type must be public and cannot be sealed.
public class DialogCallProxy : DispatchProxy
{
public Func<MethodInfo, object?[]?, object?> InvokeCall { get; set; } = null!;
protected override object? Invoke(MethodInfo? targetMethod, object?[]? args) => this.InvokeCall(targetMethod!, args);
}
@@ -92,7 +92,7 @@ public sealed class SemanticSearchToolUnreachableTests
}
[Test]
public void TheResultNamesAnUnreachableDataSourceWithoutItsID()
public void TheResultNamesAnUnreachableDataSourceWithoutItsId()
{
var result = SemanticSearchTool.DescribeUnreachableResult(INTRANET);
+60 -3
View File
@@ -62,6 +62,49 @@ public sealed class UnreachableDataSourceTests
AssertTheIndexWasKept(outcome, statuses, manifest, 2, folder);
}
[Test]
public async Task AMissingRelocationTargetKeepsTheOriginalIndex()
{
var folder = Path.Combine(this.testRoot, "moved-but-unavailable");
var (outcome, statuses, manifest) = await RunAsync(Folder(folder), Path.Combine(this.testRoot, "original", "report.pdf"));
AssertTheIndexWasKept(outcome, statuses, manifest, 1, folder);
}
[TestCase(false)]
[TestCase(true)]
public async Task AReadableEmptyMountDirectoryKeepsItsIndex(bool emptySubfolder)
{
var folder = Path.Combine(this.testRoot, "mounted-drive");
Directory.CreateDirectory(emptySubfolder ? Path.Combine(folder, "documents") : folder);
var unfinishedFile = Path.Combine(folder, "unfinished.pdf");
var (outcome, statuses, manifest) = await RunAsync(Folder(folder), [Path.Combine(folder, "report.pdf")], [unfinishedFile]);
AssertTheIndexWasKept(outcome, statuses, manifest, 1, folder);
Assert.Multiple(() =>
{
Assert.That(manifest.UnfinishedFiles, Does.Contain(unfinishedFile), "An unmounted drive must not discard interrupted work either.");
Assert.That(statuses[^1].LastError, Does.Contain("appears empty").And.Contain("remove this data source"));
});
}
[Test]
public async Task ANewEmptyFolderCanFinishWithoutAnIndex()
{
var folder = Path.Combine(this.testRoot, "new-empty-folder");
Directory.CreateDirectory(folder);
var (outcome, statuses, manifest) = await RunAsync(Folder(folder));
Assert.Multiple(() =>
{
Assert.That(outcome, Is.EqualTo(IndexedRunOutcome.DONE));
Assert.That(statuses[^1].State, Is.EqualTo(DataSourceEmbeddingState.COMPLETED));
Assert.That(manifest.Files, Is.Empty);
});
}
[Test]
public async Task AMissingFileKeepsItsIndex()
{
@@ -130,6 +173,15 @@ public sealed class UnreachableDataSourceTests
Assert.That(goneFiles, Is.EquivalentTo([deleted, besideSubfolder]), "A folder whose name only begins like the one which could not be read is no part of it.");
}
[Test]
public void AnUnreadableDriveRootKeepsItsIndexedFiles()
{
var root = Path.GetPathRoot(Path.GetFullPath(Path.GetTempPath()))!;
var file = Path.Combine(root, "documents", "notes.md");
Assert.That(FileSourceIndexer.GetGoneFiles([file], [], [root]), Is.Empty, "A drive root already ends in its separator; adding another would fail to protect its files.");
}
private static DataSourceLocalDirectory Folder(string path) => new()
{
Num = 1,
@@ -152,6 +204,7 @@ public sealed class UnreachableDataSourceTests
Assert.That(statuses[^1].State, Is.EqualTo(DataSourceEmbeddingState.FAILED));
Assert.That(statuses[^1].IndexedDocuments, Is.EqualTo(indexedFiles), "The index still holds these files, and the page has to say so.");
Assert.That(statuses[^1].LastError, Does.Contain(unreachablePath));
Assert.That(statuses[^1].Attention, Is.EqualTo(DataSourceAttention.LOCATION_UNREACHABLE), "The embeddings page offers to edit the data source, e.g. to select the new location of a moved folder.");
});
}
@@ -181,12 +234,16 @@ public sealed class UnreachableDataSourceTests
var statuses = new List<DataSourceEmbeddingStatus>();
var manifest = new DataSourceEmbeddingManifest { SourceHash = STORED_SOURCE_HASH };
foreach (var file in indexedFiles)
manifest.Files[file] = new EmbeddedFileRecord("FINGERPRINT", 1024, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 4);
manifest.Files[file] = new EmbeddedFileRecord("FINGERPRINT", 1024, DateTimeOffset.UnixEpoch, DateTimeOffset.UnixEpoch, 4,
RelativePath: Path.GetFileName(file), ParentFileId: IndexedDocumentIds.CreateParentId(dataSource.Id, file));
manifest.UnfinishedFiles.UnionWith(unfinishedFiles);
var scannedFile = Path.Combine(Path.GetDirectoryName(indexedFiles[0])!, "scan.pdf");
manifest.PermanentFailures[scannedFile] = new PermanentIndexingFailureRecord("FINGERPRINT", FileExtractionErrorCode.NO_TEXT_EXTRACTED, "No text could be read.", DateTimeOffset.UnixEpoch);
if (indexedFiles.Length > 0)
{
var scannedFile = Path.Combine(Path.GetDirectoryName(indexedFiles[0])!, "scan.pdf");
manifest.PermanentFailures[scannedFile] = new PermanentIndexingFailureRecord("FINGERPRINT", FileExtractionErrorCode.NO_TEXT_EXTRACTED, "No text could be read.", DateTimeOffset.UnixEpoch);
}
var embeddingProvider = new EmbeddingProvider(1, EMBEDDING_ID, "Test embeddings", LLMProviders.OPEN_AI, new("text-embedding-3-small", "text-embedding-3-small"));
var settingsManager = new SettingsManager(NullLogger<SettingsManager>.Instance, null!);
+144 -4
View File
@@ -11,9 +11,9 @@ use qdrant_edge::external::uuid::Uuid;
use qdrant_edge::{
Condition, Distance, EdgeConfig, EdgeOptimizersConfig, EdgeShard, EdgeVectorParams,
FieldCondition, Filter, HasIdCondition, HnswIndexConfig, Match, MatchValue, NamedQuery,
Payload, PointId, PointInsertOperations, PointOperations, PointStruct, QueryEnum,
Payload, PayloadOps, PointId, PointInsertOperations, PointOperations, PointStruct, QueryEnum,
QueryRequest, ScoredPoint, ScoringQuery, UpdateOperation, ValueVariants, VectorInternal,
Vectors, WithPayloadInterface, WithVector,
SetPayloadOp, Vectors, WithPayloadInterface, WithVector,
};
use serde::{Deserialize, Serialize};
use tauri::Manager;
@@ -131,6 +131,17 @@ pub struct SearchQdrantEdgeEmbeddingRequest {
pub struct DeleteQdrantEdgeEmbeddingByFileRequest {
pub store_name: String,
pub file_path: String,
pub parent_file_id: String,
}
#[derive(Deserialize)]
pub struct RelocateQdrantEdgeEmbeddingFileRequest {
pub store_name: String,
pub parent_file_id: String,
pub absolute_path: String,
pub file_name: String,
pub relative_path: String,
pub fingerprint: String,
}
#[derive(Deserialize)]
@@ -459,11 +470,17 @@ impl QdrantEdgeDatabase {
.collect())
}
fn delete_embedding_by_file(&mut self, store_name: &str, file_path: &str) -> QdrantEdgeResult<()> {
fn delete_embedding_by_file(&mut self, store_name: &str, file_path: &str, parent_file_id: &str) -> QdrantEdgeResult<()> {
let Some(shard) = self.get_existing_store(store_name)? else {
return Ok(());
};
// The location may already have changed if a relocation stopped before SQLite caught up.
if !parent_file_id.is_empty() {
shard.update(UpdateOperation::PointOperation(
PointOperations::DeletePointsByFilter(match_keyword_filter("parent_file_id", parent_file_id)?),
))?;
}
shard.update(UpdateOperation::PointOperation(
PointOperations::DeletePointsByFilter(match_keyword_filter("file_path", file_path)?),
))?;
@@ -471,6 +488,33 @@ impl QdrantEdgeDatabase {
Ok(())
}
fn relocate_embedding_file(&mut self, request: RelocateQdrantEdgeEmbeddingFileRequest) -> QdrantEdgeResult<()> {
if request.parent_file_id.is_empty() {
return Err("Relocating a document requires its parent file id.".into());
}
let Some(shard) = self.get_existing_store(&request.store_name)? else {
return Err("The vector store for the relocated document is missing.".into());
};
let payload = json!({
"file_path": request.absolute_path,
"absolute_path": request.absolute_path,
"file_name": request.file_name,
"relative_path": request.relative_path,
"fingerprint": request.fingerprint,
});
// Parent and chunk ids are identities, even when they were originally derived from a path.
// Keep them and the vectors intact; only the document's location changes.
shard.update(UpdateOperation::PayloadOperation(PayloadOps::SetPayload(SetPayloadOp {
payload: qdrant_edge::external::serde_json::from_value(payload)?,
points: None,
filter: Some(match_keyword_filter("parent_file_id", &request.parent_file_id)?),
key: None,
})))?;
shard.flush()?;
Ok(())
}
fn optimize_store(&mut self, store_name: &str) -> QdrantEdgeResult<()> {
let Some(shard) = self.get_existing_store(store_name)? else {
return Ok(());
@@ -776,7 +820,13 @@ pub async fn search_qdrant_edge_embeddings(Json(request): Json<SearchQdrantEdgeE
pub async fn delete_qdrant_edge_embedding_by_file(Json(request): Json<DeleteQdrantEdgeEmbeddingByFileRequest>) -> Json<QdrantEdgeResponse<()>> {
run_qdrant_edge_request(move |database| {
database.delete_embedding_by_file(&request.store_name, &request.file_path)
database.delete_embedding_by_file(&request.store_name, &request.file_path, &request.parent_file_id)
}).await
}
pub async fn relocate_qdrant_edge_embedding_file(Json(request): Json<RelocateQdrantEdgeEmbeddingFileRequest>) -> Json<QdrantEdgeResponse<()>> {
run_qdrant_edge_request(move |database| {
database.relocate_embedding_file(request)
}).await
}
@@ -1489,6 +1539,49 @@ mod tests {
);
}
#[test]
fn relocating_a_document_preserves_its_vectors_and_can_be_retried() {
let test_directory = test_directory("relocation");
let mut database = QdrantEdgeDatabase::new(test_directory.clone());
let mut point = test_point(POINT_X, search_vector());
point.parent_file_id = "original-parent".to_string();
point.page_number = Some(2);
database.insert_embedding(SEARCH_STORE, vec![point]).unwrap();
for _ in 0..2 {
database.relocate_embedding_file(RelocateQdrantEdgeEmbeddingFileRequest {
store_name: SEARCH_STORE.to_string(),
parent_file_id: "original-parent".to_string(),
absolute_path: "/tmp/moved/renamed.md".to_string(),
file_name: "renamed.md".to_string(),
relative_path: "renamed.md".to_string(),
fingerprint: "new-fingerprint".to_string(),
}).unwrap();
}
drop(database);
let mut database = QdrantEdgeDatabase::new(test_directory.clone());
let found = database.search_embedding(SEARCH_STORE, search_vector(), 10, None).unwrap();
assert_eq!(found.len(), 1);
assert_eq!(found[0].point_id, POINT_X);
assert_eq!(found[0].parent_file_id, "original-parent");
assert_eq!(found[0].file_path, "/tmp/moved/renamed.md");
assert_eq!(found[0].absolute_path, "/tmp/moved/renamed.md");
assert_eq!(found[0].file_name, "renamed.md");
assert_eq!(found[0].relative_path, "renamed.md");
assert_eq!(found[0].fingerprint, "new-fingerprint");
assert_eq!(found[0].text, "Some text.");
assert_eq!(found[0].page_number, Some(2));
assert!((found[0].score - 1.0).abs() < 0.0001);
// SQLite may still know the old path when a relocation was interrupted. Its stable parent
// id must be enough to remove the vectors if the document changed before the next retry.
database.delete_embedding_by_file(SEARCH_STORE, "/tmp/test-data/notes.md", "original-parent").unwrap();
assert!(database.search_embedding(SEARCH_STORE, search_vector(), 10, None).unwrap().is_empty());
drop(database);
fs::remove_dir_all(test_directory).unwrap();
}
#[test]
fn obsolete_wal_segments_keeps_the_newest_closed_segment() {
let file_names = ["closed-120", "open-7", "closed-0", "tmp-open-8", "closed-40", ".wal", "first-index", "closed-abc"].map(String::from);
@@ -1651,6 +1744,53 @@ mod tests {
fs::remove_dir_all(data_directory).unwrap();
}
#[test]
fn a_restored_store_can_relocate_foreign_paths_without_replacing_its_vectors() {
for old_path in [r"E:\Documents\nested\notes.md", r"\\server\share\nested\notes.md", "/Volumes/External drive/nested/notes.md"] {
let data_directory = test_directory("backup-relocation");
let base_path = data_directory.join("databases").join("vector_database");
let mut database = QdrantEdgeDatabase::new(base_path);
let mut point = test_point(POINT_X, vec![1.0, 0.0, 0.0]);
point.parent_file_id = "original-parent".to_string();
point.file_path = old_path.to_string();
point.absolute_path = old_path.to_string();
point.embedded_at_utc = "2026-10-01T12:00:00Z".to_string();
database.insert_embedding(SEARCH_STORE, vec![point]).unwrap();
let preparation = database.prepare_backup(&data_directory).unwrap();
database.copy_store_for_backup(SEARCH_STORE, &data_directory, &preparation.backup_id).unwrap();
let restored_base = PathBuf::from(&preparation.staging_path).join("databases").join("vector_database");
let mut restored = QdrantEdgeDatabase::new(restored_base.clone());
let new_path = data_directory.join("mounted drive").join("nested").join("notes.md").to_string_lossy().to_string();
restored.relocate_embedding_file(RelocateQdrantEdgeEmbeddingFileRequest {
store_name: SEARCH_STORE.to_string(),
parent_file_id: "original-parent".to_string(),
absolute_path: new_path.clone(),
file_name: "notes.md".to_string(),
relative_path: Path::new("nested").join("notes.md").to_string_lossy().to_string(),
fingerprint: "relocated-fingerprint".to_string(),
}).unwrap();
drop(restored);
let mut restored = QdrantEdgeDatabase::new(restored_base);
let found = restored.search_embedding(SEARCH_STORE, vec![1.0, 0.0, 0.0], 10, None).unwrap();
assert_eq!(found.len(), 1);
assert_eq!(found[0].point_id, POINT_X);
assert_eq!(found[0].parent_file_id, "original-parent");
assert_eq!(found[0].file_path, new_path);
assert_eq!(found[0].absolute_path, new_path);
assert_eq!(found[0].text, "Some text.");
assert_eq!(found[0].embedded_at_utc, "2026-10-01T12:00:00Z");
assert!((found[0].score - 1.0).abs() < 0.0001);
let original = database.search_embedding(SEARCH_STORE, vec![1.0, 0.0, 0.0], 10, None).unwrap();
assert_eq!(original[0].file_path, old_path, "relocating a restored copy must not change the original store");
drop(restored);
drop(database);
fs::remove_dir_all(data_directory).unwrap();
}
}
#[test]
fn a_backup_id_cannot_name_another_directory() {
let data_directory = Path::new("/data");
+1
View File
@@ -42,6 +42,7 @@ fn create_router() -> Router {
.route("/system/qdrant-edge/insert", post(crate::qdrant_edge_database::insert_qdrant_edge_embedding))
.route("/system/qdrant-edge/search", post(crate::qdrant_edge_database::search_qdrant_edge_embeddings))
.route("/system/qdrant-edge/delete-file", post(crate::qdrant_edge_database::delete_qdrant_edge_embedding_by_file))
.route("/system/qdrant-edge/relocate-file", post(crate::qdrant_edge_database::relocate_qdrant_edge_embedding_file))
.route("/system/qdrant-edge/optimize", post(crate::qdrant_edge_database::optimize_qdrant_edge_store))
.route("/system/qdrant-edge/delete-store", post(crate::qdrant_edge_database::delete_qdrant_edge_store))
.route("/system/qdrant-edge/backup/prepare", post(crate::qdrant_edge_database::prepare_qdrant_edge_backup))