mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-14 03:33:48 +00:00
Added local RAG (#756)
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Paul KoudelkaandThorsten Sommer authored and 
GitHub committed
1 parent
d043fbc8f0
commit
c7b42bee96
179 files changed
+276977
-1421
No files matched your search
@@ -0,0 +1,176 @@
|
||||
@attribute [Route(Routes.EMBEDDINGS)]
|
||||
@inherits MSGComponentBase
|
||||
|
||||
<MudStack Spacing="3" Class="pr-2 pb-4" Style="height: 100%; min-height: 0; overflow-y: auto;">
|
||||
<MudPaper Class="pa-4 border-dashed border rounded-lg" Elevation="0">
|
||||
<MudStack Row="true" AlignItems="AlignItems.Center">
|
||||
<MudText Typo="Typo.h5">@T("Background embeddings")</MudText>
|
||||
<MudSpacer/>
|
||||
<MudTooltip Text="@T("Manage your data sources")" Placement="Placement.Top">
|
||||
<MudIconButton Icon="@Icons.Material.Filled.Settings" OnClick="@this.OpenDataSourceSettings"/>
|
||||
</MudTooltip>
|
||||
</MudStack>
|
||||
<MudText Typo="Typo.body1" Class="mt-2">
|
||||
@T("AI Studio indexes local RAG data sources in the background. Finished files stay recorded so unchanged files can be skipped after a restart, while added or deleted files are detected during the next run. The same applies to documents without readable text, such as scanned pages: AI Studio remembers them and reads them again only once they change.")
|
||||
</MudText>
|
||||
<MudStack Row="true" Class="mt-3" Wrap="Wrap.Wrap" Spacing="2">
|
||||
<MudChip T="string" Color="Color.Success" Variant="Variant.Filled">@string.Format(T("Indexed files: {0}"), this.TotalIndexedFiles)</MudChip>
|
||||
<MudChip T="string" Color="Color.Info" Variant="Variant.Filled">@string.Format(T("Pending files: {0}"), this.TotalPendingFiles)</MudChip>
|
||||
<MudChip T="string" Color="Color.Default" Variant="Variant.Filled">@string.Format(T("Skipped files: {0}"), this.TotalPermanentlySkippedFiles)</MudChip>
|
||||
<MudChip T="string" Color="Color.Error" Variant="Variant.Filled">@string.Format(T("Failed files: {0}"), this.TotalFailedFiles)</MudChip>
|
||||
</MudStack>
|
||||
</MudPaper>
|
||||
|
||||
@if (this.Statuses.Count == 0)
|
||||
{
|
||||
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined">
|
||||
@T("No local data source has been queued for embedding yet.")
|
||||
</MudAlert>
|
||||
}
|
||||
else
|
||||
{
|
||||
@*
|
||||
One panel per data source, and only one of them open: a folder with thousands of files
|
||||
fills the page by itself. What a closed panel still has to say stays in its header, so
|
||||
nobody has to open every one of them to see how their data sources are doing.
|
||||
*@
|
||||
<MudExpansionPanels>
|
||||
@foreach (var status in this.Statuses)
|
||||
{
|
||||
<MudExpansionPanel Class="border-solid border rounded-lg" Expanded="@(this.expandedDataSourceId == status.DataSourceId)" ExpandedChanged="@(isExpanded => this.DataSourcePanelExpandedChanged(status, isExpanded))">
|
||||
<TitleContent>
|
||||
<div class="d-flex align-center">
|
||||
<MudText Typo="Typo.h6">@string.Format(T("Data source: {0}"), status.DataSourceName)</MudText>
|
||||
<MudSpacer/>
|
||||
<MudChip T="string" Color="@GetStatusColor(status)" Variant="Variant.Filled">@status.StateLabel</MudChip>
|
||||
@if (CanRefresh(status))
|
||||
{
|
||||
@*
|
||||
This opens or closes the panel along the way. The header is what
|
||||
toggles it, and neither stopping the event nor swallowing the
|
||||
click works in this project — see the end button of our own
|
||||
ExpansionPanel component, which behaves the same way.
|
||||
*@
|
||||
<MudTooltip Text="@T("Refresh this data source")">
|
||||
<MudIconButton Icon="@Icons.Material.Filled.Sync" Color="Color.Info" Size="Size.Small" OnClick="@(async () => await this.RefreshDataSource(status))" />
|
||||
</MudTooltip>
|
||||
}
|
||||
</div>
|
||||
</TitleContent>
|
||||
<ChildContent>
|
||||
<MudStack Spacing="2">
|
||||
<MudProgressLinear Value="@status.ProgressPercent" Rounded="@true" Color="@GetStatusColor(status)" />
|
||||
|
||||
<MudText Typo="Typo.body2">
|
||||
@string.Format(T("{0} of {1} files are indexed."), status.IndexedFiles, status.TotalFiles)
|
||||
</MudText>
|
||||
|
||||
@if (status.PermanentlySkippedFiles > 0)
|
||||
{
|
||||
<MudText Typo="Typo.body2">
|
||||
@string.Format(T("Skipped files: {0}. AI Studio reads them again once they change."), status.PermanentlySkippedFiles)
|
||||
</MudText>
|
||||
}
|
||||
|
||||
@if (status.FailedFiles > 0)
|
||||
{
|
||||
<MudText Typo="Typo.body2">
|
||||
@string.Format(T("Failed files: {0}"), status.FailedFiles)
|
||||
</MudText>
|
||||
}
|
||||
|
||||
@if (!string.IsNullOrWhiteSpace(status.CurrentFile))
|
||||
{
|
||||
<MudText Typo="Typo.body2">
|
||||
@string.Format(T("Current file: {0}"), status.CurrentFile)
|
||||
</MudText>
|
||||
}
|
||||
|
||||
@*
|
||||
One panel per cause, the most pressing one first. A folder in which nine
|
||||
hundred scanned documents were skipped otherwise buries the handful of
|
||||
files somebody has to look at.
|
||||
*@
|
||||
@if (status.Failures.Count > 0)
|
||||
{
|
||||
<MudExpansionPanels MultiExpansion="@true" Class="mt-2">
|
||||
@foreach (var group in this.GetFailureGroups(status))
|
||||
{
|
||||
<ExpansionPanel HeaderIcon="@group.Cause.Icon" IconColor="@group.Cause.Color" IconSize="Size.Small" HeaderTypo="Typo.subtitle1" HeaderClass="expansion-panel-header-compact" HeaderText="@GetGroupHeader(group)">
|
||||
<MudStack Spacing="3">
|
||||
<MudStack Row="@true" Wrap="Wrap.Wrap" Spacing="2" AlignItems="AlignItems.Center">
|
||||
<MudChip T="string" Size="Size.Small" Color="@group.Cause.Color" Variant="Variant.Outlined">
|
||||
@(group.Cause.IsPermanent ? T("Skipped until the file changes") : T("Tried again during the next run"))
|
||||
</MudChip>
|
||||
@if (!string.IsNullOrWhiteSpace(group.EmbeddingProviderName))
|
||||
{
|
||||
<MudText Typo="Typo.caption">@string.Format(T("Embedding provider: {0}"), group.EmbeddingProviderName)</MudText>
|
||||
}
|
||||
@if (group.Cause.NeedsProviderSettings)
|
||||
{
|
||||
<MudButton Variant="Variant.Text" Size="Size.Small" Color="Color.Primary" StartIcon="@Icons.Material.Filled.Settings" OnClick="@this.OpenEmbeddingProviderSettings">
|
||||
@T("Open the settings")
|
||||
</MudButton>
|
||||
}
|
||||
</MudStack>
|
||||
<MudTable Items="@group.Failures" Dense="@true" Hover="@true" Elevation="0" Breakpoint="Breakpoint.None" RowsPerPage="10" Class="border-dashed border rounded-lg">
|
||||
<ColGroup>
|
||||
<col/>
|
||||
<col style="width: 11em;"/>
|
||||
<col style="width: 7em;"/>
|
||||
</ColGroup>
|
||||
<HeaderContent>
|
||||
<MudTh>@T("File")</MudTh>
|
||||
<MudTh>@T("Noticed")</MudTh>
|
||||
<MudTh>@T("Actions")</MudTh>
|
||||
</HeaderContent>
|
||||
<RowTemplate>
|
||||
<MudTd Style="white-space: normal; overflow-wrap: anywhere;">
|
||||
@* Captions render as spans, so each of them needs to be told to take its own line. *@
|
||||
<MudText Typo="Typo.body2">@GetFileName(context.FilePath)</MudText>
|
||||
<MudText Typo="Typo.caption" Class="mud-text-secondary d-block">@context.FilePath</MudText>
|
||||
@if (group.Cause.ShowsMessagePerFile)
|
||||
{
|
||||
<MudText Typo="Typo.caption" Color="Color.Warning" Class="d-block">@context.Reason</MudText>
|
||||
}
|
||||
</MudTd>
|
||||
<MudTd Style="white-space: nowrap;">
|
||||
<MudText Typo="Typo.caption">@GetOccurrenceText(context)</MudText>
|
||||
</MudTd>
|
||||
<MudTd>
|
||||
@if (CanShowInFileManager(context))
|
||||
{
|
||||
<MudTooltip Text="@T("Show this file in the file browser of your system")">
|
||||
<MudIconButton Icon="@Icons.Material.Filled.FolderOpen" Color="Color.Info" Size="Size.Small" OnClick="@(async () => await this.ShowInFileManager(context))"/>
|
||||
</MudTooltip>
|
||||
}
|
||||
</MudTd>
|
||||
</RowTemplate>
|
||||
<PagerContent>
|
||||
<MudTablePager PageSizeOptions="@PAGE_SIZE_OPTIONS"/>
|
||||
</PagerContent>
|
||||
</MudTable>
|
||||
</MudStack>
|
||||
</ExpansionPanel>
|
||||
}
|
||||
</MudExpansionPanels>
|
||||
}
|
||||
|
||||
@*
|
||||
Shown next to the list, not instead of it: the list says which files failed,
|
||||
while this is the one sentence about the data source as a whole. Hiding it
|
||||
as soon as a single file failed is what made it invisible in practice.
|
||||
*@
|
||||
@if (!string.IsNullOrWhiteSpace(status.LastError))
|
||||
{
|
||||
<MudAlert Severity="Severity.Warning" Variant="Variant.Text">
|
||||
@status.LastError
|
||||
</MudAlert>
|
||||
}
|
||||
</MudStack>
|
||||
</ChildContent>
|
||||
</MudExpansionPanel>
|
||||
}
|
||||
</MudExpansionPanels>
|
||||
}
|
||||
</MudStack>
|
||||
@@ -0,0 +1,277 @@
|
||||
using AIStudio.Components;
|
||||
using AIStudio.Dialogs.Settings;
|
||||
using AIStudio.Provider;
|
||||
using AIStudio.Settings.DataModel;
|
||||
using AIStudio.Tools.Rust;
|
||||
using AIStudio.Tools.Services;
|
||||
|
||||
using Microsoft.AspNetCore.Components;
|
||||
|
||||
using DialogOptions = AIStudio.Dialogs.DialogOptions;
|
||||
|
||||
namespace AIStudio.Pages;
|
||||
|
||||
public partial class Embeddings : MSGComponentBase
|
||||
{
|
||||
private static readonly int[] PAGE_SIZE_OPTIONS = [10, 25, 50, 100];
|
||||
|
||||
[Inject]
|
||||
private DataSourceEmbeddingService DataSourceEmbeddingService { get; init; } = null!;
|
||||
|
||||
[Inject]
|
||||
private NavigationManager NavigationManager { get; init; } = null!;
|
||||
|
||||
[Inject]
|
||||
private RustService RustService { get; init; } = null!;
|
||||
|
||||
[Inject]
|
||||
private IDialogService DialogService { get; init; } = null!;
|
||||
|
||||
[Inject]
|
||||
private ILogger<Embeddings> Logger { get; init; } = null!;
|
||||
|
||||
private IReadOnlyList<DataSourceEmbeddingStatus> Statuses { get; set; } = [];
|
||||
|
||||
private string? expandedDataSourceId;
|
||||
private bool userChoseExpansion;
|
||||
|
||||
private int TotalIndexedFiles => this.Statuses.Sum(status => status.IndexedFiles);
|
||||
|
||||
private int TotalPendingFiles => this.Statuses.Sum(status => Math.Max(0, status.TotalFiles - status.IndexedFiles - status.FailedFiles - status.PermanentlySkippedFiles));
|
||||
|
||||
private int TotalFailedFiles => this.Statuses.Sum(status => status.FailedFiles);
|
||||
|
||||
private int TotalPermanentlySkippedFiles => this.Statuses.Sum(status => status.PermanentlySkippedFiles);
|
||||
|
||||
protected override async Task OnInitializedAsync()
|
||||
{
|
||||
//
|
||||
// This page belongs to the local RAG preview feature. Unlike the other preview pages, it
|
||||
// has a route of its own, so it can be reached by typing the address even while the feature
|
||||
// is switched off. There is nothing to show in that case.
|
||||
//
|
||||
if (!PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
|
||||
{
|
||||
this.NavigationManager.NavigateTo(Routes.HOME);
|
||||
return;
|
||||
}
|
||||
|
||||
this.ApplyFilters([], [ Event.RAG_EMBEDDING_STATUS_CHANGED, Event.CONFIGURATION_CHANGED ]);
|
||||
await base.OnInitializedAsync();
|
||||
this.ReloadStatuses();
|
||||
}
|
||||
|
||||
protected override Task ProcessIncomingMessage<T>(ComponentBase? sendingComponent, Event triggeredEvent, T? data) where T : default
|
||||
{
|
||||
if (triggeredEvent is Event.RAG_EMBEDDING_STATUS_CHANGED or Event.CONFIGURATION_CHANGED)
|
||||
{
|
||||
this.ReloadStatuses();
|
||||
this.StateHasChanged();
|
||||
}
|
||||
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
private void ReloadStatuses()
|
||||
{
|
||||
this.Statuses = this.DataSourceEmbeddingService
|
||||
.GetStatuses()
|
||||
.OrderBy(status => status.SortOrder)
|
||||
.ThenBy(status => status.DataSourceName, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
this.UpdateAutoExpansion();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Opens the data source which is worth reading, as long as the user has not chosen one.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// It never closes what is open. A data source which finishes its run while somebody is reading
|
||||
/// it would otherwise fold up at the very moment its result becomes interesting. From the first
|
||||
/// click on, the page stops rearranging itself at all.
|
||||
/// </remarks>
|
||||
private void UpdateAutoExpansion()
|
||||
{
|
||||
if (this.userChoseExpansion)
|
||||
return;
|
||||
|
||||
// The list is sorted by state, so the first match is the most pressing one: a running data
|
||||
// source before a queued one, and a failed one before a completed one:
|
||||
var worthOpening = this.Statuses.FirstOrDefault(IsWorthOpening);
|
||||
if (worthOpening is null)
|
||||
return;
|
||||
|
||||
this.expandedDataSourceId = worthOpening.DataSourceId;
|
||||
}
|
||||
|
||||
/// <remarks>
|
||||
/// Whoever opens this page does so because a run is under way or because something went wrong.
|
||||
/// Meeting nothing but closed panels would be a step back from the version which showed every
|
||||
/// data source at once.
|
||||
/// </remarks>
|
||||
private static bool IsWorthOpening(DataSourceEmbeddingStatus status) =>
|
||||
status.State is DataSourceEmbeddingState.RUNNING or DataSourceEmbeddingState.QUEUED or DataSourceEmbeddingState.FAILED || status.FailedFiles > 0;
|
||||
|
||||
/// <remarks>
|
||||
/// MudBlazor keeps track of which panel is open on its own, so this only records the decision.
|
||||
/// Both events of a switch arrive, in either order — the one closing the old panel and the one
|
||||
/// opening the new one — which is why the closing event only clears what it actually named.
|
||||
/// </remarks>
|
||||
private void DataSourcePanelExpandedChanged(DataSourceEmbeddingStatus status, bool isExpanded)
|
||||
{
|
||||
this.userChoseExpansion = true;
|
||||
|
||||
if (isExpanded)
|
||||
this.expandedDataSourceId = status.DataSourceId;
|
||||
else if (this.expandedDataSourceId == status.DataSourceId)
|
||||
this.expandedDataSourceId = null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Opens the data source settings, the same dialog the chat offers next to its data source selection.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Nothing is left to do once it closes: the dialog writes the settings itself and publishes
|
||||
/// CONFIGURATION_CHANGED, which this page already listens to.
|
||||
/// </remarks>
|
||||
private async Task OpenDataSourceSettings()
|
||||
{
|
||||
var dialogParameters = new DialogParameters();
|
||||
var dialogReference = await this.DialogService.ShowAsync<SettingsDialogDataSources>(null, dialogParameters, DialogOptions.FULLSCREEN);
|
||||
await dialogReference.Result;
|
||||
}
|
||||
|
||||
private static Color GetStatusColor(DataSourceEmbeddingStatus status) => status.State switch
|
||||
{
|
||||
DataSourceEmbeddingState.RUNNING => Color.Warning,
|
||||
DataSourceEmbeddingState.QUEUED => Color.Info,
|
||||
DataSourceEmbeddingState.FAILED => Color.Error,
|
||||
DataSourceEmbeddingState.COMPLETED when status.FailedFiles > 0 => Color.Warning,
|
||||
DataSourceEmbeddingState.COMPLETED => Color.Success,
|
||||
_ => Color.Default,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// What a group of failures has in common.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Also the key the failures are grouped by: two of them belong together exactly when the
|
||||
/// list would say the same thing about both.
|
||||
/// </remarks>
|
||||
private sealed record FailureCause(int Priority, string Title, string Icon, Color Color, bool IsPermanent, bool NeedsProviderSettings, bool ShowsMessagePerFile);
|
||||
|
||||
private sealed record FailureGroup(FailureCause Cause, string EmbeddingProviderName, IReadOnlyList<DataSourceEmbeddingFailure> Failures);
|
||||
|
||||
/// <summary>
|
||||
/// Puts the failures of a data source into one group per cause, the most pressing one first.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Within a priority, the largest group comes first: it is the one telling the user the most
|
||||
/// about their folder.
|
||||
/// </remarks>
|
||||
private IReadOnlyList<FailureGroup> GetFailureGroups(DataSourceEmbeddingStatus status) => status.Failures
|
||||
.GroupBy(this.GetFailureCause)
|
||||
.Select(group => new FailureGroup(group.Key, GetEmbeddingProviderName(group), group.OrderBy(failure => failure.FilePath, StringComparer.OrdinalIgnoreCase).ToList()))
|
||||
.OrderBy(group => group.Cause.Priority)
|
||||
.ThenByDescending(group => group.Failures.Count)
|
||||
.ThenBy(group => group.Cause.Title, StringComparer.OrdinalIgnoreCase)
|
||||
.ToList();
|
||||
|
||||
private FailureCause GetFailureCause(DataSourceEmbeddingFailure failure)
|
||||
{
|
||||
//
|
||||
// Anything the provider answered comes first: it stops the entire data source, while a
|
||||
// file nobody can read costs that one file:
|
||||
//
|
||||
if (failure.FailureReason is not ProviderRequestFailureReason.NONE)
|
||||
{
|
||||
var isFixedInSettings = failure.FailureReason.IsFixedInProviderSettings();
|
||||
return new FailureCause(isFixedInSettings ? 0 : 1, failure.FailureReason.GetName(), isFixedInSettings ? Icons.Material.Filled.Key : Icons.Material.Filled.CloudOff, Color.Error, false, isFixedInSettings, true);
|
||||
}
|
||||
|
||||
//
|
||||
// Codes without a name of their own carry everything they know in the message of the
|
||||
// single file, which is why those groups show that message per file:
|
||||
//
|
||||
var causeName = failure.ExtractionCode.GetIndexingCauseName();
|
||||
var hasCauseName = !string.IsNullOrWhiteSpace(causeName);
|
||||
var title = hasCauseName ? causeName : T("Other cause");
|
||||
|
||||
return failure.IsPermanent
|
||||
? new FailureCause(3, title, Icons.Material.Filled.SkipNext, Color.Default, true, false, !hasCauseName)
|
||||
: new FailureCause(2, title, Icons.Material.Filled.ReportProblem, Color.Warning, false, false, !hasCauseName);
|
||||
}
|
||||
|
||||
private static string GetEmbeddingProviderName(IEnumerable<DataSourceEmbeddingFailure> failures) => failures
|
||||
.Select(failure => failure.EmbeddingProviderName)
|
||||
.FirstOrDefault(name => !string.IsNullOrWhiteSpace(name)) ?? string.Empty;
|
||||
|
||||
private static string GetGroupHeader(FailureGroup group) => $"{group.Cause.Title} ({group.Failures.Count})";
|
||||
|
||||
private static string GetFileName(string filePath)
|
||||
{
|
||||
var fileName = Path.GetFileName(filePath);
|
||||
return string.IsNullOrWhiteSpace(fileName) ? filePath : fileName;
|
||||
}
|
||||
|
||||
private static string GetOccurrenceText(DataSourceEmbeddingFailure failure) => failure.OccurredAtUtc > DateTimeOffset.MinValue ? failure.OccurredAtUtc.ToLocalTime().ToString("g") : string.Empty;
|
||||
|
||||
/// <remarks>
|
||||
/// A failure which was not about one file, such as a folder which is gone, carries the name of
|
||||
/// the data source instead of a path. There is nothing to show for those.
|
||||
/// </remarks>
|
||||
private static bool CanShowInFileManager(DataSourceEmbeddingFailure failure) => !string.IsNullOrWhiteSpace(failure.FilePath) && Path.IsPathRooted(failure.FilePath);
|
||||
|
||||
/// <summary>
|
||||
/// Opens the file browser of the system and selects the file in it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Reading that a file could not be indexed is where the work starts, not where it ends: the
|
||||
/// file has to be opened, replaced, or run through an OCR. This is the same way out the log
|
||||
/// viewer offers for the log files.
|
||||
/// </remarks>
|
||||
private async Task ShowInFileManager(DataSourceEmbeddingFailure failure)
|
||||
{
|
||||
OpenPathResponse response;
|
||||
try
|
||||
{
|
||||
response = await this.RustService.TryOpenPathInRuntimeFileManager(failure.FilePath);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.Logger.LogWarning(e, "Could not show a file of the embedding failure list in the file manager.");
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, T("Could not open the file location.")));
|
||||
return;
|
||||
}
|
||||
|
||||
if (response.Success)
|
||||
return;
|
||||
|
||||
var issue = string.IsNullOrWhiteSpace(response.Issue) ? T("Unknown error") : response.Issue;
|
||||
await this.MessageBus.SendError(new(Icons.Material.Filled.Folder, string.Format(T("Could not open the file location: {0}"), issue)));
|
||||
}
|
||||
|
||||
private bool CanRefresh(DataSourceEmbeddingStatus status)
|
||||
{
|
||||
return this.DataSourceEmbeddingService.CanRefreshDataSource(status.DataSourceId) &&
|
||||
status.State is not DataSourceEmbeddingState.RUNNING and not DataSourceEmbeddingState.QUEUED &&
|
||||
(status.State is DataSourceEmbeddingState.FAILED || status.FailedFiles > 0);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Takes the user to the settings, where the embedding providers are configured.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Offered only for the failures a setting fixes, such as a rejected API key. Reading what
|
||||
/// went wrong and then having to find the right page is where people give up.
|
||||
/// </remarks>
|
||||
private void OpenEmbeddingProviderSettings() => this.NavigationManager.NavigateTo(Routes.SETTINGS);
|
||||
|
||||
private async Task RefreshDataSource(DataSourceEmbeddingStatus status)
|
||||
{
|
||||
await this.DataSourceEmbeddingService.RetryDataSourceAsync(status.DataSourceId);
|
||||
this.ReloadStatuses();
|
||||
await this.InvokeAsync(this.StateHasChanged);
|
||||
}
|
||||
}
|
||||
@@ -351,6 +351,7 @@
|
||||
}
|
||||
|
||||
<ThirdPartyComponent Name="Qdrant Edge" Developer="Andrey Vasnetsov, Tim Visée, Arnaud Gourlay, Luis Cossío, Ivan Pleshkov, Roman Titov, xzfc, JojiiOfficial & Open Source Community" LicenseName="Apache-2.0" LicenseUrl="https://github.com/qdrant/qdrant/blob/master/LICENSE" RepositoryUrl="https://github.com/qdrant/qdrant" UseCase="@T("Qdrant Edge is an embedded vector database and vector similarity search engine. We use it to realize local RAG—retrieval-augmented generation—within AI Studio. Thanks for the effort and great work that has been and is being put into Qdrant.")"/>
|
||||
<ThirdPartyComponent Name="SQLite" Developer="Richard Hipp & Team" LicenseName="None (public domain)" LicenseUrl="https://www.sqlite.org/copyright.html" RepositoryUrl="https://www.sqlite.org/src/" UseCase="@T("SQLite stores local RAG indexing metadata, searchable chunk text, and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation.")"/>
|
||||
<ThirdPartyComponent Name="regex" Developer="Andrew Gallant, Alex Crichton, The Rust Project Developers & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/rust-lang/regex/blob/master/LICENSE-MIT" RepositoryUrl="https://github.com/rust-lang/regex" UseCase="@T("The regex crate detects structural and obfuscated prompt-injection patterns in untrusted document content. Its linear-time matching without backtracking keeps these scans predictable, even for hostile input.")"/>
|
||||
<ThirdPartyComponent Name="aho-corasick" Developer="Alfred V. Aho, Margaret J. Corasick, Andrew Gallant & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/BurntSushi/aho-corasick/blob/master/LICENSE-MIT" RepositoryUrl="https://github.com/BurntSushi/aho-corasick" UseCase="@T("The aho-corasick crate searches the fixed catalog of prompt-injection phrases in one pass, allowing AI Studio to scan large documents efficiently. We thank Alfred V. Aho and Margaret J. Corasick for publishing the algorithm in 1975, and Andrew Gallant and the Open Source Community for bringing it to Rust.")"/>
|
||||
<ThirdPartyComponent Name="toml" Developer="Ed Page, Alex Crichton, ordian, Eric Huss & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/toml-rs/toml/blob/main/LICENSE-MIT" RepositoryUrl="https://github.com/toml-rs/toml" UseCase="@T("The toml crate parses the embedded prompt-injection phrase catalog when the runtime starts. This keeps the detection rules separate from the Rust implementation and easier to maintain.")"/>
|
||||
@@ -389,6 +390,8 @@
|
||||
<ThirdPartyComponent Name="sysinfo" Developer="Guillaume Gomez & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/GuillaumeGomez/sysinfo/blob/main/LICENSE" RepositoryUrl="https://github.com/GuillaumeGomez/sysinfo" UseCase="@T("This library is used to manage sidecar processes and to ensure that stale or zombie sidecars are detected and terminated.")"/>
|
||||
<ThirdPartyComponent Name="tempfile" Developer="Steven Allen, Ashley Mannix & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/Stebalien/tempfile/blob/master/LICENSE-MIT" RepositoryUrl="https://github.com/Stebalien/tempfile" UseCase="@T("This library is used to create temporary folders in runtime tests and supporting filesystem operations.")"/>
|
||||
<ThirdPartyComponent Name="Lua-CSharp" Developer="Yusuke Nakada & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/nuskey8/Lua-CSharp/blob/main/LICENSE" RepositoryUrl="https://github.com/nuskey8/Lua-CSharp" UseCase="@T("We use Lua as the language for plugins. Lua-CSharp lets Lua scripts communicate with AI Studio and vice versa. Thank you, Yusuke Nakada, for this great library.")" />
|
||||
<ThirdPartyComponent Name="DeepSeek-V3.2 Tokenizer" Developer="DeepSeek-AI" LicenseName="MIT" LicenseUrl="https://huggingface.co/datasets/choosealicense/licenses/blob/main/markdown/mit.md" RepositoryUrl="https://huggingface.co/deepseek-ai/DeepSeek-V3.2/tree/main" UseCase="@T("We use the DeepSeek Tokenizer to estimate the number of tokens an input will generate.")" />
|
||||
<ThirdPartyComponent Name="Tokenizer" Developer="Anthony Moi, Nicolas Patry, Pierric Cistac, Arthur Zucker & Open Source Community" LicenseName="Apache-2.0" LicenseUrl="https://github.com/huggingface/tokenizers/blob/main/LICENSE" RepositoryUrl="https://github.com/huggingface/tokenizers" UseCase="@T("The Tokenizer library serves as the base framework for integrating the DeepSeek tokenizer.")" />
|
||||
<ThirdPartyComponent Name="HtmlAgilityPack" Developer="ZZZ Projects & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/zzzprojects/html-agility-pack/blob/master/LICENSE" RepositoryUrl="https://github.com/zzzprojects/html-agility-pack" UseCase="@T("We use the HtmlAgilityPack to extract content from the web. This is necessary, e.g., when you provide a URL as input for an assistant.")"/>
|
||||
<ThirdPartyComponent Name="ReverseMarkdown" Developer="Babu Annamalai & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/mysticmind/reversemarkdown-net/blob/master/LICENSE" RepositoryUrl="https://github.com/mysticmind/reversemarkdown-net" UseCase="@T("This library is used to convert HTML to Markdown. This is necessary, e.g., when you provide a URL as input for an assistant.")"/>
|
||||
<ThirdPartyComponent Name="wikEd diff" Developer="Cacycle & Open Source Community" LicenseName="None (public domain)" LicenseUrl="https://en.wikipedia.org/wiki/User:Cacycle/diff#License" RepositoryUrl="https://en.wikipedia.org/wiki/User:Cacycle/diff" UseCase="@T("This library is used to display the differences between two texts. This is necessary, e.g., for the grammar and spelling assistant.")"/>
|
||||
|
||||
@@ -24,6 +24,10 @@
|
||||
<SettingsPanelApp AvailableLLMProvidersFunc="@(() => this.availableLLMProviders)"/>
|
||||
|
||||
<SettingsPanelTools />
|
||||
@if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
|
||||
{
|
||||
<SettingsPanelDataSources/>
|
||||
}
|
||||
|
||||
@if (PreviewFeatures.PRE_RAG_2024.IsEnabled(this.SettingsManager))
|
||||
{
|
||||
@@ -35,4 +39,4 @@
|
||||
<SettingsPanelAgentAssistantAudit AvailableLLMProvidersFunc="@(() => this.availableLLMProviders)"/>
|
||||
</MudExpansionPanels>
|
||||
</InnerScrolling>
|
||||
</div>
|
||||
</div>
|
||||
Reference in new issue
Block a user