MindWorkAI/AI-Studio
MindWorkAI/AI-Studio
1
0
Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-08-11 17:32:11 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

added metadata for bm25 and also confidence levels

Browse Source
This commit is contained in:
PaulKoudelka 2026-08-03 19:45:47 +02:00
parent b3ba945b98
commit c78a8f05ef
27 changed files with 511 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

115
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
View File

@ -3655,6 +3655,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1DIALOG::T3804576966"] = "Por
-- Connection failed.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1DIALOG::T3820825672"] = "Connection failed."
-- Compliance level
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1DIALOG::T3995796156"] = "Compliance level"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1DIALOG::T4081226330"] = "Your security policy"
@ -3772,6 +3775,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T3448155331"] =
-- ERI server port
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T3843835535"] = "ERI server port"
-- Compliance level
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T3995796156"] = "Compliance level"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T4081226330"] = "Your security policy"
@ -3787,6 +3793,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T470340825"] = "
-- the security requirements of the data provider
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T503852885"] = "the security requirements of the data provider"
-- the compliance level
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T607609781"] = "the compliance level"
-- When to use
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T629595477"] = "When to use"
@ -3838,6 +3847,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1312296210"
-- No embedding selected
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1359179968"] = "No embedding selected"
-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
-- Tokenizer
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1696723386"] = "Tokenizer"
@ -3856,29 +3868,26 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252"
-- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
-- The overlap must be smaller than the effective token limit.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2101951526"] = "The overlap must be smaller than the effective token limit."
-- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap."
-- Please enter 0 or a positive max embedding size.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2571963846"] = "Please enter 0 or a positive max embedding size."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add"
-- Max embedding size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2812251281"] = "Max embedding size"
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
-- The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2834874159"] = "The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."
-- Embedding
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"] = "Embedding"
-- Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T287150655"] = "Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting."
-- The overlap must be smaller than the effective max embedding size.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3100817060"] = "The overlap must be smaller than the effective max embedding size."
-- Token limit
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2961294165"] = "Token limit"
-- Please enter 0 or a positive overlap length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
@ -3889,8 +3898,11 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900"
-- Show Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings"
-- Embedding overlap
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T375910361"] = "Embedding overlap"
-- Please enter 0 or a positive token limit.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3659673500"] = "Please enter 0 or a positive token limit."
-- Compliance level
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3995796156"] = "Compliance level"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"] = "Your security policy"
@ -3898,15 +3910,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"
-- Select the base directory
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory"
-- The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T607909514"] = "The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting."
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name"
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T900713019"] = "Cancel"
-- Token overlap
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T981382809"] = "Token overlap"
-- the total directory size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T1082241458"] = "the total directory size"
@ -3967,6 +3979,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3602384
-- Path
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3949388886"] = "Path"
-- Compliance level
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3995796156"] = "Compliance level"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T4081226330"] = "Your security policy"
@ -3979,6 +3994,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T4438734
-- The directory chosen for the data source exists.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T445858624"] = "The directory chosen for the data source exists."
-- the compliance level
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T607609781"] = "the compliance level"
-- Hide Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1108876344"] = "Hide Expert Settings"
@ -3994,6 +4012,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1220918127"] = "
-- No embedding selected
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1359179968"] = "No embedding selected"
-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting."
-- Tokenizer
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1696723386"] = "Tokenizer"
@ -4009,32 +4030,29 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1847791252"] = "
-- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method."
-- The overlap must be smaller than the effective token limit.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2101951526"] = "The overlap must be smaller than the effective token limit."
-- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap."
-- Please enter 0 or a positive max embedding size.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2571963846"] = "Please enter 0 or a positive max embedding size."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add"
-- Max embedding size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2812251281"] = "Max embedding size"
-- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud."
-- The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2834874159"] = "The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting."
-- Embedding
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2838542994"] = "Embedding"
-- Describe what data this file contains to help the AI select it.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = "Describe what data this file contains to help the AI select it."
-- Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T287150655"] = "Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting."
-- The overlap must be smaller than the effective max embedding size.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3100817060"] = "The overlap must be smaller than the effective max embedding size."
-- Token limit
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2961294165"] = "Token limit"
-- Please enter 0 or a positive overlap length.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length."
@ -4045,11 +4063,14 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = "
-- Show Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings"
-- Please enter 0 or a positive token limit.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3659673500"] = "Please enter 0 or a positive token limit."
-- Select the file
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file"
-- Embedding overlap
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T375910361"] = "Embedding overlap"
-- Compliance level
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3995796156"] = "Compliance level"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = "Your security policy"
@ -4057,9 +4078,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = "
-- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model."
-- The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T607909514"] = "The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting."
-- Data Source Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name"
@ -4069,6 +4087,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T900713019"] = "C
-- Selected file path for this data source
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T939749563"] = "Selected file path for this data source"
-- Token overlap
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T981382809"] = "Token overlap"
-- The file chosen for the data source exists.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T1294177559"] = "The file chosen for the data source exists."
@ -4123,9 +4144,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T3650018664"]
-- The embedding runs in the cloud. All your data within the file '{0}' will be sent to the cloud.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T3688254408"] = "The embedding runs in the cloud. All your data within the file '{0}' will be sent to the cloud."
-- Compliance level
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T3995796156"] = "Compliance level"
-- Your security policy
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T4081226330"] = "Your security policy"
-- the compliance level
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T607609781"] = "the compliance level"
-- Markdown View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1373123357"] = "Markdown View"
@ -4270,9 +4297,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(O
-- Invalid tokenizer:
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:"
-- Please enter a max embedding size greater than 0.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2608328491"] = "Please enter a max embedding size greater than 0."
-- Add
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add"
@ -4282,15 +4306,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T278585345"] = "Sel
-- No models loaded or available.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2810182573"] = "No models loaded or available."
-- Max embedding size
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2812251281"] = "Max embedding size"
-- Instance Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "Instance Name"
-- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually."
-- Token limit
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "Token limit"
-- Please enter a token limit greater than 0.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Please enter a token limit greater than 0."
-- Show Expert Settings
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3361153305"] = "Show Expert Settings"
@ -6394,6 +6421,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "The Tokenizer li
-- ID mismatch: the plugin ID differs from the enterprise configuration ID.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID mismatch: the plugin ID differs from the enterprise configuration ID."
-- SQLite stores local RAG indexing metadata, searchable chunk text, and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T117115925"] = "SQLite stores local RAG indexing metadata, searchable chunk text, and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation."
-- This is a private AI Studio installation. It runs without an enterprise configuration.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1209549230"] = "This is a private AI Studio installation. It runs without an enterprise configuration."
@ -6481,9 +6511,6 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2173617769"] = "This library is
-- For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2174764529"] = "For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose."
-- SQLite stores local RAG indexing metadata and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation.
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T224267373"] = "SQLite stores local RAG indexing metadata and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation."
-- OK
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2246359087"] = "OK"
@ -6994,6 +7021,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::CONFIDENCELEVELEXTENSIONS::T3063224793"] =
-- High
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::CONFIDENCELEVELEXTENSIONS::T3188327965"] = "High"
-- Unknown
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::CONFIDENCELEVELEXTENSIONS::T3424652889"] = "Unknown"
-- Very Low
UI_TEXT_CONTENT["AISTUDIO::PROVIDER::CONFIDENCELEVELEXTENSIONS::T786675843"] = "Very Low"
@ -7459,6 +7489,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTAT
-- Indexed files
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T2235289713"] = "Indexed files"
-- Search chunks
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T2333737457"] = "Search chunks"
-- Indexed data sources
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T3524534748"] = "Indexed data sources"

10
app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor
View File

@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Provider
@using AIStudio.Tools.ERIClient.DataModel
@inherits MSGComponentBase
@ -119,6 +120,15 @@
</MudSelectItem>
}
</MudSelect>
<MudSelect @bind-Value="@this.dataComplianceLevel" Text="@this.dataComplianceLevel.GetName()" Label="@T("Compliance level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start">
@foreach (var level in this.ComplianceLevels)
{
<MudSelectItem Value="@level.Value">
@level.Name
</MudSelectItem>
}
</MudSelect>
<MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />

7
app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs
View File

@ -1,5 +1,7 @@
using AIStudio.Assistants.ERI;
using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.ERIClient;
using AIStudio.Tools.ERIClient.DataModel;
@ -46,6 +48,7 @@ public partial class DataSourceERI_V1Dialog : MSGComponentBase, ISecretId
private string dataEditingPreviousInstanceName = string.Empty;
private List<AuthMethod> availableAuthMethods = [];
private DataSourceSecurity dataSecurityPolicy;
private ConfidenceLevel dataComplianceLevel = ConfidenceLevel.UNKNOWN;
private SecurityRequirements dataSourceSecurityRequirements;
private ushort dataMaxMatches = 10;
private bool connectionTested;
@ -105,6 +108,7 @@ public partial class DataSourceERI_V1Dialog : MSGComponentBase, ISecretId
this.dataAuthMethod = this.DataSource.AuthMethod;
this.dataUsername = this.DataSource.Username;
this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
this.dataComplianceLevel = this.DataSource.ComplianceLevel;
this.dataMaxMatches = this.DataSource.MaxMatches;
// We cannot load the retrieval processes now, since we have
@ -157,6 +161,8 @@ public partial class DataSourceERI_V1Dialog : MSGComponentBase, ISecretId
#endregion
private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ComplianceLevels => ConfigurationSelectDataFactory.GetDataSourceComplianceLevelsData();
private DataSourceERI_V1 CreateDataSource()
{
var cleanedHostname = this.dataHostname.Trim();
@ -172,6 +178,7 @@ public partial class DataSourceERI_V1Dialog : MSGComponentBase, ISecretId
UsernamePasswordMode = DataSourceERIUsernamePasswordMode.USER_MANAGED,
Type = DataSourceType.ERI_V1,
SecurityPolicy = this.dataSecurityPolicy,
ComplianceLevel = this.dataComplianceLevel,
SelectedRetrievalId = this.dataSelectedRetrievalProcess.Id,
MaxMatches = this.dataMaxMatches,
};

2
app/MindWork AI Studio/Dialogs/DataSourceERI_V1InfoDialog.razor
View File

@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Provider
@using AIStudio.Tools.ERIClient.DataModel
@inherits MSGComponentBase
<MudDialog>
@ -27,6 +28,7 @@
<TextInfoLines Label="@T("Server description")" MaxLines="14" Value="@this.serverDescription" ClipboardTooltipSubject="@T("the server description")"/>
<TextInfoLines Label="@T("Security requirements of the data provider")" MaxLines="3" Value="@this.securityRequirements.Explain()" ClipboardTooltipSubject="@T("the security requirements of the data provider")"/>
<TextInfoLines Label="@T("Your security policy")" MaxLines="3" Value="@this.DataSource.SecurityPolicy.ToInfoText()" Color="@this.DataSource.SecurityPolicy.GetColor()" ClipboardTooltipSubject="@T("your security policy")"/>
<TextInfoLine Label="@T("Compliance level")" Value="@this.DataSource.ComplianceLevel.GetName()" ClipboardTooltipSubject="@T("the compliance level")"/>
<TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
<MudText Typo="Typo.h5" Class="mt-6">

10
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor
View File

@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Provider
@inherits MSGComponentBase
<MudDialog>
@ -136,6 +137,15 @@
</MudSelectItem>
}
</MudSelect>
<MudSelect @bind-Value="@this.dataComplianceLevel" Text="@this.dataComplianceLevel.GetName()" Label="@T("Compliance level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start">
@foreach (var level in this.ComplianceLevels)
{
<MudSelectItem Value="@level.Value">
@level.Name
</MudSelectItem>
}
</MudSelect>
<MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />
</MudForm>

6
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs
View File

@ -1,4 +1,5 @@
using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Validation;
@ -49,6 +50,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
private ushort dataMaxMatches = 10;
private bool showExpertSettings;
private DataSourceSecurity dataSecurityPolicy;
private ConfidenceLevel dataComplianceLevel = ConfidenceLevel.UNKNOWN;
// We get the form reference from Blazor code to validate it manually:
private MudForm form = null!;
@ -86,6 +88,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
this.dataComplianceLevel = this.DataSource.ComplianceLevel;
this.dataMaxMatches = this.DataSource.MaxMatches;
this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
}
@ -112,6 +115,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ComplianceLevels => ConfigurationSelectDataFactory.GetDataSourceComplianceLevelsData();
private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
? T("No embedding selected")
: string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
@ -130,6 +135,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
SecurityPolicy = this.dataSecurityPolicy,
ComplianceLevel = this.dataComplianceLevel,
MaxMatches = this.dataMaxMatches,
};

2
app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor
View File

@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Provider
@inherits MSGComponentBase
<MudDialog>
@ -38,6 +39,7 @@
}
<TextInfoLines Label="@T("Your security policy")" MaxLines="3" Value="@this.DataSource.SecurityPolicy.ToInfoText()" Color="@this.DataSource.SecurityPolicy.GetColor()" ClipboardTooltipSubject="@T("your security policy")"/>
<TextInfoLine Label="@T("Compliance level")" Value="@this.DataSource.ComplianceLevel.GetName()" ClipboardTooltipSubject="@T("the compliance level")"/>
<TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
<TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("Number of files")" Value="@this.NumberFilesInDirectory" ClipboardTooltipSubject="@T("the number of files in the directory")"/>

10
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor
View File

@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Provider
@inherits MSGComponentBase
<MudDialog>
@ -136,6 +137,15 @@
</MudSelectItem>
}
</MudSelect>
<MudSelect @bind-Value="@this.dataComplianceLevel" Text="@this.dataComplianceLevel.GetName()" Label="@T("Compliance level")" Class="mb-3" OpenIcon="@Icons.Material.Filled.ExpandMore" AdornmentColor="Color.Info" Adornment="Adornment.Start">
@foreach (var level in this.ComplianceLevels)
{
<MudSelectItem Value="@level.Value">
@level.Name
</MudSelectItem>
}
</MudSelect>
<MudNumericField T="ushort" Min="10" @bind-Value="@this.dataMaxMatches" Label="@T("How many matches do you want at most per query?")" Variant="Variant.Outlined" Step="10" />
</MudForm>

6
app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs
View File

@ -1,4 +1,5 @@
using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Validation;
@ -49,6 +50,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
private ushort dataMaxMatches = 10;
private bool showExpertSettings;
private DataSourceSecurity dataSecurityPolicy;
private ConfidenceLevel dataComplianceLevel = ConfidenceLevel.UNKNOWN;
// We get the form reference from Blazor code to validate it manually:
private MudForm form = null!;
@ -86,6 +88,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength;
this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength;
this.dataSecurityPolicy = this.DataSource.SecurityPolicy;
this.dataComplianceLevel = this.DataSource.ComplianceLevel;
this.dataMaxMatches = this.DataSource.MaxMatches;
this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0;
}
@ -112,6 +115,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding;
private IEnumerable<ConfigurationSelectData<ConfidenceLevel>> ComplianceLevels => ConfigurationSelectDataFactory.GetDataSourceComplianceLevelsData();
private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null
? T("No embedding selected")
: string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath)
@ -130,6 +135,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase
MaxChunkTokenLength = this.dataMaxChunkTokenLength,
ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength,
SecurityPolicy = this.dataSecurityPolicy,
ComplianceLevel = this.dataComplianceLevel,
MaxMatches = this.dataMaxMatches,
};

2
app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor
View File

@ -1,4 +1,5 @@
@using AIStudio.Settings.DataModel
@using AIStudio.Provider
@inherits MSGComponentBase
<MudDialog>
@ -38,6 +39,7 @@
}
<TextInfoLines Label="@T("Your security policy")" MaxLines="3" Value="@this.DataSource.SecurityPolicy.ToInfoText()" Color="@this.DataSource.SecurityPolicy.GetColor()" ClipboardTooltipSubject="@T("your security policy")"/>
<TextInfoLine Label="@T("Compliance level")" Value="@this.DataSource.ComplianceLevel.GetName()" ClipboardTooltipSubject="@T("the compliance level")"/>
<TextInfoLine Label="@T("Maximum matches per query")" Value="@this.DataSource.MaxMatches.ToString()" ClipboardTooltipSubject="@T("the maximum number of matches per query")"/>
<TextInfoLine Icon="@Icons.Material.Filled.SquareFoot" Label="@T("File size")" Value="@this.FileSize" ClipboardTooltipSubject="@T("the file size")"/>
</DialogContent>

2
app/MindWork AI Studio/Pages/Information.razor
View File

@ -290,7 +290,7 @@
}
<ThirdPartyComponent Name="Qdrant Edge" Developer="Andrey Vasnetsov, Tim Visée, Arnaud Gourlay, Luis Cossío, Ivan Pleshkov, Roman Titov, xzfc, JojiiOfficial & Open Source Community" LicenseName="Apache-2.0" LicenseUrl="https://github.com/qdrant/qdrant/blob/master/LICENSE" RepositoryUrl="https://github.com/qdrant/qdrant" UseCase="@T("Qdrant Edge is an embedded vector database and vector similarity search engine. We use it to realize local RAG—retrieval-augmented generation—within AI Studio. Thanks for the effort and great work that has been and is being put into Qdrant.")"/>
<ThirdPartyComponent Name="SQLite" Developer="SQLite developers & Open Source Community" LicenseName="Public Domain" LicenseUrl="https://www.sqlite.org/copyright.html" RepositoryUrl="https://www.sqlite.org/src/" UseCase="@T("SQLite stores local RAG indexing metadata and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation.")"/>
<ThirdPartyComponent Name="SQLite" Developer="SQLite developers & Open Source Community" LicenseName="Public Domain" LicenseUrl="https://www.sqlite.org/copyright.html" RepositoryUrl="https://www.sqlite.org/src/" UseCase="@T("SQLite stores local RAG indexing metadata, searchable chunk text, and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation.")"/>
<ThirdPartyComponent Name="axum" Developer="David Pedersen, Jonas Platte, tottoto, David Mládek, Yann Simon, Tobias Bieniek, Open Source Community & Tokio Project" LicenseName="MIT" LicenseUrl="https://github.com/tokio-rs/axum/blob/main/LICENSE" RepositoryUrl="https://github.com/tokio-rs/axum" UseCase="@T("Axum is used to provide the small internal service that connects the Rust runtime with the app's user interface. This lets both parts of AI Studio exchange information while the app is running.")"/>
<ThirdPartyComponent Name="axum-server" Developer="Eray Karatay, Adi Salimgereyev, daxpedda & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/programatik29/axum-server/blob/master/LICENSE" RepositoryUrl="https://github.com/programatik29/axum-server" UseCase="@T("Axum server runs the internal axum service over a secure local connection. This helps AI Studio protect the communication between the Rust runtime and the user interface.")"/>
<ThirdPartyComponent Name="Rustls" Developer="Joe Birr-Pixton, Dirkjan Ochtman, Daniel McCarney, Brian Smith, Jacob Hoffman-Andrews, Jorge Aparicio & Open Source Community" LicenseName="MIT" LicenseUrl="https://github.com/rustls/rustls/blob/main/LICENSE-MIT" RepositoryUrl="https://github.com/rustls/rustls" UseCase="@T("Rustls helps secure the internal connection between the app's user interface and the Rust runtime. This protects the local communication that AI Studio needs while it is running.")"/>

4
app/MindWork AI Studio/Plugins/configuration/plugin.lua
View File

@ -150,6 +150,7 @@ CONFIG["EMBEDDING_PROVIDERS"] = {}
-- ERI v1 data sources for retrieval-augmented generation:
CONFIG["DATA_SOURCES"] = {}
-- Allowed compliance levels are: UNTRUSTED, UNKNOWN, VERY_LOW, LOW, MODERATE, MEDIUM, HIGH
-- Example: ERI v1 data source with a shared access token.
-- CONFIG["DATA_SOURCES"][#CONFIG["DATA_SOURCES"]+1] = {
@ -161,6 +162,7 @@ CONFIG["DATA_SOURCES"] = {}
-- ["AuthMethod"] = "TOKEN",
-- ["Token"] = "ENC:v1:<base64-encoded encrypted token>",
-- ["SecurityPolicy"] = "SELF_HOSTED",
-- ["ComplianceLevel"] = "UNKNOWN",
-- ["SelectedRetrievalId"] = "<retrieval process ID from the ERI server>",
-- ["MaxMatches"] = 10,
-- }
@ -177,6 +179,7 @@ CONFIG["DATA_SOURCES"] = {}
-- ["Username"] = "<shared username>",
-- ["Password"] = "ENC:v1:<base64-encoded encrypted password>",
-- ["SecurityPolicy"] = "SELF_HOSTED",
-- ["ComplianceLevel"] = "UNKNOWN",
-- ["SelectedRetrievalId"] = "<retrieval process ID from the ERI server>",
-- ["MaxMatches"] = 10,
-- }
@ -192,6 +195,7 @@ CONFIG["DATA_SOURCES"] = {}
-- ["UsernamePasswordMode"] = "OS_USERNAME_SHARED_PASSWORD",
-- ["Password"] = "ENC:v1:<base64-encoded encrypted password>",
-- ["SecurityPolicy"] = "SELF_HOSTED",
-- ["ComplianceLevel"] = "UNKNOWN",
-- ["SelectedRetrievalId"] = "<retrieval process ID from the ERI server>",
-- ["MaxMatches"] = 10,
-- }

1
app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs
View File

@ -12,6 +12,7 @@ public static class ConfidenceLevelExtensions
ConfidenceLevel.NONE => TB("No provider selected"),
ConfidenceLevel.UNTRUSTED => TB("Untrusted"),
ConfidenceLevel.UNKNOWN => TB("Unknown"),
ConfidenceLevel.VERY_LOW => TB("Very Low"),
ConfidenceLevel.LOW => TB("Low"),
ConfidenceLevel.MODERATE => TB("Moderate"),

11
app/MindWork AI Studio/Settings/ConfigurationSelectDataFactory.cs
View File

@ -294,6 +294,17 @@ public static class ConfigurationSelectDataFactory
}
}
}
public static IEnumerable<ConfigurationSelectData<ConfidenceLevel>> GetDataSourceComplianceLevelsData()
{
foreach (var level in Enum.GetValues<ConfidenceLevel>())
{
if (level is ConfidenceLevel.NONE)
continue;
yield return new(level.GetName(), level);
}
}
public static IEnumerable<ConfigurationSelectData<Themes>> GetThemesData()
{

16
app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs
View File

@ -2,6 +2,7 @@
using AIStudio.Assistants.ERI;
using AIStudio.Chat;
using AIStudio.Provider;
using AIStudio.Tools.ERIClient;
using AIStudio.Tools.ERIClient.DataModel;
using AIStudio.Tools.PluginSystem;
@ -57,6 +58,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
/// <inheritdoc />
public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
/// <inheritdoc />
public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
/// <inheritdoc />
public bool IsEnterpriseConfiguration { get; init; }
@ -206,6 +210,16 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
return false;
}
var complianceLevel = ConfidenceLevel.UNKNOWN;
if (table.TryGetValue("ComplianceLevel", out var complianceLevelValue) &&
(!complianceLevelValue.TryRead<string>(out var complianceLevelText) ||
!Enum.TryParse(complianceLevelText, true, out complianceLevel) ||
complianceLevel is ConfidenceLevel.NONE))
{
LOGGER.LogWarning($"The configured data source {idx} contains an invalid compliance level. Falling back to UNKNOWN. (Plugin ID: {configPluginId})");
complianceLevel = ConfidenceLevel.UNKNOWN;
}
if (!table.TryGetValue("SelectedRetrievalId", out var selectedRetrievalIdValue) || !selectedRetrievalIdValue.TryRead<string>(out var selectedRetrievalId) || string.IsNullOrWhiteSpace(selectedRetrievalId))
{
LOGGER.LogWarning($"The configured data source {idx} must specify a selected retrieval ID. (Plugin ID: {configPluginId})");
@ -263,6 +277,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
Username = username,
UsernamePasswordMode = usernamePasswordMode,
SecurityPolicy = securityPolicy,
ComplianceLevel = complianceLevel,
Version = ERIVersion.V1,
SelectedRetrievalId = selectedRetrievalId,
MaxMatches = (ushort)maxMatches,
@ -323,6 +338,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
{{usernameLine}}
{{secretLine}}
["SecurityPolicy"] = "{{this.SecurityPolicy}}",
["ComplianceLevel"] = "{{this.ComplianceLevel}}",
["SelectedRetrievalId"] = "{{LuaTools.EscapeLuaString(this.SelectedRetrievalId)}}",
["MaxMatches"] = {{this.MaxMatches}},
}

4
app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs
View File

@ -1,4 +1,5 @@
using AIStudio.Chat;
using AIStudio.Provider;
using AIStudio.Tools.RAG;
namespace AIStudio.Settings.DataModel;
@ -42,6 +43,9 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource
/// <inheritdoc />
public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
/// <inheritdoc />
public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
/// <inheritdoc />
public bool IsEnterpriseConfiguration { get; init; }

4
app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs
View File

@ -1,4 +1,5 @@
using AIStudio.Chat;
using AIStudio.Provider;
using AIStudio.Tools.RAG;
namespace AIStudio.Settings.DataModel;
@ -42,6 +43,9 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource
/// <inheritdoc />
public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED;
/// <inheritdoc />
public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN;
/// <inheritdoc />
public bool IsEnterpriseConfiguration { get; init; }

6
app/MindWork AI Studio/Settings/IDataSource.cs
View File

@ -1,6 +1,7 @@
using System.Text.Json.Serialization;
using AIStudio.Chat;
using AIStudio.Provider;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG;
@ -25,6 +26,11 @@ public interface IDataSource : IConfigurationObject
/// Which data security policy is applied to this data source?
/// </summary>
public DataSourceSecurity SecurityPolicy { get; init; }
/// <summary>
/// Which compliance level is assigned to this data source?
/// </summary>
public ConfidenceLevel ComplianceLevel { get; init; }
/// <summary>
/// The maximum number of matches to return when retrieving data from the ERI server.

2
app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs
View File

@ -65,7 +65,7 @@ public sealed partial class DatabaseClientProvider(RustService rustService, ILog
return new NoEmbeddingStateClient(
client.Name,
"The configured database client does not support embedding state operations.",
"The configured database client does not support local RAG index operations.",
client.Status);
}

19
app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs
View File

@ -24,15 +24,30 @@ public abstract class EmbeddingStateClient(string name, string path) : DatabaseC
public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token);
public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token);
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
}
public sealed record EmbeddingStateFile(
string FilePath,
string ParentFileId,
string AbsolutePath,
string FileName,
string RelativePath,
string FileType,
string Fingerprint,
long FileSize,
DateTime CreationUtc,
DateTime LastWriteUtc,
DateTime EmbeddedAtUtc,
int ChunkCount);
int ChunkCount,
string ComplianceLevel,
int ComplianceLevelRank);
public sealed record EmbeddingStateChunk(
string ChunkId,
string ParentFileId,
int? PageNumber,
int ChunkIndex,
string ChunkText,
DateTime EmbeddedAtUtc);

2
app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs
View File

@ -44,6 +44,8 @@ public sealed class NoEmbeddingStateClient(string name, string? unavailableReaso
public override Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask;
public override Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token) => Task.CompletedTask;
public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask;
public override void Dispose()

217
app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs
View File

@ -14,8 +14,8 @@ public sealed class SqliteEmbeddingStateClientImplementation(
string basePath,
string version) : EmbeddingStateClient(name, basePath)
{
private const string DATABASE_NAME = "SQLite";
private const string DATABASE_FILENAME = "rag-embedding-state.sqlite3";
private const string DATABASE_NAME = "Local RAG Index";
private const string DATABASE_FILENAME = "rag-index.sqlite3";
private readonly string databasePath = databasePath;
private readonly string connectionString = new SqliteConnectionStringBuilder
@ -57,7 +57,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
}
catch (Exception exception)
{
logger.LogWarning(exception, "{DatabaseName} is not available. Indexed file fingerprints are disabled.", DATABASE_NAME);
logger.LogWarning(exception, "{DatabaseName} is not available. Indexed file fingerprints and search chunks are disabled.", DATABASE_NAME);
return CreateNoEmbeddingStateClient(DATABASE_NAME, exception.Message, DatabaseClientStatus.UNAVAILABLE, databaseClientLogger);
}
}
@ -69,6 +69,7 @@ public sealed class SqliteEmbeddingStateClientImplementation(
yield return (TB("Storage size"), this.GetStorageSize());
yield return (TB("Indexed data sources"), (await this.CountAsync("data_sources", CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
yield return (TB("Indexed files"), (await this.CountAsync("embedded_files", CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
yield return (TB("Search chunks"), (await this.CountAsync("embedding_chunks", CancellationToken.None)).ToString(CultureInfo.InvariantCulture));
}
public override async Task<DataSourceEmbeddingManifest> GetManifestAsync(string dataSourceId, CancellationToken token)
@ -94,9 +95,10 @@ public sealed class SqliteEmbeddingStateClientImplementation(
}
await using (var command = CreateCommand(connection, """
SELECT file_path, fingerprint, file_size, last_write_utc, embedded_at_utc, chunk_count
SELECT absolute_path, fingerprint, file_size, last_write_utc, embedded_at_utc, chunk_count
FROM embedded_files
WHERE data_source_id = $dataSourceId
AND chunk_count > 0
"""))
{
command.Parameters.AddWithValue("$dataSourceId", dataSourceId);
@ -197,43 +199,64 @@ public sealed class SqliteEmbeddingStateClientImplementation(
await using var connection = await this.OpenConnectionAsync(token);
await ExecuteNonQueryAsync(connection, """
INSERT INTO embedded_files (
parent_file_id,
data_source_id,
file_path,
absolute_path,
file_name,
relative_path,
file_type,
fingerprint,
file_size,
creation_utc,
last_write_utc,
embedded_at_utc,
chunk_count)
chunk_count,
compliance_level,
compliance_level_rank)
VALUES (
$parentFileId,
$dataSourceId,
$filePath,
$absolutePath,
$fileName,
$relativePath,
$fileType,
$fingerprint,
$fileSize,
$creationUtc,
$lastWriteUtc,
$embeddedAtUtc,
$chunkCount)
ON CONFLICT(data_source_id, file_path) DO UPDATE SET
$chunkCount,
$complianceLevel,
$complianceLevelRank)
ON CONFLICT(parent_file_id) DO UPDATE SET
data_source_id = excluded.data_source_id,
absolute_path = excluded.absolute_path,
file_name = excluded.file_name,
relative_path = excluded.relative_path,
file_type = excluded.file_type,
fingerprint = excluded.fingerprint,
file_size = excluded.file_size,
creation_utc = excluded.creation_utc,
last_write_utc = excluded.last_write_utc,
embedded_at_utc = excluded.embedded_at_utc,
chunk_count = excluded.chunk_count
chunk_count = excluded.chunk_count,
compliance_level = excluded.compliance_level,
compliance_level_rank = excluded.compliance_level_rank
""", token,
("$parentFileId", file.ParentFileId),
("$dataSourceId", dataSourceId),
("$filePath", file.FilePath),
("$absolutePath", file.AbsolutePath),
("$fileName", file.FileName),
("$relativePath", file.RelativePath),
("$fileType", file.FileType),
("$fingerprint", file.Fingerprint),
("$fileSize", file.FileSize),
("$creationUtc", ToUtcText(file.CreationUtc)),
("$lastWriteUtc", ToUtcText(file.LastWriteUtc)),
("$embeddedAtUtc", ToUtcText(file.EmbeddedAtUtc)),
("$chunkCount", file.ChunkCount));
("$chunkCount", file.ChunkCount),
("$complianceLevel", file.ComplianceLevel),
("$complianceLevelRank", file.ComplianceLevelRank));
}
public override async Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token)
@ -242,12 +265,57 @@ public sealed class SqliteEmbeddingStateClientImplementation(
await ExecuteNonQueryAsync(connection, """
DELETE FROM embedded_files
WHERE data_source_id = $dataSourceId
AND file_path = $filePath
AND absolute_path = $filePath
""", token,
("$dataSourceId", dataSourceId),
("$filePath", filePath));
}
public override async Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token)
{
if (chunks.Count == 0)
return;
await using var connection = await this.OpenConnectionAsync(token);
await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(token);
foreach (var chunk in chunks)
{
token.ThrowIfCancellationRequested();
await ExecuteNonQueryAsync(connection, transaction, """
INSERT INTO embedding_chunks (
chunk_id,
parent_file_id,
page_number,
chunk_index,
chunk_text,
embedded_at_utc)
VALUES (
$chunkId,
$parentFileId,
$pageNumber,
$chunkIndex,
$chunkText,
$embeddedAtUtc)
ON CONFLICT(chunk_id) DO UPDATE SET
parent_file_id = excluded.parent_file_id,
page_number = excluded.page_number,
chunk_index = excluded.chunk_index,
chunk_text = excluded.chunk_text,
embedded_at_utc = excluded.embedded_at_utc
""", token,
("$chunkId", chunk.ChunkId),
("$parentFileId", chunk.ParentFileId),
("$pageNumber", chunk.PageNumber),
("$chunkIndex", chunk.ChunkIndex),
("$chunkText", chunk.ChunkText),
("$embeddedAtUtc", ToUtcText(chunk.EmbeddedAtUtc)));
}
await transaction.CommitAsync(token);
}
public override async Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token)
{
await using var connection = await this.OpenConnectionAsync(token);
@ -281,26 +349,109 @@ public sealed class SqliteEmbeddingStateClientImplementation(
);
CREATE TABLE IF NOT EXISTS embedded_files (
parent_file_id TEXT PRIMARY KEY,
data_source_id TEXT NOT NULL,
file_path TEXT COLLATE NOCASE NOT NULL,
absolute_path TEXT COLLATE NOCASE NOT NULL,
file_name TEXT NOT NULL,
relative_path TEXT NOT NULL,
file_type TEXT NOT NULL,
fingerprint TEXT NOT NULL,
file_size INTEGER NOT NULL,
creation_utc TEXT NOT NULL,
last_write_utc TEXT NOT NULL,
embedded_at_utc TEXT NOT NULL,
chunk_count INTEGER NOT NULL,
PRIMARY KEY (data_source_id, file_path),
compliance_level TEXT NOT NULL,
compliance_level_rank INTEGER NOT NULL,
FOREIGN KEY (data_source_id)
REFERENCES data_sources(data_source_id)
ON DELETE CASCADE
ON DELETE CASCADE,
UNIQUE(data_source_id, absolute_path)
);
CREATE TABLE IF NOT EXISTS embedding_chunks (
id INTEGER PRIMARY KEY AUTOINCREMENT,
chunk_id TEXT NOT NULL UNIQUE,
parent_file_id TEXT NOT NULL,
page_number INTEGER NULL,
chunk_index INTEGER NOT NULL,
chunk_text TEXT NOT NULL,
embedded_at_utc TEXT NOT NULL,
FOREIGN KEY (parent_file_id)
REFERENCES embedded_files(parent_file_id)
ON DELETE CASCADE,
UNIQUE(parent_file_id, chunk_index)
);
CREATE INDEX IF NOT EXISTS idx_embedded_files_data_source
ON embedded_files(data_source_id);
""", token);
await EnsureColumnAsync(connection, "data_sources", "source_hash", "TEXT NOT NULL DEFAULT ''", token);
CREATE INDEX IF NOT EXISTS idx_embedded_files_absolute_path
ON embedded_files(absolute_path);
CREATE INDEX IF NOT EXISTS idx_embedded_files_file_type
ON embedded_files(file_type);
CREATE INDEX IF NOT EXISTS idx_embedded_files_compliance
ON embedded_files(compliance_level_rank);
CREATE INDEX IF NOT EXISTS idx_embedding_chunks_parent_file
ON embedding_chunks(parent_file_id);
CREATE INDEX IF NOT EXISTS idx_embedding_chunks_page
ON embedding_chunks(page_number);
CREATE VIRTUAL TABLE IF NOT EXISTS embedding_chunks_fts
USING fts5(chunk_id UNINDEXED, file_name, chunk_text);
CREATE TRIGGER IF NOT EXISTS embedding_chunks_ai
AFTER INSERT ON embedding_chunks
BEGIN
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
VALUES (
new.id,
new.chunk_id,
(SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id),
new.chunk_text);
END;
CREATE TRIGGER IF NOT EXISTS embedding_chunks_ad
AFTER DELETE ON embedding_chunks
BEGIN
DELETE FROM embedding_chunks_fts
WHERE rowid = old.id;
END;
CREATE TRIGGER IF NOT EXISTS embedding_chunks_au
AFTER UPDATE ON embedding_chunks
BEGIN
DELETE FROM embedding_chunks_fts
WHERE rowid = old.id;
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
VALUES (
new.id,
new.chunk_id,
(SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id),
new.chunk_text);
END;
CREATE TRIGGER IF NOT EXISTS embedded_files_file_name_au
AFTER UPDATE OF file_name ON embedded_files
BEGIN
DELETE FROM embedding_chunks_fts
WHERE rowid IN (
SELECT id
FROM embedding_chunks
WHERE parent_file_id = new.parent_file_id
);
INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text)
SELECT id, chunk_id, new.file_name, chunk_text
FROM embedding_chunks
WHERE parent_file_id = new.parent_file_id;
END;
""", token);
}
private async Task<string> GetSqliteVersionAsync(CancellationToken token)
@ -319,21 +470,6 @@ public sealed class SqliteEmbeddingStateClientImplementation(
return Convert.ToInt64(countObject, CultureInfo.InvariantCulture);
}
private static async Task EnsureColumnAsync(SqliteConnection connection, string tableName, string columnName, string columnDefinition, CancellationToken token)
{
await using (var command = CreateCommand(connection, $"PRAGMA table_info({tableName})"))
{
await using var reader = await command.ExecuteReaderAsync(token);
while (await reader.ReadAsync(token))
{
if (string.Equals(reader.GetString(1), columnName, StringComparison.OrdinalIgnoreCase))
return;
}
}
await ExecuteNonQueryAsync(connection, $"ALTER TABLE {tableName} ADD COLUMN {columnName} {columnDefinition}", token);
}
private async Task<SqliteConnection> OpenConnectionAsync(CancellationToken token)
{
var connection = new SqliteConnection(this.connectionString);
@ -361,6 +497,21 @@ public sealed class SqliteEmbeddingStateClientImplementation(
await command.ExecuteNonQueryAsync(token);
}
private static async Task ExecuteNonQueryAsync(
SqliteConnection connection,
SqliteTransaction transaction,
string commandText,
CancellationToken token,
params (string Name, object? Value)[] parameters)
{
await using var command = CreateCommand(connection, commandText);
command.Transaction = transaction;
foreach (var (name, value) in parameters)
command.Parameters.AddWithValue(name, value ?? DBNull.Value);
await command.ExecuteNonQueryAsync(token);
}
private static string ToUtcText(DateTime dateTime)
{
var utc = dateTime.Kind is DateTimeKind.Utc ? dateTime : dateTime.ToUniversalTime();

10
app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs
View File

@ -6,11 +6,19 @@ public sealed record VectorStoragePoint(
string DataSourceId,
string DataSourceName,
string DataSourceType,
string ChunkId,
string ParentFileId,
string FilePath,
string AbsolutePath,
string FileName,
string RelativePath,
string FileType,
int? PageNumber,
int ChunkIndex,
string Text,
string Fingerprint,
DateTime CreationUtc,
DateTime LastWriteUtc,
DateTime EmbeddedAtUtc);
DateTime EmbeddedAtUtc,
string ComplianceLevel,
int ComplianceLevelRank);

67
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs
View File

@ -2,8 +2,10 @@ using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Databases.EmbeddingState;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Rust;
@ -27,6 +29,8 @@ public sealed partial class DataSourceEmbeddingService
private sealed record ExtractedFileContent(string Text, IReadOnlyList<string> SourceSegments);
private sealed record EmbeddingChunkDraft(string ChunkId, string Text, int ChunkIndex, int? PageNumber);
private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength);
private sealed record ChunkingStrategy(string Name, IReadOnlyList<ChunkingRule> Rules);
@ -748,6 +752,7 @@ public sealed partial class DataSourceEmbeddingService
embeddingProvider.Hostname,
embeddingProvider.TokenizerPath,
embeddingProvider.EffectiveTokenLimit,
GetDataSourceComplianceLevel(dataSource).ToString(),
dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0,
dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : 0,
chunkingOptions.MaxChunkTokenLength,
@ -847,6 +852,59 @@ public sealed partial class DataSourceEmbeddingService
return Convert.ToHexString(bytes);
}
private EmbeddingStateFile CreateEmbeddingStateFile(IDataSource dataSource, FileInfo file, string fingerprint, int chunkCount, DateTime embeddedAtUtc)
{
file.Refresh();
var absolutePath = Path.GetFullPath(file.FullName);
var complianceLevel = GetDataSourceComplianceLevel(dataSource);
return new(
this.CreateParentFileId(dataSource.Id, absolutePath),
absolutePath,
file.Name,
this.TryGetRelativePath(dataSource, file),
GetFileType(file),
fingerprint,
file.Exists ? file.Length : 0,
file.Exists ? file.CreationTimeUtc : DateTime.UnixEpoch,
file.Exists ? file.LastWriteTimeUtc : DateTime.UnixEpoch,
embeddedAtUtc,
chunkCount,
complianceLevel.ToString(),
(int)complianceLevel);
}
private IReadOnlyList<EmbeddingStateChunk> CreateEmbeddingStateChunks(EmbeddingStateFile parentFile, IReadOnlyList<EmbeddingChunkDraft> batch, DateTime embeddedAtUtc)
{
return batch
.Select(chunk => new EmbeddingStateChunk(
chunk.ChunkId,
parentFile.ParentFileId,
chunk.PageNumber,
chunk.ChunkIndex,
chunk.Text,
embeddedAtUtc))
.ToList();
}
private static ConfidenceLevel GetDataSourceComplianceLevel(IDataSource dataSource) =>
dataSource.ComplianceLevel is ConfidenceLevel.NONE
? ConfidenceLevel.UNKNOWN
: dataSource.ComplianceLevel;
private static string GetFileType(FileInfo file)
{
var extension = file.Extension.TrimStart('.').ToLowerInvariant();
return string.IsNullOrWhiteSpace(extension) ? "unknown" : extension;
}
private static int? TryExtractPageNumber(string chunk)
{
var match = Regex.Match(chunk, @"^\s*#\s+Page\s+(\d+)\b", RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
return match.Success && int.TryParse(match.Groups[1].Value, out var pageNumber) && pageNumber > 0
? pageNumber
: null;
}
private string GetCollectionName(string dataSourceName, string dataSourceId)
{
var safeId = dataSourceId
@ -864,9 +922,14 @@ public sealed partial class DataSourceEmbeddingService
return $"rag_{safeName}_{safeId}";
}
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex)
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) =>
CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
private string CreateParentFileId(string dataSourceId, string absolutePath) =>
CreateStableGuid($"{dataSourceId}:parent-file:{absolutePath}");
private static string CreateStableGuid(string source)
{
var source = $"{dataSourceId}:{fingerprint}:{chunkIndex}";
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(source));
var guidBytes = hash[..16].ToArray();

4
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs
View File

@ -17,11 +17,11 @@ public sealed partial class DataSourceEmbeddingService
embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token);
if (!embeddingState.IsAvailable)
{
logger.LogWarning("Could not delete SQLite embedding state for data source '{DataSourceId}' because the embedding state database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
logger.LogWarning("Could not delete local RAG index state for data source '{DataSourceId}' because the database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
return;
}
await embeddingState.DeleteDataSourceAsync(dataSourceId, token);
logger.LogInformation("Reset persisted embedding state for data source '{DataSourceId}'.", dataSourceId);
logger.LogInformation("Reset persisted local RAG index state for data source '{DataSourceId}'.", dataSourceId);
}
}

66
app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs
View File

@ -162,7 +162,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token);
if (!embeddingState.IsAvailable)
{
logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the embedding state database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name);
return true;
}
@ -395,7 +395,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
dataSource.Id,
embeddingState.Name);
token.ThrowIfCancellationRequested();
this.UpsertStatus(this.GetFallbackStatus(dataSource, "The SQLite embedding state database is not available."));
this.UpsertStatus(this.GetFallbackStatus(dataSource, "The local RAG index database is not available."));
return;
}
@ -522,15 +522,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
chunkCount);
await embeddingState.UpsertFileAsync(
dataSource.Id,
new EmbeddingStateFile(
file.FullName,
file.Name,
this.TryGetRelativePath(dataSource, file),
fingerprint,
file.Length,
file.LastWriteTimeUtc,
embeddedAtUtc,
chunkCount),
this.CreateEmbeddingStateFile(dataSource, file, fingerprint, chunkCount, embeddedAtUtc),
token);
manifest.Files[file.FullName] = record;
completedFiles++;
@ -603,22 +595,26 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
file.FullName,
collectionName);
await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token);
await embeddingState.DeleteFileAsync(dataSource.Id, file.FullName, token);
var parentFile = this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTime.UtcNow);
await embeddingState.UpsertFileAsync(dataSource.Id, parentFile, token);
var embeddingBatchSize = embeddingProvider.EffectiveEmbeddingBatchSize;
var batch = new List<(string Text, int ChunkIndex)>(embeddingBatchSize);
var batch = new List<EmbeddingChunkDraft>(embeddingBatchSize);
var totalChunkCount = 0;
await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, dataSource, embeddingProvider, token))
{
batch.Add((chunk, totalChunkCount));
batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk, totalChunkCount, TryExtractPageNumber(chunk)));
totalChunkCount++;
if (batch.Count >= embeddingBatchSize)
await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, collectionName, batch, token);
await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, collectionName, batch, token);
}
if (batch.Count > 0)
await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, collectionName, batch, token);
await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, collectionName, batch, token);
if (totalChunkCount == 0)
throw new InvalidOperationException($"The file '{file.Name}' did not yield any text chunks.");
@ -639,11 +635,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
IDataSource dataSource,
FileInfo file,
string fingerprint,
EmbeddingStateFile parentFile,
EmbeddingProvider embeddingProvider,
IProvider provider,
DataSourceEmbeddingManifest manifest,
string collectionName,
List<(string Text, int ChunkIndex)> batch,
List<EmbeddingChunkDraft> batch,
CancellationToken token)
{
logger.LogDebug(
@ -694,15 +691,22 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
}
token.ThrowIfCancellationRequested();
var embeddedAtUtc = DateTime.UtcNow;
await this.UpsertPointsAsync(
vectorStore,
collectionName,
dataSource,
file,
fingerprint,
parentFile,
batch,
vectors,
this.TryGetRelativePath(dataSource, file),
embeddedAtUtc,
token);
token.ThrowIfCancellationRequested();
await embeddingState.UpsertChunksAsync(
dataSource.Id,
this.CreateEmbeddingStateChunks(parentFile, batch, embeddedAtUtc),
token);
logger.LogDebug(
@ -725,26 +729,34 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
IDataSource dataSource,
FileInfo file,
string fingerprint,
IReadOnlyList<(string Text, int ChunkIndex)> batch,
EmbeddingStateFile parentFile,
IReadOnlyList<EmbeddingChunkDraft> batch,
IReadOnlyList<IReadOnlyList<float>> vectors,
string relativePath,
DateTime embeddedAtUtc,
CancellationToken token)
{
var embeddedAtUtc = DateTime.UtcNow;
var points = batch.Select((item, index) => new VectorStoragePoint(
this.CreatePointId(dataSource.Id, fingerprint, item.ChunkIndex),
item.ChunkId,
vectors[index],
dataSource.Id,
dataSource.Name,
dataSource.Type.ToString(),
item.ChunkId,
parentFile.ParentFileId,
file.FullName,
file.Name,
relativePath,
parentFile.AbsolutePath,
parentFile.FileName,
parentFile.RelativePath,
parentFile.FileType,
item.PageNumber,
item.ChunkIndex,
item.Text,
fingerprint,
file.LastWriteTimeUtc,
embeddedAtUtc)).ToList();
parentFile.CreationUtc,
parentFile.LastWriteUtc,
embeddedAtUtc,
parentFile.ComplianceLevel,
parentFile.ComplianceLevelRank)).ToList();
await vectorStore.InsertEmbedding(collectionName, points, token);
}
@ -805,7 +817,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
.ToList();
logger.LogInformation(
"Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed embedding state will be retried during this pass. File watchers will be activated after this check completes.",
"Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed local RAG index state will be retried during this pass. File watchers will be activated after this check completes.",
supportedDataSources.Count);
foreach (var dataSource in supportedDataSources)
@ -875,7 +887,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token);
logger.LogInformation(
"Loaded persisted embedding manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.",
"Loaded persisted local RAG index manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.",
dataSource.Name,
dataSource.Id,
manifest.Files.Count,

16
runtime/src/qdrant_edge_database.rs
View File

@ -67,14 +67,22 @@ pub struct QdrantEdgeStoragePoint {
pub data_source_id: String,
pub data_source_name: String,
pub data_source_type: String,
pub chunk_id: String,
pub parent_file_id: String,
pub file_path: String,
pub absolute_path: String,
pub file_name: String,
pub relative_path: String,
pub file_type: String,
pub page_number: Option<i32>,
pub chunk_index: i32,
pub text: String,
pub fingerprint: String,
pub creation_utc: String,
pub last_write_utc: String,
pub embedded_at_utc: String,
pub compliance_level: String,
pub compliance_level_rank: i32,
}
#[derive(Deserialize)]
@ -509,14 +517,22 @@ fn to_qdrant_edge_point(point: QdrantEdgeStoragePoint) -> qdrant_edge::PointStru
"data_source_id": point.data_source_id,
"data_source_name": point.data_source_name,
"data_source_type": point.data_source_type,
"chunk_id": point.chunk_id,
"parent_file_id": point.parent_file_id,
"file_path": point.file_path,
"absolute_path": point.absolute_path,
"file_name": point.file_name,
"relative_path": point.relative_path,
"file_type": point.file_type,
"page_number": point.page_number,
"chunk_index": point.chunk_index,
"text": point.text,
"fingerprint": point.fingerprint,
"creation_utc": point.creation_utc,
"last_write_utc": point.last_write_utc,
"embedded_at_utc": point.embedded_at_utc,
"compliance_level": point.compliance_level,
"compliance_level_rank": point.compliance_level_rank,
}),
)
.into()