diff --git a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua index a8b31fb8..68ba78fe 100644 --- a/app/MindWork AI Studio/Assistants/I18N/allTexts.lua +++ b/app/MindWork AI Studio/Assistants/I18N/allTexts.lua @@ -3655,6 +3655,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1DIALOG::T3804576966"] = "Por -- Connection failed. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1DIALOG::T3820825672"] = "Connection failed." +-- Compliance level +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1DIALOG::T3995796156"] = "Compliance level" + -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1DIALOG::T4081226330"] = "Your security policy" @@ -3772,6 +3775,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T3448155331"] = -- ERI server port UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T3843835535"] = "ERI server port" +-- Compliance level +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T3995796156"] = "Compliance level" + -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T4081226330"] = "Your security policy" @@ -3787,6 +3793,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T470340825"] = " -- the security requirements of the data provider UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T503852885"] = "the security requirements of the data provider" +-- the compliance level +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T607609781"] = "the compliance level" + -- When to use UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCEERI_V1INFODIALOG::T629595477"] = "When to use" @@ -3838,6 +3847,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1312296210" -- No embedding selected UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1359179968"] = "No embedding selected" +-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting." + -- Tokenizer UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1696723386"] = "Tokenizer" @@ -3856,29 +3868,26 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1847791252" -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method." +-- The overlap must be smaller than the effective token limit. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2101951526"] = "The overlap must be smaller than the effective token limit." + -- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap." --- Please enter 0 or a positive max embedding size. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2571963846"] = "Please enter 0 or a positive max embedding size." - -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2646845972"] = "Add" --- Max embedding size -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2812251281"] = "Max embedding size" - -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud." +-- The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2834874159"] = "The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting." + -- Embedding UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2838542994"] = "Embedding" --- Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T287150655"] = "Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting." - --- The overlap must be smaller than the effective max embedding size. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3100817060"] = "The overlap must be smaller than the effective max embedding size." +-- Token limit +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T2961294165"] = "Token limit" -- Please enter 0 or a positive overlap length. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length." @@ -3889,8 +3898,11 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3359366900" -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3361153305"] = "Show Expert Settings" --- Embedding overlap -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T375910361"] = "Embedding overlap" +-- Please enter 0 or a positive token limit. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3659673500"] = "Please enter 0 or a positive token limit." + +-- Compliance level +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T3995796156"] = "Compliance level" -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330"] = "Your security policy" @@ -3898,15 +3910,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T4081226330" -- Select the base directory UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T562479068"] = "Select the base directory" --- The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T607909514"] = "The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting." - -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T813773421"] = "Data Source Name" -- Cancel UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T900713019"] = "Cancel" +-- Token overlap +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYDIALOG::T981382809"] = "Token overlap" + -- the total directory size UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T1082241458"] = "the total directory size" @@ -3967,6 +3979,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3602384 -- Path UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3949388886"] = "Path" +-- Compliance level +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T3995796156"] = "Compliance level" + -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T4081226330"] = "Your security policy" @@ -3979,6 +3994,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T4438734 -- The directory chosen for the data source exists. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T445858624"] = "The directory chosen for the data source exists." +-- the compliance level +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALDIRECTORYINFODIALOG::T607609781"] = "the compliance level" + -- Hide Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1108876344"] = "Hide Expert Settings" @@ -3994,6 +4012,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1220918127"] = " -- No embedding selected UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1359179968"] = "No embedding selected" +-- Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1529596777"] = "Maximum number of tokens per chunk for this data source. Use 0 to use the embedding provider setting." + -- Tokenizer UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1696723386"] = "Tokenizer" @@ -4009,32 +4030,29 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1847791252"] = " -- In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T1948697886"] = "In order for the AI to be able to determine the appropriate data at any time, you must choose an embedding method." +-- The overlap must be smaller than the effective token limit. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2101951526"] = "The overlap must be smaller than the effective token limit." + -- Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2263831524"] = "Number of tokens repeated at the start of the next chunk. Use 0 to disable overlap." --- Please enter 0 or a positive max embedding size. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2571963846"] = "Please enter 0 or a positive max embedding size." - -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2646845972"] = "Add" --- Max embedding size -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2812251281"] = "Max embedding size" - -- The embedding you selected runs locally or in your organization. Your data is not sent to the cloud. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2814869210"] = "The embedding you selected runs locally or in your organization. Your data is not sent to the cloud." +-- The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2834874159"] = "The data source token limit must be smaller than the embedding provider token limit ({0}). Use 0 to use the provider setting." + -- Embedding UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2838542994"] = "Embedding" -- Describe what data this file contains to help the AI select it. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2859265837"] = "Describe what data this file contains to help the AI select it." --- Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T287150655"] = "Maximum number of tokens per embedded chunk for this data source. Use 0 to use the embedding provider setting." - --- The overlap must be smaller than the effective max embedding size. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3100817060"] = "The overlap must be smaller than the effective max embedding size." +-- Token limit +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T2961294165"] = "Token limit" -- Please enter 0 or a positive overlap length. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3242265813"] = "Please enter 0 or a positive overlap length." @@ -4045,11 +4063,14 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3359366900"] = " -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3361153305"] = "Show Expert Settings" +-- Please enter 0 or a positive token limit. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3659673500"] = "Please enter 0 or a positive token limit." + -- Select the file UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3740148848"] = "Select the file" --- Embedding overlap -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T375910361"] = "Embedding overlap" +-- Compliance level +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T3995796156"] = "Compliance level" -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = "Your security policy" @@ -4057,9 +4078,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T4081226330"] = " -- This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T458267883"] = "This data source already has indexed embeddings. Delete and recreate it to change the file path or embedding model." --- The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T607909514"] = "The data source max embedding size must be smaller than the embedding provider limit ({0}). Use 0 to use the provider setting." - -- Data Source Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T813773421"] = "Data Source Name" @@ -4069,6 +4087,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T900713019"] = "C -- Selected file path for this data source UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T939749563"] = "Selected file path for this data source" +-- Token overlap +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEDIALOG::T981382809"] = "Token overlap" + -- The file chosen for the data source exists. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T1294177559"] = "The file chosen for the data source exists." @@ -4123,9 +4144,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T3650018664"] -- The embedding runs in the cloud. All your data within the file '{0}' will be sent to the cloud. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T3688254408"] = "The embedding runs in the cloud. All your data within the file '{0}' will be sent to the cloud." +-- Compliance level +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T3995796156"] = "Compliance level" + -- Your security policy UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T4081226330"] = "Your security policy" +-- the compliance level +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DATASOURCELOCALFILEINFODIALOG::T607609781"] = "the compliance level" + -- Markdown View UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1373123357"] = "Markdown View" @@ -4270,9 +4297,6 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2331453405"] = "(O -- Invalid tokenizer: UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2448302543"] = "Invalid tokenizer:" --- Please enter a max embedding size greater than 0. -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2608328491"] = "Please enter a max embedding size greater than 0." - -- Add UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2646845972"] = "Add" @@ -4282,15 +4306,18 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T278585345"] = "Sel -- No models loaded or available. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2810182573"] = "No models loaded or available." --- Max embedding size -UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2812251281"] = "Max embedding size" - -- Instance Name UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2842060373"] = "Instance Name" -- Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually. UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T290547799"] = "Currently, we cannot query the embedding models for the selected provider and/or host. Therefore, please enter the model name manually." +-- Token limit +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T2961294165"] = "Token limit" + +-- Please enter a token limit greater than 0. +UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3316544737"] = "Please enter a token limit greater than 0." + -- Show Expert Settings UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGPROVIDERDIALOG::T3361153305"] = "Show Expert Settings" @@ -6394,6 +6421,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1132433749"] = "The Tokenizer li -- ID mismatch: the plugin ID differs from the enterprise configuration ID. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1137744461"] = "ID mismatch: the plugin ID differs from the enterprise configuration ID." +-- SQLite stores local RAG indexing metadata, searchable chunk text, and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation. +UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T117115925"] = "SQLite stores local RAG indexing metadata, searchable chunk text, and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation." + -- This is a private AI Studio installation. It runs without an enterprise configuration. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1209549230"] = "This is a private AI Studio installation. It runs without an enterprise configuration." @@ -6481,9 +6511,6 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2173617769"] = "This library is -- For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose. UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2174764529"] = "For the secure communication between the user interface and the runtime, we need to create certificates. This Rust library is great for this purpose." --- SQLite stores local RAG indexing metadata and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation. -UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T224267373"] = "SQLite stores local RAG indexing metadata and the file fingerprints used to decide whether local files need to be indexed again, without requiring a separate database server or a system SQLite installation." - -- OK UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T2246359087"] = "OK" @@ -6994,6 +7021,9 @@ UI_TEXT_CONTENT["AISTUDIO::PROVIDER::CONFIDENCELEVELEXTENSIONS::T3063224793"] = -- High UI_TEXT_CONTENT["AISTUDIO::PROVIDER::CONFIDENCELEVELEXTENSIONS::T3188327965"] = "High" +-- Unknown +UI_TEXT_CONTENT["AISTUDIO::PROVIDER::CONFIDENCELEVELEXTENSIONS::T3424652889"] = "Unknown" + -- Very Low UI_TEXT_CONTENT["AISTUDIO::PROVIDER::CONFIDENCELEVELEXTENSIONS::T786675843"] = "Very Low" @@ -7459,6 +7489,9 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTAT -- Indexed files UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T2235289713"] = "Indexed files" +-- Search chunks +UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T2333737457"] = "Search chunks" + -- Indexed data sources UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::EMBEDDINGSTATE::SQLITEEMBEDDINGSTATECLIENTIMPLEMENTATION::T3524534748"] = "Indexed data sources" diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor index 754d15f3..c955c1fc 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor @@ -1,4 +1,5 @@ @using AIStudio.Settings.DataModel +@using AIStudio.Provider @using AIStudio.Tools.ERIClient.DataModel @inherits MSGComponentBase @@ -119,6 +120,15 @@ } + + + @foreach (var level in this.ComplianceLevels) + { + + @level.Name + + } + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs index 8bec772a..cda00e32 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1Dialog.razor.cs @@ -1,5 +1,7 @@ using AIStudio.Assistants.ERI; using AIStudio.Components; +using AIStudio.Provider; +using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.ERIClient; using AIStudio.Tools.ERIClient.DataModel; @@ -46,6 +48,7 @@ public partial class DataSourceERI_V1Dialog : MSGComponentBase, ISecretId private string dataEditingPreviousInstanceName = string.Empty; private List availableAuthMethods = []; private DataSourceSecurity dataSecurityPolicy; + private ConfidenceLevel dataComplianceLevel = ConfidenceLevel.UNKNOWN; private SecurityRequirements dataSourceSecurityRequirements; private ushort dataMaxMatches = 10; private bool connectionTested; @@ -105,6 +108,7 @@ public partial class DataSourceERI_V1Dialog : MSGComponentBase, ISecretId this.dataAuthMethod = this.DataSource.AuthMethod; this.dataUsername = this.DataSource.Username; this.dataSecurityPolicy = this.DataSource.SecurityPolicy; + this.dataComplianceLevel = this.DataSource.ComplianceLevel; this.dataMaxMatches = this.DataSource.MaxMatches; // We cannot load the retrieval processes now, since we have @@ -157,6 +161,8 @@ public partial class DataSourceERI_V1Dialog : MSGComponentBase, ISecretId #endregion + private IEnumerable> ComplianceLevels => ConfigurationSelectDataFactory.GetDataSourceComplianceLevelsData(); + private DataSourceERI_V1 CreateDataSource() { var cleanedHostname = this.dataHostname.Trim(); @@ -172,6 +178,7 @@ public partial class DataSourceERI_V1Dialog : MSGComponentBase, ISecretId UsernamePasswordMode = DataSourceERIUsernamePasswordMode.USER_MANAGED, Type = DataSourceType.ERI_V1, SecurityPolicy = this.dataSecurityPolicy, + ComplianceLevel = this.dataComplianceLevel, SelectedRetrievalId = this.dataSelectedRetrievalProcess.Id, MaxMatches = this.dataMaxMatches, }; diff --git a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1InfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1InfoDialog.razor index fa9766fe..b35d183c 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceERI_V1InfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceERI_V1InfoDialog.razor @@ -1,4 +1,5 @@ @using AIStudio.Settings.DataModel +@using AIStudio.Provider @using AIStudio.Tools.ERIClient.DataModel @inherits MSGComponentBase @@ -27,6 +28,7 @@ + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor index c718e103..42729a2d 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor @@ -1,4 +1,5 @@ @using AIStudio.Settings.DataModel +@using AIStudio.Provider @inherits MSGComponentBase @@ -136,6 +137,15 @@ } + + + @foreach (var level in this.ComplianceLevels) + { + + @level.Name + + } + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs index 9b2bfd82..81909c57 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryDialog.razor.cs @@ -1,4 +1,5 @@ using AIStudio.Components; +using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.Validation; @@ -49,6 +50,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase private ushort dataMaxMatches = 10; private bool showExpertSettings; private DataSourceSecurity dataSecurityPolicy; + private ConfidenceLevel dataComplianceLevel = ConfidenceLevel.UNKNOWN; // We get the form reference from Blazor code to validate it manually: private MudForm form = null!; @@ -86,6 +88,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength; this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength; this.dataSecurityPolicy = this.DataSource.SecurityPolicy; + this.dataComplianceLevel = this.DataSource.ComplianceLevel; this.dataMaxMatches = this.DataSource.MaxMatches; this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0; } @@ -112,6 +115,8 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding; + private IEnumerable> ComplianceLevels => ConfigurationSelectDataFactory.GetDataSourceComplianceLevelsData(); + private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null ? T("No embedding selected") : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath) @@ -130,6 +135,7 @@ public partial class DataSourceLocalDirectoryDialog : MSGComponentBase MaxChunkTokenLength = this.dataMaxChunkTokenLength, ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, SecurityPolicy = this.dataSecurityPolicy, + ComplianceLevel = this.dataComplianceLevel, MaxMatches = this.dataMaxMatches, }; diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor index e80bad6a..b75c8fdd 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalDirectoryInfoDialog.razor @@ -1,4 +1,5 @@ @using AIStudio.Settings.DataModel +@using AIStudio.Provider @inherits MSGComponentBase @@ -38,6 +39,7 @@ } + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor index 5088864c..e7260a72 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor @@ -1,4 +1,5 @@ @using AIStudio.Settings.DataModel +@using AIStudio.Provider @inherits MSGComponentBase @@ -136,6 +137,15 @@ } + + + @foreach (var level in this.ComplianceLevels) + { + + @level.Name + + } + diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs index 822e1ace..e5fc5bd5 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileDialog.razor.cs @@ -1,4 +1,5 @@ using AIStudio.Components; +using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; using AIStudio.Tools.Validation; @@ -49,6 +50,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase private ushort dataMaxMatches = 10; private bool showExpertSettings; private DataSourceSecurity dataSecurityPolicy; + private ConfidenceLevel dataComplianceLevel = ConfidenceLevel.UNKNOWN; // We get the form reference from Blazor code to validate it manually: private MudForm form = null!; @@ -86,6 +88,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase this.dataMaxChunkTokenLength = this.DataSource.MaxChunkTokenLength; this.dataChunkOverlapTokenLength = this.DataSource.ChunkOverlapTokenLength; this.dataSecurityPolicy = this.DataSource.SecurityPolicy; + this.dataComplianceLevel = this.DataSource.ComplianceLevel; this.dataMaxMatches = this.DataSource.MaxMatches; this.showExpertSettings = this.dataMaxChunkTokenLength > 0 || this.dataChunkOverlapTokenLength > 0; } @@ -112,6 +115,8 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase private bool CanChangeSourceAndEmbedding => !this.IsEditing || !this.LockSourceAndEmbedding; + private IEnumerable> ComplianceLevels => ConfigurationSelectDataFactory.GetDataSourceComplianceLevelsData(); + private string SelectedEmbeddingTokenizerText => this.SelectedEmbedding is null ? T("No embedding selected") : string.IsNullOrWhiteSpace(this.SelectedEmbedding.TokenizerPath) @@ -130,6 +135,7 @@ public partial class DataSourceLocalFileDialog : MSGComponentBase MaxChunkTokenLength = this.dataMaxChunkTokenLength, ChunkOverlapTokenLength = this.dataChunkOverlapTokenLength, SecurityPolicy = this.dataSecurityPolicy, + ComplianceLevel = this.dataComplianceLevel, MaxMatches = this.dataMaxMatches, }; diff --git a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor index 61d07916..9b997ec3 100644 --- a/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor +++ b/app/MindWork AI Studio/Dialogs/DataSourceLocalFileInfoDialog.razor @@ -1,4 +1,5 @@ @using AIStudio.Settings.DataModel +@using AIStudio.Provider @inherits MSGComponentBase @@ -38,6 +39,7 @@ } + diff --git a/app/MindWork AI Studio/Pages/Information.razor b/app/MindWork AI Studio/Pages/Information.razor index e32f6a0b..b304a1d9 100644 --- a/app/MindWork AI Studio/Pages/Information.razor +++ b/app/MindWork AI Studio/Pages/Information.razor @@ -290,7 +290,7 @@ } - + diff --git a/app/MindWork AI Studio/Plugins/configuration/plugin.lua b/app/MindWork AI Studio/Plugins/configuration/plugin.lua index 5cf7172f..c2403aae 100644 --- a/app/MindWork AI Studio/Plugins/configuration/plugin.lua +++ b/app/MindWork AI Studio/Plugins/configuration/plugin.lua @@ -150,6 +150,7 @@ CONFIG["EMBEDDING_PROVIDERS"] = {} -- ERI v1 data sources for retrieval-augmented generation: CONFIG["DATA_SOURCES"] = {} +-- Allowed compliance levels are: UNTRUSTED, UNKNOWN, VERY_LOW, LOW, MODERATE, MEDIUM, HIGH -- Example: ERI v1 data source with a shared access token. -- CONFIG["DATA_SOURCES"][#CONFIG["DATA_SOURCES"]+1] = { @@ -161,6 +162,7 @@ CONFIG["DATA_SOURCES"] = {} -- ["AuthMethod"] = "TOKEN", -- ["Token"] = "ENC:v1:", -- ["SecurityPolicy"] = "SELF_HOSTED", +-- ["ComplianceLevel"] = "UNKNOWN", -- ["SelectedRetrievalId"] = "", -- ["MaxMatches"] = 10, -- } @@ -177,6 +179,7 @@ CONFIG["DATA_SOURCES"] = {} -- ["Username"] = "", -- ["Password"] = "ENC:v1:", -- ["SecurityPolicy"] = "SELF_HOSTED", +-- ["ComplianceLevel"] = "UNKNOWN", -- ["SelectedRetrievalId"] = "", -- ["MaxMatches"] = 10, -- } @@ -192,6 +195,7 @@ CONFIG["DATA_SOURCES"] = {} -- ["UsernamePasswordMode"] = "OS_USERNAME_SHARED_PASSWORD", -- ["Password"] = "ENC:v1:", -- ["SecurityPolicy"] = "SELF_HOSTED", +-- ["ComplianceLevel"] = "UNKNOWN", -- ["SelectedRetrievalId"] = "", -- ["MaxMatches"] = 10, -- } diff --git a/app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs b/app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs index 915b37cf..426bdffb 100644 --- a/app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs +++ b/app/MindWork AI Studio/Provider/ConfidenceLevelExtensions.cs @@ -12,6 +12,7 @@ public static class ConfidenceLevelExtensions ConfidenceLevel.NONE => TB("No provider selected"), ConfidenceLevel.UNTRUSTED => TB("Untrusted"), + ConfidenceLevel.UNKNOWN => TB("Unknown"), ConfidenceLevel.VERY_LOW => TB("Very Low"), ConfidenceLevel.LOW => TB("Low"), ConfidenceLevel.MODERATE => TB("Moderate"), diff --git a/app/MindWork AI Studio/Settings/ConfigurationSelectDataFactory.cs b/app/MindWork AI Studio/Settings/ConfigurationSelectDataFactory.cs index 7b2704ac..ddeaf5d5 100644 --- a/app/MindWork AI Studio/Settings/ConfigurationSelectDataFactory.cs +++ b/app/MindWork AI Studio/Settings/ConfigurationSelectDataFactory.cs @@ -294,6 +294,17 @@ public static class ConfigurationSelectDataFactory } } } + + public static IEnumerable> GetDataSourceComplianceLevelsData() + { + foreach (var level in Enum.GetValues()) + { + if (level is ConfidenceLevel.NONE) + continue; + + yield return new(level.GetName(), level); + } + } public static IEnumerable> GetThemesData() { diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs index 3fb7bd1a..a3fc8694 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs @@ -2,6 +2,7 @@ using AIStudio.Assistants.ERI; using AIStudio.Chat; +using AIStudio.Provider; using AIStudio.Tools.ERIClient; using AIStudio.Tools.ERIClient.DataModel; using AIStudio.Tools.PluginSystem; @@ -57,6 +58,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource /// public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; + /// + public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN; + /// public bool IsEnterpriseConfiguration { get; init; } @@ -206,6 +210,16 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource return false; } + var complianceLevel = ConfidenceLevel.UNKNOWN; + if (table.TryGetValue("ComplianceLevel", out var complianceLevelValue) && + (!complianceLevelValue.TryRead(out var complianceLevelText) || + !Enum.TryParse(complianceLevelText, true, out complianceLevel) || + complianceLevel is ConfidenceLevel.NONE)) + { + LOGGER.LogWarning($"The configured data source {idx} contains an invalid compliance level. Falling back to UNKNOWN. (Plugin ID: {configPluginId})"); + complianceLevel = ConfidenceLevel.UNKNOWN; + } + if (!table.TryGetValue("SelectedRetrievalId", out var selectedRetrievalIdValue) || !selectedRetrievalIdValue.TryRead(out var selectedRetrievalId) || string.IsNullOrWhiteSpace(selectedRetrievalId)) { LOGGER.LogWarning($"The configured data source {idx} must specify a selected retrieval ID. (Plugin ID: {configPluginId})"); @@ -263,6 +277,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource Username = username, UsernamePasswordMode = usernamePasswordMode, SecurityPolicy = securityPolicy, + ComplianceLevel = complianceLevel, Version = ERIVersion.V1, SelectedRetrievalId = selectedRetrievalId, MaxMatches = (ushort)maxMatches, @@ -323,6 +338,7 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource {{usernameLine}} {{secretLine}} ["SecurityPolicy"] = "{{this.SecurityPolicy}}", + ["ComplianceLevel"] = "{{this.ComplianceLevel}}", ["SelectedRetrievalId"] = "{{LuaTools.EscapeLuaString(this.SelectedRetrievalId)}}", ["MaxMatches"] = {{this.MaxMatches}}, } diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs index 3fd2363a..3c5c0c47 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalDirectory.cs @@ -1,4 +1,5 @@ using AIStudio.Chat; +using AIStudio.Provider; using AIStudio.Tools.RAG; namespace AIStudio.Settings.DataModel; @@ -42,6 +43,9 @@ public readonly record struct DataSourceLocalDirectory : IInternalDataSource /// public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; + /// + public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN; + /// public bool IsEnterpriseConfiguration { get; init; } diff --git a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs index 0f666f32..641c5355 100644 --- a/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs +++ b/app/MindWork AI Studio/Settings/DataModel/DataSourceLocalFile.cs @@ -1,4 +1,5 @@ using AIStudio.Chat; +using AIStudio.Provider; using AIStudio.Tools.RAG; namespace AIStudio.Settings.DataModel; @@ -42,6 +43,9 @@ public readonly record struct DataSourceLocalFile : IInternalDataSource /// public DataSourceSecurity SecurityPolicy { get; init; } = DataSourceSecurity.NOT_SPECIFIED; + /// + public ConfidenceLevel ComplianceLevel { get; init; } = ConfidenceLevel.UNKNOWN; + /// public bool IsEnterpriseConfiguration { get; init; } diff --git a/app/MindWork AI Studio/Settings/IDataSource.cs b/app/MindWork AI Studio/Settings/IDataSource.cs index 7e29123d..87a24d88 100644 --- a/app/MindWork AI Studio/Settings/IDataSource.cs +++ b/app/MindWork AI Studio/Settings/IDataSource.cs @@ -1,6 +1,7 @@ using System.Text.Json.Serialization; using AIStudio.Chat; +using AIStudio.Provider; using AIStudio.Settings.DataModel; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.RAG; @@ -25,6 +26,11 @@ public interface IDataSource : IConfigurationObject /// Which data security policy is applied to this data source? /// public DataSourceSecurity SecurityPolicy { get; init; } + + /// + /// Which compliance level is assigned to this data source? + /// + public ConfidenceLevel ComplianceLevel { get; init; } /// /// The maximum number of matches to return when retrieving data from the ERI server. diff --git a/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs b/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs index 2bcca341..f28c0779 100644 --- a/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs +++ b/app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs @@ -65,7 +65,7 @@ public sealed partial class DatabaseClientProvider(RustService rustService, ILog return new NoEmbeddingStateClient( client.Name, - "The configured database client does not support embedding state operations.", + "The configured database client does not support local RAG index operations.", client.Status); } diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs index 848082b9..5ebf4488 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/EmbeddingStateClient.cs @@ -24,15 +24,30 @@ public abstract class EmbeddingStateClient(string name, string path) : DatabaseC public abstract Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token); + public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList chunks, CancellationToken token); + public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token); } public sealed record EmbeddingStateFile( - string FilePath, + string ParentFileId, + string AbsolutePath, string FileName, string RelativePath, + string FileType, string Fingerprint, long FileSize, + DateTime CreationUtc, DateTime LastWriteUtc, DateTime EmbeddedAtUtc, - int ChunkCount); + int ChunkCount, + string ComplianceLevel, + int ComplianceLevelRank); + +public sealed record EmbeddingStateChunk( + string ChunkId, + string ParentFileId, + int? PageNumber, + int ChunkIndex, + string ChunkText, + DateTime EmbeddedAtUtc); diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs index 38bd49f6..e043b38a 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/NoEmbeddingStateClient.cs @@ -44,6 +44,8 @@ public sealed class NoEmbeddingStateClient(string name, string? unavailableReaso public override Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) => Task.CompletedTask; + public override Task UpsertChunksAsync(string dataSourceId, IReadOnlyList chunks, CancellationToken token) => Task.CompletedTask; + public override Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) => Task.CompletedTask; public override void Dispose() diff --git a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs index b93e80e0..a6fe24c6 100644 --- a/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs +++ b/app/MindWork AI Studio/Tools/Databases/EmbeddingState/SqliteEmbeddingStateClientImplementation.cs @@ -14,8 +14,8 @@ public sealed class SqliteEmbeddingStateClientImplementation( string basePath, string version) : EmbeddingStateClient(name, basePath) { - private const string DATABASE_NAME = "SQLite"; - private const string DATABASE_FILENAME = "rag-embedding-state.sqlite3"; + private const string DATABASE_NAME = "Local RAG Index"; + private const string DATABASE_FILENAME = "rag-index.sqlite3"; private readonly string databasePath = databasePath; private readonly string connectionString = new SqliteConnectionStringBuilder @@ -57,7 +57,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( } catch (Exception exception) { - logger.LogWarning(exception, "{DatabaseName} is not available. Indexed file fingerprints are disabled.", DATABASE_NAME); + logger.LogWarning(exception, "{DatabaseName} is not available. Indexed file fingerprints and search chunks are disabled.", DATABASE_NAME); return CreateNoEmbeddingStateClient(DATABASE_NAME, exception.Message, DatabaseClientStatus.UNAVAILABLE, databaseClientLogger); } } @@ -69,6 +69,7 @@ public sealed class SqliteEmbeddingStateClientImplementation( yield return (TB("Storage size"), this.GetStorageSize()); yield return (TB("Indexed data sources"), (await this.CountAsync("data_sources", CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); yield return (TB("Indexed files"), (await this.CountAsync("embedded_files", CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); + yield return (TB("Search chunks"), (await this.CountAsync("embedding_chunks", CancellationToken.None)).ToString(CultureInfo.InvariantCulture)); } public override async Task GetManifestAsync(string dataSourceId, CancellationToken token) @@ -94,9 +95,10 @@ public sealed class SqliteEmbeddingStateClientImplementation( } await using (var command = CreateCommand(connection, """ - SELECT file_path, fingerprint, file_size, last_write_utc, embedded_at_utc, chunk_count + SELECT absolute_path, fingerprint, file_size, last_write_utc, embedded_at_utc, chunk_count FROM embedded_files WHERE data_source_id = $dataSourceId + AND chunk_count > 0 """)) { command.Parameters.AddWithValue("$dataSourceId", dataSourceId); @@ -197,43 +199,64 @@ public sealed class SqliteEmbeddingStateClientImplementation( await using var connection = await this.OpenConnectionAsync(token); await ExecuteNonQueryAsync(connection, """ INSERT INTO embedded_files ( + parent_file_id, data_source_id, - file_path, + absolute_path, file_name, relative_path, + file_type, fingerprint, file_size, + creation_utc, last_write_utc, embedded_at_utc, - chunk_count) + chunk_count, + compliance_level, + compliance_level_rank) VALUES ( + $parentFileId, $dataSourceId, - $filePath, + $absolutePath, $fileName, $relativePath, + $fileType, $fingerprint, $fileSize, + $creationUtc, $lastWriteUtc, $embeddedAtUtc, - $chunkCount) - ON CONFLICT(data_source_id, file_path) DO UPDATE SET + $chunkCount, + $complianceLevel, + $complianceLevelRank) + ON CONFLICT(parent_file_id) DO UPDATE SET + data_source_id = excluded.data_source_id, + absolute_path = excluded.absolute_path, file_name = excluded.file_name, relative_path = excluded.relative_path, + file_type = excluded.file_type, fingerprint = excluded.fingerprint, file_size = excluded.file_size, + creation_utc = excluded.creation_utc, last_write_utc = excluded.last_write_utc, embedded_at_utc = excluded.embedded_at_utc, - chunk_count = excluded.chunk_count + chunk_count = excluded.chunk_count, + compliance_level = excluded.compliance_level, + compliance_level_rank = excluded.compliance_level_rank """, token, + ("$parentFileId", file.ParentFileId), ("$dataSourceId", dataSourceId), - ("$filePath", file.FilePath), + ("$absolutePath", file.AbsolutePath), ("$fileName", file.FileName), ("$relativePath", file.RelativePath), + ("$fileType", file.FileType), ("$fingerprint", file.Fingerprint), ("$fileSize", file.FileSize), + ("$creationUtc", ToUtcText(file.CreationUtc)), ("$lastWriteUtc", ToUtcText(file.LastWriteUtc)), ("$embeddedAtUtc", ToUtcText(file.EmbeddedAtUtc)), - ("$chunkCount", file.ChunkCount)); + ("$chunkCount", file.ChunkCount), + ("$complianceLevel", file.ComplianceLevel), + ("$complianceLevelRank", file.ComplianceLevelRank)); } public override async Task DeleteFileAsync(string dataSourceId, string filePath, CancellationToken token) @@ -242,12 +265,57 @@ public sealed class SqliteEmbeddingStateClientImplementation( await ExecuteNonQueryAsync(connection, """ DELETE FROM embedded_files WHERE data_source_id = $dataSourceId - AND file_path = $filePath + AND absolute_path = $filePath """, token, ("$dataSourceId", dataSourceId), ("$filePath", filePath)); } + public override async Task UpsertChunksAsync(string dataSourceId, IReadOnlyList chunks, CancellationToken token) + { + if (chunks.Count == 0) + return; + + await using var connection = await this.OpenConnectionAsync(token); + await using var transaction = (SqliteTransaction)await connection.BeginTransactionAsync(token); + + foreach (var chunk in chunks) + { + token.ThrowIfCancellationRequested(); + + await ExecuteNonQueryAsync(connection, transaction, """ + INSERT INTO embedding_chunks ( + chunk_id, + parent_file_id, + page_number, + chunk_index, + chunk_text, + embedded_at_utc) + VALUES ( + $chunkId, + $parentFileId, + $pageNumber, + $chunkIndex, + $chunkText, + $embeddedAtUtc) + ON CONFLICT(chunk_id) DO UPDATE SET + parent_file_id = excluded.parent_file_id, + page_number = excluded.page_number, + chunk_index = excluded.chunk_index, + chunk_text = excluded.chunk_text, + embedded_at_utc = excluded.embedded_at_utc + """, token, + ("$chunkId", chunk.ChunkId), + ("$parentFileId", chunk.ParentFileId), + ("$pageNumber", chunk.PageNumber), + ("$chunkIndex", chunk.ChunkIndex), + ("$chunkText", chunk.ChunkText), + ("$embeddedAtUtc", ToUtcText(chunk.EmbeddedAtUtc))); + } + + await transaction.CommitAsync(token); + } + public override async Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token) { await using var connection = await this.OpenConnectionAsync(token); @@ -281,26 +349,109 @@ public sealed class SqliteEmbeddingStateClientImplementation( ); CREATE TABLE IF NOT EXISTS embedded_files ( + parent_file_id TEXT PRIMARY KEY, data_source_id TEXT NOT NULL, - file_path TEXT COLLATE NOCASE NOT NULL, + absolute_path TEXT COLLATE NOCASE NOT NULL, file_name TEXT NOT NULL, relative_path TEXT NOT NULL, + file_type TEXT NOT NULL, fingerprint TEXT NOT NULL, file_size INTEGER NOT NULL, + creation_utc TEXT NOT NULL, last_write_utc TEXT NOT NULL, embedded_at_utc TEXT NOT NULL, chunk_count INTEGER NOT NULL, - PRIMARY KEY (data_source_id, file_path), + compliance_level TEXT NOT NULL, + compliance_level_rank INTEGER NOT NULL, FOREIGN KEY (data_source_id) REFERENCES data_sources(data_source_id) - ON DELETE CASCADE + ON DELETE CASCADE, + UNIQUE(data_source_id, absolute_path) + ); + + CREATE TABLE IF NOT EXISTS embedding_chunks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + chunk_id TEXT NOT NULL UNIQUE, + parent_file_id TEXT NOT NULL, + page_number INTEGER NULL, + chunk_index INTEGER NOT NULL, + chunk_text TEXT NOT NULL, + embedded_at_utc TEXT NOT NULL, + FOREIGN KEY (parent_file_id) + REFERENCES embedded_files(parent_file_id) + ON DELETE CASCADE, + UNIQUE(parent_file_id, chunk_index) ); CREATE INDEX IF NOT EXISTS idx_embedded_files_data_source ON embedded_files(data_source_id); - """, token); - await EnsureColumnAsync(connection, "data_sources", "source_hash", "TEXT NOT NULL DEFAULT ''", token); + CREATE INDEX IF NOT EXISTS idx_embedded_files_absolute_path + ON embedded_files(absolute_path); + + CREATE INDEX IF NOT EXISTS idx_embedded_files_file_type + ON embedded_files(file_type); + + CREATE INDEX IF NOT EXISTS idx_embedded_files_compliance + ON embedded_files(compliance_level_rank); + + CREATE INDEX IF NOT EXISTS idx_embedding_chunks_parent_file + ON embedding_chunks(parent_file_id); + + CREATE INDEX IF NOT EXISTS idx_embedding_chunks_page + ON embedding_chunks(page_number); + + CREATE VIRTUAL TABLE IF NOT EXISTS embedding_chunks_fts + USING fts5(chunk_id UNINDEXED, file_name, chunk_text); + + CREATE TRIGGER IF NOT EXISTS embedding_chunks_ai + AFTER INSERT ON embedding_chunks + BEGIN + INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text) + VALUES ( + new.id, + new.chunk_id, + (SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id), + new.chunk_text); + END; + + CREATE TRIGGER IF NOT EXISTS embedding_chunks_ad + AFTER DELETE ON embedding_chunks + BEGIN + DELETE FROM embedding_chunks_fts + WHERE rowid = old.id; + END; + + CREATE TRIGGER IF NOT EXISTS embedding_chunks_au + AFTER UPDATE ON embedding_chunks + BEGIN + DELETE FROM embedding_chunks_fts + WHERE rowid = old.id; + + INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text) + VALUES ( + new.id, + new.chunk_id, + (SELECT file_name FROM embedded_files WHERE parent_file_id = new.parent_file_id), + new.chunk_text); + END; + + CREATE TRIGGER IF NOT EXISTS embedded_files_file_name_au + AFTER UPDATE OF file_name ON embedded_files + BEGIN + DELETE FROM embedding_chunks_fts + WHERE rowid IN ( + SELECT id + FROM embedding_chunks + WHERE parent_file_id = new.parent_file_id + ); + + INSERT INTO embedding_chunks_fts(rowid, chunk_id, file_name, chunk_text) + SELECT id, chunk_id, new.file_name, chunk_text + FROM embedding_chunks + WHERE parent_file_id = new.parent_file_id; + END; + """, token); } private async Task GetSqliteVersionAsync(CancellationToken token) @@ -319,21 +470,6 @@ public sealed class SqliteEmbeddingStateClientImplementation( return Convert.ToInt64(countObject, CultureInfo.InvariantCulture); } - private static async Task EnsureColumnAsync(SqliteConnection connection, string tableName, string columnName, string columnDefinition, CancellationToken token) - { - await using (var command = CreateCommand(connection, $"PRAGMA table_info({tableName})")) - { - await using var reader = await command.ExecuteReaderAsync(token); - while (await reader.ReadAsync(token)) - { - if (string.Equals(reader.GetString(1), columnName, StringComparison.OrdinalIgnoreCase)) - return; - } - } - - await ExecuteNonQueryAsync(connection, $"ALTER TABLE {tableName} ADD COLUMN {columnName} {columnDefinition}", token); - } - private async Task OpenConnectionAsync(CancellationToken token) { var connection = new SqliteConnection(this.connectionString); @@ -361,6 +497,21 @@ public sealed class SqliteEmbeddingStateClientImplementation( await command.ExecuteNonQueryAsync(token); } + private static async Task ExecuteNonQueryAsync( + SqliteConnection connection, + SqliteTransaction transaction, + string commandText, + CancellationToken token, + params (string Name, object? Value)[] parameters) + { + await using var command = CreateCommand(connection, commandText); + command.Transaction = transaction; + foreach (var (name, value) in parameters) + command.Parameters.AddWithValue(name, value ?? DBNull.Value); + + await command.ExecuteNonQueryAsync(token); + } + private static string ToUtcText(DateTime dateTime) { var utc = dateTime.Kind is DateTimeKind.Utc ? dateTime : dateTime.ToUniversalTime(); diff --git a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs index fc95ed38..d6695c18 100644 --- a/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs +++ b/app/MindWork AI Studio/Tools/Databases/VectorStore/VectorStoragePoint.cs @@ -6,11 +6,19 @@ public sealed record VectorStoragePoint( string DataSourceId, string DataSourceName, string DataSourceType, + string ChunkId, + string ParentFileId, string FilePath, + string AbsolutePath, string FileName, string RelativePath, + string FileType, + int? PageNumber, int ChunkIndex, string Text, string Fingerprint, + DateTime CreationUtc, DateTime LastWriteUtc, - DateTime EmbeddedAtUtc); + DateTime EmbeddedAtUtc, + string ComplianceLevel, + int ComplianceLevelRank); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs index 2d04bc07..0682ba37 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.Files.cs @@ -2,8 +2,10 @@ using System.Security.Cryptography; using System.Text; using System.Text.RegularExpressions; +using AIStudio.Provider; using AIStudio.Settings; using AIStudio.Settings.DataModel; +using AIStudio.Tools.Databases.EmbeddingState; using AIStudio.Tools.PluginSystem; using AIStudio.Tools.Rust; @@ -27,6 +29,8 @@ public sealed partial class DataSourceEmbeddingService private sealed record ExtractedFileContent(string Text, IReadOnlyList SourceSegments); + private sealed record EmbeddingChunkDraft(string ChunkId, string Text, int ChunkIndex, int? PageNumber); + private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength); private sealed record ChunkingStrategy(string Name, IReadOnlyList Rules); @@ -748,6 +752,7 @@ public sealed partial class DataSourceEmbeddingService embeddingProvider.Hostname, embeddingProvider.TokenizerPath, embeddingProvider.EffectiveTokenLimit, + GetDataSourceComplianceLevel(dataSource).ToString(), dataSource is IInternalDataSource internalDataSource ? internalDataSource.MaxChunkTokenLength : 0, dataSource is IInternalDataSource overlapDataSource ? overlapDataSource.ChunkOverlapTokenLength : 0, chunkingOptions.MaxChunkTokenLength, @@ -847,6 +852,59 @@ public sealed partial class DataSourceEmbeddingService return Convert.ToHexString(bytes); } + private EmbeddingStateFile CreateEmbeddingStateFile(IDataSource dataSource, FileInfo file, string fingerprint, int chunkCount, DateTime embeddedAtUtc) + { + file.Refresh(); + var absolutePath = Path.GetFullPath(file.FullName); + var complianceLevel = GetDataSourceComplianceLevel(dataSource); + return new( + this.CreateParentFileId(dataSource.Id, absolutePath), + absolutePath, + file.Name, + this.TryGetRelativePath(dataSource, file), + GetFileType(file), + fingerprint, + file.Exists ? file.Length : 0, + file.Exists ? file.CreationTimeUtc : DateTime.UnixEpoch, + file.Exists ? file.LastWriteTimeUtc : DateTime.UnixEpoch, + embeddedAtUtc, + chunkCount, + complianceLevel.ToString(), + (int)complianceLevel); + } + + private IReadOnlyList CreateEmbeddingStateChunks(EmbeddingStateFile parentFile, IReadOnlyList batch, DateTime embeddedAtUtc) + { + return batch + .Select(chunk => new EmbeddingStateChunk( + chunk.ChunkId, + parentFile.ParentFileId, + chunk.PageNumber, + chunk.ChunkIndex, + chunk.Text, + embeddedAtUtc)) + .ToList(); + } + + private static ConfidenceLevel GetDataSourceComplianceLevel(IDataSource dataSource) => + dataSource.ComplianceLevel is ConfidenceLevel.NONE + ? ConfidenceLevel.UNKNOWN + : dataSource.ComplianceLevel; + + private static string GetFileType(FileInfo file) + { + var extension = file.Extension.TrimStart('.').ToLowerInvariant(); + return string.IsNullOrWhiteSpace(extension) ? "unknown" : extension; + } + + private static int? TryExtractPageNumber(string chunk) + { + var match = Regex.Match(chunk, @"^\s*#\s+Page\s+(\d+)\b", RegexOptions.CultureInvariant | RegexOptions.IgnoreCase); + return match.Success && int.TryParse(match.Groups[1].Value, out var pageNumber) && pageNumber > 0 + ? pageNumber + : null; + } + private string GetCollectionName(string dataSourceName, string dataSourceId) { var safeId = dataSourceId @@ -864,9 +922,14 @@ public sealed partial class DataSourceEmbeddingService return $"rag_{safeName}_{safeId}"; } - private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) + private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) => + CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}"); + + private string CreateParentFileId(string dataSourceId, string absolutePath) => + CreateStableGuid($"{dataSourceId}:parent-file:{absolutePath}"); + + private static string CreateStableGuid(string source) { - var source = $"{dataSourceId}:{fingerprint}:{chunkIndex}"; var hash = SHA256.HashData(Encoding.UTF8.GetBytes(source)); var guidBytes = hash[..16].ToArray(); diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs index 520b737d..2d8f478d 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.State.cs @@ -17,11 +17,11 @@ public sealed partial class DataSourceEmbeddingService embeddingState ??= await databaseClientProvider.GetEmbeddingStateAsync(token); if (!embeddingState.IsAvailable) { - logger.LogWarning("Could not delete SQLite embedding state for data source '{DataSourceId}' because the embedding state database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name); + logger.LogWarning("Could not delete local RAG index state for data source '{DataSourceId}' because the database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name); return; } await embeddingState.DeleteDataSourceAsync(dataSourceId, token); - logger.LogInformation("Reset persisted embedding state for data source '{DataSourceId}'.", dataSourceId); + logger.LogInformation("Reset persisted local RAG index state for data source '{DataSourceId}'.", dataSourceId); } } diff --git a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs index 18f08099..bbd5fdb7 100644 --- a/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs +++ b/app/MindWork AI Studio/Tools/Services/DataSourceEmbeddingService.cs @@ -162,7 +162,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM var embeddingState = await databaseClientProvider.GetEmbeddingStateAsync(token); if (!embeddingState.IsAvailable) { - logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the embedding state database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name); + logger.LogWarning("Locking identity settings for data source '{DataSourceId}' because the local RAG index database '{DatabaseName}' is unavailable.", dataSourceId, embeddingState.Name); return true; } @@ -395,7 +395,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM dataSource.Id, embeddingState.Name); token.ThrowIfCancellationRequested(); - this.UpsertStatus(this.GetFallbackStatus(dataSource, "The SQLite embedding state database is not available.")); + this.UpsertStatus(this.GetFallbackStatus(dataSource, "The local RAG index database is not available.")); return; } @@ -522,15 +522,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM chunkCount); await embeddingState.UpsertFileAsync( dataSource.Id, - new EmbeddingStateFile( - file.FullName, - file.Name, - this.TryGetRelativePath(dataSource, file), - fingerprint, - file.Length, - file.LastWriteTimeUtc, - embeddedAtUtc, - chunkCount), + this.CreateEmbeddingStateFile(dataSource, file, fingerprint, chunkCount, embeddedAtUtc), token); manifest.Files[file.FullName] = record; completedFiles++; @@ -603,22 +595,26 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM file.FullName, collectionName); await this.DeleteFilePointsAsync(vectorStore, collectionName, file.FullName, token); + await embeddingState.DeleteFileAsync(dataSource.Id, file.FullName, token); + + var parentFile = this.CreateEmbeddingStateFile(dataSource, file, fingerprint, 0, DateTime.UtcNow); + await embeddingState.UpsertFileAsync(dataSource.Id, parentFile, token); var embeddingBatchSize = embeddingProvider.EffectiveEmbeddingBatchSize; - var batch = new List<(string Text, int ChunkIndex)>(embeddingBatchSize); + var batch = new List(embeddingBatchSize); var totalChunkCount = 0; await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, dataSource, embeddingProvider, token)) { - batch.Add((chunk, totalChunkCount)); + batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk, totalChunkCount, TryExtractPageNumber(chunk))); totalChunkCount++; if (batch.Count >= embeddingBatchSize) - await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, collectionName, batch, token); + await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, collectionName, batch, token); } if (batch.Count > 0) - await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, embeddingProvider, provider, manifest, collectionName, batch, token); + await this.FlushBatchAsync(embeddingState, vectorStore, dataSource, file, fingerprint, parentFile, embeddingProvider, provider, manifest, collectionName, batch, token); if (totalChunkCount == 0) throw new InvalidOperationException($"The file '{file.Name}' did not yield any text chunks."); @@ -639,11 +635,12 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM IDataSource dataSource, FileInfo file, string fingerprint, + EmbeddingStateFile parentFile, EmbeddingProvider embeddingProvider, IProvider provider, DataSourceEmbeddingManifest manifest, string collectionName, - List<(string Text, int ChunkIndex)> batch, + List batch, CancellationToken token) { logger.LogDebug( @@ -694,15 +691,22 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM } token.ThrowIfCancellationRequested(); + var embeddedAtUtc = DateTime.UtcNow; await this.UpsertPointsAsync( vectorStore, collectionName, dataSource, file, fingerprint, + parentFile, batch, vectors, - this.TryGetRelativePath(dataSource, file), + embeddedAtUtc, + token); + token.ThrowIfCancellationRequested(); + await embeddingState.UpsertChunksAsync( + dataSource.Id, + this.CreateEmbeddingStateChunks(parentFile, batch, embeddedAtUtc), token); logger.LogDebug( @@ -725,26 +729,34 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM IDataSource dataSource, FileInfo file, string fingerprint, - IReadOnlyList<(string Text, int ChunkIndex)> batch, + EmbeddingStateFile parentFile, + IReadOnlyList batch, IReadOnlyList> vectors, - string relativePath, + DateTime embeddedAtUtc, CancellationToken token) { - var embeddedAtUtc = DateTime.UtcNow; var points = batch.Select((item, index) => new VectorStoragePoint( - this.CreatePointId(dataSource.Id, fingerprint, item.ChunkIndex), + item.ChunkId, vectors[index], dataSource.Id, dataSource.Name, dataSource.Type.ToString(), + item.ChunkId, + parentFile.ParentFileId, file.FullName, - file.Name, - relativePath, + parentFile.AbsolutePath, + parentFile.FileName, + parentFile.RelativePath, + parentFile.FileType, + item.PageNumber, item.ChunkIndex, item.Text, fingerprint, - file.LastWriteTimeUtc, - embeddedAtUtc)).ToList(); + parentFile.CreationUtc, + parentFile.LastWriteUtc, + embeddedAtUtc, + parentFile.ComplianceLevel, + parentFile.ComplianceLevelRank)).ToList(); await vectorStore.InsertEmbedding(collectionName, points, token); } @@ -805,7 +817,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM .ToList(); logger.LogInformation( - "Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed embedding state will be retried during this pass. File watchers will be activated after this check completes.", + "Starting initial persisted hash check for {DataSourceCount} supported internal data source(s). Incomplete or failed local RAG index state will be retried during this pass. File watchers will be activated after this check completes.", supportedDataSources.Count); foreach (var dataSource in supportedDataSources) @@ -875,7 +887,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM var manifest = await embeddingState.GetManifestAsync(dataSource.Id, token); logger.LogInformation( - "Loaded persisted embedding manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.", + "Loaded persisted local RAG index manifest for data source '{DataSourceName}' ({DataSourceId}). StoredFiles={StoredFiles}, StoredSourceHashPrefix={StoredSourceHashPrefix}, StoredSignaturePrefix={StoredSignaturePrefix}, CurrentSignaturePrefix={CurrentSignaturePrefix}.", dataSource.Name, dataSource.Id, manifest.Files.Count, diff --git a/runtime/src/qdrant_edge_database.rs b/runtime/src/qdrant_edge_database.rs index 0c495cb4..a77b3213 100644 --- a/runtime/src/qdrant_edge_database.rs +++ b/runtime/src/qdrant_edge_database.rs @@ -67,14 +67,22 @@ pub struct QdrantEdgeStoragePoint { pub data_source_id: String, pub data_source_name: String, pub data_source_type: String, + pub chunk_id: String, + pub parent_file_id: String, pub file_path: String, + pub absolute_path: String, pub file_name: String, pub relative_path: String, + pub file_type: String, + pub page_number: Option, pub chunk_index: i32, pub text: String, pub fingerprint: String, + pub creation_utc: String, pub last_write_utc: String, pub embedded_at_utc: String, + pub compliance_level: String, + pub compliance_level_rank: i32, } #[derive(Deserialize)] @@ -509,14 +517,22 @@ fn to_qdrant_edge_point(point: QdrantEdgeStoragePoint) -> qdrant_edge::PointStru "data_source_id": point.data_source_id, "data_source_name": point.data_source_name, "data_source_type": point.data_source_type, + "chunk_id": point.chunk_id, + "parent_file_id": point.parent_file_id, "file_path": point.file_path, + "absolute_path": point.absolute_path, "file_name": point.file_name, "relative_path": point.relative_path, + "file_type": point.file_type, + "page_number": point.page_number, "chunk_index": point.chunk_index, "text": point.text, "fingerprint": point.fingerprint, + "creation_utc": point.creation_utc, "last_write_utc": point.last_write_utc, "embedded_at_utc": point.embedded_at_utc, + "compliance_level": point.compliance_level, + "compliance_level_rank": point.compliance_level_rank, }), ) .into()