MindWorkAI/AI-Studio
1
0 Fork 0
mirror of https://github.com/MindWorkAI/AI-Studio.git synced 2026-10-30 21:04:26 +00:00
Code Issues Packages Projects Releases Wiki Activity

Added a way to read what the AI received from your sources (#1100)

Browse Source 
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
Paul KoudelkaandThorsten Sommer authored and GitHub committed 2026-10-26 10:00:01 +01:00
1 parent e0d8e18f34
commit cbc4ce5413
229 files changed
+12739 -1269
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

app/MindWork AI Studio/Agents/AgentRetrievalContextValidation.cs
+8 -11
View File
@@ -5,7 +5,6 @@ using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.RAG;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services;
namespace AIStudio.Agents;
@@ -165,11 +164,10 @@ public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalConte
/// </summary>
/// <param name="lastUserPrompt">The last user prompt.</param>
/// <param name="chatThread">The chat thread.</param>
/// <param name="retrievalContexts">All retrieval contexts to validate.</param>
/// <param name="reportingScope">The action the contexts are validated for, or null when they are validated for none.</param>
/// <param name="retrievalContexts">All retrieval contexts to validate, prepared for the AI and thus already filtered for prompt injections.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The validation results.</returns>
public async Task<IReadOnlyList<RetrievalContextValidationResult>> ValidateRetrievalContextsAsync(IContent lastUserPrompt, ChatThread chatThread, IReadOnlyList<IRetrievalContext> retrievalContexts, PromptInjectionReportingScope? reportingScope = null, CancellationToken token = default)
public async Task<IReadOnlyList<RetrievalContextValidationResult>> ValidateRetrievalContextsAsync(IContent lastUserPrompt, ChatThread chatThread, IReadOnlyList<PreparedRetrievalContext> retrievalContexts, CancellationToken token = default)
{
// Check if the retrieval context validation is enabled:
if (!this.SettingsManager.ConfigurationData.AgentRetrievalContextValidation.EnableRetrievalContextValidation)
@@ -195,7 +193,7 @@ public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalConte
await semaphore.WaitAsync(token);
// Start the next validation task:
validationTasks.Add(this.ValidateRetrievalContextAsync(lastUserPrompt, chatThread, retrievalContext, semaphore, reportingScope, token));
validationTasks.Add(this.ValidateRetrievalContextAsync(lastUserPrompt, chatThread, retrievalContext, semaphore, token));
}
// Wait for all validation tasks to complete:
@@ -212,12 +210,11 @@ public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalConte
/// </remarks>
/// <param name="lastUserPrompt">The last user prompt.</param>
/// <param name="chatThread">The chat thread.</param>
/// <param name="retrievalContext">The retrieval context to validate.</param>
/// <param name="retrievalContext">The retrieval context to validate, prepared for the AI and thus already filtered for prompt injections.</param>
/// <param name="semaphore">The optional semaphore to limit the number of parallel validations.</param>
/// <param name="reportingScope">The action the context is validated for, or null when it is validated for none.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The validation result.</returns>
public async Task<RetrievalContextValidationResult> ValidateRetrievalContextAsync(IContent lastUserPrompt, ChatThread chatThread, IRetrievalContext retrievalContext, SemaphoreSlim? semaphore = null, PromptInjectionReportingScope? reportingScope = null, CancellationToken token = default)
public async Task<RetrievalContextValidationResult> ValidateRetrievalContextAsync(IContent lastUserPrompt, ChatThread chatThread, PreparedRetrievalContext retrievalContext, SemaphoreSlim? semaphore = null, CancellationToken token = default)
{
try
{
@@ -252,11 +249,11 @@ public sealed class AgentRetrievalContextValidation (ILogger<AgentRetrievalConte
}
//
// 2. Prepare the retrieval context for the agent:
// 2. Prepare the retrieval context for the agent. It reads the context exactly as the
// AI would, filtered once for both, so the check judges what is actually sent:
//
var additionalData = new Dictionary<string, string>();
var markdownRetrievalContext = await retrievalContext.AsMarkdown(reportingScope: reportingScope, token: token);
additionalData.Add("retrievalContext", markdownRetrievalContext);
additionalData.Add("retrievalContext", retrievalContext.ToMarkdown());
//
// 3. Let the agent validate the retrieval context:
app/MindWork AI Studio/Agents/RetrievalContextValidationResult.cs
+2 -2
View File
@@ -8,5 +8,5 @@ namespace AIStudio.Agents;
/// <param name="Decision">Whether the retrieval context is useful or not.</param>
/// <param name="Reason">The reason for the decision.</param>
/// <param name="Confidence">The confidence of the decision.</param>
/// <param name="RetrievalContext">The retrieval context that was validated.</param>
public readonly record struct RetrievalContextValidationResult(bool Decision, string Reason, float Confidence, IRetrievalContext? RetrievalContext) : IConfidence;
/// <param name="RetrievalContext">The retrieval context that was validated, as it is prepared for the AI.</param>
public readonly record struct RetrievalContextValidationResult(bool Decision, string Reason, float Confidence, PreparedRetrievalContext? RetrievalContext) : IConfidence;
app/MindWork AI Studio/Assistants/AssistantBase.razor.cs
+5
View File
@@ -851,6 +851,11 @@ public abstract partial class AssistantBase<TSettings> : AssistantLowerBase wher
{
var convertedChatThread = this.ConvertToChatThread;
convertedChatThread = convertedChatThread with { SelectedProvider = this.ProviderSettings.Id };
// Some assistants start the chat under an id of its own, so what their answers received goes along. A chat without an id received nothing:
if (this.ChatThread is { } assistantChat && convertedChatThread.ChatId != Guid.Empty && assistantChat.ChatId != convertedChatThread.ChatId)
await ChatProvenance.CopyChatAsync(assistantChat.ChatId, convertedChatThread.ChatId);
MessageBus.INSTANCE.DeferMessage(this, sendToData.Event, new ChatStartRequest(convertedChatThread));
}
break;
app/MindWork AI Studio/Assistants/I18N/allTexts.lua
+224 -50
View File
@@ -3658,6 +3658,9 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T3378120620"] = "We could load mod
-- Your data sources could not be used. This answer was created without them.
UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T373499115"] = "Your data sources could not be used. This answer was created without them."
-- AI Studio could not store what the AI received separately from your chat. It remains in the chat itself, so you can still read it.
UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T713722424"] = "AI Studio could not store what the AI received separately from your chat. It remains in the chat itself, so you can still read it."
-- The local image file does not exist. Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T255679918"] = "The local image file does not exist. Skipping the image."
@@ -4060,6 +4063,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T4081853237"] = "C
-- Configure Keyboard Shortcut
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T636303786"] = "Configure Keyboard Shortcut"
-- Image
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONTENTVIEWER::T1494001562"] = "Image"
-- Plain text
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONTENTVIEWER::T3086748382"] = "Plain text"
-- Formatted
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONTENTVIEWER::T3618541955"] = "Formatted"
-- Keep the existing chat
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1274903302"] = "Keep the existing chat"
@@ -4267,6 +4279,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3655837348"] = "Back up the
-- Files which AI Studio needs only for a moment and deletes again.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T366353671"] = "Files which AI Studio needs only for a moment and deletes again."
-- What the AI received in your chats besides your messages, such as the content of attached files, the results of tools, and passages of your data sources. It is part of your chats.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3806894665"] = "What the AI received in your chats besides your messages, such as the content of attached files, the results of tools, and passages of your data sources. It is part of your chats."
-- Plugins which come with AI Studio, such as the language packs. AI Studio brings them back by itself.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3877928356"] = "Plugins which come with AI Studio, such as the language packs. AI Studio brings them back by itself."
@@ -4708,26 +4723,26 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERSETTINGSACTION::T244391
-- Change what this tile opens
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERSETTINGSACTION::T4272203100"] = "Change what this tile opens"
-- Could not open the file location.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1118835751"] = "Could not open the file location."
-- Copy the path: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T1140628382"] = "Copy the path: {0}"
-- Could not open the file: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1290686840"] = "Could not open the file: {0}"
-- Show this file in the file manager of your system
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T1587653504"] = "Show this file in the file manager of your system"
-- Open with the default program
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1332010305"] = "Open with the default program"
-- Copy the address: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T1663694701"] = "Copy the address: {0}"
-- Could not open the file location: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1455637941"] = "Could not open the file location: {0}"
-- AI Studio cannot find the file {0} on this computer. It may have been moved, renamed, or deleted, or it is on another computer, such as the server of a data source.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T21766676"] = "AI Studio cannot find the file {0} on this computer. It may have been moved, renamed, or deleted, or it is on another computer, such as the server of a data source."
-- Show in the file manager
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T3007835369"] = "Show in the file manager"
-- Open the file at page {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T3524349202"] = "Open the file at page {0}"
-- Unknown error
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T3461425987"] = "Unknown error"
-- Open this web page in your browser
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T3596972550"] = "Open this web page in your browser"
-- Could not open the file.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T362546438"] = "Could not open the file."
-- Open the file in the program your system uses for it
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T4218155268"] = "Open the file in the program your system uses for it"
-- Saving '{0}'...
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTRUNNINGMESSAGE::T1063135893"] = "Saving '{0}'..."
@@ -5851,26 +5866,38 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T78
-- Are you sure you want to delete the transcription provider '{0}'?
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T789660305"] = "Are you sure you want to delete the transcription provider '{0}'?"
-- Could not open the file location.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1118835751"] = "Could not open the file location."
-- Transcript of an audio file
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T1797766177"] = "Transcript of an audio file"
-- Could not open the file location: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1455637941"] = "Could not open the file location: {0}"
-- Transcript of a video file
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T3408352114"] = "Transcript of a video file"
-- Show this file in the file manager of your system
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1587653504"] = "Show this file in the file manager of your system"
-- At the moment, AI Studio cannot read the text of this passage that the AI received, e.g., while AI Studio is still starting.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T3678367303"] = "At the moment, AI Studio cannot read the text of this passage that the AI received, e.g., while AI Studio is still starting."
-- Opens this document in the program your system uses for it
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3169582185"] = "Opens this document in the program your system uses for it"
-- AI Studio no longer has the text of this passage that the AI received. Only where it was found is still known.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T3950319181"] = "AI Studio no longer has the text of this passage that the AI received. Only where it was found is still known."
-- Unknown error
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3461425987"] = "Unknown error"
-- Text the AI received
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T46215024"] = "Text the AI received"
-- Could not open the document.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3570758363"] = "Could not open the document."
-- Try again
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T557748474"] = "Try again"
-- Could not open the document: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T945417289"] = "Could not open the document: {0}"
-- Surrounding content
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T884772048"] = "Surrounding content"
-- Transcript
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T92473707"] = "Transcript"
-- An image matched here. The AI was told about it and where to find it, but it did not receive the image itself.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T989893925"] = "An image matched here. The AI was told about it and where to find it, but it did not receive the image itself."
-- Show the {0} passages the AI received from this source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T2556408625"] = "Show the {0} passages the AI received from this source"
-- Show the passage the AI received from this source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3350418006"] = "Show the passage the AI received from this source"
-- Copy {0} to the clipboard
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TEXTINFOLINE::T2206391442"] = "Copy {0} to the clipboard"
@@ -7831,21 +7858,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DIRECTCHATLAUNCHERSETTINGSDIALOG::T900713019
-- Please wait while we load the content of your file. Depending on the file type and size, this may take a moment.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1205126512"] = "Please wait while we load the content of your file. Depending on the file type and size, this may take a moment."
-- Markdown View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1373123357"] = "Markdown View"
-- This is the text the AI received from your file when you first sent it. Later changes to the file do not reach the AI in this chat.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T160563952"] = "This is the text the AI received from your file when you first sent it. Later changes to the file do not reach the AI in this chat."
-- Load file
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2129302565"] = "Load file"
-- Image View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2199753423"] = "Image View"
-- Your document is large, so we show you only its beginning. We hide the remaining {0:N0} characters here. Rest assured: the AI still receives your entire document.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2468296835"] = "Your document is large, so we show you only its beginning. We hide the remaining {0:N0} characters here. Rest assured: the AI still receives your entire document."
-- You can drag another file into this window. We attach it right away and show it here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2822249202"] = "You can drag another file into this window. We attach it right away and show it here."
-- Your document is large, so we show you only its beginning. We hide the remaining {0} characters here. Rest assured: the AI still receives your entire document.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2853647837"] = "Your document is large, so we show you only its beginning. We hide the remaining {0} characters here. Rest assured: the AI still receives your entire document."
-- The file is no longer in its original location. Below is the text the AI received from it when you sent it.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3044324335"] = "The file is no longer in its original location. Below is the text the AI received from it when you sent it."
-- See how we load your file. Review the content before we process it further.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3271853346"] = "See how we load your file. Review the content before we process it further."
@@ -7855,18 +7882,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3448155331"] = "Close"
-- Loaded Content
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3529911749"] = "Loaded Content"
-- Simple View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T428485200"] = "Simple View"
-- The file could not be found. It may have been moved, renamed, or deleted, or it is otherwise inaccessible.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T430625309"] = "The file could not be found. It may have been moved, renamed, or deleted, or it is otherwise inaccessible."
-- Text the AI received
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T46215024"] = "Text the AI received"
-- This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T652739927"] = "This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected."
-- File Path
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T729508546"] = "File Path"
-- The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T973777830"] = "The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible."
-- Embedding Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T1427271797"] = "Embedding Name"
@@ -8815,6 +8839,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3160778981"] = "Pr
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3448155331"] = "Close"
-- Show the text the AI received from this file.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3884104588"] = "Show the text the AI received from this file."
-- Your attached files
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3909191077"] = "Your attached files"
@@ -9205,6 +9232,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T82
-- Restore default
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T913168756"] = "Restore default"
-- Files are sent as they were first read
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1320692831"] = "Files are sent as they were first read"
-- Preselect one of your chat templates?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1402022556"] = "Preselect one of your chat templates?"
@@ -9232,6 +9262,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2510376349"]
-- Control how the LLM provider for added chats is selected.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T263621180"] = "Control how the LLM provider for added chats is selected."
-- Changed files are read again for the next message
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T264953820"] = "Changed files are read again for the next message"
-- Provider selection when loading a chat and sending assistant results to chat
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2868379953"] = "Provider selection when loading a chat and sending assistant results to chat"
@@ -9241,6 +9274,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2913693228"]
-- Do you want to use any shortcut to send your input?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2936560092"] = "Do you want to use any shortcut to send your input?"
-- Read changed attachments again?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3079510748"] = "Read changed attachments again?"
-- Would you like to set one of your chat templates as the default for chats?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3234927721"] = "Would you like to set one of your chat templates as the default for chats?"
@@ -9265,6 +9301,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3755993611"]
-- Do you want to apply the default data source options when sending assistant results to chat?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4033153439"] = "Do you want to apply the default data source options when sending assistant results to chat?"
-- When enabled, AI Studio reads an attached file again when it changed since it was sent, and the AI receives the new content with the next message. Viewing the attachment then shows the content that was sent last. When disabled, the AI keeps receiving the content a file had when it was first sent, also when the file changed or is gone.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4060508809"] = "When enabled, AI Studio reads an attached file again when it changed since it was sent, and the AI receives the new content with the next message. Viewing the attachment then shows the content that was sent last. When disabled, the AI keeps receiving the content a file had when it was first sent, also when the file changed or is gone."
-- When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T477675197"] = "When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider."
@@ -10336,6 +10375,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SINGLEINPUTDIALOG::T4030229154"] = "Your Inp
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SINGLEINPUTDIALOG::T900713019"] = "Cancel"
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SOURCEPASSAGESDIALOG::T3448155331"] = "Close"
-- This is exactly what the AI received from this source for its answer, after AI Studio removed any suspicious instructions.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SOURCEPASSAGESDIALOG::T406940705"] = "This is exactly what the AI received from this source for its answer, after AI Studio removed any suspicious instructions."
-- Passages
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SOURCEPASSAGESDIALOG::T4291613392"] = "Passages"
-- Hugging Face Inference Provider
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1085481431"] = "Hugging Face Inference Provider"
@@ -11068,6 +11116,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1421513382"] = "This library is
-- Trademarks & Brand Assets
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1421823619"] = "Trademarks & Brand Assets"
-- Provenance database
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1496928875"] = "Provenance database"
-- Copies the allowed host pattern to the clipboard
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1513592659"] = "Copies the allowed host pattern to the clipboard"
@@ -12721,6 +12772,69 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T3662391977"] = "
-- Status
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T6222351"] = "Status"
-- Reason
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T1093747001"] = "Reason"
-- Starting
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T1233211769"] = "Starting"
-- unknown
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T2608177081"] = "unknown"
-- Unavailable
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T3662391977"] = "Unavailable"
-- Process architecture
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T563161633"] = "Process architecture"
-- Status
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T6222351"] = "Status"
-- Native library
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T634426545"] = "Native library"
-- no migration applied
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T1014567907"] = "no migration applied"
-- Storage size
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T1230141403"] = "Storage size"
-- Stored items across all chats
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T1569215203"] = "Stored items across all chats"
-- {0} ({1} applied)
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2286846332"] = "{0} ({1} applied)"
-- Journal mode
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2411639995"] = "Journal mode"
-- Attached files
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T246258678"] = "Attached files"
-- Mails
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2534642695"] = "Mails"
-- unknown
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2608177081"] = "unknown"
-- Reported version
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T3556099842"] = "Reported version"
-- Schema version
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T424290830"] = "Schema version"
-- Passages of data sources
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T686479917"] = "Passages of data sources"
-- Web pages
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T748543057"] = "Web pages"
-- Chats with stored items
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T826028599"] = "Chats with stored items"
-- {0} ({1} applied, {2} pending)
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T983802795"] = "{0} ({1} applied, {2} pending)"
-- Reason
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::NOVECTORSTORECLIENT::T1093747001"] = "Reason"
@@ -14944,6 +15058,39 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T304
-- AI source selection with AI retrieval context validation
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T3775725978"] = "AI source selection with AI retrieval context validation"
-- Pages {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T1095604049"] = "Pages {0}"
-- {0}: rows {1}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T1424805945"] = "{0}: rows {1}"
-- {0}: row {1}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T1620414946"] = "{0}: row {1}"
-- Rows {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T2222697074"] = "Rows {0}"
-- Line {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T278321977"] = "Line {0}"
-- {0} passages
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T344316214"] = "{0} passages"
-- Lines {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T3536135902"] = "Lines {0}"
-- Part {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T3742022998"] = "Part {0}"
-- Row {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T4122835077"] = "Row {0}"
-- Page {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T4127287940"] = "Page {0}"
-- Parts {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T826267883"] = "Parts {0}"
-- Text
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text"
@@ -15325,12 +15472,6 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T26552229
-- Completed
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T3968379570"] = "Completed"
-- Chunk {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T2544251224"] = "Chunk {0}"
-- Page {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Page {0}"
-- The following data sources selected by the assistant chat launcher are currently unavailable or not permitted for the selected provider: {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DIRECTCHATSERVICE::T103791004"] = "The following data sources selected by the assistant chat launcher are currently unavailable or not permitted for the selected provider: {0}"
@@ -15760,6 +15901,21 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::UPDATESERVICE::T1064148123"] = "Fail
-- Failed to install update automatically. Please try again manually.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::UPDATESERVICE::T3709709946"] = "Failed to install update automatically. Please try again manually."
-- Could not open the file location.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T1118835751"] = "Could not open the file location."
-- Could not open the file location: {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T1455637941"] = "Could not open the file location: {0}"
-- Unknown error
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T3461425987"] = "Unknown error"
-- Could not open the document.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T3570758363"] = "Could not open the document."
-- Could not open the document: {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T945417289"] = "Could not open the document: {0}"
-- Sources
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEEXTENSIONS::T2730980305"] = "Sources"
@@ -15856,6 +16012,24 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS:
-- To use this tool, add a mailbox to your data sources first.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLCONFIGURATION::T1547505353"] = "To use this tool, add a mailbox to your data sources first."
-- To
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T1106613828"] = "To"
-- Text from the attachment
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T1262266390"] = "Text from the attachment"
-- Received
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T1559762898"] = "Received"
-- Subject
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T2692293871"] = "Subject"
-- From
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T410269077"] = "From"
-- Attachments
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T498946337"] = "Attachments"
-- Read Mail
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::READMAILTOOL::T2299936968"] = "Read Mail"
app/MindWork AI Studio/Chat/ChatDraft.cs
+62
View File
@@ -42,6 +42,19 @@ public sealed class ChatDraft
[JsonInclude]
public IReadOnlyList<FileAttachment> FileAttachments { get; private set; } = [];
/// <summary>
/// The texts a preview read of the attached files, which sending takes along instead of reading the files again.
/// </summary>
/// <remarks>
/// Kept apart from the attachments, which stay as the user attached them: the composer, this
/// draft, and the attachment dialogs tell attachments apart by their values, and one which changed
/// underneath them would count as removed and added. Stored with the chat like everything else of
/// the draft, so a preview still counts when the user comes back to the chat. The provenance
/// store holds the texts themselves.
/// </remarks>
[JsonInclude]
public IReadOnlyList<PreviewedText> PreviewedTexts { get; private set; } = [];
/// <summary>
/// Whether the draft holds neither text nor attachments.
/// </summary>
@@ -140,15 +153,64 @@ public sealed class ChatDraft
}
}
/// <summary>
/// Finds the text a preview read of an attached file.
/// </summary>
/// <param name="filePath">The path of the file.</param>
/// <returns>The text, or null when no preview read the file.</returns>
public PreviewedText? GetPreviewedText(string filePath)
{
lock (this.stateLock)
return this.PreviewedTexts.FirstOrDefault(text => string.Equals(text.FilePath, filePath, StringComparison.Ordinal));
}
/// <summary>
/// Notes the text a preview read of an attached file, in place of one noted for the same file before.
/// </summary>
/// <param name="previewedText">The text.</param>
/// <returns>The entry of the text noted before, which nothing refers to any longer, or null.</returns>
public Guid? NotePreviewedText(PreviewedText previewedText)
{
lock (this.stateLock)
{
var replaced = this.PreviewedTexts.FirstOrDefault(text => string.Equals(text.FilePath, previewedText.FilePath, StringComparison.Ordinal));
this.PreviewedTexts = [..this.PreviewedTexts.Where(text => !ReferenceEquals(text, replaced)), previewedText];
return replaced?.ProvenanceId;
}
}
/// <summary>
/// Lets go of the texts a preview read of files which are no longer attached, e.g., since the user took them out.
/// </summary>
/// <returns>The entries of those texts, which nothing refers to any longer, so the caller removes them from the provenance store.</returns>
public IReadOnlyList<Guid> ForgetPreviewsOfDetachedFiles()
{
lock (this.stateLock)
{
var attachedPaths = this.FileAttachments.Select(attachment => attachment.FilePath).ToHashSet(StringComparer.Ordinal);
var detached = this.PreviewedTexts.Where(text => !attachedPaths.Contains(text.FilePath)).ToList();
if (detached.Count is 0)
return [];
this.PreviewedTexts = this.PreviewedTexts.Except(detached).ToList();
return detached.Select(text => text.ProvenanceId).ToList();
}
}
/// <summary>
/// Empties the draft, which is what sending it does: its attachments belong to the new message from then on.
/// </summary>
/// <remarks>
/// The texts a preview read go as well. Whoever sends the draft removes those the message went
/// without from the provenance store.
/// </remarks>
public void Clear()
{
lock (this.stateLock)
{
this.Text = string.Empty;
this.FileAttachments = [];
this.PreviewedTexts = [];
}
}
app/MindWork AI Studio/Chat/ChatProvenance.cs
+115
View File
@@ -0,0 +1,115 @@
using AIStudio.Tools.Databases;
using AIStudio.Tools.Databases.ProvenanceStore;
namespace AIStudio.Chat;
/// <summary>
/// Keeps what chats received in the provenance store in step with the chats themselves: it goes
/// when its chat goes, and it comes along when its chat is copied.
/// </summary>
/// <remarks>
/// Called from the places which delete or copy chats. Nothing here may stop them: a chat is
/// deleted or copied either way, and what the store keeps too long is removed by the cleanup at
/// the next start, see ProvenanceCleanup.
/// </remarks>
public static class ChatProvenance
{
private static readonly ILogger LOGGER = Program.LOGGER_FACTORY.CreateLogger(nameof(ChatProvenance));
/// <summary>
/// Collects the entries of the provenance store which a chat refers to.
/// </summary>
/// <remarks>
/// The passages of its answers, the documents attached to its messages, and those attached to
/// the message being written, which may come from a message sent before, e.g., one being edited,
/// together with the texts a preview read of them.
/// </remarks>
/// <param name="thread">The chat.</param>
/// <returns>The ids of the entries.</returns>
public static IReadOnlySet<Guid> CollectProvenanceIds(this ChatThread thread)
{
var provenanceIds = new HashSet<Guid>();
foreach (var block in thread.Blocks)
{
if (block.Content is ContentText text)
provenanceIds.UnionWith(text.SourcePassages.Select(passage => passage.ProvenanceId).OfType<Guid>());
if (block.Content is not null)
provenanceIds.UnionWith(block.Content.FileAttachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>());
}
provenanceIds.UnionWith(thread.Draft.FileAttachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>());
provenanceIds.UnionWith(thread.Draft.PreviewedTexts.Select(text => text.ProvenanceId));
return provenanceIds;
}
/// <summary>
/// Deletes what the given chats received from the provenance store, for chats which were deleted.
/// </summary>
/// <param name="chatIds">The deleted chats.</param>
public static async Task ForgetChatsAsync(IReadOnlyCollection<Guid> chatIds)
{
if (chatIds.Count is 0 || await GetStoreAsync() is not { } store)
return;
try
{
var deleted = await store.DeleteChatsAsync(chatIds, CancellationToken.None);
if (deleted > 0)
LOGGER.LogInformation("Deleted {Count} provenance entries of {ChatCount} deleted chats.", deleted, chatIds.Count);
}
catch (Exception exception)
{
LOGGER.LogWarning(exception, "Failed to delete the provenance entries of {ChatCount} deleted chats. The next start removes them.", chatIds.Count);
}
}
/// <summary>
/// Copies what a chat received to its copy, which refers to the same entries.
/// </summary>
/// <param name="sourceChatId">The chat which was copied.</param>
/// <param name="targetChatId">The copy.</param>
public static async Task CopyChatAsync(Guid sourceChatId, Guid targetChatId)
{
if (await GetStoreAsync() is not { } store)
return;
try
{
await store.CopyChatAsync(sourceChatId, targetChatId, CancellationToken.None);
}
catch (Exception exception)
{
LOGGER.LogWarning(exception, "Failed to copy the provenance entries of the chat '{SourceChatId}' to its copy '{TargetChatId}'. The copy shows them as long as the original keeps them.", sourceChatId, targetChatId);
}
}
/// <summary>
/// Gets the provenance store for chats and their messages, or nothing when there is none to use.
/// </summary>
/// <remarks>
/// The one way to the store for code which the app's services do not create, such as a chat or a
/// message. Components and services get the DatabaseClientProvider injected instead. It never
/// throws: there is no store while the app has not started its services, e.g., in a test, or when
/// the store is not available. Every caller carries on without it then.
/// </remarks>
/// <returns>The provenance store, or null when there is none.</returns>
internal static async Task<ProvenanceStoreClient?> GetStoreAsync()
{
try
{
// Declared as never null, but it is null until the app has started its services:
// ReSharper disable once ConditionalAccessQualifierIsNonNullableAccordingToAPIContract
if (Program.SERVICE_PROVIDER?.GetService<DatabaseClientProvider>() is not { } databaseClientProvider)
return null;
var store = await databaseClientProvider.GetProvenanceStoreAsync();
return store is NoProvenanceStoreClient ? null : store;
}
catch (Exception exception)
{
LOGGER.LogWarning(exception, "The provenance store is not available to the chats.");
return null;
}
}
}
app/MindWork AI Studio/Chat/ChatThread.cs
+46
View File
@@ -5,6 +5,7 @@ using AIStudio.Components;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Security;
using AIStudio.Tools.ToolCallingSystem;
using AIStudio.Tools.ERIClient.DataModel;
using AIStudio.Tools.Web;
@@ -102,6 +103,15 @@ public sealed record ChatThread
/// <summary>
/// The augmented data for this chat thread. Will be inserted into the system prompt.
/// </summary>
/// <remarks>
/// Not stored: it belongs to one request. The classic RAG process empties and writes it anew for
/// every message, a regenerated answer included, and a rollback empties it, so nothing reads it
/// once the answer is there. What the AI read of it is kept with the answer instead, as the
/// passages of its sources, see ContentText.StoreReceivedContentAsync. Stored, it was often the
/// largest part of a chat file. Older chat files still hold it; it is ignored on loading and
/// left out on their next save.
/// </remarks>
[JsonIgnore]
public string AugmentedData { get; set; } = string.Empty;
/// <summary>
@@ -515,6 +525,42 @@ public sealed record ChatThread
this.Blocks.Remove(block);
}
/// <summary>
/// Stores the documents attached to the messages of this chat which were never sent, so that each
/// one is read from the disk once.
/// </summary>
/// <remarks>
/// Called before every request of the chat, see AIJobService. The request then sends the stored
/// texts, see ContentText.PrepareTextContentForAI, and so does every later one. Assistants do not
/// call this: their messages are made anew with every run, so every run would store its documents
/// once more.<br/><br/>
///
/// Nothing here may stop the request. A document which is not stored, e.g., since the store is not
/// available, is read when the message is built, as every request did before.
/// </remarks>
/// <param name="readChangedAgain">Whether documents whose files changed since they were stored are read again, see DataChat.ReadChangedAttachmentsAgain.</param>
public async Task StoreAttachedDocumentsAsync(bool readChangedAgain)
{
var messages = this.Blocks.Select(block => block.Content).OfType<ContentText>().Where(text => text.HasDocumentsToStore(readChangedAgain)).ToList();
if (messages.Count is 0)
return;
try
{
if (await ChatProvenance.GetStoreAsync() is not { } store)
return;
// One report for everything filtered out of the documents, as for the request itself:
await using var reportingScope = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>().BeginAction();
foreach (var message in messages)
await message.StoreAttachedDocumentsAsync(store, this.ChatId, Program.RUST_SERVICE, reportingScope, readChangedAgain);
}
catch (Exception exception)
{
LOGGER.LogError(exception, "Failed to store the attached documents of the chat '{ChatId}'. They are read from the disk with every request.", this.ChatId);
}
}
/// <summary>
/// Rolls this chat thread back, so that the given content becomes the last block of the conversation.
/// </summary>
app/MindWork AI Studio/Chat/ContentBlock.cs
+20
View File
@@ -38,4 +38,24 @@ public class ContentBlock
Role = this.Role,
HideFromUser = changeHideState ? hideFromUser : this.HideFromUser,
};
/// <summary>
/// Clones this block for a place apart from its chat, e.g., the example conversation of a chat template.
/// </summary>
/// <remarks>
/// Without anything which refers to the provenance store: its entries belong to the chat and are
/// deleted with it. The passages of an answer stay behind, and every chat which receives an
/// attached document reads it anew from its file, cf. FileAttachment.ProvenanceId.
/// </remarks>
public ContentBlock CloneApartFromChat(bool changeHideState = false, bool hideFromUser = true)
{
var clone = this.DeepClone(changeHideState, hideFromUser);
if (clone.Content is ContentText text)
text.SourcePassages = [];
if (clone.Content is not null)
clone.Content.FileAttachments = clone.Content.FileAttachments.Select(attachment => attachment.WithoutStoredText()).ToList();
return clone;
}
}
app/MindWork AI Studio/Chat/ContentBlockComponent.razor
+1 -1
View File
@@ -229,7 +229,7 @@
}
@if (textContent.Sources.Count > 0)
{
<SourcesList @ref="this.sourcesList" Sources="@textContent.Sources"/>
<SourcesList @ref="this.sourcesList" Sources="@textContent.Sources" Passages="@textContent.SourcePassages"/>
}
</div>
}
app/MindWork AI Studio/Chat/ContentBlockComponent.razor.cs
+1 -7
View File
@@ -360,6 +360,7 @@ public partial class ContentBlockComponent : MSGComponentBase
hash.Add(textValue.Length);
hash.Add(textValue.GetHashCode(StringComparison.Ordinal));
hash.Add(text.Sources.Count);
hash.Add(text.SourcePassages.Count);
hash.Add(text.ToolInvocations.Count);
hash.Add(text.ToolRuntimeStatus.IsRunning);
hash.Add(text.ToolRuntimeStatus.Message);
@@ -399,8 +400,6 @@ public partial class ContentBlockComponent : MSGComponentBase
private bool HasToolTrace => this.Role is ChatRole.AI && this.GetToolInvocations().Count > 0;
private CodeBlockTheme CodeColorPalette => this.SettingsManager.IsDarkMode ? CodeBlockTheme.Dark : CodeBlockTheme.Default;
private static Color GetTraceColor(ToolInvocationTraceStatus status) => status switch
{
ToolInvocationTraceStatus.SUCCESS => Color.Success,
@@ -446,11 +445,6 @@ public partial class ContentBlockComponent : MSGComponentBase
? this.T("No result")
: invocation.Result;
private MudMarkdownStyling MarkdownStyling => new()
{
CodeBlock = { Theme = this.CodeColorPalette },
};
private MarkdownRenderPlan GetMarkdownRenderPlan(string text)
{
if (ReferenceEquals(this.cachedMarkdownRenderPlanInput, text) || string.Equals(this.cachedMarkdownRenderPlanInput, text, StringComparison.Ordinal))
app/MindWork AI Studio/Chat/ContentText.cs
+377 -55
View File
@@ -4,10 +4,13 @@ using System.Text.Json.Serialization;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Tools.Databases.ProvenanceStore;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG;
using AIStudio.Tools.RAG.RAGProcesses;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Security;
using AIStudio.Tools.Services;
using AIStudio.Tools.ToolCallingSystem;
using AIStudio.Tools.Web;
@@ -54,6 +57,128 @@ public sealed class ContentText : IContent
public List<ToolInvocationTrace> ToolInvocations { get; set; } = [];
/// <summary>
/// The passages the data sources handed to the AI for this answer, as the AI received them.
/// </summary>
/// <remarks>
/// Stored with the answer, so the user can still read what it rests on after the chat was
/// reloaded. Each passage belongs to one of the sources, see SourcePassageExtensions.GetPassagesOf.
/// Empty for every answer written before passages were kept, and for every answer no data
/// source contributed to.
/// </remarks>
public List<SourcePassage> SourcePassages { get; set; } = [];
/// <summary>
/// Adds sources to this answer, together with the passages the AI received from them.
/// </summary>
/// <remarks>
/// Both at once, because the title of a source names where its passages are, see
/// SourcePassageExtensions.NamePassagesInTitles. A passage added on its own would leave a title
/// behind which names only the pages known before it. Both lists are replaced as a whole rather
/// than changed, as in StoreReceivedContentAsync: the user interface may read them meanwhile.
/// </remarks>
/// <param name="sources">The sources to add.</param>
/// <param name="passages">The passages to add.</param>
public void AddSources(IEnumerable<ISource> sources, IEnumerable<SourcePassage> passages)
{
var mergedSources = new List<Source>(this.Sources);
mergedSources.MergeSources(sources);
var mergedPassages = new List<SourcePassage>(this.SourcePassages);
mergedPassages.MergePassages(passages);
mergedSources.NamePassagesInTitles(mergedPassages);
// The passages first, so that every source the user interface finds has its passages:
this.SourcePassages = mergedPassages;
this.Sources = mergedSources;
}
/// <summary>
/// Whether the user was told that a text stays in the chat file, since the provenance store could not take it.
/// </summary>
/// <remarks>
/// Once per run of the app: a store which cannot be written stays so for a while, and a message
/// with every answer would tell the user nothing new.
/// </remarks>
private static int UNSTORED_CONTENT_REPORTED;
/// <summary>
/// Moves what the AI received for this answer into the provenance store, so the chat keeps only references to it.
/// </summary>
/// <remarks>
/// Called once the answer is complete, also when it was stopped or failed: until then, a passage
/// found twice is told apart by its text, see SourcePassageExtensions.MergePassages, and only the
/// passages which made it into the answer get stored. Nothing here may stop the answer from being
/// stored, so every failure ends in the passages keeping their texts.
/// </remarks>
/// <param name="chatId">The chat this answer belongs to.</param>
public async Task StoreReceivedContentAsync(Guid chatId)
{
try
{
// Most answers, those of the agents among them, received nothing to store, and need no database:
if (!this.SourcePassages.Any(passage => passage.HoldsTextsToStore()))
return;
if (await ChatProvenance.GetStoreAsync() is { } store && await this.StoreReceivedContentAsync(store, chatId))
return;
}
catch (Exception exception)
{
LOGGER.LogError(exception, "Failed to store what the AI received for an answer of the chat '{ChatId}'. It stays in the chat file.", chatId);
}
if (Interlocked.Exchange(ref UNSTORED_CONTENT_REPORTED, 1) is 0)
await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Storage, TB("AI Studio could not store what the AI received separately from your chat. It remains in the chat itself, so you can still read it.")));
}
/// <summary>
/// Moves what the AI received for this answer into the given provenance store.
/// </summary>
/// <remarks>
/// The list of passages is replaced as a whole rather than changed: the user interface may read
/// it meanwhile. A passage which the store could not take keeps its texts.
/// </remarks>
/// <param name="store">The provenance store.</param>
/// <param name="chatId">The chat this answer belongs to.</param>
/// <returns>True when everything was stored, false when something stays in the chat itself.</returns>
internal async Task<bool> StoreReceivedContentAsync(ProvenanceStoreClient store, Guid chatId)
{
var allStored = true;
var passages = new List<SourcePassage>(this.SourcePassages.Count);
foreach (var passage in this.SourcePassages)
{
if (!passage.HoldsTextsToStore())
{
passages.Add(passage);
continue;
}
Guid? provenanceId = null;
try
{
// Not cancellable: the answer is complete, also when the user stopped it.
provenanceId = await store.StoreAsync(chatId, passage.GetProvenanceKind(), passage.Text, passage.SurroundingContent, CancellationToken.None);
}
catch (Exception exception)
{
LOGGER.LogWarning(exception, "Failed to store a passage of the source '{SourceUrl}'. It stays in the chat file.", passage.SourceUrl);
}
if (provenanceId is null)
{
allStored = false;
passages.Add(passage);
continue;
}
passages.Add(passage with { ProvenanceId = provenanceId, Text = string.Empty, SurroundingContent = [] });
}
this.SourcePassages = passages;
return allStored;
}
/// <summary>
/// What the provider said everything sent along with this answer cost, where it said anything.
/// </summary>
@@ -139,6 +264,31 @@ public sealed class ContentText : IContent
};
}
/// <summary>
/// Adds what one chunk of the stream brings to this answer: its text, its sources together with
/// their passages, and what the provider says the request cost.
/// </summary>
/// <remarks>
/// The chat and the job service both write answers from a stream, and they used to do so step
/// by step each. A step one of them missed, such as the passages, would have been missing from
/// every answer written that way.
/// </remarks>
/// <param name="chunk">The chunk of the stream.</param>
/// <param name="modelId">The model the request went to.</param>
/// <param name="blockCount">How many blocks the conversation had when the request went out, this answer included.</param>
public void ApplyStreamChunk(ContentStreamChunk chunk, string modelId, int blockCount)
{
// The first chunk ends the waiting animation:
this.InitialRemoteWait = false;
this.IsStreaming = true;
this.Text += chunk;
if (chunk.Sources.Count > 0 || chunk.Passages.Count > 0)
this.AddSources(chunk.Sources, chunk.Passages);
this.RecordReportedUsage(chunk.Usage, modelId, blockCount);
}
/// <inheritdoc />
public async Task<ChatThread> CreateFromProviderAsync(IProvider provider, Model chatModel, IContent? lastUserPrompt, ChatThread? chatThread, CancellationToken token = default)
{
@@ -241,18 +391,8 @@ public sealed class ContentText : IContent
chunkCount++;
timeToFirstChunk ??= Stopwatch.GetElapsedTime(requestStartedAt);
// Stop the waiting animation:
this.InitialRemoteWait = false;
this.IsStreaming = true;
// Add the response to the text:
this.Text += contentStreamChunk;
// Merge the sources:
this.Sources.MergeSources(contentStreamChunk.Sources);
// Keep what the provider says the request cost:
this.RecordReportedUsage(contentStreamChunk.Usage, chatModel.Id, blocksSent);
// Add the response, its sources, and what it cost:
this.ApplyStreamChunk(contentStreamChunk, chatModel.Id, blocksSent);
// Notify the UI that the content has changed,
// depending on the energy saving mode:
@@ -295,6 +435,9 @@ public sealed class ContentText : IContent
this.Text = this.Text.RemoveThinkTags().Trim();
this.EndToolRun();
// Before whoever is told next stores the chat, so that it refers to what the AI received instead of holding it:
await this.StoreReceivedContentAsync(chatThread.ChatId);
// Inform the UI that the streaming is done:
await this.StreamingDone();
}
@@ -395,6 +538,7 @@ public sealed class ContentText : IContent
InitialRemoteWait = this.InitialRemoteWait,
IsStreaming = this.IsStreaming,
Sources = [..this.Sources],
SourcePassages = [..this.SourcePassages],
FileAttachments = [..this.FileAttachments],
ToolInvocations = [..this.ToolInvocations.Select(x => new ToolInvocationTrace
{
@@ -419,20 +563,204 @@ public sealed class ContentText : IContent
/// WebAddresses.CreateRequestKey.
/// </summary>
/// <remarks>
/// The documents are read from disk only when a message is sent, so their addresses are
/// collected right then, see PrepareTextContentForAI, and count as given to the model, see
/// ChatThread.IsWebAddressGivenToTheModel. Not stored: every request reads the documents anew,
/// and a document changed or gone since no longer gives an address.
/// The texts of the documents are known only when a message is sent, from the provenance store
/// or from the disk, so their addresses are collected right then, see PrepareTextContentForAI,
/// and count as given to the model, see ChatThread.IsWebAddressGivenToTheModel. Not stored:
/// every request collects them anew from what it sends.
/// </remarks>
[JsonIgnore]
public IReadOnlySet<string> RuntimeAttachmentWebAddresses { get; internal set; } = new HashSet<string>(StringComparer.Ordinal);
/// <summary>
/// Whether this message holds attached documents which were never stored, see StoreAttachedDocumentsAsync.
/// </summary>
/// <param name="readChangedAgain">Whether documents whose files changed since they were stored count as well, see DataChat.ReadChangedAttachmentsAgain.</param>
internal bool HasDocumentsToStore(bool readChangedAgain) => this.FileAttachments.Any(attachment => IsToStore(attachment, readChangedAgain));
/// <summary>
/// Whether an attachment is a document to read and store: one which was never stored, or, when
/// the user chose so, one whose file changed since.
/// </summary>
private static bool IsToStore(FileAttachment attachment, bool readChangedAgain) => attachment.Type is FileAttachmentType.DOCUMENT && (attachment.ProvenanceId is null || readChangedAgain && attachment.ChangedSinceStored());
/// <summary>
/// Stores the attached documents of this message which were never sent, reading each one from the disk once.
/// </summary>
/// <remarks>
/// A document which cannot be read or stored right now keeps no entry: the message reads it when
/// it is built, as every message did before, and tells the user then what went wrong. The list of
/// attachments is replaced as a whole rather than changed, since the user interface may read it
/// meanwhile.
/// </remarks>
/// <param name="store">The provenance store.</param>
/// <param name="chatId">The chat of this message.</param>
/// <param name="rustService">The runtime, which reads the documents.</param>
/// <param name="reportingScope">The request the documents are read for, which tells the user once about everything filtered out of them.</param>
/// <param name="readChangedAgain">Whether documents whose files changed since they were stored are read again, see DataChat.ReadChangedAttachmentsAgain.</param>
internal async Task StoreAttachedDocumentsAsync(ProvenanceStoreClient store, Guid chatId, RustService rustService, PromptInjectionReportingScope? reportingScope, bool readChangedAgain)
{
var attachments = new List<FileAttachment>(this.FileAttachments.Count);
var changed = false;
foreach (var attachment in this.FileAttachments)
{
var storedAttachment = await StoreAttachedDocumentAsync(attachment, store, chatId, rustService, reportingScope, readChangedAgain);
changed |= !ReferenceEquals(storedAttachment, attachment);
attachments.Add(storedAttachment);
}
if (changed)
this.FileAttachments = attachments;
}
/// <summary>
/// Stores one attached document which was never sent, or whose file changed since, when the user chose so.
/// </summary>
/// <remarks>
/// A document read again gets an entry of its own: the one before belongs to what the AI
/// received then, and the cleanup at the next start removes it once nothing refers to it.
/// </remarks>
/// <returns>The attachment with the entry of its text, or the attachment as it was when there is nothing to store, or nothing could be.</returns>
private static async Task<FileAttachment> StoreAttachedDocumentAsync(FileAttachment attachment, ProvenanceStoreClient store, Guid chatId, RustService rustService, PromptInjectionReportingScope? reportingScope, bool readChangedAgain)
{
if (!IsToStore(attachment, readChangedAgain))
return attachment;
var document = attachment.Normalize();
if (document.Type is not FileAttachmentType.DOCUMENT || !document.Exists)
return attachment;
try
{
// Quietly: building the message tells the user about a missing Pandoc, when it reads the document after all.
if (FileTypes.RequiresPandoc(document.FilePath))
{
var pandocState = await Pandoc.CheckAvailabilityAsync(rustService, showMessages: false, showSuccessMessage: false);
if (pandocState is not { IsAvailable: true, CheckWasSuccessful: true })
return attachment;
}
// Taken before reading, so that a change while reading counts as a change:
var version = FileVersion.Of(document.FilePath);
var extraction = await rustService.ReadArbitraryFileData(document.FilePath, int.MaxValue, reportingScope: reportingScope);
if (!extraction.HasUsableContent)
return attachment;
// Told here, since no later request reads the document again:
await ReportIncompleteReadingAsync(document, extraction);
// Not cancellable: what was read is what the request sends.
var provenanceId = await store.StoreAsync(chatId, ProvenanceKind.ATTACHMENT, extraction.Content, null, CancellationToken.None);
return provenanceId is null ? attachment : attachment with { ProvenanceId = provenanceId, Version = version };
}
catch (Exception exception)
{
LOGGER.LogWarning(exception, "Failed to store the file attachment '{FilePath}'. It is read from the disk with every request.", document.FilePath);
return attachment;
}
}
/// <summary>
/// Tells the user when a document could be read, but not as it should have been.
/// </summary>
/// <param name="document">The document.</param>
/// <param name="extraction">What reading it gave.</param>
private static async Task ReportIncompleteReadingAsync(FileAttachment document, FileExtractionResult extraction)
{
//
// The file is usable, but we lost parts of it. The user has to know which
// parts are missing, because the answer will be based on the rest.
//
if (extraction.Outcome is FileExtractionOutcome.PARTIAL)
{
LOGGER.LogWarning("Parts of the file attachment '{FilePath}' could not be read: pages={FailedPages}.", document.FilePath, string.Join(", ", extraction.FailedPages));
await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Description, extraction.ToPartialUserMessage(document.FileName)));
}
// The file was read correctly, but its extension lies about what it contains:
if (extraction.HasExtensionMismatch)
{
LOGGER.LogWarning("The file attachment '{FilePath}' is actually a '{DetectedFormat}'.", document.FilePath, extraction.DetectedFormat);
await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.RuleFolder, extraction.ToExtensionMismatchUserMessage(document.FileName)));
}
}
/// <summary>
/// Reads the stored texts of the attached documents.
/// </summary>
/// <param name="attachments">The attachments.</param>
/// <param name="getProvenanceStore">Gets the provenance store, or null when there is none. It is asked only when a document has an entry.</param>
/// <returns>The texts by the ids of their entries. An entry the store lacks, or cannot read, is missing.</returns>
private static async Task<IReadOnlyDictionary<Guid, string>> ReadStoredDocumentTextsAsync(IEnumerable<FileAttachment> attachments, Func<Task<ProvenanceStoreClient?>> getProvenanceStore)
{
var provenanceIds = attachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>().Distinct().ToList();
if (provenanceIds.Count is 0)
return new Dictionary<Guid, string>();
try
{
if (await getProvenanceStore() is not { } store)
return new Dictionary<Guid, string>();
var entries = await store.GetEntriesAsync(provenanceIds, CancellationToken.None);
return entries.ToDictionary(entry => entry.Key, entry => entry.Value.Content);
}
catch (Exception exception)
{
LOGGER.LogWarning(exception, "Failed to read the stored texts of the file attachments. Their files are read instead.");
return new Dictionary<Guid, string>();
}
}
/// <summary>
/// Filters the stored texts of the attached documents for prompt injections once more.
/// </summary>
/// <remarks>
/// The texts were filtered when they were read from their files. Filtering them again with every
/// request applies the rules of today, as when every request read the files anew. All documents of
/// the message go in one request to the runtime, and the user hears about them with the request.
/// </remarks>
/// <param name="storedTexts">The stored texts by the ids of their entries.</param>
/// <param name="attachments">The attachments, which name the files of the texts.</param>
/// <param name="getPromptInjectionGuard">Gets the prompt injection filter.</param>
/// <param name="reportingScope">The request the texts are sent with.</param>
/// <returns>The filtered texts by the ids of their entries.</returns>
private static async Task<IReadOnlyDictionary<Guid, string>> FilterStoredTextsAsync(IReadOnlyDictionary<Guid, string> storedTexts, IReadOnlyList<FileAttachment> attachments, Func<PromptInjectionGuardService> getPromptInjectionGuard, PromptInjectionReportingScope? reportingScope)
{
if (storedTexts.Count is 0)
return storedTexts;
var filePaths = attachments
.Where(attachment => attachment.ProvenanceId is not null)
.GroupBy(attachment => attachment.ProvenanceId!.Value)
.ToDictionary(group => group.Key, group => group.First().FilePath);
var ids = storedTexts.Keys.ToList();
var filteredTexts = await getPromptInjectionGuard().SanitizeAsync(ids.Select(id => new PromptInjectionText(storedTexts[id], PromptInjectionSource.ChatAttachment(filePaths[id]))).ToList(), reportingScope);
return ids.Zip(filteredTexts).ToDictionary(pair => pair.First, pair => pair.Second);
}
/// <summary>
/// Builds the text the AI gets for this message, with the content of its attached documents.
/// </summary>
/// <param name="reportingScope">The request this message is part of, which tells the user once about everything filtered out of its attachments, or null when the request reads no documents.</param>
/// <returns>The text for the AI.</returns>
public async Task<string> PrepareTextContentForAI(PromptInjectionReportingScope? reportingScope)
public Task<string> PrepareTextContentForAI(PromptInjectionReportingScope? reportingScope) => this.PrepareTextContentForAI(reportingScope, Program.RUST_SERVICE, ChatProvenance.GetStoreAsync, () => Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>());
/// <summary>
/// Builds the text the AI gets for this message, with the content of its attached documents.
/// </summary>
/// <remarks>
/// A document stored at its first send comes from the provenance store, also when its file
/// changed or is gone by now, see FileAttachment.ProvenanceId: the AI gets the same text with
/// every request. Only a document without a stored text is read from the disk, as before. A stored
/// text passes the prompt injection filter once more, see FilterStoredTextsAsync.
/// </remarks>
/// <param name="reportingScope">The request this message is part of, which tells the user once about everything filtered out of its attachments, or null when the request reads no documents.</param>
/// <param name="rustService">The runtime, which reads the documents without a stored text.</param>
/// <param name="getProvenanceStore">Gets the provenance store, or null when there is none. It is asked only when a document has an entry.</param>
/// <param name="getPromptInjectionGuard">Gets the prompt injection filter, which is asked only when a document has a stored text.</param>
/// <returns>The text for the AI.</returns>
internal async Task<string> PrepareTextContentForAI(PromptInjectionReportingScope? reportingScope, RustService rustService, Func<Task<ProvenanceStoreClient?>> getProvenanceStore, Func<PromptInjectionGuardService> getPromptInjectionGuard)
{
var sb = new StringBuilder();
var attachmentWebAddresses = new HashSet<string>(StringComparer.Ordinal);
@@ -444,8 +772,10 @@ public sealed class ContentText : IContent
.Select(attachment => attachment.Normalize())
.ToList();
// Get the list of existing documents:
var existingDocuments = normalizedAttachments.Where(x => x.Type is FileAttachmentType.DOCUMENT && x.Exists).ToList();
var storedTexts = await FilterStoredTextsAsync(await ReadStoredDocumentTextsAsync(normalizedAttachments, getProvenanceStore), normalizedAttachments, getPromptInjectionGuard, reportingScope);
// Get the list of existing documents, those with a stored text included, whether their file is still there or not:
var existingDocuments = normalizedAttachments.Where(x => x.Type is FileAttachmentType.DOCUMENT && (StoredTextOf(x) is not null || x.Exists)).ToList();
//
// Report missing files. We tell the user about them instead of only logging: on a
@@ -468,9 +798,9 @@ public sealed class ContentText : IContent
// Pandoc installation must not stop them.
//
var pandocIsUsable = true;
if (existingDocuments.Any(document => FileTypes.RequiresPandoc(document.FilePath)))
if (existingDocuments.Any(document => StoredTextOf(document) is null && FileTypes.RequiresPandoc(document.FilePath)))
{
var pandocState = await Pandoc.CheckAvailabilityAsync(Program.RUST_SERVICE, showMessages: true, showSuccessMessage: false);
var pandocState = await Pandoc.CheckAvailabilityAsync(rustService, showMessages: true, showSuccessMessage: false);
pandocIsUsable = pandocState is { IsAvailable: true, CheckWasSuccessful: true };
if (!pandocState.IsAvailable)
@@ -487,42 +817,32 @@ public sealed class ContentText : IContent
var documentBlocks = new StringBuilder();
foreach(var document in existingDocuments)
{
if (document.IsForbidden)
var content = StoredTextOf(document);
if (content is null)
{
LOGGER.LogWarning("File attachment '{FilePath}' has a forbidden file type and will be skipped.", document.FilePath);
continue;
}
if (document.IsForbidden)
{
LOGGER.LogWarning("File attachment '{FilePath}' has a forbidden file type and will be skipped.", document.FilePath);
continue;
}
if (!pandocIsUsable && FileTypes.RequiresPandoc(document.FilePath))
{
LOGGER.LogWarning("The file attachment '{FilePath}' needs Pandoc and will be skipped.", document.FilePath);
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, FileExtractionErrorCode.PANDOC_UNAVAILABLE.ToUserMessage(document.FileName)));
continue;
}
if (!pandocIsUsable && FileTypes.RequiresPandoc(document.FilePath))
{
LOGGER.LogWarning("The file attachment '{FilePath}' needs Pandoc and will be skipped.", document.FilePath);
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, FileExtractionErrorCode.PANDOC_UNAVAILABLE.ToUserMessage(document.FileName)));
continue;
}
var extraction = await Program.RUST_SERVICE.ReadArbitraryFileData(document.FilePath, int.MaxValue, reportingScope: reportingScope);
if (!extraction.HasUsableContent)
{
LOGGER.LogError("Reading the file attachment '{FilePath}' failed and it will not be sent: code={ErrorCode}, message='{ErrorMessage}'.", document.FilePath, extraction.ErrorCode, extraction.ErrorMessage);
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, extraction.ToUserMessage(document.FileName)));
continue;
}
var extraction = await rustService.ReadArbitraryFileData(document.FilePath, int.MaxValue, reportingScope: reportingScope);
if (!extraction.HasUsableContent)
{
LOGGER.LogError("Reading the file attachment '{FilePath}' failed and it will not be sent: code={ErrorCode}, message='{ErrorMessage}'.", document.FilePath, extraction.ErrorCode, extraction.ErrorMessage);
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Description, extraction.ToUserMessage(document.FileName)));
continue;
}
//
// The file is usable, but we lost parts of it. The user has to know which
// parts are missing, because the answer will be based on the rest.
//
if (extraction.Outcome is FileExtractionOutcome.PARTIAL)
{
LOGGER.LogWarning("Parts of the file attachment '{FilePath}' could not be read: pages={FailedPages}.", document.FilePath, string.Join(", ", extraction.FailedPages));
await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.Description, extraction.ToPartialUserMessage(document.FileName)));
}
// The file was read correctly, but its extension lies about what it contains:
if (extraction.HasExtensionMismatch)
{
LOGGER.LogWarning("The file attachment '{FilePath}' is actually a '{DetectedFormat}'.", document.FilePath, extraction.DetectedFormat);
await MessageBus.INSTANCE.SendWarning(new(Icons.Material.Filled.RuleFolder, extraction.ToExtensionMismatchUserMessage(document.FileName)));
await ReportIncompleteReadingAsync(document, extraction);
content = extraction.Content;
}
documentBlocks.AppendLine();
@@ -530,10 +850,10 @@ public sealed class ContentText : IContent
documentBlocks.AppendLine($"File path: {document.FilePath}");
documentBlocks.AppendLine("File content:");
documentBlocks.AppendLine("````");
documentBlocks.AppendLine(extraction.Content);
documentBlocks.AppendLine(content);
documentBlocks.AppendLine("````");
foreach (var address in WebAddresses.Find(extraction.Content))
foreach (var address in WebAddresses.Find(content))
if (WebAddresses.TryCreateRequestKey(address, out var requestKey))
attachmentWebAddresses.Add(requestKey);
}
@@ -553,6 +873,8 @@ public sealed class ContentText : IContent
sb.AppendLine("Please consider them as part of the message content and use them to answer accordingly.");
}
}
string? StoredTextOf(FileAttachment document) => document.ProvenanceId is { } provenanceId ? storedTexts.GetValueOrDefault(provenanceId) : null;
}
this.RuntimeAttachmentWebAddresses = attachmentWebAddresses;
app/MindWork AI Studio/Chat/FileAttachment.cs
+43 -2
View File
@@ -58,10 +58,51 @@ public record FileAttachment(FileAttachmentType Type, string FileName, [property
[JsonIgnore]
public bool Exists => File.Exists(this.FilePath);
/// <summary>
/// The entry of the provenance store which holds the text this document was first sent with, or
/// null while it was never sent, or could not be stored.
/// </summary>
/// <remarks>
/// A document is read from the disk once, see ChatThread.StoreAttachedDocumentsAsync. Every later
/// request of the chat sends the stored text, also when the file changed or is gone by now, so
/// what the AI received stays the same and can be shown as it was. Left out of the chat file, and
/// of chat templates, while null.
/// </remarks>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public Guid? ProvenanceId { get; init; }
/// <summary>
/// The state of the file which the stored text was read from, see ProvenanceId, or null while
/// there is no stored text, or when it was stored before AI Studio noted the state.
/// </summary>
/// <remarks>
/// Tells whether the file changed since, for a chat which reads changed files again, see
/// DataChat.ReadChangedAttachmentsAgain. Left out of the chat file while null.
/// </remarks>
[JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
public FileVersion? Version { get; init; }
/// <summary>
/// Whether the file changed since its text was stored.
/// </summary>
/// <remarks>
/// Reads the state of the file. A file which is gone did not change: the stored text is all
/// there is of it. Without a noted state, nothing tells, and the stored text stays as well.
/// </remarks>
public bool ChangedSinceStored() => this.ProvenanceId is not null && this.Version is { } version && FileVersion.Of(this.FilePath) is { } current && current != version;
/// <summary>
/// The same attachment without its stored text, e.g., for a place apart from its chat, where the file is read anew.
/// </summary>
public FileAttachment WithoutStoredText() => this with { ProvenanceId = null, Version = null };
/// <summary>
/// Rebuilds the attachment from its current file path so file type detection uses the latest rules.
/// </summary>
public virtual FileAttachment Normalize() => FromPath(this.FilePath);
/// <remarks>
/// The entry of its text stays: it belongs to the attachment, not to the rules.
/// </remarks>
public virtual FileAttachment Normalize() => FromPath(this.FilePath) with { ProvenanceId = this.ProvenanceId, Version = this.Version };
/// <summary>
/// Makes the same attachment for its file at another place, e.g. after the chat holding the file moved.
@@ -72,7 +113,7 @@ public record FileAttachment(FileAttachmentType Type, string FileName, [property
/// </remarks>
/// <param name="filePath">The path of the file now.</param>
/// <returns>An attachment of the same kind, with the new path.</returns>
public virtual FileAttachment WithFilePath(string filePath) => new(this.Type, this.FileName, filePath, this.FileSizeBytes);
public virtual FileAttachment WithFilePath(string filePath) => new(this.Type, this.FileName, filePath, this.FileSizeBytes) { ProvenanceId = this.ProvenanceId, Version = this.Version };
/// <summary>
/// Creates a FileAttachment from a file path by automatically determining the type,
app/MindWork AI Studio/Chat/FileAttachmentImage.cs
+2
View File
@@ -28,5 +28,7 @@ public record FileAttachmentImage(string FileName, string FilePath, long FileSiz
{
SourceType = this.SourceType,
Source = this.SourceType is ContentImageSource.LOCAL_PATH ? filePath : this.Source,
ProvenanceId = this.ProvenanceId,
Version = this.Version,
};
}
app/MindWork AI Studio/Chat/FileVersion.cs
+24
View File
@@ -0,0 +1,24 @@
namespace AIStudio.Chat;
/// <summary>
/// The state of a file which a text was read from, to tell whether the file changed since.
/// </summary>
/// <remarks>
/// The size counts as well as the time: some file systems keep the time coarsely, e.g., FAT to two
/// seconds, and a network drive may report the time of another clock.
/// </remarks>
/// <param name="LastWriteUtc">When the file was last written.</param>
/// <param name="Length">The size of the file in bytes.</param>
public sealed record FileVersion(DateTime LastWriteUtc, long Length)
{
/// <summary>
/// Reads the state of a file as it is now.
/// </summary>
/// <param name="filePath">The path of the file.</param>
/// <returns>The state, or null when the file is gone.</returns>
public static FileVersion? Of(string filePath)
{
var file = new FileInfo(filePath);
return file.Exists ? new(file.LastWriteTimeUtc, file.Length) : null;
}
}
app/MindWork AI Studio/Chat/ManagedTranscriptAttachment.cs
+1 -1
View File
@@ -34,7 +34,7 @@ public sealed record ManagedTranscriptAttachment(string FileName, string FilePat
}
/// <inheritdoc />
public override FileAttachment WithFilePath(string filePath) => new ManagedTranscriptAttachment(this.FileName, filePath, this.FileSizeBytes, this.OriginalFileName, this.IsStaged);
public override FileAttachment WithFilePath(string filePath) => new ManagedTranscriptAttachment(this.FileName, filePath, this.FileSizeBytes, this.OriginalFileName, this.IsStaged) { ProvenanceId = this.ProvenanceId, Version = this.Version };
/// <summary>Creates a transcript in an operation-specific staging directory.</summary>
/// <param name="originalPath">Original media path.</param>
app/MindWork AI Studio/Chat/PreviewedText.cs
+11
View File
@@ -0,0 +1,11 @@
using System.Text.Json.Serialization;
namespace AIStudio.Chat;
/// <summary>
/// The text a preview read of a file attached to the draft of a chat, which sending takes along instead of reading the file again.
/// </summary>
/// <param name="FilePath">The path of the file, as the attachment names it.</param>
/// <param name="ProvenanceId">The entry of the provenance store which holds the text.</param>
/// <param name="Version">The state of the file before the preview read it. A file which changed since is read again.</param>
public sealed record PreviewedText([property: JsonConverter(typeof(DataDirectoryRelativePathConverter))] string FilePath, Guid ProvenanceId, FileVersion Version);
app/MindWork AI Studio/Chat/ProvenanceCleanup.cs
+91
View File
@@ -0,0 +1,91 @@
using AIStudio.Tools.Databases.ProvenanceStore;
namespace AIStudio.Chat;
/// <summary>
/// Removes from the provenance store what no stored chat refers to anymore.
/// </summary>
/// <remarks>
/// Deleting a chat takes its entries along right away, see ChatProvenance, and so does the
/// maintenance of the temporary chats. What is left over comes from changes inside a chat, or from
/// runs which ended before they could clean up:
/// <list type="bullet">
/// <item>a message which was removed, edited, or answered anew, together with its passages and attached documents;</item>
/// <item>a text which a preview stored, while its chat was never stored with it, or sending could not remove it;</item>
/// <item>an assistant whose conversation was never sent to a chat;</item>
/// <item>a chat which was never stored, or deleted while the store was not available.</item>
/// </list>
/// Each chat file is read for its references, which only a cleanup at the start can do safely:
/// while the app runs, a chat in memory may refer to an entry its file does not know yet. For the
/// same reason, only entries stored before the start are removed.<br/><br/>
///
/// Reading every chat at every start would cost much on a small computer, so a chat is read only
/// when something changed since the last complete cleanup: its file was written since, or it
/// received an entry since. Otherwise, the last cleanup removed all it could, and nothing it kept
/// has become unreferenced. A chat which is not read keeps its entries; skipping never removes
/// anything. Chats which no longer exist cost no reading and are always taken care of.
/// </remarks>
public static class ProvenanceCleanup
{
/// <summary>
/// How long before the last cleanup a chat file may have been written and still counts as changed.
/// </summary>
/// <remarks>
/// The time of a file may be coarse, e.g., two seconds on FAT, or come from the clock of another
/// computer, for a data folder on a network drive. Reading a few chats too many costs little,
/// while missing one leaves what it no longer refers to until it changes again.
/// </remarks>
private static readonly TimeSpan WRITE_TIME_TOLERANCE = TimeSpan.FromMinutes(5);
/// <summary>
/// Removes the entries of chats which no longer exist, and those their chat no longer refers to.
/// </summary>
/// <param name="store">The provenance store.</param>
/// <param name="chatLocations">The workspace of every chat on the disk by its ID, Guid.Empty for a temporary chat.</param>
/// <param name="loadChat">Reads a chat by its workspace and ID, or answers null when it cannot be read.</param>
/// <param name="getChatWrittenAt">Tells when a chat file was last written, by its workspace and ID, or answers null when that is not known.</param>
/// <param name="createdBefore">The start of this run: entries stored since may belong to a chat which is not on the disk yet.</param>
/// <param name="lastCleanup">When the last complete cleanup began, or null to read every chat.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>What the cleanup did.</returns>
public static async Task<ProvenanceCleanupResult> CleanUpAsync(ProvenanceStoreClient store, IReadOnlyDictionary<Guid, Guid> chatLocations, Func<Guid, Guid, Task<ChatThread?>> loadChat, Func<Guid, Guid, DateTimeOffset?> getChatWrittenAt, DateTimeOffset createdBefore, DateTimeOffset? lastCleanup, CancellationToken token)
{
// The chats which received an entry since the last cleanup, in one query, which the last cleanup had to leave alone:
var chatsWithNewEntries = lastCleanup is { } since ? (await store.GetChatIdsAsync(token, since)).ToHashSet() : [];
var unreferenced = 0;
var skipped = 0;
var unreadable = 0;
var orphanedChats = new List<Guid>();
foreach (var chatId in await store.GetChatIdsAsync(token))
{
token.ThrowIfCancellationRequested();
if (!chatLocations.TryGetValue(chatId, out var workspaceId))
{
orphanedChats.Add(chatId);
continue;
}
// A chat whose time of writing is not known counts as changed:
if (lastCleanup is { } previousStart && !chatsWithNewEntries.Contains(chatId) && getChatWrittenAt(workspaceId, chatId) is { } writtenAt && writtenAt < previousStart - WRITE_TIME_TOLERANCE)
{
skipped++;
continue;
}
// A chat which cannot be read keeps everything: nothing tells what it refers to.
var chat = await loadChat(workspaceId, chatId);
if (chat is null)
{
unreadable++;
continue;
}
unreferenced += await store.DeleteChatEntriesExceptAsync(chatId, chat.CollectProvenanceIds(), token, createdBefore);
}
// The chats which no longer exist all at once, since there may be many of them after a long time without a start:
var orphaned = await store.DeleteChatsAsync(orphanedChats, token, createdBefore);
return new(orphaned, unreferenced, skipped, unreadable);
}
}
app/MindWork AI Studio/Chat/ProvenanceCleanupResult.cs
+10
View File
@@ -0,0 +1,10 @@
namespace AIStudio.Chat;
/// <summary>
/// What a cleanup of the provenance store did, see ProvenanceCleanup.
/// </summary>
/// <param name="Orphaned">How many entries of chats which no longer exist were removed.</param>
/// <param name="Unreferenced">How many entries their chats no longer refer to were removed.</param>
/// <param name="Skipped">How many chats were left unread, since nothing changed in them since the last cleanup.</param>
/// <param name="Unreadable">How many chats could not be read, and thus kept all their entries.</param>
public sealed record ProvenanceCleanupResult(int Orphaned, int Unreferenced, int Skipped, int Unreadable);
app/MindWork AI Studio/Chat/SetFileAttachmentExtensions.cs
+21
View File
@@ -0,0 +1,21 @@
namespace AIStudio.Chat;
public static class SetFileAttachmentExtensions
{
/// <summary>
/// Attaches a file in place of the attachment of the same file the set held before.
/// </summary>
/// <remarks>
/// The attachments differ by more than their files: one of a message being edited holds the
/// entry of the text it was sent with, and the size of the file may have changed since. Both
/// would stay in the set, and the message would go with the old and the new text of the file.
/// Attaching a file again means the file as it is now.
/// </remarks>
/// <param name="attachments">The attachments of a message.</param>
/// <param name="attachment">The attachment of the file just attached.</param>
public static void AttachInPlaceOfSameFile(this HashSet<FileAttachment> attachments, FileAttachment attachment)
{
attachments.RemoveWhere(existing => string.Equals(existing.FilePath, attachment.FilePath, StringComparison.Ordinal));
attachments.Add(attachment);
}
}
app/MindWork AI Studio/Components/AttachDocuments.razor.cs
+13 -2
View File
@@ -81,6 +81,16 @@ public partial class AttachDocuments : MSGComponentBase
[Parameter]
public Func<string, Task<ChatThread?>> EnsureOwnerChatAsync { get; set; } = _ => Task.FromResult<ChatThread?>(null);
/// <summary>
/// Keeps what the preview read of an attached document, so that sending it does not read the file again.
/// </summary>
/// <remarks>
/// Only the chat hands one over, cf. DocumentCheckDialog.KeepReadText. Elsewhere, e.g., in the
/// assistants, the documents are read for every run anyway.
/// </remarks>
[Parameter]
public Func<FileAttachment, string, FileVersion, Task>? KeepPreviewedText { get; set; }
[Inject]
private ILogger<AttachDocuments> Logger { get; set; } = null!;
@@ -263,7 +273,7 @@ public partial class AttachDocuments : MSGComponentBase
return;
var previousAttachments = this.DocumentPaths.ToHashSet();
this.DocumentPaths = await ReviewAttachmentsDialog.OpenDialogAsync(this.DialogService, this.DocumentPaths, this.AttachDroppedPathsAsync, () => this.IsUnavailable);
this.DocumentPaths = await ReviewAttachmentsDialog.OpenDialogAsync(this.DialogService, this.DocumentPaths, this.AttachDroppedPathsAsync, () => this.IsUnavailable, this.KeepPreviewedText);
var removedAttachments = previousAttachments.Except(this.DocumentPaths).ToList();
foreach (var removedAttachment in removedAttachments)
ManagedTranscriptAttachment.TryDeleteOwnedFile(removedAttachment);
@@ -383,7 +393,7 @@ public partial class AttachDocuments : MSGComponentBase
// about whether it happened to be new to us.
//
var attachment = FileAttachment.FromPath(path);
this.DocumentPaths.Add(attachment);
this.DocumentPaths.AttachInPlaceOfSameFile(attachment);
attached.Add(attachment);
}
@@ -452,6 +462,7 @@ public partial class AttachDocuments : MSGComponentBase
{ x => x.Document, fileAttachment },
{ x => x.AttachPaths, this.AttachDroppedPathsAsync },
{ x => x.IsAttachingUnavailable, () => this.IsUnavailable },
{ x => x.KeepReadText, this.KeepPreviewedText },
};
await this.DialogService.ShowAsync<DocumentCheckDialog>(T("Document Preview"), dialogParameters, DialogOptions.FULLSCREEN);
app/MindWork AI Studio/Components/ChatComponent.razor
+1 -1
View File
@@ -109,7 +109,7 @@
</MudTooltip>
}
<AttachDocuments Name="File Attachments" DocumentPaths="@this.ComposerState.FileAttachments" DocumentPathsChanged="@this.ComposerAttachmentsChanged" CatchAllDocuments="true" UseSmallForm="true" ShowMediaStatus="false" Provider="@this.Provider" OwnerChat="@this.ChatThread" EnsureOwnerChatAsync="@this.EnsureMediaImportChatAsync" Disabled="@(!this.CanAttachFiles)"/>
<AttachDocuments Name="File Attachments" DocumentPaths="@this.ComposerState.FileAttachments" DocumentPathsChanged="@this.ComposerAttachmentsChanged" CatchAllDocuments="true" UseSmallForm="true" ShowMediaStatus="false" Provider="@this.Provider" OwnerChat="@this.ChatThread" EnsureOwnerChatAsync="@this.EnsureMediaImportChatAsync" KeepPreviewedText="@this.KeepPreviewedTextAsync" Disabled="@(!this.CanAttachFiles)"/>
<MudDivider Vertical="true" Style="height: 24px; align-self: center;"/>
app/MindWork AI Studio/Components/ChatComponent.razor.cs
+128 -4
View File
@@ -5,6 +5,8 @@ using AIStudio.Dialogs;
using AIStudio.Provider;
using AIStudio.Settings;
using AIStudio.Settings.DataModel;
using AIStudio.Tools.Databases;
using AIStudio.Tools.Databases.ProvenanceStore;
using AIStudio.Tools.ToolCallingSystem;
using AIStudio.Tools.AIJobs;
using AIStudio.Tools.Media;
@@ -84,6 +86,9 @@ public partial class ChatComponent : MSGComponentBase
[Inject]
private NavigationManager NavigationManager { get; init; } = null!;
[Inject]
private DatabaseClientProvider DatabaseClientProvider { get; init; } = null!;
private const Placement TOOLBAR_TOOLTIP_PLACEMENT = Placement.Top;
private static readonly Dictionary<string, object?> USER_INPUT_ATTRIBUTES = new();
@@ -1273,7 +1278,7 @@ public partial class ChatComponent : MSGComponentBase
this.hasUnsavedChanges = true;
}
private void ComposerAttachmentsChanged(HashSet<FileAttachment> attachments)
private async Task ComposerAttachmentsChanged(HashSet<FileAttachment> attachments)
{
if (!ReferenceEquals(this.ComposerState.FileAttachments, attachments))
this.ComposerState.ReplaceFileAttachments(attachments);
@@ -1281,6 +1286,10 @@ public partial class ChatComponent : MSGComponentBase
this.ComposerState.MarkUserDraft();
this.KeepComposerInDraft();
this.hasUnsavedChanges = true;
// A file the user took out goes without the text its preview read:
if (this.ChatThread is { } chat)
await this.ForgetPreviewedTextsAsync(chat.ChatId, chat.Draft.ForgetPreviewsOfDetachedFiles());
}
/// <summary>
@@ -1428,7 +1437,117 @@ public partial class ChatComponent : MSGComponentBase
}
/// <summary>Creates and stores a stable draft immediately after media import confirmation.</summary>
private async Task<ChatThread?> EnsureMediaImportChatAsync(string firstMediaPath)
private Task<ChatThread?> EnsureMediaImportChatAsync(string firstMediaPath) => this.EnsureChatAsync(string.Format(T("Transcription: {0}"), Path.GetFileName(firstMediaPath)));
/// <summary>
/// Keeps the text the preview read of an attached document, so that sending the message does not read the file again.
/// </summary>
/// <remarks>
/// The text belongs to a chat, and its draft notes it, see ChatDraft.PreviewedTexts. Only a chat
/// which exists keeps it: looking at a file must not create a chat, which the user may never have
/// meant to keep. In a new chat, sending reads the file, as it did before. Another preview of a
/// file which did not change keeps nothing new; one of a file which changed replaces the text kept
/// before. Nothing here may stop the preview: should the text not be kept, sending reads the file
/// as before.
/// </remarks>
/// <param name="attachment">The attachment, as the composer holds it.</param>
/// <param name="text">The text the preview read, after the prompt injection filter.</param>
/// <param name="version">The state of the file before the preview read it.</param>
private async Task KeepPreviewedTextAsync(FileAttachment attachment, string text, FileVersion version)
{
try
{
if (this.ChatThread is not { } chat)
return;
if (chat.Draft.GetPreviewedText(attachment.FilePath)?.Version == version)
return;
var store = await this.DatabaseClientProvider.GetProvenanceStoreAsync();
var provenanceId = await store.StoreAsync(chat.ChatId, ProvenanceKind.ATTACHMENT, text, null, CancellationToken.None);
if (provenanceId is null)
return;
// Stored with the chat like a change in the composer, so the text still counts after leaving the chat:
var replacedId = chat.Draft.NotePreviewedText(new(attachment.FilePath, provenanceId.Value, version));
this.KeepComposerInDraft();
if (replacedId is { } id)
await this.ForgetPreviewedTextsAsync(chat.ChatId, [id]);
}
catch (Exception exception)
{
this.Logger.LogWarning(exception, "Failed to keep the previewed text of the attachment '{FilePath}'. Sending reads the file instead.", attachment.FilePath);
}
}
/// <summary>
/// Gives an attachment of the message being sent the text its preview kept, as long as its file did not change since.
/// </summary>
/// <remarks>
/// A file which changed, or which is gone, is read when the message is sent, as it would be
/// without a preview. An attachment of a message sent before has its text already.
/// </remarks>
/// <param name="attachment">The attachment, normalized.</param>
/// <param name="draft">The draft being sent, which notes the texts its previews read.</param>
/// <returns>The attachment with the entry of its text, or the attachment as it was.</returns>
private static FileAttachment WithPreviewedText(FileAttachment attachment, ChatDraft draft)
{
if (attachment.ProvenanceId is not null || draft.GetPreviewedText(attachment.FilePath) is not { } previewed)
return attachment;
return FileVersion.Of(attachment.FilePath) == previewed.Version
? attachment with { ProvenanceId = previewed.ProvenanceId, Version = previewed.Version }
: attachment;
}
/// <summary>
/// Removes the texts the previews read for a message which was just sent from the provenance store, when the message went without them.
/// </summary>
/// <remarks>
/// The message went without the text of an attachment which the user removed, and without that
/// of a file which changed since its preview or is gone.
/// </remarks>
/// <param name="chatId">The chat the message went to.</param>
/// <param name="previewedTexts">The texts the draft of the message noted.</param>
/// <param name="sentAttachments">The attachments of the message.</param>
private Task ForgetUnsentPreviewsAsync(Guid chatId, IEnumerable<PreviewedText> previewedTexts, IEnumerable<FileAttachment> sentAttachments)
{
var sentIds = sentAttachments.Select(attachment => attachment.ProvenanceId).OfType<Guid>().ToHashSet();
return this.ForgetPreviewedTextsAsync(chatId, previewedTexts.Select(text => text.ProvenanceId).Where(id => !sentIds.Contains(id)).ToList());
}
/// <summary>
/// Removes texts the preview kept from the provenance store, since no message will be sent with them.
/// </summary>
/// <remarks>
/// Should that fail, the next start removes them, since the chat does not refer to them.
/// </remarks>
/// <param name="chatId">The chat the texts were kept for.</param>
/// <param name="provenanceIds">The entries of the texts.</param>
private async Task ForgetPreviewedTextsAsync(Guid chatId, IReadOnlyCollection<Guid> provenanceIds)
{
if (provenanceIds.Count is 0)
return;
try
{
var store = await this.DatabaseClientProvider.GetProvenanceStoreAsync();
await store.DeleteEntriesAsync(chatId, provenanceIds, CancellationToken.None);
}
catch (Exception exception)
{
this.Logger.LogWarning(exception, "Failed to remove {Count} previewed texts which no message of the chat '{ChatId}' was sent with. The next start removes them.", provenanceIds.Count, chatId);
}
}
/// <summary>
/// Creates and stores the chat the composer belongs to, when there is none yet, e.g., for what an
/// attachment brings along before the first message is sent.
/// </summary>
/// <param name="nameWithoutInput">The name of the chat while the user typed nothing yet.</param>
/// <returns>The chat of the composer.</returns>
private async Task<ChatThread?> EnsureChatAsync(string nameWithoutInput)
{
if (this.ChatThread is not null)
return this.ChatThread;
@@ -1436,7 +1555,7 @@ public partial class ChatComponent : MSGComponentBase
this.RefreshCurrentProfileAndChatTemplate();
var promptName = this.ExtractThreadName(this.ComposerState.UserInput);
var threadName = string.IsNullOrWhiteSpace(this.ComposerState.UserInput)
? string.Format(T("Transcription: {0}"), Path.GetFileName(firstMediaPath))
? nameWithoutInput
: promptName;
var chat = this.NewChatThread(threadName, await this.ResolveWorkspaceOfFirstThreadAsync());
@@ -1516,9 +1635,11 @@ public partial class ChatComponent : MSGComponentBase
IContent? lastUserPrompt;
if (!reuseLastUserPrompt)
{
var draft = this.ChatThread.Draft;
var normalizedAttachments = this.ComposerState.FileAttachments
.Select(attachment => attachment.Normalize())
.Where(attachment => attachment.IsValid)
.Select(attachment => WithPreviewedText(attachment, draft))
.ToList();
lastUserPrompt = new ContentText
@@ -1526,7 +1647,10 @@ public partial class ChatComponent : MSGComponentBase
Text = this.ComposerState.UserInput,
FileAttachments = normalizedAttachments,
};
this.ChatThread.Draft.Clear();
var previewedTexts = draft.PreviewedTexts;
draft.Clear();
await this.ForgetUnsentPreviewsAsync(this.ChatThread.ChatId, previewedTexts, normalizedAttachments);
//
// Add the user message to the thread:
app/MindWork AI Studio/Components/ContentViewer.razor
+55
View File
@@ -0,0 +1,55 @@
@inherits MSGComponentBase
@* One element around everything, so that a layout around the viewer can give it its place. *@
<div class="content-viewer">
@if (this.cutOffCharacters > 0 && this.CutOffNotice is not null)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="my-2">
@this.CutOffNotice(this.cutOffCharacters)
</MudAlert>
}
<MudPaper Outlined="@true" Class="content-viewer-frame">
<div class="content-viewer-toolbar">
<div class="content-viewer-toolbar-start">
@this.Toolbar
</div>
@if (this.views.Count > 1)
{
<MudToggleGroup T="ContentView" @bind-Value="@this.view" SelectionMode="SelectionMode.SingleSelection" Size="Size.Small" Color="Color.Primary">
@foreach (var offeredView in this.views)
{
<MudToggleItem Value="@offeredView" Text="@this.GetViewName(offeredView)"/>
}
</MudToggleGroup>
}
</div>
@* MathJax leaves this alone, even when something else on the page has it typeset everything:
it would replace elements Blazor renders here, and the next render of them would fail. *@
<div class="content-viewer-body mathjax_ignore" style="@($"max-height: {this.MaxHeight};")">
@switch (this.view)
{
case ContentView.MEDIA:
<MudImage ObjectFit="ObjectFit.ScaleDown" Style="width: 100%;" Src="@this.MediaUrl"/>
break;
case ContentView.FORMATTED:
<MudMarkdown Value="@this.shownText" Props="Markdown.DefaultConfig" Styling="@this.MarkdownStyling" MarkdownPipeline="Markdown.CONTENT_MARKDOWN_PIPELINE"/>
break;
default:
<pre class="content-viewer-plain">@this.shownText</pre>
break;
}
</div>
</MudPaper>
@if (!string.IsNullOrWhiteSpace(this.HelperText))
{
<MudText Typo="Typo.caption" Class="mt-1 px-1 mud-text-secondary">
@this.HelperText
</MudText>
}
</div>
app/MindWork AI Studio/Components/ContentViewer.razor.cs
+185
View File
@@ -0,0 +1,185 @@
using Microsoft.AspNetCore.Components;
namespace AIStudio.Components;
/// <summary>
/// Shows a piece of content for the user to check: text formatted and as plain text, and an image
/// as itself.
/// </summary>
/// <remarks>
/// Every place which lets the user see what AI Studio reads or hands to the AI shows it the same
/// way, which is why this lives here instead of in each of them. A slim bar above the content says
/// what it is, and switches between the ways to see it; the caller fills its start.<br/><br/>
/// The content scrolls inside its frame, so a dialog around it does not have to scroll as well. In a
/// dialog which fits the window, the frame gives way down to the room left over.<br/><br/>
/// The text is shown in full unless the caller sets a limit. Whether the beginning of a text is
/// enough depends on what the user wants to check: how a whole document was read, or exactly what
/// the AI received. Only the caller knows which, and how to explain a cut.
/// </remarks>
public partial class ContentViewer : MSGComponentBase
{
/// <summary>
/// The ways to see the content.
/// </summary>
/// <remarks>
/// Internal rather than private: Razor infers the type of the toggle items in a class of its own.
/// </remarks>
internal enum ContentView
{
MEDIA,
FORMATTED,
PLAIN,
}
/// <summary>
/// What kind of content this is.
/// </summary>
[Parameter]
public ContentViewerKind Kind { get; set; } = ContentViewerKind.TEXT;
/// <summary>
/// The text to show. For an image, the text which belongs to it, or nothing.
/// </summary>
[Parameter]
public string Text { get; set; } = string.Empty;
/// <summary>
/// The address an image is loaded from. Not used for text.
/// </summary>
[Parameter]
public string MediaUrl { get; set; } = string.Empty;
/// <summary>
/// What the bar above the content shows at its start, e.g., what the content is, where it comes
/// from, and a button to open it there.
/// </summary>
[Parameter]
public RenderFragment? Toolbar { get; set; }
/// <summary>
/// The hint below the text, or nothing.
/// </summary>
[Parameter]
public string HelperText { get; set; } = string.Empty;
/// <summary>
/// How many characters of the text we show at most, or null to show all of it.
/// </summary>
[Parameter]
public int? CharacterLimit { get; set; }
/// <summary>
/// Explains that the text was cut, given the number of characters left out. Null shows no explanation.
/// </summary>
[Parameter]
public Func<int, string>? CutOffNotice { get; set; }
/// <summary>
/// The height which does not limit the content: it may take all the room its place leaves over,
/// e.g., in a dialog which fits the window, see dialog-content-fit in app.css.
/// </summary>
public const string NO_HEIGHT_LIMIT = "none";
/// <summary>
/// How high the content may grow before it scrolls, unless its place says otherwise.
/// </summary>
public const string DEFAULT_MAX_HEIGHT = "40vh";
/// <summary>
/// How high the content may grow before it scrolls, as a CSS length.
/// </summary>
[Parameter]
public string MaxHeight { get; set; } = DEFAULT_MAX_HEIGHT;
/// <summary>
/// The ways to see this content, in the order they are offered; the first one is shown first.
/// </summary>
private readonly List<ContentView> views = [];
/// <summary>
/// The way the content is shown right now.
/// </summary>
private ContentView view;
/// <summary>
/// The text and the limit the current cut was made of, so that we cut once per text instead of once per render.
/// </summary>
private string? cutText;
private int? cutLimit;
/// <summary>
/// What we show of the text: all of it, or its beginning.
/// </summary>
private string shownText = string.Empty;
/// <summary>
/// How many characters we left out. Zero when we show the entire text.
/// </summary>
private int cutOffCharacters;
#region Overrides of ComponentBase
protected override void OnParametersSet()
{
if (!ReferenceEquals(this.cutText, this.Text) || this.cutLimit != this.CharacterLimit)
{
this.cutText = this.Text;
this.cutLimit = this.CharacterLimit;
this.shownText = CutAtLineBreak(this.Text, this.CharacterLimit ?? int.MaxValue, out this.cutOffCharacters);
}
//
// An image is seen as itself first. Text is seen formatted and as plain text, for text and
// for an image which brings some along. The way the user chose stays as long as it is offered:
//
this.views.Clear();
if (this.Kind is not ContentViewerKind.TEXT)
this.views.Add(ContentView.MEDIA);
if (this.Kind is ContentViewerKind.TEXT || !string.IsNullOrWhiteSpace(this.Text))
this.views.AddRange([ContentView.FORMATTED, ContentView.PLAIN]);
if (!this.views.Contains(this.view))
this.view = this.views[0];
base.OnParametersSet();
}
#endregion
private string GetViewName(ContentView contentView) => contentView switch
{
ContentView.MEDIA => T("Image"),
ContentView.FORMATTED => T("Formatted"),
_ => T("Plain text"),
};
/// <summary>
/// Takes the beginning of a text which is too long to show.
/// </summary>
/// <remarks>
/// We cut at the last line break before the limit. Otherwise, we might tear apart a Markdown
/// construct like a table row or a code fence in the middle of a line. A text without any line
/// break before the limit is cut at the limit itself.
/// </remarks>
/// <param name="text">The text.</param>
/// <param name="characterLimit">How many characters to keep at most.</param>
/// <param name="cutOffCharacters">How many characters were left out; zero when nothing was.</param>
/// <returns>The text itself when it fits, otherwise its beginning.</returns>
internal static string CutAtLineBreak(string text, int characterLimit, out int cutOffCharacters)
{
if (text.Length <= characterLimit)
{
cutOffCharacters = 0;
return text;
}
var cutIndex = text.LastIndexOf('\n', characterLimit - 1) + 1;
if (cutIndex < 1)
cutIndex = characterLimit;
cutOffCharacters = text.Length - cutIndex;
return text[..cutIndex];
}
}
app/MindWork AI Studio/Components/ContentViewerKind.cs
+17
View File
@@ -0,0 +1,17 @@
namespace AIStudio.Components;
/// <summary>
/// What kind of content a ContentViewer shows.
/// </summary>
public enum ContentViewerKind
{
/// <summary>
/// Text, shown as rendered Markdown and as plain text.
/// </summary>
TEXT,
/// <summary>
/// An image, shown from its address.
/// </summary>
IMAGE,
}
app/MindWork AI Studio/Components/DataBackupPanel.razor
+1 -1
View File
@@ -37,7 +37,7 @@
<MudText Typo="Typo.body2">
@string.Format(T("Backed up {0} files into: {1}"), backupResult.FileCount, backupResult.ArchivePath)
</MudText>
<FileExportActions FilePath="@backupResult.ArchivePath" OffersOpening="@false"/>
<FileActions Path="@backupResult.ArchivePath" ShowCopy="@false" ShowOpen="@false"/>
</MudStack>
@if (!backupResult.IncludesDataSourceIndexes)
{
app/MindWork AI Studio/Components/DataBackupPanel.razor.cs
+6 -1
View File
@@ -2,6 +2,7 @@ using System.Globalization;
using AIStudio.Dialogs;
using AIStudio.Tools.ChatArchive;
using AIStudio.Tools.Databases;
using AIStudio.Tools.DataBackup;
using AIStudio.Tools.Rust;
using AIStudio.Tools.Services;
@@ -43,6 +44,9 @@ public partial class DataBackupPanel : MSGComponentBase
[Inject]
private DataRestoreService DataRestoreService { get; init; } = null!;
[Inject]
private DatabaseClientProvider DatabaseClientProvider { get; init; } = null!;
private ChatArchiveCollisionBehavior collisionBehavior = ChatArchiveCollisionBehavior.SKIP;
private CancellationTokenSource? importCancellation;
private ChatArchiveProgress importProgress;
@@ -253,7 +257,8 @@ public partial class DataBackupPanel : MSGComponentBase
var archivePath = this.selectedArchivePath;
var token = this.importCancellation.Token;
var reporter = this.CreateImportProgressReporter();
this.importResult = await Task.Run(() => ChatArchiveImporter.ImportAsync(archivePath, this.collisionBehavior, reporter, token), token);
var provenanceStore = await this.DatabaseClientProvider.GetProvenanceStoreAsync(token);
this.importResult = await Task.Run(() => ChatArchiveImporter.ImportAsync(archivePath, provenanceStore, this.collisionBehavior, reporter, token), token);
//
// Drop the preview once every chat was added. Otherwise, another click on the button
app/MindWork AI Studio/Components/DataFolders.razor.cs
+1
View File
@@ -78,6 +78,7 @@ public partial class DataFolders : MSGComponentBase
DataBackupEntryKind.SETTINGS => T("Your settings."),
DataBackupEntryKind.WORKSPACES => T("Your chats in workspaces, along with their transcripts."),
DataBackupEntryKind.TEMPORARY_CHATS => T("Your temporary chats, along with their transcripts."),
DataBackupEntryKind.CHAT_PROVENANCE => T("What the AI received in your chats besides your messages, such as the content of attached files, the results of tools, and passages of your data sources. It is part of your chats."),
DataBackupEntryKind.PLUGINS => T("Your plugins, such as the assistants you built, apart from the folders listed below."),
DataBackupEntryKind.TOKENIZERS => T("The tokenizers you chose for your providers."),
DataBackupEntryKind.VISUAL_BRIEFINGS => T("Your projects in the Visual Briefing Assistant."),
app/MindWork AI Studio/Components/FileActions.razor
+62
View File
@@ -0,0 +1,62 @@
@inherits MSGComponentBase
@*
The tooltips of the buttons which open something show on hover only: the focus comes back to the
button from the program, the browser, or the file manager it opened, and a tooltip shown on focus
would then stay until the user clicks somewhere else.
*@
@switch (this.kind)
{
case FileActionsKind.WEB_PAGE:
@if (this.ShowCopy)
{
<MudTooltip Text="@string.Format(T("Copy the address: {0}"), this.Path)" Placement="@this.TooltipPlacement">
<MudIconButton Icon="@Icons.Material.Outlined.ContentCopy" Size="Size.Small" Class="icon-button-compact" OnClick="@this.CopyPath"/>
</MudTooltip>
}
@if (this.ShowOpen)
{
<MudTooltip Text="@T("Open this web page in your browser")" Placement="@this.TooltipPlacement" ShowOnFocus="@false">
<MudIconButton Icon="@Icons.Material.Outlined.OpenInNew" Size="Size.Small" Class="icon-button-compact" Href="@this.Path" Target="_blank"/>
</MudTooltip>
}
break;
case FileActionsKind.FILE:
@if (this.ShowCopy)
{
<MudTooltip Text="@string.Format(T("Copy the path: {0}"), this.Path)" Placement="@this.TooltipPlacement">
<MudIconButton Icon="@Icons.Material.Outlined.ContentCopy" Size="Size.Small" Class="icon-button-compact" OnClick="@this.CopyPath"/>
</MudTooltip>
}
@if (this.isMissing)
{
@* Instead of buttons which would only fail, and only where there would have been one: *@
@if (this.WarnIfMissing && (this.ShowFileManager || this.ShowOpen))
{
@* Not necessarily gone: the server of an ERI data source may name a file of its own, which was never on this computer. *@
<MudTooltip Text="@string.Format(T("AI Studio cannot find the file {0} on this computer. It may have been moved, renamed, or deleted, or it is on another computer, such as the server of a data source."), this.Path)" Placement="@this.TooltipPlacement">
<MudIcon Icon="@Icons.Material.Outlined.WarningAmber" Color="Color.Warning" Size="Size.Small" Class="file-action-warning"/>
</MudTooltip>
}
}
else
{
@if (this.ShowFileManager)
{
<MudTooltip Text="@T("Show this file in the file manager of your system")" Placement="@this.TooltipPlacement" ShowOnFocus="@false">
<MudIconButton Icon="@Icons.Material.Outlined.FolderOpen" Size="Size.Small" Class="icon-button-compact" OnClick="@this.ShowInFileManager"/>
</MudTooltip>
}
@if (this.ShowOpen)
{
<MudTooltip Text="@GetOpenTooltip(this.PageNumber)" Placement="@this.TooltipPlacement" ShowOnFocus="@false">
<MudIconButton Icon="@Icons.Material.Outlined.OpenInNew" Size="Size.Small" Class="icon-button-compact" OnClick="@this.Open"/>
</MudTooltip>
}
}
break;
}
app/MindWork AI Studio/Components/FileActions.razor.cs
+155
View File
@@ -0,0 +1,155 @@
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Components;
/// <summary>
/// What the user can do with a file or a web page which a source, a passage, or an attachment
/// names: copy its path, find it in the file manager, open it, at a page where there is one.
/// </summary>
/// <remarks>
/// Every place which names a file offers the same buttons the same way, which is why they live
/// here. Each place decides which of them it shows, e.g., a title copies and finds, while the bar
/// above a text opens.<br/><br/>
///
/// A file which is no longer at its place gets a warning instead of the buttons which would only
/// fail; copying its path stays, since it tells where the file was. Whether it is there is asked
/// off the UI thread and once per path: on an unreachable network share, the answer takes until
/// the timeout of the share, and a list of sources renders anew with every streamed chunk.<br/><br/>
///
/// A web page is copied and opened in the browser. Anything else, such as a mail, which only
/// AI Studio can find by its address, or a relative path, gets no buttons at all.<br/><br/>
///
/// Every place which opens a file names it the same way, see GetOpenTooltip, also where a title
/// opens it instead of a button.
/// </remarks>
public partial class FileActions : MSGComponentBase
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(FileActions).Namespace, nameof(FileActions));
/// <summary>
/// What a path names.
/// </summary>
internal enum FileActionsKind
{
NONE,
FILE,
WEB_PAGE,
}
/// <summary>
/// The path of the file, or the address of the web page.
/// </summary>
[Parameter]
public string Path { get; set; } = string.Empty;
/// <summary>
/// The page to open a file at, or null to open it at its beginning.
/// </summary>
[Parameter]
public int? PageNumber { get; set; }
[Parameter]
public bool ShowCopy { get; set; } = true;
[Parameter]
public bool ShowFileManager { get; set; } = true;
[Parameter]
public bool ShowOpen { get; set; } = true;
/// <summary>
/// Whether a file which is no longer at its place gets a warning. False where the caller says so
/// itself, or where another place on the screen warns already.
/// </summary>
[Parameter]
public bool WarnIfMissing { get; set; } = true;
[Parameter]
public Placement TooltipPlacement { get; set; } = Placement.Top;
/// <summary>
/// Tells the caller whether the file is at its place, once that is known, e.g., so that a title
/// which opens it stops being a link.
/// </summary>
[Parameter]
public EventCallback<bool> OnExistenceChecked { get; set; }
[Inject]
private RustService RustService { get; init; } = null!;
private FileActionsKind kind;
/// <summary>
/// The path whose existence was asked last, so that a render with the same path does not ask again.
/// </summary>
private string? checkedPath;
/// <summary>
/// Whether the file is known to be gone. Until the answer arrives, the buttons show: most files are there.
/// </summary>
private bool isMissing;
/// <summary>
/// The tooltip of opening a file, at its page where there is one.
/// </summary>
/// <param name="pageNumber">The page the file opens at, or null when it opens at its beginning.</param>
/// <returns>The tooltip.</returns>
internal static string GetOpenTooltip(int? pageNumber) => pageNumber is { } page
? string.Format(TB("Open the file at page {0}"), page)
: TB("Open the file in the program your system uses for it");
#region Overrides of ComponentBase
protected override async Task OnParametersSetAsync()
{
this.kind = DetermineKind(this.Path);
if (this.kind is FileActionsKind.FILE && !string.Equals(this.checkedPath, this.Path, StringComparison.Ordinal))
{
var path = this.Path;
this.checkedPath = path;
this.isMissing = false;
var exists = await Task.Run(() => File.Exists(path));
// Another path took over meanwhile, which asks for itself:
if (!string.Equals(this.checkedPath, path, StringComparison.Ordinal))
return;
this.isMissing = !exists;
await this.OnExistenceChecked.InvokeAsync(exists);
}
await base.OnParametersSetAsync();
}
#endregion
/// <summary>
/// Tells a web page from a file on this computer by its path.
/// </summary>
/// <remarks>
/// Only an absolute path names a file: a relative one would name another file depending on
/// where it is opened from.
/// </remarks>
internal static FileActionsKind DetermineKind(string path)
{
if (string.IsNullOrWhiteSpace(path))
return FileActionsKind.NONE;
if (Uri.TryCreate(path, UriKind.Absolute, out var uri) && uri.Scheme is "http" or "https")
return FileActionsKind.WEB_PAGE;
return System.IO.Path.IsPathRooted(path) && (uri is null || uri.IsFile)
? FileActionsKind.FILE
: FileActionsKind.NONE;
}
private async Task CopyPath() => await this.RustService.CopyText2Clipboard(this.Path);
private async Task ShowInFileManager() => await new SourceDocumentLocation(this.Path, null).ShowInFileManagerAsync(this.RustService);
private async Task Open() => await new SourceDocumentLocation(this.Path, this.PageNumber).OpenAsync(this.RustService);
}
app/MindWork AI Studio/Components/FileDialogTitle.razor
+11
View File
@@ -0,0 +1,11 @@
@inherits MSGComponentBase
@* The name takes the free width, so the buttons stand right. *@
<div class="file-dialog-title">
<MudText Typo="Typo.h6" Class="flex-grow-1">
@this.Title
</MudText>
@* Opening a file belongs to the bar above its content; a web page has no such bar, so it opens from here: *@
<FileActions Path="@this.Path" ShowOpen="@this.OpensWebPages" WarnIfMissing="@this.WarnIfMissing" TooltipPlacement="Placement.Bottom"/>
</div>
app/MindWork AI Studio/Components/FileDialogTitle.razor.cs
+39
View File
@@ -0,0 +1,39 @@
using Microsoft.AspNetCore.Components;
namespace AIStudio.Components;
/// <summary>
/// The title of a dialog about one file or web page: its name, and at the side, ways to copy its
/// path and to find it in the file manager, see FileActions.
/// </summary>
/// <remarks>
/// The path is what the copy button copies, so that is where it shows, in the tooltip of that
/// button, instead of taking a line of its own. Every dialog about a file names it the same way,
/// which is why this lives here instead of in each of them. A file which is no longer at its place
/// gets a warning instead of the button of the file manager.
/// </remarks>
public partial class FileDialogTitle : MSGComponentBase
{
/// <summary>
/// The title, usually the name of the file.
/// </summary>
[Parameter]
public string Title { get; set; } = string.Empty;
/// <summary>
/// The path of the file or the address of the web page, or nothing when there is neither to name.
/// </summary>
[Parameter]
public string Path { get; set; } = string.Empty;
/// <summary>
/// Whether a file which is no longer at its place gets a warning, see FileActions.WarnIfMissing.
/// </summary>
[Parameter]
public bool WarnIfMissing { get; set; } = true;
/// <summary>
/// Whether the path is the address of a web page, which this title then opens in the browser.
/// </summary>
private bool OpensWebPages => FileActions.DetermineKind(this.Path) is FileActions.FileActionsKind.WEB_PAGE;
}
app/MindWork AI Studio/Components/FileExportActions.razor
-13
View File
@@ -1,13 +0,0 @@
@inherits MSGComponentBase
<MudStack Row="@true" AlignItems="@AlignItems.Center" Spacing="0">
<MudTooltip Text="@T("Show in the file manager")" Placement="Placement.Top">
<MudIconButton Icon="@Icons.Material.Filled.FolderOpen" Size="Size.Small" OnClick="@this.ShowInFileManager"/>
</MudTooltip>
@if (this.OffersOpening)
{
<MudTooltip Text="@T("Open with the default program")" Placement="Placement.Top">
<MudIconButton Icon="@Icons.Material.Filled.OpenInNew" Size="Size.Small" OnClick="@this.OpenWithDefaultProgram"/>
</MudTooltip>
}
</MudStack>
app/MindWork AI Studio/Components/FileExportActions.razor.cs
-73
View File
@@ -1,73 +0,0 @@
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Components;
/// <summary>
/// The ways from an exported file to the file itself: the file manager, which shows it, and the default
/// program, which opens it.
/// </summary>
/// <remarks>
/// Whoever chose where a file goes wants to get to it afterward, and a message which merely says that
/// the export succeeded leaves them searching. Both ways go through the runtime, like those of the list
/// of sources: under Linux, the file manager is reached through the desktop portal, and the runtime
/// refuses to start programs and launchers.
/// </remarks>
public partial class FileExportActions : MSGComponentBase
{
/// <summary>
/// The exported file.
/// </summary>
[Parameter]
public string FilePath { get; set; } = string.Empty;
/// <summary>
/// Whether the file may be opened with the default program. False for an archive, which that
/// program would only unpack, at least on macOS.
/// </summary>
[Parameter]
public bool OffersOpening { get; set; }
[Inject]
private RustService RustService { get; init; } = null!;
[Inject]
private ILogger<FileExportActions> Logger { get; init; } = null!;
private async Task ShowInFileManager()
{
try
{
var response = await this.RustService.TryOpenPathInRuntimeFileManager(this.FilePath);
if (response.Success)
return;
var issue = string.IsNullOrWhiteSpace(response.Issue) ? T("Unknown error") : response.Issue;
await this.MessageBus.SendError(new(Icons.Material.Filled.FolderOpen, string.Format(T("Could not open the file location: {0}"), issue)));
}
catch (Exception e)
{
this.Logger.LogWarning(e, "Could not show an exported file in the file manager.");
await this.MessageBus.SendError(new(Icons.Material.Filled.FolderOpen, T("Could not open the file location.")));
}
}
private async Task OpenWithDefaultProgram()
{
try
{
var response = await this.RustService.TryOpenDocumentInSystemViewer(this.FilePath, null);
if (response.Success)
return;
var issue = string.IsNullOrWhiteSpace(response.Issue) ? T("Unknown error") : response.Issue;
await this.MessageBus.SendError(new(Icons.Material.Filled.OpenInNew, string.Format(T("Could not open the file: {0}"), issue)));
}
catch (Exception e)
{
this.Logger.LogWarning(e, "Could not open an exported file with the default program.");
await this.MessageBus.SendError(new(Icons.Material.Filled.OpenInNew, T("Could not open the file.")));
}
}
}
app/MindWork AI Studio/Components/FileExportSuccessMessage.razor
+1 -1
View File
@@ -4,5 +4,5 @@
<div>
@this.Message
</div>
<FileExportActions FilePath="@this.FilePath" OffersOpening="@this.OffersOpening"/>
<FileActions Path="@this.FilePath" ShowCopy="@false" ShowOpen="@this.OffersOpening"/>
</MudStack>
app/MindWork AI Studio/Components/MSGComponentBase.cs
+13
View File
@@ -23,6 +23,19 @@ public abstract class MSGComponentBase : ComponentBase, IDisposable, IAsyncDispo
private ILanguagePlugin Lang { get; set; } = PluginFactory.BaseLanguage;
/// <summary>
/// How Markdown rendered by this component looks: its code blocks follow the color theme of the app.
/// </summary>
/// <remarks>
/// Every component which renders Markdown wants the same, so it is written down once here instead
/// of in each of them. Read on every render: a change of the color theme renders every component
/// again, cf. ProcessMessage, and the code blocks change along with it.
/// </remarks>
protected MudMarkdownStyling MarkdownStyling => new()
{
CodeBlock = { Theme = this.SettingsManager.IsDarkMode ? CodeBlockTheme.Dark : CodeBlockTheme.Default },
};
#region Overrides of ComponentBase
protected override async Task OnInitializedAsync()
app/MindWork AI Studio/Components/SourcePassageView.razor
+45
View File
@@ -0,0 +1,45 @@
@using AIStudio.Tools.RAG
@inherits MSGComponentBase
@if (this.Passage.Category is RetrievalContentCategory.IMAGE)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined">
@T("An image matched here. The AI was told about it and where to find it, but it did not receive the image itself.")
</MudAlert>
}
else
{
@if (this.isStoreUnavailable)
{
<MudAlert Severity="Severity.Warning" Variant="Variant.Outlined" Class="mb-2">
<MudStack Row="true" AlignItems="AlignItems.Center" Justify="Justify.SpaceBetween" Wrap="Wrap.Wrap">
<MudText>@T("At the moment, AI Studio cannot read the text of this passage that the AI received, e.g., while AI Studio is still starting.")</MudText>
<MudButton Variant="Variant.Outlined" Color="Color.Warning" StartIcon="@Icons.Material.Filled.Refresh" OnClick="@this.ReadTextAgainAsync">
@T("Try again")
</MudButton>
</MudStack>
</MudAlert>
}
else if (this.isTextMissing)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="mb-2">
@T("AI Studio no longer has the text of this passage that the AI received. Only where it was found is still known.")
</MudAlert>
}
else if (this.isReadingText)
{
<MudProgressLinear Indeterminate="@true" Color="Color.Primary" Size="Size.Small" Class="mb-2"/>
}
<ContentViewer Text="@this.text" MaxHeight="@this.MaxHeight">
<Toolbar>
@* The heading takes the free width, so the button stands right, next to the views. The
title of the dialog warns about a file which is no longer at its place. *@
<MudText Typo="Typo.subtitle2" Class="flex-grow-1">
@this.heading
</MudText>
<FileActions Path="@this.openPath" PageNumber="@this.openPage" ShowCopy="@false" ShowFileManager="@false" WarnIfMissing="@false"/>
</Toolbar>
</ContentViewer>
}
app/MindWork AI Studio/Components/SourcePassageView.razor.cs
+201
View File
@@ -0,0 +1,201 @@
using AIStudio.Tools.Databases;
using AIStudio.Tools.Databases.ProvenanceStore;
using AIStudio.Tools.RAG;
using AIStudio.Tools.Rust;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Components;
/// <summary>
/// Shows one passage the AI received from a data source, in full and the way the AI read it.
/// </summary>
/// <remarks>
/// The content is the filtered one, see SourcePassage. The bar above it says where in the document
/// the passage stands, and whether it is the transcript of a recording, since the AI then read the
/// words of the recording rather than the recording itself. An image the AI was only told about is
/// said to be one, rather than shown as if the AI had seen it.<br/><br/>
///
/// The texts of a passage lie in the provenance store once its answer is complete, and are read
/// from there. Should the store no longer hold them, the view says so instead of showing nothing.
/// A store which cannot be read, e.g., while it is still starting, is no proof of that: the view
/// then offers to read the texts again.
/// </remarks>
public partial class SourcePassageView : MSGComponentBase
{
/// <summary>
/// What stands between the parts of the text, as Markdown: a horizontal rule.
/// </summary>
private const string PART_SEPARATOR = "\n\n---\n\n";
/// <summary>
/// The passage to show.
/// </summary>
[Parameter]
public SourcePassage Passage { get; set; } = new();
/// <summary>
/// The document the passage is from, which can be opened at its page, or null when there is no
/// document to open.
/// </summary>
[Parameter]
public SourceDocumentLocation? Document { get; set; }
/// <summary>
/// How high the text may grow before it scrolls, as a CSS length.
/// </summary>
[Parameter]
public string MaxHeight { get; set; } = ContentViewer.DEFAULT_MAX_HEIGHT;
[Inject]
private DatabaseClientProvider DatabaseClientProvider { get; init; } = null!;
[Inject]
private ILogger<SourcePassageView> Logger { get; init; } = null!;
private string heading = string.Empty;
private string text = string.Empty;
/// <summary>
/// What the bar above the text opens: the document, or the web page of the passage; empty when there is neither.
/// </summary>
private string openPath = string.Empty;
/// <summary>
/// The page of the passage, which a document opens at, or null.
/// </summary>
private int? openPage;
/// <summary>
/// The passage whose texts are shown or being read, so that a render with the same passage does not read them again.
/// </summary>
private SourcePassage? shownPassage;
/// <summary>
/// Numbers the reads, so that a read which another passage overtook leaves the texts alone.
/// </summary>
private int readNumber;
private bool isReadingText;
/// <summary>
/// Whether the provenance store no longer holds the texts of the passage.
/// </summary>
private bool isTextMissing;
/// <summary>
/// Whether the provenance store could not be read, so that it is unknown whether it holds the texts of the passage.
/// </summary>
private bool isStoreUnavailable;
#region Overrides of ComponentBase
protected override async Task OnParametersSetAsync()
{
var parts = new[] { this.Passage.GetLocationLabel(), this.GetTranscriptLabel() }.Where(part => part.Length > 0).ToList();
this.heading = parts.Count > 0 ? string.Join(" · ", parts) : T("Text the AI received");
//
// A document opens at the page of the passage. One without pages, such as a CSV file, opens
// at its beginning: no program can be sent to a part, but the user may still want to look.
// Without a document, the passage of a web page opens its page; any other passage, such as
// that of a mail, has nothing to open, see FileActions:
//
var page = this.Passage.PageNumber is > 0 ? this.Passage.PageNumber : null;
this.openPath = this.Document?.Path ?? this.Passage.Path;
this.openPage = this.Document is null ? null : page;
if (!ReferenceEquals(this.shownPassage, this.Passage))
{
this.shownPassage = this.Passage;
await this.ReadTextAsync(this.Passage);
}
await base.OnParametersSetAsync();
}
#endregion
/// <summary>
/// Reads the texts of a passage: from the provenance store once they lie there, otherwise from the passage itself.
/// </summary>
/// <param name="passage">The passage.</param>
private async Task ReadTextAsync(SourcePassage passage)
{
var currentRead = ++this.readNumber;
this.isTextMissing = false;
this.isStoreUnavailable = false;
if (passage.ProvenanceId is not { } provenanceId)
{
this.isReadingText = false;
this.text = this.ComposeText(passage.Text, passage.SurroundingContent);
return;
}
this.isReadingText = true;
this.text = string.Empty;
ProvenanceEntry? entry = null;
bool storeCannotBeRead;
try
{
// A store which is not available answers every read with nothing, which would read as a deleted text:
var store = await this.DatabaseClientProvider.GetProvenanceStoreAsync();
storeCannotBeRead = !store.IsAvailable;
if (!storeCannotBeRead)
entry = (await store.GetEntriesAsync([provenanceId], CancellationToken.None)).GetValueOrDefault(provenanceId);
}
catch (Exception exception)
{
this.Logger.LogError(exception, "Failed to read the passage '{ProvenanceId}' from the provenance store.", provenanceId);
storeCannotBeRead = true;
}
// The user went on to another passage meanwhile, which shows its own texts:
if (currentRead != this.readNumber)
return;
this.isReadingText = false;
this.isStoreUnavailable = storeCannotBeRead;
this.isTextMissing = !storeCannotBeRead && entry is null;
this.text = entry is null ? string.Empty : this.ComposeText(entry.Content, entry.SurroundingContent);
}
/// <summary>
/// Reads the texts of the shown passage once more, after the provenance store could not be read.
/// </summary>
private Task ReadTextAgainAsync() => this.ReadTextAsync(this.Passage);
/// <summary>
/// Writes the texts of a passage as one text.
/// </summary>
/// <remarks>
/// The surrounding content follows the matched text, the way the AI received both, so that all
/// of it reads in one place.
/// </remarks>
private string ComposeText(string matchedText, IReadOnlyList<string> surroundingContent) => surroundingContent.Count is 0
? matchedText
: $"{matchedText}{PART_SEPARATOR}**{T("Surrounding content")}**{PART_SEPARATOR}{string.Join(PART_SEPARATOR, surroundingContent)}";
/// <summary>
/// Says what the passage is a transcript of, or nothing when it is no transcript.
/// </summary>
/// <remarks>
/// Asked by the extension of the path only: the file may be gone by now, and reading it would
/// tell nothing more about what the AI received.
/// </remarks>
private string GetTranscriptLabel()
{
if (!this.Passage.IsTranscript())
return string.Empty;
var extension = Path.GetExtension(this.Passage.Path);
if (FileTypes.IsAllowedExtension(extension, FileTypes.VIDEO))
return T("Transcript of a video file");
return FileTypes.IsAllowedExtension(extension, FileTypes.AUDIO)
? T("Transcript of an audio file")
: T("Transcript");
}
}
app/MindWork AI Studio/Components/SourcesList.razor
+78 -24
View File
@@ -1,3 +1,4 @@
@using AIStudio.Tools.RAG
@inherits MSGComponentBase
@* The class is what the Markdown renderer wraps its own output in, so the headings and the list
@@ -13,32 +14,85 @@
<ul>
@foreach (var entry in group.Entries)
{
<li>
@($"[{entry.Number}] ")
@if (entry.Document is { } document)
{
<MudTooltip Text="@T("Opens this document in the program your system uses for it")" Placement="Placement.Top">
<MudLink Typo="Typo.body1" OnClick="@(() => this.OpenDocument(document))">
<li class="source-entry">
<div class="source-entry-title">
@($"[{entry.Number}] ")
@if (this.IsMissing(entry))
{
@* A link would only fail. The warning next to the entry says why: *@
<MudText Typo="Typo.body1" Inline="true">
@entry.Title
</MudText>
}
else if (entry.Document is { } pagedDocument && entry.Location is { } location)
{
@* Each page the AI read is a link of its own, which opens the document there. The
name would only open it at one of them, so it is no link. *@
<MudText Typo="Typo.body1" Inline="true">
@($"{entry.Name} ({location.Prefix}")
</MudText>
@for (var rangeIndex = 0; rangeIndex < location.Ranges.Count; rangeIndex++)
{
var pageRange = location.Ranges[rangeIndex];
if (rangeIndex > 0)
{
<MudText Typo="Typo.body1" Inline="true">
@PassageLocation.RANGE_SEPARATOR
</MudText>
}
<MudTooltip Text="@FileActions.GetOpenTooltip(pageRange.First)" Placement="Placement.Top">
<MudLink Typo="Typo.body1" OnClick="@(() => this.OpenDocument(pagedDocument with { PageNumber = pageRange.First }))">
@pageRange
</MudLink>
</MudTooltip>
}
<MudText Typo="Typo.body1" Inline="true">
@($"{location.Suffix})")
</MudText>
}
else if (entry.Document is { } document)
{
@* Inline rather than the usual inline block: a long title, e.g., one naming many
parts, then wraps like text right after its number instead of moving below it. *@
<MudTooltip Text="@FileActions.GetOpenTooltip(null)" Placement="Placement.Top" RootClass="d-inline">
<MudLink Typo="Typo.body1" OnClick="@(() => this.OpenDocument(document))">
@entry.Title
</MudLink>
</MudTooltip>
}
else if (entry.HasNoLink)
{
@* Nothing opens a mail by its address yet, nor what only a data source knows, so a link would lead nowhere. *@
<MudText Typo="Typo.body1" Inline="true">
@entry.Title
</MudText>
}
else
{
<MudLink Href="@entry.Link" Target="_blank" Typo="Typo.body1">
@entry.Title
</MudLink>
</MudTooltip>
<MudTooltip Text="@T("Show this file in the file manager of your system")" Placement="Placement.Top">
<MudIconButton Icon="@Icons.Material.Filled.FolderOpen" Size="Size.Small" OnClick="@(() => this.ShowInFileManager(document))"/>
</MudTooltip>
}
else if (entry.IsMail)
{
@* Nothing opens a mail by its address yet, so a link would lead nowhere. *@
<MudText Typo="Typo.body1" Inline="true">
@entry.Title
</MudText>
}
else
{
<MudLink Href="@entry.Link" Target="_blank" Typo="Typo.body1">
@entry.Title
</MudLink>
}
}
</div>
@* The passages come last, so their button lines up at the right edge also where a
source has no file to show. The tooltips open to the left, where there is room
for them; above a button at the edge, they would be pushed around and cover the
row above. They show on hover only: the focus comes back to the button when the
dialog closes, or the file manager does, and a tooltip shown on focus would then
stay until the user clicks somewhere else. The title opens the file or the page,
so the actions only copy and find it; a mail gets none, see FileActions. *@
<div class="source-entry-actions">
<FileActions Path="@(entry.Document?.Path ?? entry.Link)" ShowOpen="@false" TooltipPlacement="Placement.Left" OnExistenceChecked="@(exists => this.NoteExistence(entry, exists))"/>
@if (entry.Passages.Count > 0)
{
<MudTooltip Text="@this.PassagesTooltip(entry)" Placement="Placement.Left" ShowOnFocus="@false">
<MudIconButton Icon="@Icons.Material.Filled.Visibility" Size="Size.Small" Class="icon-button-compact" OnClick="@(() => this.ShowPassages(entry))"/>
</MudTooltip>
}
</div>
</li>
}
</ul>
app/MindWork AI Studio/Components/SourcesList.razor.cs
+46 -56
View File
@@ -1,4 +1,5 @@
using AIStudio.Tools.Rust;
using AIStudio.Dialogs;
using AIStudio.Tools.RAG;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
@@ -29,14 +30,20 @@ public partial class SourcesList : MSGComponentBase
[Parameter]
public IList<Source> Sources { get; set; } = [];
/// <summary>
/// The passages the AI received from the sources, which the user can read below each of them.
/// </summary>
[Parameter]
public IReadOnlyList<SourcePassage> Passages { get; set; } = [];
[Inject]
private RustService RustService { get; init; } = null!;
[Inject]
private IJSRuntime JsRuntime { get; init; } = null!;
private IDialogService DialogService { get; init; } = null!;
[Inject]
private ILogger<SourcesList> Logger { get; init; } = null!;
private IJSRuntime JsRuntime { get; init; } = null!;
private readonly List<SourceEntryGroup> groups = [];
@@ -73,77 +80,57 @@ public partial class SourcesList : MSGComponentBase
private void RebuildGroups()
{
this.groups.Clear();
var passagesBySource = this.Passages.SortBySource();
foreach (var group in this.Sources.GroupSources())
{
var entries = new List<SourceEntry>(group.Sources.Count);
foreach (var numberedSource in group.Sources)
{
var document = numberedSource.Source.TryGetDocumentLocation(out var location) ? location : (SourceDocumentLocation?)null;
entries.Add(new(numberedSource.Number, numberedSource.Source.Title, numberedSource.Source.URL, document, numberedSource.Source.IsMailSource()));
var passages = passagesBySource.GetPassagesOf(numberedSource.Source);
//
// Pages are links only into a document, and only under the name the title of the
// source was made with, see SourcePassageExtensions.NamePassagesInTitles. Parts of a
// document without pages are nothing a program can open, so their title stays as it is:
//
var name = passages.Count > 0 ? passages[0].DataSourceName : string.Empty;
var pages = document is not null && !string.IsNullOrWhiteSpace(name) && passages.GetLocation() is { ArePages: true } pageLocation ? pageLocation : null;
entries.Add(new(numberedSource.Number, numberedSource.Source.Title, numberedSource.Source.URL, document, numberedSource.Source.HasNoLink(), passages, name, pages));
}
this.groups.Add(new(group.Heading, entries));
}
}
/// <summary>
/// Opens a document in the program the system uses for it.
/// </summary>
/// <remarks>
/// Whether the program can be sent to a page is the runtime's business, and it says afterwards
/// whether it managed to. Nothing is shown about that here: the document is open, and the title
/// of the source names the page anyway.
/// </remarks>
/// <param name="document">The document to open, and the page to show.</param>
private async Task OpenDocument(SourceDocumentLocation document)
{
OpenDocumentResponse response;
try
{
response = await this.RustService.TryOpenDocumentInSystemViewer(document.Path, document.PageNumber);
}
catch (Exception e)
{
this.Logger.LogWarning(e, "Could not open a source document.");
await this.MessageBus.SendError(new(Icons.Material.Filled.Description, T("Could not open the document.")));
return;
}
private string PassagesTooltip(SourceEntry entry) => entry.Passages.Count is 1
? T("Show the passage the AI received from this source")
: string.Format(T("Show the {0} passages the AI received from this source"), entry.Passages.Count.CompactCount());
if (response.Success)
return;
private async Task ShowPassages(SourceEntry entry) => await SourcePassagesDialog.OpenDialogAsync(this.DialogService, entry.Passages, entry.Document);
var issue = string.IsNullOrWhiteSpace(response.Issue) ? T("Unknown error") : response.Issue;
await this.MessageBus.SendError(new(Icons.Material.Filled.Description, string.Format(T("Could not open the document: {0}"), issue)));
}
private async Task OpenDocument(SourceDocumentLocation document) => await document.OpenAsync(this.RustService);
/// <summary>
/// Opens the file browser of the system and selects the document in it.
/// The files of sources which are no longer at their place, by their paths, as FileActions found out.
/// </summary>
/// <remarks>
/// The second way out of the list: a document which the system opens in the wrong program, or
/// which the user wants to move or send on instead of read, is reached from here without being
/// opened. This is the same way out the embeddings page offers for a file it could not read.
/// </remarks>
/// <param name="document">The document to show.</param>
private async Task ShowInFileManager(SourceDocumentLocation document)
private readonly HashSet<string> missingFiles = new(PathTools.COMPARER);
private bool IsMissing(SourceEntry entry) => entry.Document is { } document && this.missingFiles.Contains(document.Path);
/// <summary>
/// Keeps whether the file of a source is at its place, so that its title stops being a link when it is not.
/// </summary>
private void NoteExistence(SourceEntry entry, bool exists)
{
OpenPathResponse response;
try
{
response = await this.RustService.TryOpenPathInRuntimeFileManager(document.Path);
}
catch (Exception e)
{
this.Logger.LogWarning(e, "Could not show a source document in the file manager.");
await this.MessageBus.SendError(new(Icons.Material.Filled.FolderOpen, T("Could not open the file location.")));
return;
}
if (response.Success)
if (entry.Document is not { } document)
return;
var issue = string.IsNullOrWhiteSpace(response.Issue) ? T("Unknown error") : response.Issue;
await this.MessageBus.SendError(new(Icons.Material.Filled.FolderOpen, string.Format(T("Could not open the file location: {0}"), issue)));
if (exists)
this.missingFiles.Remove(document.Path);
else
this.missingFiles.Add(document.Path);
}
/// <summary>
@@ -160,6 +147,9 @@ public partial class SourcesList : MSGComponentBase
/// <param name="Title">The title of the source.</param>
/// <param name="Link">The address of the source, which a web source is opened by.</param>
/// <param name="Document">The document the source names, or null when it names none.</param>
/// <param name="IsMail">Whether the source names a mail, which is shown as text.</param>
private readonly record struct SourceEntry(int Number, string Title, string Link, SourceDocumentLocation? Document, bool IsMail);
/// <param name="HasNoLink">Whether nothing can open the source by its address, e.g., a mail, so that it is shown as text.</param>
/// <param name="Passages">The passages the AI received from the source; empty when it received none, or the chat is older than these.</param>
/// <param name="Name">The name of the source without its location, or empty when it has no passages.</param>
/// <param name="Location">The pages of the passages in the document, each to be opened on its own, or null when there are none.</param>
private readonly record struct SourceEntry(int Number, string Title, string Link, SourceDocumentLocation? Document, bool HasNoLink, IReadOnlyList<SourcePassage> Passages, string Name, PassageLocation? Location);
}
app/MindWork AI Studio/Dialogs/AssistantDraftDialog.razor.cs
-7
View File
@@ -14,11 +14,4 @@ public partial class AssistantDraftDialog : MSGComponentBase
private void Cancel() => this.MudDialog.Cancel();
private void Confirm() => this.MudDialog.Close(DialogResult.Ok(this.DraftMarkdown));
private CodeBlockTheme CodeColorPalette => this.SettingsManager.IsDarkMode ? CodeBlockTheme.Dark : CodeBlockTheme.Default;
private MudMarkdownStyling MarkdownStyling => new()
{
CodeBlock = { Theme = this.CodeColorPalette },
};
}
app/MindWork AI Studio/Dialogs/ChatTemplateDialog.razor.cs
+3 -2
View File
@@ -183,7 +183,7 @@ public partial class ChatTemplateDialog : MSGComponentBase
if (this.CreateFromExistingChatThread && this.ExistingChatThread is not null)
{
this.DataSystemPrompt = this.ExistingChatThread.SystemPrompt;
this.dataExampleConversation = this.ExistingChatThread.Blocks.Select(n => n.DeepClone(true)).ToList();
this.dataExampleConversation = this.ExistingChatThread.Blocks.Select(n => n.CloneApartFromChat(true)).ToList();
this.DataName = this.ExistingChatThread.Name;
}
@@ -211,7 +211,8 @@ public partial class ChatTemplateDialog : MSGComponentBase
SystemPrompt = this.DataSystemPrompt,
PredefinedUserPrompt = this.PredefinedUserPrompt,
ExampleConversation = this.dataExampleConversation,
FileAttachments = this.fileAttachments.Select(attachment => attachment.Normalize()).ToList(),
// Without the entries of their texts: every chat started from the template reads its files anew, cf. FileAttachment.ProvenanceId.
FileAttachments = this.fileAttachments.Select(attachment => attachment.Normalize().WithoutStoredText()).ToList(),
AllowProfileUsage = this.AllowProfileUsage,
ToolIds = this.preselectTools ? new HashSet<string>(this.selectedToolIds, StringComparer.Ordinal) : null,
DataSourceOptions = this.preselectDataSources ? this.templateDataSourceOptions.CreateCopy() : null,
app/MindWork AI Studio/Dialogs/DocumentCheckDialog.razor
+42 -77
View File
@@ -1,12 +1,18 @@
@inherits MSGComponentBase
<MudDialog>
<MudDialog ContentClass="dialog-content-fit">
<TitleContent>
@* Named after the document once there is one, and after what the caller opened us for until then.
A file which is no longer at its place is told about below, together with what is shown instead: *@
<FileDialogTitle Title="@(this.document?.FileName ?? this.MudDialog.Title ?? string.Empty)" Path="@(this.document?.FilePath ?? string.Empty)" WarnIfMissing="@false"/>
</TitleContent>
<DialogContent>
@* A drop anywhere in this dialog belongs to the dialog, not to the page behind it: *@
<PathDropZone IsArea="@true"
IdPrefix="document-check"
OnPathsDropped="@this.DropCallback"
Disabled="@this.IsZoneDisabled"
Class="dialog-fit"
Context="isDropTarget">
<MudJustifiedText Typo="Typo.body1" Class="mb-3">
@T("See how we load your file. Review the content before we process it further.")
@@ -21,38 +27,13 @@
@if (this.document is null)
{
<ReadFileContent Text="@T("Load file")" FileContent="@this.fileContent" FileContentChanged="@this.ApplyLoadedFileContent" EnableDragDrop="true" CatchAllDocuments="true"/>
<ReadFileContent Text="@T("Load file")" @bind-FileContent="@this.fileContent" EnableDragDrop="true" CatchAllDocuments="true"/>
}
else
{
@* Keys have to be unique among siblings, no matter the component: this field and the
tabs below both stand for the document and would otherwise collide on its path. *@
<MudTextField
@key="@($"file-path-{this.document.FilePath}")"
T="string"
Text="@this.document.FilePath"
AdornmentIcon="@Icons.Material.Filled.FileOpen"
Adornment="Adornment.Start"
Immediate="@true"
Label="@T("File Path")"
Variant="Variant.Outlined"
Lines="1"
MaxLines="3"
Class="mt-10"
ReadOnly="true"
/>
}
@* The frame shows where a dropped file would land. It is drawn only while this dialog can
take one, and keeps its width in both states so that nothing jumps during a drag: *@
<div class="@this.PreviewAreaClass(isDropTarget)">
@if (!this.document?.Exists ?? false)
{
<MudAlert Severity="Severity.Error" Variant="Variant.Filled" Class="my-2">
@T("The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible.")
</MudAlert>
}
else if (this.isLoadingContent)
<div class="dialog-fit @this.PreviewAreaClass(isDropTarget)">
@if (this.isLoadingContent)
{
<MudJustifiedText Typo="Typo.body1" Class="my-3">
@T("Please wait while we load the content of your file. Depending on the file type and size, this may take a moment.")
@@ -62,6 +43,12 @@
<MudSkeleton Width="100%"/>
<MudSkeleton Width="90%"/>
}
else if (this.IsFileMissing && !this.showsStoredText)
{
<MudAlert Severity="Severity.Error" Variant="Variant.Filled" Class="my-2">
@T("The file could not be found. It may have been moved, renamed, or deleted, or it is otherwise inaccessible.")
</MudAlert>
}
else if (this.loadFailureMessage is not null)
{
<MudAlert Severity="Severity.Error" Variant="Variant.Filled" Class="my-2">
@@ -70,56 +57,34 @@
}
else
{
@if (this.previewCutOffCharacters > 0)
@if (this.IsFileMissing)
{
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Class="my-2">
@string.Format(T("Your document is large, so we show you only its beginning. We hide the remaining {0:N0} characters here. Rest assured: the AI still receives your entire document."), this.previewCutOffCharacters)
@* The text the AI received outlives the file, so the file being gone is worth a note, not an error: *@
<MudAlert Severity="Severity.Info" Variant="Variant.Outlined" Dense="@true" Class="my-2">
@T("The file is no longer in its original location. Below is the text the AI received from it when you sent it.")
</MudAlert>
}
@* Keyed by the document: a switch from an image to a text file changes which panels
exist, and a leftover active panel would point at one that is gone. *@
<MudTabs @key="@($"preview-tabs-{this.document?.FilePath}")" Elevation="0" Rounded="true" ApplyEffectsToContainer="true" Outlined="true" PanelClass="pa-2" Class="mb-2">
@if (this.document?.IsImage ?? false)
{
<MudTabPanel Text="@T("Image View")" Icon="@Icons.Material.Filled.Image">
<MudImage ObjectFit="ObjectFit.ScaleDown" Style="width: 100%;" Src="@this.document.FilePathAsUrl"/>
</MudTabPanel>
}
else
{
<MudTabPanel Text="@T("Markdown View")" Icon="@Icons.Material.Filled.TextSnippet">
<MudField
Variant="Variant.Outlined"
AdornmentIcon="@Icons.Material.Filled.Article"
Adornment="Adornment.Start"
Label="@T("Loaded Content")"
FullWidth="true"
Class="ma-2 pe-4"
HelperText="@T("This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected.")">
<div style="max-height: 40vh; overflow-y: auto;">
<MudMarkdown Value="@this.previewContent" Props="Markdown.DefaultConfig" Styling="@this.MarkdownStyling" MarkdownPipeline="Markdown.SAFE_MARKDOWN_PIPELINE"/>
</div>
</MudField>
</MudTabPanel>
<MudTabPanel Text="@T("Simple View")" Icon="@Icons.Material.Filled.Terminal">
<MudTextField
T="string"
Text="@this.previewContent"
AdornmentIcon="@Icons.Material.Filled.Article"
Adornment="Adornment.Start"
Immediate="@true"
Label="@T("Loaded Content")"
Variant="Variant.Outlined"
Lines="6"
AutoGrow="@true"
MaxLines="25"
ReadOnly="true"
Class="ma-2"
HelperText="@T("This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected.")"/>
</MudTabPanel>
}
</MudTabs>
@* Keyed by the document: every document starts on the first panel of its own. *@
<ContentViewer
@key="@($"preview-tabs-{this.document?.FilePath}")"
Kind="@this.ViewerKind"
Text="@this.fileContent"
MediaUrl="@(this.document?.FilePathAsUrl ?? string.Empty)"
HelperText="@(this.showsStoredText ? T("This is the text the AI received from your file when you first sent it. Later changes to the file do not reach the AI in this chat.") : T("This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected."))"
CharacterLimit="@PREVIEW_CHARACTER_LIMIT"
CutOffNotice="@this.CutOffNotice"
MaxHeight="@ContentViewer.NO_HEIGHT_LIMIT">
<Toolbar>
@* The label takes the free width, so the button stands right, next to the views. *@
<MudText Typo="Typo.subtitle2" Class="flex-grow-1">
@(this.showsStoredText ? T("Text the AI received") : T("Loaded Content"))
</MudText>
@* At its beginning: the preview shows how we read the file, not a place in it. *@
<FileActions Path="@(this.document?.FilePath ?? string.Empty)" ShowCopy="@false" ShowFileManager="@false" WarnIfMissing="@false"/>
</Toolbar>
</ContentViewer>
}
</div>
</PathDropZone>
app/MindWork AI Studio/Dialogs/DocumentCheckDialog.razor.cs
+96 -52
View File
@@ -1,5 +1,6 @@
using AIStudio.Chat;
using AIStudio.Components;
using AIStudio.Tools.Databases;
using AIStudio.Tools.Services;
using Microsoft.AspNetCore.Components;
@@ -43,6 +44,19 @@ public partial class DocumentCheckDialog : MSGComponentBase
[Parameter]
public Func<bool>? IsAttachingUnavailable { get; set; }
/// <summary>
/// Keeps what this dialog read of a document which was never sent, so that sending it does not
/// read the file again, see FileAttachment.ProvenanceId.
/// </summary>
/// <remarks>
/// Gets the document, the text read from it, and the state of its file before it was read, so
/// that a file which changed afterward is read again; never throws. Only the chat hands
/// one over, for the attachments of the message being written: a sent message has its texts
/// already, and elsewhere, nothing is sent later.
/// </remarks>
[Parameter]
public Func<FileAttachment, string, FileVersion, Task>? KeepReadText { get; set; }
/// <summary>
/// The document we show right now. It starts out as the one we were opened with and changes
/// whenever the user drops another file onto this dialog.
@@ -74,17 +88,6 @@ public partial class DocumentCheckDialog : MSGComponentBase
/// </summary>
private string? loadFailureMessage;
/// <summary>
/// What we show to the user: either the entire file content, or its beginning. We keep this in
/// its own field so that we cut the content only once, instead of on every render.
/// </summary>
private string previewContent = string.Empty;
/// <summary>
/// How many characters we cut off from the preview. Zero when we show the entire content.
/// </summary>
private int previewCutOffCharacters;
/// <summary>
/// Ends the extraction when this dialog is gone, or when another document took the place of
/// the one being read, before that file was read completely.
@@ -113,6 +116,20 @@ public partial class DocumentCheckDialog : MSGComponentBase
/// </summary>
private bool isLoadingContent;
/// <summary>
/// True when the content is the text the AI received from the document when it was sent, rather
/// than what the file holds now, see FileAttachment.ProvenanceId.
/// </summary>
private bool showsStoredText;
/// <summary>
/// Whether the document we show is no longer at its place.
/// </summary>
private bool IsFileMissing => this.document is not null && !this.document.Exists;
[Inject]
private DatabaseClientProvider DatabaseClientProvider { get; init; } = null!;
[Inject]
private RustService RustService { get; init; } = null!;
@@ -128,7 +145,6 @@ public partial class DocumentCheckDialog : MSGComponentBase
this.fileContent = this.FileContent;
this.isLoadingContent = this.NeedsExtraction();
this.UpdatePreview();
await base.OnInitializedAsync();
}
@@ -148,13 +164,14 @@ public partial class DocumentCheckDialog : MSGComponentBase
/// <summary>
/// Whether the document we show has to be read before we can show anything of it. Images are
/// shown as they are, a missing file shows its own message, and content a caller already handed
/// us is reused instead of being extracted a second time.
/// shown as they are, a missing file shows its own message unless the text the AI received from
/// it is stored, and content a caller already handed us is reused instead of being extracted a
/// second time.
/// </summary>
private bool NeedsExtraction() =>
this.document is not null &&
!this.document.IsImage &&
this.document.Exists &&
(this.document.ProvenanceId is not null || this.document.Exists) &&
string.IsNullOrWhiteSpace(this.fileContent);
/// <summary>
@@ -187,8 +204,36 @@ public partial class DocumentCheckDialog : MSGComponentBase
if (this.isDisposed || generation != this.loadGeneration)
return;
string? textToKeep = null;
FileVersion? version = null;
try
{
//
// A document sent before shows the text the AI received from it, which the provenance
// store keeps, also when the file changed or is gone by now. Without it, we read the file:
//
if (documentToLoad.ProvenanceId is { } provenanceId)
{
var storedText = await this.ReadStoredTextAsync(provenanceId);
if (this.isDisposed || generation != this.loadGeneration)
return;
if (storedText is not null)
{
this.fileContent = storedText;
this.showsStoredText = true;
return;
}
// Nothing left to read, which the dialog says:
if (!documentToLoad.Exists)
return;
}
// Taken before reading, so that a change while reading counts as a change:
if (this.KeepReadText is not null)
version = await Task.Run(() => FileVersion.Of(documentToLoad.FilePath), cancellationToken);
var extraction = await UserFile.LoadFileData(documentToLoad.FilePath, this.RustService, this.PandocAvailability, cancellationToken);
if (this.isDisposed || generation != this.loadGeneration)
return;
@@ -201,6 +246,8 @@ public partial class DocumentCheckDialog : MSGComponentBase
//
if (!extraction.HasUsableContent)
this.loadFailureMessage = extraction.ToUserMessage(documentToLoad.FileName);
else if (this.KeepReadText is not null && documentToLoad.ProvenanceId is null)
textToKeep = extraction.Content;
}
catch (OperationCanceledException)
{
@@ -221,10 +268,17 @@ public partial class DocumentCheckDialog : MSGComponentBase
if (!this.isDisposed && generation == this.loadGeneration)
{
this.isLoadingContent = false;
this.UpdatePreview();
this.StateHasChanged();
}
}
//
// Kept once the user sees the text, since keeping may first have to create the chat. Reading
// the file told the user already what could not be read, so sending the kept text tells
// nothing of it again:
//
if (textToKeep is not null && version is not null)
await this.KeepReadText!(documentToLoad, textToKeep, version);
}
/// <summary>
@@ -300,45 +354,42 @@ public partial class DocumentCheckDialog : MSGComponentBase
this.document = attachment;
this.fileContent = string.Empty;
this.loadFailureMessage = null;
this.showsStoredText = false;
this.isLoadingContent = this.NeedsExtraction();
this.UpdatePreview();
}
/// <summary>
/// Called when the user loads a file through this dialog. We don't use a two-way binding here,
/// since we have to refresh the preview whenever the content changes.
/// Reads the text the AI received from a document when it was sent.
/// </summary>
/// <param name="loadedContent">The content of the file the user has loaded.</param>
private void ApplyLoadedFileContent(string loadedContent)
/// <param name="provenanceId">The entry of the text.</param>
/// <returns>The text, or null when the provenance store no longer holds it, or cannot be read.</returns>
private async Task<string?> ReadStoredTextAsync(Guid provenanceId)
{
this.fileContent = loadedContent;
this.UpdatePreview();
}
/// <summary>
/// Determines what part of the file content we show to the user.
/// </summary>
private void UpdatePreview()
{
if (this.fileContent.Length <= PREVIEW_CHARACTER_LIMIT)
try
{
this.previewContent = this.fileContent;
this.previewCutOffCharacters = 0;
return;
var store = await this.DatabaseClientProvider.GetProvenanceStoreAsync();
var entries = await store.GetEntriesAsync([provenanceId], CancellationToken.None);
return entries.GetValueOrDefault(provenanceId)?.Content;
}
catch (Exception exception)
{
this.Logger.LogWarning(exception, "Failed to read the stored text of the document '{ProvenanceId}'. Its file is read instead.", provenanceId);
return null;
}
//
// We cut at the last line break before our limit. Otherwise, we might tear apart a Markdown
// construct like a table row or a code fence in the middle of a line:
//
var cutIndex = this.fileContent.LastIndexOf('\n', PREVIEW_CHARACTER_LIMIT - 1) + 1;
if (cutIndex < 1)
cutIndex = PREVIEW_CHARACTER_LIMIT;
this.previewContent = this.fileContent[..cutIndex];
this.previewCutOffCharacters = this.fileContent.Length - cutIndex;
}
/// <summary>
/// An image is shown as it is; everything else is shown as the text we read from it.
/// </summary>
private ContentViewerKind ViewerKind => (this.document?.IsImage ?? false) ? ContentViewerKind.IMAGE : ContentViewerKind.TEXT;
/// <summary>
/// Explains why we show only the beginning of a large document. The preview only answers how we
/// read the file, so it may leave out the rest; the AI does not.
/// </summary>
/// <param name="cutOffCharacters">How many characters the preview leaves out.</param>
private string CutOffNotice(int cutOffCharacters) => string.Format(T("Your document is large, so we show you only its beginning. We hide the remaining {0} characters here. Rest assured: the AI still receives your entire document."), cutOffCharacters.CompactCount());
/// <summary>
/// Ends a running extraction. Without this, reading a large document would continue after the
/// user closed this dialog and would keep this component, the extracted content, and the
@@ -357,11 +408,4 @@ public partial class DocumentCheckDialog : MSGComponentBase
base.DisposeResources();
}
private CodeBlockTheme CodeColorPalette => this.SettingsManager.IsDarkMode ? CodeBlockTheme.Dark : CodeBlockTheme.Default;
private MudMarkdownStyling MarkdownStyling => new()
{
CodeBlock = { Theme = this.CodeColorPalette },
};
}
app/MindWork AI Studio/Dialogs/PromptingGuidelineDialog.razor.cs
-7
View File
@@ -12,11 +12,4 @@ public partial class PromptingGuidelineDialog : MSGComponentBase
public string GuidelineMarkdown { get; set; } = string.Empty;
private void Close() => this.MudDialog.Cancel();
private CodeBlockTheme CodeColorPalette => this.SettingsManager.IsDarkMode ? CodeBlockTheme.Dark : CodeBlockTheme.Default;
private MudMarkdownStyling MarkdownStyling => new()
{
CodeBlock = { Theme = this.CodeColorPalette },
};
}
app/MindWork AI Studio/Dialogs/ReviewAttachmentsDialog.razor
+16 -4
View File
@@ -89,10 +89,22 @@
</div>
<MudToolBar WrapContent="true" Gutters="false" Class="ml-2" Style="flex-shrink: 0; min-height: 1em;">
<MudIconButton Icon="@Icons.Material.Filled.Search"
Color="Color.Primary"
Disabled="true"/>
@* A file sent before keeps the text the AI received from it, which outlives the file: *@
@if (fileAttachment.ProvenanceId is not null)
{
<MudTooltip Text="@T("Show the text the AI received from this file.")" Placement="Placement.Bottom">
<MudIconButton Icon="@Icons.Material.Filled.Search"
Color="Color.Primary"
OnClick="@(() => this.InvestigateFile(fileAttachment))"/>
</MudTooltip>
}
else
{
<MudIconButton Icon="@Icons.Material.Filled.Search"
Color="Color.Primary"
Disabled="true"/>
}
<MudTooltip Text="@T("Remove this attachment.")" Placement="Placement.Bottom">
<MudIconButton Icon="@Icons.Material.Filled.Delete"
Color="Color.Error"
app/MindWork AI Studio/Dialogs/ReviewAttachmentsDialog.razor.cs
+14 -1
View File
@@ -38,6 +38,15 @@ public partial class ReviewAttachmentsDialog : MSGComponentBase
[Parameter]
public Func<bool>? IsAttachingUnavailable { get; set; }
/// <summary>
/// Keeps what the preview reads of an attachment, so that sending it does not read the file again, cf. DocumentCheckDialog.KeepReadText.
/// </summary>
/// <remarks>
/// Null for a message which was already sent, and wherever nothing is sent later.
/// </remarks>
[Parameter]
public Func<FileAttachment, string, FileVersion, Task>? KeepReadText { get; set; }
[Inject]
private IDialogService DialogService { get; set; } = null!;
@@ -120,7 +129,7 @@ public partial class ReviewAttachmentsDialog : MSGComponentBase
private async Task PathsDropped(List<string> paths) => await this.AttachPathsAsync(paths);
public static async Task<HashSet<FileAttachment>> OpenDialogAsync(IDialogService dialogService, HashSet<FileAttachment> documentPaths, Func<List<string>, Task<IReadOnlyList<FileAttachment>>>? attachPaths = null, Func<bool>? isAttachingUnavailable = null)
public static async Task<HashSet<FileAttachment>> OpenDialogAsync(IDialogService dialogService, HashSet<FileAttachment> documentPaths, Func<List<string>, Task<IReadOnlyList<FileAttachment>>>? attachPaths = null, Func<bool>? isAttachingUnavailable = null, Func<FileAttachment, string, FileVersion, Task>? keepReadText = null)
{
var dialogParameters = new DialogParameters<ReviewAttachmentsDialog>
{
@@ -133,6 +142,9 @@ public partial class ReviewAttachmentsDialog : MSGComponentBase
if (isAttachingUnavailable is not null)
dialogParameters.Add(x => x.IsAttachingUnavailable, isAttachingUnavailable);
if (keepReadText is not null)
dialogParameters.Add(x => x.KeepReadText, keepReadText);
var dialogReference = await dialogService.ShowAsync<ReviewAttachmentsDialog>(TB("Your attached files"), dialogParameters, DialogOptions.FULLSCREEN);
var dialogResult = await dialogReference.Result;
if (dialogResult is null || dialogResult.Canceled)
@@ -161,6 +173,7 @@ public partial class ReviewAttachmentsDialog : MSGComponentBase
var dialogParameters = new DialogParameters<DocumentCheckDialog>
{
{ x => x.Document, fileAttachment },
{ x => x.KeepReadText, this.KeepReadText },
};
//
app/MindWork AI Studio/Dialogs/Settings/SettingsDialogChat.razor
+1
View File
@@ -12,6 +12,7 @@
<DialogContent>
<ConfigurationSelect OptionDescription="@T("Shortcut to send input")" SelectedValue="@(() => this.SettingsManager.ConfigurationData.Chat.ShortcutSendBehavior)" Data="@ConfigurationSelectDataFactory.GetSendBehaviorData()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.Chat.ShortcutSendBehavior = selectedValue)" OptionHelp="@T("Do you want to use any shortcut to send your input?")"/>
<ConfigurationOption OptionDescription="@T("Show the latest message after loading?")" LabelOn="@T("Latest message is shown, after loading a chat")" LabelOff="@T("First (oldest) message is shown, after loading a chat")" State="@(() => this.SettingsManager.ConfigurationData.Chat.ShowLatestMessageAfterLoading)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.Chat.ShowLatestMessageAfterLoading = updatedState)" OptionHelp="@T("When enabled, the latest message is shown after loading a chat. When disabled, the first (oldest) message is shown.")"/>
<ConfigurationOption OptionDescription="@T("Read changed attachments again?")" LabelOn="@T("Changed files are read again for the next message")" LabelOff="@T("Files are sent as they were first read")" State="@(() => this.SettingsManager.ConfigurationData.Chat.ReadChangedAttachmentsAgain)" StateUpdate="@(updatedState => this.SettingsManager.ConfigurationData.Chat.ReadChangedAttachmentsAgain = updatedState)" OptionHelp="@T("When enabled, AI Studio reads an attached file again when it changed since it was sent, and the AI receives the new content with the next message. Viewing the attachment then shows the content that was sent last. When disabled, the AI keeps receiving the content a file had when it was first sent, also when the file changed or is gone.")" IsLocked="() => ManagedConfiguration.TryGet(x => x.Chat, x => x.ReadChangedAttachmentsAgain, out var meta) && meta.IsLocked"/>
<ConfigurationSelect OptionDescription="@T("Provider selection when creating new chats")" SelectedValue="@(() => this.SettingsManager.ConfigurationData.Chat.AddChatProviderBehavior)" Data="@ConfigurationSelectDataFactory.GetAddChatProviderBehavior()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.Chat.AddChatProviderBehavior = selectedValue)" OptionHelp="@T("Control how the LLM provider for added chats is selected.")"/>
<ConfigurationSelect OptionDescription="@T("Provider selection when loading a chat and sending assistant results to chat")" SelectedValue="@(() => this.SettingsManager.ConfigurationData.Chat.LoadingProviderBehavior)" Data="@ConfigurationSelectDataFactory.GetLoadingChatProviderBehavior()" SelectionUpdate="@(selectedValue => this.SettingsManager.ConfigurationData.Chat.LoadingProviderBehavior = selectedValue)" OptionHelp="@T("Control how the LLM provider for loaded chats is selected and when assistant results are sent to chat.")"/>
<MudPaper Class="pa-3 mb-8 border-dashed border rounded-lg">
app/MindWork AI Studio/Dialogs/SourcePassagesDialog.razor
+35
View File
@@ -0,0 +1,35 @@
@using AIStudio.Tools.RAG
@inherits MSGComponentBase
<MudDialog ContentClass="dialog-content-fit">
<TitleContent>
<FileDialogTitle Title="@this.fileName" Path="@this.path"/>
</TitleContent>
<DialogContent>
<MudText Typo="Typo.body2" Class="mb-4">
@T("This is exactly what the AI received from this source for its answer, after AI Studio removed any suspicious instructions.")
</MudText>
<div class="source-passages">
@* Numbers are enough here: the bar above the passage names its page or part. *@
@if (this.passagesInDocumentOrder.Count > 1)
{
<MudList T="SourcePassage" @bind-SelectedValue="@this.selectedPassage" SelectionMode="SelectionMode.SingleSelection" Color="Color.Primary" Dense="@true" Class="source-passages-menu">
@for (var index = 0; index < this.passagesInDocumentOrder.Count; index++)
{
<MudListItem T="SourcePassage" Value="@this.passagesInDocumentOrder[index]" Text="@($"{index + 1}")"/>
}
</MudList>
}
<div class="source-passages-content dialog-fit">
<SourcePassageView Passage="@this.selectedPassage" Document="@this.Document" MaxHeight="@ContentViewer.NO_HEIGHT_LIMIT"/>
</div>
</div>
</DialogContent>
<DialogActions>
<MudButton OnClick="@this.Close" Variant="Variant.Filled" Color="Color.Primary">
@T("Close")
</MudButton>
</DialogActions>
</MudDialog>
app/MindWork AI Studio/Dialogs/SourcePassagesDialog.razor.cs
+93
View File
@@ -0,0 +1,93 @@
using AIStudio.Components;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.RAG;
using Microsoft.AspNetCore.Components;
namespace AIStudio.Dialogs;
/// <summary>
/// Shows the passages the AI received from one file, so the user can check what an answer rests on.
/// </summary>
/// <remarks>
/// Opened from the list of sources below an answer, and meant for every other place which hands
/// passages of data sources to the AI and wants to show them, such as an assistant. The file is
/// named once, above everything else; its passages are numbered at the side when there are several,
/// in the order of the document, and the one picked is shown in full.<br/><br/>
/// Only the text of a passage scrolls: the dialog fits the window, see dialog-content-fit in app.css,
/// so that it never has to scroll as well.
/// </remarks>
public partial class SourcePassagesDialog : MSGComponentBase
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(SourcePassagesDialog).Namespace, nameof(SourcePassagesDialog));
[CascadingParameter]
private IMudDialogInstance MudDialog { get; set; } = null!;
/// <summary>
/// The passages to show, in the order the AI received them; they are shown in the order of the document.
/// </summary>
[Parameter]
public IReadOnlyList<SourcePassage> Passages { get; set; } = [];
/// <summary>
/// The file the passages are from, or null when it is no file this computer can open.
/// </summary>
[Parameter]
public SourceDocumentLocation? Document { get; set; }
/// <summary>
/// The passages in the order of the document. A passage which does not say where it is comes
/// last, in the order the AI received it.
/// </summary>
/// <remarks>
/// The order the AI received them in tells the user nothing: the AI gets no weight with them,
/// and each one counts as much as every other. The order of the document is what lets the user
/// read them alongside it.
/// </remarks>
private List<SourcePassage> passagesInDocumentOrder = [];
private SourcePassage selectedPassage = new();
private string fileName = string.Empty;
private string path = string.Empty;
#region Overrides of ComponentBase
protected override async Task OnInitializedAsync()
{
this.passagesInDocumentOrder = [..this.Passages.OrderBy(passage => passage.GetLocationRange()?.First ?? int.MaxValue)];
this.selectedPassage = this.passagesInDocumentOrder.Count > 0 ? this.passagesInDocumentOrder[0] : new();
this.fileName = GetFileName(this.Passages);
this.path = this.Document?.Path ?? this.selectedPassage.Path;
await base.OnInitializedAsync();
}
#endregion
/// <summary>
/// The name of the file the passages are from, without their pages: those are what the dialog lets the user pick.
/// </summary>
private static string GetFileName(IReadOnlyList<SourcePassage> passages) => passages is [{ DataSourceName.Length: > 0 } firstPassage, ..] ? firstPassage.DataSourceName : TB("Passages");
private void Close() => this.MudDialog.Cancel();
/// <summary>
/// Shows the passages the AI received from a file.
/// </summary>
/// <param name="dialogService">The dialog service.</param>
/// <param name="passages">The passages to show, in any order: they are shown in the order of the document.</param>
/// <param name="document">The file the passages are from, or null when it is no file this computer can open.</param>
public static async Task OpenDialogAsync(IDialogService dialogService, IReadOnlyList<SourcePassage> passages, SourceDocumentLocation? document = null)
{
var dialogParameters = new DialogParameters<SourcePassagesDialog>
{
{ x => x.Passages, passages },
{ x => x.Document, document },
};
var dialogReference = await dialogService.ShowAsync<SourcePassagesDialog>(GetFileName(passages), dialogParameters, DialogOptions.FULLSCREEN);
await dialogReference.Result;
}
}
app/MindWork AI Studio/Pages/Information.razor
+21 -37
View File
@@ -20,44 +20,28 @@
<MudListItem T="string" Icon="@Icons.Material.Outlined.Build" Text="@this.VersionDotnetSdk"/>
<MudListItem T="string" Icon="@Icons.Material.Outlined.Memory" Text="@this.VersionDotnetRuntime"/>
<MudListItem T="string" Icon="@Icons.Material.Outlined.Build" Text="@this.VersionRust"/>
<MudListItem T="string" Icon="@Icons.Material.Outlined.Storage">
<MudText Typo="Typo.body1">
@this.DatabaseHeaderText(this.vectorStoreSection)
</MudText>
<MudCollapse Expanded="@this.vectorStoreSection.ShowDetails">
<MudText Typo="Typo.body1" Class="mt-2 mb-2">
@foreach (var item in this.BuildDatabaseInfoItems(this.vectorStoreSection))
{
<ConfigInfoRow Item="@item"/>
}
@foreach (var database in this.databaseSections)
{
<MudListItem T="string" Icon="@database.Icon">
<MudText Typo="Typo.body1">
@this.DatabaseHeaderText(database)
</MudText>
</MudCollapse>
<MudButton StartIcon="@(this.vectorStoreSection.ShowDetails ? Icons.Material.Filled.ExpandLess : Icons.Material.Filled.ExpandMore)"
Size="Size.Small"
Variant="Variant.Text"
OnClick="@(() => this.ToggleDatabaseDetails(this.vectorStoreSection))">
@(this.vectorStoreSection.ShowDetails ? T("Hide Details") : T("Show Details"))
</MudButton>
</MudListItem>
<MudListItem T="string" Icon="@Icons.Material.Outlined.ManageSearch">
<MudText Typo="Typo.body1">
@this.DatabaseHeaderText(this.indexStoreSection)
</MudText>
<MudCollapse Expanded="@this.indexStoreSection.ShowDetails">
<MudText Typo="Typo.body1" Class="mt-2 mb-2">
@foreach (var item in this.BuildDatabaseInfoItems(this.indexStoreSection))
{
<ConfigInfoRow Item="@item"/>
}
</MudText>
</MudCollapse>
<MudButton StartIcon="@(this.indexStoreSection.ShowDetails ? Icons.Material.Filled.ExpandLess : Icons.Material.Filled.ExpandMore)"
Size="Size.Small"
Variant="Variant.Text"
OnClick="@(() => this.ToggleDatabaseDetails(this.indexStoreSection))">
@(this.indexStoreSection.ShowDetails ? T("Hide Details") : T("Show Details"))
</MudButton>
</MudListItem>
<MudCollapse Expanded="@database.ShowDetails">
<MudText Typo="Typo.body1" Class="mt-2 mb-2">
@foreach (var item in this.BuildDatabaseInfoItems(database))
{
<ConfigInfoRow Item="@item"/>
}
</MudText>
</MudCollapse>
<MudButton StartIcon="@(database.ShowDetails ? Icons.Material.Filled.ExpandLess : Icons.Material.Filled.ExpandMore)"
Size="Size.Small"
Variant="Variant.Text"
OnClick="@(() => this.ToggleDatabaseDetails(database))">
@(database.ShowDetails ? T("Hide Details") : T("Show Details"))
</MudButton>
</MudListItem>
}
<MudListItem T="string" Icon="@Icons.Material.Outlined.DocumentScanner" Text="@this.VersionPdfium"/>
<MudListItem T="string" Icon="@Icons.Material.Outlined.Article" Text="@this.versionPandoc"/>
<MudListItem T="string" Icon="@Icons.Material.Outlined.Widgets" Text="@MudBlazorVersion"/>
app/MindWork AI Studio/Pages/Information.razor.cs
+23 -15
View File
@@ -102,7 +102,7 @@ public partial class Information : MSGComponentBase
/// </summary>
/// <remarks>
/// The vector store names the version from the build metadata, which is what this page always
/// showed. The index store names the one its client read from the running database instead:
/// showed. The SQLite stores name the one their client read from the running database instead:
/// there, which SQLite build actually got loaded is the whole point of showing it.
/// </remarks>
private string DatabaseHeaderText(DatabaseSection section)
@@ -110,6 +110,7 @@ public partial class Information : MSGComponentBase
var nameLabel = section.Role switch
{
DatabaseRole.VECTOR_STORE => T("Vector database"),
DatabaseRole.PROVENANCE_STORE => T("Provenance database"),
_ => T("Index database"),
};
@@ -163,13 +164,16 @@ public partial class Information : MSGComponentBase
/// Everything one database block on this page needs to show itself.
/// </summary>
/// <remarks>
/// Both blocks work the same way, so they share their state and their methods and differ only
/// in their role. Whoever adds a third database adds one field here, not another set of methods.
/// The blocks work the same way, so they share their state, their methods, and their markup, and
/// differ only in their role and icon. Whoever adds another database adds one entry to
/// databaseSections, not another set of methods.
/// </remarks>
private sealed class DatabaseSection(DatabaseRole role)
private sealed class DatabaseSection(DatabaseRole role, string icon)
{
public DatabaseRole Role => role;
public string Icon => icon;
public DatabaseClient? Client { get; set; }
public bool ShowDetails { get; set; }
@@ -179,8 +183,15 @@ public partial class Information : MSGComponentBase
public CancellationTokenSource? RefreshCancellationTokenSource { get; set; }
}
private readonly DatabaseSection vectorStoreSection = new(DatabaseRole.VECTOR_STORE);
private readonly DatabaseSection indexStoreSection = new(DatabaseRole.INDEX_STORE);
/// <summary>
/// The database blocks, in the order this page shows them.
/// </summary>
private readonly IReadOnlyList<DatabaseSection> databaseSections =
[
new(DatabaseRole.VECTOR_STORE, Icons.Material.Outlined.Storage),
new(DatabaseRole.INDEX_STORE, Icons.Material.Outlined.ManageSearch),
new(DatabaseRole.PROVENANCE_STORE, Icons.Material.Outlined.Source),
];
private bool HasAnyActiveEnvironment => this.enterpriseEnvironments.Any(e => e.IsActive);
@@ -229,14 +240,11 @@ public partial class Information : MSGComponentBase
// The index store goes first: the vector store asks it for the number of stored vectors,
// and this way that client is already cached when it does.
await this.RefreshDatabaseInfo(this.indexStoreSection, CancellationToken.None);
await this.RefreshDatabaseInfo(this.vectorStoreSection, CancellationToken.None);
foreach (var section in this.databaseSections.OrderByDescending(section => section.Role is DatabaseRole.INDEX_STORE))
await this.RefreshDatabaseInfo(section, CancellationToken.None);
if (this.indexStoreSection.Client?.Status is DatabaseClientStatus.STARTING)
this.StartShortDatabaseRefreshLoop(this.indexStoreSection);
if (this.vectorStoreSection.Client?.Status is DatabaseClientStatus.STARTING)
this.StartShortDatabaseRefreshLoop(this.vectorStoreSection);
foreach (var section in this.databaseSections.Where(section => section.Client?.Status is DatabaseClientStatus.STARTING))
this.StartShortDatabaseRefreshLoop(section);
// Determine the Pandoc version may take some time, so we start it here
// without waiting for the result:
@@ -588,8 +596,8 @@ public partial class Information : MSGComponentBase
protected override void DisposeResources()
{
this.CancelDatabaseRefreshLoop(this.vectorStoreSection);
this.CancelDatabaseRefreshLoop(this.indexStoreSection);
foreach (var section in this.databaseSections)
this.CancelDatabaseRefreshLoop(section);
base.DisposeResources();
}
app/MindWork AI Studio/Plugins/configuration/plugin.lua
+9
View File
@@ -663,6 +663,14 @@ CONFIG["SETTINGS"] = {}
-- Allowed values are: NO_DATA_SOURCES, APPLY_STANDARD_CHAT_DATA_SOURCE_OPTIONS
-- CONFIG["SETTINGS"]["DataChat.SendToChatDataSourceBehavior"] = "APPLY_STANDARD_CHAT_DATA_SOURCE_OPTIONS"
--
-- Configure whether a chat reads an attached file again when the file changed since it was sent.
-- By default, every later message of a chat sends the AI the content a file had when it was first
-- sent, also when the file changed or is gone by now, and users can look up what the AI received.
-- When enabled, the AI receives the new content of a changed file with the next message. A file
-- which is gone keeps the content it was sent with either way.
-- Allowed values are: true, false
-- CONFIG["SETTINGS"]["DataChat.ReadChangedAttachmentsAgain"] = true
--
-- Allow users to change any configured chat default locally.
-- Allowed values are: true, false
-- CONFIG["SETTINGS"]["DataChat.PreselectOptions.AllowUserOverride"] = true
@@ -675,6 +683,7 @@ CONFIG["SETTINGS"] = {}
-- CONFIG["SETTINGS"]["DataChat.PreselectedDataSourcesAutomaticValidation.AllowUserOverride"] = true
-- CONFIG["SETTINGS"]["DataChat.PreselectedDataSourceIds.AllowUserOverride"] = true
-- CONFIG["SETTINGS"]["DataChat.SendToChatDataSourceBehavior.AllowUserOverride"] = true
-- CONFIG["SETTINGS"]["DataChat.ReadChangedAttachmentsAgain.AllowUserOverride"] = true
-- Configure defaults for the Slide Assistant.
-- Preselection must be enabled for the template, the picture library, and the other defaults
app/MindWork AI Studio/Plugins/languages/de-de-43065dbc-78d0-45b7-92be-f14c2926e2dc/plugin.lua
+226 -52
View File
@@ -3660,6 +3660,9 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T3378120620"] = "Wir konnten Model
-- Your data sources could not be used. This answer was created without them.
UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T373499115"] = "Ihre Datenquellen konnten nicht verwendet werden. Diese Antwort wurde ohne sie erstellt."
-- AI Studio could not store what the AI received separately from your chat. It remains in the chat itself, so you can still read it.
UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T713722424"] = "AI Studio konnte die Inhalte, die die KI erhalten hat, nicht getrennt von Ihrem Chat speichern. Diese bleiben im Chat selbst, sodass Sie sie weiterhin lesen können."
-- The local image file does not exist. Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T255679918"] = "Die lokale Bilddatei existiert nicht. Das Bild wird übersprungen."
@@ -4062,6 +4065,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T4081853237"] = "T
-- Configure Keyboard Shortcut
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T636303786"] = "Tastaturkurzbefehl konfigurieren"
-- Image
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONTENTVIEWER::T1494001562"] = "Bild"
-- Plain text
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONTENTVIEWER::T3086748382"] = "Unformatiert"
-- Formatted
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONTENTVIEWER::T3618541955"] = "Formatiert"
-- Keep the existing chat
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1274903302"] = "Vorhandenen Chat behalten"
@@ -4269,6 +4281,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3655837348"] = "Sichern Sie
-- Files which AI Studio needs only for a moment and deletes again.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T366353671"] = "Dateien, die AI Studio nur kurz benötigt und anschließend wieder löscht."
-- What the AI received in your chats besides your messages, such as the content of attached files, the results of tools, and passages of your data sources. It is part of your chats.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3806894665"] = "Was die KI in Ihren Chats zusätzlich zu Ihren Nachrichten erhalten hat, z. B. den Inhalt angehängter Dateien, die Ergebnisse von Werkzeugen und Textstellen aus Ihren Datenquellen. Diese Inhalte sind Teil Ihrer Chats."
-- Plugins which come with AI Studio, such as the language packs. AI Studio brings them back by itself.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3877928356"] = "Plugins, die mit AI Studio mitgeliefert werden, etwa Sprachpakete. AI Studio stellt sie automatisch wieder her."
@@ -4710,26 +4725,26 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERSETTINGSACTION::T244391
-- Change what this tile opens
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERSETTINGSACTION::T4272203100"] = "Ändern, was diese Kachel öffnet"
-- Could not open the file location.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1118835751"] = "Der Speicherort der Datei konnte nicht geöffnet werden."
-- Copy the path: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T1140628382"] = "Pfad kopieren: {0}"
-- Could not open the file: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1290686840"] = "Die Datei konnte nicht geöffnet werden: {0}"
-- Show this file in the file manager of your system
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T1587653504"] = "Diese Datei im Dateimanager Ihres Systems anzeigen"
-- Open with the default program
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1332010305"] = "Mit dem Standardprogramm öffnen"
-- Copy the address: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T1663694701"] = "Adresse kopieren: {0}"
-- Could not open the file location: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1455637941"] = "Der Speicherort der Datei konnte nicht geöffnet werden: {0}"
-- AI Studio cannot find the file {0} on this computer. It may have been moved, renamed, or deleted, or it is on another computer, such as the server of a data source.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T21766676"] = "AI Studio kann die Datei {0} auf diesem Rechner nicht finden. Möglicherweise wurde sie verschoben, umbenannt oder gelöscht, oder sie liegt auf einem anderen Rechner, z. B. auf dem Server einer Datenquelle."
-- Show in the file manager
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T3007835369"] = "Im Dateimanager anzeigen"
-- Open the file at page {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T3524349202"] = "Datei auf Seite {0} öffnen"
-- Unknown error
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T3461425987"] = "Unbekannter Fehler"
-- Open this web page in your browser
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T3596972550"] = "Diese Webseite im Browser öffnen"
-- Could not open the file.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T362546438"] = "Die Datei konnte nicht geöffnet werden."
-- Open the file in the program your system uses for it
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T4218155268"] = "Datei in dem Programm öffnen, das Ihr System dafür verwendet"
-- Saving '{0}'...
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTRUNNINGMESSAGE::T1063135893"] = "„{0}“ wird gespeichert …"
@@ -5853,26 +5868,38 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T78
-- Are you sure you want to delete the transcription provider '{0}'?
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T789660305"] = "Möchten Sie den Anbieter für Transkriptionen „{0}“ wirklich löschen?"
-- Could not open the file location.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1118835751"] = "Der Speicherort der Datei konnte nicht geöffnet werden."
-- Transcript of an audio file
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T1797766177"] = "Transkript einer Audiodatei"
-- Could not open the file location: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1455637941"] = "Der Speicherort der Datei konnte nicht geöffnet werden: {0}"
-- Transcript of a video file
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T3408352114"] = "Transkript einer Videodatei"
-- Show this file in the file manager of your system
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1587653504"] = "Diese Datei im Dateimanager Ihres Systems anzeigen"
-- At the moment, AI Studio cannot read the text of this passage that the AI received, e.g., while AI Studio is still starting.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T3678367303"] = "Derzeit kann AI Studio den Text dieser Textstelle, den die KI erhalten hat, nicht lesen, z. B. solange AI Studio noch startet."
-- Opens this document in the program your system uses for it
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3169582185"] = "Öffnet dieses Dokument in dem Programm, das Ihr System dafür verwendet."
-- AI Studio no longer has the text of this passage that the AI received. Only where it was found is still known.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T3950319181"] = "AI Studio liegt der Text dieser Textstelle, den die KI erhalten hat, nicht mehr vor. Bekannt ist nur noch, wo sie gefunden wurde."
-- Unknown error
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3461425987"] = "Unbekannter Fehler"
-- Text the AI received
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T46215024"] = "Text, den die KI erhalten hat"
-- Could not open the document.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3570758363"] = "Das Dokument konnte nicht geöffnet werden."
-- Try again
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T557748474"] = "Erneut versuchen"
-- Could not open the document: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T945417289"] = "Dokument konnte nicht geöffnet werden: {0}"
-- Surrounding content
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T884772048"] = "Umgebender Text"
-- Transcript
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T92473707"] = "Transkript"
-- An image matched here. The AI was told about it and where to find it, but it did not receive the image itself.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T989893925"] = "Hier wurde ein passendes Bild gefunden. Die KI hat davon erfahren und weiß, wo es zu finden ist, hat das Bild selbst aber nicht erhalten."
-- Show the {0} passages the AI received from this source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T2556408625"] = "Die {0} Textstellen anzeigen, die die KI aus dieser Quelle erhalten hat"
-- Show the passage the AI received from this source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3350418006"] = "Die Textstelle anzeigen, die die KI aus dieser Quelle erhalten hat"
-- Copy {0} to the clipboard
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TEXTINFOLINE::T2206391442"] = "Kopiere {0} in die Zwischenablage"
@@ -7833,21 +7860,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DIRECTCHATLAUNCHERSETTINGSDIALOG::T900713019
-- Please wait while we load the content of your file. Depending on the file type and size, this may take a moment.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1205126512"] = "Bitte warten Sie, während wir den Inhalt Ihrer Datei laden. Je nach Dateityp und -größe kann dies einen Moment dauern."
-- Markdown View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1373123357"] = "Markdown-Ansicht"
-- This is the text the AI received from your file when you first sent it. Later changes to the file do not reach the AI in this chat.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T160563952"] = "Dies ist der Text, den die KI aus Ihrer Datei erhalten hat, als Sie die Datei zum ersten Mal gesendet haben. Spätere Änderungen an der Datei erreichen die KI in diesem Chat nicht."
-- Load file
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2129302565"] = "Datei laden"
-- Image View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2199753423"] = "Bildansicht"
-- Your document is large, so we show you only its beginning. We hide the remaining {0:N0} characters here. Rest assured: the AI still receives your entire document.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2468296835"] = "Ihr Dokument ist groß, daher zeigen wir Ihnen hier nur den Anfang. Die verbleibenden {0:N0} Zeichen werden ausgeblendet. Keine Sorge: Die KI erhält trotzdem Ihr gesamtes Dokument."
-- You can drag another file into this window. We attach it right away and show it here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2822249202"] = "Sie können eine weitere Datei in dieses Fenster ziehen. Wir hängen sie sofort an und zeigen sie Ihnen hier."
-- Your document is large, so we show you only its beginning. We hide the remaining {0} characters here. Rest assured: the AI still receives your entire document.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2853647837"] = "Ihr Dokument ist groß, daher zeigen wir Ihnen hier nur den Anfang. Die verbleibenden {0} Zeichen werden ausgeblendet. Keine Sorge: Die KI erhält trotzdem Ihr gesamtes Dokument."
-- The file is no longer in its original location. Below is the text the AI received from it when you sent it.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3044324335"] = "Die Datei befindet sich nicht mehr an ihrem ursprünglichen Speicherort. Unten sehen Sie den Text, den die KI erhalten hat, als Sie die Datei gesendet haben."
-- See how we load your file. Review the content before we process it further.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3271853346"] = "So wird Ihre Datei geladen. Überprüfen Sie den Inhalt, bevor wir ihn weiterverarbeiten."
@@ -7857,18 +7884,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3448155331"] = "Schlie
-- Loaded Content
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3529911749"] = "Geladener Inhalt"
-- Simple View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T428485200"] = "Einfache Ansicht"
-- The file could not be found. It may have been moved, renamed, or deleted, or it is otherwise inaccessible.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T430625309"] = "Die Datei konnte nicht gefunden werden. Möglicherweise wurde sie verschoben, umbenannt oder gelöscht oder ist aus einem anderen Grund nicht zugänglich."
-- Text the AI received
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T46215024"] = "Text, den die KI erhalten hat"
-- This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T652739927"] = "Dies ist der Inhalt, den wir aus Ihrer Datei geladen haben – einschließlich Überschriften, Listen und Formatierung. Verwenden Sie dies, um zu überprüfen, ob Ihre Datei wie erwartet geladen wird."
-- File Path
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T729508546"] = "Dateipfad"
-- The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T973777830"] = "Die angegebene Datei konnte nicht gefunden werden. Die Datei wurde möglicherweise verschoben, gelöscht, umbenannt oder ist anderweitig nicht zugänglich."
-- Embedding Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T1427271797"] = "Name der Einbettung"
@@ -8817,6 +8841,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3160778981"] = "Vo
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3448155331"] = "Schließen"
-- Show the text the AI received from this file.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3884104588"] = "Den Text anzeigen, den die KI aus dieser Datei erhalten hat."
-- Your attached files
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3909191077"] = "Ihre angehängten Dateien"
@@ -9207,6 +9234,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T82
-- Restore default
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T913168756"] = "Standard wiederherstellen"
-- Files are sent as they were first read
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1320692831"] = "Dateien werden so gesendet, wie sie zuerst eingelesen wurden"
-- Preselect one of your chat templates?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1402022556"] = "Eine ihrer Chat-Vorlagen vorab auswählen?"
@@ -9234,6 +9264,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2510376349"]
-- Control how the LLM provider for added chats is selected.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T263621180"] = "Steuern Sie, wie der LLM-Anbieter für hinzugefügte Chats ausgewählt wird."
-- Changed files are read again for the next message
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T264953820"] = "Geänderte Dateien werden für die nächste Nachricht erneut eingelesen"
-- Provider selection when loading a chat and sending assistant results to chat
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2868379953"] = "Anbieterauswahl beim Laden eines Chats und beim Senden von Assistentenergebnissen in den Chat"
@@ -9243,6 +9276,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2913693228"]
-- Do you want to use any shortcut to send your input?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2936560092"] = "Möchten Sie eine Tastenkombination verwenden, um Ihre Eingabe zu senden?"
-- Read changed attachments again?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3079510748"] = "Geänderte Anhänge erneut einlesen?"
-- Would you like to set one of your chat templates as the default for chats?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3234927721"] = "Möchten Sie eine Ihrer Chat-Vorlagen als Standard für alle Chats festlegen?"
@@ -9267,6 +9303,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3755993611"]
-- Do you want to apply the default data source options when sending assistant results to chat?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4033153439"] = "Möchten Sie die Standardoptionen für Datenquellen verwenden, wenn die Ergebnisse des Assistenten an den Chat gesendet werden?"
-- When enabled, AI Studio reads an attached file again when it changed since it was sent, and the AI receives the new content with the next message. Viewing the attachment then shows the content that was sent last. When disabled, the AI keeps receiving the content a file had when it was first sent, also when the file changed or is gone.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4060508809"] = "Wenn diese Option aktiviert ist, liest AI Studio eine angehängte Datei erneut ein, wenn sie seit dem Senden geändert wurde. Mit der nächsten Nachricht erhält die KI den neuen Inhalt. Die Ansicht des Anhangs zeigt dann den zuletzt gesendeten Inhalt. Ist die Option deaktiviert, erhält die KI weiterhin den Inhalt, den die Datei beim ersten Senden hatte – auch wenn sie sich inzwischen geändert hat oder nicht mehr vorhanden ist."
-- When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T477675197"] = "Wenn aktiviert, können Sie Chat-Optionen im Voraus auswählen. Das kann nützlich sein, wenn Sie einen bestimmten Anbieter bevorzugen."
@@ -10338,6 +10377,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SINGLEINPUTDIALOG::T4030229154"] = "Ihre Ein
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SINGLEINPUTDIALOG::T900713019"] = "Abbrechen"
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SOURCEPASSAGESDIALOG::T3448155331"] = "Schließen"
-- This is exactly what the AI received from this source for its answer, after AI Studio removed any suspicious instructions.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SOURCEPASSAGESDIALOG::T406940705"] = "Genau das hat die KI für ihre Antwort aus dieser Quelle erhalten, nachdem AI Studio etwaige verdächtige Anweisungen entfernt hatte."
-- Passages
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SOURCEPASSAGESDIALOG::T4291613392"] = "Textstellen"
-- Hugging Face Inference Provider
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1085481431"] = "Hugging Face-Inferenzanbieter"
@@ -11070,6 +11118,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1421513382"] = "Diese Bibliothek
-- Trademarks & Brand Assets
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1421823619"] = "Marken und Markenressourcen"
-- Provenance database
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1496928875"] = "Herkunftsdatenbank"
-- Copies the allowed host pattern to the clipboard
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1513592659"] = "Kopiert das zulässige Hostmuster in die Zwischenablage"
@@ -12634,7 +12685,7 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABACKUP::DATARESTOREPREPARER::T3812849494"]
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1093747001"] = "Grund"
-- Starting
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1233211769"] = "Starten"
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T1233211769"] = "Wird gestartet"
-- unknown
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::INDEXSTORE::NOINDEXSTORECLIENT::T2608177081"] = "Unbekannt"
@@ -12723,11 +12774,74 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T3662391977"] = "
-- Status
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T6222351"] = "Status"
-- Reason
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T1093747001"] = "Grund"
-- Starting
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T1233211769"] = "Wird gestartet"
-- unknown
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T2608177081"] = "Unbekannt"
-- Unavailable
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T3662391977"] = "Nicht verfügbar"
-- Process architecture
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T563161633"] = "Prozessarchitektur"
-- Status
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T6222351"] = "Status"
-- Native library
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T634426545"] = "Native Bibliothek"
-- no migration applied
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T1014567907"] = "Keine Migration angewendet"
-- Storage size
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T1230141403"] = "Speichergröße"
-- Stored items across all chats
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T1569215203"] = "Gespeicherte Inhalte in allen Chats"
-- {0} ({1} applied)
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2286846332"] = "{0} ({1} angewendet)"
-- Journal mode
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2411639995"] = "Journalmodus"
-- Attached files
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T246258678"] = "Angehängte Dateien"
-- Mails
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2534642695"] = "E-Mails"
-- unknown
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2608177081"] = "Unbekannt"
-- Reported version
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T3556099842"] = "Gemeldete Version"
-- Schema version
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T424290830"] = "Schemaversion"
-- Passages of data sources
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T686479917"] = "Textstellen aus Datenquellen"
-- Web pages
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T748543057"] = "Webseiten"
-- Chats with stored items
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T826028599"] = "Chats mit gespeicherten Inhalten"
-- {0} ({1} applied, {2} pending)
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T983802795"] = "{0} ({1} angewendet, {2} ausstehend)"
-- Reason
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::NOVECTORSTORECLIENT::T1093747001"] = "Grund"
-- Starting
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::NOVECTORSTORECLIENT::T1233211769"] = "Starten"
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::NOVECTORSTORECLIENT::T1233211769"] = "Wird gestartet"
-- Unavailable
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::NOVECTORSTORECLIENT::T3662391977"] = "Nicht verfügbar"
@@ -14946,6 +15060,39 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T304
-- AI source selection with AI retrieval context validation
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T3775725978"] = "KI-basierte Datenquellen-Auswahl mit Validierung des Abrufkontexts"
-- Pages {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T1095604049"] = "Seiten {0}"
-- {0}: rows {1}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T1424805945"] = "{0}: Zeilen {1}"
-- {0}: row {1}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T1620414946"] = "{0}: Zeile {1}"
-- Rows {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T2222697074"] = "Zeilen {0}"
-- Line {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T278321977"] = "Zeile {0}"
-- {0} passages
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T344316214"] = "{0} Textstellen"
-- Lines {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T3536135902"] = "Zeilen {0}"
-- Part {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T3742022998"] = "Teil {0}"
-- Row {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T4122835077"] = "Zeile {0}"
-- Page {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T4127287940"] = "Seite {0}"
-- Parts {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T826267883"] = "Teile {0}"
-- Text
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text"
@@ -15327,12 +15474,6 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T26552229
-- Completed
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T3968379570"] = "Abgeschlossen"
-- Chunk {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T2544251224"] = "Block {0}"
-- Page {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Seite {0}"
-- The following data sources selected by the assistant chat launcher are currently unavailable or not permitted for the selected provider: {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DIRECTCHATSERVICE::T103791004"] = "Die folgenden vom Chat-Schnellstart-Assistenten ausgewählten Datenquellen sind derzeit nicht verfügbar oder für den ausgewählten Anbieter nicht zugelassen: {0}"
@@ -15762,6 +15903,21 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::UPDATESERVICE::T1064148123"] = "Die
-- Failed to install update automatically. Please try again manually.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::UPDATESERVICE::T3709709946"] = "Fehler bei der automatischen Installation des Updates. Bitte versuchen Sie es manuell erneut."
-- Could not open the file location.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T1118835751"] = "Der Speicherort der Datei konnte nicht geöffnet werden."
-- Could not open the file location: {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T1455637941"] = "Der Speicherort der Datei konnte nicht geöffnet werden: {0}"
-- Unknown error
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T3461425987"] = "Unbekannter Fehler"
-- Could not open the document.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T3570758363"] = "Das Dokument konnte nicht geöffnet werden."
-- Could not open the document: {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T945417289"] = "Das Dokument konnte nicht geöffnet werden: {0}"
-- Sources
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEEXTENSIONS::T2730980305"] = "Quellen"
@@ -15858,6 +16014,24 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS:
-- To use this tool, add a mailbox to your data sources first.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLCONFIGURATION::T1547505353"] = "Um dieses Werkzeug zu verwenden, fügen Sie zuerst ein Postfach zu Ihren Datenquellen hinzu."
-- To
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T1106613828"] = "An"
-- Text from the attachment
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T1262266390"] = "Text aus dem Anhang"
-- Received
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T1559762898"] = "Empfangen"
-- Subject
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T2692293871"] = "Betreff"
-- From
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T410269077"] = "Von"
-- Attachments
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T498946337"] = "Anhänge"
-- Read Mail
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::READMAILTOOL::T2299936968"] = "E-Mails lesen"
app/MindWork AI Studio/Plugins/languages/en-us-97dfb1ba-50c4-4440-8dfa-6575daf543c8/plugin.lua
+224 -50
View File
@@ -3660,6 +3660,9 @@ UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T3378120620"] = "We could load mod
-- Your data sources could not be used. This answer was created without them.
UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T373499115"] = "Your data sources could not be used. This answer was created without them."
-- AI Studio could not store what the AI received separately from your chat. It remains in the chat itself, so you can still read it.
UI_TEXT_CONTENT["AISTUDIO::CHAT::CONTENTTEXT::T713722424"] = "AI Studio could not store what the AI received separately from your chat. It remains in the chat itself, so you can still read it."
-- The local image file does not exist. Skipping the image.
UI_TEXT_CONTENT["AISTUDIO::CHAT::IIMAGESOURCEEXTENSIONS::T255679918"] = "The local image file does not exist. Skipping the image."
@@ -4062,6 +4065,15 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T4081853237"] = "C
-- Configure Keyboard Shortcut
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONFIGURATIONSHORTCUT::T636303786"] = "Configure Keyboard Shortcut"
-- Image
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONTENTVIEWER::T1494001562"] = "Image"
-- Plain text
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONTENTVIEWER::T3086748382"] = "Plain text"
-- Formatted
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::CONTENTVIEWER::T3618541955"] = "Formatted"
-- Keep the existing chat
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATABACKUPPANEL::T1274903302"] = "Keep the existing chat"
@@ -4269,6 +4281,9 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3655837348"] = "Back up the
-- Files which AI Studio needs only for a moment and deletes again.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T366353671"] = "Files which AI Studio needs only for a moment and deletes again."
-- What the AI received in your chats besides your messages, such as the content of attached files, the results of tools, and passages of your data sources. It is part of your chats.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3806894665"] = "What the AI received in your chats besides your messages, such as the content of attached files, the results of tools, and passages of your data sources. It is part of your chats."
-- Plugins which come with AI Studio, such as the language packs. AI Studio brings them back by itself.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DATAFOLDERS::T3877928356"] = "Plugins which come with AI Studio, such as the language packs. AI Studio brings them back by itself."
@@ -4710,26 +4725,26 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERSETTINGSACTION::T244391
-- Change what this tile opens
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::DIRECTCHATLAUNCHERSETTINGSACTION::T4272203100"] = "Change what this tile opens"
-- Could not open the file location.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1118835751"] = "Could not open the file location."
-- Copy the path: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T1140628382"] = "Copy the path: {0}"
-- Could not open the file: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1290686840"] = "Could not open the file: {0}"
-- Show this file in the file manager of your system
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T1587653504"] = "Show this file in the file manager of your system"
-- Open with the default program
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1332010305"] = "Open with the default program"
-- Copy the address: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T1663694701"] = "Copy the address: {0}"
-- Could not open the file location: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T1455637941"] = "Could not open the file location: {0}"
-- AI Studio cannot find the file {0} on this computer. It may have been moved, renamed, or deleted, or it is on another computer, such as the server of a data source.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T21766676"] = "AI Studio cannot find the file {0} on this computer. It may have been moved, renamed, or deleted, or it is on another computer, such as the server of a data source."
-- Show in the file manager
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T3007835369"] = "Show in the file manager"
-- Open the file at page {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T3524349202"] = "Open the file at page {0}"
-- Unknown error
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T3461425987"] = "Unknown error"
-- Open this web page in your browser
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T3596972550"] = "Open this web page in your browser"
-- Could not open the file.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTACTIONS::T362546438"] = "Could not open the file."
-- Open the file in the program your system uses for it
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEACTIONS::T4218155268"] = "Open the file in the program your system uses for it"
-- Saving '{0}'...
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::FILEEXPORTRUNNINGMESSAGE::T1063135893"] = "Saving '{0}'..."
@@ -5853,26 +5868,38 @@ UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T78
-- Are you sure you want to delete the transcription provider '{0}'?
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SETTINGS::SETTINGSPANELTRANSCRIPTION::T789660305"] = "Are you sure you want to delete the transcription provider '{0}'?"
-- Could not open the file location.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1118835751"] = "Could not open the file location."
-- Transcript of an audio file
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T1797766177"] = "Transcript of an audio file"
-- Could not open the file location: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1455637941"] = "Could not open the file location: {0}"
-- Transcript of a video file
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T3408352114"] = "Transcript of a video file"
-- Show this file in the file manager of your system
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T1587653504"] = "Show this file in the file manager of your system"
-- At the moment, AI Studio cannot read the text of this passage that the AI received, e.g., while AI Studio is still starting.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T3678367303"] = "At the moment, AI Studio cannot read the text of this passage that the AI received, e.g., while AI Studio is still starting."
-- Opens this document in the program your system uses for it
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3169582185"] = "Opens this document in the program your system uses for it"
-- AI Studio no longer has the text of this passage that the AI received. Only where it was found is still known.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T3950319181"] = "AI Studio no longer has the text of this passage that the AI received. Only where it was found is still known."
-- Unknown error
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3461425987"] = "Unknown error"
-- Text the AI received
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T46215024"] = "Text the AI received"
-- Could not open the document.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3570758363"] = "Could not open the document."
-- Try again
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T557748474"] = "Try again"
-- Could not open the document: {0}
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T945417289"] = "Could not open the document: {0}"
-- Surrounding content
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T884772048"] = "Surrounding content"
-- Transcript
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T92473707"] = "Transcript"
-- An image matched here. The AI was told about it and where to find it, but it did not receive the image itself.
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCEPASSAGEVIEW::T989893925"] = "An image matched here. The AI was told about it and where to find it, but it did not receive the image itself."
-- Show the {0} passages the AI received from this source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T2556408625"] = "Show the {0} passages the AI received from this source"
-- Show the passage the AI received from this source
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::SOURCESLIST::T3350418006"] = "Show the passage the AI received from this source"
-- Copy {0} to the clipboard
UI_TEXT_CONTENT["AISTUDIO::COMPONENTS::TEXTINFOLINE::T2206391442"] = "Copy {0} to the clipboard"
@@ -7833,21 +7860,21 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DIRECTCHATLAUNCHERSETTINGSDIALOG::T900713019
-- Please wait while we load the content of your file. Depending on the file type and size, this may take a moment.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1205126512"] = "Please wait while we load the content of your file. Depending on the file type and size, this may take a moment."
-- Markdown View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T1373123357"] = "Markdown View"
-- This is the text the AI received from your file when you first sent it. Later changes to the file do not reach the AI in this chat.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T160563952"] = "This is the text the AI received from your file when you first sent it. Later changes to the file do not reach the AI in this chat."
-- Load file
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2129302565"] = "Load file"
-- Image View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2199753423"] = "Image View"
-- Your document is large, so we show you only its beginning. We hide the remaining {0:N0} characters here. Rest assured: the AI still receives your entire document.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2468296835"] = "Your document is large, so we show you only its beginning. We hide the remaining {0:N0} characters here. Rest assured: the AI still receives your entire document."
-- You can drag another file into this window. We attach it right away and show it here.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2822249202"] = "You can drag another file into this window. We attach it right away and show it here."
-- Your document is large, so we show you only its beginning. We hide the remaining {0} characters here. Rest assured: the AI still receives your entire document.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T2853647837"] = "Your document is large, so we show you only its beginning. We hide the remaining {0} characters here. Rest assured: the AI still receives your entire document."
-- The file is no longer in its original location. Below is the text the AI received from it when you sent it.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3044324335"] = "The file is no longer in its original location. Below is the text the AI received from it when you sent it."
-- See how we load your file. Review the content before we process it further.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3271853346"] = "See how we load your file. Review the content before we process it further."
@@ -7857,18 +7884,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3448155331"] = "Close"
-- Loaded Content
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T3529911749"] = "Loaded Content"
-- Simple View
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T428485200"] = "Simple View"
-- The file could not be found. It may have been moved, renamed, or deleted, or it is otherwise inaccessible.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T430625309"] = "The file could not be found. It may have been moved, renamed, or deleted, or it is otherwise inaccessible."
-- Text the AI received
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T46215024"] = "Text the AI received"
-- This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T652739927"] = "This is the content we loaded from your file — including headings, lists, and formatting. Use this to verify your file loads as expected."
-- File Path
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T729508546"] = "File Path"
-- The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::DOCUMENTCHECKDIALOG::T973777830"] = "The specified file could not be found. The file have been moved, deleted, renamed, or is otherwise inaccessible."
-- Embedding Name
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::EMBEDDINGMETHODDIALOG::T1427271797"] = "Embedding Name"
@@ -8817,6 +8841,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3160778981"] = "Pr
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3448155331"] = "Close"
-- Show the text the AI received from this file.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3884104588"] = "Show the text the AI received from this file."
-- Your attached files
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::REVIEWATTACHMENTSDIALOG::T3909191077"] = "Your attached files"
@@ -9207,6 +9234,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T82
-- Restore default
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGBATCHPROCESSING::T913168756"] = "Restore default"
-- Files are sent as they were first read
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1320692831"] = "Files are sent as they were first read"
-- Preselect one of your chat templates?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T1402022556"] = "Preselect one of your chat templates?"
@@ -9234,6 +9264,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2510376349"]
-- Control how the LLM provider for added chats is selected.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T263621180"] = "Control how the LLM provider for added chats is selected."
-- Changed files are read again for the next message
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T264953820"] = "Changed files are read again for the next message"
-- Provider selection when loading a chat and sending assistant results to chat
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2868379953"] = "Provider selection when loading a chat and sending assistant results to chat"
@@ -9243,6 +9276,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2913693228"]
-- Do you want to use any shortcut to send your input?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T2936560092"] = "Do you want to use any shortcut to send your input?"
-- Read changed attachments again?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3079510748"] = "Read changed attachments again?"
-- Would you like to set one of your chat templates as the default for chats?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3234927721"] = "Would you like to set one of your chat templates as the default for chats?"
@@ -9267,6 +9303,9 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T3755993611"]
-- Do you want to apply the default data source options when sending assistant results to chat?
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4033153439"] = "Do you want to apply the default data source options when sending assistant results to chat?"
-- When enabled, AI Studio reads an attached file again when it changed since it was sent, and the AI receives the new content with the next message. Viewing the attachment then shows the content that was sent last. When disabled, the AI keeps receiving the content a file had when it was first sent, also when the file changed or is gone.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T4060508809"] = "When enabled, AI Studio reads an attached file again when it changed since it was sent, and the AI receives the new content with the next message. Viewing the attachment then shows the content that was sent last. When disabled, the AI keeps receiving the content a file had when it was first sent, also when the file changed or is gone."
-- When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SETTINGS::SETTINGSDIALOGCHAT::T477675197"] = "When enabled, you can preselect chat options. This is might be useful when you prefer a specific provider."
@@ -10338,6 +10377,15 @@ UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SINGLEINPUTDIALOG::T4030229154"] = "Your Inp
-- Cancel
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SINGLEINPUTDIALOG::T900713019"] = "Cancel"
-- Close
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SOURCEPASSAGESDIALOG::T3448155331"] = "Close"
-- This is exactly what the AI received from this source for its answer, after AI Studio removed any suspicious instructions.
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SOURCEPASSAGESDIALOG::T406940705"] = "This is exactly what the AI received from this source for its answer, after AI Studio removed any suspicious instructions."
-- Passages
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::SOURCEPASSAGESDIALOG::T4291613392"] = "Passages"
-- Hugging Face Inference Provider
UI_TEXT_CONTENT["AISTUDIO::DIALOGS::TRANSCRIPTIONPROVIDERDIALOG::T1085481431"] = "Hugging Face Inference Provider"
@@ -11070,6 +11118,9 @@ UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1421513382"] = "This library is
-- Trademarks & Brand Assets
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1421823619"] = "Trademarks & Brand Assets"
-- Provenance database
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1496928875"] = "Provenance database"
-- Copies the allowed host pattern to the clipboard
UI_TEXT_CONTENT["AISTUDIO::PAGES::INFORMATION::T1513592659"] = "Copies the allowed host pattern to the clipboard"
@@ -12723,6 +12774,69 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T3662391977"] = "
-- Status
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::NODATABASECLIENT::T6222351"] = "Status"
-- Reason
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T1093747001"] = "Reason"
-- Starting
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T1233211769"] = "Starting"
-- unknown
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T2608177081"] = "unknown"
-- Unavailable
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T3662391977"] = "Unavailable"
-- Process architecture
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T563161633"] = "Process architecture"
-- Status
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T6222351"] = "Status"
-- Native library
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::NOPROVENANCESTORECLIENT::T634426545"] = "Native library"
-- no migration applied
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T1014567907"] = "no migration applied"
-- Storage size
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T1230141403"] = "Storage size"
-- Stored items across all chats
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T1569215203"] = "Stored items across all chats"
-- {0} ({1} applied)
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2286846332"] = "{0} ({1} applied)"
-- Journal mode
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2411639995"] = "Journal mode"
-- Attached files
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T246258678"] = "Attached files"
-- Mails
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2534642695"] = "Mails"
-- unknown
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T2608177081"] = "unknown"
-- Reported version
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T3556099842"] = "Reported version"
-- Schema version
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T424290830"] = "Schema version"
-- Passages of data sources
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T686479917"] = "Passages of data sources"
-- Web pages
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T748543057"] = "Web pages"
-- Chats with stored items
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T826028599"] = "Chats with stored items"
-- {0} ({1} applied, {2} pending)
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::PROVENANCESTORE::SQLITEPROVENANCESTORECLIENTIMPLEMENTATION::T983802795"] = "{0} ({1} applied, {2} pending)"
-- Reason
UI_TEXT_CONTENT["AISTUDIO::TOOLS::DATABASES::VECTORSTORE::NOVECTORSTORECLIENT::T1093747001"] = "Reason"
@@ -14946,6 +15060,39 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T304
-- AI source selection with AI retrieval context validation
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::RAGPROCESSES::AISRCSELWITHRETCTXVAL::T3775725978"] = "AI-based data source selection with AI retrieval context validation"
-- Pages {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T1095604049"] = "Pages {0}"
-- {0}: rows {1}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T1424805945"] = "{0}: rows {1}"
-- {0}: row {1}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T1620414946"] = "{0}: row {1}"
-- Rows {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T2222697074"] = "Rows {0}"
-- Line {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T278321977"] = "Line {0}"
-- {0} passages
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T344316214"] = "{0} passages"
-- Lines {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T3536135902"] = "Lines {0}"
-- Part {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T3742022998"] = "Part {0}"
-- Row {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T4122835077"] = "Row {0}"
-- Page {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T4127287940"] = "Page {0}"
-- Parts {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RAG::SOURCEPASSAGEEXTENSIONS::T826267883"] = "Parts {0}"
-- Text
UI_TEXT_CONTENT["AISTUDIO::TOOLS::RUST::FILETYPES::T1041509726"] = "Text"
@@ -15327,12 +15474,6 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T26552229
-- Completed
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCEEMBEDDINGSTATUS::T3968379570"] = "Completed"
-- Chunk {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T2544251224"] = "Chunk {0}"
-- Page {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DATASOURCELOCALRETRIEVALSERVICE::T4127287940"] = "Page {0}"
-- The following data sources selected by the assistant chat launcher are currently unavailable or not permitted for the selected provider: {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::DIRECTCHATSERVICE::T103791004"] = "The following data sources selected by the assistant chat launcher are currently unavailable or not permitted for the selected provider: {0}"
@@ -15762,6 +15903,21 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::UPDATESERVICE::T1064148123"] = "Fail
-- Failed to install update automatically. Please try again manually.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SERVICES::UPDATESERVICE::T3709709946"] = "Failed to install update automatically. Please try again manually."
-- Could not open the file location.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T1118835751"] = "Could not open the file location."
-- Could not open the file location: {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T1455637941"] = "Could not open the file location: {0}"
-- Unknown error
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T3461425987"] = "Unknown error"
-- Could not open the document.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T3570758363"] = "Could not open the document."
-- Could not open the document: {0}
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEDOCUMENTLOCATIONEXTENSIONS::T945417289"] = "Could not open the document: {0}"
-- Sources
UI_TEXT_CONTENT["AISTUDIO::TOOLS::SOURCEEXTENSIONS::T2730980305"] = "Sources"
@@ -15858,6 +16014,24 @@ UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS:
-- To use this tool, add a mailbox to your data sources first.
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLCONFIGURATION::T1547505353"] = "To use this tool, add a mailbox to your data sources first."
-- To
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T1106613828"] = "To"
-- Text from the attachment
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T1262266390"] = "Text from the attachment"
-- Received
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T1559762898"] = "Received"
-- Subject
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T2692293871"] = "Subject"
-- From
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T410269077"] = "From"
-- Attachments
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::MAILTOOLRESULTS::T498946337"] = "Attachments"
-- Read Mail
UI_TEXT_CONTENT["AISTUDIO::TOOLS::TOOLCALLINGSYSTEM::TOOLCALLINGIMPLEMENTATIONS::MAILBOXES::READMAILTOOL::T2299936968"] = "Read Mail"
app/MindWork AI Studio/Program.cs
+1
View File
@@ -230,6 +230,7 @@ internal sealed class Program
builder.Services.AddHostedService<UpdateService>();
builder.Services.AddHostedService<TemporaryChatService>();
builder.Services.AddHostedService<TranscriptStagingCleanupService>();
builder.Services.AddHostedService<ProvenanceCleanupService>();
builder.Services.AddHostedService<DataRestoreFollowUpService>();
// Compatibility shim, see documentation/compatibility-shims/2026-10-relative-data-paths.md. Remove after 2027-04-14:
app/MindWork AI Studio/Provider/ContentStreamChunk.cs
+11
View File
@@ -1,3 +1,5 @@
using AIStudio.Tools.RAG;
namespace AIStudio.Provider;
/// <summary>
@@ -13,6 +15,15 @@ namespace AIStudio.Provider;
/// <param name="Usage">What the provider said the request cost, where it said anything.</param>
public sealed record ContentStreamChunk(string Content, IList<ISource> Sources, TokenUsage Usage = default)
{
/// <summary>
/// The passages of the sources, as the model received them from a tool.
/// </summary>
/// <remarks>
/// They travel with the sources they belong to, the way the tools hand both over at the end
/// of their conversation, cf. ToolCallingLoop. A provider's own stream brings none.
/// </remarks>
public IReadOnlyList<SourcePassage> Passages { get; init; } = [];
/// <summary>
/// Implicit conversion to string.
/// </summary>
app/MindWork AI Studio/Settings/DataModel/DataChat.cs
+10
View File
@@ -104,4 +104,14 @@ public sealed class DataChat(Expression<Func<Data, DataChat>>? configSelection =
/// Should we show the latest message after loading? When false, we show the first (aka oldest) message.
/// </summary>
public bool ShowLatestMessageAfterLoading { get; set; } = true;
/// <summary>
/// Whether a chat reads an attached file again when the file changed since it was sent.
/// </summary>
/// <remarks>
/// Off by default: every later message of a chat sends the text a file had when it was first
/// sent, so that what the AI received stays the same and can be shown as it was, see
/// FileAttachment.ProvenanceId. A file which is gone keeps its text either way.
/// </remarks>
public bool ReadChangedAttachmentsAgain { get; set; } = ManagedConfiguration.Register(configSelection, n => n.ReadChangedAttachmentsAgain, false);
}
app/MindWork AI Studio/Settings/DataModel/DataSourceERI_V1.cs
+11 -9
View File
@@ -155,7 +155,9 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
if(retrievalResponse is { Successful: true, Data: not null })
{
//
// Next, we have to transform the ERI context back to our generic retrieval context:
// Next, we have to transform the ERI context back to our generic retrieval context.
// The server writes the JSON, and a field it leaves out arrives as null, whatever the
// contract says. Everything after this point counts on texts and lists:
//
var genericRetrievalContexts = new List<IRetrievalContext>(retrievalResponse.Data.Count);
foreach (var eriContext in retrievalResponse.Data)
@@ -167,26 +169,26 @@ public readonly record struct DataSourceERI_V1 : IERIDataSource
{
Path = eriContext.Path ?? string.Empty,
Type = eriContext.ToRetrievalContentType(),
Links = eriContext.Links,
Links = eriContext.Links ?? [],
Category = eriContext.Type.ToRetrievalContentCategory(),
SourceCategory = eriContext.Category ?? string.Empty,
MatchedText = eriContext.MatchedContent,
DataSourceName = eriContext.Name,
SurroundingContent = eriContext.SurroundingContent,
MatchedText = eriContext.MatchedContent ?? string.Empty,
DataSourceName = eriContext.Name ?? string.Empty,
SurroundingContent = eriContext.SurroundingContent ?? [],
});
break;
case ContentType.IMAGE:
genericRetrievalContexts.Add(new RetrievalImageContext
{
Path = eriContext.Path ?? string.Empty,
Type = eriContext.ToRetrievalContentType(),
Links = eriContext.Links,
Source = eriContext.MatchedContent,
Links = eriContext.Links ?? [],
Source = eriContext.MatchedContent ?? string.Empty,
Category = eriContext.Type.ToRetrievalContentCategory(),
SourceCategory = eriContext.Category ?? string.Empty,
SourceType = ContentImageSource.BASE64,
DataSourceName = eriContext.Name,
DataSourceName = eriContext.Name ?? string.Empty,
});
break;
app/MindWork AI Studio/Tools/AIJobs/AIJobService.cs
+12 -6
View File
@@ -305,6 +305,10 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes
token.ThrowIfCancellationRequested();
// Read each attached document once, before the request sends it, see ChatThread.StoreAttachedDocumentsAsync:
await chatThread.StoreAttachedDocumentsAsync(settingsManager.ConfigurationData.Chat.ReadChangedAttachmentsAgain);
token.ThrowIfCancellationRequested();
var lastStreamingEvent = DateTimeOffset.MinValue;
if (!TrySetWaitingForRemote(state, token))
return;
@@ -371,6 +375,13 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes
RemoveEmptyAIResponse(state);
//
// Before the last checkpoint, so that the stored chat refers to what the AI received instead
// of holding it. An answer which was removed for being empty keeps nothing:
//
if (request.ChatThread.Blocks.Any(block => ReferenceEquals(block.Content, aiText)))
await aiText.StoreReceivedContentAsync(request.ChatThread.ChatId);
lock (state.SyncRoot)
{
state.Snapshot = state.Snapshot with
@@ -461,12 +472,7 @@ public sealed class AIJobService(SettingsManager settingsManager, MessageBus mes
if (state.IsCompletionStarted || token.IsCancellationRequested || state.ChatGenerationRequest is null)
return false;
var aiText = state.ChatGenerationRequest.AIText;
aiText.InitialRemoteWait = false;
aiText.IsStreaming = true;
aiText.Text += contentStreamChunk;
aiText.Sources.MergeSources(contentStreamChunk.Sources);
aiText.RecordReportedUsage(contentStreamChunk.Usage, state.ChatGenerationRequest.ProviderSettings.Model.Id, blocksSent);
state.ChatGenerationRequest.AIText.ApplyStreamChunk(contentStreamChunk, state.ChatGenerationRequest.ProviderSettings.Model.Id, blocksSent);
if (state.Snapshot.Status is not AIJobStatus.RUNNING)
{
app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveFormat.cs
+6
View File
@@ -1,3 +1,4 @@
using AIStudio.Tools.Databases.ProvenanceStore;
using AIStudio.Tools.DataBackup;
namespace AIStudio.Tools.ChatArchive;
@@ -21,6 +22,11 @@ public static class ChatArchiveFormat
/// </summary>
public static readonly string TEMPORARY_CHATS_PREFIX = DataBackupFormat.DATA_ROOT + DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.TEMPORARY_CHATS).RelativePath;
/// <summary>
/// Where a backup keeps the provenance store, which holds what the chats received besides their messages.
/// </summary>
public static readonly string PROVENANCE_STORE_ENTRY = DataBackupFormat.DATA_ROOT + DataBackupContents.ENTRIES.First(entry => entry.Kind is DataBackupEntryKind.CHAT_PROVENANCE).RelativePath + SqliteProvenanceStoreClientImplementation.DATABASE_FILENAME;
/// <summary>
/// The file storing the workspace or chat name.
/// </summary>
app/MindWork AI Studio/Tools/ChatArchive/ChatArchiveImporter.cs
+128 -11
View File
@@ -3,6 +3,7 @@ using System.Text.Json;
using AIStudio.Chat;
using AIStudio.Settings;
using AIStudio.Tools.Databases.ProvenanceStore;
using AIStudio.Tools.DataBackup;
using AIStudio.Tools.PluginSystem;
@@ -24,6 +25,11 @@ public static class ChatArchiveImporter
private const int BUFFER_SIZE = 64 * 1024;
/// <summary>
/// What SQLite appends to the name of a database for its write-ahead log.
/// </summary>
private const string WRITE_AHEAD_LOG_SUFFIX = "-wal";
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(ChatArchiveImporter).Namespace, nameof(ChatArchiveImporter));
/// <summary>
@@ -31,6 +37,13 @@ public static class ChatArchiveImporter
/// </summary>
private sealed record BackupEntry(string Name, ZipArchiveEntry Entry);
/// <summary>
/// Where the imported chats get what they received besides their messages from.
/// </summary>
/// <param name="Store">The provenance store of this computer, which receives the entries.</param>
/// <param name="BackupStorePath">The provenance store of the backup, unpacked into a file of its own.</param>
private sealed record ProvenanceImport(ProvenanceStoreClient Store, string BackupStorePath);
/// <summary>
/// What one import did so far.
/// </summary>
@@ -84,22 +97,26 @@ public static class ChatArchiveImporter
/// Adds all chats of a backup to the chat storage.
/// </summary>
/// <param name="archivePath">The backup.</param>
/// <param name="provenanceStore">The provenance store, which receives what the imported chats received besides their messages.</param>
/// <param name="collisionBehavior">What to do with chats which exist already.</param>
/// <param name="progress">Receives the progress.</param>
/// <param name="token">Cancels the import. Chats added before the cancellation are kept.</param>
/// <returns>What the import did.</returns>
public static async Task<ChatArchiveImportResult> ImportAsync(string archivePath, ChatArchiveCollisionBehavior collisionBehavior, IProgress<ChatArchiveProgress>? progress, CancellationToken token)
public static async Task<ChatArchiveImportResult> ImportAsync(string archivePath, ProvenanceStoreClient provenanceStore, ChatArchiveCollisionBehavior collisionBehavior, IProgress<ChatArchiveProgress>? progress, CancellationToken token)
{
var tally = new ImportTally();
var preview = await ReadPreviewAsync(archivePath, token);
if (!preview.Success)
return new(false, 0, 0, 0, 0, preview.Issue, false);
// Next to the copies a backup makes, which the next start clears in case AI Studio stops halfway:
var provenanceDirectory = Path.Join(DataRestoreFormat.DirectoryIn(SettingsManager.DataDirectory!, DataBackupFormat.STAGING_DIRECTORY), $"provenance-import-{Guid.NewGuid():N}");
try
{
var manifest = preview.Manifest;
using var archive = ZipFile.OpenRead(archivePath);
var entriesByChat = GroupEntriesByChat(archive);
var provenance = await ExtractProvenanceStoreAsync(archive, provenanceStore, provenanceDirectory, token);
// Which workspaces exist already is asked once per workspace of the backup, so read
// the tree once instead of building a full snapshot every time:
@@ -134,10 +151,10 @@ public static class ChatArchiveImporter
}
var workspaceEntryPrefix = $"{ChatArchiveFormat.WORKSPACES_PREFIX}{workspace.WorkspaceId}{ZipEntryNames.SEPARATOR}";
await ImportChatsAsync(tally, workspaceEntryPrefix, targetWorkspaceId, workspace.Chats, true, manifest.TotalChatCount, entriesByChat, collisionBehavior, existingChatIds, progress, token);
await ImportChatsAsync(tally, workspaceEntryPrefix, targetWorkspaceId, workspace.Chats, true, manifest.TotalChatCount, entriesByChat, collisionBehavior, existingChatIds, provenance, progress, token);
}
await ImportChatsAsync(tally, ChatArchiveFormat.TEMPORARY_CHATS_PREFIX, Guid.Empty, manifest.TemporaryChats, false, manifest.TotalChatCount, entriesByChat, collisionBehavior, existingChatIds, progress, token);
await ImportChatsAsync(tally, ChatArchiveFormat.TEMPORARY_CHATS_PREFIX, Guid.Empty, manifest.TemporaryChats, false, manifest.TotalChatCount, entriesByChat, collisionBehavior, existingChatIds, provenance, progress, token);
progress?.Report(new(tally.ProcessedChats, manifest.TotalChatCount));
LOG.LogInformation("Imported {ImportedChats} chats from the backup '{ArchivePath}'. Skipped {SkippedChats} existing chats, {FailedChats} chats failed.", tally.ImportedChats, archivePath, tally.SkippedChats, tally.FailedChats);
@@ -158,12 +175,111 @@ public static class ChatArchiveImporter
LOG.LogError(exception, "Failed to import the backup '{ArchivePath}'.", archivePath);
return new(false, tally.ImportedWorkspaces, tally.ImportedChats, tally.SkippedChats, tally.FailedChats, string.Format(TB("Unexpected error: {0}"), exception.Message), false);
}
finally
{
TryDeleteDirectory(provenanceDirectory);
}
}
/// <summary>
/// Unpacks the provenance store of a backup, so that every imported chat can take its entries from it.
/// </summary>
/// <remarks>
/// A backup of a version before the store holds none, and its chats arrive as they were: their
/// sources without the contents behind them.
/// </remarks>
/// <param name="archive">The backup.</param>
/// <param name="provenanceStore">The provenance store of this computer.</param>
/// <param name="directory">The directory to unpack the store into. The caller deletes it afterward.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>Where the entries come from, or null when the backup holds no store, or this computer cannot keep one.</returns>
private static async Task<ProvenanceImport?> ExtractProvenanceStoreAsync(ZipArchive archive, ProvenanceStoreClient provenanceStore, string directory, CancellationToken token)
{
if (provenanceStore is NoProvenanceStoreClient)
{
LOG.LogWarning("The provenance store is not available, so the chats of the backup arrive without what they received besides their messages.");
return null;
}
var entry = archive.GetEntry(ChatArchiveFormat.PROVENANCE_STORE_ENTRY);
if (entry is null)
{
LOG.LogInformation("The backup holds no provenance store, so its chats arrive without what they received besides their messages.");
return null;
}
try
{
Directory.CreateDirectory(directory);
var path = Path.Join(directory, SqliteProvenanceStoreClientImplementation.DATABASE_FILENAME);
await ExtractEntryAsync(entry, path, token);
//
// A backup which took the store as it lay in the data folder may hold its newest entries
// in the write-ahead log next to it, which SQLite reads along:
//
if (archive.GetEntry(ChatArchiveFormat.PROVENANCE_STORE_ENTRY + WRITE_AHEAD_LOG_SUFFIX) is { } logEntry)
await ExtractEntryAsync(logEntry, path + WRITE_AHEAD_LOG_SUFFIX, token);
return new(provenanceStore, path);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception exception)
{
// As in ImportProvenanceAsync: without the store, the chats lose the contents behind their sources, not themselves.
LOG.LogWarning(exception, "The provenance store of the backup could not be unpacked, so its chats arrive without what they received besides their messages.");
return null;
}
}
/// <summary>
/// Writes one entry of the backup into a file, as a stream.
/// </summary>
/// <param name="entry">The entry.</param>
/// <param name="destinationPath">The file, which is replaced when it exists.</param>
/// <param name="token">The cancellation token.</param>
private static async Task ExtractEntryAsync(ZipArchiveEntry entry, string destinationPath, CancellationToken token)
{
await using var entryStream = entry.Open();
await using var fileStream = new FileStream(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan);
await entryStream.CopyToAsync(fileStream, token);
}
/// <summary>
/// Adds what an imported chat received besides its messages, from the provenance store of the backup.
/// </summary>
/// <remarks>
/// The chat is stored already, so this ends neither by a cancellation nor by a failure: the next
/// chat checks for the cancellation. A chat without its entries still has its messages and
/// sources; it only lacks the contents behind them, and the store of the backup may be damaged
/// or of a newer form.
/// </remarks>
/// <param name="provenance">Where the entries come from, or null when they come from nowhere.</param>
/// <param name="sourceChatId">The chat in the backup.</param>
/// <param name="targetChatId">The chat as it was imported, possibly under a new id.</param>
private static async Task ImportProvenanceAsync(ProvenanceImport? provenance, Guid sourceChatId, Guid targetChatId)
{
if (provenance is null)
return;
try
{
var count = await provenance.Store.ImportChatAsync(provenance.BackupStorePath, sourceChatId, targetChatId, CancellationToken.None);
LOG.LogDebug("Imported {Count} provenance entries of the chat '{ChatId}'.", count, targetChatId);
}
catch (Exception exception)
{
LOG.LogWarning(exception, "The chat '{ChatId}' was imported without what it received besides its messages, because the provenance store of the backup could not be read.", targetChatId);
}
}
/// <summary>
/// Adds the chats of one workspace of the backup, or its temporary chats.
/// </summary>
private static async Task ImportChatsAsync(ImportTally tally, string entryPrefix, Guid targetWorkspaceId, IReadOnlyList<ChatArchiveManifestChat> chats, bool countsAsWorkspace, int totalChats, Dictionary<string, List<BackupEntry>> entriesByChat, ChatArchiveCollisionBehavior collisionBehavior, HashSet<Guid> existingChatIds, IProgress<ChatArchiveProgress>? progress, CancellationToken token)
private static async Task ImportChatsAsync(ImportTally tally, string entryPrefix, Guid targetWorkspaceId, IReadOnlyList<ChatArchiveManifestChat> chats, bool countsAsWorkspace, int totalChats, Dictionary<string, List<BackupEntry>> entriesByChat, ChatArchiveCollisionBehavior collisionBehavior, HashSet<Guid> existingChatIds, ProvenanceImport? provenance, IProgress<ChatArchiveProgress>? progress, CancellationToken token)
{
var isFirstImportOfWorkspace = true;
foreach (var chat in chats)
@@ -175,7 +291,7 @@ public static class ChatArchiveImporter
var chatEntryPrefix = $"{entryPrefix}{chat.ChatId}{ZipEntryNames.SEPARATOR}";
var chatEntries = entriesByChat.GetValueOrDefault(chatEntryPrefix, []);
switch (await ImportChatAsync(chatEntries, chatEntryPrefix, targetWorkspaceId, chat, collisionBehavior, existingChatIds, token))
switch (await ImportChatAsync(chatEntries, chatEntryPrefix, targetWorkspaceId, chat, collisionBehavior, existingChatIds, provenance, token))
{
case ChatArchiveImportOutcome.IMPORTED:
tally.ImportedChats++;
@@ -383,8 +499,9 @@ public static class ChatArchiveImporter
/// <param name="chat">The chat as the preview lists it.</param>
/// <param name="collisionBehavior">What to do when a chat with the same ID exists already.</param>
/// <param name="existingChatIds">The IDs of all chats in the storage. An imported chat is added.</param>
/// <param name="provenance">Where the chat gets what it received besides its messages from, or null when from nowhere.</param>
/// <param name="token">Cancels the import.</param>
private static async Task<ChatArchiveImportOutcome> ImportChatAsync(IReadOnlyList<BackupEntry> chatEntries, string chatEntryPrefix, Guid targetWorkspaceId, ChatArchiveManifestChat chat, ChatArchiveCollisionBehavior collisionBehavior, HashSet<Guid> existingChatIds, CancellationToken token)
private static async Task<ChatArchiveImportOutcome> ImportChatAsync(IReadOnlyList<BackupEntry> chatEntries, string chatEntryPrefix, Guid targetWorkspaceId, ChatArchiveManifestChat chat, ChatArchiveCollisionBehavior collisionBehavior, HashSet<Guid> existingChatIds, ProvenanceImport? provenance, CancellationToken token)
{
var threadEntry = chatEntries.FirstOrDefault(entry => entry.Name.Equals($"{chatEntryPrefix}{ChatArchiveFormat.THREAD_FILE_NAME}", StringComparison.OrdinalIgnoreCase));
if (threadEntry is null)
@@ -427,10 +544,7 @@ public static class ChatArchiveImporter
var destinationPath = ZipEntryNames.GetDestinationPath(chatDirectory, relativePath);
Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)!);
await using var entryStream = entry.Entry.Open();
await using var fileStream = new FileStream(destinationPath, FileMode.Create, FileAccess.Write, FileShare.None, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan);
await entryStream.CopyToAsync(fileStream, token);
await ExtractEntryAsync(entry.Entry, destinationPath, token);
extractedPaths.Add(relativePath);
}
@@ -477,6 +591,9 @@ public static class ChatArchiveImporter
}
existingChatIds.Add(targetChatId);
// Under the id of the chat in the backup, which is what the store of the backup knows it by:
await ImportProvenanceAsync(provenance, chat.ChatId, targetChatId);
return ChatArchiveImportOutcome.IMPORTED;
}
catch (OperationCanceledException)
@@ -533,7 +650,7 @@ public static class ChatArchiveImporter
}
catch (Exception exception)
{
LOG.LogWarning(exception, "Failed to clean up the incomplete chat directory '{ChatDirectory}'.", directory);
LOG.LogWarning(exception, "Failed to clean up the directory '{Directory}'.", directory);
}
}
}
app/MindWork AI Studio/Tools/ContentStreamProcessedEvent.cs
+6 -6
View File
@@ -12,8 +12,8 @@ namespace AIStudio.Tools;
/// <param name="Error">The reported failure, or null when the event was processed successfully.</param>
/// <param name="PromptInjection">What the runtime filtered out of the content, or null when it filtered nothing.</param>
/// <param name="TokenCount">The number of tokens of the content, or null when it is unknown.</param>
/// <param name="PageNumber">The page the content came from, or null when it has none.</param>
public readonly record struct ContentStreamProcessedEvent(string? Content, ContentStreamErrorDetails? Error, ContentStreamPromptInjectionDetails? PromptInjection = null, int? TokenCount = null, int? PageNumber = null)
/// <param name="Location">Where the content came from: its page, its sheet and row, or its line.</param>
public readonly record struct ContentStreamProcessedEvent(string? Content, ContentStreamErrorDetails? Error, ContentStreamPromptInjectionDetails? PromptInjection = null, int? TokenCount = null, SourceLocation Location = default)
{
/// <summary>
/// An event which neither produced content nor reported a failure.
@@ -21,18 +21,18 @@ public readonly record struct ContentStreamProcessedEvent(string? Content, Conte
public static readonly ContentStreamProcessedEvent NOTHING = new(null, null);
/// <summary>
/// An event which produced content, with the token count and the page of that very content.
/// An event which produced content, with the token count and the location of that very content.
/// </summary>
/// <remarks>
/// The count travels with the content because a reader may hold content back across several
/// events: pairing it with the count of the event which released it would size it by the
/// wrong text. The page travels along for the same reason, and so that whoever indexes the
/// wrong text. The location travels along for the same reason, and so that whoever indexes the
/// content is told where it came from instead of having to read it back out of the text.
/// </remarks>
/// <param name="content">The content to append.</param>
/// <param name="tokenCount">The number of tokens of that content, or null when it is unknown.</param>
/// <param name="pageNumber">The page that content came from, or null when it has none.</param>
public static ContentStreamProcessedEvent FromContent(string? content, int? tokenCount = null, int? pageNumber = null) => new(content, null, TokenCount: tokenCount, PageNumber: pageNumber);
/// <param name="location">Where that content came from, or nothing when the runtime stated nothing.</param>
public static ContentStreamProcessedEvent FromContent(string? content, int? tokenCount = null, SourceLocation location = default) => new(content, null, TokenCount: tokenCount, Location: location);
public static ContentStreamProcessedEvent FromError(ContentStreamErrorDetails? error) => new(null, error);
app/MindWork AI Studio/Tools/ContentStreamSseHandler.cs
+9 -5
View File
@@ -16,8 +16,8 @@ public static class ContentStreamSseHandler
case { Content: not null, Metadata: not null }:
switch (sseEvent.Metadata)
{
case ContentStreamTextMetadata:
return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount);
case ContentStreamTextMetadata textMetadata:
return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount, SourceLocation.OnLine(textMetadata.Text?.LineNumber));
//
// The heading tells the AI which page it is reading. The number is handed on
@@ -31,8 +31,12 @@ public static class ContentStreamSseHandler
# Page {pageNumber}
{sseEvent.Content}
""", sseEvent.TokenCount, pageNumber > 0 ? pageNumber : null);
""", sseEvent.TokenCount, SourceLocation.OnPage(pageNumber));
//
// Row 0 is the heading of a sheet together with the start of its table, which
// is no row of the sheet, cf. SourceLocation.InSheet:
//
case ContentStreamSpreadsheetMetadata spreadsheetMetadata:
var sheetName = spreadsheetMetadata.Spreadsheet?.SheetName;
var rowNumber = spreadsheetMetadata.Spreadsheet?.RowNumber;
@@ -44,7 +48,7 @@ public static class ContentStreamSseHandler
}
spreadSheetResult.Append(sseEvent.Content);
return ContentStreamProcessedEvent.FromContent(spreadSheetResult.ToString(), sseEvent.TokenCount);
return ContentStreamProcessedEvent.FromContent(spreadSheetResult.ToString(), sseEvent.TokenCount, SourceLocation.InSheet(sheetName, rowNumber));
//
// Documents which the runtime reads page by page are buffered, so the images of
@@ -62,7 +66,7 @@ public static class ContentStreamSseHandler
var documentManager = DOCUMENT_MANAGERS.GetOrAdd(sseEvent.StreamId!, _ => new());
var documentContent = documentManager.AddPage(documentMetadata, sseEvent.Content, sseEvent.TokenCount, extractImages);
return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent.Value.Content, documentContent.Value.TokenCount, documentContent.Value.PageNumber);
return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent.Value.Content, documentContent.Value.TokenCount, SourceLocation.OnPage(documentContent.Value.PageNumber));
case ContentStreamImageMetadata:
return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount);
app/MindWork AI Studio/Tools/DataBackup/DataBackupContents.cs
+1
View File
@@ -41,6 +41,7 @@ public static class DataBackupContents
new(DataBackupLocation.CONFIG_DIRECTORY, "settings.json", DataBackupEntryKind.SETTINGS),
new(DataBackupLocation.DATA_DIRECTORY, "workspaces/", DataBackupEntryKind.WORKSPACES),
new(DataBackupLocation.DATA_DIRECTORY, "tempChats/", DataBackupEntryKind.TEMPORARY_CHATS),
new(DataBackupLocation.DATA_DIRECTORY, "databases/provenance/", DataBackupEntryKind.CHAT_PROVENANCE),
new(DataBackupLocation.DATA_DIRECTORY, PLUGINS_FOLDER, DataBackupEntryKind.PLUGINS),
new(DataBackupLocation.DATA_DIRECTORY, "tokenizers/", DataBackupEntryKind.TOKENIZERS),
new(DataBackupLocation.DATA_DIRECTORY, "visualBriefings/", DataBackupEntryKind.VISUAL_BRIEFINGS),
app/MindWork AI Studio/Tools/DataBackup/DataBackupEntry.cs
+11
View File
@@ -14,6 +14,7 @@ public sealed record DataBackupEntry(DataBackupLocation Location, string Relativ
public bool IsIncluded => this.Kind is DataBackupEntryKind.SETTINGS
or DataBackupEntryKind.WORKSPACES
or DataBackupEntryKind.TEMPORARY_CHATS
or DataBackupEntryKind.CHAT_PROVENANCE
or DataBackupEntryKind.PLUGINS
or DataBackupEntryKind.TOKENIZERS
or DataBackupEntryKind.VISUAL_BRIEFINGS
@@ -33,6 +34,16 @@ public sealed record DataBackupEntry(DataBackupLocation Location, string Relativ
or DataBackupEntryKind.VECTOR_STORES
or DataBackupEntryKind.MEDIA_TRANSCRIPTS;
/// <summary>
/// Whether a backup takes the entry from the copy it makes of the provenance store, rather than from the data folder.
/// </summary>
/// <remarks>
/// The chats write to the store at any time, so a file-by-file copy could catch it halfway
/// through a write. Unlike the indexes, it is copied for every backup, since it belongs to the
/// chats, which every backup holds. Cf. DataBackupSources.CopyProvenanceStoreAsync.
/// </remarks>
public bool ComesFromProvenanceCopy => this.Kind is DataBackupEntryKind.CHAT_PROVENANCE;
/// <summary>
/// Whether the entry is a folder rather than a single file.
/// </summary>
app/MindWork AI Studio/Tools/DataBackup/DataBackupEntryKind.cs
+6
View File
@@ -20,6 +20,12 @@ public enum DataBackupEntryKind
/// </summary>
TEMPORARY_CHATS,
/// <summary>
/// What the chats received besides their messages, such as the content of attached files and the
/// results of tools. It belongs to the chats, so every backup takes it along, like them.
/// </summary>
CHAT_PROVENANCE,
/// <summary>
/// The plugins of the user. A backup leaves out the plugins of the organization.
/// </summary>
app/MindWork AI Studio/Tools/DataBackup/DataBackupSources.cs
+2 -1
View File
@@ -8,4 +8,5 @@ namespace AIStudio.Tools.DataBackup;
/// <param name="SettingsJson">The settings as the backup takes them along, cf. DataBackupSettings. They come from memory rather than from the disk, since the backup leaves out the objects of configuration plugins.</param>
/// <param name="IndexCopiesDirectory">The copies of the indexes, in the layout of the data folder, or an empty string when the backup leaves the indexes out. The indexes in the data folder change while AI Studio runs, so the backup never reads them there.</param>
/// <param name="OrganizationPluginFolders">The plugins an organization deployed among the user's own ones, cf. DataBackupContents.FindOrganizationPluginsAmongUserPlugins.</param>
public sealed record DataBackupSources(string DataDirectory, string ConfigDirectory, string SettingsJson, string IndexCopiesDirectory, IReadOnlyList<string> OrganizationPluginFolders);
/// <param name="CopyProvenanceStoreAsync">Copies the provenance store once the chats are written, and answers the directory of the copy, in the layout of the data folder, or an empty string when there is none. Copied after the chats, the store holds everything the chats of the backup refer to, also what an answer completed meanwhile received; what no chat of the backup refers to is removed after a restore. Without a copy, the backup takes the store from the data folder, which is consistent only while nothing writes to it, i.e., when AI Studio could not open the store at all.</param>
public sealed record DataBackupSources(string DataDirectory, string ConfigDirectory, string SettingsJson, string IndexCopiesDirectory, IReadOnlyList<string> OrganizationPluginFolders, Func<CancellationToken, Task<string>>? CopyProvenanceStoreAsync = null);
app/MindWork AI Studio/Tools/DataBackup/DataBackupWriter.cs
+48 -26
View File
@@ -43,8 +43,9 @@ public static class DataBackupWriter
/// <returns>The manifest as the backup holds it.</returns>
public static async Task<DataBackupManifest> WriteAsync(Stream target, DataBackupSources sources, DataBackupManifest manifest, IProgress<DataBackupProgress>? progress, CancellationToken token)
{
var files = CollectFiles(sources, token);
var totalFiles = files.Count + 1;
// The provenance store follows the chats, cf. DataBackupSources.CopyProvenanceStoreAsync. Until it is copied, it counts as one file:
var files = CollectFiles(sources, entry => !entry.ComesFromProvenanceCopy, string.Empty, token);
var totalFiles = files.Count + 2;
var entries = new List<DataBackupManifestEntry>(totalFiles);
var buffer = new byte[BUFFER_SIZE];
@@ -56,25 +57,12 @@ public static class DataBackupWriter
entries.Add(await WriteEntryAsync(archive, DataBackupFormat.EntryName(settingsEntry.Location, settingsEntry.RelativePath), settingsStream, DateTimeOffset.Now, buffer, token));
progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles));
foreach (var file in files)
{
token.ThrowIfCancellationRequested();
await WriteFilesAsync(archive, files, entries, totalFiles, progress, buffer, token);
FileStream source;
try
{
source = new FileStream(file.SourcePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan);
}
catch (Exception exception) when (exception is FileNotFoundException or DirectoryNotFoundException)
{
continue;
}
await using (source)
entries.Add(await WriteEntryAsync(archive, file.EntryName, source, File.GetLastWriteTime(file.SourcePath), buffer, token));
progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles));
}
var provenanceCopyDirectory = sources.CopyProvenanceStoreAsync is null ? string.Empty : await sources.CopyProvenanceStoreAsync(token);
var provenanceFiles = CollectFiles(sources, entry => entry.ComesFromProvenanceCopy, provenanceCopyDirectory, token);
totalFiles = files.Count + 1 + provenanceFiles.Count;
await WriteFilesAsync(archive, provenanceFiles, entries, totalFiles, progress, buffer, token);
manifest = manifest with { Entries = entries };
await using (var manifestStream = new MemoryStream(JsonSerializer.SerializeToUtf8Bytes(manifest, DataBackupFormat.JSON_OPTIONS), writable: false))
@@ -88,9 +76,14 @@ public static class DataBackupWriter
/// Collects the files of the data folder which belong into a backup, cf. DataBackupContents.
/// </summary>
/// <remarks>
/// The settings are no part of it: they come from memory. The indexes come from their copies.
/// The settings are no part of it: they come from memory. The indexes and the provenance store
/// come from their copies.
/// </remarks>
private static List<BackupFile> CollectFiles(DataBackupSources sources, CancellationToken token)
/// <param name="sources">Where the files come from.</param>
/// <param name="include">Which entries of the list to collect.</param>
/// <param name="provenanceCopyDirectory">The copy of the provenance store, or an empty string to take it from the data folder.</param>
/// <param name="token">Cancels collecting.</param>
private static List<BackupFile> CollectFiles(DataBackupSources sources, Func<DataBackupEntry, bool> include, string provenanceCopyDirectory, CancellationToken token)
{
var excludedPaths = DataBackupContents.ENTRIES
.Where(entry => !entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY)
@@ -99,11 +92,14 @@ public static class DataBackupWriter
.ToList();
var files = new List<BackupFile>();
foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY))
foreach (var entry in DataBackupContents.ENTRIES.Where(entry => entry.IsIncluded && entry.Location is DataBackupLocation.DATA_DIRECTORY && include(entry)))
{
var root = entry.ComesFromIndexCopies
? sources.IndexCopiesDirectory
: sources.DataDirectory;
var root = entry switch
{
{ ComesFromIndexCopies: true } => sources.IndexCopiesDirectory,
{ ComesFromProvenanceCopy: true } when !string.IsNullOrWhiteSpace(provenanceCopyDirectory) => provenanceCopyDirectory,
_ => sources.DataDirectory,
};
if (string.IsNullOrWhiteSpace(root))
continue;
@@ -141,6 +137,32 @@ public static class DataBackupWriter
return files;
}
/// <summary>
/// Writes files into the backup, and notes each one in the entries of the manifest.
/// </summary>
private static async Task WriteFilesAsync(ZipArchive archive, IReadOnlyList<BackupFile> files, List<DataBackupManifestEntry> entries, int totalFiles, IProgress<DataBackupProgress>? progress, byte[] buffer, CancellationToken token)
{
foreach (var file in files)
{
token.ThrowIfCancellationRequested();
FileStream source;
try
{
source = new FileStream(file.SourcePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, BUFFER_SIZE, FileOptions.Asynchronous | FileOptions.SequentialScan);
}
catch (Exception exception) when (exception is FileNotFoundException or DirectoryNotFoundException)
{
continue;
}
await using (source)
entries.Add(await WriteEntryAsync(archive, file.EntryName, source, File.GetLastWriteTime(file.SourcePath), buffer, token));
progress?.Report(new(DataBackupPhase.WRITING, entries.Count, totalFiles));
}
}
/// <summary>
/// Writes one entry, and notes its size and hash while it streams through.
/// </summary>
app/MindWork AI Studio/Tools/Databases/AGENTS.md
+11 -3
View File
@@ -5,7 +5,7 @@ These instructions cover the databases in `app/MindWork AI Studio/Tools/Database
## Databases
Local RAG runs on two databases, addressed through `DatabaseRole`:
AI Studio keeps three databases, addressed through `DatabaseRole`. Local RAG runs on the first two:
- **`VECTOR_STORE`** — Qdrant Edge through the `qdrant-edge` crate, running **in-process inside the
Rust runtime**. There is no sidecar process, no port 6333 and no Qdrant API key; .NET reaches it
@@ -15,6 +15,13 @@ Local RAG runs on two databases, addressed through `DatabaseRole`:
- **`INDEX_STORE`** — SQLite at `<data directory>/databases/sqlite/rag-index.sqlite3`, reached
through EF Core. It holds the data sources, the file fingerprints, the chunk texts and an FTS5
index over them, plus the files which permanently failed to index.
- **`PROVENANCE_STORE`** — SQLite at `<data directory>/databases/provenance/provenance.sqlite3`,
reached through EF Core with migrations of its own. It holds what chats received besides their
messages, exactly as the AI received it: attached files, tool results, and passages of data
sources. A chat keeps only the ids. Never put this into the index store: the index can be
rebuilt from the files and is treated that way, while what a chat received cannot. Its
connections run with `secure_delete`, see `SecureDeleteInterceptor`, so that the text of a
deleted chat does not stay readable in the file; a new way to connect to it keeps that.
When working on these, keep in mind:
@@ -32,7 +39,8 @@ When working on these, keep in mind:
- **Raw SQL against SQLite goes through `context.Database.GetDbConnection()`**, not through
`SqlQueryRaw<T>`: that one expects a column named `Value` and wraps the statement, so a `PRAGMA`
never works with it.
- **A new EF Core migration needs a `[DynamicDependency]`** in `IndexStoreSchemaMigrator`, because
`PublishTrimmed` is on and the migration type would otherwise be trimmed away. The "Schema version"
- **A new EF Core migration needs a `[DynamicDependency]`** in the schema migrator of its store,
`IndexStoreSchemaMigrator` or `ProvenanceStoreSchemaMigrator`, because `PublishTrimmed` is on and
the migration type would otherwise be trimmed away. The "Schema version"
line on the information page shows the applied and pending counts, so a forgotten entry becomes
visible there.
app/MindWork AI Studio/Tools/Databases/ChunkLocation.cs
+27
View File
@@ -0,0 +1,27 @@
namespace AIStudio.Tools.Databases;
/// <summary>
/// Where a stored chunk is in its document, as the index store keeps it.
/// </summary>
/// <remarks>
/// Each value is null when the document has no such thing, and for a chunk stored before the value
/// was kept: it reads as unknown then. The vector store keeps only the pages, so a hit of the vector
/// search learns the rest from the index store, cf. LocalIndexSearchService.
/// </remarks>
/// <param name="PageNumber">The page the chunk starts on.</param>
/// <param name="EndPageNumber">The page the chunk ends on.</param>
/// <param name="SheetName">The sheet of a spreadsheet the chunk is from.</param>
/// <param name="StartRow">The row of its sheet the chunk starts at.</param>
/// <param name="EndRow">The row of its sheet the chunk ends at.</param>
/// <param name="StartLine">The line the chunk starts at, for text, CSV and source code.</param>
/// <param name="EndLine">The line the chunk ends at.</param>
/// <param name="AttachmentPosition">Which attachment of a mail the chunk is from, as mail_parts numbers them, or null for the text of the mail itself and for any file.</param>
public sealed record ChunkLocation(
int? PageNumber = null,
int? EndPageNumber = null,
string? SheetName = null,
int? StartRow = null,
int? EndRow = null,
int? StartLine = null,
int? EndLine = null,
int? AttachmentPosition = null);
app/MindWork AI Studio/Tools/Databases/DatabaseClientProvider.cs
+15
View File
@@ -1,4 +1,5 @@
using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.Databases.ProvenanceStore;
using AIStudio.Tools.Databases.VectorStore;
using AIStudio.Tools.Services;
@@ -69,6 +70,18 @@ public sealed class DatabaseClientProvider(RustService rustService, ILoggerFacto
client.Status);
}
public async Task<ProvenanceStoreClient> GetProvenanceStoreAsync(CancellationToken cancellationToken = default)
{
var client = await this.GetClientAsync(DatabaseRole.PROVENANCE_STORE, cancellationToken);
if (client is ProvenanceStoreClient provenanceStore)
return provenanceStore;
return new NoProvenanceStoreClient(
client.Name,
"The configured database client does not support provenance operations.",
client.Status);
}
/// <summary>
/// Builds the client which stands in for a database role that cannot serve right now.
/// </summary>
@@ -85,6 +98,7 @@ public sealed class DatabaseClientProvider(RustService rustService, ILoggerFacto
{
DatabaseRole.VECTOR_STORE => new NoVectorStoreClient(name, reason, status),
DatabaseRole.INDEX_STORE => new NoIndexStoreClient(name, reason, status),
DatabaseRole.PROVENANCE_STORE => new NoProvenanceStoreClient(name, reason, status),
_ => new NoDatabaseClient(name, reason, status)
};
@@ -125,6 +139,7 @@ public sealed class DatabaseClientProvider(RustService rustService, ILoggerFacto
{
DatabaseRole.VECTOR_STORE => await QdrantEdgeClientImplementation.CreateAsync(rustService, this.GetIndexStoreAsync, this.logger, this.databaseClientLogger, cancellationToken),
DatabaseRole.INDEX_STORE => await SqliteIndexStoreClientImplementation.CreateAsync(this.logger, this.databaseClientLogger, cancellationToken),
DatabaseRole.PROVENANCE_STORE => await SqliteProvenanceStoreClientImplementation.CreateAsync(this.logger, this.databaseClientLogger, cancellationToken),
_ => new NoDatabaseClient(databaseRole.ToString(), "The requested database role is not supported.")
};
app/MindWork AI Studio/Tools/Databases/DatabaseRole.cs
+1
View File
@@ -4,4 +4,5 @@ public enum DatabaseRole
{
VECTOR_STORE,
INDEX_STORE,
PROVENANCE_STORE,
}
app/MindWork AI Studio/Tools/Databases/HiddenChunks.cs
+15
View File
@@ -0,0 +1,15 @@
namespace AIStudio.Tools.Databases;
/// <summary>
/// The chunks of one document which no search may find.
/// </summary>
/// <remarks>
/// A document is searched by the generation its index row names, and only once that generation is
/// complete. Indexing it again writes the next generation next to it, and an attempt which failed
/// may stay there afterward, so the next one can reuse its vectors, cf. IndexedRunContext. A
/// document indexed for the first time has no searchable generation until its first one is
/// complete. The index store knows all of this, and the vector store leaves out what it names.
/// </remarks>
/// <param name="ParentFileId">The parent id of the document.</param>
/// <param name="SearchableGeneration">The generation which can be searched, so every other one is hidden. Null while the document has none, so all of its chunks are hidden.</param>
public sealed record HiddenChunks(string ParentFileId, int? SearchableGeneration);
app/MindWork AI Studio/Tools/Databases/IndexStore/ChunkTextHash.cs
+23
View File
@@ -0,0 +1,23 @@
using System.Security.Cryptography;
using System.Text;
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// The hash of the text of a chunk, exactly as it was embedded.
/// </summary>
/// <remarks>
/// A vector depends on nothing but the text it was made from and the embedding signature, and all
/// vectors of one store share that signature. So two chunks of a store with the same hash may share
/// a vector, which is what spares a document the embedding provider for every chunk that did not
/// change. The text is hashed with its overlap and its heading line, as the provider received it.
/// </remarks>
internal static class ChunkTextHash
{
/// <summary>
/// SHA-256 over the UTF-8 bytes of the text, as lowercase hex.
/// </summary>
/// <param name="text">The text of the chunk.</param>
/// <returns>The hash.</returns>
public static string Of(string text) => Convert.ToHexStringLower(SHA256.HashData(Encoding.UTF8.GetBytes(text)));
}
app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunk.cs
+39 -1
View File
@@ -1,3 +1,41 @@
namespace AIStudio.Tools.Databases.IndexStore;
public sealed record EmbeddingStateChunk(string ChunkId, string ParentFileId, int? PageNumber, int ChunkIndex, string ChunkText, DateTimeOffset EmbeddedAtUtc);
/// <summary>
/// One chunk as the index stores it.
/// </summary>
/// <remarks>
/// Where a chunk is goes into the typed columns its document has: pages, sheet and rows, or lines.
/// The vector store keeps only the pages, cf. VectorStoragePoint: a search completes its hits from
/// here by their chunk id.
/// </remarks>
/// <param name="ChunkId">The id of the chunk, which is the id of its point in the vector store.</param>
/// <param name="ParentFileId">The document the chunk belongs to.</param>
/// <param name="PageNumber">The page the chunk starts on, or null.</param>
/// <param name="EndPageNumber">The page the chunk ends on, or null.</param>
/// <param name="ChunkIndex">The place of the chunk in its document, counted from zero.</param>
/// <param name="ChunkText">The text which was embedded.</param>
/// <param name="EmbeddedAtUtc">When the chunk was stored.</param>
/// <param name="TextHash">The hash of the text, cf. ChunkTextHash.</param>
/// <param name="Generation">Which indexing of the document wrote the chunk.</param>
/// <param name="SheetName">The sheet the chunk is from, or null. A chunk never runs from one sheet into the next.</param>
/// <param name="StartRow">The row of its sheet the chunk starts at, or null.</param>
/// <param name="EndRow">The row of its sheet the chunk ends at, or null.</param>
/// <param name="StartLine">The line the chunk starts at, or null.</param>
/// <param name="EndLine">The line the chunk ends at, or null.</param>
/// <param name="AttachmentPosition">Which attachment of a mail the chunk is from, as mail_parts numbers it, or null for anything else.</param>
public sealed record EmbeddingStateChunk(
string ChunkId,
string ParentFileId,
int? PageNumber,
int? EndPageNumber,
int ChunkIndex,
string ChunkText,
DateTimeOffset EmbeddedAtUtc,
string? TextHash = null,
int Generation = 0,
string? SheetName = null,
int? StartRow = null,
int? EndRow = null,
int? StartLine = null,
int? EndLine = null,
int? AttachmentPosition = null);
app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateChunkEntity.cs
+58
View File
@@ -10,10 +10,68 @@ internal sealed class EmbeddingStateChunkEntity
public int? PageNumber { get; set; }
public int? EndPageNumber { get; set; }
public int ChunkIndex { get; set; }
public string ChunkText { get; set; } = string.Empty;
/// <summary>
/// The hash of the chunk text, cf. ChunkTextHash, or null for a chunk stored before v26.10.1.
/// </summary>
/// <remarks>
/// Stored so that a document read again need not hash every old chunk text before it can tell
/// which vectors to keep. A chunk without it is hashed from its text instead.
/// </remarks>
public string? TextHash { get; set; }
/// <summary>
/// How many times the document was indexed before this chunk was written, cf. IndexedDocumentIds.CreateChunkIdSource.
/// </summary>
/// <remarks>
/// The chunk id includes it, so a document indexed again never overwrites its old chunks with
/// new ones, even where its fingerprint stays the same, as a mail's does. Chunks stored before
/// v26.10.1 are generation 0.
/// </remarks>
public int Generation { get; set; }
/// <summary>
/// The sheet the chunk is from, or null when its document has no sheets or the chunk was stored before v26.10.1.
/// </summary>
/// <remarks>
/// A chunk never runs from one sheet into the next, cf. TextChunker.SplitBySheets. The location
/// columns hold what the runtime states about the pieces of a document. A chunk stored before
/// they existed keeps NULL in all of them, which reads as unknown, and gains them once its
/// document is indexed again.
/// </remarks>
public string? SheetName { get; set; }
/// <summary>
/// The row of its sheet the chunk starts at, counted from one as a spreadsheet program counts it.
/// </summary>
public int? StartRow { get; set; }
/// <summary>
/// The row of its sheet the chunk ends at.
/// </summary>
public int? EndRow { get; set; }
/// <summary>
/// The line the chunk starts at, for text, CSV and source code, counted from one.
/// </summary>
public int? StartLine { get; set; }
/// <summary>
/// The line the chunk ends at.
/// </summary>
public int? EndLine { get; set; }
/// <summary>
/// Which attachment of a mail the chunk is from, as mail_parts numbers the attachments of a
/// mail, or null for the text of the mail itself and for every chunk of a file.
/// </summary>
public int? AttachmentPosition { get; set; }
public DateTimeOffset EmbeddedAtUtc { get; set; }
public EmbeddingStateFileEntity? File { get; set; }
app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFile.cs
+3 -1
View File
@@ -12,4 +12,6 @@ public sealed record EmbeddingStateFile(
DateTimeOffset LastWriteUtc,
DateTimeOffset EmbeddedAtUtc,
int ChunkCount,
string? TranscriptKey = null);
string? TranscriptKey = null,
string? ContentHash = null,
int Generation = 0);
app/MindWork AI Studio/Tools/Databases/IndexStore/EmbeddingStateFileEntity.cs
+22
View File
@@ -28,6 +28,28 @@ internal sealed class EmbeddingStateFileEntity
public string? TranscriptKey { get; set; }
/// <summary>
/// The SHA-256 hash of the raw bytes of the file when it was indexed, cf. FileContentHash, or
/// null for a file indexed before v26.10.1 and for every mail.
/// </summary>
/// <remarks>
/// The fingerprint says only that path, size or write time changed. When it does, the content
/// hash tells whether the content changed as well, or whether the file was merely saved again,
/// synchronized or copied, which needs neither extraction nor embedding.
/// </remarks>
public string? ContentHash { get; set; }
/// <summary>
/// The generation of the chunks the document is searched by, cf. IndexedDocumentIds.CreateChunkIdSource.
/// </summary>
/// <remarks>
/// A document read again writes its chunks under the next generation, while the chunks of this
/// one stay. Only once all new chunks are stored does this switch to the new generation, and
/// the old chunks go. An attempt which fails leaves this as it was, and only its own chunks go,
/// so the document keeps the version it had.
/// </remarks>
public int Generation { get; set; }
public EmbeddingStateDataSourceEntity? DataSource { get; set; }
public List<EmbeddingStateChunkEntity> Chunks { get; set; } = [];
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreClient.cs
+63
View File
@@ -42,6 +42,69 @@ public abstract partial class IndexStoreClient(string name, string path) : Datab
public abstract Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token);
/// <summary>
/// The chunks a document has stored now, each with the hash of its text and its generation.
/// </summary>
/// <remarks>
/// The same chunks DeleteFileAsync removes for the document. A chunk stored before its hash was
/// kept is hashed from its text here, and only such a chunk has its text read at all.
/// </remarks>
/// <param name="dataSourceId">The data source the document belongs to.</param>
/// <param name="filePath">The key of the document, as DeleteFileAsync takes it.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The chunks, in no particular order.</returns>
public abstract Task<IReadOnlyList<StoredChunk>> GetStoredChunksAsync(string dataSourceId, string filePath, CancellationToken token);
/// <summary>
/// The ids of the chunks of some generations of a document.
/// </summary>
/// <param name="dataSourceId">The data source the document belongs to.</param>
/// <param name="filePath">The key of the document, as DeleteFileAsync takes it.</param>
/// <param name="generations">The generations.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The chunk ids, in no particular order.</returns>
public abstract Task<IReadOnlyList<string>> GetChunkIdsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token);
/// <summary>
/// Removes the chunks of every generation of a document but the given ones.
/// </summary>
/// <remarks>
/// Keeping the new generation completes a document read again. Keeping the stored one, and an
/// attempt which failed but holds the vectors the document needs, makes room for the next attempt.
/// </remarks>
/// <param name="dataSourceId">The data source the document belongs to.</param>
/// <param name="filePath">The key of the document, as DeleteFileAsync takes it.</param>
/// <param name="generations">The generations which stay.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task DeleteOtherGenerationsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token);
/// <summary>
/// Where some chunks of a data source are in their documents, by their chunk ids.
/// </summary>
/// <remarks>
/// The vector store keeps only the pages of a chunk, so its hits learn the rest from here.
/// </remarks>
/// <param name="dataSourceId">The data source.</param>
/// <param name="chunkIds">The ids of the chunks.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The location of every chunk the index store knows, by its id.</returns>
public abstract Task<IReadOnlyDictionary<string, ChunkLocation>> GetChunkLocationsAsync(string dataSourceId, IReadOnlyCollection<string> chunkIds, CancellationToken token);
/// <summary>
/// The chunks of a data source which no search may find, document by document.
/// </summary>
/// <remarks>
/// A document is searched by the generation its row names, once its chunk count says that
/// generation is complete. Every other chunk of it is hidden: a generation which is still being
/// written, an attempt which failed and stays for the vectors of the next one, and all chunks of
/// a document whose first indexing is not complete. The keyword search applies the same rule
/// itself, cf. SearchChunksAsync. Only documents with such chunks are listed, which are few.
/// </remarks>
/// <param name="dataSourceId">The data source.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>One entry per document with hidden chunks, in no particular order.</returns>
public abstract Task<IReadOnlyList<HiddenChunks>> GetHiddenChunksAsync(string dataSourceId, CancellationToken token);
public abstract Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token);
public abstract Task DeleteDataSourceAsync(string dataSourceId, CancellationToken token);
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreDbContext.cs
+12 -1
View File
@@ -81,6 +81,8 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
entity.Property(file => file.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
entity.Property(file => file.ChunkCount).HasColumnName("chunk_count");
entity.Property(file => file.TranscriptKey).HasColumnName("transcript_key");
entity.Property(file => file.ContentHash).HasColumnName("content_hash");
entity.Property(file => file.Generation).HasColumnName("generation");
entity.HasIndex(file => file.DataSourceId).HasDatabaseName("idx_embedded_files_data_source");
entity.HasIndex(file => file.AbsolutePath).HasDatabaseName("idx_embedded_files_absolute_path");
@@ -103,14 +105,23 @@ internal sealed class IndexStoreDbContext(DbContextOptions<IndexStoreDbContext>
entity.Property(chunk => chunk.ChunkId).HasColumnName("chunk_id").IsRequired();
entity.Property(chunk => chunk.ParentFileId).HasColumnName("parent_file_id").IsRequired();
entity.Property(chunk => chunk.PageNumber).HasColumnName("page_number");
entity.Property(chunk => chunk.EndPageNumber).HasColumnName("end_page_number");
entity.Property(chunk => chunk.ChunkIndex).HasColumnName("chunk_index");
entity.Property(chunk => chunk.ChunkText).HasColumnName("chunk_text").IsRequired();
entity.Property(chunk => chunk.TextHash).HasColumnName("text_hash");
entity.Property(chunk => chunk.Generation).HasColumnName("generation");
entity.Property(chunk => chunk.SheetName).HasColumnName("sheet_name");
entity.Property(chunk => chunk.StartRow).HasColumnName("start_row");
entity.Property(chunk => chunk.EndRow).HasColumnName("end_row");
entity.Property(chunk => chunk.StartLine).HasColumnName("start_line");
entity.Property(chunk => chunk.EndLine).HasColumnName("end_line");
entity.Property(chunk => chunk.AttachmentPosition).HasColumnName("attachment_position");
entity.Property(chunk => chunk.EmbeddedAtUtc).HasColumnName("embedded_at_utc").HasConversion(utcDateTimeOffsetConverter).IsRequired();
entity.HasIndex(chunk => chunk.ChunkId).HasDatabaseName("idx_embedding_chunks_chunk_id").IsUnique();
entity.HasIndex(chunk => chunk.ParentFileId).HasDatabaseName("idx_embedding_chunks_parent_file");
entity.HasIndex(chunk => chunk.PageNumber).HasDatabaseName("idx_embedding_chunks_page");
entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index").IsUnique();
entity.HasIndex(chunk => new { chunk.ParentFileId, chunk.Generation, chunk.ChunkIndex }).HasDatabaseName("idx_embedding_chunks_parent_file_generation_chunk_index").IsUnique();
});
modelBuilder.Entity<IndexingFailureEntity>(entity =>
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSchemaMigrator.cs
+1
View File
@@ -12,6 +12,7 @@ internal static class IndexStoreSchemaMigrator
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.DropDataSourceName))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.MailboxIndex))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.MediaTranscriptKey))]
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.ChunkMetadata))]
public static async Task MigrateAsync(IndexStoreDbContext context, CancellationToken token)
{
await context.Database.MigrateAsync(token);
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResult.cs
+14 -1
View File
@@ -10,6 +10,7 @@ public sealed record IndexStoreSearchResult(
string RelativePath,
string FileType,
int? PageNumber,
int? EndPageNumber,
int ChunkIndex,
string ChunkText,
double Score,
@@ -18,4 +19,16 @@ public sealed record IndexStoreSearchResult(
DateTimeOffset CreationUtc,
DateTimeOffset LastWriteUtc,
DateTimeOffset EmbeddedAtUtc,
int ChunkCount);
int ChunkCount,
string? SheetName = null,
int? StartRow = null,
int? EndRow = null,
int? StartLine = null,
int? EndLine = null,
int? AttachmentPosition = null)
{
/// <summary>
/// Where the chunk is in its document.
/// </summary>
public ChunkLocation Location => new(this.PageNumber, this.EndPageNumber, this.SheetName, this.StartRow, this.EndRow, this.StartLine, this.EndLine, this.AttachmentPosition);
}
app/MindWork AI Studio/Tools/Databases/IndexStore/IndexStoreSearchResultEntity.cs
+14
View File
@@ -20,6 +20,20 @@ internal sealed class IndexStoreSearchResultEntity
public int? PageNumber { get; set; }
public int? EndPageNumber { get; set; }
public string? SheetName { get; set; }
public int? StartRow { get; set; }
public int? EndRow { get; set; }
public int? StartLine { get; set; }
public int? EndLine { get; set; }
public int? AttachmentPosition { get; set; }
public int ChunkIndex { get; set; }
public string ChunkText { get; set; } = string.Empty;
app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/20261022000000_ChunkMetadata.cs
+83
View File
@@ -0,0 +1,83 @@
#nullable disable
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
namespace AIStudio.Tools.Databases.IndexStore.Migrations;
/// <summary>
/// Adds what a chunk and its file know beyond their text: where a chunk is, the hash of its text,
/// its generation, and the hash of the content of its file.
/// </summary>
/// <remarks>
/// Each chunk gains:
/// <list type="bullet">
/// <item>the page it ends on, next to the page it starts on, and its sheet and rows, its lines, and
/// the attachment of a mail it is from, cf. SourceLocation;</item>
/// <item>the hash of its text, so a document read again keeps the vector of every chunk whose text
/// did not change, cf. ChunkTextHash;</item>
/// <item>its generation, so a document read again gets its new chunks next to the old ones and
/// keeps the old ones until the new ones are complete, cf. IndexedDocumentIds.CreateChunkIdSource.
/// Two generations may hold the same position, so the unique index over parent and position
/// gains the generation as well.</item>
/// </list>
/// Each file gains the hash of its content, so a file saved again without changes needs neither
/// extraction nor embedding, and the generation it is searched by.<br/><br/>
/// Chunks and files stored before keep NULL, which reads as unknown, and generation 0, which is
/// the generation every one of them has: an index built before has one generation per document.
/// They gain the rest once their document is indexed again, so no index has to be rebuilt, cf.
/// DataSourceEmbeddingService.CHUNK_METADATA_VERSION.
/// </remarks>
[DbContext(typeof(IndexStoreDbContext))]
[Migration("20261022000000_ChunkMetadata")]
public partial class ChunkMetadata : Migration
{
/// <remarks>
/// Both directions go through raw SQL, like MediaTranscriptKey: the SQLite provider answers
/// DropColumn by rebuilding the table, which would drop the triggers that keep the full-text
/// index of the chunks in step, and dropping the old embedded_files table would let the cascade
/// of the foreign key in embedding_chunks take every chunk with it. Adding a column and
/// replacing an index are native in SQLite and leave the rows alone.
/// </remarks>
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN end_page_number INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN sheet_name TEXT NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN start_row INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN end_row INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN start_line INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN end_line INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN attachment_position INTEGER NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN text_hash TEXT NULL;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks ADD COLUMN generation INTEGER NOT NULL DEFAULT 0;");
migrationBuilder.Sql("ALTER TABLE embedded_files ADD COLUMN content_hash TEXT NULL;");
migrationBuilder.Sql("ALTER TABLE embedded_files ADD COLUMN generation INTEGER NOT NULL DEFAULT 0;");
migrationBuilder.Sql("DROP INDEX IF EXISTS idx_embedding_chunks_parent_file_chunk_index;");
migrationBuilder.Sql("CREATE UNIQUE INDEX idx_embedding_chunks_parent_file_generation_chunk_index ON embedding_chunks (parent_file_id, generation, chunk_index);");
}
/// <remarks>
/// Going back needs every document to have a single generation again, as it has whenever no
/// indexing run is under way and no failed attempt is kept.
/// </remarks>
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.Sql("DROP INDEX IF EXISTS idx_embedding_chunks_parent_file_generation_chunk_index;");
migrationBuilder.Sql("CREATE UNIQUE INDEX idx_embedding_chunks_parent_file_chunk_index ON embedding_chunks (parent_file_id, chunk_index);");
migrationBuilder.Sql("ALTER TABLE embedded_files DROP COLUMN generation;");
migrationBuilder.Sql("ALTER TABLE embedded_files DROP COLUMN content_hash;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN generation;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN text_hash;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN attachment_position;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN end_line;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN start_line;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN end_row;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN start_row;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN sheet_name;");
migrationBuilder.Sql("ALTER TABLE embedding_chunks DROP COLUMN end_page_number;");
}
}
app/MindWork AI Studio/Tools/Databases/IndexStore/Migrations/IndexStoreDbContextModelSnapshot.cs
+67 -2
View File
@@ -75,6 +75,10 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnType("INTEGER")
.HasColumnName("chunk_count");
entity.Property<string>("ContentHash")
.HasColumnType("TEXT")
.HasColumnName("content_hash");
entity.Property<DateTimeOffset>("CreationUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
@@ -109,6 +113,10 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnType("TEXT")
.HasColumnName("fingerprint");
entity.Property<int>("Generation")
.HasColumnType("INTEGER")
.HasColumnName("generation");
entity.Property<DateTimeOffset>("LastWriteUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
@@ -149,6 +157,10 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnName("id")
.HasAnnotation("Sqlite:Autoincrement", true);
entity.Property<int?>("AttachmentPosition")
.HasColumnType("INTEGER")
.HasColumnName("attachment_position");
entity.Property<string>("ChunkId")
.IsRequired()
.HasColumnType("TEXT")
@@ -168,6 +180,22 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnType("TEXT")
.HasColumnName("embedded_at_utc");
entity.Property<int?>("EndLine")
.HasColumnType("INTEGER")
.HasColumnName("end_line");
entity.Property<int?>("EndPageNumber")
.HasColumnType("INTEGER")
.HasColumnName("end_page_number");
entity.Property<int?>("EndRow")
.HasColumnType("INTEGER")
.HasColumnName("end_row");
entity.Property<int>("Generation")
.HasColumnType("INTEGER")
.HasColumnName("generation");
entity.Property<int?>("PageNumber")
.HasColumnType("INTEGER")
.HasColumnName("page_number");
@@ -177,6 +205,22 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasColumnType("TEXT")
.HasColumnName("parent_file_id");
entity.Property<string>("SheetName")
.HasColumnType("TEXT")
.HasColumnName("sheet_name");
entity.Property<int?>("StartLine")
.HasColumnType("INTEGER")
.HasColumnName("start_line");
entity.Property<int?>("StartRow")
.HasColumnType("INTEGER")
.HasColumnName("start_row");
entity.Property<string>("TextHash")
.HasColumnType("TEXT")
.HasColumnName("text_hash");
entity.HasKey("Id");
entity.HasIndex("ChunkId")
@@ -189,9 +233,9 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
entity.HasIndex("ParentFileId")
.HasDatabaseName("idx_embedding_chunks_parent_file");
entity.HasIndex("ParentFileId", "ChunkIndex")
entity.HasIndex("ParentFileId", "Generation", "ChunkIndex")
.IsUnique()
.HasDatabaseName("idx_embedding_chunks_parent_file_chunk_index");
.HasDatabaseName("idx_embedding_chunks_parent_file_generation_chunk_index");
entity.ToTable("embedding_chunks");
});
@@ -251,6 +295,9 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.IsRequired()
.HasColumnType("TEXT");
entity.Property<int?>("AttachmentPosition")
.HasColumnType("INTEGER");
entity.Property<int>("ChunkCount")
.HasColumnType("INTEGER");
@@ -281,6 +328,15 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT");
entity.Property<int?>("EndLine")
.HasColumnType("INTEGER");
entity.Property<int?>("EndPageNumber")
.HasColumnType("INTEGER");
entity.Property<int?>("EndRow")
.HasColumnType("INTEGER");
entity.Property<string>("FileName")
.IsRequired()
.HasColumnType("TEXT");
@@ -314,6 +370,15 @@ partial class IndexStoreDbContextModelSnapshot : ModelSnapshot
entity.Property<double>("Score")
.HasColumnType("REAL");
entity.Property<string>("SheetName")
.HasColumnType("TEXT");
entity.Property<int?>("StartLine")
.HasColumnType("INTEGER");
entity.Property<int?>("StartRow")
.HasColumnType("INTEGER");
entity.HasNoKey();
entity.ToView("embedding_chunk_search_results");
app/MindWork AI Studio/Tools/Databases/IndexStore/NoIndexStoreClient.cs
+10
View File
@@ -63,6 +63,16 @@ public sealed partial class NoIndexStoreClient(string name, string? unavailableR
public override Task UpsertChunksAsync(string dataSourceId, IReadOnlyList<EmbeddingStateChunk> chunks, CancellationToken token) => Task.CompletedTask;
public override Task<IReadOnlyList<StoredChunk>> GetStoredChunksAsync(string dataSourceId, string filePath, CancellationToken token) => Task.FromResult<IReadOnlyList<StoredChunk>>([]);
public override Task<IReadOnlyList<string>> GetChunkIdsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token) => Task.FromResult<IReadOnlyList<string>>([]);
public override Task DeleteOtherGenerationsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token) => Task.CompletedTask;
public override Task<IReadOnlyDictionary<string, ChunkLocation>> GetChunkLocationsAsync(string dataSourceId, IReadOnlyCollection<string> chunkIds, CancellationToken token) => Task.FromResult<IReadOnlyDictionary<string, ChunkLocation>>(new Dictionary<string, ChunkLocation>(StringComparer.Ordinal));
public override Task<IReadOnlyList<HiddenChunks>> GetHiddenChunksAsync(string dataSourceId, CancellationToken token) => Task.FromResult<IReadOnlyList<HiddenChunks>>([]);
public override Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token) =>
Task.FromResult<IReadOnlyList<IndexStoreSearchResult>>([]);
app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.MailQueries.cs
+9 -2
View File
@@ -35,8 +35,8 @@ public sealed partial class SqliteIndexStoreClientImplementation
{
await using var context = this.CreateContext();
var mailIds = FilterMails(context, dataSourceId, filter).Select(mail => mail.ParentFileId);
return await context.EmbeddingChunks
.AsNoTracking()
return await SearchableChunks(context)
.Where(chunk => mailIds.Contains(chunk.ParentFileId))
.Select(chunk => chunk.ChunkId)
.ToListAsync(token);
@@ -177,6 +177,13 @@ public sealed partial class SqliteIndexStoreClientImplementation
.FirstOrDefaultAsync(token);
}
/// <summary>
/// The chunks a search may find: those of the generation their document is searched by, cf. GetHiddenChunksAsync.
/// </summary>
private static IQueryable<EmbeddingStateChunkEntity> SearchableChunks(IndexStoreDbContext context) => context.EmbeddingChunks
.AsNoTracking()
.Where(chunk => context.EmbeddedFiles.Any(file => file.ParentFileId == chunk.ParentFileId && file.ChunkCount > 0 && file.Generation == chunk.Generation));
/// <summary>
/// The mails of a mailbox which meet the conditions, as a query to build on.
/// </summary>
app/MindWork AI Studio/Tools/Databases/IndexStore/SqliteIndexStoreClientImplementation.cs
+143 -65
View File
@@ -7,7 +7,6 @@ using AIStudio.Tools.Mail;
using AIStudio.Tools.PluginSystem;
using AIStudio.Tools.Services;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
namespace AIStudio.Tools.Databases.IndexStore;
@@ -151,7 +150,9 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
file.ChunkCount,
file.TranscriptKey,
file.RelativePath,
file.ParentFileId);
file.ParentFileId,
file.ContentHash,
file.Generation);
}
var permanentFailures = await context.PermanentIndexingFailures
@@ -336,6 +337,110 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
await transaction.CommitAsync(token);
}
public override async Task<IReadOnlyList<StoredChunk>> GetStoredChunksAsync(string dataSourceId, string filePath, CancellationToken token)
{
await using var context = this.CreateContext();
var parentFileIds = context.EmbeddedFiles
.Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath)
.Select(file => file.ParentFileId);
//
// Only a description of the query: the rows are read in the loop below, one at a time. A
// chunk without a stored hash brings its whole text along, and a large document has
// thousands of them, so they must never be in memory together. A chunk with a hash brings
// no text at all.
//
var chunkQuery = context.EmbeddingChunks
.AsNoTracking()
.Where(chunk => parentFileIds.Contains(chunk.ParentFileId))
.Select(chunk => new { chunk.ChunkId, chunk.TextHash, chunk.Generation, ChunkText = chunk.TextHash == null ? chunk.ChunkText : null })
.AsAsyncEnumerable();
var storedChunks = new List<StoredChunk>();
await foreach (var chunk in chunkQuery.WithCancellation(token))
storedChunks.Add(new(chunk.ChunkId, chunk.TextHash ?? ChunkTextHash.Of(chunk.ChunkText ?? string.Empty), chunk.Generation));
return storedChunks;
}
public override async Task<IReadOnlyList<string>> GetChunkIdsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token)
{
await using var context = this.CreateContext();
var parentFileIds = context.EmbeddedFiles
.Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath)
.Select(file => file.ParentFileId);
return await context.EmbeddingChunks
.Where(chunk => parentFileIds.Contains(chunk.ParentFileId) && generations.Contains(chunk.Generation))
.Select(chunk => chunk.ChunkId)
.ToListAsync(token);
}
public override async Task DeleteOtherGenerationsAsync(string dataSourceId, string filePath, IReadOnlyCollection<int> generations, CancellationToken token)
{
await using var context = this.CreateContext();
var parentFileIds = context.EmbeddedFiles
.Where(file => file.DataSourceId == dataSourceId && file.AbsolutePath == filePath)
.Select(file => file.ParentFileId);
await context.EmbeddingChunks
.Where(chunk => parentFileIds.Contains(chunk.ParentFileId) && !generations.Contains(chunk.Generation))
.ExecuteDeleteAsync(token);
}
public override async Task<IReadOnlyDictionary<string, ChunkLocation>> GetChunkLocationsAsync(string dataSourceId, IReadOnlyCollection<string> chunkIds, CancellationToken token)
{
if (chunkIds.Count is 0)
return new Dictionary<string, ChunkLocation>(StringComparer.Ordinal);
await using var context = this.CreateContext();
var parentFileIds = context.EmbeddedFiles
.Where(file => file.DataSourceId == dataSourceId)
.Select(file => file.ParentFileId);
// The locations only, never the texts of the chunks:
return await context.EmbeddingChunks
.AsNoTracking()
.Where(chunk => chunkIds.Contains(chunk.ChunkId) && parentFileIds.Contains(chunk.ParentFileId))
.Select(chunk => new
{
chunk.ChunkId,
Location = new ChunkLocation(chunk.PageNumber, chunk.EndPageNumber, chunk.SheetName, chunk.StartRow, chunk.EndRow, chunk.StartLine, chunk.EndLine, chunk.AttachmentPosition),
})
.ToDictionaryAsync(chunk => chunk.ChunkId, chunk => chunk.Location, StringComparer.Ordinal, token);
}
/// <inheritdoc />
/// <remarks>
/// On a thread of its own, cf. the Databases AGENTS.md: it runs before every vector search,
/// which a message in the chat may start, and it looks at every document of the data source.
/// </remarks>
public override Task<IReadOnlyList<HiddenChunks>> GetHiddenChunksAsync(string dataSourceId, CancellationToken token) => Task.Run<IReadOnlyList<HiddenChunks>>(async () =>
{
await using var context = this.CreateContext();
var documents = await DocumentsWithHiddenChunks(context, dataSourceId)
.Select(file => new { file.ParentFileId, file.ChunkCount, file.Generation })
.ToListAsync(token);
return documents.Select(document => new HiddenChunks(document.ParentFileId, document.ChunkCount > 0 ? document.Generation : null)).ToList();
}, token);
/// <summary>
/// The documents of a data source with chunks a search must not find: those whose row names no
/// generation yet, and those with chunks of another generation than the one their row names.
/// </summary>
/// <remarks>
/// Asked before every vector search. For a document without hidden chunks, which is nearly
/// every document, this costs two lookups in the index over parent, generation and position:
/// one below the generation of its row and one above it.
/// </remarks>
private static IQueryable<EmbeddingStateFileEntity> DocumentsWithHiddenChunks(IndexStoreDbContext context, string dataSourceId) => context.EmbeddedFiles
.AsNoTracking()
.Where(file => file.DataSourceId == dataSourceId)
.Where(file => file.ChunkCount <= 0
|| context.EmbeddingChunks.Any(chunk => chunk.ParentFileId == file.ParentFileId && chunk.Generation < file.Generation)
|| context.EmbeddingChunks.Any(chunk => chunk.ParentFileId == file.ParentFileId && chunk.Generation > file.Generation));
public override async Task<IReadOnlyList<IndexStoreSearchResult>> SearchChunksAsync(string dataSourceId, string query, int maxMatches, CancellationToken token)
{
if (maxMatches <= 0)
@@ -358,7 +463,9 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
/// </summary>
/// <remarks>
/// Kept free of ORDER BY and LIMIT, so EF Core can wrap it and filter it further before either
/// of them applies, cf. InSearchOrder.
/// of them applies, cf. InSearchOrder.<br/><br/>
/// Only the generation the row of a document names can be found, and only once the row says it
/// is complete. These are the same chunks the vector search finds, cf. GetHiddenChunksAsync.
/// </remarks>
private static IQueryable<IndexStoreSearchResultEntity> MatchChunks(IndexStoreDbContext context, string dataSourceId, string ftsQuery) => context.SearchResults
.FromSqlInterpolated($"""
@@ -372,6 +479,13 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
f.relative_path AS RelativePath,
f.file_type AS FileType,
c.page_number AS PageNumber,
c.end_page_number AS EndPageNumber,
c.sheet_name AS SheetName,
c.start_row AS StartRow,
c.end_row AS EndRow,
c.start_line AS StartLine,
c.end_line AS EndLine,
c.attachment_position AS AttachmentPosition,
c.chunk_index AS ChunkIndex,
c.chunk_text AS ChunkText,
bm25(embedding_chunks_fts) AS Score,
@@ -386,6 +500,8 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
JOIN embedded_files f ON f.parent_file_id = c.parent_file_id
JOIN data_sources ds ON ds.data_source_id = f.data_source_id
WHERE ds.data_source_id = {dataSourceId}
AND f.chunk_count > 0
AND c.generation = f.generation
AND embedding_chunks_fts MATCH {ftsQuery}
""")
.AsNoTracking();
@@ -609,70 +725,14 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
/// <inheritdoc />
/// <remarks>
/// VACUUM INTO writes a consistent snapshot of the database, including what the write-ahead log
/// still holds, and in WAL mode, reading it holds back no writer.<br/><br/>
///
/// It runs on connections of their own, although the databases guide has raw SQL go through
/// GetDbConnection. The connections of the context share one cache, and a shared cache locks
/// whole tables: reading the whole database for the copy would hold back every writer until the
/// copy is done. Without pooling, nothing keeps the copy open afterward, so the backup can
/// delete it on every OS.<br/><br/>
///
/// SQLite runs its async methods synchronously, and copying and checking a large index takes a
/// while. The copy therefore runs on a thread of its own, so that it never holds up its caller.
/// A consistent snapshot which holds back no writer, made on a thread of its own, cf. SqliteDatabaseCopy.
/// </remarks>
public override Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token) => Task.Run(() => this.CopyAndCheckAsync(targetDirectory, token), token);
private async Task<string> CopyAndCheckAsync(string targetDirectory, CancellationToken token)
public override Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token) => Task.Run(async () =>
{
Directory.CreateDirectory(targetDirectory);
var targetPath = Path.Join(targetDirectory, DATABASE_FILENAME);
// VACUUM INTO refuses to write into a file which holds anything:
if (File.Exists(targetPath))
File.Delete(targetPath);
try
{
await using (var connection = new SqliteConnection(BuildBackupConnectionString(this.databasePath, SqliteOpenMode.ReadWrite)))
{
await connection.OpenAsync(token);
await using var command = connection.CreateCommand();
command.CommandText = "VACUUM INTO $targetPath;";
command.Parameters.AddWithValue("$targetPath", targetPath);
await command.ExecuteNonQueryAsync(token);
}
await using (var connection = new SqliteConnection(BuildBackupConnectionString(targetPath, SqliteOpenMode.ReadOnly)))
{
await connection.OpenAsync(token);
await using var command = connection.CreateCommand();
command.CommandText = "PRAGMA quick_check;";
var checkResult = (await command.ExecuteScalarAsync(token))?.ToString();
if (checkResult is not "ok")
throw new InvalidOperationException($"The copy of the index at '{targetPath}' failed its check: '{checkResult}'.");
}
return targetPath;
}
catch
{
if (File.Exists(targetPath))
File.Delete(targetPath);
throw;
}
}
private static string BuildBackupConnectionString(string path, SqliteOpenMode mode) => new SqliteConnectionStringBuilder
{
DataSource = path,
Mode = mode,
Cache = SqliteCacheMode.Private,
Pooling = false,
DefaultTimeout = 30,
}.ToString();
await SqliteDatabaseCopy.CopyAndCheckAsync(this.databasePath, targetPath, token);
return targetPath;
}, token);
private IndexStoreDbContext CreateContext() => new(this.dbContextOptions);
@@ -706,6 +766,8 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
fileEntity.EmbeddedAtUtc = file.EmbeddedAtUtc;
fileEntity.ChunkCount = file.ChunkCount;
fileEntity.TranscriptKey = file.TranscriptKey;
fileEntity.ContentHash = file.ContentHash;
fileEntity.Generation = file.Generation;
}
private static void ApplyPermanentFailure(IndexingFailureEntity failureEntity, string dataSourceId, PermanentIndexingFailure failure)
@@ -731,8 +793,17 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
chunkEntity.ChunkId = chunk.ChunkId;
chunkEntity.ParentFileId = chunk.ParentFileId;
chunkEntity.PageNumber = chunk.PageNumber;
chunkEntity.EndPageNumber = chunk.EndPageNumber;
chunkEntity.ChunkIndex = chunk.ChunkIndex;
chunkEntity.ChunkText = chunk.ChunkText;
chunkEntity.TextHash = chunk.TextHash;
chunkEntity.Generation = chunk.Generation;
chunkEntity.SheetName = chunk.SheetName;
chunkEntity.StartRow = chunk.StartRow;
chunkEntity.EndRow = chunk.EndRow;
chunkEntity.StartLine = chunk.StartLine;
chunkEntity.EndLine = chunk.EndLine;
chunkEntity.AttachmentPosition = chunk.AttachmentPosition;
chunkEntity.EmbeddedAtUtc = chunk.EmbeddedAtUtc;
}
@@ -746,6 +817,7 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
result.RelativePath,
result.FileType,
result.PageNumber,
result.EndPageNumber,
result.ChunkIndex,
result.ChunkText,
result.Score,
@@ -754,7 +826,13 @@ public sealed partial class SqliteIndexStoreClientImplementation(string name, st
result.CreationUtc,
result.LastWriteUtc,
result.EmbeddedAtUtc,
result.ChunkCount);
result.ChunkCount,
result.SheetName,
result.StartRow,
result.EndRow,
result.StartLine,
result.EndLine,
result.AttachmentPosition);
private static string BuildFtsQuery(string query)
{
app/MindWork AI Studio/Tools/Databases/IndexStore/StoredChunk.cs
+9
View File
@@ -0,0 +1,9 @@
namespace AIStudio.Tools.Databases.IndexStore;
/// <summary>
/// A chunk the index store holds, as far as indexing its document again needs to know it.
/// </summary>
/// <param name="ChunkId">The id of the chunk, which is also the id of its point in the vector store.</param>
/// <param name="TextHash">The hash of its text, cf. ChunkTextHash.</param>
/// <param name="Generation">How many times its document was indexed before it, cf. IndexedDocumentIds.CreateChunkIdSource.</param>
public sealed record StoredChunk(string ChunkId, string TextHash, int Generation);
app/MindWork AI Studio/Tools/Databases/ProvenanceStore/Migrations/20261024000000_InitialProvenanceStore.cs
+43
View File
@@ -0,0 +1,43 @@
#nullable disable
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.EntityFrameworkCore.Migrations;
namespace AIStudio.Tools.Databases.ProvenanceStore.Migrations;
/// <summary>
/// Creates the table provenance, which holds what chats received besides their messages.
/// </summary>
[DbContext(typeof(ProvenanceStoreDbContext))]
[Migration("20261024000000_InitialProvenanceStore")]
public partial class InitialProvenanceStore : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "provenance",
columns: table => new
{
chat_id = table.Column<string>(type: "TEXT", nullable: false),
id = table.Column<string>(type: "TEXT", nullable: false),
kind = table.Column<string>(type: "TEXT", nullable: false),
content = table.Column<string>(type: "TEXT", nullable: false),
surrounding_content = table.Column<string>(type: "TEXT", nullable: true),
created_at_utc = table.Column<string>(type: "TEXT", nullable: false),
},
constraints: table =>
{
table.PrimaryKey("PK_provenance", entry => new { entry.chat_id, entry.id });
});
migrationBuilder.CreateIndex(
name: "idx_provenance_id",
table: "provenance",
column: "id");
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(name: "provenance");
}
}
app/MindWork AI Studio/Tools/Databases/ProvenanceStore/Migrations/ProvenanceStoreDbContextModelSnapshot.cs
+60
View File
@@ -0,0 +1,60 @@
#nullable disable
using AIStudio.Tools.Databases.IndexStore;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
namespace AIStudio.Tools.Databases.ProvenanceStore.Migrations;
// EF Core writes this file and declares the type as partial. Dropping the keyword would only
// last until the next migration is added, so the inspection is silenced instead.
// ReSharper disable once PartialTypeWithSinglePart
[DbContext(typeof(ProvenanceStoreDbContext))]
partial class ProvenanceStoreDbContextModelSnapshot : ModelSnapshot
{
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "9.0.18");
var utcDateTimeOffsetConverter = new IndexStoreDateTimeOffsetConverter();
modelBuilder.Entity("AIStudio.Tools.Databases.ProvenanceStore.ProvenanceEntity", entity =>
{
entity.Property<string>("ChatId")
.HasColumnType("TEXT")
.HasColumnName("chat_id");
entity.Property<string>("Id")
.HasColumnType("TEXT")
.HasColumnName("id");
entity.Property<string>("Content")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("content");
entity.Property<DateTimeOffset>("CreatedAtUtc")
.HasConversion(utcDateTimeOffsetConverter)
.HasColumnType("TEXT")
.HasColumnName("created_at_utc");
entity.Property<string>("Kind")
.IsRequired()
.HasColumnType("TEXT")
.HasColumnName("kind");
entity.Property<string>("SurroundingContent")
.HasColumnType("TEXT")
.HasColumnName("surrounding_content");
entity.HasKey("ChatId", "Id");
entity.HasIndex("Id")
.HasDatabaseName("idx_provenance_id");
entity.ToTable("provenance");
});
#pragma warning restore 612, 618
}
}
app/MindWork AI Studio/Tools/Databases/ProvenanceStore/NoProvenanceStoreClient.cs
+69
View File
@@ -0,0 +1,69 @@
using AIStudio.Tools.Databases.IndexStore;
using AIStudio.Tools.PluginSystem;
namespace AIStudio.Tools.Databases.ProvenanceStore;
/// <summary>
/// Stands in for the provenance store while it cannot serve: it keeps nothing and finds nothing.
/// </summary>
/// <remarks>
/// Storing answers with no id, which tells the caller to keep the content in the chat itself.
/// </remarks>
public sealed class NoProvenanceStoreClient(string name, string? unavailableReason, DatabaseClientStatus status = DatabaseClientStatus.UNAVAILABLE) : ProvenanceStoreClient(name, string.Empty)
{
private static string TB(string fallbackEN) => I18N.I.T(fallbackEN, typeof(NoProvenanceStoreClient).Namespace, nameof(NoProvenanceStoreClient));
public override DatabaseClientStatus Status => status;
public override async IAsyncEnumerable<(string Label, string Value)> GetDisplayInfo()
{
yield return (TB("Status"), status switch
{
DatabaseClientStatus.STARTING => TB("Starting"),
_ => TB("Unavailable")
});
if (!string.IsNullOrWhiteSpace(unavailableReason))
yield return (TB("Reason"), unavailableReason);
// As in NoIndexStoreClient: which library this process bound to matters most when SQLite cannot be loaded at all.
yield return (TB("Native library"), OrUnknown(SqliteRuntimeInfo.GetNativeLibraryName()));
yield return (TB("Process architecture"), OrUnknown(SqliteRuntimeInfo.GetProcessArchitecture()));
await Task.CompletedTask;
}
private static string OrUnknown(string value) => string.IsNullOrWhiteSpace(value) ? TB("unknown") : value;
public override Task<Guid?> StoreAsync(Guid chatId, ProvenanceKind kind, string content, IReadOnlyList<string>? surroundingContent, CancellationToken token) => Task.FromResult<Guid?>(null);
public override Task<IReadOnlyDictionary<Guid, ProvenanceEntry>> GetEntriesAsync(IReadOnlyCollection<Guid> ids, CancellationToken token) => Task.FromResult<IReadOnlyDictionary<Guid, ProvenanceEntry>>(new Dictionary<Guid, ProvenanceEntry>());
public override Task CopyChatAsync(Guid sourceChatId, Guid targetChatId, CancellationToken token) => Task.CompletedTask;
public override Task<int> DeleteChatsAsync(IReadOnlyCollection<Guid> chatIds, CancellationToken token, DateTimeOffset? createdBefore = null) => Task.FromResult(0);
public override Task<int> DeleteChatEntriesExceptAsync(Guid chatId, IReadOnlyCollection<Guid> keptIds, CancellationToken token, DateTimeOffset? createdBefore = null) => Task.FromResult(0);
public override Task<int> DeleteEntriesAsync(Guid chatId, IReadOnlyCollection<Guid> ids, CancellationToken token) => Task.FromResult(0);
public override Task<IReadOnlyList<Guid>> GetChatIdsAsync(CancellationToken token, DateTimeOffset? createdSince = null) => Task.FromResult<IReadOnlyList<Guid>>([]);
public override Task<DateTimeOffset?> GetLastCleanupAsync(CancellationToken token) => Task.FromResult<DateTimeOffset?>(null);
public override Task SetLastCleanupAsync(DateTimeOffset startedAt, CancellationToken token) => Task.CompletedTask;
/// <inheritdoc />
/// <remarks>
/// Unlike the other methods, this one does not pretend success: a backup would take an empty
/// copy for the real one. Whoever makes the backup decides what to do instead.
/// </remarks>
public override Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token) =>
Task.FromException<string>(new InvalidOperationException(unavailableReason ?? "The provenance store is not available."));
public override Task<int> ImportChatAsync(string sourceDatabasePath, Guid sourceChatId, Guid targetChatId, CancellationToken token) => Task.FromResult(0);
public override void Dispose()
{
}
}
app/MindWork AI Studio/Tools/Databases/ProvenanceStore/ProvenanceEntity.cs
+30
View File
@@ -0,0 +1,30 @@
namespace AIStudio.Tools.Databases.ProvenanceStore;
/// <summary>
/// One row of the table provenance.
/// </summary>
/// <remarks>
/// The key is the chat together with the id. A chat which is copied, e.g., an assistant's
/// conversation which the user carries over into a chat, takes its rows along under the same ids,
/// so its references stay valid, and deleting one of the two chats leaves the other one whole.
/// </remarks>
internal sealed class ProvenanceEntity
{
public string ChatId { get; set; } = string.Empty;
public string Id { get; set; } = string.Empty;
/// <summary>
/// The ProvenanceKind, stored by name.
/// </summary>
public string Kind { get; set; } = string.Empty;
public string Content { get; set; } = string.Empty;
/// <summary>
/// The content around an ERI passage as a JSON array of strings, or null when there is none.
/// </summary>
public string? SurroundingContent { get; set; }
public DateTimeOffset CreatedAtUtc { get; set; }
}
app/MindWork AI Studio/Tools/Databases/ProvenanceStore/ProvenanceEntry.cs
+16
View File
@@ -0,0 +1,16 @@
namespace AIStudio.Tools.Databases.ProvenanceStore;
/// <summary>
/// One piece of content a chat received besides its messages, exactly as the AI received it.
/// </summary>
/// <remarks>
/// Where the content comes from, such as the file, the page, or the tool, is kept by the chat
/// itself, next to the id of this entry. The chat needs it to list its sources even without this
/// store, and it would only be stored twice here.
/// </remarks>
/// <param name="Id">The id the chat refers to the entry by.</param>
/// <param name="Kind">What sort of content this is.</param>
/// <param name="Content">The content, after the prompt injection filter.</param>
/// <param name="SurroundingContent">The content an ERI server sent along around its passage, empty for everything else.</param>
/// <param name="CreatedAtUtc">When the entry was stored.</param>
public sealed record ProvenanceEntry(Guid Id, ProvenanceKind Kind, string Content, IReadOnlyList<string> SurroundingContent, DateTimeOffset CreatedAtUtc);
app/MindWork AI Studio/Tools/Databases/ProvenanceStore/ProvenanceKind.cs
+32
View File
@@ -0,0 +1,32 @@
namespace AIStudio.Tools.Databases.ProvenanceStore;
/// <summary>
/// What sort of content a chat received, apart from its messages.
/// </summary>
/// <remarks>
/// Stored by name, like the kinds of mail parts in the index: a new member may then go anywhere
/// in this list, and the database reads on its own.
/// </remarks>
public enum ProvenanceKind
{
/// <summary>
/// The content of a file the user attached to a message.
/// </summary>
ATTACHMENT,
/// <summary>
/// A passage a data source handed to the AI: a local one or an ERI server, through the classic
/// process or the semantic search.
/// </summary>
DATA_SOURCE_PASSAGE,
/// <summary>
/// A mail, or a part of one, which the AI received from a mailbox.
/// </summary>
MAIL,
/// <summary>
/// A web page the AI received, e.g., which a tool read for it.
/// </summary>
WEB_PAGE,
}
app/MindWork AI Studio/Tools/Databases/ProvenanceStore/ProvenanceStoreClient.cs
+129
View File
@@ -0,0 +1,129 @@
namespace AIStudio.Tools.Databases.ProvenanceStore;
/// <summary>
/// Keeps what chats received besides their messages: the content of attached files, the results
/// of tools, and the passages of data sources, each exactly as the AI received it.
/// </summary>
/// <remarks>
/// A chat keeps only the ids of its entries, next to where each one came from. The content lives
/// here, so that a chat file stays small and an attached file is read only once.<br/><br/>
///
/// A database of its own rather than a table of the index: the index of the data sources can be
/// rebuilt from the files and is treated that way, e.g., a backup may leave it out. What a chat
/// received cannot be rebuilt, since the files, pages and mails may have changed since.<br/><br/>
///
/// Every entry belongs to one chat. The same id may stand for an entry of several chats, after a
/// chat was copied, and then it holds the same content in each.
/// </remarks>
public abstract class ProvenanceStoreClient(string name, string path) : DatabaseClient(name, path)
{
/// <summary>
/// Stores one piece of content a chat received.
/// </summary>
/// <param name="chatId">The chat which received the content.</param>
/// <param name="kind">What sort of content it is.</param>
/// <param name="content">The content, exactly as the AI received it.</param>
/// <param name="surroundingContent">The content an ERI server sent along around its passage, or nothing.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The id of the new entry, or null when this store cannot keep anything right now.</returns>
public abstract Task<Guid?> StoreAsync(Guid chatId, ProvenanceKind kind, string content, IReadOnlyList<string>? surroundingContent, CancellationToken token);
/// <summary>
/// Reads entries by their ids.
/// </summary>
/// <remarks>
/// By the ids alone, without the chat: the place which shows an entry seldom knows its chat,
/// and every copy of an entry holds the same content.
/// </remarks>
/// <param name="ids">The ids to read.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The entries found, by their ids. An id without an entry is missing from it.</returns>
public abstract Task<IReadOnlyDictionary<Guid, ProvenanceEntry>> GetEntriesAsync(IReadOnlyCollection<Guid> ids, CancellationToken token);
/// <summary>
/// Copies every entry of a chat to another chat, keeping their ids.
/// </summary>
/// <remarks>
/// The copy is what a chat made from another one refers to. Entries the target holds already
/// stay as they are.
/// </remarks>
/// <param name="sourceChatId">The chat to copy from.</param>
/// <param name="targetChatId">The chat to copy to.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task CopyChatAsync(Guid sourceChatId, Guid targetChatId, CancellationToken token);
/// <summary>
/// Deletes every entry of some chats.
/// </summary>
/// <remarks>
/// All of them at once: deleting a workspace or cleaning up at the start may concern many chats.
/// </remarks>
/// <param name="chatIds">The chats.</param>
/// <param name="token">The cancellation token.</param>
/// <param name="createdBefore">Deletes only the entries stored before this moment, or all of them when null.</param>
/// <returns>How many entries were deleted.</returns>
public abstract Task<int> DeleteChatsAsync(IReadOnlyCollection<Guid> chatIds, CancellationToken token, DateTimeOffset? createdBefore = null);
/// <summary>
/// Deletes the entries of a chat which it no longer refers to.
/// </summary>
/// <param name="chatId">The chat.</param>
/// <param name="keptIds">The ids the chat still refers to.</param>
/// <param name="token">The cancellation token.</param>
/// <param name="createdBefore">Deletes only the entries stored before this moment, or all of them when null.</param>
/// <returns>How many entries were deleted.</returns>
public abstract Task<int> DeleteChatEntriesExceptAsync(Guid chatId, IReadOnlyCollection<Guid> keptIds, CancellationToken token, DateTimeOffset? createdBefore = null);
/// <summary>
/// Deletes some entries of a chat, e.g., texts which a preview kept for a message that went without them.
/// </summary>
/// <param name="chatId">The chat.</param>
/// <param name="ids">The ids of the entries.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>How many entries were deleted.</returns>
public abstract Task<int> DeleteEntriesAsync(Guid chatId, IReadOnlyCollection<Guid> ids, CancellationToken token);
/// <summary>
/// Lists the chats this store holds entries of.
/// </summary>
/// <param name="token">The cancellation token.</param>
/// <param name="createdSince">Lists only the chats with an entry stored at or after this moment, or all of them when null.</param>
/// <returns>The ids of the chats, Guid.Empty included when entries were stored for a chat without an id.</returns>
public abstract Task<IReadOnlyList<Guid>> GetChatIdsAsync(CancellationToken token, DateTimeOffset? createdSince = null);
/// <summary>
/// Reads when the last complete cleanup of this store began, see ProvenanceCleanup.
/// </summary>
/// <param name="token">The cancellation token.</param>
/// <returns>The moment, or null when no cleanup was noted, e.g., for a store a restore brought back.</returns>
public abstract Task<DateTimeOffset?> GetLastCleanupAsync(CancellationToken token);
/// <summary>
/// Notes when a complete cleanup of this store began, see ProvenanceCleanup.
/// </summary>
/// <param name="startedAt">The moment the cleanup began.</param>
/// <param name="token">The cancellation token.</param>
public abstract Task SetLastCleanupAsync(DateTimeOffset startedAt, CancellationToken token);
/// <summary>
/// Writes a copy of this store into a directory, for a backup.
/// </summary>
/// <param name="targetDirectory">The directory of the copy. A copy written there before is replaced.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>The path of the copy.</returns>
public abstract Task<string> BackupToDirectoryAsync(string targetDirectory, CancellationToken token);
/// <summary>
/// Adds the entries of one chat from a copy of this store, such as the one a backup holds.
/// </summary>
/// <remarks>
/// The entries keep their ids, so the chat's references stay valid, also when the chat is added
/// under another id. Entries the target chat holds already stay as they are.
/// </remarks>
/// <param name="sourceDatabasePath">The copy to read from. It is only read.</param>
/// <param name="sourceChatId">The chat in the copy.</param>
/// <param name="targetChatId">The chat in this store.</param>
/// <param name="token">The cancellation token.</param>
/// <returns>How many entries were added.</returns>
public abstract Task<int> ImportChatAsync(string sourceDatabasePath, Guid sourceChatId, Guid targetChatId, CancellationToken token);
}
app/MindWork AI Studio/Tools/Databases/ProvenanceStore/ProvenanceStoreDbContext.cs
+45
View File
@@ -0,0 +1,45 @@
using AIStudio.Tools.Databases.IndexStore;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
namespace AIStudio.Tools.Databases.ProvenanceStore;
internal sealed class ProvenanceStoreDbContext(DbContextOptions<ProvenanceStoreDbContext> options) : DbContext(options)
{
private static readonly SecureDeleteInterceptor SECURE_DELETE = new();
public static DbContextOptions<ProvenanceStoreDbContext> CreateOptions(string databasePath) => new DbContextOptionsBuilder<ProvenanceStoreDbContext>()
.UseSqlite(BuildConnectionString(databasePath))
.AddInterceptors(SECURE_DELETE)
.Options;
public DbSet<ProvenanceEntity> Entries => this.Set<ProvenanceEntity>();
protected override void OnModelCreating(ModelBuilder modelBuilder)
{
modelBuilder.Entity<ProvenanceEntity>(entity =>
{
entity.ToTable("provenance");
entity.HasKey(entry => new { entry.ChatId, entry.Id });
entity.Property(entry => entry.ChatId).HasColumnName("chat_id");
entity.Property(entry => entry.Id).HasColumnName("id");
entity.Property(entry => entry.Kind).HasColumnName("kind").IsRequired();
entity.Property(entry => entry.Content).HasColumnName("content").IsRequired();
entity.Property(entry => entry.SurroundingContent).HasColumnName("surrounding_content");
entity.Property(entry => entry.CreatedAtUtc).HasColumnName("created_at_utc").HasConversion(new IndexStoreDateTimeOffsetConverter()).IsRequired();
// A chat reads its entries by their ids alone, see ProvenanceStoreClient.GetEntriesAsync:
entity.HasIndex(entry => entry.Id).HasDatabaseName("idx_provenance_id");
});
}
private static string BuildConnectionString(string databasePath) => new SqliteConnectionStringBuilder
{
DataSource = databasePath,
Mode = SqliteOpenMode.ReadWriteCreate,
Cache = SqliteCacheMode.Shared,
DefaultTimeout = 30,
}.ToString();
}
app/MindWork AI Studio/Tools/Databases/ProvenanceStore/ProvenanceStoreDesignTimeDbContextFactory.cs
+15
View File
@@ -0,0 +1,15 @@
using Microsoft.EntityFrameworkCore.Design;
namespace AIStudio.Tools.Databases.ProvenanceStore;
internal sealed class ProvenanceStoreDesignTimeDbContextFactory : IDesignTimeDbContextFactory<ProvenanceStoreDbContext>
{
public ProvenanceStoreDbContext CreateDbContext(string[] args)
{
var databasePath = args.FirstOrDefault(argument => argument.EndsWith(".sqlite3", StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrWhiteSpace(databasePath))
databasePath = Path.Combine(Path.GetTempPath(), "mindwork-ai-studio-provenance-design.sqlite3");
return new ProvenanceStoreDbContext(ProvenanceStoreDbContext.CreateOptions(databasePath));
}
}
app/MindWork AI Studio/Tools/Databases/ProvenanceStore/ProvenanceStoreSchemaMigrator.cs
+15
View File
@@ -0,0 +1,15 @@
using System.Diagnostics.CodeAnalysis;
using Microsoft.EntityFrameworkCore;
namespace AIStudio.Tools.Databases.ProvenanceStore;
internal static class ProvenanceStoreSchemaMigrator
{
[DynamicDependency(DynamicallyAccessedMemberTypes.All, typeof(Migrations.InitialProvenanceStore))]
public static async Task MigrateAsync(ProvenanceStoreDbContext context, CancellationToken token)
{
await context.Database.MigrateAsync(token);
await context.Database.ExecuteSqlRawAsync("PRAGMA journal_mode=WAL;", token);
}
}
Loaded 100 of 229 files, more files were not shown because too many files have changed in this diff. Show more