Added retrieval integration, part I (#281)

app/MindWork AI Studio/Tools/RAG/IRetrievalContext.cs

@@ -0,0 +1,45 @@
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// The common interface for any retrieval context.
+/// </summary>
+public interface IRetrievalContext
+{
+    /// <summary>
+    /// The name of the data source.
+    /// </summary>
+    /// <remarks>
+    /// Depending on the configuration, the AI is selecting the appropriate data source.
+    /// In order to inform the user about where the information is coming from, the data
+    /// source name is necessary.
+    /// </remarks>
+    public string DataSourceName { get; init; }
+
+    /// <summary>
+    /// The category of the content, like e.g., text, audio, image, etc.
+    /// </summary>
+    public RetrievalContentCategory Category { get; init; }
+
+    /// <summary>
+    /// What type of content is being retrieved? Like e.g., a project proposal, spreadsheet, art, etc.
+    /// </summary>
+    public RetrievalContentType Type { get; init; }
+
+    /// <summary>
+    /// The path to the content, e.g., a URL, a file path, a path in a graph database, etc.
+    /// </summary>
+    public string Path { get; init; }
+
+    /// <summary>
+    /// Links to related content, e.g., links to Wikipedia articles, links to sources, etc.
+    /// </summary>
+    /// <remarks>
+    /// Why would you need links for retrieval? You are right that not all retrieval
+    /// contexts need links. But think about a web search feature, where we want to
+    /// query a search engine and get back a list of links to the most relevant
+    /// matches. Think about a continuous web crawler that is constantly looking for
+    /// new information and adding it to the knowledge base. In these cases, links
+    /// are essential.
+    /// </remarks>
+    public IReadOnlyList<string> Links { get; init; }
+}

app/MindWork AI Studio/Tools/RAG/RetrievalContentCategory.cs

@@ -0,0 +1,12 @@
+namespace AIStudio.Tools.RAG;
+
+public enum RetrievalContentCategory
+{
+    NONE,
+    UNKNOWN,
+    
+    TEXT,
+    IMAGE,
+    VIDEO,
+    AUDIO,
+}

app/MindWork AI Studio/Tools/RAG/RetrievalContentCategoryExtensions.cs

@@ -0,0 +1,24 @@
+using AIStudio.Tools.ERIClient.DataModel;
+
+namespace AIStudio.Tools.RAG;
+
+public static class RetrievalContentCategoryExtensions
+{
+    /// <summary>
+    /// Converts an ERI content type to a common retrieval content category.
+    /// </summary>
+    /// <param name="contentType">The content type yielded by the ERI server.</param>
+    /// <returns>The corresponding retrieval content category.</returns>
+    public static RetrievalContentCategory ToRetrievalContentCategory(ContentType contentType) => contentType switch
+    {
+        ContentType.NONE => RetrievalContentCategory.NONE,
+        ContentType.UNKNOWN => RetrievalContentCategory.UNKNOWN,
+        ContentType.TEXT => RetrievalContentCategory.TEXT,
+        ContentType.IMAGE => RetrievalContentCategory.IMAGE,
+        ContentType.VIDEO => RetrievalContentCategory.VIDEO,
+        ContentType.AUDIO => RetrievalContentCategory.AUDIO,
+        ContentType.SPEECH => RetrievalContentCategory.AUDIO,
+        
+        _ => RetrievalContentCategory.UNKNOWN,
+    };
+}

app/MindWork AI Studio/Tools/RAG/RetrievalContentType.cs

@@ -0,0 +1,105 @@
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// The type of the retrieved content.
+/// </summary>
+public enum RetrievalContentType
+{
+    NOT_SPECIFIED,
+    UNKNOWN,
+    
+    //
+    // Text Content:
+    //
+    TEXT_DOCUMENT,
+    TEXT_ARTICLE,
+    TEXT_BOOK,
+    TEXT_CHAPTER,
+    TEXT_PAPER,
+    TEXT_THESIS,
+    TEXT_BUSINESS_CONCEPT,
+    TEXT_DICTIONARY,
+    TEXT_ENCYCLOPEDIA,
+    TEXT_GLOSSARY,
+    TEXT_JOURNAL,
+    TEXT_MAGAZINE,
+    TEXT_NEWSPAPER,
+    TEXT_REPORT,
+    TEXT_REVIEW,
+    TEXT_WEBSITE,
+    TEXT_IDEA,
+    TEXT_CONCEPT,
+    TEXT_DEFINITION,
+    TEXT_EXAMPLE,
+    TEXT_QUOTE,
+    TEXT_DRAFT,
+    TEXT_SCRIPT,
+    TEXT_TRANSCRIPT,
+    TEXT_SUBTITLE,
+    TEXT_CAPTION,
+    TEXT_DIALOGUE,
+    TEXT_PROJECT_PROPOSAL,
+    TEXT_PROJECT_PLAN,
+    TEXT_SPREADSHEET,
+    TEXT_PRESENTATION,
+    TEXT_MEETING_MINUTES,
+    TEXT_EMAIL,
+    TEXT_PROTOCOL,
+    
+    //
+    // Image Content:
+    //
+    IMAGE_PHOTO,
+    IMAGE_ILLUSTRATION,
+    IMAGE_DIAGRAM,
+    IMAGE_CHART,
+    IMAGE_ART,
+    IMAGE_DRAWING,
+    IMAGE_PAINTING,
+    IMAGE_SKETCH,
+    IMAGE_MAP,
+    IMAGE_CHARACTER,
+    IMAGE_SCENE,
+    IMAGE_LANDSCAPE,
+    IMAGE_PORTRAIT,
+    IMAGE_POSTER,
+    IMAGE_LOGO,
+    IMAGE_ICON,
+    IMAGE_SATELLITE_IMAGE,
+    
+    //
+    // Audio Content:
+    //
+    AUDIO_SPEECH,
+    AUDIO_PODCAST,
+    AUDIO_BOOK,
+    AUDIO_INTERVIEW,
+    AUDIO_LECTURE,
+    AUDIO_TALK,
+    AUDIO_SONG,
+    AUDIO_MUSIC,
+    AUDIO_SOUND,
+    AUDIO_CALL,
+    AUDIO_VOICE_ACTING,
+    AUDIO_DESCRIPTION,
+    AUDIO_GUIDE,
+    AUDIO_DIALOGUE,
+    
+    //
+    // Video Content:
+    //
+    VIDEO_MOVIE,
+    VIDEO_FILM,
+    VIDEO_TV_SHOW,
+    VIDEO_SERIES,
+    VIDEO_EPISODE,
+    VIDEO_DOCUMENTARY,
+    VIDEO_TUTORIAL,
+    VIDEO_LECTURE,
+    VIDEO_WEBINAR,
+    VIDEO_GAME,
+    VIDEO_ANIMATION,
+    VIDEO_CUTSCENE,
+    VIDEO_TRAILER,
+    VIDEO_ADVERTISEMENT,
+}

app/MindWork AI Studio/Tools/RAG/RetrievalContentTypeExtensions.cs

@@ -0,0 +1,141 @@
+using AIStudio.Tools.ERIClient.DataModel;
+
+namespace AIStudio.Tools.RAG;
+
+public static class RetrievalContentTypeExtensions
+{
+    public static RetrievalContentType ToRetrievalContentType(Context eriContext)
+    {
+        //
+        // Right now, we have to parse the category string along the type enum to
+        // determine the common retrieval content type. In future ERI versions, we
+        // might use the same enum.
+        //
+        
+        var lowerCategory = eriContext.Category.ToLowerInvariant();
+        var type = eriContext.Type;
+        return type switch
+        {
+            ContentType.TEXT when lowerCategory.Contains("book") => RetrievalContentType.TEXT_BOOK,
+            ContentType.TEXT when lowerCategory.Contains("paper") => RetrievalContentType.TEXT_PAPER,
+            ContentType.TEXT when lowerCategory.Contains("dictionary") => RetrievalContentType.TEXT_DICTIONARY,
+            ContentType.TEXT when lowerCategory.Contains("encyclopedia") => RetrievalContentType.TEXT_ENCYCLOPEDIA,
+            ContentType.TEXT when lowerCategory.Contains("glossary") => RetrievalContentType.TEXT_GLOSSARY,
+            ContentType.TEXT when lowerCategory.Contains("journal") => RetrievalContentType.TEXT_JOURNAL,
+            ContentType.TEXT when lowerCategory.Contains("magazine") => RetrievalContentType.TEXT_MAGAZINE,
+            ContentType.TEXT when lowerCategory.Contains("newspaper") => RetrievalContentType.TEXT_NEWSPAPER,
+            ContentType.TEXT when lowerCategory.Contains("report") => RetrievalContentType.TEXT_REPORT,
+            ContentType.TEXT when lowerCategory.Contains("review") => RetrievalContentType.TEXT_REVIEW,
+            ContentType.TEXT when lowerCategory.Contains("website") => RetrievalContentType.TEXT_WEBSITE,
+            ContentType.TEXT when lowerCategory.Contains("idea") => RetrievalContentType.TEXT_IDEA,
+            
+            ContentType.TEXT when lowerCategory.Contains("business concept") => RetrievalContentType.TEXT_BUSINESS_CONCEPT,
+            ContentType.TEXT when lowerCategory.Contains("concept") => RetrievalContentType.TEXT_CONCEPT,
+            
+            ContentType.TEXT when lowerCategory.Contains("definition") => RetrievalContentType.TEXT_DEFINITION,
+            ContentType.TEXT when lowerCategory.Contains("example") => RetrievalContentType.TEXT_EXAMPLE,
+            ContentType.TEXT when lowerCategory.Contains("quote") => RetrievalContentType.TEXT_QUOTE,
+            ContentType.TEXT when lowerCategory.Contains("article") => RetrievalContentType.TEXT_ARTICLE,
+            ContentType.TEXT when lowerCategory.Contains("chapter") => RetrievalContentType.TEXT_CHAPTER,
+            
+            ContentType.TEXT when lowerCategory.Contains("thesis") => RetrievalContentType.TEXT_THESIS,
+            ContentType.TEXT when lowerCategory.Contains("dissertation") => RetrievalContentType.TEXT_THESIS,
+            
+            ContentType.TEXT when lowerCategory.Contains("draft") => RetrievalContentType.TEXT_DRAFT,
+            ContentType.TEXT when lowerCategory.Contains("script") => RetrievalContentType.TEXT_SCRIPT,
+            ContentType.TEXT when lowerCategory.Contains("transcript") => RetrievalContentType.TEXT_TRANSCRIPT,
+            ContentType.TEXT when lowerCategory.Contains("subtitle") => RetrievalContentType.TEXT_SUBTITLE,
+            ContentType.TEXT when lowerCategory.Contains("caption") => RetrievalContentType.TEXT_CAPTION,
+            ContentType.TEXT when lowerCategory.Contains("dialogue") => RetrievalContentType.TEXT_DIALOGUE,
+            ContentType.TEXT when lowerCategory.Contains("project proposal") => RetrievalContentType.TEXT_PROJECT_PROPOSAL,
+            ContentType.TEXT when lowerCategory.Contains("project plan") => RetrievalContentType.TEXT_PROJECT_PLAN,
+            ContentType.TEXT when lowerCategory.Contains("spreadsheet") => RetrievalContentType.TEXT_SPREADSHEET,
+            
+            ContentType.TEXT when lowerCategory.Contains("presentation") => RetrievalContentType.TEXT_PRESENTATION,
+            ContentType.TEXT when lowerCategory.Contains("powerpoint") => RetrievalContentType.TEXT_PRESENTATION,
+            ContentType.TEXT when lowerCategory.Contains("slide") => RetrievalContentType.TEXT_PRESENTATION,
+            
+            ContentType.TEXT when lowerCategory.Contains("meeting minutes") => RetrievalContentType.TEXT_MEETING_MINUTES,
+            ContentType.TEXT when lowerCategory.Contains("email") => RetrievalContentType.TEXT_EMAIL,
+            ContentType.TEXT when lowerCategory.Contains("protocol") => RetrievalContentType.TEXT_PROTOCOL,
+            
+            ContentType.TEXT => RetrievalContentType.TEXT_DOCUMENT,
+            
+            
+            ContentType.IMAGE when lowerCategory.Contains("photo") => RetrievalContentType.IMAGE_PHOTO,
+            ContentType.IMAGE when lowerCategory.Contains("illustration") => RetrievalContentType.IMAGE_ILLUSTRATION,
+            ContentType.IMAGE when lowerCategory.Contains("diagram") => RetrievalContentType.IMAGE_DIAGRAM,
+            ContentType.IMAGE when lowerCategory.Contains("chart") => RetrievalContentType.IMAGE_CHART,
+            ContentType.IMAGE when lowerCategory.Contains("art") => RetrievalContentType.IMAGE_ART,
+            ContentType.IMAGE when lowerCategory.Contains("drawing") => RetrievalContentType.IMAGE_DRAWING,
+            ContentType.IMAGE when lowerCategory.Contains("painting") => RetrievalContentType.IMAGE_PAINTING,
+            ContentType.IMAGE when lowerCategory.Contains("sketch") => RetrievalContentType.IMAGE_SKETCH,
+            ContentType.IMAGE when lowerCategory.Contains("map") => RetrievalContentType.IMAGE_MAP,
+            ContentType.IMAGE when lowerCategory.Contains("scene") => RetrievalContentType.IMAGE_SCENE,
+            ContentType.IMAGE when lowerCategory.Contains("character") => RetrievalContentType.IMAGE_CHARACTER,
+            ContentType.IMAGE when lowerCategory.Contains("landscape") => RetrievalContentType.IMAGE_LANDSCAPE,
+            ContentType.IMAGE when lowerCategory.Contains("portrait") => RetrievalContentType.IMAGE_PORTRAIT,
+            ContentType.IMAGE when lowerCategory.Contains("poster") => RetrievalContentType.IMAGE_POSTER,
+            ContentType.IMAGE when lowerCategory.Contains("logo") => RetrievalContentType.IMAGE_LOGO,
+            ContentType.IMAGE when lowerCategory.Contains("icon") => RetrievalContentType.IMAGE_ICON,
+            
+            ContentType.IMAGE when lowerCategory.Contains("satellite") => RetrievalContentType.IMAGE_SATELLITE_IMAGE,
+            ContentType.IMAGE when lowerCategory.Contains("EO") => RetrievalContentType.IMAGE_SATELLITE_IMAGE,
+            ContentType.IMAGE when lowerCategory.Contains("earth observation") => RetrievalContentType.IMAGE_SATELLITE_IMAGE,
+            
+            ContentType.IMAGE => RetrievalContentType.NOT_SPECIFIED,
+            
+            
+            ContentType.AUDIO when lowerCategory.Contains("speech") => RetrievalContentType.AUDIO_SPEECH,
+            
+            ContentType.AUDIO when lowerCategory.Contains("podcast") => RetrievalContentType.AUDIO_PODCAST,
+            ContentType.SPEECH when lowerCategory.Contains("podcast") => RetrievalContentType.AUDIO_PODCAST,
+            
+            ContentType.AUDIO when lowerCategory.Contains("book") => RetrievalContentType.AUDIO_BOOK,
+            ContentType.SPEECH when lowerCategory.Contains("book") => RetrievalContentType.AUDIO_BOOK,
+            
+            ContentType.AUDIO when lowerCategory.Contains("interview") => RetrievalContentType.AUDIO_INTERVIEW,
+            ContentType.SPEECH when lowerCategory.Contains("interview") => RetrievalContentType.AUDIO_INTERVIEW,
+            
+            ContentType.AUDIO when lowerCategory.Contains("lecture") => RetrievalContentType.AUDIO_LECTURE,
+            ContentType.SPEECH when lowerCategory.Contains("lecture") => RetrievalContentType.AUDIO_LECTURE,
+            
+            ContentType.AUDIO when lowerCategory.Contains("talk") => RetrievalContentType.AUDIO_TALK,
+            ContentType.SPEECH when lowerCategory.Contains("talk") => RetrievalContentType.AUDIO_TALK,
+            
+            ContentType.AUDIO when lowerCategory.Contains("song") => RetrievalContentType.AUDIO_SONG,
+            ContentType.AUDIO when lowerCategory.Contains("music") => RetrievalContentType.AUDIO_MUSIC,
+            ContentType.AUDIO when lowerCategory.Contains("sound") => RetrievalContentType.AUDIO_SOUND,
+            ContentType.AUDIO when lowerCategory.Contains("call") => RetrievalContentType.AUDIO_CALL,
+            ContentType.AUDIO when lowerCategory.Contains("voice acting") => RetrievalContentType.AUDIO_VOICE_ACTING,
+            ContentType.AUDIO when lowerCategory.Contains("description") => RetrievalContentType.AUDIO_DESCRIPTION,
+            ContentType.AUDIO when lowerCategory.Contains("guide") => RetrievalContentType.AUDIO_GUIDE,
+            ContentType.AUDIO when lowerCategory.Contains("dialogue") => RetrievalContentType.AUDIO_DIALOGUE,
+            
+            ContentType.SPEECH => RetrievalContentType.AUDIO_SPEECH,
+            ContentType.AUDIO => RetrievalContentType.NOT_SPECIFIED,
+            
+            
+            ContentType.VIDEO when lowerCategory.Contains("movie") => RetrievalContentType.VIDEO_MOVIE,
+            ContentType.VIDEO when lowerCategory.Contains("film") => RetrievalContentType.VIDEO_FILM,
+            ContentType.VIDEO when lowerCategory.Contains("tv show") => RetrievalContentType.VIDEO_TV_SHOW,
+            ContentType.VIDEO when lowerCategory.Contains("series") => RetrievalContentType.VIDEO_SERIES,
+            ContentType.VIDEO when lowerCategory.Contains("episode") => RetrievalContentType.VIDEO_EPISODE,
+            ContentType.VIDEO when lowerCategory.Contains("documentary") => RetrievalContentType.VIDEO_DOCUMENTARY,
+            ContentType.VIDEO when lowerCategory.Contains("tutorial") => RetrievalContentType.VIDEO_TUTORIAL,
+            ContentType.VIDEO when lowerCategory.Contains("lecture") => RetrievalContentType.VIDEO_LECTURE,
+            ContentType.VIDEO when lowerCategory.Contains("webinar") => RetrievalContentType.VIDEO_WEBINAR,
+            ContentType.VIDEO when lowerCategory.Contains("game") => RetrievalContentType.VIDEO_GAME,
+            ContentType.VIDEO when lowerCategory.Contains("animation") => RetrievalContentType.VIDEO_ANIMATION,
+            ContentType.VIDEO when lowerCategory.Contains("cutscene") => RetrievalContentType.VIDEO_CUTSCENE,
+            ContentType.VIDEO when lowerCategory.Contains("trailer") => RetrievalContentType.VIDEO_TRAILER,
+            ContentType.VIDEO when lowerCategory.Contains("advertisement") => RetrievalContentType.VIDEO_ADVERTISEMENT,
+            
+            ContentType.VIDEO => RetrievalContentType.NOT_SPECIFIED,
+            
+            ContentType.NONE => RetrievalContentType.NOT_SPECIFIED,
+            ContentType.UNKNOWN => RetrievalContentType.UNKNOWN,
+            _ => RetrievalContentType.UNKNOWN,
+        };
+    }
+}

app/MindWork AI Studio/Tools/RAG/RetrievalImageContext.cs

@@ -0,0 +1,38 @@
+using AIStudio.Chat;
+
+namespace AIStudio.Tools.RAG;
+
+public sealed class RetrievalImageContext : IRetrievalContext
+{
+    #region Implementation of IRetrievalContext
+
+    /// <inheritdoc />
+    public required string DataSourceName { get; init; }
+
+    /// <inheritdoc />
+    public required RetrievalContentCategory Category { get; init; }
+
+    /// <inheritdoc />
+    public required RetrievalContentType Type { get; init; }
+
+    /// <inheritdoc />
+    public required string Path { get; init; }
+
+    /// <inheritdoc />
+    public IReadOnlyList<string> Links { get; init; } = [];
+
+    #endregion
+
+    /// <summary>
+    /// The type of the image source.
+    /// </summary>
+    /// <remarks>
+    /// Is the image source a URL, a local file path, a base64 string, etc.?
+    /// </remarks>
+    public required ContentImageSource SourceType { get; init; }
+
+    /// <summary>
+    /// The image source.
+    /// </summary>
+    public required string Source { get; set; }
+}

app/MindWork AI Studio/Tools/RAG/RetrievalTextContext.cs

@@ -0,0 +1,43 @@
+namespace AIStudio.Tools.RAG;
+
+/// <summary>
+/// The retrieval context for text data.
+/// </summary>
+public sealed class RetrievalTextContext : IRetrievalContext
+{
+    #region Implementation of IRetrievalContext
+    
+    /// <inheritdoc />
+    public required string DataSourceName { get; init; }
+    
+    /// <inheritdoc />
+    public required RetrievalContentCategory Category { get; init; }
+    
+    /// <inheritdoc />
+    public required RetrievalContentType Type { get; init; }
+    
+    /// <inheritdoc />
+    public required string Path { get; init; }
+
+    /// <inheritdoc />
+    public IReadOnlyList<string> Links { get; init; } = [];
+
+    #endregion
+
+    /// <summary>
+    /// The text content.
+    /// </summary>
+    /// <remarks>
+    /// Should contain the matched text and some small context around it.
+    /// </remarks>
+    public required string MatchedText { get; set; }
+
+    /// <summary>
+    /// The surrounding content of the matched text.
+    /// </summary>
+    /// <remarks>
+    /// Might give the user some context about the matched text.
+    /// For example, one sentence or paragraph before and after the matched text.
+    /// </remarks>
+    public IReadOnlyList<string> SurroundingContent { get; set; } = [];
+}