mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-10-13 14:53:49 +00:00
Let the sources of your own documents open at the page they were found on (#974)
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions
This commit is contained in:
1 parent
6ce7d856a3
commit
1ebe8eb2a4
37 files changed
+1831
-120
No files matched your search
@@ -1,17 +1,21 @@
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Content which a reader held back, together with the token count of exactly that content.
|
||||
/// Content which a reader held back, together with the token count and the page of exactly that
|
||||
/// content.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Readers which assemble a page or a slide from several stream events cannot pass their content
|
||||
/// on right away. Its token count has to travel with it: the count describes the content, not the
|
||||
/// event which happened to arrive at the moment the content was released. Keeping the two together
|
||||
/// is what stops a page from being sized by the text of the page after it.
|
||||
/// is what stops a page from being sized by the text of the page after it. The page number travels
|
||||
/// for the very same reason, and because a number the runtime already stated must not be derived
|
||||
/// from the text again further down the line.
|
||||
/// </remarks>
|
||||
/// <param name="Content">The assembled content.</param>
|
||||
/// <param name="TokenCount">The number of tokens of that content, or null when it is unknown.</param>
|
||||
public readonly record struct ContentStreamPendingContent(string Content, int? TokenCount)
|
||||
/// <param name="PageNumber">The page that content came from, or null when it has none.</param>
|
||||
public readonly record struct ContentStreamPendingContent(string Content, int? TokenCount, int? PageNumber = null)
|
||||
{
|
||||
/// <summary>
|
||||
/// Adds up two token counts, where an unknown count makes the sum unknown as well.
|
||||
|
||||
@@ -12,7 +12,8 @@ namespace AIStudio.Tools;
|
||||
/// <param name="Error">The reported failure, or null when the event was processed successfully.</param>
|
||||
/// <param name="PromptInjection">What the runtime filtered out of the content, or null when it filtered nothing.</param>
|
||||
/// <param name="TokenCount">The number of tokens of the content, or null when it is unknown.</param>
|
||||
public readonly record struct ContentStreamProcessedEvent(string? Content, ContentStreamErrorDetails? Error, ContentStreamPromptInjectionDetails? PromptInjection = null, int? TokenCount = null)
|
||||
/// <param name="PageNumber">The page the content came from, or null when it has none.</param>
|
||||
public readonly record struct ContentStreamProcessedEvent(string? Content, ContentStreamErrorDetails? Error, ContentStreamPromptInjectionDetails? PromptInjection = null, int? TokenCount = null, int? PageNumber = null)
|
||||
{
|
||||
/// <summary>
|
||||
/// An event which neither produced content nor reported a failure.
|
||||
@@ -20,16 +21,18 @@ public readonly record struct ContentStreamProcessedEvent(string? Content, Conte
|
||||
public static readonly ContentStreamProcessedEvent NOTHING = new(null, null);
|
||||
|
||||
/// <summary>
|
||||
/// An event which produced content, with the token count of that very content.
|
||||
/// An event which produced content, with the token count and the page of that very content.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The count travels with the content because a reader may hold content back across several
|
||||
/// events: pairing it with the count of the event which released it would size it by the
|
||||
/// wrong text.
|
||||
/// wrong text. The page travels along for the same reason, and so that whoever indexes the
|
||||
/// content is told where it came from instead of having to read it back out of the text.
|
||||
/// </remarks>
|
||||
/// <param name="content">The content to append.</param>
|
||||
/// <param name="tokenCount">The number of tokens of that content, or null when it is unknown.</param>
|
||||
public static ContentStreamProcessedEvent FromContent(string? content, int? tokenCount = null) => new(content, null, TokenCount: tokenCount);
|
||||
/// <param name="pageNumber">The page that content came from, or null when it has none.</param>
|
||||
public static ContentStreamProcessedEvent FromContent(string? content, int? tokenCount = null, int? pageNumber = null) => new(content, null, TokenCount: tokenCount, PageNumber: pageNumber);
|
||||
|
||||
public static ContentStreamProcessedEvent FromError(ContentStreamErrorDetails? error) => new(null, error);
|
||||
|
||||
|
||||
@@ -19,13 +19,19 @@ public static class ContentStreamSseHandler
|
||||
case ContentStreamTextMetadata:
|
||||
return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount);
|
||||
|
||||
//
|
||||
// The heading tells the AI which page it is reading. The number is handed on
|
||||
// separately as well, because whoever indexes this content needs it as a
|
||||
// number: reading it back out of the heading would mean guessing at something
|
||||
// the runtime already stated.
|
||||
//
|
||||
case ContentStreamPdfMetadata pdfMetadata:
|
||||
var pageNumber = pdfMetadata.Pdf?.PageNumber ?? 0;
|
||||
return ContentStreamProcessedEvent.FromContent($"""
|
||||
# Page {pageNumber}
|
||||
{sseEvent.Content}
|
||||
|
||||
""", sseEvent.TokenCount);
|
||||
""", sseEvent.TokenCount, pageNumber > 0 ? pageNumber : null);
|
||||
|
||||
case ContentStreamSpreadsheetMetadata spreadsheetMetadata:
|
||||
var sheetName = spreadsheetMetadata.Spreadsheet?.SheetName;
|
||||
@@ -45,9 +51,10 @@ public static class ContentStreamSseHandler
|
||||
// a page can follow its Markdown. Documents converted as a whole, e.g. by Pandoc,
|
||||
// carry no page number and are passed on unchanged.
|
||||
//
|
||||
// The buffering is why the count comes back from the reader rather than from
|
||||
// this event: the page which is released here arrived one event ago, and this
|
||||
// event's count belongs to the page which is now being buffered.
|
||||
// The buffering is why the count and the page come back from the reader rather
|
||||
// than from this event: the page which is released here arrived one event ago,
|
||||
// and this event's count and number belong to the page which is now being
|
||||
// buffered.
|
||||
//
|
||||
case ContentStreamDocumentMetadata documentMetadata:
|
||||
if (documentMetadata.Document?.PageNumber is not > 0)
|
||||
@@ -55,7 +62,7 @@ public static class ContentStreamSseHandler
|
||||
|
||||
var documentManager = DOCUMENT_MANAGERS.GetOrAdd(sseEvent.StreamId!, _ => new());
|
||||
var documentContent = documentManager.AddPage(documentMetadata, sseEvent.Content, sseEvent.TokenCount, extractImages);
|
||||
return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent.Value.Content, documentContent.Value.TokenCount);
|
||||
return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent.Value.Content, documentContent.Value.TokenCount, documentContent.Value.PageNumber);
|
||||
|
||||
case ContentStreamImageMetadata:
|
||||
return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount);
|
||||
@@ -184,7 +191,9 @@ public static class ContentStreamSseHandler
|
||||
/// <remarks>
|
||||
/// The readers which assemble pages or slides always keep the last one of them: nothing tells
|
||||
/// them that no further image is coming. It is released here, and it carries its own token
|
||||
/// count, because a chunk without one cannot be sized by the caller.
|
||||
/// count, because a chunk without one cannot be sized by the caller. Only the page reader
|
||||
/// states a page; a stream is read by one of them, so there is no second number to weigh
|
||||
/// against.
|
||||
/// </remarks>
|
||||
/// <param name="streamId">The stream to release and forget.</param>
|
||||
/// <returns>The content which was held back, or null when there was none.</returns>
|
||||
@@ -195,6 +204,7 @@ public static class ContentStreamSseHandler
|
||||
|
||||
var finalContentChunk = new StringBuilder();
|
||||
int? tokenCount = 0;
|
||||
int? pageNumber = null;
|
||||
if(SLIDE_MANAGERS.TryGetValue(streamId, out var slideManager)
|
||||
&& slideManager.GetAllSlidesInOrder() is { } slides
|
||||
&& !string.IsNullOrWhiteSpace(slides.Content))
|
||||
@@ -209,6 +219,7 @@ public static class ContentStreamSseHandler
|
||||
{
|
||||
finalContentChunk.Append(page.Content);
|
||||
tokenCount = ContentStreamPendingContent.AddTokenCounts(tokenCount, page.TokenCount);
|
||||
pageNumber = page.PageNumber;
|
||||
}
|
||||
|
||||
SLIDE_MANAGERS.TryRemove(streamId, out _);
|
||||
@@ -217,6 +228,6 @@ public static class ContentStreamSseHandler
|
||||
foreach (var key in CHUNKED_IMAGES.Keys.Where(k => k.StartsWith(imageIdPrefix, StringComparison.InvariantCultureIgnoreCase)))
|
||||
CHUNKED_IMAGES.TryRemove(key, out _);
|
||||
|
||||
return finalContentChunk.Length > 0 ? new ContentStreamPendingContent(finalContentChunk.ToString(), tokenCount) : null;
|
||||
return finalContentChunk.Length > 0 ? new ContentStreamPendingContent(finalContentChunk.ToString(), tokenCount, pageNumber) : null;
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ public sealed class DocumentManager
|
||||
{
|
||||
private StringBuilder? currentPageContent;
|
||||
private int? currentPageTokenCount;
|
||||
private int? currentPageNumber;
|
||||
|
||||
public ContentStreamPendingContent? AddPage(ContentStreamDocumentMetadata metadata, string? content, int? tokenCount, bool extractImages)
|
||||
{
|
||||
@@ -36,9 +37,12 @@ public sealed class DocumentManager
|
||||
|
||||
//
|
||||
// The count waits here together with the page it belongs to. Handing it out along with
|
||||
// the page we just completed would size that page by the text of this one.
|
||||
// the page we just completed would size that page by the text of this one. The page
|
||||
// number waits for the same reason: it belongs to the page being buffered, not to the
|
||||
// one leaving here.
|
||||
//
|
||||
this.currentPageTokenCount = tokenCount;
|
||||
this.currentPageNumber = pageNumber;
|
||||
return completedPage;
|
||||
}
|
||||
|
||||
@@ -72,8 +76,10 @@ public sealed class DocumentManager
|
||||
|
||||
var result = this.currentPageContent.ToString();
|
||||
var tokenCount = this.currentPageTokenCount;
|
||||
var pageNumber = this.currentPageNumber;
|
||||
this.currentPageContent = null;
|
||||
this.currentPageTokenCount = null;
|
||||
return string.IsNullOrWhiteSpace(result) ? null : new ContentStreamPendingContent(result, tokenCount);
|
||||
this.currentPageNumber = null;
|
||||
return string.IsNullOrWhiteSpace(result) ? null : new ContentStreamPendingContent(result, tokenCount, pageNumber);
|
||||
}
|
||||
}
|
||||
@@ -204,6 +204,26 @@ public static class FileExportFormatExtensions
|
||||
_ => WITHOUT_BYTE_ORDER_MARK,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Determines whether a link into a local file may name the page it points at.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A page is named by the fragment of the link, the way the PDF open parameters call for. A
|
||||
/// browser and a PDF reader follow that and open the document on the page; Word and LibreOffice
|
||||
/// take the fragment for part of the file name, look for a file which does not exist, and refuse
|
||||
/// the link altogether. There the page is dropped, so the link at least opens the document --
|
||||
/// which page it was stays in the title of the source. Verified on 2026-09-15 with LibreOffice
|
||||
/// on an exported .odt. A format added later keeps the page unless it is known to stumble too.
|
||||
/// </remarks>
|
||||
/// <param name="format">The format.</param>
|
||||
/// <returns>True, when a reader of this format follows such a link.</returns>
|
||||
public static bool FollowsPageAnchors(this FileExportFormat format) => format switch
|
||||
{
|
||||
FileExportFormat.MICROSOFT_WORD or FileExportFormat.OPEN_DOCUMENT_TEXT => false,
|
||||
|
||||
_ => true,
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
/// Returns the name Pandoc knows the format by.
|
||||
/// </summary>
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// A source together with the number it is listed under.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The number runs through the whole list rather than starting over per group, because that is how
|
||||
/// an answer refers to a source. It is assigned once, where the groups are formed, so the chat and
|
||||
/// an exported document cannot end up numbering the same list differently.
|
||||
/// </remarks>
|
||||
/// <param name="Number">The number this source is listed under, counted from one.</param>
|
||||
/// <param name="Source">The source itself.</param>
|
||||
public readonly record struct NumberedSource(int Number, Source Source);
|
||||
@@ -118,7 +118,7 @@ public static class PandocExport
|
||||
// We read the text before we ask for a path: when there is nothing to convert, the user
|
||||
// should learn that right away instead of picking a file first and getting an error afterwards.
|
||||
//
|
||||
if (!markdownContent.TryGetExportMarkdown(out var markdownText))
|
||||
if (!markdownContent.TryGetExportMarkdown(out var markdownText, format.FollowsPageAnchors()))
|
||||
{
|
||||
LOGGER.LogWarning("Cannot export the content as {ExportFormat}, because it carries no text.", format);
|
||||
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Cancel, TB("Only text messages can be exported.")));
|
||||
|
||||
@@ -8,7 +8,36 @@ namespace AIStudio.Tools.RAG;
|
||||
public static class IRetrievalContextExtensions
|
||||
{
|
||||
private static readonly ILogger<IRetrievalContext> LOGGER = Program.LOGGER_FACTORY.CreateLogger<IRetrievalContext>();
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Writes what the AI is told about a retrieval context, before its content follows.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The location is what lets the AI say where an answer comes from. Naming only the file is
|
||||
/// not enough in a document of two hundred pages, and we know the page: it travels from the
|
||||
/// runtime through the index into the context. A slide or a sheet has no page, and then
|
||||
/// nothing is claimed rather than something made up.
|
||||
/// </remarks>
|
||||
/// <param name="contextBuilder">The builder to write into.</param>
|
||||
/// <param name="retrievalContext">The context to describe.</param>
|
||||
internal static void AppendContextDescription(StringBuilder contextBuilder, IRetrievalContext retrievalContext)
|
||||
{
|
||||
contextBuilder.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
|
||||
contextBuilder.AppendLine($"Content category: {retrievalContext.Category}");
|
||||
contextBuilder.AppendLine($"Content type: {retrievalContext.Type}");
|
||||
contextBuilder.AppendLine($"Content path: {retrievalContext.Path}");
|
||||
|
||||
if(retrievalContext is RetrievalTextContext { PageNumber: > 0 } locatedContext)
|
||||
contextBuilder.AppendLine($"Content location: page {locatedContext.PageNumber}");
|
||||
|
||||
if(retrievalContext.Links.Count is 0)
|
||||
return;
|
||||
|
||||
contextBuilder.AppendLine("Additional links:");
|
||||
foreach(var link in retrievalContext.Links)
|
||||
contextBuilder.AppendLine($"- {link}");
|
||||
}
|
||||
|
||||
public static async Task<string> AsMarkdown(this IReadOnlyList<IRetrievalContext> retrievalContexts, StringBuilder? sb = null, CancellationToken token = default)
|
||||
{
|
||||
sb ??= new StringBuilder();
|
||||
@@ -49,17 +78,7 @@ public static class IRetrievalContextExtensions
|
||||
break;
|
||||
}
|
||||
|
||||
contextBuilder.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
|
||||
contextBuilder.AppendLine($"Content category: {retrievalContext.Category}");
|
||||
contextBuilder.AppendLine($"Content type: {retrievalContext.Type}");
|
||||
contextBuilder.AppendLine($"Content path: {retrievalContext.Path}");
|
||||
|
||||
if(retrievalContext.Links.Count > 0)
|
||||
{
|
||||
contextBuilder.AppendLine("Additional links:");
|
||||
foreach(var link in retrievalContext.Links)
|
||||
contextBuilder.AppendLine($"- {link}");
|
||||
}
|
||||
AppendContextDescription(contextBuilder, retrievalContext);
|
||||
|
||||
var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
|
||||
var source = PromptInjectionSource.RetrievalContext(retrievalContext.DataSourceName, retrievalContext.Path);
|
||||
|
||||
@@ -50,4 +50,14 @@ public sealed class RetrievalTextContext : IRetrievalContext
|
||||
/// Optional link used when this context is displayed as a source reference.
|
||||
/// </summary>
|
||||
public string ReferenceLink { get; init; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// The page this passage was found on, or null when it has none.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Kept as a number rather than only inside the reference title: the AI is told the page so it
|
||||
/// can say where an answer comes from, and a source has to name a page a program can be sent
|
||||
/// to. A slide or a sheet has no page and leaves this empty.
|
||||
/// </remarks>
|
||||
public int? PageNumber { get; init; }
|
||||
}
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace AIStudio.Tools.Rust;
|
||||
|
||||
/// <summary>
|
||||
/// Asks the runtime to open a document in the program the system uses for it.
|
||||
/// </summary>
|
||||
/// <param name="Path">The document to open.</param>
|
||||
/// <param name="Page">The page to show, counted from one, or null when the document has none.</param>
|
||||
public readonly record struct OpenDocumentRequest(string Path, int? Page);
|
||||
@@ -0,0 +1,14 @@
|
||||
namespace AIStudio.Tools.Rust;
|
||||
|
||||
/// <summary>
|
||||
/// Says how opening a document went.
|
||||
/// </summary>
|
||||
/// <param name="Success">Whether the document was opened at all.</param>
|
||||
/// <param name="PageApplied">
|
||||
/// Whether the document was handed to its program together with the page. False means it opens on
|
||||
/// its first page: no page was asked for, the system uses a program which cannot be told one, or
|
||||
/// starting that program failed. None of these is an error, so this belongs in the log rather than
|
||||
/// in front of the user, who is told the page by the source itself.
|
||||
/// </param>
|
||||
/// <param name="Issue">Why the document could not be opened, or an empty text when it was.</param>
|
||||
public readonly record struct OpenDocumentResponse(bool Success, bool PageApplied, string Issue);
|
||||
@@ -1,3 +1,9 @@
|
||||
namespace AIStudio.Tools.Services;
|
||||
|
||||
public sealed record ArbitraryFileDataSegment(string Content, int TokenCount);
|
||||
/// <summary>
|
||||
/// One piece of an extracted file, as the runtime delivered it.
|
||||
/// </summary>
|
||||
/// <param name="Content">The extracted text.</param>
|
||||
/// <param name="TokenCount">The number of tokens of that text.</param>
|
||||
/// <param name="PageNumber">The page that text came from, or null when it has none. Presentations and spreadsheets have none.</param>
|
||||
public sealed record ArbitraryFileDataSegment(string Content, int TokenCount, int? PageNumber);
|
||||
@@ -16,6 +16,22 @@ public sealed partial class DataSourceEmbeddingService
|
||||
internal const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300;
|
||||
private const bool IMAGE_EMBEDDING_ENABLED = false;
|
||||
|
||||
/// <summary>
|
||||
/// What this build writes next to a chunk besides its text. Raise it whenever that changes.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A stored chunk keeps the metadata of the run which wrote it, and nothing recomputes it: the
|
||||
/// fingerprint of a file says whether the file changed, not whether we got better at reading
|
||||
/// it. Raising this number makes the embedding signature differ, which drops the index and
|
||||
/// builds it again — the only way corrected page numbers reach a data source somebody indexed
|
||||
/// earlier.
|
||||
///
|
||||
/// Version 2: the page of a chunk is taken from the runtime metadata instead of being read back
|
||||
/// out of the chunk text, which is what left Word and OpenDocument files, and passages
|
||||
/// continuing across a page break, without a page.
|
||||
/// </remarks>
|
||||
private const string CHUNK_METADATA_VERSION = "2";
|
||||
|
||||
private enum RagFileIndexingDecision
|
||||
{
|
||||
INDEXABLE,
|
||||
@@ -23,10 +39,23 @@ public sealed partial class DataSourceEmbeddingService
|
||||
UNSUPPORTED,
|
||||
}
|
||||
|
||||
private sealed record ExtractedFileSegment(string Text, int? TokenCount);
|
||||
private sealed record ExtractedFileSegment(string Text, int? TokenCount, int? PageNumber);
|
||||
|
||||
private sealed record ExtractedFileContent(string Text, IReadOnlyList<ExtractedFileSegment> SourceSegments);
|
||||
|
||||
/// <summary>
|
||||
/// One chunk as the chunking produced it, together with the page it starts on.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The page is carried rather than read back out of the chunk text. The runtime states it, and
|
||||
/// the chunking knows which source segment a chunk begins in, so nothing has to be derived from
|
||||
/// a marker in the text — which is what used to leave Word files and continued passages without
|
||||
/// a page.
|
||||
/// </remarks>
|
||||
/// <param name="Text">The chunk itself, overlap prefix included.</param>
|
||||
/// <param name="PageNumber">The page the chunk's own content starts on, or null when it has none.</param>
|
||||
private sealed record EmbeddingChunk(string Text, int? PageNumber);
|
||||
|
||||
private sealed record EmbeddingChunkDraft(string ChunkId, string Text, int ChunkIndex, int? PageNumber);
|
||||
|
||||
private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength);
|
||||
@@ -37,7 +66,7 @@ public sealed partial class DataSourceEmbeddingService
|
||||
|
||||
private sealed record DataSourceMetadataSnapshot(string SourceHash, IReadOnlyDictionary<string, string> FileHashes);
|
||||
|
||||
private async IAsyncEnumerable<string> StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
|
||||
private async IAsyncEnumerable<EmbeddingChunk> StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
var options = this.GetChunkingOptions(dataSource, embeddingProvider);
|
||||
var strategy = this.GetChunkingStrategy(filePath);
|
||||
@@ -55,26 +84,31 @@ public sealed partial class DataSourceEmbeddingService
|
||||
{
|
||||
var normalized = NormalizeChunkSegment(segment.Content);
|
||||
if (!string.IsNullOrWhiteSpace(normalized))
|
||||
segments.Add(new(normalized, segment.TokenCount));
|
||||
segments.Add(new(normalized, segment.TokenCount, segment.PageNumber));
|
||||
}
|
||||
|
||||
return new(string.Join("\n", segments.Select(segment => segment.Text)).Trim(), segments);
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<string> SplitByChunkingStrategyAsync(ExtractedFileContent content, ChunkingStrategy strategy, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
|
||||
private async IAsyncEnumerable<EmbeddingChunk> SplitByChunkingStrategyAsync(ExtractedFileContent content, ChunkingStrategy strategy, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
|
||||
{
|
||||
var estimatedTokenCount = SumTokenCounts(content.SourceSegments);
|
||||
await foreach (var chunk in this.SplitTextByRulesAsync(content.Text, content.SourceSegments, strategy, 0, options, embeddingProvider, token, estimatedTokenCount: estimatedTokenCount))
|
||||
|
||||
// The whole text starts where the first segment starts, so that is the page it is on until
|
||||
// the splitting reaches a segment boundary:
|
||||
var firstPageNumber = content.SourceSegments.Count > 0 ? content.SourceSegments[0].PageNumber : null;
|
||||
await foreach (var chunk in this.SplitTextByRulesAsync(content.Text, content.SourceSegments, strategy, 0, options, embeddingProvider, firstPageNumber, token, estimatedTokenCount: estimatedTokenCount))
|
||||
yield return chunk;
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<string> SplitTextByRulesAsync(
|
||||
private async IAsyncEnumerable<EmbeddingChunk> SplitTextByRulesAsync(
|
||||
string text,
|
||||
IReadOnlyList<ExtractedFileSegment> sourceSegments,
|
||||
ChunkingStrategy strategy,
|
||||
int ruleIndex,
|
||||
ChunkingOptions options,
|
||||
EmbeddingProvider embeddingProvider,
|
||||
int? currentPageNumber,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token,
|
||||
string requiredOverlapPrefix = "",
|
||||
int? estimatedTokenCount = null)
|
||||
@@ -91,14 +125,14 @@ public sealed partial class DataSourceEmbeddingService
|
||||
tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, textWithOverlap, token);
|
||||
if (tokenCount <= options.MaxChunkTokenLength)
|
||||
{
|
||||
yield return textWithOverlap;
|
||||
yield return new(textWithOverlap, currentPageNumber);
|
||||
yield break;
|
||||
}
|
||||
}
|
||||
|
||||
if (ruleIndex >= strategy.Rules.Count)
|
||||
{
|
||||
await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token, requiredOverlapPrefix, estimatedTokenCount))
|
||||
await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, currentPageNumber, token, requiredOverlapPrefix, estimatedTokenCount))
|
||||
yield return hardChunk;
|
||||
|
||||
yield break;
|
||||
@@ -107,7 +141,7 @@ public sealed partial class DataSourceEmbeddingService
|
||||
var rule = strategy.Rules[ruleIndex];
|
||||
if (rule.Split is null)
|
||||
{
|
||||
await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token, requiredOverlapPrefix, estimatedTokenCount))
|
||||
await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, currentPageNumber, token, requiredOverlapPrefix, estimatedTokenCount))
|
||||
yield return hardChunk;
|
||||
|
||||
yield break;
|
||||
@@ -116,7 +150,7 @@ public sealed partial class DataSourceEmbeddingService
|
||||
var units = NormalizeSplitUnits(rule.Split(text, sourceSegments.Select(segment => segment.Text).ToList()), text);
|
||||
if (units.Count <= 1)
|
||||
{
|
||||
await foreach (var chunk in this.SplitTextByRulesAsync(text, sourceSegments, strategy, ruleIndex + 1, options, embeddingProvider, token, requiredOverlapPrefix, estimatedTokenCount))
|
||||
await foreach (var chunk in this.SplitTextByRulesAsync(text, sourceSegments, strategy, ruleIndex + 1, options, embeddingProvider, currentPageNumber, token, requiredOverlapPrefix, estimatedTokenCount))
|
||||
yield return chunk;
|
||||
|
||||
yield break;
|
||||
@@ -135,6 +169,15 @@ public sealed partial class DataSourceEmbeddingService
|
||||
var overlapPrefix = requiredOverlapPrefix;
|
||||
var unitTokenCounts = EstimateSplitUnitTokenCounts(units, sourceSegments, rule.UsesSourceSegmentCounts, estimatedTokenCount);
|
||||
|
||||
//
|
||||
// The first rule of every strategy cuts along the segments the runtime delivered, so there
|
||||
// a unit is a segment and carries that segment's page. Every later rule cuts inside a
|
||||
// single segment, where all units share the page they were handed. This is what ties a
|
||||
// chunk to a page without anybody reading the text.
|
||||
//
|
||||
var unitsAreSourceSegments = rule.UsesSourceSegmentCounts && sourceSegments.Count == units.Count;
|
||||
int? PageOfUnit(int unitIndex) => unitsAreSourceSegments ? sourceSegments[unitIndex].PageNumber ?? currentPageNumber : currentPageNumber;
|
||||
|
||||
while (index < units.Count)
|
||||
{
|
||||
token.ThrowIfCancellationRequested();
|
||||
@@ -145,8 +188,14 @@ public sealed partial class DataSourceEmbeddingService
|
||||
var rawChunk = string.Concat(units.Skip(index).Take(unitCount)).Trim();
|
||||
var chunk = AddOverlapPrefix(rawChunk, overlapPrefix);
|
||||
overlapPrefix = string.Empty;
|
||||
|
||||
//
|
||||
// The page of the first unit this chunk covers, not of the overlap prefix in front
|
||||
// of it: the prefix repeats what the chunk before already said, while the page has
|
||||
// to name where this chunk's own content begins.
|
||||
//
|
||||
if (!string.IsNullOrWhiteSpace(chunk))
|
||||
yield return chunk;
|
||||
yield return new(chunk, PageOfUnit(index));
|
||||
|
||||
var nextIndex = index + unitCount;
|
||||
if (nextIndex >= units.Count)
|
||||
@@ -178,9 +227,10 @@ public sealed partial class DataSourceEmbeddingService
|
||||
|
||||
string? lastSplitUnit = null;
|
||||
var unitTokenCount = unitTokenCounts?[index];
|
||||
await foreach (var splitUnit in this.SplitTextByRulesAsync(units[index], [new(units[index], unitTokenCount)], strategy, ruleIndex + 1, options, embeddingProvider, token, overlapPrefix, unitTokenCount))
|
||||
var unitPageNumber = PageOfUnit(index);
|
||||
await foreach (var splitUnit in this.SplitTextByRulesAsync(units[index], [new(units[index], unitTokenCount, unitPageNumber)], strategy, ruleIndex + 1, options, embeddingProvider, unitPageNumber, token, overlapPrefix, unitTokenCount))
|
||||
{
|
||||
lastSplitUnit = splitUnit;
|
||||
lastSplitUnit = splitUnit.Text;
|
||||
yield return splitUnit;
|
||||
}
|
||||
|
||||
@@ -372,10 +422,15 @@ public sealed partial class DataSourceEmbeddingService
|
||||
return bestStartIndex <= chunkStartIndex ? chunkEndIndex : bestStartIndex;
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<string> SplitTextByHardCutAsync(
|
||||
/// <remarks>
|
||||
/// The hard cut is only ever reached inside a single piece of text which no rule could split
|
||||
/// any further, so every chunk it produces sits on the page that piece was handed.
|
||||
/// </remarks>
|
||||
private async IAsyncEnumerable<EmbeddingChunk> SplitTextByHardCutAsync(
|
||||
string text,
|
||||
ChunkingOptions options,
|
||||
EmbeddingProvider embeddingProvider,
|
||||
int? currentPageNumber,
|
||||
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token,
|
||||
string requiredOverlapPrefix = "",
|
||||
int? estimatedTokenCount = null)
|
||||
@@ -455,7 +510,7 @@ public sealed partial class DataSourceEmbeddingService
|
||||
|
||||
var chunk = AddOverlapPrefix(text[startIndex..bestEndIndex].Trim(), overlapPrefix);
|
||||
if (!string.IsNullOrWhiteSpace(chunk))
|
||||
yield return chunk;
|
||||
yield return new(chunk, currentPageNumber);
|
||||
|
||||
if (bestEndIndex >= text.Length)
|
||||
yield break;
|
||||
@@ -934,6 +989,7 @@ public sealed partial class DataSourceEmbeddingService
|
||||
private string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions)
|
||||
{
|
||||
return string.Join('|',
|
||||
CHUNK_METADATA_VERSION,
|
||||
embeddingProvider.Id,
|
||||
embeddingProvider.UsedLLMProvider,
|
||||
embeddingProvider.Model.Id,
|
||||
@@ -1086,14 +1142,6 @@ public sealed partial class DataSourceEmbeddingService
|
||||
return string.IsNullOrWhiteSpace(extension) ? "unknown" : extension;
|
||||
}
|
||||
|
||||
private static int? TryExtractPageNumber(string chunk)
|
||||
{
|
||||
var match = Regex.Match(chunk, @"^\s*#\s+Page\s+(\d+)\b", RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
|
||||
return match.Success && int.TryParse(match.Groups[1].Value, out var pageNumber) && pageNumber > 0
|
||||
? pageNumber
|
||||
: null;
|
||||
}
|
||||
|
||||
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) =>
|
||||
CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
|
||||
|
||||
|
||||
@@ -843,7 +843,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
|
||||
|
||||
await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, dataSource, embeddingProvider, token))
|
||||
{
|
||||
batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk, totalChunkCount, TryExtractPageNumber(chunk)));
|
||||
batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber));
|
||||
totalChunkCount++;
|
||||
|
||||
if (batch.Count >= embeddingBatchSize)
|
||||
|
||||
@@ -395,6 +395,7 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
SurroundingContent = [],
|
||||
ReferenceTitle = BuildReferenceTitle(hit),
|
||||
ReferenceLink = referenceLink,
|
||||
PageNumber = hit.PageNumber is > 0 ? hit.PageNumber : null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -413,11 +414,19 @@ public sealed class DataSourceLocalRetrievalService(
|
||||
return $"{sourceName} ({location})";
|
||||
}
|
||||
|
||||
/// <remarks>
|
||||
/// A known page is written as the fragment `#page=N`, which is what the PDF open parameters
|
||||
/// call for: a program which understands them opens the document where the passage is. Without
|
||||
/// a page there is nothing to send a program to, and the chunk stays in the link so the
|
||||
/// reference still points at something.
|
||||
/// </remarks>
|
||||
private static string BuildReferenceLink(string path, LocalRetrievalHit hit)
|
||||
{
|
||||
var link = NormalizeLocalReferencePath(path);
|
||||
var separator = link.Contains('#', StringComparison.Ordinal) ? "&" : "#";
|
||||
return $"{link}{separator}chunk={hit.ChunkIndex}";
|
||||
return hit.PageNumber is > 0
|
||||
? $"{link}{separator}page={hit.PageNumber}"
|
||||
: $"{link}{separator}chunk={hit.ChunkIndex}";
|
||||
}
|
||||
|
||||
private static string NormalizeLocalReferencePath(string path)
|
||||
|
||||
@@ -168,4 +168,66 @@ public sealed partial class RustService
|
||||
result.Dispose();
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Opens a document in the program the system uses for it, on the given page where possible.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// The page is best effort and never decides whether this succeeded. Which programs can be
|
||||
/// told a page is the runtime's business, and it says afterwards whether it managed to.
|
||||
/// </remarks>
|
||||
/// <param name="path">The document to open.</param>
|
||||
/// <param name="pageNumber">The page to show, counted from one, or null when there is none.</param>
|
||||
/// <returns>Whether the document was opened, whether the page was applied, and what went wrong.</returns>
|
||||
public async Task<OpenDocumentResponse> TryOpenDocumentInSystemViewer(string path, int? pageNumber)
|
||||
{
|
||||
HttpResponseMessage result;
|
||||
try
|
||||
{
|
||||
result = await this.http.PostAsJsonAsync("/open/document", new OpenDocumentRequest(path, pageNumber), this.jsonRustSerializerOptions);
|
||||
}
|
||||
catch (HttpRequestException e)
|
||||
{
|
||||
this.logger!.LogWarning(e, "Failed to reach the Rust runtime document endpoint.");
|
||||
return new OpenDocumentResponse(false, false, TB("The runtime document endpoint is not available."));
|
||||
}
|
||||
catch (TaskCanceledException e)
|
||||
{
|
||||
this.logger!.LogWarning(e, "Timed out while reaching the Rust runtime document endpoint.");
|
||||
return new OpenDocumentResponse(false, false, TB("The runtime document endpoint is not available."));
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (!result.IsSuccessStatusCode)
|
||||
{
|
||||
this.logger!.LogWarning("Failed to open a document through the Rust runtime: '{StatusCode}'", result.StatusCode);
|
||||
return new OpenDocumentResponse(false, false, string.Format(TB("The runtime document endpoint returned '{0}'."), result.StatusCode));
|
||||
}
|
||||
|
||||
var response = await result.Content.ReadFromJsonAsync<OpenDocumentResponse>(this.jsonRustSerializerOptions);
|
||||
if (response.Success)
|
||||
{
|
||||
//
|
||||
// A page which was asked for but not applied is noted here and nowhere else: the
|
||||
// document is open, and the source the user clicked names the page anyway.
|
||||
//
|
||||
if (pageNumber is > 0 && !response.PageApplied)
|
||||
this.logger!.LogInformation("Opened a document without the requested page {PageNumber}, because the system uses a program which cannot be told one.", pageNumber);
|
||||
|
||||
return response;
|
||||
}
|
||||
|
||||
return new OpenDocumentResponse(false, false, string.IsNullOrWhiteSpace(response.Issue) ? TB("The runtime document endpoint failed without details.") : response.Issue);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
this.logger!.LogWarning(e, "Failed to process the Rust runtime document endpoint response.");
|
||||
return new OpenDocumentResponse(false, false, TB("The runtime document endpoint failed without details."));
|
||||
}
|
||||
finally
|
||||
{
|
||||
result.Dispose();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -278,7 +278,7 @@ public sealed partial class RustService
|
||||
{
|
||||
if (segment.TokenCount is { } tokenCount)
|
||||
{
|
||||
yield return new(segment.Content, tokenCount);
|
||||
yield return new(segment.Content, tokenCount, segment.PageNumber);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -291,7 +291,7 @@ public sealed partial class RustService
|
||||
var countedSegment = await this.GetTokenCount(embeddingProvider, segment.Content, token);
|
||||
if (countedSegment is { Success: true } counted)
|
||||
{
|
||||
yield return new(segment.Content, counted.TokenCount);
|
||||
yield return new(segment.Content, counted.TokenCount, segment.PageNumber);
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -303,7 +303,7 @@ public sealed partial class RustService
|
||||
}
|
||||
}
|
||||
|
||||
private async IAsyncEnumerable<(string Content, int? TokenCount)> StreamArbitraryFileDataCore(
|
||||
private async IAsyncEnumerable<(string Content, int? TokenCount, int? PageNumber)> StreamArbitraryFileDataCore(
|
||||
string path,
|
||||
bool extractImages,
|
||||
bool includeTokenCount,
|
||||
@@ -420,12 +420,13 @@ public sealed partial class RustService
|
||||
}
|
||||
|
||||
//
|
||||
// The count comes from the processed event, not from the event which was just read:
|
||||
// a reader may hold content back across several events, and the count of the content
|
||||
// it releases is the count of that content, not of the event that released it.
|
||||
// The count and the page come from the processed event, not from the event which
|
||||
// was just read: a reader may hold content back across several events, and the
|
||||
// count and page of the content it releases describe that content, not the event
|
||||
// that released it.
|
||||
//
|
||||
if (!string.IsNullOrWhiteSpace(processedEvent.Content))
|
||||
yield return (processedEvent.Content, processedEvent.TokenCount);
|
||||
yield return (processedEvent.Content, processedEvent.TokenCount, processedEvent.PageNumber);
|
||||
}
|
||||
}
|
||||
finally
|
||||
@@ -434,7 +435,7 @@ public sealed partial class RustService
|
||||
}
|
||||
|
||||
if (finalContentChunk is { } pendingContent && !string.IsNullOrWhiteSpace(pendingContent.Content))
|
||||
yield return (pendingContent.Content, pendingContent.TokenCount);
|
||||
yield return (pendingContent.Content, pendingContent.TokenCount, pendingContent.PageNumber);
|
||||
|
||||
if (promptInjectionRedactedCount is 0)
|
||||
yield break;
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// Where a source points in the file system, and where inside the document it was found.
|
||||
/// </summary>
|
||||
/// <param name="Path">The document in the file system, spelled the way this system spells a path.</param>
|
||||
/// <param name="PageNumber">The page the passage stands on, counted from one, or null when no page is known.</param>
|
||||
public readonly record struct SourceDocumentLocation(string Path, int? PageNumber);
|
||||
@@ -1,3 +1,4 @@
|
||||
using System.Globalization;
|
||||
using System.Text;
|
||||
using System.Text.RegularExpressions;
|
||||
|
||||
@@ -80,72 +81,81 @@ public static partial class SourceExtensions
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts a list of sources to a markdown-formatted string.
|
||||
/// Sorts a list of sources into the groups it is shown in, and numbers them.
|
||||
/// </summary>
|
||||
/// <param name="sources">The list of sources to convert.</param>
|
||||
/// <returns>A markdown-formatted string representing the sources.</returns>
|
||||
public static string ToMarkdown(this IList<Source> sources)
|
||||
/// <remarks>
|
||||
/// The order of the groups and the running number are what a reader follows, and they have to
|
||||
/// be the same wherever the list appears: in the chat, in an exported document, and in the
|
||||
/// clipboard. This is why both the chat and the Markdown below ask here instead of sorting the
|
||||
/// list themselves.
|
||||
/// </remarks>
|
||||
/// <param name="sources">The list of sources to sort.</param>
|
||||
/// <returns>The groups which have sources, in the order they are shown; empty when there are none.</returns>
|
||||
public static IReadOnlyList<SourceGroup> GroupSources(this IList<Source> sources)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
var ragSources = new List<ISource>();
|
||||
var toolSources = new List<ISource>();
|
||||
var sourceNum = 0;
|
||||
var addedLLMHeaders = false;
|
||||
var llmSources = new List<Source>();
|
||||
var toolSources = new List<Source>();
|
||||
var ragSources = new List<Source>();
|
||||
foreach (var source in sources)
|
||||
{
|
||||
switch (source.Origin)
|
||||
{
|
||||
case SourceOrigin.RAG:
|
||||
ragSources.Add(source);
|
||||
break;
|
||||
|
||||
case SourceOrigin.LLM:
|
||||
if (!addedLLMHeaders)
|
||||
{
|
||||
sb.Append("## ");
|
||||
sb.AppendLine(TB("Sources provided by the AI"));
|
||||
addedLLMHeaders = true;
|
||||
}
|
||||
|
||||
sb.Append($"- [{++sourceNum}] ");
|
||||
AppendMarkdownLink(sb, source.Title, source.URL);
|
||||
sb.AppendLine();
|
||||
llmSources.Add(source);
|
||||
break;
|
||||
|
||||
case SourceOrigin.TOOL:
|
||||
toolSources.Add(source);
|
||||
break;
|
||||
|
||||
case SourceOrigin.RAG:
|
||||
ragSources.Add(source);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if(toolSources.Count > 0)
|
||||
var groups = new List<SourceGroup>(3);
|
||||
var sourceNum = 0;
|
||||
AddGroup(groups, TB("Sources provided by the AI"), llmSources, ref sourceNum);
|
||||
AddGroup(groups, TB("Sources used by tools"), toolSources, ref sourceNum);
|
||||
AddGroup(groups, TB("Sources provided by the data providers"), ragSources, ref sourceNum);
|
||||
return groups;
|
||||
}
|
||||
|
||||
private static void AddGroup(ICollection<SourceGroup> groups, string heading, IReadOnlyList<Source> sources, ref int sourceNum)
|
||||
{
|
||||
if (sources.Count == 0)
|
||||
return;
|
||||
|
||||
var numberedSources = new List<NumberedSource>(sources.Count);
|
||||
foreach (var source in sources)
|
||||
numberedSources.Add(new(++sourceNum, source));
|
||||
|
||||
groups.Add(new(heading, numberedSources));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts a list of sources to a markdown-formatted string.
|
||||
/// </summary>
|
||||
/// <param name="sources">The list of sources to convert.</param>
|
||||
/// <param name="keepPageAnchors">Whether a link into a local file may name its page; see the method below.</param>
|
||||
/// <returns>A markdown-formatted string representing the sources.</returns>
|
||||
public static string ToMarkdown(this IList<Source> sources, bool keepPageAnchors = true)
|
||||
{
|
||||
var sb = new StringBuilder();
|
||||
foreach (var group in sources.GroupSources())
|
||||
{
|
||||
if(sb.Length > 0)
|
||||
if (sb.Length > 0)
|
||||
sb.AppendLine();
|
||||
|
||||
sb.Append("## ");
|
||||
sb.AppendLine(TB("Sources used by tools"));
|
||||
sb.AppendLine(group.Heading);
|
||||
|
||||
foreach (var source in toolSources)
|
||||
foreach (var numberedSource in group.Sources)
|
||||
{
|
||||
sb.Append($"- [{++sourceNum}] ");
|
||||
AppendMarkdownLink(sb, source.Title, source.URL);
|
||||
sb.AppendLine();
|
||||
}
|
||||
}
|
||||
|
||||
if(ragSources.Count > 0)
|
||||
{
|
||||
if(sb.Length > 0)
|
||||
sb.AppendLine();
|
||||
|
||||
sb.Append("## ");
|
||||
sb.AppendLine(TB("Sources provided by the data providers"));
|
||||
|
||||
foreach (var source in ragSources)
|
||||
{
|
||||
sb.Append($"- [{++sourceNum}] ");
|
||||
AppendMarkdownLink(sb, source.Title, source.URL);
|
||||
var url = keepPageAnchors ? numberedSource.Source.URL : WithoutPageAnchor(numberedSource.Source.URL);
|
||||
sb.Append($"- [{numberedSource.Number}] ");
|
||||
AppendMarkdownLink(sb, numberedSource.Source.Title, url);
|
||||
sb.AppendLine();
|
||||
}
|
||||
}
|
||||
@@ -153,6 +163,29 @@ public static partial class SourceExtensions
|
||||
return sb.ToString();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Takes the page off a link into a local file, for a reader which cannot follow it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Everything a local link carries in its fragment is dropped, not only a page: a chunk is no
|
||||
/// use to any reader either, and what breaks such a link is the fragment itself rather than what
|
||||
/// stands in it. A web address keeps its fragment untouched, because there the fragment is part
|
||||
/// of the address and naming a section of a page is exactly what it is for.
|
||||
/// </remarks>
|
||||
/// <param name="url">The link of the source.</param>
|
||||
/// <returns>The link without its fragment, or the link itself when it carries none.</returns>
|
||||
private static string WithoutPageAnchor(string url)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(url))
|
||||
return url;
|
||||
|
||||
var cleanedUrl = url.Trim().Replace("\r", string.Empty).Replace("\n", string.Empty);
|
||||
if (!Uri.TryCreate(cleanedUrl, UriKind.Absolute, out var absoluteUri) || !absoluteUri.IsFile || absoluteUri.Fragment.Length == 0)
|
||||
return url;
|
||||
|
||||
return absoluteUri.GetComponents(UriComponents.AbsoluteUri & ~UriComponents.Fragment, UriFormat.UriEscaped);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Converts a list of sources to a markdown-formatted string, headed by a title of its own.
|
||||
/// </summary>
|
||||
@@ -163,16 +196,66 @@ public static partial class SourceExtensions
|
||||
/// for this and the chat does not.
|
||||
/// </remarks>
|
||||
/// <param name="sources">The list of sources to convert.</param>
|
||||
/// <param name="keepPageAnchors">Whether a link into a local file may name its page.</param>
|
||||
/// <returns>A markdown-formatted string representing the sources, or an empty string when there are none.</returns>
|
||||
public static string ToExportMarkdown(this IList<Source> sources)
|
||||
public static string ToExportMarkdown(this IList<Source> sources, bool keepPageAnchors = true)
|
||||
{
|
||||
var sourcesMarkdown = sources.ToMarkdown();
|
||||
var sourcesMarkdown = sources.ToMarkdown(keepPageAnchors);
|
||||
if (string.IsNullOrWhiteSpace(sourcesMarkdown))
|
||||
return string.Empty;
|
||||
|
||||
return $"# {TB("Sources")}{Environment.NewLine}{Environment.NewLine}{sourcesMarkdown}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Reads which document a source names, and which page of it.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// Only a source which names a file has such a location; a web source is opened by the browser
|
||||
/// and never asks. The page rides in the fragment of the link as `page=N`, which is what the PDF
|
||||
/// open parameters call for. A chat written before v26.9.1 carries `chunk=N` instead, which names
|
||||
/// nothing a program could be sent to: such a source keeps its document and loses only the page.
|
||||
/// </remarks>
|
||||
/// <param name="source">The source to read.</param>
|
||||
/// <param name="location">The document and its page, or the default when the source names no file.</param>
|
||||
/// <returns>Whether the source names a file.</returns>
|
||||
public static bool TryGetDocumentLocation(this ISource source, out SourceDocumentLocation location)
|
||||
{
|
||||
location = default;
|
||||
if (string.IsNullOrWhiteSpace(source.URL))
|
||||
return false;
|
||||
|
||||
var cleanedUrl = source.URL.Trim().Replace("\r", string.Empty).Replace("\n", string.Empty);
|
||||
if (!Uri.TryCreate(cleanedUrl, UriKind.Absolute, out var absoluteUri) || !absoluteUri.IsFile)
|
||||
return false;
|
||||
|
||||
//
|
||||
// The link was made from a path of this system, so reading it back gives that path again --
|
||||
// percent-encoded spaces and umlauts included, and with the separators this system uses.
|
||||
//
|
||||
var path = absoluteUri.LocalPath;
|
||||
if (string.IsNullOrWhiteSpace(path))
|
||||
return false;
|
||||
|
||||
location = new(path, ReadPageFromFragment(absoluteUri.Fragment));
|
||||
return true;
|
||||
}
|
||||
|
||||
private static int? ReadPageFromFragment(string fragment)
|
||||
{
|
||||
const string PAGE_PARAMETER = "page=";
|
||||
foreach (var parameter in fragment.TrimStart('#').Split('&', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
|
||||
{
|
||||
if (!parameter.StartsWith(PAGE_PARAMETER, StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
if (int.TryParse(parameter.AsSpan(PAGE_PARAMETER.Length), NumberStyles.None, CultureInfo.InvariantCulture, out var pageNumber) && pageNumber > 0)
|
||||
return pageNumber;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Merges a list of added sources into an existing list of sources, avoiding duplicates based on normalized URLs.
|
||||
/// </summary>
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
namespace AIStudio.Tools;
|
||||
|
||||
/// <summary>
|
||||
/// One group of a source list: a heading and the sources below it.
|
||||
/// </summary>
|
||||
/// <param name="Heading">The heading above the group.</param>
|
||||
/// <param name="Sources">The sources of the group, in the order they are shown.</param>
|
||||
public readonly record struct SourceGroup(string Heading, IReadOnlyList<NumberedSource> Sources);
|
||||
Reference in new issue
Block a user