Let the sources of your own documents open at the page they were found on (#974)
Build and Release / Determine run mode (push) Waiting to run
Build and Release / Read metadata (push) Blocked by required conditions
Build and Release / Sync Flatpak repo (push) Blocked by required conditions
Build and Release / Collect Flatpak artifacts (push) Blocked by required conditions
Build and Release / Verify (push) Waiting to run
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-apple-darwin, osx-arm64, macos-latest, aarch64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-pc-windows-msvc.exe, win-arm64, windows-latest, aarch64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-aarch64-unknown-linux-gnu, linux-arm64, ubuntu-22.04-arm, aarch64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-apple-darwin, osx-x64, macos-latest, x86_64-apple-darwin, dmg,app,updater, dmg) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-pc-windows-msvc.exe, win-x64, windows-latest, x86_64-pc-windows-msvc, nsis,updater, nsis) (push) Blocked by required conditions
Build and Release / Build app (${{ matrix.dotnet_runtime }}) (-x86_64-unknown-linux-gnu, linux-x64, ubuntu-22.04, x86_64-unknown-linux-gnu, appimage,updater, appimage) (push) Blocked by required conditions
Build and Release / Prepare & create release (push) Blocked by required conditions
Build and Release / Publish release (push) Blocked by required conditions

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2026-09-15 19:11:39 +02:00
1 parent 6ce7d856a3
commit 1ebe8eb2a4
37 files changed
+1831 -120

No files matched your search

@@ -1,17 +1,21 @@
namespace AIStudio.Tools;
/// <summary>
/// Content which a reader held back, together with the token count of exactly that content.
/// Content which a reader held back, together with the token count and the page of exactly that
/// content.
/// </summary>
/// <remarks>
/// Readers which assemble a page or a slide from several stream events cannot pass their content
/// on right away. Its token count has to travel with it: the count describes the content, not the
/// event which happened to arrive at the moment the content was released. Keeping the two together
/// is what stops a page from being sized by the text of the page after it.
/// is what stops a page from being sized by the text of the page after it. The page number travels
/// for the very same reason, and because a number the runtime already stated must not be derived
/// from the text again further down the line.
/// </remarks>
/// <param name="Content">The assembled content.</param>
/// <param name="TokenCount">The number of tokens of that content, or null when it is unknown.</param>
public readonly record struct ContentStreamPendingContent(string Content, int? TokenCount)
/// <param name="PageNumber">The page that content came from, or null when it has none.</param>
public readonly record struct ContentStreamPendingContent(string Content, int? TokenCount, int? PageNumber = null)
{
/// <summary>
/// Adds up two token counts, where an unknown count makes the sum unknown as well.
@@ -12,7 +12,8 @@ namespace AIStudio.Tools;
/// <param name="Error">The reported failure, or null when the event was processed successfully.</param>
/// <param name="PromptInjection">What the runtime filtered out of the content, or null when it filtered nothing.</param>
/// <param name="TokenCount">The number of tokens of the content, or null when it is unknown.</param>
public readonly record struct ContentStreamProcessedEvent(string? Content, ContentStreamErrorDetails? Error, ContentStreamPromptInjectionDetails? PromptInjection = null, int? TokenCount = null)
/// <param name="PageNumber">The page the content came from, or null when it has none.</param>
public readonly record struct ContentStreamProcessedEvent(string? Content, ContentStreamErrorDetails? Error, ContentStreamPromptInjectionDetails? PromptInjection = null, int? TokenCount = null, int? PageNumber = null)
{
/// <summary>
/// An event which neither produced content nor reported a failure.
@@ -20,16 +21,18 @@ public readonly record struct ContentStreamProcessedEvent(string? Content, Conte
public static readonly ContentStreamProcessedEvent NOTHING = new(null, null);
/// <summary>
/// An event which produced content, with the token count of that very content.
/// An event which produced content, with the token count and the page of that very content.
/// </summary>
/// <remarks>
/// The count travels with the content because a reader may hold content back across several
/// events: pairing it with the count of the event which released it would size it by the
/// wrong text.
/// wrong text. The page travels along for the same reason, and so that whoever indexes the
/// content is told where it came from instead of having to read it back out of the text.
/// </remarks>
/// <param name="content">The content to append.</param>
/// <param name="tokenCount">The number of tokens of that content, or null when it is unknown.</param>
public static ContentStreamProcessedEvent FromContent(string? content, int? tokenCount = null) => new(content, null, TokenCount: tokenCount);
/// <param name="pageNumber">The page that content came from, or null when it has none.</param>
public static ContentStreamProcessedEvent FromContent(string? content, int? tokenCount = null, int? pageNumber = null) => new(content, null, TokenCount: tokenCount, PageNumber: pageNumber);
public static ContentStreamProcessedEvent FromError(ContentStreamErrorDetails? error) => new(null, error);
@@ -19,13 +19,19 @@ public static class ContentStreamSseHandler
case ContentStreamTextMetadata:
return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount);
//
// The heading tells the AI which page it is reading. The number is handed on
// separately as well, because whoever indexes this content needs it as a
// number: reading it back out of the heading would mean guessing at something
// the runtime already stated.
//
case ContentStreamPdfMetadata pdfMetadata:
var pageNumber = pdfMetadata.Pdf?.PageNumber ?? 0;
return ContentStreamProcessedEvent.FromContent($"""
# Page {pageNumber}
{sseEvent.Content}
""", sseEvent.TokenCount);
""", sseEvent.TokenCount, pageNumber > 0 ? pageNumber : null);
case ContentStreamSpreadsheetMetadata spreadsheetMetadata:
var sheetName = spreadsheetMetadata.Spreadsheet?.SheetName;
@@ -45,9 +51,10 @@ public static class ContentStreamSseHandler
// a page can follow its Markdown. Documents converted as a whole, e.g. by Pandoc,
// carry no page number and are passed on unchanged.
//
// The buffering is why the count comes back from the reader rather than from
// this event: the page which is released here arrived one event ago, and this
// event's count belongs to the page which is now being buffered.
// The buffering is why the count and the page come back from the reader rather
// than from this event: the page which is released here arrived one event ago,
// and this event's count and number belong to the page which is now being
// buffered.
//
case ContentStreamDocumentMetadata documentMetadata:
if (documentMetadata.Document?.PageNumber is not > 0)
@@ -55,7 +62,7 @@ public static class ContentStreamSseHandler
var documentManager = DOCUMENT_MANAGERS.GetOrAdd(sseEvent.StreamId!, _ => new());
var documentContent = documentManager.AddPage(documentMetadata, sseEvent.Content, sseEvent.TokenCount, extractImages);
return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent.Value.Content, documentContent.Value.TokenCount);
return documentContent is null ? ContentStreamProcessedEvent.NOTHING : ContentStreamProcessedEvent.FromContent(documentContent.Value.Content, documentContent.Value.TokenCount, documentContent.Value.PageNumber);
case ContentStreamImageMetadata:
return ContentStreamProcessedEvent.FromContent(sseEvent.Content, sseEvent.TokenCount);
@@ -184,7 +191,9 @@ public static class ContentStreamSseHandler
/// <remarks>
/// The readers which assemble pages or slides always keep the last one of them: nothing tells
/// them that no further image is coming. It is released here, and it carries its own token
/// count, because a chunk without one cannot be sized by the caller.
/// count, because a chunk without one cannot be sized by the caller. Only the page reader
/// states a page; a stream is read by one of them, so there is no second number to weigh
/// against.
/// </remarks>
/// <param name="streamId">The stream to release and forget.</param>
/// <returns>The content which was held back, or null when there was none.</returns>
@@ -195,6 +204,7 @@ public static class ContentStreamSseHandler
var finalContentChunk = new StringBuilder();
int? tokenCount = 0;
int? pageNumber = null;
if(SLIDE_MANAGERS.TryGetValue(streamId, out var slideManager)
&& slideManager.GetAllSlidesInOrder() is { } slides
&& !string.IsNullOrWhiteSpace(slides.Content))
@@ -209,6 +219,7 @@ public static class ContentStreamSseHandler
{
finalContentChunk.Append(page.Content);
tokenCount = ContentStreamPendingContent.AddTokenCounts(tokenCount, page.TokenCount);
pageNumber = page.PageNumber;
}
SLIDE_MANAGERS.TryRemove(streamId, out _);
@@ -217,6 +228,6 @@ public static class ContentStreamSseHandler
foreach (var key in CHUNKED_IMAGES.Keys.Where(k => k.StartsWith(imageIdPrefix, StringComparison.InvariantCultureIgnoreCase)))
CHUNKED_IMAGES.TryRemove(key, out _);
return finalContentChunk.Length > 0 ? new ContentStreamPendingContent(finalContentChunk.ToString(), tokenCount) : null;
return finalContentChunk.Length > 0 ? new ContentStreamPendingContent(finalContentChunk.ToString(), tokenCount, pageNumber) : null;
}
}
@@ -10,6 +10,7 @@ public sealed class DocumentManager
{
private StringBuilder? currentPageContent;
private int? currentPageTokenCount;
private int? currentPageNumber;
public ContentStreamPendingContent? AddPage(ContentStreamDocumentMetadata metadata, string? content, int? tokenCount, bool extractImages)
{
@@ -36,9 +37,12 @@ public sealed class DocumentManager
//
// The count waits here together with the page it belongs to. Handing it out along with
// the page we just completed would size that page by the text of this one.
// the page we just completed would size that page by the text of this one. The page
// number waits for the same reason: it belongs to the page being buffered, not to the
// one leaving here.
//
this.currentPageTokenCount = tokenCount;
this.currentPageNumber = pageNumber;
return completedPage;
}
@@ -72,8 +76,10 @@ public sealed class DocumentManager
var result = this.currentPageContent.ToString();
var tokenCount = this.currentPageTokenCount;
var pageNumber = this.currentPageNumber;
this.currentPageContent = null;
this.currentPageTokenCount = null;
return string.IsNullOrWhiteSpace(result) ? null : new ContentStreamPendingContent(result, tokenCount);
this.currentPageNumber = null;
return string.IsNullOrWhiteSpace(result) ? null : new ContentStreamPendingContent(result, tokenCount, pageNumber);
}
}
@@ -204,6 +204,26 @@ public static class FileExportFormatExtensions
_ => WITHOUT_BYTE_ORDER_MARK,
};
/// <summary>
/// Determines whether a link into a local file may name the page it points at.
/// </summary>
/// <remarks>
/// A page is named by the fragment of the link, the way the PDF open parameters call for. A
/// browser and a PDF reader follow that and open the document on the page; Word and LibreOffice
/// take the fragment for part of the file name, look for a file which does not exist, and refuse
/// the link altogether. There the page is dropped, so the link at least opens the document --
/// which page it was stays in the title of the source. Verified on 2026-09-15 with LibreOffice
/// on an exported .odt. A format added later keeps the page unless it is known to stumble too.
/// </remarks>
/// <param name="format">The format.</param>
/// <returns>True, when a reader of this format follows such a link.</returns>
public static bool FollowsPageAnchors(this FileExportFormat format) => format switch
{
FileExportFormat.MICROSOFT_WORD or FileExportFormat.OPEN_DOCUMENT_TEXT => false,
_ => true,
};
/// <summary>
/// Returns the name Pandoc knows the format by.
/// </summary>
@@ -0,0 +1,13 @@
namespace AIStudio.Tools;
/// <summary>
/// A source together with the number it is listed under.
/// </summary>
/// <remarks>
/// The number runs through the whole list rather than starting over per group, because that is how
/// an answer refers to a source. It is assigned once, where the groups are formed, so the chat and
/// an exported document cannot end up numbering the same list differently.
/// </remarks>
/// <param name="Number">The number this source is listed under, counted from one.</param>
/// <param name="Source">The source itself.</param>
public readonly record struct NumberedSource(int Number, Source Source);
+1 -1
View File
@@ -118,7 +118,7 @@ public static class PandocExport
// We read the text before we ask for a path: when there is nothing to convert, the user
// should learn that right away instead of picking a file first and getting an error afterwards.
//
if (!markdownContent.TryGetExportMarkdown(out var markdownText))
if (!markdownContent.TryGetExportMarkdown(out var markdownText, format.FollowsPageAnchors()))
{
LOGGER.LogWarning("Cannot export the content as {ExportFormat}, because it carries no text.", format);
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Cancel, TB("Only text messages can be exported.")));
@@ -8,7 +8,36 @@ namespace AIStudio.Tools.RAG;
public static class IRetrievalContextExtensions
{
private static readonly ILogger<IRetrievalContext> LOGGER = Program.LOGGER_FACTORY.CreateLogger<IRetrievalContext>();
/// <summary>
/// Writes what the AI is told about a retrieval context, before its content follows.
/// </summary>
/// <remarks>
/// The location is what lets the AI say where an answer comes from. Naming only the file is
/// not enough in a document of two hundred pages, and we know the page: it travels from the
/// runtime through the index into the context. A slide or a sheet has no page, and then
/// nothing is claimed rather than something made up.
/// </remarks>
/// <param name="contextBuilder">The builder to write into.</param>
/// <param name="retrievalContext">The context to describe.</param>
internal static void AppendContextDescription(StringBuilder contextBuilder, IRetrievalContext retrievalContext)
{
contextBuilder.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
contextBuilder.AppendLine($"Content category: {retrievalContext.Category}");
contextBuilder.AppendLine($"Content type: {retrievalContext.Type}");
contextBuilder.AppendLine($"Content path: {retrievalContext.Path}");
if(retrievalContext is RetrievalTextContext { PageNumber: > 0 } locatedContext)
contextBuilder.AppendLine($"Content location: page {locatedContext.PageNumber}");
if(retrievalContext.Links.Count is 0)
return;
contextBuilder.AppendLine("Additional links:");
foreach(var link in retrievalContext.Links)
contextBuilder.AppendLine($"- {link}");
}
public static async Task<string> AsMarkdown(this IReadOnlyList<IRetrievalContext> retrievalContexts, StringBuilder? sb = null, CancellationToken token = default)
{
sb ??= new StringBuilder();
@@ -49,17 +78,7 @@ public static class IRetrievalContextExtensions
break;
}
contextBuilder.AppendLine($"Data source name: {retrievalContext.DataSourceName}");
contextBuilder.AppendLine($"Content category: {retrievalContext.Category}");
contextBuilder.AppendLine($"Content type: {retrievalContext.Type}");
contextBuilder.AppendLine($"Content path: {retrievalContext.Path}");
if(retrievalContext.Links.Count > 0)
{
contextBuilder.AppendLine("Additional links:");
foreach(var link in retrievalContext.Links)
contextBuilder.AppendLine($"- {link}");
}
AppendContextDescription(contextBuilder, retrievalContext);
var guardService = Program.SERVICE_PROVIDER.GetRequiredService<PromptInjectionGuardService>();
var source = PromptInjectionSource.RetrievalContext(retrievalContext.DataSourceName, retrievalContext.Path);
@@ -50,4 +50,14 @@ public sealed class RetrievalTextContext : IRetrievalContext
/// Optional link used when this context is displayed as a source reference.
/// </summary>
public string ReferenceLink { get; init; } = string.Empty;
/// <summary>
/// The page this passage was found on, or null when it has none.
/// </summary>
/// <remarks>
/// Kept as a number rather than only inside the reference title: the AI is told the page so it
/// can say where an answer comes from, and a source has to name a page a program can be sent
/// to. A slide or a sheet has no page and leaves this empty.
/// </remarks>
public int? PageNumber { get; init; }
}
@@ -0,0 +1,8 @@
namespace AIStudio.Tools.Rust;
/// <summary>
/// Asks the runtime to open a document in the program the system uses for it.
/// </summary>
/// <param name="Path">The document to open.</param>
/// <param name="Page">The page to show, counted from one, or null when the document has none.</param>
public readonly record struct OpenDocumentRequest(string Path, int? Page);
@@ -0,0 +1,14 @@
namespace AIStudio.Tools.Rust;
/// <summary>
/// Says how opening a document went.
/// </summary>
/// <param name="Success">Whether the document was opened at all.</param>
/// <param name="PageApplied">
/// Whether the document was handed to its program together with the page. False means it opens on
/// its first page: no page was asked for, the system uses a program which cannot be told one, or
/// starting that program failed. None of these is an error, so this belongs in the log rather than
/// in front of the user, who is told the page by the source itself.
/// </param>
/// <param name="Issue">Why the document could not be opened, or an empty text when it was.</param>
public readonly record struct OpenDocumentResponse(bool Success, bool PageApplied, string Issue);
@@ -1,3 +1,9 @@
namespace AIStudio.Tools.Services;
public sealed record ArbitraryFileDataSegment(string Content, int TokenCount);
/// <summary>
/// One piece of an extracted file, as the runtime delivered it.
/// </summary>
/// <param name="Content">The extracted text.</param>
/// <param name="TokenCount">The number of tokens of that text.</param>
/// <param name="PageNumber">The page that text came from, or null when it has none. Presentations and spreadsheets have none.</param>
public sealed record ArbitraryFileDataSegment(string Content, int TokenCount, int? PageNumber);
@@ -16,6 +16,22 @@ public sealed partial class DataSourceEmbeddingService
internal const int DEFAULT_CHUNK_OVERLAP_TOKEN_LENGTH = 300;
private const bool IMAGE_EMBEDDING_ENABLED = false;
/// <summary>
/// What this build writes next to a chunk besides its text. Raise it whenever that changes.
/// </summary>
/// <remarks>
/// A stored chunk keeps the metadata of the run which wrote it, and nothing recomputes it: the
/// fingerprint of a file says whether the file changed, not whether we got better at reading
/// it. Raising this number makes the embedding signature differ, which drops the index and
/// builds it again — the only way corrected page numbers reach a data source somebody indexed
/// earlier.
///
/// Version 2: the page of a chunk is taken from the runtime metadata instead of being read back
/// out of the chunk text, which is what left Word and OpenDocument files, and passages
/// continuing across a page break, without a page.
/// </remarks>
private const string CHUNK_METADATA_VERSION = "2";
private enum RagFileIndexingDecision
{
INDEXABLE,
@@ -23,10 +39,23 @@ public sealed partial class DataSourceEmbeddingService
UNSUPPORTED,
}
private sealed record ExtractedFileSegment(string Text, int? TokenCount);
private sealed record ExtractedFileSegment(string Text, int? TokenCount, int? PageNumber);
private sealed record ExtractedFileContent(string Text, IReadOnlyList<ExtractedFileSegment> SourceSegments);
/// <summary>
/// One chunk as the chunking produced it, together with the page it starts on.
/// </summary>
/// <remarks>
/// The page is carried rather than read back out of the chunk text. The runtime states it, and
/// the chunking knows which source segment a chunk begins in, so nothing has to be derived from
/// a marker in the text — which is what used to leave Word files and continued passages without
/// a page.
/// </remarks>
/// <param name="Text">The chunk itself, overlap prefix included.</param>
/// <param name="PageNumber">The page the chunk's own content starts on, or null when it has none.</param>
private sealed record EmbeddingChunk(string Text, int? PageNumber);
private sealed record EmbeddingChunkDraft(string ChunkId, string Text, int ChunkIndex, int? PageNumber);
private sealed record ChunkingOptions(int MaxChunkTokenLength, int OverlapTokenLength);
@@ -37,7 +66,7 @@ public sealed partial class DataSourceEmbeddingService
private sealed record DataSourceMetadataSnapshot(string SourceHash, IReadOnlyDictionary<string, string> FileHashes);
private async IAsyncEnumerable<string> StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
private async IAsyncEnumerable<EmbeddingChunk> StreamEmbeddingChunksAsync(string filePath, IDataSource dataSource, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
{
var options = this.GetChunkingOptions(dataSource, embeddingProvider);
var strategy = this.GetChunkingStrategy(filePath);
@@ -55,26 +84,31 @@ public sealed partial class DataSourceEmbeddingService
{
var normalized = NormalizeChunkSegment(segment.Content);
if (!string.IsNullOrWhiteSpace(normalized))
segments.Add(new(normalized, segment.TokenCount));
segments.Add(new(normalized, segment.TokenCount, segment.PageNumber));
}
return new(string.Join("\n", segments.Select(segment => segment.Text)).Trim(), segments);
}
private async IAsyncEnumerable<string> SplitByChunkingStrategyAsync(ExtractedFileContent content, ChunkingStrategy strategy, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
private async IAsyncEnumerable<EmbeddingChunk> SplitByChunkingStrategyAsync(ExtractedFileContent content, ChunkingStrategy strategy, ChunkingOptions options, EmbeddingProvider embeddingProvider, [System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token)
{
var estimatedTokenCount = SumTokenCounts(content.SourceSegments);
await foreach (var chunk in this.SplitTextByRulesAsync(content.Text, content.SourceSegments, strategy, 0, options, embeddingProvider, token, estimatedTokenCount: estimatedTokenCount))
// The whole text starts where the first segment starts, so that is the page it is on until
// the splitting reaches a segment boundary:
var firstPageNumber = content.SourceSegments.Count > 0 ? content.SourceSegments[0].PageNumber : null;
await foreach (var chunk in this.SplitTextByRulesAsync(content.Text, content.SourceSegments, strategy, 0, options, embeddingProvider, firstPageNumber, token, estimatedTokenCount: estimatedTokenCount))
yield return chunk;
}
private async IAsyncEnumerable<string> SplitTextByRulesAsync(
private async IAsyncEnumerable<EmbeddingChunk> SplitTextByRulesAsync(
string text,
IReadOnlyList<ExtractedFileSegment> sourceSegments,
ChunkingStrategy strategy,
int ruleIndex,
ChunkingOptions options,
EmbeddingProvider embeddingProvider,
int? currentPageNumber,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token,
string requiredOverlapPrefix = "",
int? estimatedTokenCount = null)
@@ -91,14 +125,14 @@ public sealed partial class DataSourceEmbeddingService
tokenCount = await this.GetEmbeddingTokenCountAsync(embeddingProvider, textWithOverlap, token);
if (tokenCount <= options.MaxChunkTokenLength)
{
yield return textWithOverlap;
yield return new(textWithOverlap, currentPageNumber);
yield break;
}
}
if (ruleIndex >= strategy.Rules.Count)
{
await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token, requiredOverlapPrefix, estimatedTokenCount))
await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, currentPageNumber, token, requiredOverlapPrefix, estimatedTokenCount))
yield return hardChunk;
yield break;
@@ -107,7 +141,7 @@ public sealed partial class DataSourceEmbeddingService
var rule = strategy.Rules[ruleIndex];
if (rule.Split is null)
{
await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, token, requiredOverlapPrefix, estimatedTokenCount))
await foreach (var hardChunk in this.SplitTextByHardCutAsync(text, options, embeddingProvider, currentPageNumber, token, requiredOverlapPrefix, estimatedTokenCount))
yield return hardChunk;
yield break;
@@ -116,7 +150,7 @@ public sealed partial class DataSourceEmbeddingService
var units = NormalizeSplitUnits(rule.Split(text, sourceSegments.Select(segment => segment.Text).ToList()), text);
if (units.Count <= 1)
{
await foreach (var chunk in this.SplitTextByRulesAsync(text, sourceSegments, strategy, ruleIndex + 1, options, embeddingProvider, token, requiredOverlapPrefix, estimatedTokenCount))
await foreach (var chunk in this.SplitTextByRulesAsync(text, sourceSegments, strategy, ruleIndex + 1, options, embeddingProvider, currentPageNumber, token, requiredOverlapPrefix, estimatedTokenCount))
yield return chunk;
yield break;
@@ -135,6 +169,15 @@ public sealed partial class DataSourceEmbeddingService
var overlapPrefix = requiredOverlapPrefix;
var unitTokenCounts = EstimateSplitUnitTokenCounts(units, sourceSegments, rule.UsesSourceSegmentCounts, estimatedTokenCount);
//
// The first rule of every strategy cuts along the segments the runtime delivered, so there
// a unit is a segment and carries that segment's page. Every later rule cuts inside a
// single segment, where all units share the page they were handed. This is what ties a
// chunk to a page without anybody reading the text.
//
var unitsAreSourceSegments = rule.UsesSourceSegmentCounts && sourceSegments.Count == units.Count;
int? PageOfUnit(int unitIndex) => unitsAreSourceSegments ? sourceSegments[unitIndex].PageNumber ?? currentPageNumber : currentPageNumber;
while (index < units.Count)
{
token.ThrowIfCancellationRequested();
@@ -145,8 +188,14 @@ public sealed partial class DataSourceEmbeddingService
var rawChunk = string.Concat(units.Skip(index).Take(unitCount)).Trim();
var chunk = AddOverlapPrefix(rawChunk, overlapPrefix);
overlapPrefix = string.Empty;
//
// The page of the first unit this chunk covers, not of the overlap prefix in front
// of it: the prefix repeats what the chunk before already said, while the page has
// to name where this chunk's own content begins.
//
if (!string.IsNullOrWhiteSpace(chunk))
yield return chunk;
yield return new(chunk, PageOfUnit(index));
var nextIndex = index + unitCount;
if (nextIndex >= units.Count)
@@ -178,9 +227,10 @@ public sealed partial class DataSourceEmbeddingService
string? lastSplitUnit = null;
var unitTokenCount = unitTokenCounts?[index];
await foreach (var splitUnit in this.SplitTextByRulesAsync(units[index], [new(units[index], unitTokenCount)], strategy, ruleIndex + 1, options, embeddingProvider, token, overlapPrefix, unitTokenCount))
var unitPageNumber = PageOfUnit(index);
await foreach (var splitUnit in this.SplitTextByRulesAsync(units[index], [new(units[index], unitTokenCount, unitPageNumber)], strategy, ruleIndex + 1, options, embeddingProvider, unitPageNumber, token, overlapPrefix, unitTokenCount))
{
lastSplitUnit = splitUnit;
lastSplitUnit = splitUnit.Text;
yield return splitUnit;
}
@@ -372,10 +422,15 @@ public sealed partial class DataSourceEmbeddingService
return bestStartIndex <= chunkStartIndex ? chunkEndIndex : bestStartIndex;
}
private async IAsyncEnumerable<string> SplitTextByHardCutAsync(
/// <remarks>
/// The hard cut is only ever reached inside a single piece of text which no rule could split
/// any further, so every chunk it produces sits on the page that piece was handed.
/// </remarks>
private async IAsyncEnumerable<EmbeddingChunk> SplitTextByHardCutAsync(
string text,
ChunkingOptions options,
EmbeddingProvider embeddingProvider,
int? currentPageNumber,
[System.Runtime.CompilerServices.EnumeratorCancellation] CancellationToken token,
string requiredOverlapPrefix = "",
int? estimatedTokenCount = null)
@@ -455,7 +510,7 @@ public sealed partial class DataSourceEmbeddingService
var chunk = AddOverlapPrefix(text[startIndex..bestEndIndex].Trim(), overlapPrefix);
if (!string.IsNullOrWhiteSpace(chunk))
yield return chunk;
yield return new(chunk, currentPageNumber);
if (bestEndIndex >= text.Length)
yield break;
@@ -934,6 +989,7 @@ public sealed partial class DataSourceEmbeddingService
private string BuildEmbeddingSignature(IDataSource dataSource, EmbeddingProvider embeddingProvider, ChunkingOptions chunkingOptions)
{
return string.Join('|',
CHUNK_METADATA_VERSION,
embeddingProvider.Id,
embeddingProvider.UsedLLMProvider,
embeddingProvider.Model.Id,
@@ -1086,14 +1142,6 @@ public sealed partial class DataSourceEmbeddingService
return string.IsNullOrWhiteSpace(extension) ? "unknown" : extension;
}
private static int? TryExtractPageNumber(string chunk)
{
var match = Regex.Match(chunk, @"^\s*#\s+Page\s+(\d+)\b", RegexOptions.CultureInvariant | RegexOptions.IgnoreCase);
return match.Success && int.TryParse(match.Groups[1].Value, out var pageNumber) && pageNumber > 0
? pageNumber
: null;
}
private string CreatePointId(string dataSourceId, string fingerprint, int chunkIndex) =>
CreateStableGuid($"{dataSourceId}:chunk:{fingerprint}:{chunkIndex}");
@@ -843,7 +843,7 @@ public sealed partial class DataSourceEmbeddingService(SettingsManager settingsM
await foreach (var chunk in this.StreamEmbeddingChunksAsync(file.FullName, dataSource, embeddingProvider, token))
{
batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk, totalChunkCount, TryExtractPageNumber(chunk)));
batch.Add(new(this.CreatePointId(dataSource.Id, fingerprint, totalChunkCount), chunk.Text, totalChunkCount, chunk.PageNumber));
totalChunkCount++;
if (batch.Count >= embeddingBatchSize)
@@ -395,6 +395,7 @@ public sealed class DataSourceLocalRetrievalService(
SurroundingContent = [],
ReferenceTitle = BuildReferenceTitle(hit),
ReferenceLink = referenceLink,
PageNumber = hit.PageNumber is > 0 ? hit.PageNumber : null,
};
}
@@ -413,11 +414,19 @@ public sealed class DataSourceLocalRetrievalService(
return $"{sourceName} ({location})";
}
/// <remarks>
/// A known page is written as the fragment `#page=N`, which is what the PDF open parameters
/// call for: a program which understands them opens the document where the passage is. Without
/// a page there is nothing to send a program to, and the chunk stays in the link so the
/// reference still points at something.
/// </remarks>
private static string BuildReferenceLink(string path, LocalRetrievalHit hit)
{
var link = NormalizeLocalReferencePath(path);
var separator = link.Contains('#', StringComparison.Ordinal) ? "&" : "#";
return $"{link}{separator}chunk={hit.ChunkIndex}";
return hit.PageNumber is > 0
? $"{link}{separator}page={hit.PageNumber}"
: $"{link}{separator}chunk={hit.ChunkIndex}";
}
private static string NormalizeLocalReferencePath(string path)
@@ -168,4 +168,66 @@ public sealed partial class RustService
result.Dispose();
}
}
/// <summary>
/// Opens a document in the program the system uses for it, on the given page where possible.
/// </summary>
/// <remarks>
/// The page is best effort and never decides whether this succeeded. Which programs can be
/// told a page is the runtime's business, and it says afterwards whether it managed to.
/// </remarks>
/// <param name="path">The document to open.</param>
/// <param name="pageNumber">The page to show, counted from one, or null when there is none.</param>
/// <returns>Whether the document was opened, whether the page was applied, and what went wrong.</returns>
public async Task<OpenDocumentResponse> TryOpenDocumentInSystemViewer(string path, int? pageNumber)
{
HttpResponseMessage result;
try
{
result = await this.http.PostAsJsonAsync("/open/document", new OpenDocumentRequest(path, pageNumber), this.jsonRustSerializerOptions);
}
catch (HttpRequestException e)
{
this.logger!.LogWarning(e, "Failed to reach the Rust runtime document endpoint.");
return new OpenDocumentResponse(false, false, TB("The runtime document endpoint is not available."));
}
catch (TaskCanceledException e)
{
this.logger!.LogWarning(e, "Timed out while reaching the Rust runtime document endpoint.");
return new OpenDocumentResponse(false, false, TB("The runtime document endpoint is not available."));
}
try
{
if (!result.IsSuccessStatusCode)
{
this.logger!.LogWarning("Failed to open a document through the Rust runtime: '{StatusCode}'", result.StatusCode);
return new OpenDocumentResponse(false, false, string.Format(TB("The runtime document endpoint returned '{0}'."), result.StatusCode));
}
var response = await result.Content.ReadFromJsonAsync<OpenDocumentResponse>(this.jsonRustSerializerOptions);
if (response.Success)
{
//
// A page which was asked for but not applied is noted here and nowhere else: the
// document is open, and the source the user clicked names the page anyway.
//
if (pageNumber is > 0 && !response.PageApplied)
this.logger!.LogInformation("Opened a document without the requested page {PageNumber}, because the system uses a program which cannot be told one.", pageNumber);
return response;
}
return new OpenDocumentResponse(false, false, string.IsNullOrWhiteSpace(response.Issue) ? TB("The runtime document endpoint failed without details.") : response.Issue);
}
catch (Exception e)
{
this.logger!.LogWarning(e, "Failed to process the Rust runtime document endpoint response.");
return new OpenDocumentResponse(false, false, TB("The runtime document endpoint failed without details."));
}
finally
{
result.Dispose();
}
}
}
@@ -278,7 +278,7 @@ public sealed partial class RustService
{
if (segment.TokenCount is { } tokenCount)
{
yield return new(segment.Content, tokenCount);
yield return new(segment.Content, tokenCount, segment.PageNumber);
continue;
}
@@ -291,7 +291,7 @@ public sealed partial class RustService
var countedSegment = await this.GetTokenCount(embeddingProvider, segment.Content, token);
if (countedSegment is { Success: true } counted)
{
yield return new(segment.Content, counted.TokenCount);
yield return new(segment.Content, counted.TokenCount, segment.PageNumber);
continue;
}
@@ -303,7 +303,7 @@ public sealed partial class RustService
}
}
private async IAsyncEnumerable<(string Content, int? TokenCount)> StreamArbitraryFileDataCore(
private async IAsyncEnumerable<(string Content, int? TokenCount, int? PageNumber)> StreamArbitraryFileDataCore(
string path,
bool extractImages,
bool includeTokenCount,
@@ -420,12 +420,13 @@ public sealed partial class RustService
}
//
// The count comes from the processed event, not from the event which was just read:
// a reader may hold content back across several events, and the count of the content
// it releases is the count of that content, not of the event that released it.
// The count and the page come from the processed event, not from the event which
// was just read: a reader may hold content back across several events, and the
// count and page of the content it releases describe that content, not the event
// that released it.
//
if (!string.IsNullOrWhiteSpace(processedEvent.Content))
yield return (processedEvent.Content, processedEvent.TokenCount);
yield return (processedEvent.Content, processedEvent.TokenCount, processedEvent.PageNumber);
}
}
finally
@@ -434,7 +435,7 @@ public sealed partial class RustService
}
if (finalContentChunk is { } pendingContent && !string.IsNullOrWhiteSpace(pendingContent.Content))
yield return (pendingContent.Content, pendingContent.TokenCount);
yield return (pendingContent.Content, pendingContent.TokenCount, pendingContent.PageNumber);
if (promptInjectionRedactedCount is 0)
yield break;
@@ -0,0 +1,8 @@
namespace AIStudio.Tools;
/// <summary>
/// Where a source points in the file system, and where inside the document it was found.
/// </summary>
/// <param name="Path">The document in the file system, spelled the way this system spells a path.</param>
/// <param name="PageNumber">The page the passage stands on, counted from one, or null when no page is known.</param>
public readonly record struct SourceDocumentLocation(string Path, int? PageNumber);
+130 -47
View File
@@ -1,3 +1,4 @@
using System.Globalization;
using System.Text;
using System.Text.RegularExpressions;
@@ -80,72 +81,81 @@ public static partial class SourceExtensions
}
/// <summary>
/// Converts a list of sources to a markdown-formatted string.
/// Sorts a list of sources into the groups it is shown in, and numbers them.
/// </summary>
/// <param name="sources">The list of sources to convert.</param>
/// <returns>A markdown-formatted string representing the sources.</returns>
public static string ToMarkdown(this IList<Source> sources)
/// <remarks>
/// The order of the groups and the running number are what a reader follows, and they have to
/// be the same wherever the list appears: in the chat, in an exported document, and in the
/// clipboard. This is why both the chat and the Markdown below ask here instead of sorting the
/// list themselves.
/// </remarks>
/// <param name="sources">The list of sources to sort.</param>
/// <returns>The groups which have sources, in the order they are shown; empty when there are none.</returns>
public static IReadOnlyList<SourceGroup> GroupSources(this IList<Source> sources)
{
var sb = new StringBuilder();
var ragSources = new List<ISource>();
var toolSources = new List<ISource>();
var sourceNum = 0;
var addedLLMHeaders = false;
var llmSources = new List<Source>();
var toolSources = new List<Source>();
var ragSources = new List<Source>();
foreach (var source in sources)
{
switch (source.Origin)
{
case SourceOrigin.RAG:
ragSources.Add(source);
break;
case SourceOrigin.LLM:
if (!addedLLMHeaders)
{
sb.Append("## ");
sb.AppendLine(TB("Sources provided by the AI"));
addedLLMHeaders = true;
}
sb.Append($"- [{++sourceNum}] ");
AppendMarkdownLink(sb, source.Title, source.URL);
sb.AppendLine();
llmSources.Add(source);
break;
case SourceOrigin.TOOL:
toolSources.Add(source);
break;
case SourceOrigin.RAG:
ragSources.Add(source);
break;
}
}
if(toolSources.Count > 0)
var groups = new List<SourceGroup>(3);
var sourceNum = 0;
AddGroup(groups, TB("Sources provided by the AI"), llmSources, ref sourceNum);
AddGroup(groups, TB("Sources used by tools"), toolSources, ref sourceNum);
AddGroup(groups, TB("Sources provided by the data providers"), ragSources, ref sourceNum);
return groups;
}
private static void AddGroup(ICollection<SourceGroup> groups, string heading, IReadOnlyList<Source> sources, ref int sourceNum)
{
if (sources.Count == 0)
return;
var numberedSources = new List<NumberedSource>(sources.Count);
foreach (var source in sources)
numberedSources.Add(new(++sourceNum, source));
groups.Add(new(heading, numberedSources));
}
/// <summary>
/// Converts a list of sources to a markdown-formatted string.
/// </summary>
/// <param name="sources">The list of sources to convert.</param>
/// <param name="keepPageAnchors">Whether a link into a local file may name its page; see the method below.</param>
/// <returns>A markdown-formatted string representing the sources.</returns>
public static string ToMarkdown(this IList<Source> sources, bool keepPageAnchors = true)
{
var sb = new StringBuilder();
foreach (var group in sources.GroupSources())
{
if(sb.Length > 0)
if (sb.Length > 0)
sb.AppendLine();
sb.Append("## ");
sb.AppendLine(TB("Sources used by tools"));
sb.AppendLine(group.Heading);
foreach (var source in toolSources)
foreach (var numberedSource in group.Sources)
{
sb.Append($"- [{++sourceNum}] ");
AppendMarkdownLink(sb, source.Title, source.URL);
sb.AppendLine();
}
}
if(ragSources.Count > 0)
{
if(sb.Length > 0)
sb.AppendLine();
sb.Append("## ");
sb.AppendLine(TB("Sources provided by the data providers"));
foreach (var source in ragSources)
{
sb.Append($"- [{++sourceNum}] ");
AppendMarkdownLink(sb, source.Title, source.URL);
var url = keepPageAnchors ? numberedSource.Source.URL : WithoutPageAnchor(numberedSource.Source.URL);
sb.Append($"- [{numberedSource.Number}] ");
AppendMarkdownLink(sb, numberedSource.Source.Title, url);
sb.AppendLine();
}
}
@@ -153,6 +163,29 @@ public static partial class SourceExtensions
return sb.ToString();
}
/// <summary>
/// Takes the page off a link into a local file, for a reader which cannot follow it.
/// </summary>
/// <remarks>
/// Everything a local link carries in its fragment is dropped, not only a page: a chunk is no
/// use to any reader either, and what breaks such a link is the fragment itself rather than what
/// stands in it. A web address keeps its fragment untouched, because there the fragment is part
/// of the address and naming a section of a page is exactly what it is for.
/// </remarks>
/// <param name="url">The link of the source.</param>
/// <returns>The link without its fragment, or the link itself when it carries none.</returns>
private static string WithoutPageAnchor(string url)
{
if (string.IsNullOrWhiteSpace(url))
return url;
var cleanedUrl = url.Trim().Replace("\r", string.Empty).Replace("\n", string.Empty);
if (!Uri.TryCreate(cleanedUrl, UriKind.Absolute, out var absoluteUri) || !absoluteUri.IsFile || absoluteUri.Fragment.Length == 0)
return url;
return absoluteUri.GetComponents(UriComponents.AbsoluteUri & ~UriComponents.Fragment, UriFormat.UriEscaped);
}
/// <summary>
/// Converts a list of sources to a markdown-formatted string, headed by a title of its own.
/// </summary>
@@ -163,16 +196,66 @@ public static partial class SourceExtensions
/// for this and the chat does not.
/// </remarks>
/// <param name="sources">The list of sources to convert.</param>
/// <param name="keepPageAnchors">Whether a link into a local file may name its page.</param>
/// <returns>A markdown-formatted string representing the sources, or an empty string when there are none.</returns>
public static string ToExportMarkdown(this IList<Source> sources)
public static string ToExportMarkdown(this IList<Source> sources, bool keepPageAnchors = true)
{
var sourcesMarkdown = sources.ToMarkdown();
var sourcesMarkdown = sources.ToMarkdown(keepPageAnchors);
if (string.IsNullOrWhiteSpace(sourcesMarkdown))
return string.Empty;
return $"# {TB("Sources")}{Environment.NewLine}{Environment.NewLine}{sourcesMarkdown}";
}
/// <summary>
/// Reads which document a source names, and which page of it.
/// </summary>
/// <remarks>
/// Only a source which names a file has such a location; a web source is opened by the browser
/// and never asks. The page rides in the fragment of the link as `page=N`, which is what the PDF
/// open parameters call for. A chat written before v26.9.1 carries `chunk=N` instead, which names
/// nothing a program could be sent to: such a source keeps its document and loses only the page.
/// </remarks>
/// <param name="source">The source to read.</param>
/// <param name="location">The document and its page, or the default when the source names no file.</param>
/// <returns>Whether the source names a file.</returns>
public static bool TryGetDocumentLocation(this ISource source, out SourceDocumentLocation location)
{
location = default;
if (string.IsNullOrWhiteSpace(source.URL))
return false;
var cleanedUrl = source.URL.Trim().Replace("\r", string.Empty).Replace("\n", string.Empty);
if (!Uri.TryCreate(cleanedUrl, UriKind.Absolute, out var absoluteUri) || !absoluteUri.IsFile)
return false;
//
// The link was made from a path of this system, so reading it back gives that path again --
// percent-encoded spaces and umlauts included, and with the separators this system uses.
//
var path = absoluteUri.LocalPath;
if (string.IsNullOrWhiteSpace(path))
return false;
location = new(path, ReadPageFromFragment(absoluteUri.Fragment));
return true;
}
private static int? ReadPageFromFragment(string fragment)
{
const string PAGE_PARAMETER = "page=";
foreach (var parameter in fragment.TrimStart('#').Split('&', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
{
if (!parameter.StartsWith(PAGE_PARAMETER, StringComparison.OrdinalIgnoreCase))
continue;
if (int.TryParse(parameter.AsSpan(PAGE_PARAMETER.Length), NumberStyles.None, CultureInfo.InvariantCulture, out var pageNumber) && pageNumber > 0)
return pageNumber;
}
return null;
}
/// <summary>
/// Merges a list of added sources into an existing list of sources, avoiding duplicates based on normalized URLs.
/// </summary>
@@ -0,0 +1,8 @@
namespace AIStudio.Tools;
/// <summary>
/// One group of a source list: a heading and the sources below it.
/// </summary>
/// <param name="Heading">The heading above the group.</param>
/// <param name="Sources">The sources of the group, in the order they are shown.</param>
public readonly record struct SourceGroup(string Heading, IReadOnlyList<NumberedSource> Sources);