mirror of
https://github.com/MindWorkAI/AI-Studio.git
synced 2026-11-05 01:07:21 +00:00
Added a way to read what the AI received from your sources (#1100)
Co-authored-by: Thorsten Sommer <SommerEngineering@users.noreply.github.com>
This commit is contained in:
1 parent
e0d8e18f34
commit
cbc4ce5413
229 files changed
+12739
-1269
No files matched your search
@@ -26,25 +26,35 @@ internal static class IndexedDocumentIds
|
||||
CreateStableGuid($"{dataSourceId}:chunk:{source}:{chunkIndex}");
|
||||
|
||||
/// <summary>
|
||||
/// What the chunk ids of a document derive from: nearly always its fingerprint.
|
||||
/// What the chunk ids of a document derive from: nearly always its fingerprint, and the generation of the chunks.
|
||||
/// </summary>
|
||||
/// <remarks>
|
||||
/// A moved file keeps its parent id and its chunk ids, but its fingerprint now names the new path.
|
||||
/// Another file may later appear at the old path with exactly the fingerprint those chunk ids came
|
||||
/// from, and the fingerprint alone would hand it the same ids. The two documents would then take
|
||||
/// over each other's chunks. Hence the parent id joins whenever a document is not stored under the
|
||||
/// id its key gives, which only happens after a move. Every other document keeps the ids it has.
|
||||
/// id its key gives, which only happens after a move. Every other document keeps the ids it has.<br/><br/>
|
||||
/// Every time a document is indexed again, its chunks get the next generation, and with it new
|
||||
/// ids, even where the fingerprint stays the same, as for a mail. So the new chunks never
|
||||
/// overwrite the old ones, which keep their vectors until the new chunks are stored, cf.
|
||||
/// ReusableVectors. The first indexing of a document is generation 0, which gives the ids
|
||||
/// documents got before generations existed.
|
||||
/// </remarks>
|
||||
/// <param name="dataSourceId">The data source the document belongs to.</param>
|
||||
/// <param name="documentKey">The key of the document.</param>
|
||||
/// <param name="parentId">The id the document is stored under.</param>
|
||||
/// <param name="fingerprint">The fingerprint of the document as it is about to be read.</param>
|
||||
/// <param name="generation">The generation of the chunks about to be written.</param>
|
||||
/// <returns>What to pass to CreateChunkId.</returns>
|
||||
public static string CreateChunkIdSource(string dataSourceId, string documentKey, string parentId, string fingerprint) =>
|
||||
string.Equals(parentId, CreateParentId(dataSourceId, documentKey), StringComparison.Ordinal)
|
||||
public static string CreateChunkIdSource(string dataSourceId, string documentKey, string parentId, string fingerprint, int generation = 0)
|
||||
{
|
||||
var source = string.Equals(parentId, CreateParentId(dataSourceId, documentKey), StringComparison.Ordinal)
|
||||
? fingerprint
|
||||
: $"{parentId}:{fingerprint}";
|
||||
|
||||
return generation is 0 ? source : $"{source}:generation:{generation}";
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The id of a document's row in the index store, which its chunks point at.
|
||||
/// </summary>
|
||||
|
||||
Reference in new issue
Block a user