using System.Security.Cryptography;
using System.Text;
namespace AIStudio.Tools.Services.Indexing;
///
/// The ids under which documents and their chunks are stored.
///
///
/// Derived rather than random, so the same document of the same data source ends up under the same
/// ids, in the vector store as well as in the index store. Once stored, they are the identity of the
/// document: a file which moves to another folder keeps its ids, although they were derived from its
/// old path and its old fingerprint, cf. FileSourceIndexer.RelocateStoredFilesAsync. The formats must
/// stay as they are: every stored id was made with them.
///
internal static class IndexedDocumentIds
{
///
/// The id of one chunk, which is also the id of its point in the vector store.
///
/// The data source the document belongs to.
/// What the chunk ids of the document derive from, cf. CreateChunkIdSource.
/// The position of the chunk within the document.
/// The chunk id.
public static string CreateChunkId(string dataSourceId, string source, int chunkIndex) =>
CreateStableGuid($"{dataSourceId}:chunk:{source}:{chunkIndex}");
///
/// What the chunk ids of a document derive from: nearly always its fingerprint.
///
///
/// A moved file keeps its parent id and its chunk ids, but its fingerprint now names the new path.
/// Another file may later appear at the old path with exactly the fingerprint those chunk ids came
/// from, and the fingerprint alone would hand it the same ids. The two documents would then take
/// over each other's chunks. Hence the parent id joins whenever a document is not stored under the
/// id its key gives, which only happens after a move. Every other document keeps the ids it has.
///
/// The data source the document belongs to.
/// The key of the document.
/// The id the document is stored under.
/// The fingerprint of the document as it is about to be read.
/// What to pass to CreateChunkId.
public static string CreateChunkIdSource(string dataSourceId, string documentKey, string parentId, string fingerprint) =>
string.Equals(parentId, CreateParentId(dataSourceId, documentKey), StringComparison.Ordinal)
? fingerprint
: $"{parentId}:{fingerprint}";
///
/// The id of a document's row in the index store, which its chunks point at.
///
/// The data source the document belongs to.
/// What the row is stored under. For a file, its absolute path.
/// The document id.
public static string CreateParentId(string dataSourceId, string absolutePath) =>
CreateStableGuid($"{dataSourceId}:parent-file:{absolutePath}");
private static string CreateStableGuid(string source)
{
var hash = SHA256.HashData(Encoding.UTF8.GetBytes(source));
var guidBytes = hash[..16].ToArray();
guidBytes[6] = (byte)((guidBytes[6] & 0x0F) | 0x40);
guidBytes[8] = (byte)((guidBytes[8] & 0x3F) | 0x80);
return new Guid(guidBytes).ToString();
}
}