namespace AIStudio.Tools.PowerPoint; /// /// Reads a folder of pictures and icons, which the LLM may choose from by their names. /// /// /// The LLM reads only the names, never the pictures: a library holds hundreds or thousands of them. /// Only PNG and JPEG pictures count, since the writer can place nothing else; many icons come as SVG, /// which is why the other files are counted, so that the user learns about them. Hidden files and /// folders are left out, as everywhere in AI Studio. /// internal static class PictureLibrary { /// /// The most pictures of a library the LLM reads about in one request. /// /// /// Every name costs tokens in every request, and 5,000 names come to about 50,000 tokens. A /// larger library is sampled at random, anew for every request, so that every picture has the /// same chance rather than those which the file system happens to list first. /// public const int MAX_LIBRARY_PICTURES = 5_000; /// /// What the name of an icon, or of one of the folders it is in, starts with, in any case. /// /// /// The library folder counts as well, so that a folder of icons, such as "Icons", makes all of /// its pictures icons. The folders above the library do not count: they say where the library /// lies, not what it holds. /// public const string ICON_PREFIX = "icon"; private static readonly HashSet PICTURE_EXTENSIONS = new(StringComparer.OrdinalIgnoreCase) { ".png", ".jpg", ".jpeg" }; /// /// Reads a library on a thread of its own, since a folder on a network drive may take a while. /// /// The library folder. /// Where the sample of a large library comes from. /// The most pictures the sample holds, e.g., MAX_LIBRARY_PICTURES, or 0 to count the pictures only. /// Told about every folder which could not be read. /// The cancellation token. /// What the library offers. public static Task ReadAsync(string folder, Random random, int maxPictures, Action? reportInaccessibleFolder, CancellationToken token) => Task.Run(() => Read(folder, random, maxPictures, reportInaccessibleFolder, token), token); /// /// Reads a library. /// /// /// The sample is drawn while the folder is read, by reservoir sampling: the first pictures fill /// the sample, and each later one replaces a random member of it with the chance that keeps every /// picture equally likely. So the list of all pictures is never held in memory. /// /// The library folder. /// Where the sample of a large library comes from. /// The most pictures the sample holds. /// Told about every folder which could not be read. /// The cancellation token. /// What the library offers. public static PictureLibraryContent Read(string folder, Random random, int maxPictures, Action? reportInaccessibleFolder, CancellationToken token) { var sample = new List(); var isIconFolder = StartsWithIconPrefix(Path.GetFileName(Path.TrimEndingDirectorySeparator(folder))); var pictureCount = 0; var iconCount = 0; var skippedFileCount = 0; foreach (var path in DirectoryFileTraversal.EnumerateFiles(folder, true, false, reportInaccessibleDirectory: reportInaccessibleFolder)) { token.ThrowIfCancellationRequested(); if (!PICTURE_EXTENSIONS.Contains(Path.GetExtension(path))) { skippedFileCount++; continue; } var name = Path.GetRelativePath(folder, path).Replace(Path.DirectorySeparatorChar, '/'); var picture = new LibraryPicture(path, name, isIconFolder || IsIcon(name)); pictureCount++; if (picture.IsIcon) iconCount++; if (sample.Count < maxPictures) sample.Add(picture); else if (random.NextInt64(pictureCount) is var slot && slot < maxPictures) sample[(int)slot] = picture; } sample.Sort((first, second) => StringComparer.OrdinalIgnoreCase.Compare(first.Name, second.Name)); return new PictureLibraryContent(sample, pictureCount, iconCount, skippedFileCount); } /// /// Whether a picture is an icon by its own path: its name or the name of one of its folders below the library starts with "icon". /// /// The path of the picture below the library folder, with forward slashes. private static bool IsIcon(string name) => name.Split('/').Any(StartsWithIconPrefix); private static bool StartsWithIconPrefix(string name) => name.StartsWith(ICON_PREFIX, StringComparison.OrdinalIgnoreCase); }