From e4fa1cd72a71728e058a5b029794d0a570941937 Mon Sep 17 00:00:00 2001 From: Thorsten Sommer Date: Thu, 11 Jun 2026 12:22:09 +0200 Subject: [PATCH] Added offline build mode for local build script (#806) --- app/Build/Commands/Pdfium.cs | 109 +++++++++++-------- app/Build/Commands/UpdateMetadataCommands.cs | 14 ++- app/Build/Tools/Environment.cs | 14 +++ 3 files changed, 85 insertions(+), 52 deletions(-) diff --git a/app/Build/Commands/Pdfium.cs b/app/Build/Commands/Pdfium.cs index 12348a4b..88effaed 100644 --- a/app/Build/Commands/Pdfium.cs +++ b/app/Build/Commands/Pdfium.cs @@ -7,74 +7,91 @@ namespace Build.Commands; public static class Pdfium { - public static async Task InstallAsync(RID rid, string version) + public static async Task InstallAsync(RID rid, string version, bool offline) { Console.Write($"- Installing Pdfium {version} for {rid.ToUserFriendlyName()} ..."); var cwd = Environment.GetRustRuntimeDirectory(); - var pdfiumTmpDownloadPath = Path.GetTempFileName(); - var pdfiumTmpExtractPath = Directory.CreateTempSubdirectory(); var pdfiumUrl = GetPdfiumDownloadUrl(rid, version); + var library = GetLibraryPath(rid); + var pdfiumLibTargetPath = Path.Join(cwd, "resources", "libraries", library.Filename); - // - // Download the file: - // - Console.Write(" downloading ..."); - using (var client = new HttpClient()) + if (offline) { - var response = await client.GetAsync(pdfiumUrl); - if (!response.IsSuccessStatusCode) + if (File.Exists(pdfiumLibTargetPath)) { - Console.WriteLine($" failed to download Pdfium {version} for {rid.ToUserFriendlyName()} from {pdfiumUrl}"); + Console.WriteLine(" offline mode enabled and library already exists, skipping download"); return; } - await using var fileStream = File.Create(pdfiumTmpDownloadPath); - await response.Content.CopyToAsync(fileStream); + Console.WriteLine($" failed because offline mode is enabled and '{pdfiumLibTargetPath}' does not exist"); + return; } - - // - // Extract the downloaded file: - // - Console.Write(" extracting ..."); - await using(var tgzStream = File.Open(pdfiumTmpDownloadPath, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - await using var uncompressedStream = new GZipStream(tgzStream, CompressionMode.Decompress); - await TarFile.ExtractToDirectoryAsync(uncompressedStream, pdfiumTmpExtractPath.FullName, true); - } - - // - // Copy the library to the target directory: - // - Console.Write(" deploying ..."); - var library = GetLibraryPath(rid); + if (string.IsNullOrWhiteSpace(library.Path)) { Console.WriteLine($" failed to find the library path for {rid.ToUserFriendlyName()}"); return; } - - var pdfiumLibSourcePath = Path.Join(pdfiumTmpExtractPath.FullName, library.Path); - var pdfiumLibTargetPath = Path.Join(cwd, "resources", "libraries", library.Filename); - if (!File.Exists(pdfiumLibSourcePath)) + + var pdfiumLibTargetDirectory = Path.Join(cwd, "resources", "libraries"); + var pdfiumLibTmpTargetPath = Path.Join(pdfiumLibTargetDirectory, $"{library.Filename}.{Guid.NewGuid():N}.tmp"); + var pdfiumLibArchivePath = library.Path.Replace('\\', '/'); + + // + // Download the file: + // + Console.Write(" downloading ..."); + using var client = new HttpClient(); + using var response = await client.GetAsync(pdfiumUrl, HttpCompletionOption.ResponseHeadersRead); + if (!response.IsSuccessStatusCode) { - Console.WriteLine($" failed to find the library file '{pdfiumLibSourcePath}'"); + Console.WriteLine($" failed to download Pdfium {version} for {rid.ToUserFriendlyName()} from {pdfiumUrl}"); return; } - - Directory.CreateDirectory(Path.Join(cwd, "resources", "libraries")); - if (File.Exists(pdfiumLibTargetPath)) - File.Delete(pdfiumLibTargetPath); - - File.Copy(pdfiumLibSourcePath, pdfiumLibTargetPath); - + // - // Cleanup: + // Extract the library from the downloaded file: // - Console.Write(" cleaning up ..."); - File.Delete(pdfiumTmpDownloadPath); - Directory.Delete(pdfiumTmpExtractPath.FullName, true); - + Console.Write(" extracting ..."); + Directory.CreateDirectory(pdfiumLibTargetDirectory); + + var foundLibrary = false; + try + { + await using var downloadStream = await response.Content.ReadAsStreamAsync(); + await using var uncompressedStream = new GZipStream(downloadStream, CompressionMode.Decompress); + await using var tarReader = new TarReader(uncompressedStream, false); + + while (await tarReader.GetNextEntryAsync(false) is { } entry) + { + if (!string.Equals(entry.Name.Replace('\\', '/'), pdfiumLibArchivePath, StringComparison.Ordinal)) + continue; + + if (entry.DataStream == null) + break; + + await using var fileStream = File.Create(pdfiumLibTmpTargetPath); + await entry.DataStream.CopyToAsync(fileStream); + foundLibrary = true; + break; + } + + if (!foundLibrary) + { + Console.WriteLine($" failed to find the library file '{pdfiumLibArchivePath}' in the Pdfium archive"); + return; + } + + Console.Write(" deploying ..."); + File.Move(pdfiumLibTmpTargetPath, pdfiumLibTargetPath, true); + } + finally + { + if (File.Exists(pdfiumLibTmpTargetPath)) + File.Delete(pdfiumLibTmpTargetPath); + } + Console.WriteLine(" done."); } diff --git a/app/Build/Commands/UpdateMetadataCommands.cs b/app/Build/Commands/UpdateMetadataCommands.cs index 303edcd5..dad05f93 100644 --- a/app/Build/Commands/UpdateMetadataCommands.cs +++ b/app/Build/Commands/UpdateMetadataCommands.cs @@ -15,7 +15,8 @@ public sealed partial class UpdateMetadataCommands [Command("release", Description = "Prepare & build the next release")] public async Task Release( [Option("action", ['a'], Description = "The release action: patch, minor, or major")] PrepareAction action = PrepareAction.NONE, - [Option("version", ['v'], Description = "Set a specific version directly, e.g., 26.1.2")] string? version = null) + [Option("version", ['v'], Description = "Set a specific version directly, e.g., 26.1.2")] string? version = null, + [Option("offline", Description = "Skip downloads and use locally available build dependencies")] bool offline = false) { if(!Environment.IsWorkingDirectoryValid()) return; @@ -42,7 +43,7 @@ public sealed partial class UpdateMetadataCommands // Build once to allow the Rust compiler to read the changed metadata // and to update all .NET artifacts: - await this.Build(); + await this.Build(offline); // Now, we update the web assets (which may were updated by the first build): new UpdateWebAssetsCommand().UpdateWebAssets(); @@ -53,7 +54,7 @@ public sealed partial class UpdateMetadataCommands // Build the final release, where Rust knows the updated metadata, the .NET // artifacts are already in place, and .NET knows the updated web assets, etc.: - await this.Build(); + await this.Build(offline); } [Command("update-versions", Description = "The command will update the package versions in the metadata file")] @@ -136,7 +137,8 @@ public sealed partial class UpdateMetadataCommands } [Command("build", Description = "Build MindWork AI Studio")] - public async Task Build() + public async Task Build( + [Option("offline", Description = "Skip downloads and use locally available build dependencies")] bool offline = false) { if(!Environment.IsWorkingDirectoryValid()) return; @@ -153,7 +155,7 @@ public sealed partial class UpdateMetadataCommands await this.UpdateVectorStoreVersion(); var pdfiumVersion = await this.ReadPdfiumVersion(); - await Pdfium.InstallAsync(rid, pdfiumVersion); + await Pdfium.InstallAsync(rid, pdfiumVersion, Environment.IsOfflineBuildRequested(offline)); Console.Write($"- Start .NET build for {rid.ToUserFriendlyName()} ..."); await this.ReadCommandOutput(pathApp, "dotnet", $"clean --configuration release --runtime {rid.AsMicrosoftRid()}"); @@ -750,4 +752,4 @@ public sealed partial class UpdateMetadataCommands [GeneratedRegex("""(?[0-9]+)\.(?[0-9]+)\.(?[0-9]+)""")] private static partial Regex AppVersionRegex(); -} \ No newline at end of file +} diff --git a/app/Build/Tools/Environment.cs b/app/Build/Tools/Environment.cs index f03ff354..39c383f1 100644 --- a/app/Build/Tools/Environment.cs +++ b/app/Build/Tools/Environment.cs @@ -7,6 +7,7 @@ namespace Build.Tools; public static class Environment { public const string DOTNET_VERSION = "net9.0"; + public const string BUILD_OFFLINE_ENVIRONMENT_VARIABLE = "AI_STUDIO_BUILD_OFFLINE"; public static readonly Encoding UTF8_NO_BOM = new UTF8Encoding(false); private static readonly Dictionary ALL_RIDS = Enum.GetValues().Select(rid => new KeyValuePair(rid, rid.AsMicrosoftRid())).ToDictionary(kvp => kvp.Key, kvp => kvp.Value); @@ -47,6 +48,19 @@ public static class Environment return Path.GetFullPath(directory); } + public static bool IsOfflineBuildRequested(bool offlineOption) + { + if (offlineOption) + return true; + + var environmentValue = global::System.Environment.GetEnvironmentVariable(BUILD_OFFLINE_ENVIRONMENT_VARIABLE); + return environmentValue?.Trim().ToLowerInvariant() switch + { + "1" or "true" or "yes" or "on" => true, + _ => false, + }; + } + public static string? GetOS() { if(RuntimeInformation.IsOSPlatform(OSPlatform.Windows))