Finished the Pandoc integration (#487)

This commit is contained in:
Thorsten Sommer authored and GitHub committed 2025-05-30 22:39:16 +02:00
1 parent 5b685193ef
commit 437da0fd9b
30 files changed
+1600 -349

No files matched your search

+142 -129
View File
@@ -1,58 +1,56 @@
using System.Diagnostics;
using System.Formats.Tar;
using System.IO.Compression;
using System.Runtime.InteropServices;
using System.Reflection;
using System.Text.RegularExpressions;
using AIStudio.Tools.Metadata;
using AIStudio.Tools.Services;
using SharedTools;
namespace AIStudio.Tools;
public static partial class Pandoc
{
private const string CPU_ARCHITECTURE = "win-x64";
private static string TB(string fallbackEN) => PluginSystem.I18N.I.T(fallbackEN, typeof(Pandoc).Namespace, nameof(Pandoc));
private static readonly Assembly ASSEMBLY = Assembly.GetExecutingAssembly();
private static readonly MetaDataArchitectureAttribute META_DATA_ARCH = ASSEMBLY.GetCustomAttribute<MetaDataArchitectureAttribute>()!;
private static readonly RID CPU_ARCHITECTURE = META_DATA_ARCH.Architecture.ToRID();
private const string DOWNLOAD_URL = "https://github.com/jgm/pandoc/releases/download";
private const string LATEST_URL = "https://github.com/jgm/pandoc/releases/latest";
private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger("PandocService");
private static readonly Version MINIMUM_REQUIRED_VERSION = new (3, 6);
private static readonly Version FALLBACK_VERSION = new (3, 7, 0, 1);
private static readonly ILogger LOG = Program.LOGGER_FACTORY.CreateLogger("Pandoc");
private static readonly Version MINIMUM_REQUIRED_VERSION = new (3, 7, 0, 2);
private static readonly Version FALLBACK_VERSION = new (3, 7, 0, 2);
/// <summary>
/// Checks if pandoc is available on the system and can be started as a process or present in AiStudio's data dir
/// Prepares a Pandoc process by using the Pandoc process builder.
/// </summary>
/// <param name="rustService">Global rust service to access file system and data dir</param>
/// <param name="showMessages">Controls if snackbars are shown to the user</param>
/// <returns>True, if pandoc is available and the minimum required version is met, else False.</returns>
public static async Task<bool> CheckAvailabilityAsync(RustService rustService, bool showMessages = true)
{
var installDir = await GetPandocDataFolder(rustService);
var subdirectories = Directory.GetDirectories(installDir);
/// <returns>The Pandoc process builder with default settings.</returns>
public static PandocProcessBuilder PreparePandocProcess() => PandocProcessBuilder.Create();
if (subdirectories.Length > 1)
{
await InstallAsync(rustService);
return true;
}
if (HasPandoc(installDir)) return true;
/// <summary>
/// Checks if pandoc is available on the system and can be started as a process or is present in AI Studio's data dir.
/// </summary>
/// <param name="rustService">Global rust service to access file system and data dir.</param>
/// <param name="showMessages">Controls if snackbars are shown to the user.</param>
/// <returns>True, if pandoc is available and the minimum required version is met, else false.</returns>
public static async Task<PandocInstallation> CheckAvailabilityAsync(RustService rustService, bool showMessages = true)
{
try
{
var startInfo = new ProcessStartInfo
{
FileName = GetPandocExecutableName(),
Arguments = "--version",
RedirectStandardOutput = true,
UseShellExecute = false,
CreateNoWindow = true
};
using var process = Process.Start(startInfo);
var preparedProcess = await PreparePandocProcess().AddArgument("--version").BuildAsync(rustService);
using var process = Process.Start(preparedProcess.StartInfo);
if (process == null)
{
if (showMessages)
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Help, "The pandoc process could not be started."));
LOG.LogInformation("The pandoc process was not started, it was null");
return false;
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Help, TB("Was not able to check the Pandoc installation.")));
LOG.LogInformation("The Pandoc process was not started, it was null");
return new(false, TB("Was not able to check the Pandoc installation."), false, string.Empty, preparedProcess.IsLocal);
}
var output = await process.StandardOutput.ReadToEndAsync();
@@ -60,65 +58,48 @@ public static partial class Pandoc
if (process.ExitCode != 0)
{
if (showMessages)
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Error, $"The pandoc process exited unexpectedly."));
LOG.LogError("The pandoc process was exited with code {ProcessExitCode}", process.ExitCode);
return false;
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Error, TB("Pandoc is not available on the system or the process had issues.")));
LOG.LogError("The Pandoc process was exited with code {ProcessExitCode}", process.ExitCode);
return new(false, TB("Pandoc is not available on the system or the process had issues."), false, string.Empty, preparedProcess.IsLocal);
}
var versionMatch = PandocCmdRegex().Match(output);
if (!versionMatch.Success)
{
if (showMessages)
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Terminal, $"pandoc --version returned an invalid format."));
LOG.LogError("pandoc --version returned an invalid format:\n {Output}", output);
return false;
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Terminal, TB("Was not able to validate the Pandoc installation.")));
LOG.LogError("Pandoc --version returned an invalid format: {Output}", output);
return new(false, TB("Was not able to validate the Pandoc installation."), false, string.Empty, preparedProcess.IsLocal);
}
var versions = versionMatch.Groups[1].Value;
var installedVersion = Version.Parse(versions);
var installedVersionString = installedVersion.ToString();
if (installedVersion >= MINIMUM_REQUIRED_VERSION)
{
if (showMessages)
await MessageBus.INSTANCE.SendSuccess(new(Icons.Material.Filled.CheckCircle, $"Pandoc {installedVersion.ToString()} is installed."));
return true;
await MessageBus.INSTANCE.SendSuccess(new(Icons.Material.Filled.CheckCircle, string.Format(TB("Pandoc v{0} is installed."), installedVersionString)));
LOG.LogInformation("Pandoc v{0} is installed and matches the required version (v{1})", installedVersionString, MINIMUM_REQUIRED_VERSION.ToString());
return new(true, string.Empty, true, installedVersionString, preparedProcess.IsLocal);
}
if (showMessages)
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Build, $"Pandoc {installedVersion.ToString()} is installed, but it doesn't match the required version ({MINIMUM_REQUIRED_VERSION.ToString()})."));
LOG.LogInformation("Pandoc {Installed} is installed, but it does not match the required version ({Requirement})", installedVersion.ToString(), MINIMUM_REQUIRED_VERSION.ToString());
return false;
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Build, string.Format(TB("Pandoc v{0} is installed, but it doesn't match the required version (v{1})."), installedVersionString, MINIMUM_REQUIRED_VERSION.ToString())));
LOG.LogWarning("Pandoc v{0} is installed, but it does not match the required version (v{1})", installedVersionString, MINIMUM_REQUIRED_VERSION.ToString());
return new(true, string.Format(TB("Pandoc v{0} is installed, but it does not match the required version (v{1})."), installedVersionString, MINIMUM_REQUIRED_VERSION.ToString()), false, installedVersionString, preparedProcess.IsLocal);
}
catch (Exception e)
{
if (showMessages)
await MessageBus.INSTANCE.SendError(new (@Icons.Material.Filled.AppsOutage, "Pandoc is not installed."));
LOG.LogError("Pandoc is not installed and threw an exception:\n {Message}", e.Message);
return false;
}
}
private static bool HasPandoc(string pandocDirectory)
{
try
{
var subdirectories = Directory.GetDirectories(pandocDirectory);
foreach (var subdirectory in subdirectories)
{
var pandocPath = Path.Combine(subdirectory, "pandoc.exe");
if (File.Exists(pandocPath))
{
return true;
}
}
return false;
}
catch (Exception ex)
{
LOG.LogInformation("Pandoc is not installed in the data directory and might have thrown and error:\n{ErrorMessage}", ex.Message);
return false;
await MessageBus.INSTANCE.SendError(new (@Icons.Material.Filled.AppsOutage, TB("It seems that Pandoc is not installed.")));
LOG.LogError("Pandoc is not installed and threw an exception: {0}", e.Message);
return new(false, TB("It seems that Pandoc is not installed."), false, string.Empty, false);
}
}
@@ -129,70 +110,79 @@ public static partial class Pandoc
/// <returns>None</returns>
public static async Task InstallAsync(RustService rustService)
{
var latestVersion = await FetchLatestVersionAsync();
var installDir = await GetPandocDataFolder(rustService);
ClearFolder(installDir);
LOG.LogInformation("Trying to install Pandoc v{0} to '{1}'...", latestVersion, installDir);
try
{
if (!Directory.Exists(installDir))
Directory.CreateDirectory(installDir);
using var client = new HttpClient();
var uri = await GenerateUriAsync();
var response = await client.GetAsync(uri);
if (!response.IsSuccessStatusCode)
// Create a temporary file to download the archive to:
var pandocTempDownloadFile = Path.GetTempFileName();
//
// Download the latest Pandoc archive from GitHub:
//
var uri = await GenerateArchiveUriAsync();
using (var client = new HttpClient())
{
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Error, $"Pandoc was not installed successfully, because the download archive was not found."));
LOG.LogError("Pandoc was not installed, the release archive was not found (Status Code {StatusCode}):\n{Uri}\n{Message}", response.StatusCode, uri, response.RequestMessage);
return;
}
var fileBytes = await response.Content.ReadAsByteArrayAsync();
var response = await client.GetAsync(uri);
if (!response.IsSuccessStatusCode)
{
await MessageBus.INSTANCE.SendError(new(Icons.Material.Filled.Error, TB("Pandoc was not installed successfully, because the archive was not found.")));
LOG.LogError("Pandoc was not installed successfully, because the archive was not found (status code {0}): url='{1}', message='{2}'", response.StatusCode, uri, response.RequestMessage);
return;
}
if (uri.Contains(".zip"))
{
var tempZipPath = Path.Join(Path.GetTempPath(), "pandoc.zip");
await File.WriteAllBytesAsync(tempZipPath, fileBytes);
ZipFile.ExtractToDirectory(tempZipPath, installDir);
File.Delete(tempZipPath);
// Download the archive to the temporary file:
await using var tempFileStream = File.Create(pandocTempDownloadFile);
await response.Content.CopyToAsync(tempFileStream);
}
else if (uri.Contains(".tar.gz"))
if (uri.EndsWith(".zip", StringComparison.OrdinalIgnoreCase))
{
var tempTarPath = Path.Join(Path.GetTempPath(), "pandoc.tar.gz");
await File.WriteAllBytesAsync(tempTarPath, fileBytes);
ZipFile.ExtractToDirectory(tempTarPath, installDir);
File.Delete(tempTarPath);
ZipFile.ExtractToDirectory(pandocTempDownloadFile, installDir);
}
else if (uri.EndsWith(".tar.gz", StringComparison.OrdinalIgnoreCase))
{
await using var tgzStream = File.Open(pandocTempDownloadFile, FileMode.Open, FileAccess.Read, FileShare.Read);
await using var uncompressedStream = new GZipStream(tgzStream, CompressionMode.Decompress);
await TarFile.ExtractToDirectoryAsync(uncompressedStream, installDir, true);
}
else
{
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Error, $"Pandoc was not installed successfully, because the download archive type is unknown."));
LOG.LogError("Pandoc was not installed, the download archive is unknown:\n {Uri}", uri);
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Error, TB("Pandoc was not installed successfully, because the archive type is unknown.")));
LOG.LogError("Pandoc was not installed, the archive is unknown: url='{0}'", uri);
return;
}
await MessageBus.INSTANCE.SendSuccess(new(Icons.Material.Filled.CheckCircle,
$"Pandoc {await FetchLatestVersionAsync()} was installed successfully."));
File.Delete(pandocTempDownloadFile);
await MessageBus.INSTANCE.SendSuccess(new(Icons.Material.Filled.CheckCircle, string.Format(TB("Pandoc v{0} was installed successfully."), latestVersion)));
LOG.LogInformation("Pandoc v{0} was installed successfully.", latestVersion);
}
catch (Exception ex)
{
Console.WriteLine($"Fehler: {ex.Message}");
LOG.LogError(ex, "An error occurred while installing Pandoc.");
}
}
private static void ClearFolder(string path)
{
if (!Directory.Exists(path)) return;
if (!Directory.Exists(path))
return;
try
{
foreach (var dir in Directory.GetDirectories(path))
{
Directory.Delete(dir, true);
}
Directory.Delete(path, true);
}
catch (Exception ex)
{
LOG.LogError(ex, "Error clearing pandoc folder.");
LOG.LogError(ex, "Error clearing pandoc installation directory.");
}
}
@@ -207,18 +197,17 @@ public static partial class Pandoc
if (!response.IsSuccessStatusCode)
{
LOG.LogError("Code {StatusCode}: Could not fetch Pandoc's latest page:\n {Response}", response.StatusCode, response.RequestMessage);
await MessageBus.INSTANCE.SendWarning(new (Icons.Material.Filled.Warning, $"The latest pandoc version was not found, installing version {FALLBACK_VERSION.ToString()} instead."));
LOG.LogError("Code {StatusCode}: Could not fetch Pandoc's latest page: {Response}", response.StatusCode, response.RequestMessage);
await MessageBus.INSTANCE.SendWarning(new (Icons.Material.Filled.Warning, string.Format(TB("The latest Pandoc version was not found, installing version {0} instead."), FALLBACK_VERSION.ToString())));
return FALLBACK_VERSION.ToString();
}
var htmlContent = await response.Content.ReadAsStringAsync();
var versionMatch = LatestVersionRegex().Match(htmlContent);
if (!versionMatch.Success)
{
LOG.LogError("The latest version regex returned nothing:\n {Value}", versionMatch.Groups.ToString());
await MessageBus.INSTANCE.SendWarning(new (Icons.Material.Filled.Warning, $"The latest pandoc version was not found, installing version {FALLBACK_VERSION.ToString()} instead."));
LOG.LogError("The latest version regex returned nothing: {0}", versionMatch.Groups.ToString());
await MessageBus.INSTANCE.SendWarning(new (Icons.Material.Filled.Warning, string.Format(TB("The latest Pandoc version was not found, installing version {0} instead."), FALLBACK_VERSION.ToString())));
return FALLBACK_VERSION.ToString();
}
@@ -227,20 +216,35 @@ public static partial class Pandoc
}
/// <summary>
/// Reads the systems architecture to find the correct archive
/// Reads the systems architecture to find the correct archive.
/// </summary>
/// <returns>Full URI to the right archive in Pandoc's repo</returns>
public static async Task<string> GenerateUriAsync()
/// <returns>Full URI to the right archive in Pandoc's repository.</returns>
public static async Task<string> GenerateArchiveUriAsync()
{
var version = await FetchLatestVersionAsync();
var baseUri = $"{DOWNLOAD_URL}/{version}/pandoc-{version}-";
return CPU_ARCHITECTURE switch
{
"win-x64" => $"{baseUri}windows-x86_64.zip",
"osx-x64" => $"{baseUri}x86_64-macOS.zip",
"osx-arm64" => $"{baseUri}arm64-macOS.zip",
"linux-x64" => $"{baseUri}linux-amd64.tar.gz",
"linux-arm" => $"{baseUri}linux-arm64.tar.gz",
//
// Unfortunately, pandoc is not yet available for ARM64 Windows systems,
// so we have to use the x86_64 version for now. ARM Windows contains
// an x86_64 emulation layer, so it should work fine for now.
//
// Pandoc would be available for ARM64 Windows, but the Haskell compiler
// does not support ARM64 Windows yet. Here are the related issues:
//
// - Haskell compiler: https://gitlab.haskell.org/ghc/ghc/-/issues/24603
// - Haskell ARM MR: https://gitlab.haskell.org/ghc/ghc/-/merge_requests/13856
// - Pandoc ARM64: https://github.com/jgm/pandoc/issues/10095
//
RID.WIN_X64 or RID.WIN_ARM64 => $"{baseUri}windows-x86_64.zip",
RID.OSX_X64 => $"{baseUri}x86_64-macOS.zip",
RID.OSX_ARM64 => $"{baseUri}arm64-macOS.zip",
RID.LINUX_X64 => $"{baseUri}linux-amd64.tar.gz",
RID.LINUX_ARM64 => $"{baseUri}linux-arm64.tar.gz",
_ => string.Empty,
};
}
@@ -256,25 +260,34 @@ public static partial class Pandoc
switch (CPU_ARCHITECTURE)
{
case "win-x64":
//
// Unfortunately, pandoc is not yet available for ARM64 Windows systems,
// so we have to use the x86_64 version for now. ARM Windows contains
// an x86_64 emulation layer, so it should work fine for now.
//
// Pandoc would be available for ARM64 Windows, but the Haskell compiler
// does not support ARM64 Windows yet. Here are the related issues:
//
// - Haskell compiler: https://gitlab.haskell.org/ghc/ghc/-/issues/24603
// - Haskell ARM MR: https://gitlab.haskell.org/ghc/ghc/-/merge_requests/13856
// - Pandoc ARM64: https://github.com/jgm/pandoc/issues/10095
//
case RID.WIN_X64 or RID.WIN_ARM64:
return $"{baseUri}windows-x86_64.msi";
case "osx-x64":
case RID.OSX_X64:
return $"{baseUri}x86_64-macOS.pkg";
case "osx-arm64":
return $"{baseUri}arm64-macOS.pkg\n";
case RID.OSX_ARM64:
return $"{baseUri}arm64-macOS.pkg";
default:
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Terminal, $"Installers are not available on {CPU_ARCHITECTURE} systems."));
await MessageBus.INSTANCE.SendError(new (Icons.Material.Filled.Terminal, string.Format(TB("Installers are not available on {0} systems."), CPU_ARCHITECTURE.ToUserFriendlyName())));
return string.Empty;
}
}
/// <summary>
/// Reads the os platform to determine the used executable name
/// </summary>
/// <returns>Name of the pandoc executable</returns>
private static string GetPandocExecutableName() => RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "pandoc.exe" : "pandoc";
private static async Task<string> GetPandocDataFolder(RustService rustService) => Path.Join(await rustService.GetDataDirectory(), "pandoc");
public static async Task<string> GetPandocDataFolder(RustService rustService) => Path.Join(await rustService.GetDataDirectory(), "pandoc");
[GeneratedRegex(@"pandoc(?:\.exe)?\s*([0-9]+\.[0-9]+(?:\.[0-9]+)?(?:\.[0-9]+)?)")]
private static partial Regex PandocCmdRegex();
@@ -0,0 +1,3 @@
namespace AIStudio.Tools;
public readonly record struct PandocExecutable(string Executable, bool IsLocalInstallation);
@@ -0,0 +1,3 @@
namespace AIStudio.Tools;
public readonly record struct PandocInstallation(bool CheckWasSuccessful, string ErrorMessage, bool IsAvailable, string Version, bool IsLocalInstallation);
@@ -0,0 +1,10 @@
using System.Diagnostics;
namespace AIStudio.Tools;
public sealed class PandocPreparedProcess(ProcessStartInfo startInfo, bool isLocal)
{
public ProcessStartInfo StartInfo => startInfo;
public bool IsLocal => isLocal;
}
@@ -0,0 +1,135 @@
using System.Diagnostics;
using System.Reflection;
using System.Text;
using AIStudio.Tools.Metadata;
using AIStudio.Tools.Services;
using SharedTools;
namespace AIStudio.Tools;
public sealed class PandocProcessBuilder
{
private static readonly Assembly ASSEMBLY = Assembly.GetExecutingAssembly();
private static readonly MetaDataArchitectureAttribute META_DATA_ARCH = ASSEMBLY.GetCustomAttribute<MetaDataArchitectureAttribute>()!;
private static readonly RID CPU_ARCHITECTURE = META_DATA_ARCH.Architecture.ToRID();
private string? providedInputFile;
private string? providedOutputFile;
private string? providedInputFormat;
private string? providedOutputFormat;
private readonly List<string> additionalArguments = new();
private PandocProcessBuilder()
{
}
public static PandocProcessBuilder Create() => new();
public PandocProcessBuilder WithInputFile(string inputFile)
{
this.providedInputFile = inputFile;
return this;
}
public PandocProcessBuilder WithOutputFile(string outputFile)
{
this.providedOutputFile = outputFile;
return this;
}
public PandocProcessBuilder WithInputFormat(string inputFormat)
{
this.providedInputFormat = inputFormat;
return this;
}
public PandocProcessBuilder WithOutputFormat(string outputFormat)
{
this.providedOutputFormat = outputFormat;
return this;
}
public PandocProcessBuilder AddArgument(string argument)
{
this.additionalArguments.Add(argument);
return this;
}
public async Task<PandocPreparedProcess> BuildAsync(RustService rustService)
{
var sbArguments = new StringBuilder();
if(!string.IsNullOrWhiteSpace(this.providedInputFile))
sbArguments.Append(this.providedInputFile);
if(!string.IsNullOrWhiteSpace(this.providedInputFormat))
sbArguments.Append($" -f {this.providedInputFormat}");
if(!string.IsNullOrWhiteSpace(this.providedOutputFormat))
sbArguments.Append($" -t {this.providedOutputFormat}");
foreach (var additionalArgument in this.additionalArguments)
sbArguments.Append($" {additionalArgument}");
if(!string.IsNullOrWhiteSpace(this.providedOutputFile))
sbArguments.Append($" -o {this.providedOutputFile}");
var pandocExecutable = await PandocExecutablePath(rustService);
return new (new ProcessStartInfo
{
FileName = pandocExecutable.Executable,
Arguments = sbArguments.ToString(),
RedirectStandardOutput = true,
RedirectStandardError = true,
UseShellExecute = false,
CreateNoWindow = true
}, pandocExecutable.IsLocalInstallation);
}
/// <summary>
/// Returns the path to the pandoc executable.
/// </summary>
/// <remarks>
/// Any local installation of pandoc will be preferred over the system-wide installation.
/// When a local installation is found, its absolute path will be returned. In case no local
/// installation is found, the name of the pandoc executable will be returned.
/// </remarks>
/// <param name="rustService">Global rust service to access file system and data dir.</param>
/// <returns>Path to the pandoc executable.</returns>
private static async Task<PandocExecutable> PandocExecutablePath(RustService rustService)
{
//
// First, we try to find the pandoc executable in the data directory.
// Any local installation should be preferred over the system-wide installation.
//
var localInstallationRootDirectory = await Pandoc.GetPandocDataFolder(rustService);
try
{
var executableName = PandocExecutableName;
var subdirectories = Directory.GetDirectories(localInstallationRootDirectory, "*", SearchOption.AllDirectories);
foreach (var subdirectory in subdirectories)
{
var pandocPath = Path.Combine(subdirectory, executableName);
if (File.Exists(pandocPath))
return new(pandocPath, true);
}
}
catch
{
// ignored
}
//
// When no local installation was found, we assume that the pandoc executable is in the system PATH.
//
return new(PandocExecutableName, false);
}
/// <summary>
/// Reads the os platform to determine the used executable name.
/// </summary>
public static string PandocExecutableName => CPU_ARCHITECTURE is RID.WIN_ARM64 or RID.WIN_X64 ? "pandoc.exe" : "pandoc";
}