using System.Text.Json; using System.Text.Json.Nodes; using AIStudio.Tools.Mathematics; using AIStudio.Tools.ToolCallingSystem.ToolCallingImplementations.Mathematics; using Microsoft.Extensions.Logging.Abstractions; using PeterO.Numbers; namespace AIStudio.Tests.Tools.Mathematics; [TestFixture] public sealed class CalculateToolTests { private static JsonObject Calculate(string expression, int digits = 20) => CalculateTool.Calculate(expression, digits, CancellationToken.None); private static string Refusal(string expression) => Assert.Throws(() => Calculate(expression))!.Message; private static string Text(JsonObject result, string property) => result[property]!.GetValue(); private static double RelativeError(JsonObject result, string expected) { var reference = EDecimal.FromString(expected); return EDecimal.FromString(Text(result, "decimal")).Subtract(reference).Abs().Divide(reference.Abs(), EContext.ForPrecision(20)).ToDouble(); } private static JsonElement Arguments(string json) { using var document = JsonDocument.Parse(json); return document.RootElement.Clone(); } [Test] public void KeepsAFractionExact() { var result = Calculate("547 / 57576"); using (Assert.EnterMultipleScope()) { Assert.That(result["expression"]!.GetValue(), Is.EqualTo("547 / 57576"), "The expression reads as written; its value is the fraction."); Assert.That(result["exact"]!.GetValue(), Is.EqualTo("547/57576")); Assert.That(result["exact_latex"]!.GetValue(), Is.EqualTo(@"\frac{547}{57576}")); Assert.That(result["decimal"]!.GetValue(), Is.EqualTo("0.0095004863137418368765")); Assert.That(result["digits"]!.GetValue(), Is.EqualTo(20)); Assert.That(result["approximate"]!.GetValue(), Is.False); Assert.That(result.ContainsKey("estimated_error"), Is.False); Assert.That(result.ContainsKey("issues"), Is.False); } } [Test] public void GivesAnIrrationalValueWithAllDigitsAskedFor() { var root = Calculate("sqrt(2)", 30); var power = Calculate("pi^4/15", 60); using (Assert.EnterMultipleScope()) { Assert.That(root["decimal"]!.GetValue(), Is.EqualTo("1.41421356237309504880168872421")); Assert.That(root.ContainsKey("exact"), Is.False); Assert.That(power["decimal"]!.GetValue(), Is.EqualTo("6.49393940226682914909602217924700741664850571151236144609786"), "Without guard digits, the last digits would be off."); Assert.That(power["digits"]!.GetValue(), Is.EqualTo(60)); } } [Test] public void DoesNotTakeANumberCloseToAnIntegerForIt() { // e^(pi*sqrt(163)) is 262537412640768743.99999999999925... var result = Calculate("e^(pi*sqrt(163))", 30); using (Assert.EnterMultipleScope()) { Assert.That(result.ContainsKey("exact"), Is.False); Assert.That(result["decimal"]!.GetValue(), Is.EqualTo("262537412640768743.999999999999")); } } [Test] public void GivesComplexAndInfiniteValues() { var root = Calculate("sqrt(-4)"); var logarithm = Calculate("ln(0)"); using (Assert.EnterMultipleScope()) { Assert.That(root["decimal"]!.GetValue(), Is.EqualTo("2i")); Assert.That(root.ContainsKey("exact"), Is.True); Assert.That(logarithm["decimal"]!.GetValue(), Is.EqualTo("-oo")); Assert.That(logarithm["digits"]!.GetValue(), Is.Zero); Assert.That(logarithm["issues"]![0]!.GetValue(), Is.EqualTo("The value is infinite.")); } } [Test] public void ReadsEveryDecimalLiteralAsItsFraction() { // AngouriMath alone reads 6.62607015e-34 as an inexact decimal and 1e-60 as 0, see AngouriMathBehaviorTests.DowncastingWhileParsingLosesTinyLiterals: var planck = Calculate("6.62607015e-34"); var tiny = Calculate("1e-60"); var equation = Calculate("1 + 1e-40 = 1"); using (Assert.EnterMultipleScope()) { Assert.That(Text(planck, "exact"), Is.EqualTo("132521403/200000000000000000000000000000000000000000")); Assert.That(Text(planck, "decimal"), Is.EqualTo("6.62607015e-34")); Assert.That(Text(tiny, "exact"), Is.EqualTo("1/1" + new string('0', 60))); Assert.That(equation["exactly_equal"]!.GetValue(), Is.False, "Simplifying would drop an inexact 1e-40 from the sum."); Assert.That(equation["numerically_equal"]!.GetValue(), Is.True, "Both sides agree in the 20 digits compared."); } } [Test] public void KeepsTheDigitsOfTinyValues() { // Each of these came out as 0 while AngouriMath downcast, see AngouriMathBehaviorTests.DowncastingTurnsTinyValuesIntoZero: var photon = Calculate("6.62607015e-34 * 299792458 / 500e-9"); var thermal = Calculate("1.380649e-23 * 300"); var exponential = Calculate("e^(-100)"); var tinier = Calculate("exp(-1000)"); var fraction = Calculate("21/10^50 * pi"); var sum = Calculate("21/10^50 + 1e-60"); var exactFraction = Calculate("1/10^200"); var integral = Calculate("integral(1e-30*x, x, 0, 1)"); var quotient = Calculate("pi/10^200"); var simplifiedToZero = Calculate("1e-2000 * pi"); using (Assert.EnterMultipleScope()) { Assert.That(RelativeError(quotient, "3.14159265358979323846264e-200"), Is.LessThan(1e-19), "AngouriMath rounds a quotient to decimal places, so the precision grows until the value shows."); Assert.That(simplifiedToZero.ContainsKey("exact"), Is.False, "AngouriMath simplifies the value to 0, see AngouriMathBehaviorTests.DowncastingWhileParsingLosesTinyLiterals."); Assert.That(RelativeError(simplifiedToZero, "3.14159265358979323846264e-2000"), Is.LessThan(1e-19)); Assert.That(Text(photon, "decimal"), Is.EqualTo("3.9728917142978574e-19")); Assert.That(Text(thermal, "decimal"), Is.EqualTo("4.141947e-21")); Assert.That(RelativeError(exponential, "3.72007597602083596295969580386e-44"), Is.LessThan(1e-19)); Assert.That(RelativeError(tinier, "5.07595889754945676529180947957e-435"), Is.LessThan(1e-19)); Assert.That(RelativeError(fraction, "6.5973445725385658007715511049e-49"), Is.LessThan(1e-19), "The fraction is rounded to decimal places unless the precision grows by its leading zeros."); Assert.That(Text(sum, "decimal"), Is.EqualTo("2.10000000001e-49")); Assert.That(Text(exactFraction, "exact"), Is.EqualTo("1/1" + new string('0', 200))); Assert.That(Text(exactFraction, "decimal"), Is.EqualTo("1e-200")); Assert.That(RelativeError(integral, "5e-31"), Is.LessThan(1e-13)); foreach (var result in new[] { photon, thermal, exponential, tinier, fraction, sum, exactFraction }) Assert.That(result["digits"]!.GetValue(), Is.EqualTo(20)); } } [Test] public void SaysWhenAZeroMayBeATinyValue() { // Simplifying gives 0, but the fraction 1e-2000 needs more digits than NumericEvaluator.MAX_PRECISION for a numerical check: var result = Calculate("1e-2000 * pi - 1e-2000 * pi"); using (Assert.EnterMultipleScope()) { Assert.That(Text(result, "decimal"), Is.EqualTo("0")); Assert.That(result["digits"]!.GetValue(), Is.Zero); Assert.That(result["issues"]!.AsArray().Select(issue => issue!.GetValue()), Has.Some.StartsWith("The value evaluates to 0 with ").And.Some.Contains("cannot show that it is exactly 0")); } } [Test] public void IntegratesADefiniteIntegralNumerically() { var result = Calculate("integral(x^3/(e^x-1), x, 0, +oo)"); var digits = result["digits"]!.GetValue(); var exact = Math.Pow(Math.PI, 4) / 15; using (Assert.EnterMultipleScope()) { Assert.That(digits, Is.InRange(12, 15), "A double carries no more, and the estimated error of the quadrature is conservative."); Assert.That(double.Parse(result["decimal"]!.GetValue(), System.Globalization.CultureInfo.InvariantCulture), Is.EqualTo(exact).Within(Math.Pow(10, 1 - digits) * exact), "Every digit reported is right."); Assert.That(result["approximate"]!.GetValue(), Is.True); Assert.That(result.ContainsKey("estimated_error"), Is.True); Assert.That(result.ContainsKey("exact"), Is.False); Assert.That(result["issues"]!.AsArray().Select(issue => issue!.GetValue()), Has.Some.Contains("was integrated numerically")); Assert.That(result["issues"]!.AsArray().Select(issue => issue!.GetValue()), Has.Some.StartsWith("Only "), "Fewer digits than asked for are certain."); } } [Test] public void ComputesAroundAnIntegral() { var result = Calculate("1 + integral(e^(-x^2), x, -oo, +oo)"); Assert.That(double.Parse(result["decimal"]!.GetValue(), System.Globalization.CultureInfo.InvariantCulture), Is.EqualTo(1 + Math.Sqrt(Math.PI)).Within(1e-13)); } [Test] public void ChecksTheExampleEquation() { var result = Calculate("integral(x^3/(e^x-1), x, 0, +oo) = pi^4/15"); using (Assert.EnterMultipleScope()) { Assert.That(result["numerically_equal"]!.GetValue(), Is.True); Assert.That(result["matching_digits"]!.GetValue(), Is.InRange(12, 15)); Assert.That(result["left"]!["approximate"]!.GetValue(), Is.True); Assert.That(result["right"]!["expression"]!.GetValue(), Is.EqualTo("pi ^ 4 / 15")); Assert.That(result.ContainsKey("exactly_equal"), Is.False, "Only two exact sides settle an equation."); } } [Test] public void SettlesAnEquationOfExactValues() { var equal = Calculate("2 + 2 = 4"); var close = Calculate("1/3 = 0.333"); var beyondTheDigits = Calculate("1/3 = 333333333333333333333333333333/1000000000000000000000000000000"); using (Assert.EnterMultipleScope()) { Assert.That(equal["exactly_equal"]!.GetValue(), Is.True); Assert.That(equal["numerically_equal"]!.GetValue(), Is.True); Assert.That(close["exactly_equal"]!.GetValue(), Is.False); Assert.That(close["numerically_equal"]!.GetValue(), Is.False); Assert.That(close["matching_digits"]!.GetValue(), Is.EqualTo(3)); Assert.That(beyondTheDigits["numerically_equal"]!.GetValue(), Is.True, "Twenty digits agree."); Assert.That(beyondTheDigits["exactly_equal"]!.GetValue(), Is.False, "The exact values differ all the same."); } } [Test] public void RefusesWhatHasNoNumericValue() { using (Assert.EnterMultipleScope()) { Assert.That(Refusal("1/0"), Does.Contain("has no value: it evaluates to NaN")); Assert.That(Refusal("x + 1"), Does.Contain("contains the variable x, so it has no numeric value. compute_symbolically works with variables.")); Assert.That(Refusal("a*b"), Does.Contain("contains the variables a, b,")); Assert.That(Refusal("1 < 2"), Does.Contain("is a statement, such as a comparison")); Assert.That(Refusal("(10^9)!"), Does.Contain("digits as an exact number")); Assert.That(() => Calculate("integral(x^2, x)"), Throws.InstanceOf()); Assert.That(Refusal("sum(1/k, k, 1, 1000)"), Is.EqualTo("'sum(1 / k, k, 1, 1000)' has no value the tool can compute as a single number, as for a sum without a closed form, an infinite sum, or a matrix."), "AngouriMath sums only in closed form; its own message tells a developer to call EvaluableNumerical."); Assert.That(Refusal("sum(1/k^2, k, 1, +oo)"), Does.Contain("has no value the tool can compute as a single number")); Assert.That(Refusal("[1, 2] + [3, 4]"), Does.Contain("has no value the tool can compute as a single number")); } } [Test] public void ComputesDivisorsMultiplesAndSumsInClosedForm() { using (Assert.EnterMultipleScope()) { Assert.That(Text(Calculate("gcd(12, 18)"), "exact"), Is.EqualTo("6")); Assert.That(Text(Calculate("lcm(4, 6, 10)"), "exact"), Is.EqualTo("60"), "Three arguments nest into two calls."); Assert.That(Text(Calculate("gcd(2^100, 6^50)"), "exact"), Is.EqualTo("1125899906842624")); Assert.That(Text(Calculate("sum(k^2, k, 1, 100)"), "exact"), Is.EqualTo("338350")); Assert.That(Text(Calculate("sum(k, k, 1, 10^7)"), "exact"), Is.EqualTo("50000005000000"), "AngouriMath sums in closed form, never term by term."); Assert.That(Text(Calculate("sum(k, k, 10, 1)"), "exact"), Is.EqualTo("0"), "An empty sum is 0."); } } [Test] public void RefusesAnIntegralItCannotIntegrate() { using (Assert.EnterMultipleScope()) { Assert.That(Refusal("integral(1/x, x, -1, 1)"), Does.StartWith("The integrand of 'integral(1 / x, x, -1, 1)' has no finite value at x = 0, inside the interval of integration.")); Assert.That(Refusal("integral(sin(x)/x, x, 0, +oo)"), Does.StartWith("The numerical integration of 'integral(sin(x) / x, x, 0, +oo)' did not settle.")); Assert.That(Refusal("integral(integral(x*y, y, 0, 1), x, 0, 1)"), Does.Contain("depends on the variable of an outer integral")); } } [Test] public async Task ExecutesThroughTheComputationService() { var tool = new CalculateTool(null!, new MathComputationService(NullLogger.Instance)); var result = await tool.ExecuteAsync(Arguments("""{"expression": "547 / 57576", "digits": 5}"""), null!); Assert.That(result.JsonContent!["decimal"]!.GetValue(), Is.EqualTo("0.0095005")); } [Test] public void RefusesDigitsOutsideTheRange() { var tool = new CalculateTool(null!, new MathComputationService(NullLogger.Instance)); var tooMany = Arguments("""{"expression": "1/3", "digits": 101}"""); var none = Arguments("""{"expression": "1/3", "digits": 0}"""); using (Assert.EnterMultipleScope()) { Assert.That(() => tool.ExecuteAsync(tooMany, null!), Throws.ArgumentException.With.Message.EqualTo("Argument 'digits' must be at most 100, but was 101. Leave it out to get 20 significant digits.")); Assert.That(() => tool.ExecuteAsync(none, null!), Throws.ArgumentException); } } [Test] public void TellsTheModelNothingAboutTheLibrary() { var definition = new CalculateTool(null!, new MathComputationService(NullLogger.Instance)).GetDefinition(); using (Assert.EnterMultipleScope()) { Assert.That(definition.SystemPromptInstructions, Does.Not.Contain("AngouriMath"), "A model would take the name for something it may call or look up."); Assert.That(definition.Function.DescriptionForLLM, Does.Not.Contain("AngouriMath")); Assert.That(definition.Function.Parameters.GetRawText(), Does.Not.Contain("AngouriMath")); } } }