Upgraded to .NET 5

.idea/.idea.ExaArray/.idea/codeStyles/codeStyleConfig.xml

@@ -0,0 +1,5 @@
+<component name="ProjectCodeStyleConfiguration">
+  <state>
+    <option name="PREFERRED_PROJECT_CODE_STYLE" value="Default" />
+  </state>
+</component>

ExaArray/ExaArray.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>netcoreapp3.1</TargetFramework>
+        <TargetFramework>net5.0</TargetFramework>
         <RootNamespace>Exa</RootNamespace>
         <PackageId>ExaArray</PackageId>
         <Authors>Thorsten Sommer</Authors>
@@ -10,9 +10,9 @@
         <PackageProjectUrl>https://github.com/SommerEngineering/ExaArray</PackageProjectUrl>
         <PackageLicenseExpression>BSD-3-Clause</PackageLicenseExpression>
         <RepositoryUrl>https://code.tsommer.org/thorsten/ExaArray</RepositoryUrl>
-        <PackageVersion>1.2.0</PackageVersion>
-        <AssemblyVersion>1.2.0</AssemblyVersion>
-        <FileVersion>1.2.0</FileVersion>
+        <PackageVersion>1.4.1-rc</PackageVersion>
+        <AssemblyVersion>1.4.1</AssemblyVersion>
+        <FileVersion>1.4.1</FileVersion>
     </PropertyGroup>
 
     <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">

ExaArray/ExaArray1D.cs

@@ -1,5 +1,8 @@
 using System;
 using System.Collections.Generic;
+using System.IO;
+using System.Runtime.Serialization;
+using System.Runtime.Serialization.Formatters.Binary;
 
 namespace Exa
 {
@@ -8,7 +11,8 @@ namespace Exa
     /// to 4.6 quintillion (4,607,183,514,018,780,000) or 4.6 exa elements. 
     /// </summary>
     /// <typeparam name="T">The desired type to use, e.g. byte, int, etc.</typeparam>
-    public sealed partial class ExaArray1D<T>
+    [Serializable]
+    public sealed partial class ExaArray1D<T> : ISerializable
     {
         /// <summary>
         /// Unfortunately, this seems to be the maximal number of entries an
@@ -218,5 +222,72 @@ namespace Exa
             for (ulong n = 0; n < this.Length; n++)
                 yield return this[n];
         }
+
+        #region Store and load
+
+        /// <summary>
+        /// Stores the exa array into a stream. <b>Please read the remarks regarding security issues.</b>
+        /// </summary>
+        /// <remarks>
+        /// The data stored in this way should never be part of a public API. Serializing and
+        /// deserializing is not secure: an attacker can manipulate the data in a targeted
+        /// manner to compromise the API server, etc.
+        ///
+        /// This method does not dispose the stream.
+        /// </remarks>
+        public void Store(Stream outputStream)
+        {
+            var formatter = new BinaryFormatter();
+            formatter.Serialize(outputStream, this);
+        }
+
+        /// <summary>
+        /// Restores an exa array from the given stream. <b>Please read the remarks regarding security issues.</b>
+        /// </summary>
+        /// <remarks>
+        /// The data loaded in this way should never be part of a public API. Serializing and
+        /// deserializing is not secure: an attacker can manipulate the data in a targeted
+        /// manner to compromise the API server, etc.
+        /// 
+        /// This method does not dispose the stream.
+        /// </remarks>
+        public static ExaArray1D<T> Restore(Stream inputStream)
+        {
+            var formatter = new BinaryFormatter();
+            return formatter.Deserialize(inputStream) as ExaArray1D<T>;
+        }
+
+        #endregion
+
+        #region Serialization
+
+        /// <summary>
+        /// This method serves for the serialization process. Do not call it manually. 
+        /// </summary>
+        public void GetObjectData(SerializationInfo info, StreamingContext context)
+        {
+            info.AddValue("version", "v1");
+            info.AddValue("strategy", this.OptimizationStrategy, typeof(Strategy));
+            info.AddValue("length", this.Length);
+            info.AddValue("chunks", this.chunks, typeof(T[][]));
+        }
+        
+        private ExaArray1D(SerializationInfo info, StreamingContext context)
+        {
+            switch (info.GetString("version"))
+            {
+                case "v1":
+                    this.Length = info.GetUInt64("length");
+                    this.chunks = info.GetValue("chunks", typeof(T[][])) as T[][];
+                    this.OptimizationStrategy = (Strategy) info.GetValue("strategy", typeof(Strategy));
+                    break;
+            }
+
+            this.chunks[0] ??= new T[0];
+            this.maxElements = this.OptimizationStrategy == Strategy.MAX_PERFORMANCE ? MAX_NUMBER_ELEMENTS_PERFORMANCE : MAX_NUMBER_ELEMENTS;
+            this.maxArrayCapacity = this.OptimizationStrategy == Strategy.MAX_PERFORMANCE ? MAX_CAPACITY_ARRAY_PERFORMANCE : MAX_CAPACITY_ARRAY;
+        }
+
+        #endregion
     }
 }

ExaArray/ExaArray2D.cs

@@ -0,0 +1,151 @@
+using System;
+using System.IO;
+using System.Runtime.Serialization;
+using System.Runtime.Serialization.Formatters.Binary;
+
+namespace Exa
+{
+    /// <summary>
+    /// The two-dimensional exa-scale array. Can grow up to 18,446,744,073,709,551,615 elements in total.
+    /// </summary>
+    [Serializable]
+    public sealed class ExaArray2D<T> : ISerializable
+    {
+        /// <summary>
+        /// The total number of possible elements.
+        /// </summary>
+        public const ulong MAX_NUMBER_ELEMENTS = ulong.MaxValue;
+
+        private ulong sumLengthOrdinates = 0;
+        
+        // Chunk storage:
+        private readonly ExaArray1D<ExaArray1D<T>> chunks = new ExaArray1D<ExaArray1D<T>>(Strategy.MAX_PERFORMANCE);
+
+        /// <summary>
+        /// Constructs a two-dimensional exa-scale array.
+        /// </summary>
+        public ExaArray2D()
+        {
+        }
+        
+        /// <summary>
+        /// Returns the current total number of elements across all dimensions. 
+        /// </summary>
+        /// <remarks>
+        /// Performance: O(1)
+        /// </remarks>
+        public ulong Length => this.sumLengthOrdinates;
+
+        /// <summary>
+        /// Gets or sets an element of the array.
+        /// </summary>
+        /// <remarks>
+        /// Getting a value: When asking for elements which are not yet allocated, returns <c>default(T)</c>. Performance: O(1).
+        /// 
+        /// Setting a value: The underlying data structure gets extended on demand as necessary. The array can and will grow
+        /// on demand per index. For example, the abscissa index 5 might have allocated memory for 15 elements while abscissa
+        /// index 16 have allocated memory for 1,000 elements.
+        ///
+        /// Performance, when the memory is already allocated: O(1)
+        /// Performance to extend on demand: O(n)
+        ///
+        /// On the abscissa, you extend up to 1,152,921,504,606,850,000 entries. Across all dimensions, you can have 18,446,744,073,709,551,615
+        /// elements on total.
+        /// </remarks>
+        /// <exception cref="ArgumentOutOfRangeException">Throws, when you tried to extend more than 1,152,921,504,606,850,000 elements on the abscissa
+        /// or you tried to extend to more than 18,446,744,073,709,551,615 elements in total.</exception>
+        public T this[ulong indexAbscissa, ulong indexOrdinate]
+        {
+            get
+            {
+                if (this.chunks.Length == 0 || indexAbscissa >= this.chunks.Length || indexOrdinate >= this.chunks[indexAbscissa]?.Length)
+                    return default(T);
+
+                return this.chunks[indexAbscissa][indexOrdinate];
+            }
+
+            set
+            {
+                if(this.chunks.Length == 0 || indexAbscissa >= this.chunks.Length)
+                    this.chunks.Extend(indexAbscissa - this.chunks.Length + 1);
+                
+                this.chunks[indexAbscissa] ??= new ExaArray1D<T>(Strategy.MAX_PERFORMANCE);
+                if(this.chunks[indexAbscissa].Length == 0 || indexOrdinate >= this.chunks[indexAbscissa].Length - 1)
+                {
+                    var extendBy = (indexOrdinate - this.chunks[indexAbscissa].Length) + 1;
+                    if(extendBy > MAX_NUMBER_ELEMENTS - this.sumLengthOrdinates)
+                        throw new ArgumentOutOfRangeException($"It is not possible to extend more than {MAX_NUMBER_ELEMENTS} total elements across all dimensions.");
+                    
+                    this.chunks[indexAbscissa].Extend(extendBy);
+                    this.sumLengthOrdinates += extendBy;
+                }
+
+                this.chunks[indexAbscissa][indexOrdinate] = value;
+            }
+        }
+        
+        #region Store and load
+
+        /// <summary>
+        /// Stores the exa array into a stream. <b>Please read the remarks regarding security issues.</b>
+        /// </summary>
+        /// <remarks>
+        /// The data stored in this way should never be part of a public API. Serializing and
+        /// deserializing is not secure: an attacker can manipulate the data in a targeted
+        /// manner to compromise the API server, etc.
+        ///
+        /// This method does not dispose the stream.
+        /// </remarks>
+        public void Store(Stream outputStream)
+        {
+            var formatter = new BinaryFormatter();
+            formatter.Serialize(outputStream, this);
+        }
+
+        /// <summary>
+        /// Restores an exa array from the given stream. <b>Please read the remarks regarding security issues.</b>
+        /// </summary>
+        /// <remarks>
+        /// The data loaded in this way should never be part of a public API. Serializing and
+        /// deserializing is not secure: an attacker can manipulate the data in a targeted
+        /// manner to compromise the API server, etc.
+        /// 
+        /// This method does not dispose the stream.
+        /// </remarks>
+        public static ExaArray2D<T> Restore(Stream inputStream)
+        {
+            var formatter = new BinaryFormatter();
+            return formatter.Deserialize(inputStream) as ExaArray2D<T>;
+        }
+
+        #endregion
+        
+        #region Serialization
+
+        /// <summary>
+        /// This method serves for the serialization process. Do not call it manually. 
+        /// </summary>
+        public void GetObjectData(SerializationInfo info, StreamingContext context)
+        {
+            info.AddValue("version", "v1");
+            info.AddValue("length", this.Length);
+            info.AddValue("chunks", this.chunks, typeof(ExaArray1D<ExaArray1D<T>>));
+        }
+        
+        private ExaArray2D(SerializationInfo info, StreamingContext context)
+        {
+            switch (info.GetString("version"))
+            {
+                case "v1":
+                    this.sumLengthOrdinates = info.GetUInt64("length");
+                    this.chunks = info.GetValue("chunks", typeof(ExaArray1D<ExaArray1D<T>>)) as ExaArray1D<ExaArray1D<T>>;
+                    break;
+
+                default:
+                    break;
+            }
+        }
+
+        #endregion
+    }
+}

ExaArrayTests/ExaArray1DTests.cs

@@ -2,6 +2,7 @@ using System;
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 using Exa;
@@ -9,11 +10,10 @@ using NUnit.Framework;
 
 namespace ExaArrayTests
 {
+    [ExcludeFromCodeCoverage]
     public class ExaArray1DTests
     {
-        [ExcludeFromCodeCoverage]
-        public class InfinityArrayTests
-        {
+        [Serializable]
         private class TestClass
         {
             public int Age { get; set; }
@@ -83,9 +83,9 @@ namespace ExaArrayTests
         {
             const uint MAX = 1_073_741_824;
             var exaA = new ExaArray1D<byte>(Strategy.MAX_PERFORMANCE);
-                exaA.Extend(MAX-2);
+            exaA.Extend(MAX - 2);
 
-                Assert.That(exaA.Length, Is.EqualTo(MAX-2));
+            Assert.That(exaA.Length, Is.EqualTo(MAX - 2));
 
             exaA.Extend(2);
             Assert.That(exaA.Length, Is.EqualTo(MAX));
@@ -272,10 +272,7 @@ namespace ExaArrayTests
 
             var next = ExaArray1D<byte>.CreateFrom(exPerf);
             Assert.That(next.Length, Is.EqualTo(exPerf.Length));
-                Assert.DoesNotThrow(() =>
-                {
-                    next[4_999_999_999] = 0xab;
-                });
+            Assert.DoesNotThrow(() => { next[4_999_999_999] = 0xab; });
 
             exPerf = null;
             next = null;
@@ -285,10 +282,7 @@ namespace ExaArrayTests
 
             next = ExaArray1D<byte>.CreateFrom(exElem);
             Assert.That(next.Length, Is.EqualTo(exElem.Length));
-                Assert.DoesNotThrow(() =>
-                {
-                    next[4_999_999_999] = 0xab;
-                });
+            Assert.DoesNotThrow(() => { next[4_999_999_999] = 0xab; });
         }
 
         [Test]
@@ -315,9 +309,9 @@ namespace ExaArrayTests
         {
             var exPerf = new ExaArray1D<TestClass>();
             exPerf.Extend(3);
-                exPerf[0] = new TestClass{Age = 5};
-                exPerf[1] = new TestClass{Age = 10};
-                exPerf[2] = new TestClass{Age = 45};
+            exPerf[0] = new TestClass {Age = 5};
+            exPerf[1] = new TestClass {Age = 10};
+            exPerf[2] = new TestClass {Age = 45};
 
             var next = ExaArray1D<TestClass>.CreateFrom(exPerf);
             Assert.That(next.Length, Is.EqualTo(3));
@@ -436,9 +430,9 @@ namespace ExaArrayTests
         {
             var exPerf = new ExaArray1D<TestClass>();
             exPerf.Extend(3);
-                exPerf[0] = new TestClass{Age = 5};
-                exPerf[1] = new TestClass{Age = 10};
-                exPerf[2] = new TestClass{Age = 45};
+            exPerf[0] = new TestClass {Age = 5};
+            exPerf[1] = new TestClass {Age = 10};
+            exPerf[2] = new TestClass {Age = 45};
 
             var next = ExaArray1D<TestClass>.CreateFrom(exPerf, 2, 2);
             Assert.That(next.Length, Is.EqualTo(1));
@@ -653,7 +647,7 @@ namespace ExaArrayTests
             var exPerf3 = list.AsEnumerable().AsExaArray(3);
             Assert.That(exPerf3.OptimizationStrategy, Is.EqualTo(Strategy.MAX_PERFORMANCE));
             Assert.That(exPerf3.Length, Is.EqualTo(3));
-                Assert.That(exPerf3.Items(), Is.EquivalentTo(new int[] {5,7,8}));
+            Assert.That(exPerf3.Items(), Is.EquivalentTo(new int[] {5, 7, 8}));
         }
 
         [Test]
@@ -742,6 +736,62 @@ namespace ExaArrayTests
 
             TestContext.WriteLine($"Performing 100M assignments took {t2Times.Average()} ms (average) by means of the max. performance strategy (min={t2Times.Min()} ms, max={t2Times.Max()} ms)");
         }
+        
+        [Test]
+        [Category("cover")]
+        [Category("normal")]
+        public void StoreAndLoad01()
+        {
+            var exaA = new ExaArray1D<byte>();
+            exaA.Extend(5_000_000);
+            exaA[4_483_124] = 0xff;
+
+            var filename = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+            using (var file = File.OpenWrite(filename))
+            {
+                exaA.Store(file);
+            }
+
+            using (var file = File.OpenRead(filename))
+            {
+                var exaB = ExaArray1D<byte>.Restore(file);
+                
+                Assert.That(exaA.Length, Is.EqualTo(exaB.Length));
+                Assert.That(exaA[4_483_124], Is.EqualTo(0xff));
+                Assert.That(exaB[4_483_124], Is.EqualTo(0xff));
+            }
+            
+            File.Delete(filename);
+        }
+        
+        [Test]
+        [Category("cover")]
+        [Category("normal")]
+        public void StoreAndLoad02()
+        {
+            var exaA = new ExaArray1D<TestClass>();
+            exaA.Extend(100);
+            exaA[66] = new TestClass
+            {
+                Age = 55,
+            };
+
+            var filename = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+            using (var file = File.OpenWrite(filename))
+            {
+                exaA.Store(file);
+            }
+
+            using (var file = File.OpenRead(filename))
+            {
+                var exaB = ExaArray1D<TestClass>.Restore(file);
+                
+                Assert.That(exaA.Length, Is.EqualTo(exaB.Length));
+                Assert.That(exaA[66].Age, Is.EqualTo(55));
+                Assert.That(exaB[66].Age, Is.EqualTo(55));
+            }
+            
+            File.Delete(filename);
         }
     }
 }

ExaArrayTests/ExaArray2DTests.cs

@@ -0,0 +1,103 @@
+using System;
+using System.Diagnostics.CodeAnalysis;
+using System.IO;
+using Exa;
+using NUnit.Framework;
+
+namespace ExaArrayTests
+{
+    [ExcludeFromCodeCoverage]
+    public class ExaArray2DTests
+    {
+        [Test]
+        [Category("cover")]
+        [Category("normal")]
+        public void CreatingNormalSize01()
+        {
+            var arr = new ExaArray2D<int>();
+            
+            // Empty array must have length = 0:
+            Assert.That(arr.Length, Is.EqualTo(0));
+            
+            // Try to access some points, which should return default(T):
+            Assert.That(arr[0, 0], Is.EqualTo(default(int)));
+            Assert.That(arr[1_000_000, 1_000_000], Is.EqualTo(default(int)));
+            
+            // Even after accessing some points, there should be no space allocated i.e. length is still 0:
+            Assert.That(arr.Length, Is.EqualTo(0));
+
+            // Write an int to a position:
+            arr[500, 500] = 4_756;
+            
+            // Now, we have 500 empty "rows" + 1 "row" with 501 (0-500) elements:
+            Assert.That(arr.Length, Is.EqualTo(501));
+            
+            // Should be possible to read out the value:
+            Assert.That(arr[500, 500], Is.EqualTo(4_756));
+            
+            // Change the value:
+            arr[500, 500] = 100;
+            Assert.That(arr[500, 500], Is.EqualTo(100));
+            
+            // Still the same size:
+            Assert.That(arr.Length, Is.EqualTo(501));
+            
+            // Add another value in the same "row":
+            arr[500, 499] = 499;
+            Assert.That(arr[500, 499], Is.EqualTo(499));
+            Assert.That(arr[500, 500], Is.EqualTo(100));
+            
+            // Now, we should have still 501 elements, because
+            // we added the new value "below" the previously:
+            Assert.That(arr.Length, Is.EqualTo(501));
+            
+            // Add another value in the same "row", but higher:
+            arr[500, 1_000] = 6;
+            Assert.That(arr[500, 499], Is.EqualTo(499));
+            Assert.That(arr[500, 500], Is.EqualTo(100));
+            Assert.That(arr[500, 1_000], Is.EqualTo(6));
+            
+            // Now we should have more elements:
+            Assert.That(arr.Length, Is.EqualTo(1_001));
+        }
+
+        [Test]
+        [Category("cover")]
+        [Category("normal")]
+        public void CreatingHugeSize01()
+        {
+            var arr = new ExaArray2D<int>();
+            arr[0, 1_000_000] = 47;
+            Assert.Throws<ArgumentOutOfRangeException>(() =>
+            {
+                arr[1, UInt64.MaxValue - 1] = 6;
+            });
+        }
+        
+        [Test]
+        [Category("cover")]
+        [Category("normal")]
+        public void StoreAndLoad01()
+        {
+            var exaA = new ExaArray2D<byte>();
+            exaA[5_000_124, 5_000_666] = 0xff;
+
+            var filename = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+            using (var file = File.OpenWrite(filename))
+            {
+                exaA.Store(file);
+            }
+
+            using (var file = File.OpenRead(filename))
+            {
+                var exaB = ExaArray2D<byte>.Restore(file);
+                
+                Assert.That(exaA.Length, Is.EqualTo(exaB.Length));
+                Assert.That(exaA[5_000_124, 5_000_666], Is.EqualTo(0xff));
+                Assert.That(exaB[5_000_124, 5_000_666], Is.EqualTo(0xff));
+            }
+            
+            File.Delete(filename);
+        }
+    }
+}

ExaArrayTests/ExaArrayTests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFramework>netcoreapp3.1</TargetFramework>
+        <TargetFramework>net5.0</TargetFramework>
 
         <IsPackable>false</IsPackable>
     </PropertyGroup>

README.md

@@ -1,7 +1,6 @@
 # ExaArray
-ExaArray is a .NET library for exa-scale array-like structures. By using this library, it becomes possible to
-add up to 4.6 quintillion i.e. 4,607,183,514,018,780,000 elements into one array. When using `byte` for `T`,
-this would need approx. 4 EB of memory.
+ExaArray is a .NET library for exa-scale array-like structures. By using this library, it becomes possible to add up to 4.6 quintillion i.e. 4,607,183,514,018,780,000 elements into a one-dimensional array. When using `byte` for `T`, this would need approx. 4 EB of memory. The two-dimensional array can grow up to 18.4 quintillion i.e. 18,446,744,073,709,551,615 elements, though.
 
-Extending the data structure performs as O(n) with O(m+n) of memory. Accessing the data performs as O(1), though.
-For the generic type `T`, any .NET type can be used: The ExaArray uses managed memory.
+Extending the data structure performs as O(n) with O(m+n) of memory. Accessing the data performs as O(1), though. For the generic type `T`, any .NET type can be used: The ExaArray uses managed memory.
+
+Storing to and loading from an arbitrary stream is supported. The data stored in this way should never be part of a public API. Serializing and deserializing is not secure: an attacker can manipulate the data in a targeted manner to compromise the API server, etc.