Merge pull request #48 from Cysharp/circular-reference

Add support Circular Reference

Author: neuecc

README.md

@@ -21,6 +21,7 @@ Other than performance, MemoryPack has these features.
 * Deserialize into existing instance
 * Polymorphism(Union) serialization
 * limited version-tolerant(fast/default) and full version-tolerant support
+* Circular reference serialization
 * PipeWriter/Reader based streaming serialization
 * TypeScript code generation and ASP.NET Core Formatter
 * Unity(2021.3) IL2CPP Support via .NET Source Generator
@@ -528,6 +529,58 @@ By checking the differences in this file, dangerous schema changes can be preven
 * member order change
 * member deletion
 
+Circular Reference
+---
+MemoryPack also supports circular reference. This allows the tree objects to be serialized as is.
+
+```csharp
+// to enable circular-reference, use GenerateType.CircularReference
+[MemoryPackable(GenerateType.CircularReference)]
+public partial class Node
+{
+    [MemoryPackOrder(0)]
+    public Node? Parent { get; set; }
+    [MemoryPackOrder(1)]
+    public Node[]? Children { get; set; }
+}
+```
+
+ For example, [System.Text.Json preserve-references](https://learn.microsoft.com/en-us/dotnet/standard/serialization/system-text-json/preserve-references) code will become like here.
+
+```csharp
+// https://learn.microsoft.com/en-us/dotnet/standard/serialization/system-text-json/preserve-references?pivots=dotnet-7-0
+Employee tyler = new()
+{
+    Name = "Tyler Stein"
+};
+
+Employee adrian = new()
+{
+    Name = "Adrian King"
+};
+
+tyler.DirectReports = new List<Employee> { adrian };
+adrian.Manager = tyler;
+
+var bin = MemoryPackSerializer.Serialize(tyler);
+Employee? tylerDeserialized = MemoryPackSerializer.Deserialize<Employee>(bin);
+
+Console.WriteLine(tylerDeserialized?.DirectReports?[0].Manager == tylerDeserialized); // true
+
+[MemoryPackable(GenerateType.CircularReference)]
+public partial class Employee
+{
+    [MemoryPackOrder(0)]
+    public string? Name { get; set; }
+    [MemoryPackOrder(1)]
+    public Employee? Manager { get; set; }
+    [MemoryPackOrder(2)]
+    public List<Employee>? DirectReports { get; set; }
+}
+```
+
+`GenerateType.CircularReference` has the same characteristics as version-tolerant. However, as an additional constraint, only parameterless constructors are allowed. Also, object reference tracking is only done for objects marked with `GenerateType.CircularReference`. If you want to track any other object, wrap it.
+
 CustomFormatter
 ---
 If implements `MemoryPackCustomFormatterAttribute<T>`, you can configure to use custom formatter to MemoryPackObject's member.
@@ -871,15 +924,16 @@ Unity version is not supported CustomFormatter and ImmutableCollections.
 
 Binary wire format specification
 ---
-The type of `T` defined in `Serialize<T>` and `Deserialize<T>` is called C# schema. MemoryPack format is not self described format. Deserialize requires the corresponding C# schema. Seven types exist as internal representations of binaries, but types cannot be determined without a C# schema.
+The type of `T` defined in `Serialize<T>` and `Deserialize<T>` is called C# schema. MemoryPack format is not self described format. Deserialize requires the corresponding C# schema. These types exist as internal representations of binaries, but types cannot be determined without a C# schema.
 
 Endian must be `Little Endian`. However reference C# implementation does not care endianness so can not use on big-endian machine. However modern computers are usually little-endian.
 
-There are seven types of format.
+There are eight types of format.
 
 * Unmanaged struct
 * Object
 * Version Tolerant Object
+* Circular Reference Object
 * Tuple
 * Collection
 * String
@@ -901,6 +955,13 @@ Object has 1byte unsigned byte as member count in header. Member count allows `0
 
 Version Tolerant Object is similar as Object but has byte length of values in header. varint follows these spec, first sbyte is value or typeCode and next X byte is value. 0 to 127 = unsigned byte value, -1 to -120 = signed byte value, -121 = byte, -122 = sbyte, -123 = ushort, -124 = short, -125 = uint, -126 = int, -127 = ulong, -128 = long.
 
+### Circular Reference Object
+
+`(byte memberCount, [varint byte-length-of-values...], varint referenceId, [values...])`
+`(250, varint referenceId)`
+
+Circular Reference Object is similar as Version Tolerant Object but if memberCount is 250, next varint(unsigned-int32) is referenceId. If not, after byte-length-of-values, varint referenceId is written.
+
 ### Tuple
 
 `(values...)`

sandbox/Benchmark/Benchmark.csproj

@@ -6,7 +6,7 @@
         <ImplicitUsings>enable</ImplicitUsings>
         <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
         <Nullable>enable</Nullable>
-
+        <TieredPGO>true</TieredPGO>
         <IsPackable>false</IsPackable>
     </PropertyGroup>
 

sandbox/Benchmark/Micro/GetLocalVsStaticField.cs

@@ -24,7 +24,8 @@ public GetLocalVsStaticField()
     [Benchmark(Baseline = true)]
     public void GetFromProvider()
     {
-        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, MemoryPackSerializeOptions.Default);
+        using var state = MemoryPackWriterOptionalStatePool.Rent(MemoryPackSerializeOptions.Default);
+        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, state);
         for (int i = 0; i < 100; i++)
         {
             writer.GetFormatter<int>().Serialize(ref writer, ref i);
@@ -35,7 +36,8 @@ public void GetFromProvider()
     [Benchmark]
     public void GetFromLocal()
     {
-        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, MemoryPackSerializeOptions.Default);
+        using var state = MemoryPackWriterOptionalStatePool.Rent(MemoryPackSerializeOptions.Default);
+        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, state);
         var provider = writer.GetFormatter<int>();
         for (int i = 0; i < 100; i++)
         {

sandbox/Benchmark/Micro/RawSerialize.cs

@@ -71,7 +71,8 @@ public byte[] HandMemoryPackWriterEmpty()
             bufWriter = staticWriter = new ReusableLinkedArrayBufferWriter(true, true);
         }
 
-        var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufWriter, bufWriter.DangerousGetFirstBuffer(), MemoryPackSerializeOptions.Default);
+        var state = MemoryPackWriterOptionalStatePool.Rent(null);
+        var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufWriter, bufWriter.DangerousGetFirstBuffer(), state);
         try
         {
             if (value == null)
@@ -94,6 +95,7 @@ public byte[] HandMemoryPackWriterEmpty()
         finally
         {
             bufWriter.Reset();
+            state.Reset();
         }
     }
 
@@ -106,7 +108,8 @@ public byte[] HandMemoryPackWriterHeaderOnly()
             bufWriter = staticWriter = new ReusableLinkedArrayBufferWriter(true, true);
         }
 
-        var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufWriter, bufWriter.DangerousGetFirstBuffer(), MemoryPackSerializeOptions.Default);
+        var state = MemoryPackWriterOptionalStatePool.Rent(null);
+        var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufWriter, bufWriter.DangerousGetFirstBuffer(), state);
         try
         {
             if (value == null)
@@ -128,6 +131,7 @@ public byte[] HandMemoryPackWriterHeaderOnly()
         finally
         {
             bufWriter.Reset();
+            state.Reset();
         }
     }
 
@@ -140,7 +144,8 @@ public byte[] HandMemoryPackWriterHeaderInt3()
             bufWriter = staticWriter = new ReusableLinkedArrayBufferWriter(true, true);
         }
 
-        var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufWriter, bufWriter.DangerousGetFirstBuffer(), MemoryPackSerializeOptions.Default);
+        var state = MemoryPackWriterOptionalStatePool.Rent(null);
+        var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufWriter, bufWriter.DangerousGetFirstBuffer(), state);
         try
         {
             if (value == null)
@@ -162,6 +167,7 @@ public byte[] HandMemoryPackWriterHeaderInt3()
         finally
         {
             bufWriter.Reset();
+            state.Reset();
         }
     }
 
@@ -174,7 +180,8 @@ public byte[] HandMemoryPackWriterHeaderInt3String1()
             bufWriter = staticWriter = new ReusableLinkedArrayBufferWriter(true, true);
         }
 
-        var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufWriter, bufWriter.DangerousGetFirstBuffer(), MemoryPackSerializeOptions.Default);
+        var state = MemoryPackWriterOptionalStatePool.Rent(null);
+        var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufWriter, bufWriter.DangerousGetFirstBuffer(), state);
         try
         {
             if (value == null)
@@ -196,6 +203,7 @@ public byte[] HandMemoryPackWriterHeaderInt3String1()
         finally
         {
             bufWriter.Reset();
+            state.Reset();
         }
     }
 
@@ -208,7 +216,8 @@ public byte[] HandMemoryPackFull()
             bufWriter = staticWriter = new ReusableLinkedArrayBufferWriter(true, true);
         }
 
-        var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufWriter, bufWriter.DangerousGetFirstBuffer(), MemoryPackSerializeOptions.Default);
+        var state = MemoryPackWriterOptionalStatePool.Rent(null);
+        var writer = new MemoryPackWriter<ReusableLinkedArrayBufferWriter>(ref bufWriter, bufWriter.DangerousGetFirstBuffer(), state);
         try
         {
             if (value == null)
@@ -230,6 +239,7 @@ public byte[] HandMemoryPackFull()
         finally
         {
             bufWriter.Reset();
+            state.Reset();
         }
     }
 

sandbox/Benchmark/Micro/StaticAbstractVsFormatter.cs

@@ -14,15 +14,17 @@ public StaticAbstractVsFormatter()
         this.value = new IntClass { Value = 999999 };
         this.bufferWriter = new ArrayBufferWriter<byte>(99999);
 
-        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, MemoryPackSerializeOptions.Default);
+        using var state = MemoryPackWriterOptionalStatePool.Rent(null);
+        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, state);
         this.formatter = writer.GetFormatter<IntClass>();
     }
 
     [Benchmark]
     public void WriteValue()
     {
         bufferWriter.Clear();
-        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, MemoryPackSerializeOptions.Default);
+        using var state = MemoryPackWriterOptionalStatePool.Rent(null);
+        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, state);
 
         writer.WriteValue(value); // GetFormatter<T>.Serialize(ref writer, ref value);
     }
@@ -31,23 +33,26 @@ public void WriteValue()
     public void FormatterSerialize()
     {
         bufferWriter.Clear();
-        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, MemoryPackSerializeOptions.Default);
+        using var state = MemoryPackWriterOptionalStatePool.Rent(null);
+        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, state);
         formatter.Serialize(ref writer, ref value!); // IMemoryPackFormatter<T>.Serialize(ref writer, rf value)
     }
 
     [Benchmark(Baseline = true)]
     public void WritePackable()
     {
         bufferWriter.Clear();
-        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, MemoryPackSerializeOptions.Default);
+        using var state = MemoryPackWriterOptionalStatePool.Rent(null);
+        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, state);
         writer.WritePackable(value); // T.Serialize(ref writer, ref value);
     }
 
     [Benchmark]
     public void Direct()
     {
         bufferWriter.Clear();
-        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, MemoryPackSerializeOptions.Default);
+        using var state = MemoryPackWriterOptionalStatePool.Rent(null);
+        var writer = new MemoryPackWriter<ArrayBufferWriter<byte>>(ref bufferWriter, state);
         writer.WriteUnmanagedWithObjectHeader(1, value.Value);
     }