Improve binary serialization, add support for typed BigInt

Author: sebaciv

core/js/src/main/scala/com/avsystem/commons/serialization/nativejs/NativeFormatOptions.scala

@@ -25,15 +25,29 @@ object NativeDateFormat extends AbstractValueEnumCompanion[NativeDateFormat] {
   final val JsDate: Value = new NativeDateFormat
 }
 
+/**
+  * Specifies format used by `NativeJsonOutput.writeBigInt` / `NativeJsonInput.readBigInt`
+  * to represent [[BigInt]].
+  *
+  * Note that [[scala.scalajs.js.JSON.stringify]] does not know how to serialize a BigInt and throws an error
+  */
+final class NativeBitIntFormat(implicit ctx: EnumCtx) extends AbstractValueEnum
+object NativeBitIntFormat extends AbstractValueEnumCompanion[NativeBitIntFormat] {
+  final val RawString: Value = new NativeBitIntFormat
+  final val JsBigInt: Value = new NativeBitIntFormat
+}
+
 /**
   * Adjusts format produced by [[NativeJsonOutput]].
   *
   * @param longFormat format used to [[Long]]
   * @param dateFormat format used to represent timestamps
+  * @param bigIntFormat format used to represent [[BigInt]]
   */
 final case class NativeFormatOptions(
   longFormat: NativeLongFormat = NativeLongFormat.RawString,
   dateFormat: NativeDateFormat = NativeDateFormat.RawString,
+  bigIntFormat: NativeBitIntFormat = NativeBitIntFormat.RawString,
 )
 object NativeFormatOptions {
   final val RawString = NativeFormatOptions()

core/js/src/main/scala/com/avsystem/commons/serialization/nativejs/NativeJsonInput.scala

@@ -32,46 +32,49 @@ class NativeJsonInput(value: js.Any, options: NativeFormatOptions) extends Input
     }
 
   override def readLong(): Long = {
-    def longFromString(s: String): Long =
+    def fromString(s: String): Long =
       try s.toLong
       catch {
         case e: NumberFormatException => throw new ReadFailure(s"Cannot read Long", e)
       }
     (value: Any) match {
-      case s: String => longFromString(s)
+      case s: String => fromString(s)
       case i: Int => i
       case d: Double if d.isWhole => d.toLong
-      case b: js.BigInt => longFromString(b.toString)
-      case b if js.typeOf(b) == "bigint" => longFromString(b.asInstanceOf[js.BigInt].toString)
+      case b: js.BigInt => fromString(b.toString)
+      case b if js.typeOf(b) == "bigint" => fromString(b.asInstanceOf[js.BigInt].toString)
       case o => throw new ReadFailure(s"Cannot read Long, got: ${js.typeOf(o)}")
     }
   }
 
   override def readBigInt(): BigInt = {
-    def fail = throw new ReadFailure(s"BigInt expected.")
+    def fromString(s: String): BigInt =
+      try BigInt(s)
+      catch {
+        case e: NumberFormatException => throw new ReadFailure(s"Cannot read BitInt", e)
+      }
+
     (value: Any) match {
-      case s: String =>
-        try BigInt(s)
-        catch {
-          case _: NumberFormatException => fail
-        }
+      case s: String => fromString(s)
       case i: Int => BigInt(i)
       case d: Double if d.isWhole => BigInt(d.toLong)
-      case _ => fail
+      case b: js.BigInt => fromString(b.toString)
+      case b if js.typeOf(b) == "bigint" => fromString(b.asInstanceOf[js.BigInt].toString)
+      case o => throw new ReadFailure(s"Cannot read BitInt, got: ${js.typeOf(o)}")
     }
   }
 
   override def readBigDecimal(): BigDecimal = {
-    def fail = throw new ReadFailure(s"BigDecimal expected.")
+    def fromString(s: String): BigDecimal =
+      try BigDecimal(s)
+      catch {
+        case e: NumberFormatException => throw new ReadFailure(s"Cannot read BigDecimal", e)
+      }
     (value: Any) match {
-      case s: String =>
-        try BigDecimal(s)
-        catch {
-          case _: NumberFormatException => fail
-        }
+      case s: String => fromString(s)
       case i: Int => BigDecimal(i)
       case d: Double => BigDecimal(d)
-      case _ => fail
+      case o => throw new ReadFailure(s"Cannot read BigDecimal, got: ${js.typeOf(o)}")
     }
   }
 
@@ -102,7 +105,9 @@ class NativeJsonInput(value: js.Any, options: NativeFormatOptions) extends Input
   override def skip(): Unit = ()
 
   override def readBinary(): Array[Byte] =
-    readList().iterator(_.readSimple().readInt().toByte).toArray
+    read("List") {
+      case array: js.Array[Int @unchecked] => array.iterator.map(_.toByte).toArray
+    }
 
   override def readCustom[T](typeMarker: TypeMarker[T]): Opt[T] =
     typeMarker match {
@@ -120,15 +125,14 @@ final class JsonListInput(list: js.Array[js.Any], options: NativeFormatOptions)
     it < list.length
 
   override def nextElement(): Input = {
-    js.BigInt
     val in = new NativeJsonInput(list(it), options)
     it += 1
     in
   }
 }
 
 final class JsonObjectInput(dict: js.Dictionary[js.Any], options: NativeFormatOptions) extends ObjectInput {
-  val it = dict.keysIterator
+  val it = dict.iterator
 
   override def hasNext: Boolean =
     it.hasNext
@@ -137,8 +141,8 @@ final class JsonObjectInput(dict: js.Dictionary[js.Any], options: NativeFormatOp
     if (dict.contains(name)) Opt(new NativeJsonFieldInput(name, dict(name), options)) else Opt.Empty
 
   override def nextField(): FieldInput = {
-    val key = it.next()
-    new NativeJsonFieldInput(key, dict.apply(key), options)
+    val (key, value) = it.next()
+    new NativeJsonFieldInput(key, value, options)
   }
 }
 

core/js/src/main/scala/com/avsystem/commons/serialization/nativejs/NativeJsonOutput.scala

@@ -30,8 +30,10 @@ final class NativeJsonOutput(
     case NativeLongFormat.JsBigInt => writeRaw(js.BigInt(long.toString))
   }
 
-  override def writeBigInt(bigInt: BigInt): Unit =
-    writeString(bigInt.toString)
+  override def writeBigInt(bigInt: BigInt): Unit = options.bigIntFormat match {
+    case NativeBitIntFormat.RawString => writeString(bigInt.toString)
+    case NativeBitIntFormat.JsBigInt => writeRaw(js.BigInt(bigInt.toString))
+  }
 
   override def writeBigDecimal(bigDecimal: BigDecimal): Unit =
     writeString(bigDecimal.toString)
@@ -46,9 +48,8 @@ final class NativeJsonOutput(
     new NativeJsonObjectOutput(valueConsumer, options)
 
   override def writeBinary(binary: Array[Byte]): Unit = {
-    val l = writeList()
-    binary.foreach(b => l.writeElement().writeSimple().writeInt(b))
-    l.finish()
+    import js.JSConverters._
+    valueConsumer(binary.toJSArray)
   }
 
   override def writeTimestamp(millis: Long): Unit = options.dateFormat match {

core/js/src/test/scala/com/avsystem/commons/serialization/nativejs/NativeJsonInputOutputTest.scala

@@ -1,7 +1,7 @@
 package com.avsystem.commons
 package serialization.nativejs
 
-import com.avsystem.commons.misc.Timestamp
+import com.avsystem.commons.misc.{Bytes, Timestamp}
 import com.avsystem.commons.serialization.json.WrappedJson
 import com.avsystem.commons.serialization.{GenCodec, HasGenCodec}
 import org.scalatest.funsuite.AnyFunSuite
@@ -15,6 +15,8 @@ object NativeJsonInputOutputTest {
     time: Timestamp,
     list: Seq[Int],
     map: Map[String, String],
+    binary: Bytes,
+    bigInt: BigInt,
     rawJson: WrappedJson,
   )
   object TestModel extends HasGenCodec[TestModel]
@@ -23,19 +25,37 @@ object NativeJsonInputOutputTest {
 class NativeJsonInputOutputTest extends AnyFunSuite {
   import NativeJsonInputOutputTest._
 
-  test("Bilateral serialization - raw string options") {
-    val options = NativeFormatOptions.RawString
-    bilateralTyped(testModel, options)
-  }
+  case class BilateralTestCase(name: String, options: NativeFormatOptions, testStringRepr: Boolean = true)
 
-  test("Bilateral serialization - number options") {
-    val options = NativeFormatOptions(longFormat = NativeLongFormat.JsNumber, dateFormat = NativeDateFormat.JsNumber)
-    bilateralTyped(testModel, options)
-  }
+  private val testCases = Seq(
+    BilateralTestCase("raw string options", NativeFormatOptions.RawString),
+    BilateralTestCase(
+      "number options",
+      NativeFormatOptions(
+        longFormat = NativeLongFormat.JsNumber,
+        dateFormat = NativeDateFormat.JsNumber),
+    ),
+    BilateralTestCase(
+      "typed options",
+      NativeFormatOptions(
+        longFormat = NativeLongFormat.JsBigInt,
+        NativeDateFormat.JsDate,
+        bigIntFormat = NativeBitIntFormat.JsBigInt,
+      ),
+      testStringRepr = false, // scala.scalajs.js.JavaScriptException: TypeError: Do not know how to serialize a BigInt
+    ),
+  )
+
+  testCases.foreach { case BilateralTestCase(name, options, testStringRepr) =>
+    test(s"Bilateral serialization - $name") {
+      bilateralTyped(testModel, options)
+    }
 
-  test("Bilateral serialization - typed options") {
-    val options = NativeFormatOptions(longFormat = NativeLongFormat.JsBigInt, dateFormat = NativeDateFormat.JsDate)
-    bilateralTyped(testModel, options)
+    if (testStringRepr) {
+      test(s"Bilateral serialization to string - $name") {
+        bilateralString(testModel, options)
+      }
+    }
   }
 
   private def testModel: TestModel = TestModel(
@@ -45,6 +65,8 @@ class NativeJsonInputOutputTest extends AnyFunSuite {
     time = Timestamp.now(),
     list = Seq(1, 2, 3),
     map = Map("Abc" -> "1", "xyz" -> "10000"),
+    binary = new Bytes(Array(1, 2, 0, 5)),
+    bigInt = BigInt("10000000000000000000"),
     rawJson = WrappedJson("""{"a":1,"b":"c"}"""),
   )
 
@@ -53,4 +75,10 @@ class NativeJsonInputOutputTest extends AnyFunSuite {
     val deserialized = NativeJsonInput.read[T](raw, options)
     assert(deserialized == input)
   }
+
+  private def bilateralString[T: GenCodec](input: T, options: NativeFormatOptions): Unit = {
+    val raw = NativeJsonOutput.writeAsString(input, options)
+    val deserialized = NativeJsonInput.readString[T](raw, options)
+    assert(deserialized == input)
+  }
 }

core/src/main/scala/com/avsystem/commons/serialization/json/WrappedJson.scala

@@ -1,6 +1,7 @@
 package com.avsystem.commons
 package serialization.json
 
+import com.avsystem.commons.misc.CaseMethods
 import com.avsystem.commons.serialization.GenCodec
 
 /**
@@ -9,7 +10,7 @@ import com.avsystem.commons.serialization.GenCodec
   * It will be serialized as JSON value when used with [[com.avsystem.commons.serialization.Output]] supporting
   * [[RawJson]] marker.
   */
-final case class WrappedJson(value: String) extends AnyVal
+final case class WrappedJson(value: String) extends AnyVal with CaseMethods
 object WrappedJson {
   implicit val codec: GenCodec[WrappedJson] = GenCodec.create(
     in => WrappedJson(in.readCustom(RawJson).getOrElse(in.readSimple().readString())),

core/src/test/scala/com/avsystem/commons/serialization/json/JsonStringInputOutputTest.scala

@@ -61,7 +61,7 @@ class JsonStringInputOutputTest extends AnyFunSuite with SerializationTestUtils
   }
 
   test("WrappedJson") {
-    val json = "{\"a\": 123, \"b\": 3.14}"
+    val json = """{"a": 123, "b": 3.14}"""
     assert(JsonStringOutput.write(WrappedJson(json)) == json)
     assert(JsonStringInput.read[WrappedJson](json) == WrappedJson(json))
   }