Add some enhancements to jsonutils.nim (#15133)

* Add some enhancements to `jsonutils.nim`

* Use `jsonutils.nim` hookable API to add a possibility to deserialize
  JSON arrays directly to `HashSet` and `OrderedSet` types and
  respectively to serialize those types to JSON arrays.

* Also add a possibility to deserialize JSON `null` objects to Nim
  option objects and respectively to serialize Nim option object to JSON
  object if some or to JSON `null` object if none.

* Move serialization/deserialization functionality for `Table` and
  `OrderedTable` types from `jsonutils.nim` to `tables.nim` via the
  hookable API.

* Add object `jsonutils.Joptions` and parameter from its type to
  `jsonutils.fromJson` procedure to control whether to allow
  deserializing JSON objects to Nim objects when the JSON has some
  extra or missing keys.

* Add unit tests for the added functionalities to `tjsonutils.nim`.

* improve fromJsonFields

* Add changelog entry for the jsonutils enhancements

* Add TODO in `jsonutils.nim`

* Added an entry to "Future directions" section in `jsonutils.nim` as
  suggestion for future support of serialization and de-serialization of
  nested variant objects.

* Added currently disabled test case in `tjsonutils.nim` for testing
  serialization and de-serialization of nested variant objects.

* Move JSON hooks to `jsonutils.nim`

Move `fromJsonHook` and `toJsonHook` procedures for different types to
`jsonutils.nim` module to avoid a dependency of collections modules to
the `json.nim` module.

The hooks are removed from the following modules:

  * `tables.nim`

  * `sets.nim`

  * `options.nim`

  * `strtabs.nim`

* Add some tests about `StringTableRef`

Add tests for `StringTableRef`'s `fromJsonHook` and `toJsonHook` to
`tjsonutils.nim`.

* Disable a warning in `jsonutils.nim`

Mark `fun` template in `jsonutils` module with `{.used.}` pragma in
order to disable `[XDeclaredButNotUsed]` hint. The template is actually
used by the `initCaseObject` macro in the same module.

Co-authored-by: Timothee Cour <timothee.cour2@gmail.com>

lib/pure/collections/sets.nim

@@ -80,6 +80,8 @@ type
     ## <#initOrderedSet,int>`_ before calling other procs on it.
     data: OrderedKeyValuePairSeq[A]
     counter, first, last: int
+  SomeSet*[A] = HashSet[A] | OrderedSet[A]
+    ## Type union representing `HashSet` or `OrderedSet`.
 
 const
   defaultInitialSize* = 64
@@ -907,8 +909,6 @@ iterator pairs*[A](s: OrderedSet[A]): tuple[a: int, b: A] =
   forAllOrderedPairs:
     yield (idx, s.data[h].key)
 
-
-
 # -----------------------------------------------------------------------
 
 

lib/pure/collections/tables.nim

@@ -1750,10 +1750,6 @@ iterator mvalues*[A, B](t: var OrderedTable[A, B]): var B =
     yield t.data[h].val
     assert(len(t) == L, "the length of the table changed while iterating over it")
 
-
-
-
-
 # ---------------------------------------------------------------------------
 # --------------------------- OrderedTableRef -------------------------------
 # ---------------------------------------------------------------------------

lib/pure/options.nim

@@ -372,7 +372,6 @@ proc unsafeGet*[T](self: Option[T]): lent T {.inline.}=
   assert self.isSome
   result = self.val
 
-
 when isMainModule:
   import unittest, sequtils
 

lib/pure/strtabs.nim

@@ -89,6 +89,7 @@ const
   growthFactor = 2
   startSize = 64
 
+proc mode*(t: StringTableRef): StringTableMode {.inline.} = t.mode
 
 iterator pairs*(t: StringTableRef): tuple[key, value: string] =
   ## Iterates over every `(key, value)` pair in the table `t`.
@@ -422,25 +423,6 @@ proc `%`*(f: string, t: StringTableRef, flags: set[FormatFlag] = {}): string {.
       add(result, f[i])
       inc(i)
 
-since (1,3,5):
-  proc fromJsonHook*[T](a: var StringTableRef, b: T) =
-    ## for json.fromJson
-    mixin jsonTo
-    var mode = jsonTo(b["mode"], StringTableMode)
-    a = newStringTable(mode)
-    let b2 = b["table"]
-    for k,v in b2: a[k] = jsonTo(v, string)
-
-  proc toJsonHook*[](a: StringTableRef): auto =
-    ## for json.toJson
-    mixin newJObject
-    mixin toJson
-    result = newJObject()
-    result["mode"] = toJson($a.mode)
-    let t = newJObject()
-    for k,v in a: t[k] = toJson(v)
-    result["table"] = t
-
 when isMainModule:
   var x = {"k": "v", "11": "22", "565": "67"}.newStringTable
   assert x["k"] == "v"

lib/std/jsonutils.nim

@@ -13,25 +13,33 @@ runnableExamples:
   let j = a.toJson
   doAssert j.jsonTo(type(a)).toJson == j
 
-import std/[json,tables,strutils]
+import std/[json,strutils,tables,sets,strtabs,options]
 
 #[
-xxx
-use toJsonHook,fromJsonHook for Table|OrderedTable
-add Options support also using toJsonHook,fromJsonHook and remove `json=>options` dependency
-
 Future directions:
 add a way to customize serialization, for eg:
-* allowing missing or extra fields in JsonNode
 * field renaming
 * allow serializing `enum` and `char` as `string` instead of `int`
   (enum is more compact/efficient, and robust to enum renamings, but string
   is more human readable)
 * handle cyclic references, using a cache of already visited addresses
+* implement support for serialization and de-serialization of nested variant
+  objects.
 ]#
 
 import std/macros
 
+type
+  Joptions* = object
+    ## Options controlling the behavior of `fromJson`.
+    allowExtraKeys*: bool
+      ## If `true` Nim's object to which the JSON is parsed is not required to
+      ## have a field for every JSON key.
+    allowMissingKeys*: bool
+      ## If `true` Nim's object to which JSON is parsed is allowed to have
+      ## fields without corresponding JSON keys.
+    # in future work: a key rename could be added
+
 proc isNamedTuple(T: typedesc): bool {.magic: "TypeTrait".}
 proc distinctBase(T: typedesc): typedesc {.magic: "TypeTrait".}
 template distinctBase[T](a: T): untyped = distinctBase(type(a))(a)
@@ -58,11 +66,11 @@ macro getDiscriminants(a: typedesc): seq[string] =
     result = quote do:
       seq[string].default
 
-macro initCaseObject(a: typedesc, fun: untyped): untyped =
+macro initCaseObject(T: typedesc, fun: untyped): untyped =
   ## does the minimum to construct a valid case object, only initializing
   ## the discriminant fields; see also `getDiscriminants`
   # maybe candidate for std/typetraits
-  var a = a.getTypeImpl
+  var a = T.getTypeImpl
   doAssert a.kind == nnkBracketExpr
   let sym = a[1]
   let t = sym.getTypeImpl
@@ -92,20 +100,81 @@ proc checkJsonImpl(cond: bool, condStr: string, msg = "") =
 template checkJson(cond: untyped, msg = "") =
   checkJsonImpl(cond, astToStr(cond), msg)
 
-template fromJsonFields(a, b, T, keys) =
-  checkJson b.kind == JObject, $(b.kind) # we could customize whether to allow JNull
-  var num = 0
-  for key, val in fieldPairs(a):
-    num.inc
-    when key notin keys:
-      if b.hasKey key:
-        fromJson(val, b[key])
-      else:
-        # we could customize to allow this
-        checkJson false, $($T, key, b)
-  checkJson b.len == num, $(b.len, num, $T, b) # could customize
+proc hasField[T](obj: T, field: string): bool =
+  for k, _ in fieldPairs(obj):
+    if k == field:
+      return true
+  return false
 
-proc fromJson*[T](a: var T, b: JsonNode) =
+macro accessField(obj: typed, name: static string): untyped = 
+  newDotExpr(obj, ident(name))
+
+template fromJsonFields(newObj, oldObj, json, discKeys, opt) =
+  type T = typeof(newObj)
+  # we could customize whether to allow JNull
+  checkJson json.kind == JObject, $json.kind
+  var num, numMatched = 0
+  for key, val in fieldPairs(newObj):
+    num.inc
+    when key notin discKeys:
+      if json.hasKey key:
+        numMatched.inc
+        fromJson(val, json[key])
+      elif opt.allowMissingKeys:
+        # if there are no discriminant keys the `oldObj` must always have the
+        # same keys as the new one. Otherwise we must check, because they could
+        # be set to different branches.
+        when typeof(oldObj) isnot typeof(nil):
+          if discKeys.len == 0 or hasField(oldObj, key):
+            val = accessField(oldObj, key)
+      else:
+        checkJson false, $($T, key, json)
+    else:
+      if json.hasKey key:
+        numMatched.inc
+
+  let ok =
+    if opt.allowExtraKeys and opt.allowMissingKeys:
+      true
+    elif opt.allowExtraKeys:
+      # This check is redundant because if here missing keys are not allowed,
+      # and if `num != numMatched` it will fail in the loop above but it is left
+      # for clarity.
+      assert num == numMatched
+      num == numMatched
+    elif opt.allowMissingKeys:
+      json.len == numMatched
+    else:
+      json.len == num and num == numMatched
+  
+  checkJson ok, $(json.len, num, numMatched, $T, json)
+
+proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions())
+
+proc discKeyMatch[T](obj: T, json: JsonNode, key: static string): bool =
+  if not json.hasKey key:
+    return true
+  let field = accessField(obj, key)
+  var jsonVal: typeof(field)
+  fromJson(jsonVal, json[key])
+  if jsonVal != field:
+    return false
+  return true
+
+macro discKeysMatchBodyGen(obj: typed, json: JsonNode,
+                           keys: static seq[string]): untyped =
+  result = newStmtList()
+  let r = ident("result")
+  for key in keys:
+    let keyLit = newLit key
+    result.add quote do:
+      `r` = `r` and discKeyMatch(`obj`, `json`, `keyLit`)
+
+proc discKeysMatch[T](obj: T, json: JsonNode, keys: static seq[string]): bool =
+  result = true
+  discKeysMatchBodyGen(obj, json, keys)
+
+proc fromJson*[T](a: var T, b: JsonNode, opt = Joptions()) =
   ## inplace version of `jsonTo`
   #[
   adding "json path" leading to `b` can be added in future work.
@@ -113,10 +182,6 @@ proc fromJson*[T](a: var T, b: JsonNode) =
   checkJson b != nil, $($T, b)
   when compiles(fromJsonHook(a, b)): fromJsonHook(a, b)
   elif T is bool: a = to(b,T)
-  elif T is Table | OrderedTable:
-    a.clear
-    for k,v in b:
-      a[k] = jsonTo(v, typeof(a[k]))
   elif T is enum:
     case b.kind
     of JInt: a = T(b.getBiggestInt())
@@ -148,14 +213,26 @@ proc fromJson*[T](a: var T, b: JsonNode) =
     for i, val in b.getElems:
       fromJson(a[i], val)
   elif T is object:
-    template fun(key, typ): untyped =
-      jsonTo(b[key], typ)
-    a = initCaseObject(T, fun)
+    template fun(key, typ): untyped {.used.} =
+      if b.hasKey key:
+        jsonTo(b[key], typ)
+      elif hasField(a, key):
+        accessField(a, key)
+      else:
+        default(typ)
     const keys = getDiscriminants(T)
-    fromJsonFields(a, b, T, keys)
+    when keys.len == 0:
+      fromJsonFields(a, nil, b, keys, opt)
+    else:
+      if discKeysMatch(a, b, keys):
+        fromJsonFields(a, nil, b, keys, opt)
+      else:
+        var newObj = initCaseObject(T, fun)
+        fromJsonFields(newObj, a, b, keys, opt)
+        a = newObj
   elif T is tuple:
     when isNamedTuple(T):
-      fromJsonFields(a, b, T, seq[string].default)
+      fromJsonFields(a, nil, b, seq[string].default, opt)
     else:
       checkJson b.kind == JArray, $(b.kind) # we could customize whether to allow JNull
       var i = 0
@@ -175,9 +252,6 @@ proc toJson*[T](a: T): JsonNode =
   ## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to
   ## customize serialization, see strtabs.toJsonHook for an example.
   when compiles(toJsonHook(a)): result = toJsonHook(a)
-  elif T is Table | OrderedTable:
-    result = newJObject()
-    for k, v in pairs(a): result[k] = toJson(v)
   elif T is object | tuple:
     when T is object or isNamedTuple(T):
       result = newJObject()
@@ -198,3 +272,145 @@ proc toJson*[T](a: T): JsonNode =
   elif T is bool: result = %(a)
   elif T is Ordinal: result = %(a.ord)
   else: result = %a
+
+proc fromJsonHook*[K, V](t: var (Table[K, V] | OrderedTable[K, V]),
+                         jsonNode: JsonNode) =
+  ## Enables `fromJson` for `Table` and `OrderedTable` types.
+  ## 
+  ## See also:
+  ## * `toJsonHook proc<#toJsonHook,(Table[K,V]|OrderedTable[K,V])>`_
+  runnableExamples:
+    import tables, json
+    var foo: tuple[t: Table[string, int], ot: OrderedTable[string, int]]
+    fromJson(foo, parseJson("""
+      {"t":{"two":2,"one":1},"ot":{"one":1,"three":3}}"""))
+    assert foo.t == [("one", 1), ("two", 2)].toTable
+    assert foo.ot == [("one", 1), ("three", 3)].toOrderedTable
+
+  assert jsonNode.kind == JObject,
+          "The kind of the `jsonNode` must be `JObject`, but its actual " &
+          "type is `" & $jsonNode.kind & "`."
+  clear(t)
+  for k, v in jsonNode:
+    t[k] = jsonTo(v, V)
+
+proc toJsonHook*[K, V](t: (Table[K, V] | OrderedTable[K, V])): JsonNode =
+  ## Enables `toJson` for `Table` and `OrderedTable` types.
+  ##
+  ## See also:
+  ## * `fromJsonHook proc<#fromJsonHook,(Table[K,V]|OrderedTable[K,V]),JsonNode>`_
+  runnableExamples:
+    import tables, json
+    let foo = (
+      t: [("two", 2)].toTable,
+      ot: [("one", 1), ("three", 3)].toOrderedTable)
+    assert $toJson(foo) == """{"t":{"two":2},"ot":{"one":1,"three":3}}"""
+
+  result = newJObject()
+  for k, v in pairs(t):
+    result[k] = toJson(v)
+
+proc fromJsonHook*[A](s: var SomeSet[A], jsonNode: JsonNode) =
+  ## Enables `fromJson` for `HashSet` and `OrderedSet` types.
+  ## 
+  ## See also:
+  ## * `toJsonHook proc<#toJsonHook,SomeSet[A]>`_
+  runnableExamples:
+    import sets, json
+    var foo: tuple[hs: HashSet[string], os: OrderedSet[string]]
+    fromJson(foo, parseJson("""
+      {"hs": ["hash", "set"], "os": ["ordered", "set"]}"""))
+    assert foo.hs == ["hash", "set"].toHashSet
+    assert foo.os == ["ordered", "set"].toOrderedSet
+
+  assert jsonNode.kind == JArray,
+          "The kind of the `jsonNode` must be `JArray`, but its actual " &
+          "type is `" & $jsonNode.kind & "`."
+  clear(s)
+  for v in jsonNode:
+    incl(s, jsonTo(v, A))
+
+proc toJsonHook*[A](s: SomeSet[A]): JsonNode =
+  ## Enables `toJson` for `HashSet` and `OrderedSet` types.
+  ##
+  ## See also:
+  ## * `fromJsonHook proc<#fromJsonHook,SomeSet[A],JsonNode>`_
+  runnableExamples:
+    import sets, json
+    let foo = (hs: ["hash"].toHashSet, os: ["ordered", "set"].toOrderedSet)
+    assert $toJson(foo) == """{"hs":["hash"],"os":["ordered","set"]}"""
+
+  result = newJArray()
+  for k in s:
+    add(result, toJson(k))
+
+proc fromJsonHook*[T](self: var Option[T], jsonNode: JsonNode) =
+  ## Enables `fromJson` for `Option` types.
+  ## 
+  ## See also:
+  ## * `toJsonHook proc<#toJsonHook,Option[T]>`_
+  runnableExamples:
+    import options, json
+    var opt: Option[string]
+    fromJsonHook(opt, parseJson("\"test\""))
+    assert get(opt) == "test"
+    fromJson(opt, parseJson("null"))
+    assert isNone(opt)
+
+  if jsonNode.kind != JNull:
+    self = some(jsonTo(jsonNode, T))
+  else:
+    self = none[T]()
+
+proc toJsonHook*[T](self: Option[T]): JsonNode =
+  ## Enables `toJson` for `Option` types.
+  ##
+  ## See also:
+  ## * `fromJsonHook proc<#fromJsonHook,Option[T],JsonNode>`_
+  runnableExamples:
+    import options, json
+    let optSome = some("test")
+    assert $toJson(optSome) == "\"test\""
+    let optNone = none[string]()
+    assert $toJson(optNone) == "null"
+
+  if isSome(self):
+    toJson(get(self))
+  else:
+    newJNull()
+
+proc fromJsonHook*(a: var StringTableRef, b: JsonNode) =
+  ## Enables `fromJson` for `StringTableRef` type.
+  ## 
+  ## See also:
+  ## * `toJsonHook` proc<#toJsonHook,StringTableRef>`_
+  runnableExamples:
+    import strtabs, json
+    var t = newStringTable(modeCaseSensitive)
+    let jsonStr = """{"mode": 0, "table": {"name": "John", "surname": "Doe"}}"""
+    fromJsonHook(t, parseJson(jsonStr))
+    assert t[] == newStringTable("name", "John", "surname", "Doe",
+                                 modeCaseSensitive)[]
+
+  var mode = jsonTo(b["mode"], StringTableMode)
+  a = newStringTable(mode)
+  let b2 = b["table"]
+  for k,v in b2: a[k] = jsonTo(v, string)
+
+proc toJsonHook*(a: StringTableRef): JsonNode =
+  ## Enables `toJson` for `StringTableRef` type.
+  ## 
+  ## See also:
+  ## * `fromJsonHook` proc<#fromJsonHook,StringTableRef,JsonNode>`_
+  runnableExamples:
+    import strtabs, json
+    let t = newStringTable("name", "John", "surname", "Doe", modeCaseSensitive)
+    let jsonStr = """{"mode": "modeCaseSensitive",
+                      "table": {"name": "John", "surname": "Doe"}}"""
+    assert toJson(t) == parseJson(jsonStr)
+
+  result = newJObject()
+  result["mode"] = toJson($a.mode)
+  let t = newJObject()
+  for k,v in a: t[k] = toJson(v)
+  result["table"] = t