register callback for marshal in VM (#19578)

* register callback for marshal in VM

* remove unrelated code

* polish

* more tests

* more tests

* add loadVM and toVM

compiler/vm.nim

@@ -15,7 +15,7 @@ import
   std/[strutils, tables, parseutils],
   msgs, vmdef, vmgen, nimsets, types, passes,
   parser, vmdeps, idents, trees, renderer, options, transf,
-  vmmarshal, gorgeimpl, lineinfos, btrees, macrocacheimpl,
+  gorgeimpl, lineinfos, btrees, macrocacheimpl,
   modulegraphs, sighashes, int128, vmprofiler
 
 import ast except getstr
@@ -1224,7 +1224,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         c.callbacks[-prc.offset-2].value(
           VmArgs(ra: ra, rb: rb, rc: rc, slots: cast[ptr UncheckedArray[TFullReg]](addr regs[0]),
                  currentException: c.currentExceptionA,
-                 currentLineInfo: c.debug[pc]))
+                 currentLineInfo: c.debug[pc])
+                 )
       elif importcCond(c, prc):
         if compiletimeFFI notin c.config.features:
           globalError(c.config, c.debug[pc], "VM not allowed to do FFI, see `compiletimeFFI`")
@@ -2100,18 +2101,6 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       while typ.kind == tyTypeDesc and typ.len > 0: typ = typ[0]
       createStr regs[ra]
       regs[ra].node.strVal = typ.typeToString(preferExported)
-    of opcMarshalLoad:
-      let ra = instr.regA
-      let rb = instr.regB
-      inc pc
-      let typ = c.types[c.code[pc].regBx - wordExcess]
-      putIntoReg(regs[ra], loadAny(regs[rb].node.strVal, typ, c.cache, c.config, c.idgen))
-    of opcMarshalStore:
-      decodeB(rkNode)
-      inc pc
-      let typ = c.types[c.code[pc].regBx - wordExcess]
-      createStrKeepNode(regs[ra])
-      storeAny(regs[ra].node.strVal, typ, regs[rb].regToNode, c.config)
 
     c.profiler.leave(c)
 

compiler/vmdef.nim

@@ -180,7 +180,6 @@ type
     opcNBindSym, opcNDynBindSym,
     opcSetType,   # dest.typ = types[Bx]
     opcTypeTrait,
-    opcMarshalLoad, opcMarshalStore,
     opcSymOwner,
     opcSymIsInstantiationOf
 
@@ -307,8 +306,8 @@ proc registerCallback*(c: PCtx; name: string; callback: VmCallback): int {.disca
 const
   firstABxInstr* = opcTJmp
   largeInstrs* = { # instructions which use 2 int32s instead of 1:
-    opcSubStr, opcConv, opcCast, opcNewSeq, opcOf,
-    opcMarshalLoad, opcMarshalStore}
+    opcSubStr, opcConv, opcCast, opcNewSeq, opcOf
+    }
   slotSomeTemp* = slotTempUnknown
   relativeJumps* = {opcTJmp, opcFJmp, opcJmp, opcJmpBack}
 

compiler/vmgen.nim

@@ -99,11 +99,6 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) =
       let idx = x.regBx-wordExcess
       result.addf("\t$#\tr$#, $# ($#)", opc.toStr, x.regA,
         c.constants[idx].renderTree, $idx)
-    elif opc in {opcMarshalLoad, opcMarshalStore}:
-      let y = c.code[i+1]
-      result.addf("\t$#\tr$#, r$#, $#", opc.toStr, x.regA, x.regB,
-        c.types[y.regBx-wordExcess].typeToString)
-      inc i
     else:
       result.addf("\t$#\tr$#, $#", opc.toStr, x.regA, x.regBx-wordExcess)
     result.add("\t# ")
@@ -1383,22 +1378,6 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     # mGCref, mGCunref,
     globalError(c.config, n.info, "cannot generate code for: " & $m)
 
-proc genMarshalLoad(c: PCtx, n: PNode, dest: var TDest) =
-  ## Signature: proc to*[T](data: string): T
-  if dest < 0: dest = c.getTemp(n.typ)
-  var tmp = c.genx(n[1])
-  c.gABC(n, opcMarshalLoad, dest, tmp)
-  c.gABx(n, opcMarshalLoad, 0, c.genType(n.typ))
-  c.freeTemp(tmp)
-
-proc genMarshalStore(c: PCtx, n: PNode, dest: var TDest) =
-  ## Signature: proc `$$`*[T](x: T): string
-  if dest < 0: dest = c.getTemp(n.typ)
-  var tmp = c.genx(n[1])
-  c.gABC(n, opcMarshalStore, dest, tmp)
-  c.gABx(n, opcMarshalStore, 0, c.genType(n[1].typ))
-  c.freeTemp(tmp)
-
 proc unneededIndirection(n: PNode): bool =
   n.typ.skipTypes(abstractInstOwned-{tyTypeDesc}).kind == tyRef
 
@@ -2054,12 +2033,6 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
       elif s.kind == skMethod:
         localError(c.config, n.info, "cannot call method " & s.name.s &
           " at compile time")
-      elif matches(s, "stdlib.marshal.to"):
-        # XXX marshal load&store should not be opcodes, but use the
-        # general callback mechanisms.
-        genMarshalLoad(c, n, dest)
-      elif matches(s, "stdlib.marshal.$$"):
-        genMarshalStore(c, n, dest)
       else:
         genCall(c, n, dest)
         clearDest(c, n, dest)

compiler/vmops.nim

@@ -32,7 +32,7 @@ from system/formatfloat import addFloatRoundtrip, addFloatSprintf
 
 
 # There are some useful procs in vmconv.
-import vmconv
+import vmconv, vmmarshal
 
 template mathop(op) {.dirty.} =
   registerCallback(c, "stdlib.math." & astToStr(op), `op Wrapper`)
@@ -152,6 +152,7 @@ when defined(nimHasInvariant):
 proc stackTrace2(c: PCtx, msg: string, n: PNode) =
   stackTrace(c, PStackFrame(prc: c.prc.sym, comesFrom: 0, next: nil), c.exceptionInstr, msg, n.info)
 
+
 proc registerAdditionalOps*(c: PCtx) =
 
   template wrapIterator(fqname: string, iter: untyped) =
@@ -344,3 +345,36 @@ proc registerAdditionalOps*(c: PCtx) =
     addFloatSprintf(p.strVal, x)
 
   wrapIterator("stdlib.os.envPairsImplSeq"): envPairs()
+
+  registerCallback c, "stdlib.marshal.toVM", proc(a: VmArgs) =
+    let typ = a.getNode(0).typ
+    case typ.kind
+    of tyInt..tyInt64, tyUInt..tyUInt64:
+      setResult(a, loadAny(a.getString(1), typ, c.cache, c.config, c.idgen).intVal)
+    of tyFloat..tyFloat128:
+      setResult(a, loadAny(a.getString(1), typ, c.cache, c.config, c.idgen).floatVal)
+    else:
+      setResult(a, loadAny(a.getString(1), typ, c.cache, c.config, c.idgen))
+
+  registerCallback c, "stdlib.marshal.loadVM", proc(a: VmArgs) =
+    let typ = a.getNode(0).typ
+    let p = a.getReg(1)
+    var res: string
+
+    var node: PNode
+    case p.kind
+    of rkNone:
+      node = newNode(nkEmpty)
+    of rkInt:
+      node = newIntNode(nkIntLit, p.intVal)
+    of rkFloat:
+      node = newFloatNode(nkFloatLit, p.floatVal)
+    of rkNode:
+      node = p.node
+    of rkRegisterAddr:
+      node = p.regAddr.node
+    of rkNodeAddr:
+      node = p.nodeAddr[]
+
+    storeAny(res, typ, node, c.config)
+    setResult(a, res)

lib/pure/marshal.nim

@@ -298,6 +298,9 @@ proc store*[T](s: Stream, data: T) =
   shallowCopy(d, data)
   storeAny(s, toAny(d), stored)
 
+proc loadVM[T](typ: typedesc[T], x: T): string =
+  discard "the implementation is in the compiler/vmops"
+
 proc `$$`*[T](x: T): string =
   ## Returns a string representation of `x` (serialization, marshalling).
   ##
@@ -313,12 +316,18 @@ proc `$$`*[T](x: T): string =
     let y = $$x
     assert y == """{"id": 1, "bar": "baz"}"""
 
-  var stored = initIntSet()
-  var d: T
-  shallowCopy(d, x)
-  var s = newStringStream()
-  storeAny(s, toAny(d), stored)
-  result = s.data
+  when nimvm:
+    result = loadVM(T, x)
+  else:
+    var stored = initIntSet()
+    var d: T
+    shallowCopy(d, x)
+    var s = newStringStream()
+    storeAny(s, toAny(d), stored)
+    result = s.data
+
+proc toVM[T](typ: typedesc[T], data: string): T =
+  discard "the implementation is in the compiler/vmops"
 
 proc to*[T](data: string): T =
   ## Reads data and transforms it to a type `T` (deserialization, unmarshalling).
@@ -335,5 +344,8 @@ proc to*[T](data: string): T =
     assert z.id == 1
     assert z.bar == "baz"
 
-  var tab = initTable[BiggestInt, pointer]()
-  loadAny(newStringStream(data), toAny(result), tab)
+  when nimvm:
+    result = toVM(T, data)
+  else:
+    var tab = initTable[BiggestInt, pointer]()
+    loadAny(newStringStream(data), toAny(result), tab)

tests/stdlib/tmarshal.nim

@@ -4,12 +4,16 @@ import std/marshal
 
 proc testit[T](x: T): string = $$to[T]($$x)
 
-let test1: array[0..1, array[0..4, string]] = [
-  ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]]
-doAssert testit(test1) ==
-  """[["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]]"""
-let test2: tuple[name: string, s: int] = ("tuple test", 56)
-doAssert testit(test2) == """{"Field0": "tuple test", "Field1": 56}"""
+template check1 =
+  let test1: array[0..1, array[0..4, string]] = [
+    ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]]
+  doAssert testit(test1) ==
+    """[["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"]]"""
+  let test2: tuple[name: string, s: int] = ("tuple test", 56)
+  doAssert testit(test2) == """{"Field0": "tuple test", "Field1": 56}"""
+
+static: check1()
+check1()
 
 type
   TE = enum
@@ -146,3 +150,29 @@ block:
 
   let a: ref A = new(B)
   doAssert $$a[] == "{}" # not "{f: 0}"
+
+template checkMarshal(data: typed) =
+  let orig = data
+  let m = $$orig
+
+  let old = to[typeof(orig)](m)
+  doAssert data == old
+
+template main() =
+  type
+    Book = object
+      page: int
+      name: string
+
+  let book = Book(page: 12, name: "persona")
+
+  checkMarshal(486)
+  checkMarshal(3.14)
+  checkMarshal("azure sky")
+  checkMarshal(book)
+  checkMarshal([1, 2, 3])
+  checkMarshal(@[1.5, 2.7, 3.9, 4.2])
+  checkMarshal(@["dream", "is", "possible"])
+
+static: main()
+main()