marshalling can be done at compile-time

2026-01-09 14:32:53 +00:00 · 2015-04-25 20:23:09 +02:00
parent 6725aa3634
commit d3fc6e1f28
8 changed files with 404 additions and 51 deletions
--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -16,7 +16,8 @@ import ast except getstr

 import
  strutils, astalgo, msgs, vmdef, vmgen, nimsets, types, passes, unsigned,
-  parser, vmdeps, idents, trees, renderer, options, transf, parseutils
+  parser, vmdeps, idents, trees, renderer, options, transf, parseutils,
+  vmmarshal

 from semfold import leValueConv, ordinalValToString
 from evaltempl import evalTemplate
@@ -371,11 +372,6 @@ template handleJmpBack() {.dirty.} =
      globalError(c.debug[pc], errTooManyIterations)
  dec(c.loopIterations)

-proc skipColon(n: PNode): PNode =
-  result = n
-  if n.kind == nkExprColonExpr:
-    result = n.sons[1]
-
 proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
  var pc = start
  var tos = tos
@@ -1369,6 +1365,19 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
      while typ.kind == tyTypeDesc and typ.len > 0: typ = typ.sons[0]
      createStr regs[ra]
      regs[ra].node.strVal = typ.typeToString(preferExported)
+    of opcMarshalLoad:
+      let ra = instr.regA
+      let rb = instr.regB
+      inc pc
+      let typ = c.types[c.code[pc].regBx - wordExcess]
+      putIntoReg(regs[ra], loadAny(regs[rb].node.strVal, typ))
+    of opcMarshalStore:
+      decodeB(rkNode)
+      inc pc
+      let typ = c.types[c.code[pc].regBx - wordExcess]
+      createStrKeepNode(regs[ra])
+      if regs[ra].node.strVal.isNil: regs[ra].node.strVal = newStringOfCap(1000)
+      storeAny(regs[ra].node.strVal, typ, regs[rb].regToNode)
    inc pc

 proc execute(c: PCtx, start: int): PNode =
--- a/compiler/vmdef.nim
+++ b/compiler/vmdef.nim
@@ -66,7 +66,8 @@ type
    opcMulSet, opcPlusSet, opcMinusSet, opcSymdiffSet, opcConcatStr,
    opcContainsSet, opcRepr, opcSetLenStr, opcSetLenSeq,
    opcSwap, opcIsNil, opcOf, opcIs,
-    opcSubStr, opcParseFloat, opcConv, opcCast, opcQuit, opcReset,
+    opcSubStr, opcParseFloat, opcConv, opcCast,
+    opcQuit, opcReset,
    opcNarrowS, opcNarrowU,

    opcAddStrCh,
@@ -132,7 +133,8 @@ type
    opcLdImmInt,  # dest = immediate value
    opcNBindSym,
    opcSetType,   # dest.typ = types[Bx]
-    opcTypeTrait
+    opcTypeTrait,
+    opcMarshalLoad, opcMarshalStore

  TBlock* = object
    label*: PSym
@@ -221,7 +223,8 @@ proc registerCallback*(c: PCtx; name: string; callback: VmCallback) =
 const
  firstABxInstr* = opcTJmp
  largeInstrs* = { # instructions which use 2 int32s instead of 1:
-    opcSubStr, opcConv, opcCast, opcNewSeq, opcOf}
+    opcSubStr, opcConv, opcCast, opcNewSeq, opcOf,
+    opcMarshalLoad, opcMarshalStore}
  slotSomeTemp* = slotTempUnknown
  relativeJumps* = {opcTJmp, opcFJmp, opcJmp, opcJmpBack}

--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -76,6 +76,11 @@ proc codeListing(c: PCtx, result: var string, start=0; last = -1) =
    elif opc in {opcLdConst, opcAsgnConst}:
      result.addf("\t$#\tr$#, $#", ($opc).substr(3), x.regA,
        c.constants[x.regBx-wordExcess].renderTree)
+    elif opc in {opcMarshalLoad, opcMarshalStore}:
+      let y = c.code[i+1]
+      result.addf("\t$#\tr$#, r$#, $#", ($opc).substr(3), x.regA, x.regB,
+        c.types[y.regBx-wordExcess].typeToString)
+      inc i
    else:
      result.addf("\t$#\tr$#, $#", ($opc).substr(3), x.regA, x.regBx-wordExcess)
    result.add("\t#")
@@ -696,8 +701,7 @@ proc genCard(c: PCtx; n: PNode; dest: var TDest) =
  c.gABC(n, opcCard, dest, tmp)
  c.freeTemp(tmp)

-proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
-  let m = n.sons[0].sym.magic
+proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
  case m
  of mAnd: c.genAndOr(n, opcFJmp, dest)
  of mOr:  c.genAndOr(n, opcTJmp, dest)
@@ -1028,6 +1032,22 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest) =
    # mGCref, mGCunref,
    internalError(n.info, "cannot generate code for: " & $m)

+proc genMarshalLoad(c: PCtx, n: PNode, dest: var TDest) =
+  ## Signature: proc to*[T](data: string): T
+  if dest < 0: dest = c.getTemp(n.typ)
+  var tmp = c.genx(n.sons[1])
+  c.gABC(n, opcMarshalLoad, dest, tmp)
+  c.gABx(n, opcMarshalLoad, 0, c.genType(n.typ))
+  c.freeTemp(tmp)
+
+proc genMarshalStore(c: PCtx, n: PNode, dest: var TDest) =
+  ## Signature: proc `$$`*[T](x: T): string
+  if dest < 0: dest = c.getTemp(n.typ)
+  var tmp = c.genx(n.sons[1])
+  c.gABC(n, opcMarshalStore, dest, tmp)
+  c.gABx(n, opcMarshalStore, 0, c.genType(n.sons[1].typ))
+  c.freeTemp(tmp)
+
 const
  atomicTypes = {tyBool, tyChar,
    tyExpr, tyStmt, tyTypeDesc, tyStatic,
@@ -1533,6 +1553,15 @@ proc matches(s: PSym; x: string): bool =
    dec L
  result = true

+proc matches(s: PSym; y: varargs[string]): bool =
+  var s = s
+  var L = y.len-1
+  while L >= 0:
+    if s == nil or y[L].cmpIgnoreStyle(s.name.s) != 0: return false
+    s = if sfFromGeneric in s.flags: s.owner.owner else: s.owner
+    dec L
+  result = true
+
 proc procIsCallback(c: PCtx; s: PSym): bool =
  if s.offset < -1: return true
  var i = -2
@@ -1570,8 +1599,17 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
    else:
      internalError(n.info, "cannot generate code for: " & s.name.s)
  of nkCallKinds:
-    if n.sons[0].kind == nkSym and n.sons[0].sym.magic != mNone:
-      genMagic(c, n, dest)
+    if n.sons[0].kind == nkSym:
+      let s = n.sons[0].sym
+      if s.magic != mNone:
+        genMagic(c, n, dest, s.magic)
+      elif matches(s, "stdlib", "marshal", "to"):
+        genMarshalLoad(c, n, dest)
+      elif matches(s, "stdlib", "marshal", "$$"):
+        genMarshalStore(c, n, dest)
+      else:
+        genCall(c, n, dest)
+        clearDest(c, n, dest)
    else:
      genCall(c, n, dest)
      clearDest(c, n, dest)
--- a/compiler/vmmarshal.nim
+++ b/compiler/vmmarshal.nim
@@ -0,0 +1,283 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Implements marshaling for the VM.
+
+import streams, json, intsets, tables, ast, astalgo, idents, types, msgs
+
+proc ptrToInt(x: PNode): int {.inline.} =
+  result = cast[int](x) # don't skip alignment
+
+proc getField(n: PNode; position: int): PSym =
+  case n.kind
+  of nkRecList:
+    for i in countup(0, sonsLen(n) - 1):
+      result = getField(n.sons[i], position)
+      if result != nil: return
+  of nkRecCase:
+    result = getField(n.sons[0], position)
+    if result != nil: return
+    for i in countup(1, sonsLen(n) - 1):
+      case n.sons[i].kind
+      of nkOfBranch, nkElse:
+        result = getField(lastSon(n.sons[i]), position)
+        if result != nil: return
+      else: internalError(n.info, "getField(record case branch)")
+  of nkSym:
+    if n.sym.position == position: result = n.sym
+  else: discard
+
+proc storeAny(s: var string; t: PType; a: PNode; stored: var IntSet)
+
+proc storeObj(s: var string; typ: PType; x: PNode; stored: var IntSet) =
+  internalAssert x.kind in {nkObjConstr, nkPar}
+  let start = ord(x.kind == nkObjConstr)
+  for i in countup(start, sonsLen(x) - 1):
+    if i > start: s.add(", ")
+    var it = x.sons[i]
+    if it.kind == nkExprColonExpr:
+      internalAssert it.sons[0].kind == nkSym
+      let field = it.sons[0].sym
+      s.add(escapeJson(field.name.s))
+      s.add(": ")
+      storeAny(s, field.typ, it.sons[1], stored)
+    elif typ.n != nil:
+      let field = getField(typ.n, i)
+      s.add(escapeJson(field.name.s))
+      s.add(": ")
+      storeAny(s, field.typ, it, stored)
+
+proc skipColon*(n: PNode): PNode =
+  result = n
+  if n.kind == nkExprColonExpr:
+    result = n.sons[1]
+
+proc storeAny(s: var string; t: PType; a: PNode; stored: var IntSet) =
+  case t.kind
+  of tyNone: assert false
+  of tyBool: s.add($(a.intVal != 0))
+  of tyChar:
+    let ch = char(a.intVal)
+    if ch < '\128':
+      s.add(escapeJson($ch))
+    else:
+      s.add($int(ch))
+  of tyArray, tySequence:
+    if t.kind == tySequence and a.kind == nkNilLit: s.add("null")
+    else:
+      s.add("[")
+      for i in 0 .. a.len-1:
+        if i > 0: s.add(", ")
+        storeAny(s, t.elemType, a[i], stored)
+      s.add("]")
+  of tyTuple:
+    s.add("{")
+    for i in 0.. <t.len:
+      if i > 0: s.add(", ")
+      s.add("\"Field" & $i)
+      s.add("\": ")
+      storeAny(s, t.sons[i], a[i].skipColon, stored)
+    s.add("}")
+  of tyObject:
+    s.add("{")
+    storeObj(s, t, a, stored)
+    s.add("}")
+  of tySet:
+    s.add("[")
+    for i in 0.. <a.len:
+      if i > 0: s.add(", ")
+      if a[i].kind == nkRange:
+        var x = copyNode(a[i][0])
+        storeAny(s, t.lastSon, x, stored)
+        while x.intVal+1 <= a[i][1].intVal:
+          s.add(", ")
+          storeAny(s, t.lastSon, x, stored)
+          inc x.intVal
+      else:
+        storeAny(s, t.lastSon, a[i], stored)
+    s.add("]")
+  of tyRange, tyGenericInst: storeAny(s, t.lastSon, a, stored)
+  of tyEnum:
+    # we need a slow linear search because of enums with holes:
+    for e in items(t.n):
+      if e.sym.position == a.intVal:
+        s.add e.sym.name.s.escapeJson
+        break
+  of tyPtr, tyRef:
+    var x = a
+    if isNil(x) or x.kind == nkNilLit: s.add("null")
+    elif stored.containsOrIncl(x.ptrToInt):
+      # already stored, so we simply write out the pointer as an int:
+      s.add($x.ptrToInt)
+    else:
+      # else as a [value, key] pair:
+      # (reversed order for convenient x[0] access!)
+      s.add("[")
+      s.add($x.ptrToInt)
+      s.add(", ")
+      storeAny(s, t.lastSon, a, stored)
+      s.add("]")
+  of tyString, tyCString:
+    if a.kind == nkNilLit or a.strVal.isNil: s.add("null")
+    else: s.add(escapeJson(a.strVal))
+  of tyInt..tyInt64, tyUInt..tyUInt64: s.add($a.intVal)
+  of tyFloat..tyFloat128: s.add($a.floatVal)
+  else:
+    internalError a.info, "cannot marshal at compile-time " & t.typeToString
+
+proc storeAny*(s: var string; t: PType; a: PNode) =
+  var stored = initIntSet()
+  storeAny(s, t, a, stored)
+
+proc loadAny(p: var JsonParser, t: PType,
+             tab: var Table[BiggestInt, PNode]): PNode =
+  case t.kind
+  of tyNone: assert false
+  of tyBool:
+    case p.kind
+    of jsonFalse: result = newIntNode(nkIntLit, 0)
+    of jsonTrue: result = newIntNode(nkIntLit, 1)
+    else: raiseParseErr(p, "'true' or 'false' expected for a bool")
+    next(p)
+  of tyChar:
+    if p.kind == jsonString:
+      var x = p.str
+      if x.len == 1:
+        result = newIntNode(nkIntLit, ord(x[0]))
+        next(p)
+        return
+    elif p.kind == jsonInt:
+      result = newIntNode(nkIntLit, getInt(p))
+      next(p)
+      return
+    raiseParseErr(p, "string of length 1 expected for a char")
+  of tyEnum:
+    if p.kind == jsonString:
+      for e in items(t.n):
+        if e.sym.name.s == p.str:
+          result = newIntNode(nkIntLit, e.sym.position)
+          next(p)
+          return
+    raiseParseErr(p, "string expected for an enum")
+  of tyArray:
+    if p.kind != jsonArrayStart: raiseParseErr(p, "'[' expected for an array")
+    next(p)
+    result = newNode(nkBracket)
+    while p.kind != jsonArrayEnd and p.kind != jsonEof:
+      result.add loadAny(p, t.elemType, tab)
+    if p.kind == jsonArrayEnd: next(p)
+    else: raiseParseErr(p, "']' end of array expected")
+  of tySequence:
+    case p.kind
+    of jsonNull:
+      result = newNode(nkNilLit)
+      next(p)
+    of jsonArrayStart:
+      next(p)
+      result = newNode(nkBracket)
+      while p.kind != jsonArrayEnd and p.kind != jsonEof:
+        result.add loadAny(p, t.elemType, tab)
+      if p.kind == jsonArrayEnd: next(p)
+      else: raiseParseErr(p, "")
+    else:
+      raiseParseErr(p, "'[' expected for a seq")
+  of tyTuple:
+    if p.kind != jsonObjectStart: raiseParseErr(p, "'{' expected for an object")
+    next(p)
+    result = newNode(nkPar)
+    var i = 0
+    while p.kind != jsonObjectEnd and p.kind != jsonEof:
+      if p.kind != jsonString:
+        raiseParseErr(p, "string expected for a field name")
+      next(p)
+      if i >= t.len:
+        raiseParseErr(p, "too many fields to tuple type " & typeToString(t))
+      result.add loadAny(p, t.sons[i], tab)
+      inc i
+    if p.kind == jsonObjectEnd: next(p)
+    else: raiseParseErr(p, "'}' end of object expected")
+  of tyObject:
+    if p.kind != jsonObjectStart: raiseParseErr(p, "'{' expected for an object")
+    next(p)
+    result = newNode(nkPar)
+    result.sons = @[]
+    while p.kind != jsonObjectEnd and p.kind != jsonEof:
+      if p.kind != jsonString:
+        raiseParseErr(p, "string expected for a field name")
+      let field = lookupInRecord(t.n, getIdent(p.str))
+      if field.isNil:
+        raiseParseErr(p, "unknown field for object of type " & typeToString(t))
+      next(p)
+      if field.position >= result.sons.len:
+        setLen(result.sons, field.position+1)
+      result.sons[field.position] = loadAny(p, field.typ, tab)
+    if p.kind == jsonObjectEnd: next(p)
+    else: raiseParseErr(p, "'}' end of object expected")
+  of tySet:
+    if p.kind != jsonArrayStart: raiseParseErr(p, "'[' expected for a set")
+    next(p)
+    result = newNode(nkCurly)
+    while p.kind != jsonArrayEnd and p.kind != jsonEof:
+      result.add loadAny(p, t.lastSon, tab)
+      next(p)
+    if p.kind == jsonArrayEnd: next(p)
+    else: raiseParseErr(p, "']' end of array expected")
+  of tyPtr, tyRef:
+    case p.kind
+    of jsonNull:
+      result = newNode(nkNilLit)
+      next(p)
+    of jsonInt:
+      result = tab[p.getInt]
+      if result.isNil:
+        raiseParseErr(p, "cannot load object with address " & $p.getInt)
+      next(p)
+    of jsonArrayStart:
+      next(p)
+      if p.kind == jsonInt:
+        let idx = p.getInt
+        next(p)
+        result = loadAny(p, t.lastSon, tab)
+        tab[idx] = result
+      else: raiseParseErr(p, "index for ref type expected")
+      if p.kind == jsonArrayEnd: next(p)
+      else: raiseParseErr(p, "']' end of ref-address pair expected")
+    else: raiseParseErr(p, "int for pointer type expected")
+  of tyString, tyCString:
+    case p.kind
+    of jsonNull:
+      result = newNode(nkNilLit)
+      next(p)
+    of jsonString:
+      result = newStrNode(nkStrLit, p.str)
+      next(p)
+    else: raiseParseErr(p, "string expected")
+  of tyInt..tyInt64, tyUInt..tyUInt64:
+    if p.kind == jsonInt:
+      result = newIntNode(nkIntLit, getInt(p))
+      next(p)
+      return
+    raiseParseErr(p, "int expected")
+  of tyFloat..tyFloat128:
+    if p.kind == jsonFloat:
+      result = newFloatNode(nkFloatLit, getFloat(p))
+      next(p)
+      return
+    raiseParseErr(p, "float expected")
+  of tyRange, tyGenericInst: result = loadAny(p, t.lastSon, tab)
+  else:
+    internalError "cannot marshal at compile-time " & t.typeToString
+
+proc loadAny*(s: string; t: PType): PNode =
+  var tab = initTable[BiggestInt, PNode]()
+  var p: JsonParser
+  open(p, newStringStream(s), "unknown file")
+  next(p)
+  result = loadAny(p, t, tab)
+  close(p)
--- a/lib/pure/marshal.nim
+++ b/lib/pure/marshal.nim
@@ -15,8 +15,8 @@
 ## type than its compiletime type:
 ##
 ## .. code-block:: nim
-## 
-##   type 
+##
+##   type
 ##     TA = object
 ##     TB = object of TA
 ##       f: int
@@ -28,6 +28,8 @@
 ##   new(b)
 ##   a = b
 ##   echo($$a[]) # produces "{}", not "{f: 0}"
+##
+## **Note**: The ``to`` and ``$$`` operations are available at compile-time!

 import streams, typeinfo, json, intsets, tables

@@ -38,7 +40,12 @@ proc storeAny(s: Stream, a: TAny, stored: var IntSet) =
  case a.kind
  of akNone: assert false
  of akBool: s.write($getBool(a))
-  of akChar: s.write(escapeJson($getChar(a)))
+  of akChar:
+    let ch = getChar(a)
+    if ch < '\128':
+      s.write(escapeJson($ch))
+    else:
+      s.write($int(ch))
  of akArray, akSequence:
    if a.kind == akSequence and isNil(a): s.write("null")
    else:
@@ -92,7 +99,7 @@ proc storeAny(s: Stream, a: TAny, stored: var IntSet) =
 proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) =
  case a.kind
  of akNone: assert false
-  of akBool: 
+  of akBool:
    case p.kind
    of jsonFalse: setBiggestInt(a, 0)
    of jsonTrue: setBiggestInt(a, 1)
@@ -105,8 +112,12 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) =
        setBiggestInt(a, ord(x[0]))
        next(p)
        return
+    elif p.kind == jsonInt:
+      setBiggestInt(a, getInt(p))
+      next(p)
+      return
    raiseParseErr(p, "string of length 1 expected for a char")
-  of akEnum: 
+  of akEnum:
    if p.kind == jsonString:
      setBiggestInt(a, getEnumOrdinal(a, p.str))
      next(p)
@@ -122,7 +133,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) =
    if p.kind == jsonArrayEnd: next(p)
    else: raiseParseErr(p, "']' end of array expected")
  of akSequence:
-    case p.kind 
+    case p.kind
    of jsonNull:
      setPointer(a, nil)
      next(p)
@@ -143,7 +154,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) =
    if p.kind != jsonObjectStart: raiseParseErr(p, "'{' expected for an object")
    next(p)
    while p.kind != jsonObjectEnd and p.kind != jsonEof:
-      if p.kind != jsonString: 
+      if p.kind != jsonString:
        raiseParseErr(p, "string expected for a field name")
      var fieldName = p.str
      next(p)
@@ -160,7 +171,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) =
    if p.kind == jsonArrayEnd: next(p)
    else: raiseParseErr(p, "']' end of array expected")
  of akPtr, akRef:
-    case p.kind 
+    case p.kind
    of jsonNull:
      setPointer(a, nil)
      next(p)
@@ -170,7 +181,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) =
    of jsonArrayStart:
      next(p)
      if a.kind == akRef: invokeNew(a)
-      else: setPointer(a, alloc0(a.baseTypeSize))      
+      else: setPointer(a, alloc0(a.baseTypeSize))
      if p.kind == jsonInt:
        t[p.getInt] = getPointer(a)
        next(p)
@@ -179,8 +190,8 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) =
      if p.kind == jsonArrayEnd: next(p)
      else: raiseParseErr(p, "']' end of ref-address pair expected")
    else: raiseParseErr(p, "int for pointer type expected")
-  of akProc, akPointer, akCString: 
-    case p.kind 
+  of akProc, akPointer, akCString:
+    case p.kind
    of jsonNull:
      setPointer(a, nil)
      next(p)
@@ -189,7 +200,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) =
      next(p)
    else: raiseParseErr(p, "int for pointer type expected")
  of akString:
-    case p.kind 
+    case p.kind
    of jsonNull:
      setPointer(a, nil)
      next(p)
@@ -197,7 +208,7 @@ proc loadAny(p: var JsonParser, a: TAny, t: var Table[BiggestInt, pointer]) =
      setString(a, p.str)
      next(p)
    else: raiseParseErr(p, "string expected")
-  of akInt..akInt64, akUInt..akUInt64: 
+  of akInt..akInt64, akUInt..akUInt64:
    if p.kind == jsonInt:
      setBiggestInt(a, getInt(p))
      next(p)
@@ -243,22 +254,22 @@ proc to*[T](data: string): T =
  ## reads data and transforms it to a ``T``.
  var tab = initTable[BiggestInt, pointer]()
  loadAny(newStringStream(data), toAny(result), tab)
-  
+
 when not defined(testing) and isMainModule:
  template testit(x: expr) = echo($$to[type(x)]($$x))

  var x: array[0..4, array[0..4, string]] = [
-    ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], 
-    ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"], 
+    ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"],
+    ["test", "1", "2", "3", "4"], ["test", "1", "2", "3", "4"],
    ["test", "1", "2", "3", "4"]]
  testit(x)
  var test2: tuple[name: string, s: uint] = ("tuple test", 56u)
  testit(test2)
-  
+
  type
    TE = enum
      blah, blah2
-  
+
    TestObj = object
      test, asd: int
      case test2: TE
@@ -266,7 +277,7 @@ when not defined(testing) and isMainModule:
        help: string
      else:
        nil
-        
+
    PNode = ref TNode
    TNode = object
      next, prev: PNode
@@ -294,7 +305,7 @@ when not defined(testing) and isMainModule:
  test4.a = "ref string test: A"
  test4.b = "ref string test: B"
  testit(test4)
-  
+
  var test5 = @[(0,1),(2,3),(4,5)]
  testit(test5)

@@ -305,7 +316,7 @@ when not defined(testing) and isMainModule:
  echo($$test7)
  testit(test7)

-  type 
+  type
    TA {.inheritable.} = object
    TB = object of TA
      f: int
--- a/lib/pure/unicode.nim
+++ b/lib/pure/unicode.nim
@@ -105,6 +105,31 @@ template fastRuneAt*(s: string, i: int, result: expr, doInc = true) =
    result = Rune(ord(s[i]))
    when doInc: inc(i)

+proc validateUtf8*(s: string): int =
+  ## returns the position of the invalid byte in ``s`` if the string ``s`` does
+  ## not hold valid UTF-8 data. Otherwise -1 is returned.
+  var i = 0
+  let L = s.len
+  while i < L:
+    if ord(s[i]) <=% 127:
+      inc(i)
+    elif ord(s[i]) shr 5 == 0b110:
+      if i+1 < L and ord(s[i+1]) shr 6 == 0b10: inc(i, 2)
+      else: return i
+    elif ord(s[i]) shr 4 == 0b1110:
+      if i+2 < L and ord(s[i+1]) shr 6 == 0b10 and ord(s[i+2]) shr 6 == 0b10:
+        inc i, 3
+      else: return i
+    elif ord(s[i]) shr 3 == 0b11110:
+      if i+3 < L and ord(s[i+1]) shr 6 == 0b10 and
+                     ord(s[i+2]) shr 6 == 0b10 and
+                     ord(s[i+3]) shr 6 == 0b10:
+        inc i, 4
+      else: return i
+    else:
+      return i
+  return -1
+
 proc runeAt*(s: string, i: Natural): Rune =
  ## returns the unicode character in `s` at byte index `i`
  fastRuneAt(s, i, result, false)
--- a/tests/stdlib/tdialogs.nim
+++ b/tests/stdlib/tdialogs.nim
@@ -1,17 +0,0 @@
-# Test the dialogs module
-
-import dialogs, gtk2
-
-gtk2.nimrod_init()
-
-var x = chooseFilesToOpen(nil)
-for a in items(x):
-  writeln(stdout, a)
-
-info(nil, "start with an info box")
-warning(nil, "now a warning ...")
-error(nil, "... and an error!")
-
-writeln(stdout, chooseFileToOpen(nil))
-writeln(stdout, chooseFileToSave(nil))
-writeln(stdout, chooseDir(nil))
--- a/web/news.txt
+++ b/web/news.txt
@@ -130,6 +130,7 @@ News
  - ``system.len`` for strings and sequences now returns 0 for nil.

  - A single underscore can now be used to discard values when unpacking tuples.
+  - ``marshal.$$`` and ``marshal.to`` can be executed at compile-time.


  Library additions