bugfix: multiple yield statements and loop body vars

lib/pure/math.nim

@@ -226,21 +226,27 @@ proc push*(s: var TRunningStat, x: float) =
   inc(s.n)
   # See Knuth TAOCP vol 2, 3rd edition, page 232
   if s.n == 1:
+    s.min = x
+    s.max = x
     s.oldM = x
     s.mean = x
     s.oldS = 0.0
   else:
+    if s.min > x: s.min = x
+    if s.max < x: s.max = x
     s.mean = s.oldM + (x - s.oldM)/toFloat(s.n)
     s.newS = s.oldS + (x - s.oldM)*(x - s.mean)
 
     # set up for next iteration:
     s.oldM = s.mean
     s.oldS = s.newS
-  
   s.sum = s.sum + x
-  if s.min > x: s.min = x
-  if s.max < x: s.max = x
-
+  
+proc push*(s: var TRunningStat, x: int) = 
+  ## pushes a value `x` for processing. `x` is simply converted to ``float``
+  ## and the other push operation is called.
+  push(s, toFloat(x))
+  
 proc variance*(s: TRunningStat): float = 
   ## computes the current variance of `s`
   if s.n > 1: result = s.newS / (toFloat(s.n - 1))

lib/pure/strutils.nim

@@ -464,6 +464,69 @@ proc align*(s: string, count: int): string {.
   else:
     result = s
 
+iterator tokenize*(s: string, seps: set[char] = Whitespace): tuple[
+  token: string, isSep: bool] =
+  ## Tokenizes the string `s` into substrings.
+  ##
+  ## Substrings are separated by a substring containing only `seps`.
+  ## Examples:
+  ##
+  ## .. code-block:: nimrod
+  ##   for word in tokenize("  this is an  example  "):
+  ##     writeln(stdout, word)
+  ##
+  ## Results in:
+  ##
+  ## .. code-block:: nimrod
+  ##   ("  ", true)
+  ##   ("this", false)
+  ##   (" ", true)
+  ##   ("is", false)
+  ##   (" ", true)
+  ##   ("an", false)
+  ##   ("  ", true)
+  ##   ("example", false)
+  ##   ("  ", true)
+  var i = 0
+  while true:
+    var j = i
+    var isSep = s[j] in seps
+    while j < s.len and (s[j] in seps) == isSep: inc(j)
+    if j > i:
+      yield (copy(s, i, j-1), isSep)
+    else:
+      break
+    i = j
+
+proc wordWrap*(s: string, maxLineWidth = 80, 
+               splitLongWords = true,
+               seps: set[char] = whitespace,
+               newLine = "\n"): string {.
+               noSideEffect, rtl, extern: "nsuWordWrap".} = 
+  ## word wraps `s`.
+  result = ""
+  var SpaceLeft = maxLineWidth
+  for word, isSep in tokenize(s, seps):
+    if len(word) > SpaceLeft:
+      if splitLongWords and len(word) > maxLineWidth:
+        result.add(copy(word, 0, spaceLeft-1))
+        var w = spaceLeft+1
+        var wordLeft = len(word) - spaceLeft
+        while wordLeft > 0: 
+          result.add(newLine)
+          var L = min(maxLineWidth, wordLeft)
+          SpaceLeft = maxLineWidth - L
+          result.add(copy(word, w, w+L-1))
+          inc(w, L)
+          dec(wordLeft, L)
+      else:
+        SpaceLeft = maxLineWidth - len(Word)
+        result.add(newLine)
+        result.add(word)
+    else:
+      SpaceLeft = SpaceLeft - len(Word)
+      result.add(word)
+
 proc startsWith*(s, prefix: string): bool {.noSideEffect,
   rtl, extern: "nsuStartsWith".} =
   ## Returns true iff ``s`` starts with ``prefix``.
@@ -879,6 +942,7 @@ when isMainModule:
   assert align("abc", 4) == " abc"
   assert align("a", 0) == "a"
   assert align("1232", 6) == "  1232"
-
+  echo wordWrap(""" this is a long text --  muchlongerthan10chars and here
+                   it goes""", 10, false)
   
   

lib/system.nim

@@ -1099,6 +1099,15 @@ iterator items*(a: cstring): char {.inline.} =
     yield a[i]
     inc(i)
 
+iterator enumerate*[TContainer, TItem](a: TContainer): tuple[
+                    index: int, item: TItem] {.inline.} = 
+  ## iterates over each item of `a` via `items` and yields an additional
+  ## counter/index starting from 0.
+  var j = 0
+  for it in items(a): 
+    yield (j, a)
+    inc j
+
 proc isNil*[T](x: seq[T]): bool {.noSideEffect, magic: "IsNil".}
 proc isNil*[T](x: ref T): bool {.noSideEffect, magic: "IsNil".}
 proc isNil*(x: string): bool {.noSideEffect, magic: "IsNil".}
@@ -1108,20 +1117,20 @@ proc isNil*(x: cstring): bool {.noSideEffect, magic: "IsNil".}
   ## Fast check whether `x` is nil. This is sometimes more efficient than
   ## ``== nil``.
 
-proc `&` *[T](x, y: openArray[T]): seq[T] {.noSideEffect.} =
+proc `&` *[T](x, y: seq[T]): seq[T] {.noSideEffect.} =
   newSeq(result, x.len + y.len)
   for i in 0..x.len-1:
     result[i] = x[i]
   for i in 0..y.len-1:
     result[i+x.len] = y[i]
 
-proc `&` *[T](x: openArray[T], y: T): seq[T] {.noSideEffect.} =
+proc `&` *[T](x: seq[T], y: T): seq[T] {.noSideEffect.} =
   newSeq(result, x.len + 1)
   for i in 0..x.len-1:
     result[i] = x[i]
   result[x.len] = y
 
-proc `&` *[T](x: T, y: openArray[T]): seq[T] {.noSideEffect.} =
+proc `&` *[T](x: T, y: seq[T]): seq[T] {.noSideEffect.} =
   newSeq(result, y.len + 1)
   for i in 0..y.len-1:
     result[i] = y[i]

rod/ast.nim

@@ -537,8 +537,8 @@ const
 
 var gId*: int
 
-proc getID*(): int
-proc setID*(id: int)
+proc getID*(): int {.inline.}
+proc setID*(id: int) {.inline.}
 proc IDsynchronizationPoint*(idRange: int)
 
 # creator procs:
@@ -568,10 +568,10 @@ proc copyStrTable*(dest: var TStrTable, src: TStrTable)
 proc copyTable*(dest: var TTable, src: TTable)
 proc copyObjectSet*(dest: var TObjectSet, src: TObjectSet)
 proc copyIdTable*(dest: var TIdTable, src: TIdTable)
-proc sonsLen*(n: PNode): int
-proc sonsLen*(n: PType): int
-proc lastSon*(n: PNode): PNode
-proc lastSon*(n: PType): PType
+proc sonsLen*(n: PNode): int {.inline.}
+proc sonsLen*(n: PType): int {.inline.}
+proc lastSon*(n: PNode): PNode {.inline.}
+proc lastSon*(n: PType): PType {.inline.}
 proc newSons*(father: PNode, length: int)
 proc newSons*(father: PType, length: int)
 proc addSon*(father, son: PNode)
@@ -903,6 +903,21 @@ proc copyNode(src: PNode): PNode =
   of nkStrLit..nkTripleStrLit: result.strVal = src.strVal
   else: nil
 
+proc shallowCopy*(src: PNode): PNode = 
+  # does not copy its sons, but provides space for them:
+  if src == nil: return nil
+  result = newNode(src.kind)
+  result.info = src.info
+  result.typ = src.typ
+  result.flags = src.flags * PersistentNodeFlags
+  case src.Kind
+  of nkCharLit..nkInt64Lit: result.intVal = src.intVal
+  of nkFloatLit, nkFloat32Lit, nkFloat64Lit: result.floatVal = src.floatVal
+  of nkSym: result.sym = src.sym
+  of nkIdent: result.ident = src.ident
+  of nkStrLit..nkTripleStrLit: result.strVal = src.strVal
+  else: newSons(result, sonsLen(src))
+
 proc copyTree(src: PNode): PNode = 
   # copy a whole syntax tree; performs deep copying
   if src == nil: 
@@ -920,7 +935,8 @@ proc copyTree(src: PNode): PNode =
   else: 
     result.sons = nil
     newSons(result, sonsLen(src))
-    for i in countup(0, sonsLen(src) - 1): result.sons[i] = copyTree(src.sons[i])
+    for i in countup(0, sonsLen(src) - 1): 
+      result.sons[i] = copyTree(src.sons[i])
   
 proc lastSon(n: PNode): PNode = 
   result = n.sons[sonsLen(n) - 1]
@@ -986,11 +1002,11 @@ proc getStrOrChar*(a: PNode): string =
     internalError(a.info, "getStrOrChar")
     result = ""
   
-proc mustRehash(length, counter: int): bool = 
+proc mustRehash(length, counter: int): bool {.inline.} = 
   assert(length > counter)
   result = (length * 2 < counter * 3) or (length - counter < 4)
 
-proc nextTry(h, maxHash: THash): THash = 
+proc nextTry(h, maxHash: THash): THash {.inline.} = 
   result = ((5 * h) + 1) and maxHash 
   # For any initial h in range(maxHash), repeating that maxHash times
   # generates each int in range(maxHash) exactly once (see any text on

rod/astalgo.nim

@@ -767,15 +767,13 @@ proc IdNodeTableRawInsert(data: var TIdNodePairSeq, key: PIdObj, val: PNode) =
   data[h].val = val
 
 proc IdNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) = 
-  var 
-    index: int
-    n: TIdNodePairSeq
-  index = IdNodeTableRawGet(t, key)
+  var index = IdNodeTableRawGet(t, key)
   if index >= 0: 
     assert(t.data[index].key != nil)
     t.data[index].val = val
   else: 
     if mustRehash(len(t.data), t.counter): 
+      var n: TIdNodePairSeq
       newSeq(n, len(t.data) * growthFactor)
       for i in countup(0, high(t.data)): 
         if t.data[i].key != nil: 

rod/sigmatch.nim

@@ -14,20 +14,20 @@ type
   TCandidateState = enum 
     csEmpty, csMatch, csNoMatch
   TCandidate{.final.} = object 
-    exactMatches*: int
-    subtypeMatches*: int
-    intConvMatches*: int      # conversions to int are not as expensive
-    convMatches*: int
-    genericMatches*: int
-    state*: TCandidateState
-    callee*: PType            # may not be nil!
-    calleeSym*: PSym          # may be nil
-    call*: PNode              # modified call
-    bindings*: TIdTable       # maps sym-ids to types
-    baseTypeMatch*: bool      # needed for conversions from T to openarray[T]
-                              # for example
+    exactMatches: int
+    subtypeMatches: int
+    intConvMatches: int      # conversions to int are not as expensive
+    convMatches: int
+    genericMatches: int
+    state: TCandidateState
+    callee: PType            # may not be nil!
+    calleeSym: PSym          # may be nil
+    call: PNode              # modified call
+    bindings: TIdTable       # maps sym-ids to types
+    baseTypeMatch: bool      # needed for conversions from T to openarray[T]
+                             # for example
   
-  TTypeRelation = enum        # order is important!
+  TTypeRelation = enum       # order is important!
     isNone, isConvertible, isIntConv, isSubtype, isGeneric, isEqual
 
 proc initCandidateAux(c: var TCandidate, callee: PType) {.inline.} = 

rod/transf.nim

@@ -29,20 +29,23 @@ proc transfPass*(): TPass
 type 
   PTransCon = ref TTransCon
   TTransCon{.final.} = object # part of TContext; stackable
-    mapping*: TIdNodeTable    # mapping from symbols to nodes
-    owner*: PSym              # current owner
-    forStmt*: PNode           # current for stmt
-    next*: PTransCon          # for stacking
+    mapping: TIdNodeTable     # mapping from symbols to nodes
+    owner: PSym               # current owner
+    forStmt: PNode            # current for stmt
+    next: PTransCon           # for stacking
   
   TTransfContext = object of passes.TPassContext
-    module*: PSym
-    transCon*: PTransCon      # top of a TransCon stack
+    module: PSym
+    transCon: PTransCon      # top of a TransCon stack
+    inlining: int            # > 0 if we are in inlining context (copy vars)
   
   PTransf = ref TTransfContext
 
-proc newTransCon(): PTransCon = 
+proc newTransCon(owner: PSym): PTransCon = 
+  assert owner != nil
   new(result)
   initIdNodeTable(result.mapping)
+  result.owner = owner
 
 proc pushTransCon(c: PTransf, t: PTransCon) = 
   t.next = c.transCon
@@ -209,9 +212,50 @@ proc transformYield(c: PTransf, n: PNode): PNode =
   else: 
     e = transform(c, copyTree(e))
     addSon(result, newAsgnStmt(c, c.transCon.forStmt.sons[0], e))
+  
+  #var tc = newTransCon(c.transCon.owner)
+  #tc.forStmt = c.transCon.forStmt
+  #pushTransCon(c, tc)
+  inc(c.inlining)
   addSon(result, transform(c, lastSon(c.transCon.forStmt)))
+  dec(c.inlining)
+  #popTransCon(c)
+
+proc transformVarSection(c: PTransf, v: PNode): PNode =
+  result = copyTree(v)
+  for i in countup(0, sonsLen(result) - 1): 
+    var it = result.sons[i]
+    if it.kind == nkCommentStmt: continue 
+    if it.kind == nkIdentDefs: 
+      if (it.sons[0].kind != nkSym):
+        InternalError(it.info, "transformVarSection")
+      var newVar = copySym(it.sons[0].sym)
+      if identEq(newVar.name, "titer2TestVar"):
+        echo "created a copy of titer2TestVar ", newVar.id, " ", 
+            it.sons[0].sym.id
+      
+      incl(newVar.flags, sfFromGeneric) 
+      # fixes a strange bug for rodgen:
+      #include(it.sons[0].sym.flags, sfFromGeneric);
+      newVar.owner = getCurrOwner(c)
+      IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar))
+      it.sons[0].sym = newVar
+      it.sons[2] = transform(c, it.sons[2])
+    else: 
+      if it.kind != nkVarTuple: 
+        InternalError(it.info, "transformVarSection: not nkVarTuple")
+      var L = sonsLen(it)
+      for j in countup(0, L - 3): 
+        var newVar = copySym(it.sons[j].sym)
+        incl(newVar.flags, sfFromGeneric)
+        newVar.owner = getCurrOwner(c)
+        IdNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar))
+        it.sons[j] = newSymNode(newVar)
+      assert(it.sons[L - 2] == nil)
+      it.sons[L - 1] = transform(c, it.sons[L - 1])
 
 proc inlineIter(c: PTransf, n: PNode): PNode = 
+  # n: iterator body
   result = n
   if n == nil: return 
   case n.kind
@@ -220,32 +264,7 @@ proc inlineIter(c: PTransf, n: PNode): PNode =
   of nkYieldStmt: 
     result = transformYield(c, n)
   of nkVarSection: 
-    result = copyTree(n)
-    for i in countup(0, sonsLen(result) - 1): 
-      var it = result.sons[i]
-      if it.kind == nkCommentStmt: continue 
-      if it.kind == nkIdentDefs: 
-        if (it.sons[0].kind != nkSym): InternalError(it.info, "inlineIter")
-        var newVar = copySym(it.sons[0].sym)
-        incl(newVar.flags, sfFromGeneric) 
-        # fixes a strange bug for rodgen:
-        #include(it.sons[0].sym.flags, sfFromGeneric);
-        newVar.owner = getCurrOwner(c)
-        IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar))
-        it.sons[0] = newSymNode(newVar)
-        it.sons[2] = transform(c, it.sons[2])
-      else: 
-        if it.kind != nkVarTuple: 
-          InternalError(it.info, "inlineIter: not nkVarTuple")
-        var L = sonsLen(it)
-        for j in countup(0, L - 3): 
-          var newVar = copySym(it.sons[j].sym)
-          incl(newVar.flags, sfFromGeneric)
-          newVar.owner = getCurrOwner(c)
-          IdNodeTablePut(c.transCon.mapping, it.sons[j].sym, newSymNode(newVar))
-          it.sons[j] = newSymNode(newVar)
-        assert(it.sons[L - 2] == nil)
-        it.sons[L - 1] = transform(c, it.sons[L - 1])
+    result = transformVarSection(c, n)
   else: 
     result = copyNode(n)
     for i in countup(0, sonsLen(n) - 1): addSon(result, inlineIter(c, n.sons[i]))
@@ -388,11 +407,11 @@ proc transformFor(c: PTransf, n: PNode): PNode =
   for i in countup(0, length - 3): 
     addVar(v, copyTree(n.sons[i])) # declare new vars
   addSon(result, v)
-  var newC = newTransCon()
   var call = n.sons[length - 2]
   if (call.kind != nkCall) or (call.sons[0].kind != nkSym): 
     InternalError(call.info, "transformFor")
-  newC.owner = call.sons[0].sym
+  
+  var newC = newTransCon(call.sons[0].sym)
   newC.forStmt = n
   if (newC.owner.kind != skIterator): 
     InternalError(call.info, "transformFor") 
@@ -486,7 +505,7 @@ proc transformLambda(c: PTransf, n: PNode): PNode =
   # all variables that are accessed should be accessed by the new closure
   # parameter:
   if sonsLen(closure) > 0: 
-    var newC = newTransCon()
+    var newC = newTransCon(c.transCon.owner)
     for i in countup(0, sonsLen(closure) - 1): 
       IdNodeTablePut(newC.mapping, closure.sons[i].sym, 
                      indirectAccess(param, closure.sons[i].sym))
@@ -613,22 +632,35 @@ proc transform(c: PTransf, n: PNode): PNode =
   of nkHiddenStdConv, nkHiddenSubConv, nkConv: 
     result = transformConv(c, n)
   of nkDiscardStmt: 
-    for i in countup(0, sonsLen(n) - 1): result.sons[i] = transform(c, n.sons[i])
+    for i in countup(0, sonsLen(n) - 1): 
+      result.sons[i] = transform(c, n.sons[i])
     if isConstExpr(result.sons[0]): result = newNode(nkCommentStmt)
   of nkCommentStmt, nkTemplateDef: 
     return 
   of nkConstSection: 
     # do not replace ``const c = 3`` with ``const 3 = 3``
     return                    
-  else: 
-    for i in countup(0, sonsLen(n) - 1): result.sons[i] = transform(c, n.sons[i])
+  of nkVarSection: 
+    if c.inlining > 0: 
+      # we need to copy the variables for multiple yield statements:
+      result = transformVarSection(c, n)
+    else:
+      result = shallowCopy(n)
+      for i in countup(0, sonsLen(n) - 1): 
+        result.sons[i] = transform(c, n.sons[i])
+  else:
+    result = shallowCopy(n)
+    for i in countup(0, sonsLen(n) - 1): 
+      result.sons[i] = transform(c, n.sons[i])
   var cnst = getConstExpr(c.module, result)
   if cnst != nil: 
     result = cnst             # do not miss an optimization  
   
 proc processTransf(context: PPassContext, n: PNode): PNode = 
   var c = PTransf(context)
+  pushTransCon(c, newTransCon(getCurrOwner(c)))
   result = transform(c, n)
+  popTransCon(c)
 
 proc openTransf(module: PSym, filename: string): PPassContext = 
   var n: PTransf

tests/accept/compile/toop.nim

@@ -0,0 +1,18 @@
+
+type
+  TA = object
+    x, y: int
+  
+  TB = object of TA
+    z: int
+    
+  TC = object of TB
+    whatever: string
+  
+proc p(a: var TA) = nil
+proc p(b: var TB) = nil
+
+var c: TC
+
+p(c)
+

tests/accept/compile/tquicksort.nim

@@ -9,7 +9,9 @@ proc QuickSort(list: seq[int]): seq[int] =
             left.add(list[i])
         elif list[i] > pivot:
             right.add(list[i])
-    result = QuickSort(left) & pivot & QuickSort(right)
+    result = QuickSort(left) & 
+      pivot & 
+      QuickSort(right)
     
 proc echoSeq(a: seq[int]) =
     for i in low(a)..high(a):

tests/accept/run/spec.csv

@@ -34,6 +34,7 @@ tisopr.nim;falsetrue
 titer2.nim;123
 titer3.nim;1231
 titer5.nim;abcxyz
+titer6.nim;000
 tlenopenarray.nim;1
 tlowhigh.nim;10
 tmatrix.nim;111

tests/accept/run/titer6.nim

@@ -0,0 +1,31 @@
+# Test iterator with more than 1 yield statement
+
+import strutils
+
+iterator tokenize2(s: string, seps: set[char] = Whitespace): tuple[
+  token: string, isSep: bool] =
+  var i = 0
+  while i < s.len:
+    var j = i
+    if s[j] in seps:
+      while j < s.len and s[j] in seps: inc(j)
+      if j > i:
+        yield (copy(s, i, j-1), true)
+    else:
+      while j < s.len and s[j] notin seps: inc(j)
+      if j > i:
+        yield (copy(s, i, j-1), false)
+    i = j
+
+for word, isSep in tokenize2("ta da", whiteSpace):
+  var titer2TestVar = 0 
+  stdout.write(titer2TestVar)
+
+proc wordWrap2(s: string, maxLineWidth = 80, 
+               splitLongWords = true,
+               seps: set[char] = whitespace,
+               newLine = "\n"): string  = 
+  result = ""
+  for word, isSep in tokenize2(s, seps):
+    var w = 0 
+

todo.txt

@@ -1,6 +1,8 @@
 High priority (version 0.9.0)
 =============================
 
+- transf should use distinct types; shallowCopy() for PNode
+
 - fix implicit generic routines
 - fix the streams implementation so that it uses methods
 - fix overloading resolution

web/news.txt

@@ -16,6 +16,8 @@ Bugfixes
 - Bugfix: ``dialogs.ChooseFilesToOpen`` did not work if only one file is
   selected.
 - Bugfix: niminst: ``nimrod`` is not default dir for *every* project.
+- Bugfix: Multiple yield statements in iterators did not cause local vars to be
+  copied.
 
 
 Additions
@@ -23,7 +25,7 @@ Additions
 
 - Added ``re.findAll``, ``pegs.findAll``.
 - Added ``os.findExe``.
-- Added ``strutils.align``. 
+- Added ``strutils.align``, ``strutils.tokenize``, ``strutils.wordWrap``. 
 - Pegs support a *captured search loop operator* ``{@}``.
 - Pegs support new built-ins: ``\letter``, ``\upper``, ``\lower``,
   ``\title``, ``\white``.