Merge branch 'devel' of https://www.github.com/Araq/Nimrod into devel

2026-06-12 06:38:11 +00:00 · 2014-01-26 01:38:45 +02:00
parent 5a6030a16b ad74332a92
commit 5d712e0d3f
45 changed files with 668 additions and 271 deletions
--- a/compiler/ccgcalls.nim
+++ b/compiler/ccgcalls.nim
@@ -146,7 +146,8 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
  proc addComma(r: PRope): PRope =
    result = if r == nil: r else: con(r, ~", ")

-  const CallPattern = "$1.ClEnv? $1.ClPrc($3$1.ClEnv) : (($4)($1.ClPrc))($2)"
+  const PatProc = "$1.ClEnv? $1.ClPrc($3$1.ClEnv):(($4)($1.ClPrc))($2)"
+  const PatIter = "$1.ClPrc($3$1.ClEnv)" # we know the env exists
  var op: TLoc
  initLocExpr(p, ri.sons[0], op)
  var pl: PRope
@@ -164,9 +165,10 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
    if i < length - 1: app(pl, ~", ")
  
  template genCallPattern {.dirty.} =
-    lineF(p, cpsStmts, CallPattern & ";$n", op.r, pl, pl.addComma, rawProc)
+    lineF(p, cpsStmts, callPattern & ";$n", op.r, pl, pl.addComma, rawProc)

  let rawProc = getRawProcType(p, typ)
+  let callPattern = if tfIterator in typ.flags: PatIter else: PatProc
  if typ.sons[0] != nil:
    if isInvalidReturnType(typ.sons[0]):
      if sonsLen(ri) > 1: app(pl, ~", ")
@@ -190,7 +192,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
      assert(d.t != nil)        # generate an assignment to d:
      var list: TLoc
      initLoc(list, locCall, d.t, OnUnknown)
-      list.r = ropef(CallPattern, op.r, pl, pl.addComma, rawProc)
+      list.r = ropef(callPattern, op.r, pl, pl.addComma, rawProc)
      genAssignment(p, d, list, {}) # no need for deep copying
  else:
    genCallPattern()
--- a/compiler/lambdalifting.nim
+++ b/compiler/lambdalifting.nim
@@ -116,9 +116,9 @@ type
  TDep = tuple[e: PEnv, field: PSym]
  TEnv {.final.} = object of TObject
    attachedNode: PNode
-    closure: PSym   # if != nil it is a used environment
+    createdVar: PSym         # if != nil it is a used environment
    capturedVars: seq[PSym] # captured variables in this environment
-    deps: seq[TDep] # dependencies
+    deps: seq[TDep]         # dependencies
    up: PEnv
    tup: PType
  
@@ -130,12 +130,99 @@ type
  TOuterContext {.final.} = object
    fn: PSym # may also be a module!
    currentEnv: PEnv
+    isIter: bool   # first class iterator?
    capturedVars, processed: TIntSet
    localsToEnv: TIdTable # PSym->PEnv mapping
    localsToAccess: TIdNodeTable
    lambdasToEnv: TIdTable # PSym->PEnv mapping
    up: POuterContext

+    closureParam, state, resultSym: PSym # only if isIter
+    tup: PType # only if isIter
+
+
+proc getStateType(iter: PSym): PType =
+  var n = newNodeI(nkRange, iter.info)
+  addSon(n, newIntNode(nkIntLit, -1))
+  addSon(n, newIntNode(nkIntLit, 0))
+  result = newType(tyRange, iter)
+  result.n = n
+  rawAddSon(result, getSysType(tyInt))
+
+proc createStateField(iter: PSym): PSym =
+  result = newSym(skField, getIdent(":state"), iter, iter.info)
+  result.typ = getStateType(iter)
+
+proc newIterResult(iter: PSym): PSym =
+  if resultPos < iter.ast.len:
+    result = iter.ast.sons[resultPos].sym
+  else:
+    # XXX a bit hacky:
+    result = newSym(skResult, getIdent":result", iter, iter.info)
+    result.typ = iter.typ.sons[0]
+    incl(result.flags, sfUsed)
+    iter.ast.add newSymNode(result)
+
+proc addHiddenParam(routine: PSym, param: PSym) =
+  var params = routine.ast.sons[paramsPos]
+  # -1 is correct here as param.position is 0 based but we have at position 0
+  # some nkEffect node:
+  param.position = params.len-1
+  addSon(params, newSymNode(param))
+  incl(routine.typ.flags, tfCapturesEnv)
+  #echo "produced environment: ", param.id, " for ", routine.name.s
+
+proc getHiddenParam(routine: PSym): PSym =
+  let params = routine.ast.sons[paramsPos]
+  let hidden = lastSon(params)
+  assert hidden.kind == nkSym
+  result = hidden.sym
+
+proc getEnvParam(routine: PSym): PSym =
+  let params = routine.ast.sons[paramsPos]
+  let hidden = lastSon(params)
+  if hidden.kind == nkSym and hidden.sym.name.s == paramName:
+    result = hidden.sym
+    
+proc addField(tup: PType, s: PSym) =
+  var field = newSym(skField, s.name, s.owner, s.info)
+  let t = skipIntLit(s.typ)
+  field.typ = t
+  field.position = sonsLen(tup)
+  addSon(tup.n, newSymNode(field))
+  rawAddSon(tup, t)
+
+proc initIterContext(c: POuterContext, iter: PSym) =
+  c.fn = iter
+  c.capturedVars = initIntSet()
+
+  var cp = getEnvParam(iter)
+  if cp == nil:
+    c.tup = newType(tyTuple, iter)
+    c.tup.n = newNodeI(nkRecList, iter.info)
+
+    cp = newSym(skParam, getIdent(paramName), iter, iter.info)
+    incl(cp.flags, sfFromGeneric)
+    cp.typ = newType(tyRef, iter)
+    rawAddSon(cp.typ, c.tup)
+    addHiddenParam(iter, cp)
+
+    c.state = createStateField(iter)
+    addField(c.tup, c.state)
+  else:
+    c.tup = cp.typ.sons[0]
+    assert c.tup.kind == tyTuple
+    if c.tup.len > 0:
+      c.state = c.tup.n[0].sym
+    else:
+      c.state = createStateField(iter)
+      addField(c.tup, c.state)
+
+  c.closureParam = cp
+  if iter.typ.sons[0] != nil:
+    c.resultSym = newIterResult(iter)
+    #iter.ast.add(newSymNode(c.resultSym))
+
 proc newOuterContext(fn: PSym, up: POuterContext = nil): POuterContext =
  new(result)
  result.fn = fn
@@ -144,12 +231,14 @@ proc newOuterContext(fn: PSym, up: POuterContext = nil): POuterContext =
  initIdNodeTable(result.localsToAccess)
  initIdTable(result.localsToEnv)
  initIdTable(result.lambdasToEnv)
+  result.isIter = fn.kind == skIterator and fn.typ.callConv == ccClosure
+  if result.isIter: initIterContext(result, fn)
  
 proc newInnerContext(fn: PSym): PInnerContext =
  new(result)
  result.fn = fn
  initIdNodeTable(result.localsToAccess)
-  
+
 proc newEnv(outerProc: PSym, up: PEnv, n: PNode): PEnv =
  new(result)
  result.deps = @[]
@@ -159,17 +248,12 @@ proc newEnv(outerProc: PSym, up: PEnv, n: PNode): PEnv =
  result.up = up
  result.attachedNode = n

-proc addField(tup: PType, s: PSym) =
-  var field = newSym(skField, s.name, s.owner, s.info)
-  let t = skipIntLit(s.typ)
-  field.typ = t
-  field.position = sonsLen(tup)
-  addSon(tup.n, newSymNode(field))
-  rawAddSon(tup, t)
-  
 proc addCapturedVar(e: PEnv, v: PSym) =
  for x in e.capturedVars:
    if x == v: return
+  # XXX meh, just add the state field for every closure for now, it's too
+  # hard to figure out if it comes from a closure iterator:
+  if e.tup.len == 0: addField(e.tup, createStateField(v.owner))
  e.capturedVars.add(v)
  addField(e.tup, v)
  
@@ -189,6 +273,7 @@ proc indirectAccess(a: PNode, b: PSym, info: TLineInfo): PNode =
  # returns a[].b as a node
  var deref = newNodeI(nkHiddenDeref, info)
  deref.typ = a.typ.sons[0]
+  assert deref.typ.kind == tyTuple
  let field = getSymFromList(deref.typ.n, b.name)
  assert field != nil, b.name.s
  addSon(deref, a)
@@ -205,33 +290,24 @@ proc newCall(a, b: PSym): PNode =
  result.add newSymNode(a)
  result.add newSymNode(b)

-proc addHiddenParam(routine: PSym, param: PSym) =
-  var params = routine.ast.sons[paramsPos]
-  # -1 is correct here as param.position is 0 based but we have at position 0
-  # some nkEffect node:
-  param.position = params.len-1
-  addSon(params, newSymNode(param))
-  incl(routine.typ.flags, tfCapturesEnv)
-  #echo "produced environment: ", param.id, " for ", routine.name.s
-
-proc getHiddenParam(routine: PSym): PSym =
-  let params = routine.ast.sons[paramsPos]
-  let hidden = lastSon(params)
-  assert hidden.kind == nkSym
-  result = hidden.sym
-
 proc isInnerProc(s, outerProc: PSym): bool {.inline.} =
-  result = s.kind in {skProc, skMethod, skConverter} and 
+  result = (s.kind in {skProc, skMethod, skConverter} or
+            s.kind == skIterator and s.typ.callConv == ccClosure) and
           s.skipGenericOwner == outerProc
  #s.typ.callConv == ccClosure

 proc addClosureParam(i: PInnerContext, e: PEnv) =
-  var cp = newSym(skParam, getIdent(paramName), i.fn, i.fn.info)
-  incl(cp.flags, sfFromGeneric)
-  cp.typ = newType(tyRef, i.fn)
-  rawAddSon(cp.typ, e.tup)
+  var cp = getEnvParam(i.fn)
+  if cp == nil:
+    cp = newSym(skParam, getIdent(paramName), i.fn, i.fn.info)
+    incl(cp.flags, sfFromGeneric)
+    cp.typ = newType(tyRef, i.fn)
+    rawAddSon(cp.typ, e.tup)
+    addHiddenParam(i.fn, cp)
+  else:
+    e.tup = cp.typ.sons[0]
+    assert e.tup.kind == tyTuple
  i.closureParam = cp
-  addHiddenParam(i.fn, i.closureParam)
  #echo "closure param added for ", i.fn.name.s, " ", i.fn.id

 proc dummyClosureParam(o: POuterContext, i: PInnerContext) =
@@ -306,7 +382,9 @@ proc gatherVars(o: POuterContext, i: PInnerContext, n: PNode) =
    var s = n.sym
    if interestingVar(s) and i.fn.id != s.owner.id:
      captureVar(o, i, s, n.info)
-    elif isInnerProc(s, o.fn) and tfCapturesEnv in s.typ.flags and s != i.fn:
+    elif s.kind in {skProc, skMethod, skConverter} and
+            s.skipGenericOwner == o.fn and 
+            tfCapturesEnv in s.typ.flags and s != i.fn:
      # call to some other inner proc; we need to track the dependencies for
      # this:
      let env = PEnv(idTableGet(o.lambdasToEnv, i.fn))
@@ -314,7 +392,7 @@ proc gatherVars(o: POuterContext, i: PInnerContext, n: PNode) =
      if o.currentEnv != env:
        discard addDep(o.currentEnv, env, i.fn)
        internalError(n.info, "too complex environment handling required")
-  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard
+  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkClosure: discard
  else:
    for k in countup(0, sonsLen(n) - 1): 
      gatherVars(o, i, n.sons[k])
@@ -366,10 +444,11 @@ proc transformInnerProc(o: POuterContext, i: PInnerContext, n: PNode): PNode =
    else:
      # captured symbol?
      result = idNodeTableGet(i.localsToAccess, n.sym)
-  of nkLambdaKinds:
-    result = transformInnerProc(o, i, n.sons[namePos])
+  of nkLambdaKinds, nkIteratorDef:
+    if n.typ != nil:
+      result = transformInnerProc(o, i, n.sons[namePos])
  of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef,
-     nkIteratorDef:
+      nkClosure:
    # don't recurse here:
    discard
  else:
@@ -400,8 +479,9 @@ proc searchForInnerProcs(o: POuterContext, n: PNode) =
      if inner.closureParam != nil:
        let ti = transformInnerProc(o, inner, body)
        if ti != nil: n.sym.ast.sons[bodyPos] = ti
-  of nkLambdaKinds:
-    searchForInnerProcs(o, n.sons[namePos])
+  of nkLambdaKinds, nkIteratorDef:
+    if n.typ != nil:
+      searchForInnerProcs(o, n.sons[namePos])
  of nkWhileStmt, nkForStmt, nkParForStmt, nkBlockStmt:
    # some nodes open a new scope, so they are candidates for the insertion
    # of closure creation; however for simplicity we merge closures between
@@ -438,7 +518,7 @@ proc searchForInnerProcs(o: POuterContext, n: PNode) =
      else:
        internalError(it.info, "transformOuter")
  of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, 
-     nkIteratorDef:
+     nkClosure:
    # don't recurse here:
    # XXX recurse here and setup 'up' pointers
    discard
@@ -463,19 +543,20 @@ proc addVar*(father, v: PNode) =
  addSon(vpart, ast.emptyNode)
  addSon(father, vpart)

-proc getClosureVar(o: POuterContext, e: PEnv): PSym =
-  if e.closure == nil:
-    result = newSym(skVar, getIdent(envName), o.fn, e.attachedNode.info)
-    incl(result.flags, sfShadowed)
-    result.typ = newType(tyRef, o.fn)
-    result.typ.rawAddSon(e.tup)
-    e.closure = result
+proc newClosureCreationVar(o: POuterContext; e: PEnv): PSym =
+  result = newSym(skVar, getIdent(envName), o.fn, e.attachedNode.info)
+  incl(result.flags, sfShadowed)
+  result.typ = newType(tyRef, o.fn)
+  result.typ.rawAddSon(e.tup)
+
+proc getClosureVar(o: POuterContext; e: PEnv): PSym =
+  if e.createdVar == nil:
+    result = newClosureCreationVar(o, e)
+    e.createdVar = result
  else:
-    result = e.closure
-
-proc generateClosureCreation(o: POuterContext, scope: PEnv): PNode =
-  var env = getClosureVar(o, scope)
+    result = e.createdVar

+proc rawClosureCreation(o: POuterContext, scope: PEnv; env: PSym): PNode =
  result = newNodeI(nkStmtList, env.info)
  var v = newNodeI(nkVarSection, env.info)
  addVar(v, newSymNode(env))
@@ -496,6 +577,65 @@ proc generateClosureCreation(o: POuterContext, scope: PEnv): PNode =
    # add ``env.up = env2``
    result.add(newAsgnStmt(indirectAccess(env, field, env.info),
               newSymNode(getClosureVar(o, e)), env.info))
+  
+proc generateClosureCreation(o: POuterContext, scope: PEnv): PNode =
+  var env = getClosureVar(o, scope)
+  result = rawClosureCreation(o, scope, env)
+
+proc generateIterClosureCreation(o: POuterContext; env: PEnv;
+                                 scope: PNode): PSym =
+  result = newClosureCreationVar(o, env)
+  let cc = rawClosureCreation(o, env, result)
+  var insertPoint = scope.sons[0]
+  if insertPoint.kind == nkEmpty: scope.sons[0] = cc
+  else:
+    assert cc.kind == nkStmtList and insertPoint.kind == nkStmtList
+    for x in cc: insertPoint.add(x)
+  if env.createdVar == nil: env.createdVar = result
+
+proc interestingIterVar(s: PSym): bool {.inline.} =
+  result = s.kind in {skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags
+
+proc transformOuterProc(o: POuterContext, n: PNode): PNode
+
+proc transformYield(c: POuterContext, n: PNode): PNode =
+  inc c.state.typ.n.sons[1].intVal
+  let stateNo = c.state.typ.n.sons[1].intVal
+
+  var stateAsgnStmt = newNodeI(nkAsgn, n.info)
+  stateAsgnStmt.add(indirectAccess(newSymNode(c.closureParam),c.state,n.info))
+  stateAsgnStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt)))
+
+  var retStmt = newNodeI(nkReturnStmt, n.info)
+  if n.sons[0].kind != nkEmpty:
+    var a = newNodeI(nkAsgn, n.sons[0].info)
+    var retVal = transformOuterProc(c, n.sons[0])
+    addSon(a, newSymNode(c.resultSym))
+    addSon(a, if retVal.isNil: n.sons[0] else: retVal)
+    retStmt.add(a)
+  else:
+    retStmt.add(emptyNode)
+  
+  var stateLabelStmt = newNodeI(nkState, n.info)
+  stateLabelStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt)))
+  
+  result = newNodeI(nkStmtList, n.info)
+  result.add(stateAsgnStmt)
+  result.add(retStmt)
+  result.add(stateLabelStmt)
+
+proc transformReturn(c: POuterContext, n: PNode): PNode =
+  result = newNodeI(nkStmtList, n.info)
+  var stateAsgnStmt = newNodeI(nkAsgn, n.info)
+  stateAsgnStmt.add(indirectAccess(newSymNode(c.closureParam),c.state,n.info))
+  stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt)))
+  result.add(stateAsgnStmt)
+  result.add(n)
+
+proc outerProcSons(o: POuterContext, n: PNode) =
+  for i in countup(0, sonsLen(n) - 1):
+    let x = transformOuterProc(o, n.sons[i])
+    if x != nil: n.sons[i] = x

 proc transformOuterProc(o: POuterContext, n: PNode): PNode =
  if n == nil: return nil
@@ -503,10 +643,25 @@ proc transformOuterProc(o: POuterContext, n: PNode): PNode =
  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard
  of nkSym:
    var local = n.sym
+
+    if o.isIter and interestingIterVar(local) and o.fn.id == local.owner.id:
+      if not containsOrIncl(o.capturedVars, local.id): addField(o.tup, local)
+      return indirectAccess(newSymNode(o.closureParam), local, n.info)
+
    var closure = PEnv(idTableGet(o.lambdasToEnv, local))
    if closure != nil:
-      # we need to replace the lambda with '(lambda, env)': 
-      let a = closure.closure
+      # we need to replace the lambda with '(lambda, env)':
+      if local.kind == skIterator and local.typ.callConv == ccClosure:
+        # consider: [i1, i2, i1]  Since we merged the iterator's closure
+        # with the captured owning variables, we need to generate the
+        # closure generation code again:
+        #if local == o.fn: message(n.info, errRecursiveDependencyX, local.name.s)
+        # XXX why doesn't this work?
+        let createdVar = generateIterClosureCreation(o, closure,
+                                                     closure.attachedNode)
+        return makeClosure(local, createdVar, n.info)
+      
+      let a = closure.createdVar
      if a != nil:
        return makeClosure(local, a, n.info)
      else:
@@ -516,7 +671,7 @@ proc transformOuterProc(o: POuterContext, n: PNode): PNode =
        if scope.sons[0].kind == nkEmpty:
          # change the empty node to contain the closure construction:
          scope.sons[0] = generateClosureCreation(o, closure)
-        let x = closure.closure
+        let x = closure.createdVar
        assert x != nil
        return makeClosure(local, x, n.info)
    
@@ -535,20 +690,47 @@ proc transformOuterProc(o: POuterContext, n: PNode): PNode =
    assert result != nil, "cannot find: " & local.name.s
    # else it is captured by copy and this means that 'outer' should continue
    # to access the local as a local.
-  of nkLambdaKinds:
-    result = transformOuterProc(o, n.sons[namePos])
-  of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef, 
-     nkIteratorDef: 
+  of nkLambdaKinds, nkIteratorDef:
+    if n.typ != nil:
+      result = transformOuterProc(o, n.sons[namePos])
+  of nkProcDef, nkMethodDef, nkConverterDef, nkMacroDef, nkTemplateDef,
+      nkClosure:
    # don't recurse here:
    discard
  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
    let x = transformOuterProc(o, n.sons[1])
    if x != nil: n.sons[1] = x
    result = transformOuterConv(n)
+  of nkYieldStmt:
+    if o.isIter: result = transformYield(o, n)
+    else: outerProcSons(o, n)
+  of nkReturnStmt:
+    if o.isIter: result = transformReturn(o, n)
+    else: outerProcSons(o, n)
  else:
-    for i in countup(0, sonsLen(n) - 1):
-      let x = transformOuterProc(o, n.sons[i])
-      if x != nil: n.sons[i] = x
+    outerProcSons(o, n)
+
+proc liftIterator(c: POuterContext, body: PNode): PNode =
+  let iter = c.fn
+  result = newNodeI(nkStmtList, iter.info)
+  var gs = newNodeI(nkGotoState, iter.info)
+  gs.add(indirectAccess(newSymNode(c.closureParam), c.state, iter.info))
+  result.add(gs)
+  var state0 = newNodeI(nkState, iter.info)
+  state0.add(newIntNode(nkIntLit, 0))
+  result.add(state0)
+  
+  let newBody = transformOuterProc(c, body)
+  if newBody != nil:
+    result.add(newBody)
+  else:
+    result.add(body)
+
+  var stateAsgnStmt = newNodeI(nkAsgn, iter.info)
+  stateAsgnStmt.add(indirectAccess(newSymNode(c.closureParam),
+                    c.state,iter.info))
+  stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt)))
+  result.add(stateAsgnStmt)

 proc liftLambdas*(fn: PSym, body: PNode): PNode =
  # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
@@ -572,8 +754,11 @@ proc liftLambdas*(fn: PSym, body: PNode): PNode =
    if resultPos < sonsLen(ast) and ast.sons[resultPos].kind == nkSym:
      idTablePut(o.localsToEnv, ast.sons[resultPos].sym, o.currentEnv)
    searchForInnerProcs(o, body)
-    discard transformOuterProc(o, body)
-    result = ex
+    if o.isIter:
+      result = liftIterator(o, ex)
+    else:
+      discard transformOuterProc(o, body)
+      result = ex

 proc liftLambdasForTopLevel*(module: PSym, body: PNode): PNode =
  if body.kind == nkEmpty or gCmd == cmdCompileToJS:
@@ -588,140 +773,15 @@ proc liftLambdasForTopLevel*(module: PSym, body: PNode): PNode =

 # ------------------- iterator transformation --------------------------------

-discard """
-  iterator chain[S, T](a, b: *S->T, args: *S): T =
-    for x in a(args): yield x
-    for x in b(args): yield x
-
-  let c = chain(f, g)
-  for x in c: echo x
-  
-  # translated to:
-  let c = chain( (f, newClosure(f)), (g, newClosure(g)), newClosure(chain))
-"""
-
-type
-  TIterContext {.final, pure.} = object
-    iter, closureParam, state, resultSym: PSym
-    capturedVars: TIntSet
-    tup: PType
-
-proc newIterResult(iter: PSym): PSym =
-  result = iter.ast.sons[resultPos].sym
-  when false:
-    result = newSym(skResult, getIdent":result", iter, iter.info)
-    result.typ = iter.typ.sons[0]
-    incl(result.flags, sfUsed)
-
-proc interestingIterVar(s: PSym): bool {.inline.} =
-  result = s.kind in {skVar, skLet, skTemp, skForVar} and sfGlobal notin s.flags
-
-proc transfIterBody(c: var TIterContext, n: PNode): PNode =
-  # gather used vars for closure generation
-  if n == nil: return nil
-  case n.kind
-  of nkSym:
-    var s = n.sym
-    if interestingIterVar(s) and c.iter.id == s.owner.id:
-      if not containsOrIncl(c.capturedVars, s.id): addField(c.tup, s)
-      result = indirectAccess(newSymNode(c.closureParam), s, n.info)
-  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: discard
-  of nkYieldStmt:
-    inc c.state.typ.n.sons[1].intVal
-    let stateNo = c.state.typ.n.sons[1].intVal
-
-    var stateAsgnStmt = newNodeI(nkAsgn, n.info)
-    stateAsgnStmt.add(indirectAccess(newSymNode(c.closureParam),c.state,n.info))
-    stateAsgnStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt)))
-
-    var retStmt = newNodeI(nkReturnStmt, n.info)
-    if n.sons[0].kind != nkEmpty:
-      var a = newNodeI(nkAsgn, n.sons[0].info)
-      var retVal = transfIterBody(c, n.sons[0])
-      addSon(a, newSymNode(c.resultSym))
-      addSon(a, if retVal.isNil: n.sons[0] else: retVal)
-      retStmt.add(a)
-    else:
-      retStmt.add(emptyNode)
-    
-    var stateLabelStmt = newNodeI(nkState, n.info)
-    stateLabelStmt.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt)))
-    
-    result = newNodeI(nkStmtList, n.info)
-    result.add(stateAsgnStmt)
-    result.add(retStmt)
-    result.add(stateLabelStmt)
-  of nkReturnStmt:
-    result = newNodeI(nkStmtList, n.info)
-    var stateAsgnStmt = newNodeI(nkAsgn, n.info)
-    stateAsgnStmt.add(indirectAccess(newSymNode(c.closureParam),c.state,n.info))
-    stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt)))
-    result.add(stateAsgnStmt)
-    result.add(n)
-  else:
-    for i in countup(0, sonsLen(n)-1):
-      let x = transfIterBody(c, n.sons[i])
-      if x != nil: n.sons[i] = x
-
-proc getStateType(iter: PSym): PType =
-  var n = newNodeI(nkRange, iter.info)
-  addSon(n, newIntNode(nkIntLit, -1))
-  addSon(n, newIntNode(nkIntLit, 0))
-  result = newType(tyRange, iter)
-  result.n = n
-  rawAddSon(result, getSysType(tyInt))
-
-proc liftIterator*(iter: PSym, body: PNode): PNode =
-  var c: TIterContext
-  c.iter = iter
-  c.capturedVars = initIntSet()
-
-  c.tup = newType(tyTuple, iter)
-  c.tup.n = newNodeI(nkRecList, iter.info)
-
-  var cp = newSym(skParam, getIdent(paramName), iter, iter.info)
-  incl(cp.flags, sfFromGeneric)
-  cp.typ = newType(tyRef, iter)
-  rawAddSon(cp.typ, c.tup)
-  c.closureParam = cp
-  addHiddenParam(iter, cp)
-
-  c.state = newSym(skField, getIdent(":state"), iter, iter.info)
-  c.state.typ = getStateType(iter)
-  addField(c.tup, c.state)
-
-  if iter.typ.sons[0] != nil:
-    c.resultSym = newIterResult(iter)
-    iter.ast.add(newSymNode(c.resultSym))
-
-  result = newNodeI(nkStmtList, iter.info)
-  var gs = newNodeI(nkGotoState, iter.info)
-  gs.add(indirectAccess(newSymNode(c.closureParam), c.state, iter.info))
-  result.add(gs)
-  var state0 = newNodeI(nkState, iter.info)
-  state0.add(newIntNode(nkIntLit, 0))
-  result.add(state0)
-  
-  let newBody = transfIterBody(c, body)
-  if newBody != nil:
-    result.add(newBody)
-  else:
-    result.add(body)
-
-  var stateAsgnStmt = newNodeI(nkAsgn, iter.info)
-  stateAsgnStmt.add(indirectAccess(newSymNode(c.closureParam),
-                    c.state,iter.info))
-  stateAsgnStmt.add(newIntTypeNode(nkIntLit, -1, getSysType(tyInt)))
-  result.add(stateAsgnStmt)
-
 proc liftIterSym*(n: PNode): PNode =
  # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env)) 
-  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
  let iter = n.sym
  assert iter.kind == skIterator
+
+  result = newNodeIT(nkStmtListExpr, n.info, n.typ)
+  
  var env = copySym(getHiddenParam(iter))
  env.kind = skLet
-
  var v = newNodeI(nkVarSection, n.info)
  addVar(v, newSymNode(env))
  result.add(v)
@@ -766,7 +826,7 @@ proc liftForLoop*(body: PNode): PNode =
  # static binding?
  var env: PSym
  if call[0].kind == nkSym and call[0].sym.kind == skIterator:
-    # createClose()
+    # createClosure()
    let iter = call[0].sym
    assert iter.kind == skIterator
    env = copySym(getHiddenParam(iter))
--- a/compiler/lexer.nim
+++ b/compiler/lexer.nim
@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2013 Andreas Rumpf
+#        (c) Copyright 2014 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
--- a/compiler/parser.nim
+++ b/compiler/parser.nim
@@ -1,7 +1,7 @@
 #
 #
 #           The Nimrod Compiler
-#        (c) Copyright 2013 Andreas Rumpf
+#        (c) Copyright 2014 Andreas Rumpf
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
@@ -31,7 +31,7 @@ type
  TParser*{.final.} = object  # a TParser object represents a module that
                              # is being parsed
    currInd: int              # current indentation
-    firstTok: bool
+    firstTok, strongSpaces: bool
    lex*: TLexer              # the lexer that is used for parsing
    tok*: TToken              # the current token
    inPragma: int
@@ -1048,6 +1048,7 @@ proc parseTypeDesc(p: var TParser): PNode =

 proc parseTypeDefAux(p: var TParser): PNode = 
  #| typeDefAux = simpleExpr
+  #|            | 'generic' typeClass
  result = simpleExpr(p, pmTypeDef)

 proc makeCall(n: PNode): PNode =
@@ -1208,8 +1209,7 @@ proc parseReturnOrRaise(p: var TParser, kind: TNodeKind): PNode =
  if p.tok.tokType == tkComment:
    skipComment(p, result)
    addSon(result, ast.emptyNode)
-  elif p.tok.indent >= 0 and p.tok.indent <= p.currInd or
-      p.tok.tokType == tkEof:
+  elif p.tok.indent >= 0 and p.tok.indent <= p.currInd or not isExprStart(p):
    # NL terminates:
    addSon(result, ast.emptyNode)
  else:
@@ -1672,6 +1672,9 @@ proc parseTypeClassParam(p: var TParser): PNode =
    result = p.parseSymbol

 proc parseTypeClass(p: var TParser): PNode =
+  #| typeClassParam = ('var')? symbol
+  #| typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')?
+  #|               &IND{>} stmt
  result = newNodeP(nkTypeClassTy, p)
  getTok(p)
  var args = newNode(nkArgList)
--- a/compiler/renderer.nim
+++ b/compiler/renderer.nim
@@ -426,6 +426,8 @@ proc lsub(n: PNode): int =
  of nkVarTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) + len("var")
  of nkDistinctTy: result = (if n.len > 0: lsub(n.sons[0])+1 else: 0) +
                                                         len("Distinct")
+  of nkStaticTy: result = (if n.len > 0: lsub(n.sons[0]) else: 0) +
+                                                         len("static[]")
  of nkTypeDef: result = lsons(n) + 3
  of nkOfInherit: result = lsub(n.sons[0]) + len("of_")
  of nkProcTy: result = lsons(n) + len("proc_")
@@ -701,6 +703,19 @@ proc gproc(g: var TSrcGen, n: PNode) =
      gcoms(g)
      dedent(g)

+proc gTypeClassTy(g: var TSrcGen, n: PNode) =
+  var c: TContext
+  initContext(c)
+  putWithSpace(g, tkGeneric, "generic")
+  gsons(g, n[0], c) # arglist
+  gsub(g, n[1]) # pragmas
+  gsub(g, n[2]) # of
+  gcoms(g)
+  indentNL(g)
+  gcoms(g)
+  gstmts(g, n[3], c)
+  dedent(g)
+
 proc gblock(g: var TSrcGen, n: PNode) = 
  var c: TContext
  initContext(c)
@@ -1054,6 +1069,12 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
      gsub(g, n.sons[0])
    else:
      put(g, tkShared, "shared")
+  of nkStaticTy:
+    put(g, tkStatic, "static")
+    put(g, tkBracketLe, "[")
+    if n.len > 0:
+      gsub(g, n.sons[0])
+    put(g, tkBracketRi, "]")    
  of nkEnumTy:
    if sonsLen(n) > 0:
      putWithSpace(g, tkEnum, "enum")
@@ -1251,6 +1272,13 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
    put(g, tkParLe, "(META|")
    gsub(g, n.sons[0])
    put(g, tkParRi, ")")
+  of nkGotoState, nkState:
+    var c: TContext
+    initContext c
+    putWithSpace g, tkSymbol, if n.kind == nkState: "state" else: "goto"
+    gsons(g, n, c)
+  of nkTypeClassTy:
+    gTypeClassTy(g, n)
  else: 
    #nkNone, nkExplicitTypeListCall: 
    internalError(n.info, "rnimsyn.gsub(" & $n.kind & ')')
--- a/compiler/semdestruct.nim
+++ b/compiler/semdestruct.nim
@@ -116,7 +116,10 @@ proc generateDestructor(c: PContext, t: PType): PNode =
      let stmt = destroyField(c, t.n.sons[s].sym, destructedObj)
      if stmt != nil: addLine(stmt)
    else:
-      internalAssert false
+      # XXX just skip it for now so that the compiler doesn't crash, but
+      # please zahary fix it! arbitrary nesting of nkRecList/nkRecCase is
+      # possible. Any thread example seems to trigger this. 
+      discard
  # base classes' destructors will be automatically called by
  # semProcAux for both auto-generated and user-defined destructors

--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -868,6 +868,8 @@ proc semProcAnnotation(c: PContext, prc: PNode): PNode =
    return semStmt(c, x)

 proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
+  # XXX semProcAux should be good enough for this now, we will eventually
+  # remove semLambda
  result = semProcAnnotation(c, n)
  if result != nil: return result
  result = n
@@ -949,11 +951,13 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
  checkSonsLen(n, bodyPos + 1)
  var s: PSym
  var typeIsDetermined = false
+  var isAnon = false
  if n[namePos].kind != nkSym:
    assert phase == stepRegisterSymbol

    if n[namePos].kind == nkEmpty:
      s = newSym(kind, idAnon, getCurrOwner(), n.info)
+      isAnon = true
    else:
      s = semIdentDef(c, n.sons[0], kind)
    n.sons[namePos] = newSymNode(s)
@@ -996,11 +1000,13 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
    rawAddSon(s.typ, nil)
  if n.sons[patternPos].kind != nkEmpty:
    n.sons[patternPos] = semPattern(c, n.sons[patternPos])
-  if s.kind == skIterator: s.typ.flags.incl(tfIterator)
+  if s.kind == skIterator: 
+    s.typ.flags.incl(tfIterator)
  
  var proto = searchForProc(c, s.scope, s)
  if proto == nil: 
-    s.typ.callConv = lastOptionEntry(c).defaultCC
+    if s.kind == skIterator and isAnon: s.typ.callConv = ccClosure
+    else: s.typ.callConv = lastOptionEntry(c).defaultCC
    # add it here, so that recursive procs are possible:
    if sfGenSym in s.flags: discard
    elif kind in OverloadableSyms:
@@ -1078,6 +1084,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
  popOwner()
  if n.sons[patternPos].kind != nkEmpty:
    c.patterns.add(s)
+  if isAnon: result.typ = s.typ

 proc determineType(c: PContext, s: PSym) =
  if s.typ != nil: return
--- a/compiler/semtypinst.nim
+++ b/compiler/semtypinst.nim
@@ -116,8 +116,8 @@ proc hasGenericArguments*(n: PNode): bool =
           (n.sym.kind == skType and
            n.sym.typ.flags * {tfGenericTypeParam, tfImplicitTypeParam} != {})
  else:
-    for s in n.sons:
-      if hasGenericArguments(s): return true
+    for i in 0.. <n.safeLen:
+      if hasGenericArguments(n.sons[i]): return true
    return false

 proc reResolveCallsWithTypedescParams(cl: var TReplTypeVars, n: PNode): PNode =
--- a/compiler/transf.nim
+++ b/compiler/transf.nim
@@ -113,8 +113,8 @@ proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode =
  result[1] = ri

 proc transformSymAux(c: PTransf, n: PNode): PNode =
-  if n.sym.kind == skIterator and n.sym.typ.callConv == ccClosure:
-    return liftIterSym(n)
+  #if n.sym.kind == skIterator and n.sym.typ.callConv == ccClosure:
+  #  return liftIterSym(n)
  var b: PNode
  var tc = c.transCon
  if sfBorrow in n.sym.flags: 
@@ -636,6 +636,8 @@ proc transform(c: PTransf, n: PNode): PTransNode =
          s.ast.sons[bodyPos] = n.sons[bodyPos]
        #n.sons[bodyPos] = liftLambdas(s, n)
        #if n.kind == nkMethodDef: methodDef(s, false)
+    #if n.kind == nkIteratorDef and n.typ != nil:
+    #  return liftIterSym(n.sons[namePos]).PTransNode
    result = PTransNode(n)
  of nkMacroDef:
    # XXX no proper closure support yet:
@@ -708,6 +710,7 @@ proc transform(c: PTransf, n: PNode): PTransNode =
    # XXX comment handling really sucks:
    if importantComments():
      PNode(result).comment = n.comment
+  of nkClosure: return PTransNode(n)
  else:
    result = transformSons(c, n)
  var cnst = getConstExpr(c.module, PNode(result))
@@ -738,8 +741,8 @@ proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
    var c = openTransf(module, "")
    result = processTransf(c, n, prc)
    result = liftLambdas(prc, result)
-    if prc.kind == skIterator and prc.typ.callConv == ccClosure:
-      result = lambdalifting.liftIterator(prc, result)
+    #if prc.kind == skIterator and prc.typ.callConv == ccClosure:
+    #  result = lambdalifting.liftIterator(prc, result)
    incl(result.flags, nfTransf)
    when useEffectSystem: trackProc(prc, result)

--- a/compiler/vm.nim
+++ b/compiler/vm.nim
@@ -1057,6 +1057,7 @@ proc fixType(result, n: PNode) {.inline.} =
  # XXX do it deeply for complex values; there seems to be no simple
  # solution except to check it deeply here.
  #if result.typ.isNil: result.typ = n.typ
+  discard

 proc execute(c: PCtx, start: int): PNode =
  var tos = PStackFrame(prc: nil, comesFrom: 0, next: nil)
@@ -1118,6 +1119,7 @@ proc evalConstExprAux(module, prc: PSym, n: PNode, mode: TEvalMode): PNode =
  var c = globalCtx
  c.mode = mode
  let start = genExpr(c, n, requiresValue = mode!=emStaticStmt)
+  if c.code[start].opcode == opcEof: return emptyNode
  assert c.code[start].opcode != opcEof
  var tos = PStackFrame(prc: prc, comesFrom: 0, next: nil)
  newSeq(tos.slots, c.prc.maxSlots)
--- a/doc/docgen.txt
+++ b/doc/docgen.txt
@@ -0,0 +1,192 @@
+===================================
+   Nimrod DocGen Tools Guide
+===================================
+
+:Author: Erik O'Leary
+:Version: |nimrodversion|
+
+.. contents::
+
+
+Introduction
+============
+
+This document describes the `documentation generation tools`:idx: built into
+the `Nimrod compiler <nimrodc.html>`_, which can generate HTML and JSON output
+from input .nim files and projects, as well as HTML and LaTeX from input RST
+(reStructuredText) files. The output documentation will include module
+dependencies (``import``), any top-level documentation comments (##), and
+exported symbols (*), including procedures, types, and variables.
+
+
+Documentation Comments
+----------------------
+
+Any comments which are preceded by a double-hash (##), are interpreted as
+documentation.  Comments are parsed as RST (see `reference
+<http://docutils.sourceforge.net/docs/user/rst/quickref.html>`_), providing
+Nimrod module authors the ability to easily generate richly formatted
+documentation with only their well-documented code.
+
+Example:
+
+.. code-block:: nimrod
+  type TPerson* = object
+    ## This type contains a description of a person
+    name: string
+    age: int
+
+Outputs::
+  TPerson* = object
+    name: string
+    age: int
+
+This type contains a description of a person
+
+Field documentation comments can be added to fields like so:
+
+.. code-block:: nimrod
+  var numValues: int ## \
+    ## `numValues` stores the number of values
+
+Note that without the `*` following the name of the type, the documentation for
+this type would not be generated. Documentation will only be generated for
+*exported* types/procedures/etc.
+
+
+Nimrod file input
+-----------------
+
+The following examples will generate documentation for the below contrived
+*Nimrod* module, aptly named 'sample.nim'
+
+sample.nim:
+
+.. code-block:: nimrod
+  ## This module is a sample.
+
+  import strutils
+
+  proc helloWorld*(times: int) =
+    ## Takes an integer and outputs
+    ## as many "hello world!"s
+
+    for i in 0 .. times-1:
+      echo "hello world!"
+
+  helloWorld(5)
+
+
+Document Types
+==============
+
+
+HTML
+----
+
+Generation of HTML documents is done via both the ``doc`` and ``doc2``
+commands. These command take either a single .nim file, outputting a single
+.html file with the same base filename, or multiple .nim files, outputting
+multiple .html files and, optionally, an index file.
+
+The ``doc`` command::
+  nimrod doc sample
+
+Partial Output::
+  ...
+  proc helloWorld*(times: int)
+  ...
+
+Output can be viewed in full here: `docgen_sample.html <docgen_sample.html>`_.
+The next command, called ``doc2``, is very similar to the ``doc`` command, but
+will be run after the compiler performs semantic checking on the input nimrod
+module(s), which allows it to process macros.
+
+The ``doc2`` command::
+  nimrod doc2 sample
+
+Partial Output::
+  ...
+  proc helloWorld(times: int) {.raises: [], tags: [].}
+  ...
+
+The full output can be seen here: `docgen_sample2.html <docgen_sample2.html>`_.
+As you can see, the tool has extracted additional information provided to it by
+the compiler beyond what the ``doc`` command provides, such as pragmas attached
+implicitly by the compiler. This type of information is not available from
+looking at the AST (Abstract Syntax Tree) prior to semantic checking, as the
+``doc`` command does.
+
+
+JSON
+----
+
+Generation of JSON documents is done via the ``jsondoc`` command. This command
+takes in a .nim file, and outputs a .json file with the same base filename.
+Note that this tool is built off of the ``doc`` command, and therefore is
+performed before semantic checking.
+
+The ``jsondoc`` command::
+  nimrod jsondoc sample
+
+Output::
+  [
+    {
+      "comment": "This module is a sample."
+    },
+    {
+      "name": "helloWorld",
+      "type": "skProc",
+      "description": "Takes an integer and outputs as many &quot;hello world!&quot;s",
+      "code": "proc helloWorld*(times: int)"
+    }
+  ]
+
+
+Related Options
+===============
+
+``--project`` switch
+::
+  nimrod doc2 --project sample
+
+This will recursively generate documentation of all nimrod modules imported
+into the input module, including system modules. Be careful with this command,
+as it may end up sprinkling html files all over your filesystem!
+
+
+``--index`` switch
+::
+  nimrod doc2 --index:on sample
+
+This will generate an index of all the exported symbols in the input Nimrod
+module, and put it into a neighboring file with the extension of `.idx`.
+
+
+Other Input Formats
+===================
+
+The *Nimrod compiler* also has support for RST (reStructuredText) files with
+the ``rst2html`` and ``rst2tex`` commands. Documents like this one are
+initially written in a dialect of RST which adds support for nimrod sourcecode
+highlighting with the ``.. code-block:: nimrod`` prefix. ``code-block`` also
+supports highlighting of C++ and some other c-like languages.
+
+Usage::
+  nimrod rst2html docgen.txt
+
+Output::
+  You're reading it!
+
+The input can be viewed here `docgen.txt <docgen.txt>`_. The ``rst2tex``
+command is invoked identically to ``rst2html``, but outputs a .tex file instead
+of .html.
+
+
+Additional Resources
+=========
+
+`Nimrod Compiler User Guide <nimrodc.html#command-line-switches>`_
+
+`RST Quick Reference
+<http://docutils.sourceforge.net/docs/user/rst/quickref.html>`_
--- a/doc/docgen_sample.nim
+++ b/doc/docgen_sample.nim
@@ -0,0 +1,12 @@
+## This module is a sample.
+
+import strutils
+
+proc helloWorld*(times: int) =
+  ## Takes an integer and outputs
+  ## as many "hello world!"s
+
+  for i in 0 .. times-1:
+    echo "hello world!"
+
+helloWorld(5)
--- a/doc/grammar.txt
+++ b/doc/grammar.txt
@@ -97,6 +97,7 @@ primary = typeKeyw typeDescK
        / 'bind' primary
 typeDesc = simpleExpr
 typeDefAux = simpleExpr
+           | 'generic' typeClass
 macroColon = ':' stmt? ( IND{=} 'of' exprList ':' stmt 
                       | IND{=} 'elif' expr ':' stmt
                       | IND{=} 'except' exprList ':' stmt
@@ -163,6 +164,9 @@ objectCase = 'case' identWithPragma ':' typeDesc ':'? COMMENT?
 objectPart = IND{>} objectPart^+IND{=} DED
           / objectWhen / objectCase / 'nil' / declColonEquals
 object = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart
+typeClassParam = ('var')? symbol
+typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')?
+              &IND{>} stmt
 distinct = 'distinct' optInd typeDesc
 typeDef = identWithPragma genericParamList? '=' optInd typeDefAux
            indAndComment?
--- a/doc/manual.txt
+++ b/doc/manual.txt
@@ -2261,8 +2261,8 @@ from different modules having the same name.
  using sdl.SetTimer

 Note that ``using`` only *adds* to the current context, it doesn't remove or
-replace, **neither** does it create a new scope. What this means is that if you
-apply this to multiple variables the compiler will find conflicts in what
+replace, **neither** does it create a new scope. What this means is that if one
+applies this to multiple variables the compiler will find conflicts in what
 variable to use:

 .. code-block:: nimrod
@@ -2275,7 +2275,7 @@ variable to use:
  echo b

 When the compiler reaches the second ``add`` call, both ``a`` and ``b`` could
-be used with the proc, so you get ``Error: expression '(a|b)' has no type (or
+be used with the proc, so one gets ``Error: expression '(a|b)' has no type (or
 is ambiguous)``. To solve this you would need to nest ``using`` with a
 ``block`` statement so as to control the reach of the ``using`` statement.

@@ -2368,8 +2368,8 @@ The `addr`:idx: operator returns the address of an l-value. If the type of the
 location is ``T``, the `addr` operator result is of the type ``ptr T``. An
 address is always an untraced reference. Taking the address of an object that
 resides on the stack is **unsafe**, as the pointer may live longer than the
-object on the stack and can thus reference a non-existing object. You can get
-the address of variables, but you can't use it on variables declared through
+object on the stack and can thus reference a non-existing object. One can get
+the address of variables, but one can't use it on variables declared through
 ``let`` statements:

 .. code-block:: nimrod
@@ -2764,7 +2764,7 @@ First class iterators
 There are 2 kinds of iterators in Nimrod: *inline* and *closure* iterators.
 An `inline iterator`:idx: is an iterator that's always inlined by the compiler 
 leading to zero overhead for the abstraction, but may result in a heavy
-increasee in code size. Inline iterators are second class
+increase in code size. Inline iterators are second class
 citizens; one cannot pass them around like first class procs.

 In contrast to that, a `closure iterator`:idx: can be passed around:
@@ -2835,7 +2835,24 @@ a `collaborative tasking`:idx: system:

 The builtin ``system.finished`` can be used to determine if an iterator has
 finished its operation; no exception is raised on an attempt to invoke an
-iterator that has already finished its work. 
+iterator that has already finished its work.
+
+Closure iterators are *resumable functions* and so one has to provide the
+arguments to every call. To get around this limitation one can capture
+parameters of an outer factory proc:
+
+.. code-block:: nimrod
+  proc mycount(a, b: int): iterator (): int =
+    return iterator (): int =
+      var x = a
+      while x <= b:
+        yield x
+        inc x
+
+  let foo = mycount 1, 4
+
+  for f in foo():
+    echo f


 Type sections
@@ -2923,9 +2940,9 @@ in an implicit try block:
  finally: close(f)
  ...

-The ``except`` statement has a limitation in this form: you can't specify the
-type of the exception, you have to catch everything. Also, if you want to use
-both ``finally`` and ``except`` you need to reverse the usual sequence of the
+The ``except`` statement has a limitation in this form: one can't specify the
+type of the exception, one has to catch everything. Also, if one wants to use
+both ``finally`` and ``except`` one needs to reverse the usual sequence of the
 statements. Example:

 .. code-block:: nimrod
@@ -3353,7 +3370,7 @@ currently matched type. These instances can act both as variables of the type,
 when used in contexts, where a value is expected, and as the type itself, when
 used in a contexts, where a type is expected.

-Please note that the ``is`` operator allows you to easily verify the precise
+Please note that the ``is`` operator allows one to easily verify the precise
 type signatures of the required operations, but since type inference and
 default parameters are still applied in the provided block, it's also possible
 to encode usage protocols that doesn't reveal implementation details.
--- a/doc/nimrodc.txt
+++ b/doc/nimrodc.txt
@@ -538,6 +538,13 @@ on Linux::
  nimrod c --dynlibOverride:lua --passL:liblua.lib program.nim


+Nimrod documentation tools
+==========================
+
+Nimrod provides the `doc`:idx: and `doc2`:idx: commands to generate HTML
+documentation from ``.nim`` source files. Only exported symbols will appear in
+the output. For more details `see the docgen documentation <docgen.html>`_.
+
 Nimrod idetools integration
 ===========================

--- a/koch.nim
+++ b/koch.nim
@@ -42,7 +42,7 @@ Possible Commands:
  csource [options]        builds the C sources for installation
  zip                      builds the installation ZIP package
  inno [options]           builds the Inno Setup installer (for Windows)
-  tests                    run the testsuite
+  tests [options]          run the testsuite
  update                   updates nimrod to the latest version from github
                           (compile koch with -d:withUpdate to enable)
  temp options             creates a temporary compiler for testing
@@ -260,11 +260,14 @@ when defined(withUpdate):

 # -------------- tests --------------------------------------------------------

+template `|`(a, b): expr = (if a.len > 0: a else: b)
+
 proc tests(args: string) =
  # we compile the tester with taintMode:on to have a basic
  # taint mode test :-)
-  exec("nimrod cc --taintMode:on tests/testament/tester")
-  exec(getCurrentDir() / "tests/testament/tester".exe & " all")
+  exec "nimrod cc --taintMode:on tests/testament/tester"
+  exec quoteShell(getCurrentDir() / "tests/testament/tester".exe) & " " &
+      (args|"all")

 proc temp(args: string) =
  var output = "compiler" / "nimrod".exe
--- a/lib/core/macros.nim
+++ b/lib/core/macros.nim
@@ -390,7 +390,7 @@ proc treeRepr*(n: PNimrodNode): string {.compileTime.} =
    res.add(($n.kind).substr(3))

    case n.kind
-    of nnkEmpty: nil # same as nil node in this representation
+    of nnkEmpty: discard # same as nil node in this representation
    of nnkNilLit: res.add(" nil")
    of nnkCharLit..nnkInt64Lit: res.add(" " & $n.intVal)
    of nnkFloatLit..nnkFloat64Lit: res.add(" " & $n.floatVal)
@@ -415,7 +415,7 @@ proc lispRepr*(n: PNimrodNode): string {.compileTime.} =
  add(result, "(")

  case n.kind
-  of nnkEmpty: nil # same as nil node in this representation
+  of nnkEmpty: discard # same as nil node in this representation
  of nnkNilLit: add(result, "nil")
  of nnkCharLit..nnkInt64Lit: add(result, $n.intVal)
  of nnkFloatLit..nnkFloat64Lit: add(result, $n.floatVal)
--- a/lib/pure/asyncio.nim
+++ b/lib/pure/asyncio.nim
@@ -139,27 +139,27 @@ type
 proc newDelegate*(): PDelegate =
  ## Creates a new delegate.
  new(result)
-  result.handleRead = (proc (h: PObject) = nil)
-  result.handleWrite = (proc (h: PObject) = nil)
-  result.handleError = (proc (h: PObject) = nil)
+  result.handleRead = (proc (h: PObject) = discard)
+  result.handleWrite = (proc (h: PObject) = discard)
+  result.handleError = (proc (h: PObject) = discard)
  result.hasDataBuffered = (proc (h: PObject): bool = return false)
-  result.task = (proc (h: PObject) = nil)
+  result.task = (proc (h: PObject) = discard)
  result.mode = fmRead

 proc newAsyncSocket(): PAsyncSocket =
  new(result)
  result.info = SockIdle

-  result.handleRead = (proc (s: PAsyncSocket) = nil)
+  result.handleRead = (proc (s: PAsyncSocket) = discard)
  result.handleWrite = nil
-  result.handleConnect = (proc (s: PAsyncSocket) = nil)
-  result.handleAccept = (proc (s: PAsyncSocket) = nil)
-  result.handleTask = (proc (s: PAsyncSocket) = nil)
+  result.handleConnect = (proc (s: PAsyncSocket) = discard)
+  result.handleAccept = (proc (s: PAsyncSocket) = discard)
+  result.handleTask = (proc (s: PAsyncSocket) = discard)

  result.lineBuffer = "".TaintedString
  result.sendBuffer = ""

-proc AsyncSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM, 
+proc asyncSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM, 
                  protocol: TProtocol = IPPROTO_TCP, 
                  buffered = true): PAsyncSocket =
  ## Initialises an AsyncSocket object. If a socket cannot be initialised
--- a/lib/pure/ftpclient.nim
+++ b/lib/pure/ftpclient.nim
@@ -95,7 +95,7 @@ type
  EInvalidReply* = object of ESynch
  EFTP* = object of ESynch

-proc FTPClient*(address: string, port = TPort(21),
+proc ftpClient*(address: string, port = TPort(21),
                user, pass = ""): PFTPClient =
  ## Create a ``PFTPClient`` object.
  new(result)
@@ -315,7 +315,7 @@ proc listDirs*(ftp: PFTPClient, dir: string = "",
  assertReply ftp.send("NLST " & dir.normalizePathSep), ["125", "150"]

  if not async:
-    while not ftp.job.prc(ftp, false): nil
+    while not ftp.job.prc(ftp, false): discard
    result = splitLines(ftp.job.lines)
    ftp.deleteJob()
  else: return @[]
@@ -390,7 +390,7 @@ proc list*(ftp: PFTPClient, dir: string = "", async = false): string =
  assertReply(ftp.send("LIST" & " " & dir.normalizePathSep), ["125", "150"])

  if not async:
-    while not ftp.job.prc(ftp, false): nil
+    while not ftp.job.prc(ftp, false): discard
    result = ftp.job.lines
    ftp.deleteJob()
  else:
@@ -405,7 +405,7 @@ proc retrText*(ftp: PFTPClient, file: string, async = false): string =
  assertReply ftp.send("RETR " & file.normalizePathSep), ["125", "150"]
  
  if not async:
-    while not ftp.job.prc(ftp, false): nil
+    while not ftp.job.prc(ftp, false): discard
    result = ftp.job.lines
    ftp.deleteJob()
  else:
@@ -460,7 +460,7 @@ proc retrFile*(ftp: PFTPClient, file, dest: string, async = false) =
  ftp.job.filename = file.normalizePathSep

  if not async:
-    while not ftp.job.prc(ftp, false): nil
+    while not ftp.job.prc(ftp, false): discard
    ftp.deleteJob()

 proc doUpload(ftp: PFTPClient, async = false): bool =
@@ -518,7 +518,7 @@ proc store*(ftp: PFTPClient, file, dest: string, async = false) =
  assertReply ftp.send("STOR " & dest.normalizePathSep), ["125", "150"]

  if not async:
-    while not ftp.job.prc(ftp, false): nil
+    while not ftp.job.prc(ftp, false): discard
    ftp.deleteJob()

 proc close*(ftp: PFTPClient) =
@@ -554,10 +554,10 @@ proc csockHandleRead(s: PAsyncSocket, ftp: PAsyncFTPClient) =
    
    ftp.handleEvent(ftp, r)

-proc AsyncFTPClient*(address: string, port = TPort(21),
+proc asyncFTPClient*(address: string, port = TPort(21),
                     user, pass = "",
    handleEvent: proc (ftp: PAsyncFTPClient, ev: TFTPEvent) {.closure.} = 
-      (proc (ftp: PAsyncFTPClient, ev: TFTPEvent) = nil)): PAsyncFTPClient =
+      (proc (ftp: PAsyncFTPClient, ev: TFTPEvent) = discard)): PAsyncFTPClient =
  ## Create a ``PAsyncFTPClient`` object.
  ##
  ## Use this if you want to use asyncio's dispatcher.
@@ -604,7 +604,7 @@ when isMainModule:
        ftp.close()
        echo d.len
      else: assert(false)
-  var ftp = AsyncFTPClient("picheta.me", user = "test", pass = "asf", handleEvent = hev)
+  var ftp = asyncFTPClient("picheta.me", user = "test", pass = "asf", handleEvent = hev)
  
  d.register(ftp)
  d.len.echo()
@@ -618,7 +618,7 @@ when isMainModule:


 when isMainModule and false:
-  var ftp = FTPClient("picheta.me", user = "asdasd", pass = "asfwq")
+  var ftp = ftpClient("picheta.me", user = "asdasd", pass = "asfwq")
  ftp.connect()
  echo ftp.pwd()
  echo ftp.list()
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -1586,7 +1586,7 @@ proc getAppFilename*(): string {.rtl, extern: "nos$1", tags: [FReadIO].} =
    # little heuristic that may work on other POSIX-like systems:
    result = string(getEnv("_"))
    if len(result) == 0:
-      result = string(ParamStr(0))
+      result = string(paramStr(0))
      # POSIX guaranties that this contains the executable
      # as it has been executed by the calling process
      if len(result) > 0 and result[0] != DirSep: # not an absolute path?
--- a/lib/pure/osproc.nim
+++ b/lib/pure/osproc.nim
@@ -660,8 +660,8 @@ elif not defined(useNimRtl):

    else:
    
-      Pid = fork()
-      if Pid < 0: osError(osLastError())
+      pid = fork()
+      if pid < 0: osError(osLastError())
      if pid == 0:
        ## child process:

@@ -685,14 +685,14 @@ elif not defined(useNimRtl):
          if env == nil:
            discard execv(command, a)
          else:
-            discard execve(command, a, ToCStringArray(env))
+            discard execve(command, a, toCStringArray(env))
        else:
          var x = addCmdArgs(command, args)
          var a = toCStringArray(["sh", "-c"], [x])
          if env == nil:
            discard execv("/bin/sh", a)
          else:
-            discard execve("/bin/sh", a, ToCStringArray(env))
+            discard execve("/bin/sh", a, toCStringArray(env))
        # too risky to raise an exception here:
        quit("execve call failed: " & $strerror(errno))
    # Parent process. Copy process information.
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -1330,7 +1330,7 @@ iterator `||`*[S, T](a: S, b: T, annotation=""): T {.
  ## such isn't aware of the parallelism in your code! Be careful! Later
  ## versions of ``||`` will get proper support by Nimrod's code generator
  ## and GC.
-  nil
+  discard

 {.push stackTrace:off.}
 proc min*(x, y: int): int {.magic: "MinI", noSideEffect.} =
--- a/lib/system/excpt.nim
+++ b/lib/system/excpt.nim
@@ -11,7 +11,7 @@
 # use the heap (and nor exceptions) do not include the GC or memory allocator.

 var
-  errorMessageWriter*: (proc(msg: string): void {.tags: [FWriteIO].})
+  errorMessageWriter*: (proc(msg: string) {.tags: [FWriteIO].})
    ## Function that will be called
    ## instead of stdmsg.write when printing stacktrace.
    ## Unstable API.
@@ -80,9 +80,9 @@ when defined(nativeStacktrace) and nativeStackTraceSupported:
  type
    TDl_info {.importc: "Dl_info", header: "<dlfcn.h>", 
               final, pure.} = object
-      dli_fname: CString
+      dli_fname: cstring
      dli_fbase: pointer
-      dli_sname: CString
+      dli_sname: cstring
      dli_saddr: pointer

  proc backtrace(symbols: ptr pointer, size: int): int {.
--- a/tests/closure/tnamedparamanonproc.nim
+++ b/tests/closure/tnamedparamanonproc.nim
@@ -4,8 +4,8 @@ type
  TButtonClicked = proc(button: PButton) {.nimcall.}

 proc newButton*(onClick: TButtonClicked) =
-  nil
-  
+  discard
+
 proc main() =
  newButton(onClick = proc(b: PButton) =
    var requestomat = 12
--- a/tests/controlflow/tblock1.nim
+++ b/tests/controlflow/tblock1.nim
--- a/tests/controlflow/tnestif.nim
+++ b/tests/controlflow/tnestif.nim
--- a/tests/iter/tanoniter1.nim
+++ b/tests/iter/tanoniter1.nim
@@ -0,0 +1,32 @@
+discard """
+  output: '''1
+2
+3
+4
+1
+2'''
+"""
+
+proc factory(a, b: int): iterator (): int =
+  iterator foo(): int =
+    var x = a
+    while x <= b:
+      yield x
+      inc x
+  return foo
+
+proc factory2(a, b: int): iterator (): int =
+  return iterator (): int =
+    var x = a
+    while x <= b:
+      yield x
+      inc x
+
+let foo = factory 1, 4
+
+for f in foo():
+  echo f
+
+let foo2 = factory2 1,2
+
+for f in foo2(): echo f
--- a/tests/iter/titer2.nim
+++ b/tests/iter/titer2.nim
@@ -1,6 +1,6 @@
 discard """
  output: '''true'''
-  cmd: "nimrod cc --gc:none --hints:on $# $#"
+  cmd: "nimrod cc --gc:none --hints:on --warnings:off $# $#"
 """

 import hashes
--- a/tests/metatype/tstaticparams.nim
+++ b/tests/metatype/tstaticparams.nim
--- a/tests/objects/tobjconstr.nim
+++ b/tests/objects/tobjconstr.nim
--- a/tests/objects/tobjconstr2.nim
+++ b/tests/objects/tobjconstr2.nim
--- a/tests/objects/tobjcov.nim
+++ b/tests/objects/tobjcov.nim
--- a/tests/objects/tobject.nim
+++ b/tests/objects/tobject.nim
--- a/tests/objects/tobject2.nim
+++ b/tests/objects/tobject2.nim
--- a/tests/objects/tobject3.nim
+++ b/tests/objects/tobject3.nim
--- a/tests/operator/tofopr.nim
+++ b/tests/operator/tofopr.nim
--- a/tests/objects/toop.nim
+++ b/tests/objects/toop.nim
--- a/tests/objects/toop1.nim
+++ b/tests/objects/toop1.nim
--- a/tests/operator/toprprec.nim
+++ b/tests/operator/toprprec.nim
--- a/tests/operator/tprecedence.nim
+++ b/tests/operator/tprecedence.nim
--- a/tests/important/tdrdobbs_examples.nim
+++ b/tests/important/tdrdobbs_examples.nim
--- a/todo.txt
+++ b/todo.txt
@@ -1,19 +1,8 @@
 version 0.9.4
 =============

- test&finish first class iterators:
-  * nested iterators
- ensure (ref T)(a, b) works as a type conversion and type constructor
+- better debugging support for writes to locations
 - document new templating symbol binding rules
- make '--implicitStatic:on' the default
- change comment handling in the AST
-
- special rule for ``[]=``
- ``=`` should be overloadable; requires specialization for ``=``; general
-  lift mechanism in the compiler is already implemented for 'fields'
- built-in 'getImpl'
- optimize 'genericReset'; 'newException' leads to code bloat
- stack-less GC
 - fix eval in macros.nim


@@ -36,12 +25,29 @@ Bugs
 version 0.9.x
 =============

- macros as type pragmas
+- ensure (ref T)(a, b) works as a type conversion and type constructor
+- optimize 'genericReset'; 'newException' leads to code bloat
+- stack-less GC
+- implement strongSpaces:on
+- make '--implicitStatic:on' the default
 - implicit deref for parameter matching
+
+- special rule for ``[]=``
+- ``=`` should be overloadable; requires specialization for ``=``; general
+  lift mechanism in the compiler is already implemented for 'fields'
+- built-in 'getImpl'
+
+- change comment handling in the AST; that's lots of work as c2nim and pas2nim
+  make use of the fast every node can have a comment!
+
+
+version 0.9.X
+=============
+
+- macros as type pragmas
 - lazy overloading resolution:
  * special case ``tyStmt``
 - FFI:
-  * test libffi on windows
  * test: times.format with the FFI
 - document NimMain and check whether it works for threading
 - 'quote' without 'do' doesn't work: parser/grammar issue; could be supported
--- a/tools/nimweb.nim
+++ b/tools/nimweb.nim
@@ -204,6 +204,18 @@ proc exec(cmd: string) =
  echo(cmd)
  if os.execShellCmd(cmd) != 0: quit("external program failed")

+proc buildDocSamples(c: var TConfigData, destPath: string) =
+  ## Special case documentation sample proc.
+  ##
+  ## The docgen sample needs to be generated twice with different commands, so
+  ## it didn't make much sense to integrate into the existing generic
+  ## documentation builders.
+  const src = "doc"/"docgen_sample.nim"
+  Exec("nimrod doc $# -o:$# $#" %
+    [c.nimrodArgs, destPath / "docgen_sample.html", src])
+  Exec("nimrod doc2 $# -o:$# $#" %
+    [c.nimrodArgs, destPath / "docgen_sample2.html", src])
+
 proc buildDoc(c: var TConfigData, destPath: string) =
  # call nim for the documentation:
  for d in items(c.doc):
@@ -352,7 +364,9 @@ proc main(c: var TConfigData) =
  copyDir("web/assets", "web/upload/assets")
  buildNewsRss(c, "web/upload")
  buildAddDoc(c, "web/upload")
+  buildDocSamples(c, "web/upload")
  buildDoc(c, "web/upload")
+  buildDocSamples(c, "doc")
  buildDoc(c, "doc")
  buildPdfDoc(c, "doc")

--- a/web/news.txt
+++ b/web/news.txt
@@ -74,6 +74,8 @@ Language Additions
  evaluable at compile-time.
 - Support for user-defined type classes has been added.
 - The *command syntax* is supported in a lot more contexts.
+- Anonymous iterators are now supported and iterators can capture variables
+  of an outer proc.


 Tools improvements
--- a/web/nimrod.ini
+++ b/web/nimrod.ini
@@ -37,7 +37,7 @@ UNIX. We don't believe this to be a coincidence. - Jeremy S. Anderson."""

 [Documentation]
 doc: "endb;intern;apis;lib;manual;tut1;tut2;nimrodc;overview;filters;trmacros"
-doc: "tools;c2nim;niminst;nimgrep;gc;estp;idetools"
+doc: "tools;c2nim;niminst;nimgrep;gc;estp;idetools;docgen"
 pdf: "manual;lib;tut1;tut2;nimrodc;c2nim;niminst;gc"
 srcdoc2: "system.nim;impure/graphics;wrappers/sdl"
 srcdoc2: "core/macros;pure/marshal;core/typeinfo;core/unsigned"