top level closures should work; transf is not a pass anymore; next steps for first class iterator support

2026-07-22 00:41:28 +00:00 · 2012-08-13 17:07:49 +02:00
parent f686647f58
commit 244c14db0b
17 changed files with 293 additions and 229 deletions
--- a/compiler/ast.nim
+++ b/compiler/ast.nim
@@ -340,7 +340,8 @@ type
                      # because for instantiations of objects, structural
                      # type equality has to be used
    tfAll,            # type class requires all constraints to be met (default)
-    tfAny             # type class requires any constraint to be met
+    tfAny,            # type class requires any constraint to be met
+    tfCapturesEnv     # whether proc really captures some environment

  TTypeFlags* = set[TTypeFlag]

@@ -867,6 +868,7 @@ proc newNode*(kind: TNodeKind, info: TLineInfo, sons: TNodeSeq = @[],
  result = newNode(kind)
  result.info = info
  result.typ = typ
+  # XXX use shallowCopy here for ownership transfer:
  result.sons = sons

 proc newNodeIT(kind: TNodeKind, info: TLineInfo, typ: PType): PNode = 
--- a/compiler/ccgstmts.nim
+++ b/compiler/ccgstmts.nim
@@ -93,10 +93,9 @@ template preserveBreakIdx(body: stmt): stmt =
  p.breakIdx = oldBreakIdx

 proc genState(p: BProc, n: PNode) =
-  internalAssert n.len == 2 and n.sons[0].kind == nkIntLit
+  internalAssert n.len == 1 and n.sons[0].kind == nkIntLit
  let idx = n.sons[0].intVal
  lineCg(p, cpsStmts, "STATE$1: ;$n", [idx.toRope])
-  genStmts(p, n.sons[1])

 proc genGotoState(p: BProc, n: PNode) =
  # we resist the temptation to translate it into duff's device as it later
--- a/compiler/cgen.nim
+++ b/compiler/cgen.nim
@@ -627,7 +627,7 @@ proc deinitFrame(p: BProc): PRope =
  result = ropecg(p.module, "\t#popFrame();$n")

 proc closureSetup(p: BProc, prc: PSym) =
-  if prc.typ.callConv != ccClosure: return
+  if tfCapturesEnv notin prc.typ.flags: return
  # prc.ast[paramsPos].last contains the type we're after:
  var ls = lastSon(prc.ast[paramsPos])
  if ls.kind != nkSym:
--- a/compiler/evals.nim
+++ b/compiler/evals.nim
@@ -1366,7 +1366,8 @@ proc evalAux(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode =
  inc(gNestedEvals)

 proc tryEval(c: PEvalContext, n: PNode): PNode =
-  var n = transform(c.module, n)
+  #internalAssert nfTransf in n.flags
+  var n = transformExpr(c.module, n)
  gWhileCounter = evalMaxIterations
  gNestedEvals = evalMaxRecDepth
  result = evalAux(c, n, {})
--- a/compiler/lambdalifting.nim
+++ b/compiler/lambdalifting.nim
@@ -211,6 +211,7 @@ proc addHiddenParam(routine: PSym, param: PSym) =
  var params = routine.ast.sons[paramsPos]
  param.position = params.len
  addSon(params, newSymNode(param))
+  incl(routine.typ.flags, tfCapturesEnv)
  #echo "produced environment: ", param.id, " for ", routine.name.s

 proc isInnerProc(s, outerProc: PSym): bool {.inline.} =
@@ -234,12 +235,22 @@ proc dummyClosureParam(o: POuterContext, i: PInnerContext) =
    IdTablePut(o.lambdasToEnv, i.fn, e)
  if i.closureParam == nil: addClosureParam(i, e)

+proc illegalCapture(s: PSym): bool {.inline.} =
+  result = skipTypes(s.typ, abstractInst).kind in {tyVar, tyOpenArray} or
+      s.kind == skResult
+
 proc captureVar(o: POuterContext, i: PInnerContext, local: PSym, 
                info: TLineInfo) =
  # for inlined variables the owner is still wrong, so it can happen that it's
  # not a captured variable at all ... *sigh* 
  var it = PEnv(IdTableGet(o.localsToEnv, local))
  if it == nil: return
+  
+  if illegalCapture(local) or o.fn.id != local.owner.id:
+    # Currently captures are restricted to a single level of nesting:
+    LocalError(info, errIllegalCaptureX, local.name.s)
+  i.fn.typ.callConv = ccClosure
+  incl(i.fn.typ.flags, tfCapturesEnv)

  # we need to remember which inner most closure belongs to this lambda:
  var e = o.currentEnv
@@ -269,12 +280,13 @@ proc interestingVar(s: PSym): bool {.inline.} =

 proc gatherVars(o: POuterContext, i: PInnerContext, n: PNode) = 
  # gather used vars for closure generation
+  if n == nil: return
  case n.kind
  of nkSym:
    var s = n.sym
    if interestingVar(s) and i.fn.id != s.owner.id:
      captureVar(o, i, s, n.info)
-    elif isInnerProc(s, o.fn) and s.typ.callConv == ccClosure and s != i.fn:
+    elif isInnerProc(s, o.fn) and tfCapturesEnv in s.typ.flags and s != i.fn:
      # call to some other inner proc; we need to track the dependencies for
      # this:
      let env = PEnv(IdTableGet(o.lambdasToEnv, i.fn))
@@ -287,6 +299,28 @@ proc gatherVars(o: POuterContext, i: PInnerContext, n: PNode) =
    for k in countup(0, sonsLen(n) - 1): 
      gatherVars(o, i, n.sons[k])

+proc generateThunk(prc: PNode, dest: PType): PNode =
+  ## Converts 'prc' into '(thunk, nil)' so that it's compatible with
+  ## a closure.
+  
+  # we cannot generate a proper thunk here for GC-safety reasons (see internal
+  # documentation):
+  if gCmd == cmdCompileToEcmaScript: return prc
+  result = newNodeIT(nkClosure, prc.info, dest)
+  var conv = newNodeIT(nkHiddenStdConv, prc.info, dest)
+  conv.add(emptyNode)
+  conv.add(prc)
+  result.add(conv)
+  result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil)))
+
+proc transformOuterConv(n: PNode): PNode =
+  # numeric types need range checks:
+  var dest = skipTypes(n.typ, abstractVarRange)
+  var source = skipTypes(n.sons[1].typ, abstractVarRange)
+  if dest.kind == tyProc:
+    if dest.callConv == ccClosure and source.callConv == ccDefault:
+      result = generateThunk(n.sons[1], dest)
+
 proc makeClosure(prc, env: PSym, info: TLineInfo): PNode =
  result = newNodeIT(nkClosure, info, prc.typ)
  result.add(newSymNode(prc))
@@ -339,6 +373,7 @@ proc searchForInnerProcs(o: POuterContext, n: PNode) =
      gatherVars(o, inner, body)
      # dummy closure param needed?
      if inner.closureParam == nil and n.sym.typ.callConv == ccClosure:
+        assert tfCapturesEnv notin n.sym.typ.flags
        dummyClosureParam(o, inner)
      # only transform if it really needs a closure:
      if inner.closureParam != nil:
@@ -437,7 +472,7 @@ proc generateClosureCreation(o: POuterContext, scope: PEnv): PNode =
               newSymNode(getClosureVar(o, e))))

 proc transformOuterProc(o: POuterContext, n: PNode): PNode =
-  # XXX I with I knew where these 'nil' nodes come from: 'array[.. |X]'
+  # XXX I wish I knew where these 'nil' nodes come from: 'array[.. |X]'
  if n == nil: return nil
  case n.kind
  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: nil
@@ -481,17 +516,20 @@ proc transformOuterProc(o: POuterContext, n: PNode): PNode =
     nkIteratorDef: 
    # don't recurse here:
    nil
+  of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+    let x = transformOuterProc(o, n.sons[1])
+    if x != nil: n.sons[1] = x
+    result = transformOuterConv(n)
  else:
    for i in countup(0, sonsLen(n) - 1):
      let x = transformOuterProc(o, n.sons[i])
      if x != nil: n.sons[i] = x

 proc liftLambdas*(fn: PSym, body: PNode): PNode =
-  if body.kind == nkEmpty:
+  if body.kind == nkEmpty or gCmd == cmdCompileToEcmaScript:
    # ignore forward declaration:
    result = body
-  elif (fn.typ == nil or fn.typ.callConv != ccClosure) and 
-      not containsNode(body, procDefs):
+  elif not containsNode(body, procDefs):
    # fast path: no inner procs, so no closure needed:
    result = body
  else:
@@ -506,40 +544,144 @@ proc liftLambdas*(fn: PSym, body: PNode): PNode =
      let param = params.sons[i].sym
      IdTablePut(o.localsToEnv, param, o.currentEnv)
    searchForInnerProcs(o, body)
-    let a = transformOuterProc(o, body)
+    discard transformOuterProc(o, body)
    result = ex
-  
-# XXX should 's' be replaced by a tuple ('s', env)?

-proc liftLambdas*(n: PNode): PNode =
-  assert n.kind in procDefs
-  var s = n.sons[namePos].sym
-  if gCmd == cmdCompileToEcmaScript: return s.getBody
-  result = liftLambdas(s, s.getBody)
-
-proc transformIterator*(fn: PSym, body: PNode): PNode =
-  if body.kind == nkEmpty:
-    # ignore forward declaration:
+proc liftLambdasForTopLevel*(module: PSym, body: PNode): PNode =
+  if body.kind == nkEmpty or gCmd == cmdCompileToEcmaScript:
    result = body
-  # it(a, b) --> (it(a, b), createClosure())
-  # it(a, b) --> ?
-  discard """
-  let c = chain(f, g)
-  
-  for x in c: echo x
-  
+  else:
+    var o = newOuterContext(module)
+    let ex = closureCreationPoint(body)
+    o.currentEnv = newEnv(module, nil, ex)
+    searchForInnerProcs(o, body)
+    discard transformOuterProc(o, body)
+    result = ex
+
+# ------------------- iterator transformation --------------------------------
+
+discard """
  iterator chain[S, T](a, b: *S->T, args: *S): T =
    for x in a(args): yield x
    for x in b(args): yield x
+
+  let c = chain(f, g)
+  for x in c: echo x
  
  # translated to:
-  
-  
-  
  let c = chain( (f, newClosure(f)), (g, newClosure(g)), newClosure(chain))
-  
-  
 """

+type
+  TIterContext {.final, pure.} = object
+    iter, closureParam, state, resultSym: PSym
+    capturedVars: TIntSet
+    tup: PType

+proc newIterResult(iter: PSym): PSym =
+  result = newSym(skResult, getIdent":result", iter)
+  result.info = iter.info
+  result.typ = iter.typ.sons[0]
+  incl(result.flags, sfUsed)

+proc transfIterBody(c: var TIterContext, n: PNode): PNode =
+  # gather used vars for closure generation
+  if n == nil: return nil
+  case n.kind
+  of nkSym:
+    var s = n.sym
+    if interestingVar(s) and c.iter.id == s.owner.id:
+      if not containsOrIncl(c.capturedVars, s.id): addField(c.tup, s)
+      result = indirectAccess(newSymNode(c.closureParam), s, n.info)
+  of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: nil
+  of nkYieldStmt:
+    inc c.state.typ.n.sons[1].intVal
+    let stateNo = c.state.typ.n.sons[1].intVal
+
+    var stateAsgnStmt = newNodeI(nkAsgn, n.info)
+    stateAsgnStmt.add(indirectAccess(newSymNode(c.closureParam),c.state,n.info))
+    stateAsgnStmt.add(newIntNode(nkIntLit, stateNo))
+
+    var retStmt = newNodeI(nkReturnStmt, n.info)
+    if n.sons[0].kind != nkEmpty:
+      var a = newNodeI(nkAsgn, n.sons[0].info)
+      addSon(a, newSymNode(c.resultSym))
+      addSon(a, n.sons[0])
+      retStmt.add(a)
+    else:
+      retStmt.add(emptyNode)
+    
+    var stateLabelStmt = newNodeI(nkState, n.info)
+    stateLabelStmt.add(newIntNode(nkIntLit, stateNo-1))
+    
+    result = newNodeI(nkStmtList, n.info)
+    result.add(stateAsgnStmt)
+    result.add(retStmt)
+    result.add(stateLabelStmt)
+  else:
+    for i in countup(0, sonsLen(n)-1):
+      let x = transfIterBody(c, n.sons[i])
+      if x != nil: n.sons[i] = x
+
+proc getStateType(iter: PSym): PType =
+  var n = newNodeI(nkRange, iter.info)
+  addSon(n, newIntNode(nkIntLit, -1))
+  addSon(n, newIntNode(nkIntLit, 0))
+  result = newType(tyRange, iter)
+  result.n = n
+  rawAddSon(result, getSysType(tyInt))
+
+proc liftIterator*(iter: PSym, body: PNode): PNode =
+  var c: TIterContext
+  c.iter = iter
+  c.capturedVars = initIntSet()
+
+  c.tup = newType(tyTuple, iter)
+  c.tup.n = newNodeI(nkRecList, iter.info)
+
+  var cp = newSym(skParam, getIdent(paramname), iter)
+  cp.info = iter.info
+  incl(cp.flags, sfFromGeneric)
+  cp.typ = newType(tyRef, iter)
+  rawAddSon(cp.typ, c.tup)
+  c.closureParam = cp
+  addHiddenParam(iter, cp)
+
+  c.state = newSym(skField, getIdent(":state"), iter)
+  c.state.typ = getStateType(iter)
+  addField(c.tup, c.state)
+
+  if iter.typ.sons[0] != nil:
+    c.resultSym = newIterResult(iter)
+    iter.ast.add(newSymNode(c.resultSym))
+
+  result = newNodeI(nkStmtList, iter.info)
+  var gs = newNodeI(nkGotoState, iter.info)
+  gs.add(indirectAccess(newSymNode(c.closureParam), c.state, iter.info))
+  result.add(gs)
+  var state0 = newNodeI(nkState, iter.info)
+  state0.add(newIntNode(nkIntLit, 0))
+  result.add(state0)
+  
+  let newBody = transfIterBody(c, body)
+  if newBody != nil:
+    result.add(newBody)
+  else:
+    result.add(body)
+    
+  var state1 = newNodeI(nkState, iter.info)
+  state1.add(newIntNode(nkIntLit, -1))
+  result.add(state1)
+
+proc transformForLoop*(iter: PSym, body: PNode): PNode =
+  discard """
+  for i in foo(): nil
+
+Is transformed to:
+  
+  cl = createClosure()
+  while true:
+    let i = foo(cl)
+    if cl.state == -1: break
+"""
+  
--- a/compiler/main.nim
+++ b/compiler/main.nim
@@ -15,7 +15,7 @@ import
  os, lists, condsyms, rodread, rodwrite, ropes, trees, 
  wordrecg, sem, semdata, idents, passes, docgen, extccomp,
  cgen, ecmasgen,
-  platform, nimconf, importer, passaux, depends, transf, evals, types, idgen,
+  platform, nimconf, importer, passaux, depends, evals, types, idgen,
  tables, docgen2

 const
@@ -90,7 +90,6 @@ proc CompileProject(projectFile = gProjectFull) =
 proc semanticPasses =
  registerPass(verbosePass())
  registerPass(sem.semPass())
-  registerPass(transf.transfPass())

 proc CommandGenDepend =
  semanticPasses()
--- a/compiler/msgs.nim
+++ b/compiler/msgs.nim
@@ -25,7 +25,8 @@ type
    errRecursiveDependencyX, errOnOrOffExpected, errNoneSpeedOrSizeExpected, 
    errInvalidPragma, errUnknownPragma, errInvalidDirectiveX, 
    errAtPopWithoutPush, errEmptyAsm, errInvalidIndentation, 
-    errExceptionExpected, errExceptionAlreadyHandled, errYieldNotAllowedHere, 
+    errExceptionExpected, errExceptionAlreadyHandled, 
+    errYieldNotAllowedHere, errYieldNotAllowedInTryStmt, 
    errInvalidNumberOfYieldExpr, errCannotReturnExpr, errAttemptToRedefine, 
    errStmtInvalidAfterReturn, errStmtExpected, errInvalidLabel, 
    errInvalidCmdLineOption, errCmdLineArgExpected, errCmdLineNoArgExpected, 
@@ -149,7 +150,8 @@ const
    errInvalidIndentation: "invalid indentation", 
    errExceptionExpected: "exception expected", 
    errExceptionAlreadyHandled: "exception already handled", 
-    errYieldNotAllowedHere: "\'yield\' only allowed in a loop of an iterator", 
+    errYieldNotAllowedHere: "'yield' only allowed in an iterator",
+    errYieldNotAllowedInTryStmt: "'yield' cannot be used within 'try' in a non-inlined iterator",
    errInvalidNumberOfYieldExpr: "invalid number of \'yield\' expresions", 
    errCannotReturnExpr: "current routine cannot return an expression", 
    errAttemptToRedefine: "redefinition of \'$1\'", 
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -14,7 +14,7 @@ import
  wordrecg, ropes, msgs, os, condsyms, idents, renderer, types, platform, math,
  magicsys, parser, nversion, nimsets, semfold, importer,
  procfind, lookups, rodread, pragmas, passes, semdata, semtypinst, sigmatch,
-  semthreads, intsets, transf, evals, idgen, aliases
+  semthreads, intsets, transf, evals, idgen, aliases, cgmeth

 proc semPass*(): TPass
 # implementation
@@ -182,6 +182,7 @@ proc myOpen(module: PSym, filename: string): PPassContext =
 proc myOpenCached(module: PSym, filename: string, 
                  rd: PRodReader): PPassContext = 
  result = myOpen(module, filename)
+  for m in items(rd.methods): methodDef(m, true)

 proc SemStmtAndGenerateGenerics(c: PContext, n: PNode): PNode = 
  result = semStmt(c, n)
@@ -193,6 +194,7 @@ proc SemStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =
      # a generic has been added to `a`:
      if result.kind != nkEmpty: addSon(a, result)
      result = a
+  result = transformStmt(c.module, result)

 proc RecoverContext(c: PContext) = 
  # clean up in case of a semantic error: We clean up the stacks, etc. This is
--- a/compiler/semdata.nim
+++ b/compiler/semdata.nim
@@ -31,6 +31,8 @@ type
    resultSym*: PSym          # the result symbol (if we are in a proc)
    nestedLoopCounter*: int   # whether we are in a loop or not
    nestedBlockCounter*: int  # whether we are in a block or not
+    InTryStmt*: int           # whether we are in a try statement; works also
+                              # in standalone ``except`` and ``finally``
    next*: PProcCon           # used for stacking procedure contexts
  
  TInstantiatedSymbol* {.final.} = object
@@ -85,36 +87,6 @@ proc newGenericsCache*(): PGenericsCache =
  initIdTable(result.InstTypes)
  result.generics = @[]

-proc tempContext*(c: PContext): PContext =
-  ## generates a temporary context so that side-effects can be rolled-back;
-  ## necessary for ``system.compiles``.
-  new(result)
-  result.module = c.module
-  result.p = c.p
-  # don't use the old cache:
-  result.generics = newGenericsCache()
-  result.friendModule = c.friendModule
-  result.InstCounter = c.InstCounter
-  result.threadEntries = @[]
-  # hrm, 'tab' is expensive to copy ... so we don't. We open a new scope
-  # instead to be able to undo scope changes. Not entirely correct for
-  # explicit 'global' vars though:
-  #shallowCopy(result.tab, c.tab)
-  assign(result.AmbiguousSymbols, c.AmbiguousSymbols)
-  result.InGenericContext = c.InGenericContext
-  result.InUnrolledContext = c.InUnrolledContext
-  result.InCompilesContext = c.InCompilesContext
-  result.converters = c.converters
-  result.semConstExpr = c.semConstExpr
-  result.semExpr = c.semExpr
-  result.semConstBoolExpr = c.semConstBoolExpr
-  assign(result.includedFiles, c.includedFiles) 
-  result.filename = c.filename
-  #shallowCopy(result.userPragmas, c.userPragmas) 
-  # XXX mark it as read-only:
-  result.evalContext = c.evalContext
-  
-
 proc newContext*(module: PSym, nimfile: string): PContext

 proc lastOptionEntry*(c: PContext): POptionEntry
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -65,10 +65,6 @@ proc inlineConst(n: PNode, s: PSym): PNode {.inline.} =
  result.typ = s.typ
  result.info = n.info

-proc illegalCapture(s: PSym): bool {.inline.} =
-  result = skipTypes(s.typ, abstractInst).kind in {tyVar, tyOpenArray} or
-      s.kind == skResult
-
 proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = 
  case s.kind
  of skProc, skMethod, skIterator, skConverter: 
@@ -111,13 +107,11 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
      incl(c.p.owner.flags, sfSideEffect)
    elif s.kind == skParam and s.typ.kind == tyExpr:
      return s.typ.n
-    elif s.owner != c.p.owner and 
-        c.p.owner.typ != nil and not IsGenericRoutine(s.owner):
-      c.p.owner.typ.callConv = ccClosure
-      if illegalCapture(s) or c.p.next.owner != s.owner:
-        # Currently captures are restricted to a single level of nesting:
-        LocalError(n.info, errIllegalCaptureX, s.name.s)
    result = newSymNode(s, n.info)
+    # We cannot check for access to outer vars for example because it's still
+    # not sure the symbol really ends up being used:
+    # var len = 0 # but won't be called
+    # genericThatUsesLen(x) # marked as taking a closure?
  of skGenericParam:
    if s.ast != nil: result = semExpr(c, s.ast)
    else:
@@ -1179,7 +1173,7 @@ proc semCompiles(c: PContext, n: PNode, flags: TExprFlags): PNode =
  
  let oldInGenericContext = c.InGenericContext
  let oldInUnrolledContext = c.InUnrolledContext
-  
+  let oldProcCon = c.p
  c.generics = newGenericsCache()
  try:
    discard semExpr(c, n.sons[1])
@@ -1190,6 +1184,7 @@ proc semCompiles(c: PContext, n: PNode, flags: TExprFlags): PNode =
  c.generics = oldGenerics
  c.InGenericContext = oldInGenericContext
  c.InUnrolledContext = oldInUnrolledContext
+  c.p = oldProcCon
  msgs.setInfoContextLen(oldContextLen)
  setlen(gOwners, oldOwnerLen)
  while c.tab.tos > oldTos: rawCloseScope(c.tab)
--- a/compiler/seminst.nim
+++ b/compiler/seminst.nim
@@ -79,8 +79,7 @@ proc instantiateBody(c: PContext, n: PNode, result: PSym) =
      addResult(c, result.typ.sons[0], n.info, result.kind)
      addResultNode(c, n)
    var b = semStmtScope(c, n.sons[bodyPos])
-    # XXX Bad hack for tests/titer2 and tests/tactiontable
-    n.sons[bodyPos] = transform(c.module, b)
+    n.sons[bodyPos] = transformBody(c.module, b, result)
    #echo "code instantiated ", result.name.s
    excl(result.flags, sfForward)
    popProcCon(c)
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -198,15 +198,17 @@ proc SemYieldVarResult(c: PContext, n: PNode, restype: PType) =
          localError(n.sons[0].info, errXExpected, "tuple constructor")
  else: nil
  
-proc SemYield(c: PContext, n: PNode): PNode = 
+proc SemYield(c: PContext, n: PNode): PNode =
  result = n
  checkSonsLen(n, 1)
-  if c.p.owner == nil or c.p.owner.kind != skIterator: 
+  if c.p.owner == nil or c.p.owner.kind != skIterator:
    LocalError(n.info, errYieldNotAllowedHere)
+  elif c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline:
+    LocalError(n.info, errYieldNotAllowedInTryStmt)
  elif n.sons[0].kind != nkEmpty:
    n.sons[0] = SemExprWithType(c, n.sons[0]) # check for type compatibility:
    var restype = c.p.owner.typ.sons[0]
-    if restype != nil: 
+    if restype != nil:
      n.sons[0] = fitNode(c, restype, n.sons[0])
      if n.sons[0].typ == nil: InternalError(n.info, "semYield")
      SemYieldVarResult(c, n, restype)
@@ -486,6 +488,7 @@ proc semRaise(c: PContext, n: PNode): PNode =

 proc semTry(c: PContext, n: PNode): PNode = 
  result = n
+  inc c.p.inTryStmt
  checkMinSonsLen(n, 2)
  n.sons[0] = semStmtScope(c, n.sons[0])
  var check = initIntSet()
@@ -511,6 +514,7 @@ proc semTry(c: PContext, n: PNode): PNode =
      illFormedAst(n) 
    # last child of an nkExcept/nkFinally branch is a statement:
    a.sons[length - 1] = semStmtScope(c, a.sons[length - 1])
+  dec c.p.inTryStmt

 proc addGenericParamListToScope(c: PContext, n: PNode) = 
  if n.kind != nkGenericParams: 
@@ -714,20 +718,33 @@ proc semLambda(c: PContext, n: PNode): PNode =
      LocalError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s)
    pushProcCon(c, s)
    addResult(c, s.typ.sons[0], n.info, skProc)
-    n.sons[bodyPos] = semStmtScope(c, n.sons[bodyPos])
+    let semBody = semStmtScope(c, n.sons[bodyPos])
+    n.sons[bodyPos] = transformBody(c.module, semBody, s)
    addResultNode(c, n)
    popProcCon(c)
-  else: 
+  else:
    LocalError(n.info, errImplOfXexpected, s.name.s)
  sideEffectsCheck(c, s)
-  if s.typ.callConv == ccClosure and s.owner.kind == skModule:
-    localError(s.info, errXCannotBeClosure, s.name.s)
  closeScope(c.tab)           # close scope for parameters
  popOwner()
  result.typ = s.typ
- 
+
 proc instantiateDestructor*(c: PContext, typ: PType): bool

+proc doDestructorStuff(c: PContext, s: PSym, n: PNode) =
+  let t = s.typ.sons[1].skipTypes({tyVar})
+  t.destructor = s
+  # automatically insert calls to base classes' destructors
+  if n.sons[bodyPos].kind != nkEmpty:
+    for i in countup(0, t.sonsLen - 1):
+      # when inheriting directly from object
+      # there will be a single nil son
+      if t.sons[i] == nil: continue
+      if instantiateDestructor(c, t.sons[i]):
+        n.sons[bodyPos].addSon(newNode(nkCall, t.sym.info, @[
+            useSym(t.sons[i].destructor),
+            n.sons[paramsPos][1][0]]))
+
 proc semProcAux(c: PContext, n: PNode, kind: TSymKind, 
                validPragmas: TSpecialWords): PNode = 
  result = semProcAnnotation(c, n)
@@ -754,7 +771,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
        n.sons[genericParamsPos] = gp
        # check for semantics again:
        # semParamList(c, n.sons[ParamsPos], nil, s)
-  else: 
+  else:
    s.typ = newTypeS(tyProc, c)
    rawAddSon(s.typ, nil)
  var proto = SearchForProc(c, s, c.tab.tos-2) # -2 because we have a scope
@@ -791,19 +808,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
    popOwner()
    pushOwner(s)
  s.options = gOptions
-  if sfDestructor in s.flags:
-    let t = s.typ.sons[1].skipTypes({tyVar})
-    t.destructor = s
-    # automatically insert calls to base classes' destructors
-    if n.sons[bodyPos].kind != nkEmpty:
-      for i in countup(0, t.sonsLen - 1):
-        # when inheriting directly from object
-        # there will be a single nil son
-        if t.sons[i] == nil: continue
-        if instantiateDestructor(c, t.sons[i]):
-          n.sons[bodyPos].addSon(newNode(nkCall, t.sym.info, @[
-              useSym(t.sons[i].destructor),
-              n.sons[paramsPos][1][0]]))
+  if sfDestructor in s.flags: doDestructorStuff(c, s, n)
  if n.sons[bodyPos].kind != nkEmpty: 
    # for DLL generation it is annoying to check for sfImportc!
    if sfBorrow in s.flags: 
@@ -815,7 +820,10 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
        addResult(c, s.typ.sons[0], n.info, kind)
      if sfImportc notin s.flags:
        # no semantic checking for importc:
-        n.sons[bodyPos] = semStmtScope(c, n.sons[bodyPos])
+        let semBody = semStmtScope(c, n.sons[bodyPos])
+        # unfortunately we cannot skip this step when in 'system.compiles'
+        # context as it may even be evaluated in 'system.compiles':
+        n.sons[bodyPos] = transformBody(c.module, semBody, s)
      if s.typ.sons[0] != nil and kind != skIterator: addResultNode(c, n)
      popProcCon(c)
    else: 
@@ -827,14 +835,12 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
    if sfImportc in s.flags: 
      # so we just ignore the body after semantic checking for importc:
      n.sons[bodyPos] = ast.emptyNode
-  else: 
+  else:
    if proto != nil: LocalError(n.info, errImplOfXexpected, proto.name.s)
    if {sfImportc, sfBorrow} * s.flags == {} and s.magic == mNone: 
      incl(s.flags, sfForward)
    elif sfBorrow in s.flags: semBorrow(c, n, s)
  sideEffectsCheck(c, s)
-  if s.typ.callConv == ccClosure and s.owner.kind == skModule:
-    localError(s.info, errXCannotBeClosure, s.name.s)
  closeScope(c.tab)           # close scope for parameters
  popOwner()
  
@@ -845,7 +851,10 @@ proc semIterator(c: PContext, n: PNode): PNode =
  if t.sons[0] == nil:
    LocalError(n.info, errXNeedsReturnType, "iterator")
  # iterators are either 'inline' or 'closure':
-  if s.typ.callConv != ccInline: s.typ.callConv = ccClosure
+  if s.typ.callConv != ccInline: 
+    s.typ.callConv = ccClosure
+    # and they always at least use the 'env' for the state field:
+    incl(s.typ.flags, tfCapturesEnv)
  if n.sons[bodyPos].kind == nkEmpty and s.magic == mNone:
    LocalError(n.info, errImplOfXexpected, s.name.s)
  
--- a/compiler/transf.nim
+++ b/compiler/transf.nim
@@ -25,7 +25,6 @@ import
 const 
  genPrefix* = ":tmp"         # prefix for generated names

-proc transfPass*(): TPass
 # implementation

 type 
@@ -109,54 +108,6 @@ proc transformSons(c: PTransf, n: PNode): PTransNode =
  for i in countup(0, sonsLen(n)-1): 
    result[i] = transform(c, n.sons[i])

-# Transforming iterators into non-inlined versions is pretty hard, but
-# unavoidable for not bloating the code too much. If we had direct access to
-# the program counter, things'd be much easier.
-# ::
-#
-#  iterator items(a: string): char =
-#    var i = 0
-#    while i < length(a):
-#      yield a[i]
-#      inc(i)
-#
-#  for ch in items("hello world"): # `ch` is an iteration variable
-#    echo(ch)
-#
-# Should be transformed into::
-#
-#  type
-#    TItemsClosure = record
-#      i: int
-#      state: int
-#  proc items(a: string, c: var TItemsClosure): char =
-#    case c.state
-#    of 0: goto L0 # very difficult without goto!
-#    of 1: goto L1 # can be implemented by GCC's computed gotos
-#
-#    block L0:
-#      c.i = 0
-#      while c.i < length(a):
-#        c.state = 1
-#        return a[i]
-#        block L1: inc(c.i)
-#
-# More efficient, but not implementable::
-#
-#  type
-#    TItemsClosure = record
-#      i: int
-#      pc: pointer
-#
-#  proc items(a: string, c: var TItemsClosure): char =
-#    goto c.pc
-#    c.i = 0
-#    while c.i < length(a):
-#      c.pc = label1
-#      return a[i]
-#      label1: inc(c.i)
-#
-
 proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode = 
  result = newTransNode(nkFastAsgn, PNode(ri).info, 2)
  result[0] = PTransNode(le)
@@ -178,15 +129,6 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
    if result != nil: return
    tc = tc.next
  result = b
-  when false:
-    case b.sym.kind
-    of skConst, skEnumField: 
-      if sfFakeConst notin b.sym.flags:
-        if skipTypes(b.sym.typ, abstractInst).kind notin ConstantDataTypes: 
-          result = getConstExpr(c.module, b)
-          if result == nil: InternalError(b.info, "transformSym: const")
-    else: 
-      nil

 proc transformSym(c: PTransf, n: PNode): PTransNode = 
  result = PTransNode(transformSymAux(c, n))
@@ -378,20 +320,6 @@ proc transformAddrDeref(c: PTransf, n: PNode, a, b: TNodeKind): PTransNode =
    if n.sons[0].kind == a or n.sons[0].kind == b:
      # addr ( deref ( x )) --> x
      result = PTransNode(n.sons[0].sons[0])
-
-proc generateThunk(c: PTransf, prc: PNode, dest: PType): PNode =
-  ## Converts 'prc' into '(thunk, nil)' so that it's compatible with
-  ## a closure.
-  
-  # we cannot generate a proper thunk here for GC-safety reasons (see internal
-  # documentation):
-  if gCmd == cmdCompileToEcmaScript: return prc
-  result = newNodeIT(nkClosure, prc.info, dest)
-  var conv = newNodeIT(nkHiddenStdConv, prc.info, dest)
-  conv.add(emptyNode)
-  conv.add(prc)
-  result.add(conv)
-  result.add(newNodeIT(nkNilLit, prc.info, getSysType(tyNil)))
  
 proc transformConv(c: PTransf, n: PNode): PTransNode = 
  # numeric types need range checks:
@@ -466,12 +394,6 @@ proc transformConv(c: PTransf, n: PNode): PTransNode =
      result[0] = transform(c, n.sons[1])
    else: 
      result = transform(c, n.sons[1])
-  of tyProc:
-    if dest.callConv == ccClosure and source.callConv == ccDefault:
-      let x = transform(c, n.sons[1]).pnode
-      result = generateThunk(c, x, dest).ptransnode
-    else:
-      result = transformSons(c, n)    
  of tyGenericParam, tyOrdinal, tyTypeClass:
    result = transform(c, n.sons[1])
    # happens sometimes for generated assignments, etc.
@@ -657,21 +579,23 @@ proc transform(c: PTransf, n: PNode): PTransNode =
    result = PTransNode(n)
  of nkBracketExpr: result = transformArrayAccess(c, n)
  of procDefs:
-    if n.sons[genericParamsPos].kind == nkEmpty:
-      var s = n.sons[namePos].sym
-      n.sons[bodyPos] = PNode(transform(c, s.getBody))
-      if s.ast.sons[bodyPos] != n.sons[bodyPos]:
-        # somehow this can happen ... :-/
-        s.ast.sons[bodyPos] = n.sons[bodyPos]
-      n.sons[bodyPos] = liftLambdas(n)
-      if n.kind == nkMethodDef: methodDef(s, false)
+    when false:
+      if n.sons[genericParamsPos].kind == nkEmpty:
+        var s = n.sons[namePos].sym
+        n.sons[bodyPos] = PNode(transform(c, s.getBody))
+        if s.ast.sons[bodyPos] != n.sons[bodyPos]:
+          # somehow this can happen ... :-/
+          s.ast.sons[bodyPos] = n.sons[bodyPos]
+        #n.sons[bodyPos] = liftLambdas(s, n)
+        #if n.kind == nkMethodDef: methodDef(s, false)
    result = PTransNode(n)
  of nkMacroDef:
    # XXX no proper closure support yet:
-    if n.sons[genericParamsPos].kind == nkEmpty:
-      var s = n.sons[namePos].sym
-      n.sons[bodyPos] = PNode(transform(c, s.getBody))
-      if n.kind == nkMethodDef: methodDef(s, false)
+    when false:
+      if n.sons[genericParamsPos].kind == nkEmpty:
+        var s = n.sons[namePos].sym
+        n.sons[bodyPos] = PNode(transform(c, s.getBody))
+        if n.kind == nkMethodDef: methodDef(s, false)
    result = PTransNode(n)
  of nkForStmt: 
    inc c.inLoop
@@ -735,38 +659,57 @@ proc transform(c: PTransf, n: PNode): PTransNode =
  if cnst != nil and not dontInlineConstant(n, cnst):
    result = PTransNode(cnst) # do not miss an optimization

-proc processTransf(context: PPassContext, n: PNode): PNode = 
+proc processTransf(c: PTransf, n: PNode): PNode = 
  # Note: For interactive mode we cannot call 'passes.skipCodegen' and skip
  # this step! We have to rely that the semantic pass transforms too errornous
  # nodes into an empty node.
-  if passes.skipCodegen(n) or context.fromCache or nfTransf in n.flags: return n
-  var c = PTransf(context)
+  if passes.skipCodegen(n) or c.fromCache or nfTransf in n.flags: return n
  pushTransCon(c, newTransCon(getCurrOwner(c)))
  result = PNode(transform(c, n))
  popTransCon(c)
  incl(result.flags, nfTransf)

-proc openTransf(module: PSym, filename: string): PPassContext = 
-  var n: PTransf
-  new(n)
-  n.contSyms = @[]
-  n.breakSyms = @[]
-  n.module = module
-  result = n
+proc openTransf(module: PSym, filename: string): PTransf = 
+  new(result)
+  result.contSyms = @[]
+  result.breakSyms = @[]
+  result.module = module

-proc openTransfCached(module: PSym, filename: string, 
-                      rd: PRodReader): PPassContext = 
-  result = openTransf(module, filename)
-  for m in items(rd.methods): methodDef(m, true)
+when false:
+  proc openTransfCached(module: PSym, filename: string, 
+                        rd: PRodReader): PPassContext = 
+    result = openTransf(module, filename)
+    for m in items(rd.methods): methodDef(m, true)

-proc transfPass(): TPass = 
-  initPass(result)
-  result.open = openTransf
-  result.openCached = openTransfCached
-  result.process = processTransf
-  result.close = processTransf # we need to process generics too!
+  proc transfPass(): TPass = 
+    initPass(result)
+    result.open = openTransf
+    result.openCached = openTransfCached
+    result.process = processTransf
+    result.close = processTransf # we need to process generics too!
  
-proc transform*(module: PSym, n: PNode): PNode =
+proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
+  if nfTransf in n.flags or prc.kind in {skTemplate, skMacro}:
+    result = n
+  else:
+    var c = openTransf(module, "")
+    result = processTransf(c, n)
+    if prc.kind != skMacro:
+      # XXX no closures yet for macros:
+      result = liftLambdas(prc, result)
+    if prc.kind == skMethod: methodDef(prc, false)
+    incl(result.flags, nfTransf)
+
+proc transformStmt*(module: PSym, n: PNode): PNode =
+  if nfTransf in n.flags:
+    result = n
+  else:
+    var c = openTransf(module, "")
+    result = processTransf(c, n)
+    result = liftLambdasForTopLevel(module, result)
+    incl(result.flags, nfTransf)
+
+proc transformExpr*(module: PSym, n: PNode): PNode =
  if nfTransf in n.flags:
    result = n
  else:
--- a/lib/core/typeinfo.nim
+++ b/lib/core/typeinfo.nim
@@ -455,7 +455,7 @@ proc getFloat*(x: TAny): float =

 proc getFloat32*(x: TAny): float32 = 
  ## retrieve the float32 value out of `x`. `x` needs to represent an float32.
-  assert skipRange(x.rawtype).kind == tyFloat64
+  assert skipRange(x.rawtype).kind == tyFloat32
  result = cast[ptr float32](x.value)[]
  
 proc getFloat64*(x: TAny): float64 = 
--- a/tests/run/tunittests.nim
+++ b/tests/run/tunittests.nim
@@ -1,3 +1 @@
-import utemplates
-
-# uclosures
+import utemplates, uclosures
--- a/tests/run/uclosures.nim
+++ b/tests/run/uclosures.nim
@@ -4,7 +4,8 @@ test "loop variables are captured by copy":
  var funcs: seq[proc (): int {.closure.}] = @[]
  
  for i in 0..10:
-    funcs.add do -> int: return i * i
+    let ii = i
+    funcs.add do -> int: return ii * ii

  check funcs[0]() == 0
  check funcs[3]() == 9
--- a/todo.txt
+++ b/todo.txt
@@ -1,14 +1,14 @@
 version 0.9.0
 =============

- closure: implement closure support for procs capturing nested
-  module vars
- implicit deref for parameter matching
+- implement "closure tuple consists of a single 'ref'" optimization
+- implement for loop transformation for first class iterators

+- implicit deref for parameter matching; implement ``varargs[T, `$`]``
 - optimize genericAssign in the code generator
+- the lookup rules for generics really are too permissive
 - fix remaining closure bugs:
  - fix evals.nim with closures
-  - implement "closure tuple consists of a single 'ref'" optimization


 Bugs