From d71f69ab50f079c03860f244f6c64b555ca403b6 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Wed, 25 Apr 2018 19:38:59 +0300
Subject: [PATCH 01/51] Closure iter transformation

---
 compiler/ccgexprs.nim                |   2 +-
 compiler/ccgstmts.nim                |  10 +-
 compiler/closureiters.nim            | 632 +++++++++++++++++++++++++++
 compiler/lambdalifting.nim           |  60 ++-
 compiler/options.nim                 |   3 +-
 compiler/renderer.nim                |  14 +-
 compiler/seminst.nim                 |   2 +-
 compiler/semstmts.nim                |   6 +-
 compiler/transf.nim                  |  25 +-
 tests/async/tasync_in_seq_constr.nim |   3 +-
 10 files changed, 713 insertions(+), 44 deletions(-)
 create mode 100644 compiler/closureiters.nim

diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim
index 96f9265f17..5b3f6c3d21 100644
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -2326,7 +2326,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
   of nkParForStmt: genParForStmt(p, n)
   of nkState: genState(p, n)
   of nkGotoState: genGotoState(p, n)
-  of nkBreakState: genBreakState(p, n)
+  of nkBreakState: genBreakState(p, n, d)
   else: internalError(n.info, "expr(" & $n.kind & "); unknown node kind")
 
 proc genNamedConstExpr(p: BProc, n: PNode): Rope =
diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim
index cb3d6dbe6e..96f5b53a77 100644
--- a/compiler/ccgstmts.nim
+++ b/compiler/ccgstmts.nim
@@ -177,17 +177,17 @@ proc genGotoState(p: BProc, n: PNode) =
     lineF(p, cpsStmts, "case $2: goto $1$2;$n", [prefix, rope(i)])
   lineF(p, cpsStmts, "}$n", [])
 
-proc genBreakState(p: BProc, n: PNode) =
+proc genBreakState(p: BProc, n: PNode, d: var TLoc) =
   var a: TLoc
+  initLoc(d, locExpr, n, OnUnknown)
+
   if n.sons[0].kind == nkClosure:
-    # XXX this produces quite inefficient code!
     initLocExpr(p, n.sons[0].sons[1], a)
-    lineF(p, cpsStmts, "if (((NI*) $1)[1] < 0) break;$n", [rdLoc(a)])
+    d.r = "(((NI*) $1)[1] < 0)" % [rdLoc(a)]
   else:
     initLocExpr(p, n.sons[0], a)
     # the environment is guaranteed to contain the 'state' field at offset 1:
-    lineF(p, cpsStmts, "if ((((NI*) $1.ClE_0)[1]) < 0) break;$n", [rdLoc(a)])
-  #  lineF(p, cpsStmts, "if (($1) < 0) break;$n", [rdLoc(a)])
+    d.r = "((((NI*) $1.ClE_0)[1]) < 0)" % [rdLoc(a)]
 
 proc genGotoVar(p: BProc; value: PNode) =
   if value.kind notin {nkCharLit..nkUInt64Lit}:
diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
new file mode 100644
index 0000000000..02795ab470
--- /dev/null
+++ b/compiler/closureiters.nim
@@ -0,0 +1,632 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2018 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# This file implements closure iterator transformations.
+# The main idea is to split the closure iterator body to top level statements.
+# The body is split by yield statement.
+#
+# Example:
+#  while a > 0:
+#    echo "hi"
+#    yield a
+#    dec a
+#
+# Should be transformed to:
+#  STATE0:
+#    if a > 0:
+#      echo "hi"
+#      :state = 1 # Next state
+#      return a # yield
+#    else:
+#      :state = 2 # Next state
+#      break :stateLoop # Proceed to the next state
+#  STATE1:
+#    dec a
+#    :state = 0 # Next state
+#    break :stateLoop # Proceed to the next state
+#  STATE2:
+#    :state = -1 # End of execution
+
+# The transformation should play well with lambdalifting, however depending
+# on situation, it can be called either before or after lambdalifting
+# transformation. As such we behave slightly differently, when accessing
+# iterator state, or using temp variables. If lambdalifting did not happen,
+# we just create local variables, so that they will be lifted further on.
+# Otherwise, we utilize existing env, created by lambdalifting.
+
+# Lambdalifting treats :state variable specially, it should always end up
+# as the first field in env. Currently C codegen depends on this behavior.
+
+# One special subtransformation is nkStmtListExpr lowering.
+# Example:
+#   template foo(): int =
+#     yield 1
+#     2
+#
+#   iterator it(): int {.closure.} =
+#     if foo() == 2:
+#       yield 3
+#
+# If a nkStmtListExpr has yield inside, it has first to be lowered to:
+#   yield 1
+#   :tmpSlLower = 2
+#   if :tmpSlLower == 2:
+#     yield 3
+
+
+import
+  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os, options,
+  idents, renderer, types, magicsys, rodread, lowerings, tables, sequtils,
+  lambdalifting
+
+type ClosureIteratorTransformationContext = object
+  fn: PSym
+  stateVarSym: PSym # :state variable. nil if env already introduced by lambdalifting
+  states: seq[PNode] # The resulting states. Every state is an nkState node.
+  blockLevel: int # Temp used to transform break and continue stmts
+  stateLoopLabel: PSym # Label to break on, when jumping between states.
+  exitStateIdx: int # index of the last state
+  tempVarId: int # unique name counter
+  tempVars: PNode # Temp var decls, nkVarSection
+  loweredStmtListExpr: PNode # Temporary used for nkStmtListExpr lowering
+
+proc newStateAssgn(ctx: var ClosureIteratorTransformationContext, stateNo: int = -2): PNode =
+  # Creates state assignmen:
+  #   :state = stateNo
+
+  result = newNode(nkAsgn)
+  if ctx.stateVarSym.isNil:
+    let state = getStateField(ctx.fn)
+    assert state != nil
+    result.add(rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)),
+                      state, result.info))
+  else:
+    result.add(newSymNode(ctx.stateVarSym))
+  result.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt)))
+
+proc setStateInAssgn(stateAssgn: PNode, stateNo: int) =
+  assert stateAssgn.kind == nkAsgn
+  assert stateAssgn[1].kind == nkIntLit
+  stateAssgn[1].intVal = stateNo
+
+proc newState(ctx: var ClosureIteratorTransformationContext, n, gotoOut: PNode): int =
+  # Creates a new state, adds it to the context fills out `gotoOut` so that it
+  # will goto this state.
+  # Returns index of the newly created state
+
+  result = ctx.states.len
+  let resLit = newIntLit(result)
+  let s = newNodeI(nkState, n.info)
+  s.add(resLit)
+  s.add(n)
+  ctx.states.add(s)
+  if not gotoOut.isNil:
+    assert(gotoOut.len == 0)
+    gotoOut.add(newIntLit(result))
+
+proc toStmtList(n: PNode): PNode =
+  result = n
+  if result.kind notin {nkStmtList, nkStmtListExpr}:
+    result = newNodeI(nkStmtList, n.info)
+    result.add(n)
+
+proc addGotoOut(n: PNode, gotoOut: PNode): PNode =
+  # Make sure `n` is a stmtlist, and ends with `gotoOut`
+
+  result = toStmtList(n)
+  if result.len != 0 and result.sons[^1].kind != nkGotoState:
+    result.add(gotoOut)
+
+proc newTempVarAccess(ctx: var ClosureIteratorTransformationContext, typ: PType, i: TLineInfo): PNode =
+  if not ctx.stateVarSym.isNil:
+    # We haven't gone through labmda lifting yet, so just create a local var,
+    # it will be lifted later
+    let s = newSym(skVar, getIdent(":tmpSlLower" & $ctx.tempVarId), ctx.fn, i)
+    s.typ = typ
+
+    if ctx.tempVars.isNil:
+      ctx.tempVars = newNode(nkVarSection)
+      addVar(ctx.tempVars, newSymNode(s))
+
+    result = newSymNode(s)
+  else:
+    # Lambda lifting is done, insert temp var to env.
+    let s = newSym(skVar, getIdent(":tmpSlLower" & $ctx.tempVarId), ctx.fn, i)
+    s.typ = typ
+    result = freshVarForClosureIter(s, ctx.fn)
+
+  inc ctx.tempVarId
+
+proc hasYields(n: PNode): bool =
+  # TODO: This is very inefficient. It traverses the node, looking for nkYieldStmt.
+  case n.kind
+  of nkYieldStmt:
+    result = true
+  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
+      nkSym, nkIdent, procDefs, nkTemplateDef:
+    discard
+  else:
+    for c in n:
+      if c.hasYields:
+        result = true
+        break
+
+proc transformBreaksAndContinuesInWhile(ctx: var ClosureIteratorTransformationContext, n: PNode, before, after: PNode): PNode =
+  result = n
+  case n.kind
+  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
+      nkSym, nkIdent, procDefs, nkTemplateDef:
+    discard
+  of nkWhileStmt: discard # Do not recurse into nested whiles
+  of nkContinueStmt:
+    result = before
+  of nkBlockStmt:
+    inc ctx.blockLevel
+    result[1] = ctx.transformBreaksAndContinuesInWhile(result[1], before, after)
+    dec ctx.blockLevel
+  of nkBreakStmt:
+    if ctx.blockLevel == 0:
+      result = after
+  else:
+    for i in 0 ..< n.len:
+      n[i] = ctx.transformBreaksAndContinuesInWhile(n[i], before, after)
+
+proc transformBreaksInBlock(ctx: var ClosureIteratorTransformationContext, n: PNode, label, after: PNode): PNode =
+  result = n
+  case n.kind
+  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
+      nkSym, nkIdent, procDefs, nkTemplateDef:
+    discard
+  of nkBlockStmt, nkWhileStmt:
+    inc ctx.blockLevel
+    result[1] = ctx.transformBreaksInBlock(result[1], label, after)
+    dec ctx.blockLevel
+  of nkBreakStmt:
+    if n[0].kind == nkEmpty:
+      if ctx.blockLevel == 0:
+        result = after
+    else:
+      if label.kind == nkSym and n[0].sym == label.sym:
+        result = after
+  else:
+    for i in 0 ..< n.len:
+      n[i] = ctx.transformBreaksInBlock(n[i], label, after)
+
+proc collectExceptState(n: PNode): PNode =
+  var ifStmt = newNode(nkIfStmt)
+  for c in n:
+    if c.kind == nkExceptBranch:
+      var ifBranch: PNode
+      var branchBody: PNode
+
+      if c[0].kind == nkType:
+        assert(c.len == 2)
+        ifBranch = newNode(nkElifBranch)
+        let expression = newNodeI(nkCall, n.info)
+        expression.add(callCodegenProc("getCurrentException", emptyNode))
+        expression.add(c[0])
+        ifBranch.add(expression)
+        branchBody = c[1]
+      else:
+        assert(c.len == 1)
+        if ifStmt.len == 0:
+          ifStmt = newNode(nkStmtList)
+          ifBranch = newNode(nkStmtList)
+        else:
+          ifBranch = newNode(nkElse)
+        branchBody = c[0]
+
+      ifBranch.add(branchBody)
+      ifStmt.add(ifBranch)
+
+  if ifStmt.len != 0:
+    result = newNode(nkStmtList)
+    result.add(ifStmt)
+  else:
+    result = emptyNode
+
+proc getFinallyNode(n: PNode): PNode =
+  result = n[^1]
+  if result.kind == nkFinally:
+    result = result[0]
+  else:
+    result = emptyNode
+
+proc hasYieldsInExpressions(n: PNode): bool =
+  case n.kind
+  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
+      nkSym, nkIdent, procDefs, nkTemplateDef:
+    discard
+  of nkStmtListExpr:
+    result = n.hasYields
+  of nkStmtList, nkWhileStmt, nkCaseStmt, nkIfStmt:
+    discard
+  else:
+    for c in n:
+      if c.hasYieldsInExpressions:
+        return true
+
+proc lowerStmtListExpr(ctx: var ClosureIteratorTransformationContext, n: PNode): PNode =
+  result = n
+  case n.kind
+  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
+      nkSym, nkIdent, procDefs, nkTemplateDef:
+    discard
+  of nkStmtListExpr:
+    if n.hasYields:
+      for i in 0 .. n.len - 2:
+        ctx.loweredStmtListExpr.add(n[i])
+
+      let tv = ctx.newTempVarAccess(n.typ, n[^1].info)
+      let asgn = newNode(nkAsgn)
+      asgn.add(tv)
+      asgn.add(n[^1])
+      ctx.loweredStmtListExpr.add(asgn)
+      result = tv
+
+  else:
+    for i in 0 ..< n.len:
+      n[i] = ctx.lowerStmtListExpr(n[i])
+
+proc transformClosureIteratorBody(ctx: var ClosureIteratorTransformationContext, n: PNode, gotoOut: PNode): PNode =
+  result = n
+  case n.kind:
+    of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
+        nkSym, nkIdent, procDefs, nkTemplateDef:
+      discard
+
+    of nkStmtList:
+      result = addGotoOut(result, gotoOut)
+      for i in 0 ..< n.len:
+        if n[i].hasYieldsInExpressions:
+          # Lower nkStmtListExpr nodes inside `n[i]` first
+          assert(ctx.loweredStmtListExpr.isNil)
+          ctx.loweredStmtListExpr = newNodeI(nkStmtList, n.info)
+          n[i] = ctx.lowerStmtListExpr(n[i])
+          ctx.loweredStmtListExpr.add(n[i])
+          n[i] = ctx.loweredStmtListExpr
+          ctx.loweredStmtListExpr = nil
+
+        if n[i].hasYields:
+          # Create a new split
+          let go = newNode(nkGotoState)
+          n[i] = ctx.transformClosureIteratorBody(n[i], go)
+
+          let s = newNode(nkStmtList)
+          for j in i + 1 ..< n.len:
+            s.add(n[j])
+
+          n.sons.setLen(i + 1)
+          discard ctx.newState(s, go)
+          discard ctx.transformClosureIteratorBody(s, gotoOut)
+          break
+
+    of nkStmtListExpr:
+      assert(false, "nkStmtListExpr not lowered")
+
+    of nkYieldStmt:
+      # echo "YIELD!"
+      result = newNodeI(nkStmtList, n.info)
+      result.add(n)
+      result.add(gotoOut)
+
+    of nkElse, nkElseExpr:
+      result[0] = addGotoOut(result[0], gotoOut)
+      result[0] = ctx.transformClosureIteratorBody(result[0], gotoOut)
+
+    of nkElifBranch, nkElifExpr, nkOfBranch:
+      result[1] = addGotoOut(result[1], gotoOut)
+      result[1] = ctx.transformClosureIteratorBody(result[1], gotoOut)
+
+    of nkIfStmt, nkCaseStmt:
+      for i in 0 ..< n.len:
+        n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
+      if n[^1].kind != nkElse:
+        # We don't have an else branch, but every possible branch has to end with
+        # gotoOut, so add else here.
+        let elseBranch = newNode(nkElse)
+        elseBranch.add(gotoOut)
+        n.add(elseBranch)
+
+    of nkWhileStmt:
+      # while e:
+      #   s
+      # ->
+      # BEGIN_STATE:
+      #   if e:
+      #     s
+      #     goto BEGIN_STATE
+      #   else:
+      #     goto OUT
+
+      result = newNodeI(nkGotoState, n.info)
+
+      let s = newNodeI(nkStmtList, n.info)
+      discard ctx.newState(s, result)
+      let ifNode = newNodeI(nkIfStmt, n.info)
+      let elifBranch = newNodeI(nkElifBranch, n.info)
+      elifBranch.add(n[0])
+
+      var body = addGotoOut(n[1], result)
+
+      body = ctx.transformBreaksAndContinuesInWhile(body, result, gotoOut)
+      body = ctx.transformClosureIteratorBody(body, result)
+
+      elifBranch.add(body)
+      ifNode.add(elifBranch)
+
+      let elseBranch = newNode(nkElse)
+      elseBranch.add(gotoOut)
+      ifNode.add(elseBranch)
+      s.add(ifNode)
+
+    of nkBlockStmt:
+      result[1] = addGotoOut(result[1], gotoOut)
+      result[1] = ctx.transformBreaksInBlock(result[1], result[0], gotoOut)
+      result[1] = ctx.transformClosureIteratorBody(result[1], gotoOut)
+
+    of nkTryStmt:
+      var tryBody = toStmtList(n[0])
+
+      # let popTry = newNode(nkPar)
+      # popTry.add(newIdentNode(getIdent("popTry"), n.info))
+      var finallyBody = newNode(nkStmtList)
+      # finallyBody.add(popTry)
+      finallyBody.add(getFinallyNode(n))
+
+      var tryCatchOut = newNode(nkGotoState)
+
+      tryBody = ctx.transformClosureIteratorBody(tryBody, tryCatchOut)
+      var exceptBody = collectExceptState(n)
+
+      var exceptIdx = -1
+      if exceptBody.kind != nkEmpty:
+        exceptBody = ctx.transformClosureIteratorBody(exceptBody, tryCatchOut)
+        exceptIdx = ctx.newState(exceptBody, nil)
+
+      finallyBody = ctx.transformClosureIteratorBody(finallyBody, gotoOut)
+      let finallyIdx = ctx.newState(finallyBody, tryCatchOut)
+
+      # let pushTry = newNode(nkPar) #newCall(newSym("pushTry"), newIntLit(exceptIdx))
+      # pushTry.add(newIdentNode(getIdent("pushTry"), n.info))
+      # pushTry.add(newIntLit(exceptIdx))
+      # pushTry.add(newIntLit(finallyIdx))
+      # tryBody.sons.insert(pushTry, 0)
+
+      result = tryBody
+
+    of nkGotoState, nkForStmt:
+      internalError("closure iter " & $n.kind)
+
+    else:
+      for i in 0 ..< n.len:
+        n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
+
+proc stateFromGotoState(n: PNode): int =
+  assert(n.kind == nkGotoState)
+  result = n[0].intVal.int
+
+proc tranformStateAssignments(ctx: var ClosureIteratorTransformationContext, n: PNode): PNode =
+  # This transforms 3 patterns:
+  ########################## 1
+  # yield e
+  # goto STATE
+  # ->
+  # :state = STATE
+  # return e
+  ########################## 2
+  # goto STATE
+  # ->
+  # :state = STATE
+  # break :stateLoop
+  ########################## 3
+  # return e
+  # ->
+  # :state = -1
+  # return e
+  #
+  result = n
+  case n.kind
+  of nkStmtList, nkStmtListExpr:
+    if n.len != 0 and n[0].kind == nkYieldStmt:
+      assert(n.len == 2)
+      assert(n[1].kind == nkGotoState)
+
+      result = newNodeI(nkStmtList, n.info)
+      result.add(ctx.newStateAssgn(stateFromGotoState(n[1])))
+
+      var retStmt = newNodeI(nkReturnStmt, n.info)
+      if n[0].sons[0].kind != nkEmpty:
+        var a = newNodeI(nkAsgn, n[0].sons[0].info)
+        var retVal = n[0].sons[0] #liftCapturedVars(n.sons[0], owner, d, c)
+        addSon(a, newSymNode(getClosureIterResult(ctx.fn)))
+        addSon(a, retVal)
+        retStmt.add(a)
+      else:
+        retStmt.add(emptyNode)
+
+      result.add(retStmt)
+    else:
+      for i in 0 ..< n.len:
+        n[i] = ctx.tranformStateAssignments(n[i])
+
+  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
+      nkSym, nkIdent, procDefs, nkTemplateDef:
+    discard
+
+  of nkReturnStmt:
+
+    result = newNodeI(nkStmtList, n.info)
+    result.add(ctx.newStateAssgn(-1))
+    result.add(n)
+
+  of nkGotoState:
+    result = newNodeI(nkStmtList, n.info)
+    result.add(ctx.newStateAssgn(stateFromGotoState(n)))
+
+    let breakState = newNodeI(nkBreakStmt, n.info)
+    breakState.add(newSymNode(ctx.stateLoopLabel))
+    result.add(breakState)
+
+  else:
+    for i in 0 ..< n.len:
+      n[i] = ctx.tranformStateAssignments(n[i])
+
+proc skipStmtList(n: PNode): PNode =
+  result = n
+  while result.kind in {nkStmtList}:
+    if result.len == 0: return emptyNode
+    result = result[0]
+
+proc skipThroughEmptyStates(ctx: var ClosureIteratorTransformationContext, n: PNode): PNode =
+  result = n
+  case n.kind
+  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
+      nkSym, nkIdent, procDefs, nkTemplateDef:
+    discard
+  of nkGotoState:
+    var maxJumps = ctx.states.len # maxJumps used only for debugging purposes.
+    result = copyTree(n)
+    while true:
+      let label = result[0].intVal.int
+      if label == ctx.exitStateIdx: break
+      var newLabel = label
+      if label == -1:
+        newLabel = ctx.exitStateIdx
+      else:
+        let fs = ctx.states[label][1].skipStmtList()
+        if fs.kind == nkGotoState:
+          newLabel = fs[0].intVal.int
+      if label == newLabel: break
+      result[0].intVal = newLabel
+      dec maxJumps
+      if maxJumps == 0:
+        assert(false, "Internal error")
+
+    let label = result[0].intVal.int
+    result[0].intVal = ctx.states[label][0].intVal
+  else:
+    for i in 0 ..< n.len:
+      n[i] = ctx.skipThroughEmptyStates(n[i])
+
+proc wrapIntoStateLoop(ctx: var ClosureIteratorTransformationContext, n: PNode): PNode =
+  result = newNode(nkWhileStmt)
+  result.add(newSymNode(getSysSym("true")))
+
+  let loopBody = newNodeI(nkStmtList, n.info)
+  result.add(loopBody)
+
+  if not ctx.stateVarSym.isNil:
+    let varSect = newNodeI(nkVarSection, n.info)
+    addVar(varSect, newSymNode(ctx.stateVarSym))
+    loopBody.add(varSect)
+
+    if not ctx.tempVars.isNil:
+      loopBody.add(ctx.tempVars)
+
+  let blockStmt = newNodeI(nkBlockStmt, n.info)
+  blockStmt.add(newSymNode(ctx.stateLoopLabel))
+
+  let blockBody = newNodeI(nkStmtList, n.info)
+  blockStmt.add(blockBody)
+
+  let gs = newNodeI(nkGotoState, n.info)
+  if ctx.stateVarSym.isNil:
+    gs.add(rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)), getStateField(ctx.fn), n.info))
+  else:
+    gs.add(newSymNode(ctx.stateVarSym))
+
+  gs.add(newIntLit(ctx.states.len - 1))
+  blockBody.add(gs)
+  blockBody.add(n)
+  # gs.add(rawIndirectAccess(newSymNode(ctx.fn.getHiddenParam), getStateField(ctx.fn), n.info))
+
+  loopBody.add(blockStmt)
+
+proc deleteEmptyStates(ctx: var ClosureIteratorTransformationContext) =
+  let goOut = newNode(nkGotoState)
+  goOut.add(newIntLit(-1))
+
+  ctx.exitStateIdx = ctx.newState(goOut, nil)
+
+  # Apply new state indexes and mark unused states with -1
+  var iValid = 0
+  for i, s in ctx.states:
+    let body = s[1].skipStmtList()
+    if body.kind == nkGotoState and i != ctx.states.len - 1:
+      # This is an empty state. Mark with -1.
+      s[0].intVal = -1
+    else:
+      s[0].intVal = iValid
+      inc iValid
+
+  for i, s in ctx.states:
+    let body = s[1].skipStmtList()
+    if body.kind != nkGotoState:
+      discard ctx.skipThroughEmptyStates(s)
+
+  var i = 0
+  while i < ctx.states.len - 1:
+    let fs = ctx.states[i][1].skipStmtList()
+    if fs.kind == nkGotoState:
+      ctx.states.delete(i)
+    else:
+      inc i
+
+proc transformClosureIterator*(fn: PSym, n: PNode): PNode =
+  var ctx: ClosureIteratorTransformationContext
+  ctx.fn = fn
+
+  if getEnvParam(fn).isNil:
+    # Lambda lifting was not done yet. Use temporary :state sym, which
+    # be handled specially by lambda lifting. Local temp vars (if needed)
+    # should folllow the same logic.
+    ctx.stateVarSym = newSym(skVar, getIdent(":state"), fn, fn.info)
+    ctx.stateVarSym.typ = createClosureIterStateType(fn)
+
+  ctx.states = @[]
+  ctx.stateLoopLabel = newSym(skLabel, getIdent(":stateLoop"), fn, fn.info)
+  let n = n.toStmtList
+
+  discard ctx.newState(n, nil)
+  let gotoOut = newNode(nkGotoState)
+  gotoOut.add(newIntLit(-1))
+
+  # Splitting transformation
+  discard ctx.transformClosureIteratorBody(n, gotoOut)
+
+  # Optimize empty states away
+  ctx.deleteEmptyStates()
+
+  # Make new body by concating the list of states
+  result = newNode(nkStmtList)
+  for i, s in ctx.states:
+    # result.add(s)
+    let body = s[1]
+    s.sons.del(1)
+    result.add(s)
+    result.add(body)
+
+  result = ctx.tranformStateAssignments(result)
+
+  # Add excpetion handling
+  var hasExceptions = false
+  if hasExceptions:
+    discard # TODO:
+    # result = wrapIntoTryCatch(result)
+
+  # while true:
+  #   block :stateLoop:
+  #     gotoState
+  #     body
+  result = ctx.wrapIntoStateLoop(result)
+
+  # echo "TRANSFORM TO STATES2: "
+  # debug(result)
+  # echo renderTree(result)
diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim
index 7757484250..a118edf00f 100644
--- a/compiler/lambdalifting.nim
+++ b/compiler/lambdalifting.nim
@@ -7,11 +7,11 @@
 #    distribution, for details about the copyright.
 #
 
-# This include file implements lambda lifting for the transformator.
+# This file implements lambda lifting for the transformator.
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os,
-  idents, renderer, types, magicsys, rodread, lowerings, tables
+  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os, options,
+  idents, renderer, types, magicsys, rodread, lowerings, tables, sequtils
 
 discard """
   The basic approach is that captured vars need to be put on the heap and
@@ -125,7 +125,7 @@ proc newCall(a: PSym, b: PNode): PNode =
   result.add newSymNode(a)
   result.add b
 
-proc createStateType(iter: PSym): PType =
+proc createClosureIterStateType*(iter: PSym): PType =
   var n = newNodeI(nkRange, iter.info)
   addSon(n, newIntNode(nkIntLit, -1))
   addSon(n, newIntNode(nkIntLit, 0))
@@ -137,7 +137,7 @@ proc createStateType(iter: PSym): PType =
 
 proc createStateField(iter: PSym): PSym =
   result = newSym(skField, getIdent(":state"), iter, iter.info)
-  result.typ = createStateType(iter)
+  result.typ = createClosureIterStateType(iter)
 
 proc createEnvObj(owner: PSym; info: TLineInfo): PType =
   # YYY meh, just add the state field for every closure for now, it's too
@@ -145,7 +145,7 @@ proc createEnvObj(owner: PSym; info: TLineInfo): PType =
   result = createObj(owner, info, final=false)
   rawAddField(result, createStateField(owner))
 
-proc getIterResult(iter: PSym): PSym =
+proc getClosureIterResult*(iter: PSym): PSym =
   if resultPos < iter.ast.len:
     result = iter.ast.sons[resultPos].sym
   else:
@@ -397,7 +397,11 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
           if not c.capturedVars.containsOrIncl(s.id):
             let obj = getHiddenParam(owner).typ.lastSon
             #let obj = c.getEnvTypeForOwner(s.owner).lastSon
-            addField(obj, s)
+
+            if s.name == getIdent(":state"):
+              obj.n[0].sym.id = -s.id
+            else:
+              addField(obj, s)
       # but always return because the rest of the proc is only relevant when
       # ow != owner:
       return
@@ -461,6 +465,7 @@ type
     processed: IntSet
     envVars: Table[int, PNode]
     inContainer: int
+    features: set[Feature]
 
 proc initLiftingPass(fn: PSym): LiftingPass =
   result.processed = initIntSet()
@@ -595,7 +600,7 @@ proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
     localError(n.info, "internal error: not part of closure object type")
     result = n
 
-proc getStateField(owner: PSym): PSym =
+proc getStateField*(owner: PSym): PSym =
   getHiddenParam(owner).typ.sons[0].n.sons[0].sym
 
 proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
@@ -621,7 +626,7 @@ proc transformYield(n: PNode; owner: PSym; d: DetectionPass;
   if n.sons[0].kind != nkEmpty:
     var a = newNodeI(nkAsgn, n.sons[0].info)
     var retVal = liftCapturedVars(n.sons[0], owner, d, c)
-    addSon(a, newSymNode(getIterResult(owner)))
+    addSon(a, newSymNode(getClosureIterResult(owner)))
     addSon(a, retVal)
     retStmt.add(a)
   else:
@@ -713,7 +718,9 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         #  echo renderTree(s.getBody, {renderIds})
         let oldInContainer = c.inContainer
         c.inContainer = 0
-        let body = wrapIterBody(liftCapturedVars(s.getBody, s, d, c), s)
+        var body = liftCapturedVars(s.getBody, s, d, c)
+        if oldIterTransf in c.features:
+          body = wrapIterBody(body, s)
         if c.envvars.getOrDefault(s.id).isNil:
           s.ast.sons[bodyPos] = body
         else:
@@ -756,9 +763,9 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
       if n[1].kind == nkClosure: result = n[1]
   else:
     if owner.isIterator:
-      if n.kind == nkYieldStmt:
+      if oldIterTransf in c.features and n.kind == nkYieldStmt:
         return transformYield(n, owner, d, c)
-      elif n.kind == nkReturnStmt:
+      elif oldIterTransf in c.features and n.kind == nkReturnStmt:
         return transformReturn(n, owner, d, c)
       elif nfLL in n.flags:
         # special case 'when nimVm' due to bug #3636:
@@ -805,7 +812,7 @@ proc liftIterToProc*(fn: PSym; body: PNode; ptrType: PType): PNode =
   fn.kind = oldKind
   fn.typ.callConv = oldCC
 
-proc liftLambdas*(fn: PSym, body: PNode; tooEarly: var bool): PNode =
+proc liftLambdas*(features: set[Feature], fn: PSym, body: PNode; tooEarly: var bool): PNode =
   # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
 
@@ -815,6 +822,7 @@ proc liftLambdas*(fn: PSym, body: PNode; tooEarly: var bool): PNode =
   if body.kind == nkEmpty or (
       gCmd == cmdCompileToJS and not isCompileTime) or
       fn.skipGenericOwner.kind != skModule:
+
     # ignore forward declaration:
     result = body
     tooEarly = true
@@ -826,10 +834,13 @@ proc liftLambdas*(fn: PSym, body: PNode; tooEarly: var bool): PNode =
       d.somethingToDo = true
     if d.somethingToDo:
       var c = initLiftingPass(fn)
-      var newBody = liftCapturedVars(body, fn, d, c)
+      c.features = features
+      result = liftCapturedVars(body, fn, d, c)
       if c.envvars.getOrDefault(fn.id) != nil:
-        newBody = newTree(nkStmtList, rawClosureCreation(fn, d, c), newBody)
-      result = wrapIterBody(newBody, fn)
+        result = newTree(nkStmtList, rawClosureCreation(fn, d, c), result)
+
+      if oldIterTransf in features:
+        result = wrapIterBody(result, fn)
     else:
       result = body
     #if fn.name.s == "get2":
@@ -870,7 +881,9 @@ proc liftForLoop*(body: PNode; owner: PSym): PNode =
       cl = createClosure()
       while true:
         let i = foo(cl)
-        nkBreakState(cl.state)
+        if cl.state < 0:
+          break
+        # nkBreakState(cl.state)
         ...
     """
   if liftingHarmful(owner): return body
@@ -930,5 +943,16 @@ proc liftForLoop*(body: PNode; owner: PSym): PNode =
   loopBody.sons[0] = v2
   var bs = newNodeI(nkBreakState, body.info)
   bs.addSon(call.sons[0])
-  loopBody.sons[1] = bs
+
+  let ibs = newNode(nkIfStmt)
+  let elifBranch = newNode(nkElifBranch)
+  elifBranch.add(bs)
+
+  let br = newNode(nkBreakStmt)
+  br.add(emptyNode)
+
+  elifBranch.add(br)
+  ibs.add(elifBranch)
+
+  loopBody.sons[1] = ibs
   loopBody.sons[2] = body[L-1]
diff --git a/compiler/options.nim b/compiler/options.nim
index f8cb735ae6..0ce2f95ce9 100644
--- a/compiler/options.nim
+++ b/compiler/options.nim
@@ -110,7 +110,8 @@ type
     callOperator,
     parallel,
     destructor,
-    notnil
+    notnil,
+    oldIterTransf
 
   ConfigRef* = ref object ## eventually all global configuration should be moved here
     linesCompiled*: int  # all lines that have been compiled
diff --git a/compiler/renderer.nim b/compiler/renderer.nim
index 996168412b..0c861bdb88 100644
--- a/compiler/renderer.nim
+++ b/compiler/renderer.nim
@@ -1411,11 +1411,21 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     put(g, tkParLe, "(ComesFrom|")
     gsub(g, n, 0)
     put(g, tkParRi, ")")
-  of nkGotoState, nkState:
+  of nkGotoState:
     var c: TContext
     initContext c
-    putWithSpace g, tkSymbol, if n.kind == nkState: "state" else: "goto"
+    putWithSpace g, tkSymbol, "goto"
     gsons(g, n, c)
+  of nkState:
+    var c: TContext
+    initContext c
+    putWithSpace g, tkSymbol, "state"
+    gsub(g, n[0], c)
+    putWithSpace(g, tkColon, ":")
+    indentNL(g)
+    gsons(g, n, c, 1)
+    dedent(g)
+
   of nkBreakState:
     put(g, tkTuple, "breakstate")
   of nkTypeClassTy:
diff --git a/compiler/seminst.nim b/compiler/seminst.nim
index 32b3853089..9ec7d87985 100644
--- a/compiler/seminst.nim
+++ b/compiler/seminst.nim
@@ -145,7 +145,7 @@ proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
     freshGenSyms(b, result, orig, symMap)
     b = semProcBody(c, b)
     b = hloBody(c, b)
-    n.sons[bodyPos] = transformBody(c.module, b, result)
+    n.sons[bodyPos] = transformBody(c, b, result)
     #echo "code instantiated ", result.name.s
     excl(result.flags, sfForward)
     dec c.inGenericInst
diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index f3cf4196f2..dfee20a99a 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -1236,7 +1236,7 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
       addResult(c, s.typ.sons[0], n.info, skProc)
       addResultNode(c, n)
       let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
-      n.sons[bodyPos] = transformBody(c.module, semBody, s)
+      n.sons[bodyPos] = transformBody(c, semBody, s)
       popProcCon(c)
     elif efOperand notin flags:
       localError(n.info, errGenericLambdaNotAllowed)
@@ -1277,7 +1277,7 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
   addResult(c, n.typ.sons[0], n.info, skProc)
   addResultNode(c, n)
   let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
-  n.sons[bodyPos] = transformBody(c.module, semBody, s)
+  n.sons[bodyPos] = transformBody(c, semBody, s)
   popProcCon(c)
   popOwner(c)
   closeScope(c)
@@ -1590,7 +1590,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
           let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
           # unfortunately we cannot skip this step when in 'system.compiles'
           # context as it may even be evaluated in 'system.compiles':
-          n.sons[bodyPos] = transformBody(c.module, semBody, s)
+          n.sons[bodyPos] = transformBody(c, semBody, s)
       else:
         if s.typ.sons[0] != nil and kind != skIterator:
           addDecl(c, newSym(skUnknown, getIdent"result", nil, n.info))
diff --git a/compiler/transf.nim b/compiler/transf.nim
index f7ec6c97f2..c0f5e5e327 100644
--- a/compiler/transf.nim
+++ b/compiler/transf.nim
@@ -19,9 +19,10 @@
 # * transforms 'defer' into a 'try finally' statement
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os,
-  idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread,
-  lambdalifting, sempass2, lowerings, lookups, destroyer, liftlocals
+  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os, lookups,
+  idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread, semdata,
+  lambdalifting, sempass2, lowerings, destroyer, liftlocals, closureiters
+
 
 type
   PTransNode* = distinct PNode
@@ -967,20 +968,22 @@ template liftDefer(c, root) =
   if c.deferDetected:
     liftDeferAux(root)
 
-proc transformBody*(module: PSym, n: PNode, prc: PSym): PNode =
-  if nfTransf in n.flags or prc.kind in {skTemplate}:
-    result = n
-  else:
-    var c = openTransf(module, "")
-    result = liftLambdas(prc, n, c.tooEarly)
-    #result = n
+proc transformBody*(ctx: PContext, n: PNode, prc: PSym): PNode =
+  result = n
+  if nfTransf notin n.flags and prc.kind notin {skTemplate}:
+    var c = openTransf(ctx.module, "")
+    result = liftLambdas(ctx.features, prc, result, c.tooEarly)
     result = processTransf(c, result, prc)
     liftDefer(c, result)
-    #result = liftLambdas(prc, result)
+
     when useEffectSystem: trackProc(prc, result)
     result = liftLocalsIfRequested(prc, result)
     if c.needsDestroyPass: #and newDestructors:
       result = injectDestructorCalls(prc, result)
+
+    if prc.isIterator and oldIterTransf notin ctx.features:
+      result = transformClosureIterator(prc, result)
+
     incl(result.flags, nfTransf)
       #if prc.name.s == "testbody":
     #  echo renderTree(result)
diff --git a/tests/async/tasync_in_seq_constr.nim b/tests/async/tasync_in_seq_constr.nim
index 46ad744512..cf9bb54516 100644
--- a/tests/async/tasync_in_seq_constr.nim
+++ b/tests/async/tasync_in_seq_constr.nim
@@ -1,6 +1,5 @@
 discard """
-  errormsg: "invalid control flow: 'yield' within a constructor"
-  line: 16
+  output: "@[1, 2, 3, 4]"
 """
 
 # bug #5314, bug #6626

From 7d38db284ba2655bf19cb9c0785240616074ee44 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Mon, 30 Apr 2018 21:39:41 +0300
Subject: [PATCH 02/51] Extended tasync_in_seq_constr test

---
 tests/async/tasync_in_seq_constr.nim | 20 ++++++++++++++------
 1 file changed, 14 insertions(+), 6 deletions(-)

diff --git a/tests/async/tasync_in_seq_constr.nim b/tests/async/tasync_in_seq_constr.nim
index cf9bb54516..3d6dae2457 100644
--- a/tests/async/tasync_in_seq_constr.nim
+++ b/tests/async/tasync_in_seq_constr.nim
@@ -1,17 +1,25 @@
 discard """
-  output: "@[1, 2, 3, 4]"
+  output: '''
+@[1, 2, 3, 4]
+123
+'''
 """
 
 # bug #5314, bug #6626
 
 import asyncdispatch
 
-proc bar(): Future[int] {.async.} =
-    await sleepAsync(500)
-    result = 3
+proc bar(i: int): Future[int] {.async.} =
+    await sleepAsync(2)
+    result = i
 
 proc foo(): Future[seq[int]] {.async.} =
-    await sleepAsync(500)
-    result = @[1, 2, await bar(), 4] # <--- The bug is here
+    await sleepAsync(2)
+    result = @[1, 2, await bar(3), 4] # <--- The bug is here
+
+proc foo2() {.async.} =
+    await sleepAsync(2)
+    echo(await bar(1), await bar(2), await bar(3))
 
 echo waitFor foo()
+waitFor foo2()

From 13167c85f6192917ebfbf224b0cc53c11ea75710 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Tue, 1 May 2018 10:04:52 +0300
Subject: [PATCH 03/51] Cosmetics

---
 compiler/closureiters.nim | 45 ++++++++++++++++++++-------------------
 1 file changed, 23 insertions(+), 22 deletions(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index 02795ab470..c94d90bd14 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -65,18 +65,19 @@ import
   idents, renderer, types, magicsys, rodread, lowerings, tables, sequtils,
   lambdalifting
 
-type ClosureIteratorTransformationContext = object
-  fn: PSym
-  stateVarSym: PSym # :state variable. nil if env already introduced by lambdalifting
-  states: seq[PNode] # The resulting states. Every state is an nkState node.
-  blockLevel: int # Temp used to transform break and continue stmts
-  stateLoopLabel: PSym # Label to break on, when jumping between states.
-  exitStateIdx: int # index of the last state
-  tempVarId: int # unique name counter
-  tempVars: PNode # Temp var decls, nkVarSection
-  loweredStmtListExpr: PNode # Temporary used for nkStmtListExpr lowering
+type
+  Ctx = object
+    fn: PSym
+    stateVarSym: PSym # :state variable. nil if env already introduced by lambdalifting
+    states: seq[PNode] # The resulting states. Every state is an nkState node.
+    blockLevel: int # Temp used to transform break and continue stmts
+    stateLoopLabel: PSym # Label to break on, when jumping between states.
+    exitStateIdx: int # index of the last state
+    tempVarId: int # unique name counter
+    tempVars: PNode # Temp var decls, nkVarSection
+    loweredStmtListExpr: PNode # Temporary used for nkStmtListExpr lowering
 
-proc newStateAssgn(ctx: var ClosureIteratorTransformationContext, stateNo: int = -2): PNode =
+proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
   # Creates state assignmen:
   #   :state = stateNo
 
@@ -95,7 +96,7 @@ proc setStateInAssgn(stateAssgn: PNode, stateNo: int) =
   assert stateAssgn[1].kind == nkIntLit
   stateAssgn[1].intVal = stateNo
 
-proc newState(ctx: var ClosureIteratorTransformationContext, n, gotoOut: PNode): int =
+proc newState(ctx: var Ctx, n, gotoOut: PNode): int =
   # Creates a new state, adds it to the context fills out `gotoOut` so that it
   # will goto this state.
   # Returns index of the newly created state
@@ -123,7 +124,7 @@ proc addGotoOut(n: PNode, gotoOut: PNode): PNode =
   if result.len != 0 and result.sons[^1].kind != nkGotoState:
     result.add(gotoOut)
 
-proc newTempVarAccess(ctx: var ClosureIteratorTransformationContext, typ: PType, i: TLineInfo): PNode =
+proc newTempVarAccess(ctx: var Ctx, typ: PType, i: TLineInfo): PNode =
   if not ctx.stateVarSym.isNil:
     # We haven't gone through labmda lifting yet, so just create a local var,
     # it will be lifted later
@@ -157,7 +158,7 @@ proc hasYields(n: PNode): bool =
         result = true
         break
 
-proc transformBreaksAndContinuesInWhile(ctx: var ClosureIteratorTransformationContext, n: PNode, before, after: PNode): PNode =
+proc transformBreaksAndContinuesInWhile(ctx: var Ctx, n: PNode, before, after: PNode): PNode =
   result = n
   case n.kind
   of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
@@ -177,7 +178,7 @@ proc transformBreaksAndContinuesInWhile(ctx: var ClosureIteratorTransformationCo
     for i in 0 ..< n.len:
       n[i] = ctx.transformBreaksAndContinuesInWhile(n[i], before, after)
 
-proc transformBreaksInBlock(ctx: var ClosureIteratorTransformationContext, n: PNode, label, after: PNode): PNode =
+proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode =
   result = n
   case n.kind
   of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
@@ -252,7 +253,7 @@ proc hasYieldsInExpressions(n: PNode): bool =
       if c.hasYieldsInExpressions:
         return true
 
-proc lowerStmtListExpr(ctx: var ClosureIteratorTransformationContext, n: PNode): PNode =
+proc lowerStmtListExpr(ctx: var Ctx, n: PNode): PNode =
   result = n
   case n.kind
   of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
@@ -274,7 +275,7 @@ proc lowerStmtListExpr(ctx: var ClosureIteratorTransformationContext, n: PNode):
     for i in 0 ..< n.len:
       n[i] = ctx.lowerStmtListExpr(n[i])
 
-proc transformClosureIteratorBody(ctx: var ClosureIteratorTransformationContext, n: PNode, gotoOut: PNode): PNode =
+proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode =
   result = n
   case n.kind:
     of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
@@ -412,7 +413,7 @@ proc stateFromGotoState(n: PNode): int =
   assert(n.kind == nkGotoState)
   result = n[0].intVal.int
 
-proc tranformStateAssignments(ctx: var ClosureIteratorTransformationContext, n: PNode): PNode =
+proc tranformStateAssignments(ctx: var Ctx, n: PNode): PNode =
   # This transforms 3 patterns:
   ########################## 1
   # yield e
@@ -484,7 +485,7 @@ proc skipStmtList(n: PNode): PNode =
     if result.len == 0: return emptyNode
     result = result[0]
 
-proc skipThroughEmptyStates(ctx: var ClosureIteratorTransformationContext, n: PNode): PNode =
+proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
   result = n
   case n.kind
   of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
@@ -515,7 +516,7 @@ proc skipThroughEmptyStates(ctx: var ClosureIteratorTransformationContext, n: PN
     for i in 0 ..< n.len:
       n[i] = ctx.skipThroughEmptyStates(n[i])
 
-proc wrapIntoStateLoop(ctx: var ClosureIteratorTransformationContext, n: PNode): PNode =
+proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   result = newNode(nkWhileStmt)
   result.add(newSymNode(getSysSym("true")))
 
@@ -549,7 +550,7 @@ proc wrapIntoStateLoop(ctx: var ClosureIteratorTransformationContext, n: PNode):
 
   loopBody.add(blockStmt)
 
-proc deleteEmptyStates(ctx: var ClosureIteratorTransformationContext) =
+proc deleteEmptyStates(ctx: var Ctx) =
   let goOut = newNode(nkGotoState)
   goOut.add(newIntLit(-1))
 
@@ -580,7 +581,7 @@ proc deleteEmptyStates(ctx: var ClosureIteratorTransformationContext) =
       inc i
 
 proc transformClosureIterator*(fn: PSym, n: PNode): PNode =
-  var ctx: ClosureIteratorTransformationContext
+  var ctx: Ctx
   ctx.fn = fn
 
   if getEnvParam(fn).isNil:

From 48d8e215d53b5b231483b208def8b3ed8e5fdf85 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Tue, 1 May 2018 10:31:49 +0300
Subject: [PATCH 04/51] Don't leak sem PContext into transf

---
 compiler/seminst.nim  |  2 +-
 compiler/semstmts.nim |  6 +++---
 compiler/transf.nim   | 11 +++++------
 3 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/compiler/seminst.nim b/compiler/seminst.nim
index 9ec7d87985..0513e23956 100644
--- a/compiler/seminst.nim
+++ b/compiler/seminst.nim
@@ -145,7 +145,7 @@ proc instantiateBody(c: PContext, n, params: PNode, result, orig: PSym) =
     freshGenSyms(b, result, orig, symMap)
     b = semProcBody(c, b)
     b = hloBody(c, b)
-    n.sons[bodyPos] = transformBody(c, b, result)
+    n.sons[bodyPos] = transformBody(c.module, c.features, b, result)
     #echo "code instantiated ", result.name.s
     excl(result.flags, sfForward)
     dec c.inGenericInst
diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim
index dfee20a99a..9f00e877f2 100644
--- a/compiler/semstmts.nim
+++ b/compiler/semstmts.nim
@@ -1236,7 +1236,7 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode =
       addResult(c, s.typ.sons[0], n.info, skProc)
       addResultNode(c, n)
       let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
-      n.sons[bodyPos] = transformBody(c, semBody, s)
+      n.sons[bodyPos] = transformBody(c.module, c.features, semBody, s)
       popProcCon(c)
     elif efOperand notin flags:
       localError(n.info, errGenericLambdaNotAllowed)
@@ -1277,7 +1277,7 @@ proc semInferredLambda(c: PContext, pt: TIdTable, n: PNode): PNode =
   addResult(c, n.typ.sons[0], n.info, skProc)
   addResultNode(c, n)
   let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
-  n.sons[bodyPos] = transformBody(c, semBody, s)
+  n.sons[bodyPos] = transformBody(c.module, c.features, semBody, s)
   popProcCon(c)
   popOwner(c)
   closeScope(c)
@@ -1590,7 +1590,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
           let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos]))
           # unfortunately we cannot skip this step when in 'system.compiles'
           # context as it may even be evaluated in 'system.compiles':
-          n.sons[bodyPos] = transformBody(c, semBody, s)
+          n.sons[bodyPos] = transformBody(c.module, c.features, semBody, s)
       else:
         if s.typ.sons[0] != nil and kind != skIterator:
           addDecl(c, newSym(skUnknown, getIdent"result", nil, n.info))
diff --git a/compiler/transf.nim b/compiler/transf.nim
index c0f5e5e327..75c7f2b6c3 100644
--- a/compiler/transf.nim
+++ b/compiler/transf.nim
@@ -20,10 +20,9 @@
 
 import
   intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os, lookups,
-  idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread, semdata,
+  idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread,
   lambdalifting, sempass2, lowerings, destroyer, liftlocals, closureiters
 
-
 type
   PTransNode* = distinct PNode
 
@@ -968,11 +967,11 @@ template liftDefer(c, root) =
   if c.deferDetected:
     liftDeferAux(root)
 
-proc transformBody*(ctx: PContext, n: PNode, prc: PSym): PNode =
+proc transformBody*(module: PSym, features: set[Feature], n: PNode, prc: PSym): PNode =
   result = n
   if nfTransf notin n.flags and prc.kind notin {skTemplate}:
-    var c = openTransf(ctx.module, "")
-    result = liftLambdas(ctx.features, prc, result, c.tooEarly)
+    var c = openTransf(module, "")
+    result = liftLambdas(features, prc, result, c.tooEarly)
     result = processTransf(c, result, prc)
     liftDefer(c, result)
 
@@ -981,7 +980,7 @@ proc transformBody*(ctx: PContext, n: PNode, prc: PSym): PNode =
     if c.needsDestroyPass: #and newDestructors:
       result = injectDestructorCalls(prc, result)
 
-    if prc.isIterator and oldIterTransf notin ctx.features:
+    if prc.isIterator and oldIterTransf notin features:
       result = transformClosureIterator(prc, result)
 
     incl(result.flags, nfTransf)

From 0ed6c3e476e421827d081e9ab0d9fcb0d3de5eb2 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Tue, 1 May 2018 13:19:01 +0300
Subject: [PATCH 05/51] Minor dry up

---
 compiler/closureiters.nim | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index c94d90bd14..7653176de2 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -125,12 +125,12 @@ proc addGotoOut(n: PNode, gotoOut: PNode): PNode =
     result.add(gotoOut)
 
 proc newTempVarAccess(ctx: var Ctx, typ: PType, i: TLineInfo): PNode =
+  let s = newSym(skVar, getIdent(":tmpSlLower" & $ctx.tempVarId), ctx.fn, i)
+  s.typ = typ
+
   if not ctx.stateVarSym.isNil:
     # We haven't gone through labmda lifting yet, so just create a local var,
     # it will be lifted later
-    let s = newSym(skVar, getIdent(":tmpSlLower" & $ctx.tempVarId), ctx.fn, i)
-    s.typ = typ
-
     if ctx.tempVars.isNil:
       ctx.tempVars = newNode(nkVarSection)
       addVar(ctx.tempVars, newSymNode(s))
@@ -138,8 +138,6 @@ proc newTempVarAccess(ctx: var Ctx, typ: PType, i: TLineInfo): PNode =
     result = newSymNode(s)
   else:
     # Lambda lifting is done, insert temp var to env.
-    let s = newSym(skVar, getIdent(":tmpSlLower" & $ctx.tempVarId), ctx.fn, i)
-    s.typ = typ
     result = freshVarForClosureIter(s, ctx.fn)
 
   inc ctx.tempVarId

From ce634909281ffc8efbc7d192f557ffe38f49e740 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Fri, 4 May 2018 15:23:47 +0300
Subject: [PATCH 06/51] Yield in try

---
 compiler/closureiters.nim  | 590 +++++++++++++++++++++++++++++++------
 compiler/semexprs.nim      |   2 +-
 lib/system/embedded.nim    |   3 +
 lib/system/excpt.nim       |   4 +
 tests/async/tasynctry2.nim |   4 +-
 tests/iter/tyieldintry.nim | 201 +++++++++++++
 6 files changed, 711 insertions(+), 93 deletions(-)
 create mode 100644 tests/iter/tyieldintry.nim

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index 7653176de2..504f70347d 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -59,6 +59,80 @@
 #   if :tmpSlLower == 2:
 #     yield 3
 
+# nkTryStmt Transformations:
+# If the iter has an nkTryStmt with a yield inside
+#  - the closure iter is promoted to have exceptions (ctx.hasExceptions = true)
+#  - exception table is created. This is a const array, where
+#    `abs(exceptionTable[i])` is a state idx to which we should jump from state
+#    `i` should exception be raised in state `i`. For all states in `try` block
+#    the target state is `except` block. For all states in `except` block
+#    the target state is `finally` block. For all other states there is no
+#    target state (0, as the first block can never be neither except nor finally).
+#    `exceptionTable[i]` is < 0 if `abs(exceptionTable[i])` is except block,
+#    and > 0, for finally block.
+#  - local variable :curExc is created
+#  - the iter body is wrapped into a
+#      try:
+#       closureIterSetupExc(:curExc)
+#       ...body...
+#      catch:
+#        :state = exceptionTable[:state]
+#        if :state == 0: raise # No state that could handle exception
+#        :unrollFinally = :state > 0 # Target state is finally
+#        if :state < 0:
+#           :state = -:state
+#        :curExc = getCurrentException()
+#
+# nkReturnStmt within a try/except/finally now has to behave differently as we
+# want the nearest finally block to be executed before the return, thus it is
+# transformed to:
+#  :tmpResult = returnValue (if return doesn't have a value, this is skipped)
+#  :unrollFinally = true
+#  goto nearestFinally (or -1 if not exists)
+#
+# Every finally block calls closureIterEndFinally() upon its successful
+# completion.
+#
+# Example:
+#
+# try:
+#  yield 0
+#  raise ...
+# except:
+#  yield 1
+#  return 3
+# finally:
+#  yield 2
+#
+# Is transformed to (yields are left in place for example simplicity,
+#    in reality the code is subdivided even more, as described above):
+#
+# STATE0: # Try
+#   yield 0
+#   raise ...
+#   :state = 2 # What would happen should we not raise
+#   break :stateLoop
+# STATE1: # Except
+#   yield 1
+#   :tmpResult = 3           # Return
+#   :unrollFinally = true # Return
+#   :state = 2 # Goto Finally
+#   break :stateLoop
+#   :state = 2 # What would happen should we not return
+#   break :stateLoop
+# STATE2: # Finally
+#   yield 2
+#   if :unrollFinally: # This node is created by `newEndFinallyNode`
+#     when nearestFinally == 0: # Pseudocode. The `when` is not emitted in reality
+#       if :curExc.isNil:
+#         return :tmpResult
+#       else:
+#         raise
+#     else:
+#       :state = nearestFinally
+#       break :stateLoop
+#   state = -1 # Goto next state. In this case we just exit
+#   break :stateLoop
 
 import
   intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os, options,
@@ -69,6 +143,10 @@ type
   Ctx = object
     fn: PSym
     stateVarSym: PSym # :state variable. nil if env already introduced by lambdalifting
+    tmpResultSym: PSym # Used when we return, but finally has to interfere
+    unrollFinallySym: PSym # Indicates that we're unrolling finally states (either exception happened or premature return)
+    curExcSym: PSym # Current exception
+
     states: seq[PNode] # The resulting states. Every state is an nkState node.
     blockLevel: int # Temp used to transform break and continue stmts
     stateLoopLabel: PSym # Label to break on, when jumping between states.
@@ -76,20 +154,66 @@ type
     tempVarId: int # unique name counter
     tempVars: PNode # Temp var decls, nkVarSection
     loweredStmtListExpr: PNode # Temporary used for nkStmtListExpr lowering
+    exceptionTable: seq[int] # For state `i` jump to state `exceptionTable[i]` if exception is raised
+    hasExceptions: bool # Does closure have yield in try?
+    curExcHandlingState: int # Negative for except, positive for finally
+    nearestFinally: int # Index of the nearest finally block. For try/except it
+                    # is their finally. For finally it is parent finally. Otherwise -1
+
+proc newStateAccess(ctx: var Ctx): PNode =
+  if ctx.stateVarSym.isNil:
+    result = rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)), getStateField(ctx.fn), ctx.fn.info)
+  else:
+    result = newSymNode(ctx.stateVarSym)
+
+proc newStateAssgn(ctx: var Ctx, toValue: PNode): PNode =
+  # Creates state assignment:
+  #   :state = toValue
+  result = newNode(nkAsgn)
+  result.add(ctx.newStateAccess())
+  result.add(toValue)
 
 proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
-  # Creates state assignmen:
+  # Creates state assignment:
   #   :state = stateNo
+  ctx.newStateAssgn(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt)))
 
-  result = newNode(nkAsgn)
-  if ctx.stateVarSym.isNil:
-    let state = getStateField(ctx.fn)
-    assert state != nil
-    result.add(rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)),
-                      state, result.info))
+proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
+  result = newSym(skVar, getIdent(name), ctx.fn, ctx.fn.info)
+  result.typ = typ
+
+  if not ctx.stateVarSym.isNil:
+    # We haven't gone through labmda lifting yet, so just create a local var,
+    # it will be lifted later
+    if ctx.tempVars.isNil:
+      ctx.tempVars = newNode(nkVarSection)
+      addVar(ctx.tempVars, newSymNode(result))
   else:
-    result.add(newSymNode(ctx.stateVarSym))
-  result.add(newIntTypeNode(nkIntLit, stateNo, getSysType(tyInt)))
+    let envParam = getEnvParam(ctx.fn)
+    # let obj = envParam.typ.lastSon
+    result = addUniqueField(envParam.typ.lastSon, result)
+
+proc newEnvVarAccess(ctx: Ctx, s: PSym): PNode =
+  if ctx.stateVarSym.isNil:
+    result = rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)), s, ctx.fn.info)
+  else:
+    result = newSymNode(s)
+
+proc newTmpResultAccess(ctx: var Ctx): PNode =
+  if ctx.tmpResultSym.isNil:
+    debug(ctx.fn.typ)
+    ctx.tmpResultSym = ctx.newEnvVar(":tmpResult", ctx.fn.typ[0])
+  ctx.newEnvVarAccess(ctx.tmpResultSym)
+
+proc newUnrollFinallyAccess(ctx: var Ctx): PNode =
+  if ctx.unrollFinallySym.isNil:
+    ctx.unrollFinallySym = ctx.newEnvVar(":unrollFinally", getSysType(tyBool))
+  ctx.newEnvVarAccess(ctx.unrollFinallySym)
+
+proc newCurExcAccess(ctx: var Ctx): PNode =
+  if ctx.curExcSym.isNil:
+    ctx.curExcSym = ctx.newEnvVar(":curExc", callCodegenProc("getCurrentException", emptyNode).typ)
+  ctx.newEnvVarAccess(ctx.curExcSym)
 
 proc setStateInAssgn(stateAssgn: PNode, stateNo: int) =
   assert stateAssgn.kind == nkAsgn
@@ -107,6 +231,8 @@ proc newState(ctx: var Ctx, n, gotoOut: PNode): int =
   s.add(resLit)
   s.add(n)
   ctx.states.add(s)
+  ctx.exceptionTable.add(ctx.curExcHandlingState)
+
   if not gotoOut.isNil:
     assert(gotoOut.len == 0)
     gotoOut.add(newIntLit(result))
@@ -119,27 +245,13 @@ proc toStmtList(n: PNode): PNode =
 
 proc addGotoOut(n: PNode, gotoOut: PNode): PNode =
   # Make sure `n` is a stmtlist, and ends with `gotoOut`
-
   result = toStmtList(n)
   if result.len != 0 and result.sons[^1].kind != nkGotoState:
     result.add(gotoOut)
 
 proc newTempVarAccess(ctx: var Ctx, typ: PType, i: TLineInfo): PNode =
-  let s = newSym(skVar, getIdent(":tmpSlLower" & $ctx.tempVarId), ctx.fn, i)
-  s.typ = typ
-
-  if not ctx.stateVarSym.isNil:
-    # We haven't gone through labmda lifting yet, so just create a local var,
-    # it will be lifted later
-    if ctx.tempVars.isNil:
-      ctx.tempVars = newNode(nkVarSection)
-      addVar(ctx.tempVars, newSymNode(s))
-
-    result = newSymNode(s)
-  else:
-    # Lambda lifting is done, insert temp var to env.
-    result = freshVarForClosureIter(s, ctx.fn)
-
+  let s = ctx.newEnvVar(":tmpSlLower" & $ctx.tempVarId, typ)
+  result = ctx.newEnvVarAccess(s)
   inc ctx.tempVarId
 
 proc hasYields(n: PNode): bool =
@@ -197,7 +309,17 @@ proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode
     for i in 0 ..< n.len:
       n[i] = ctx.transformBreaksInBlock(n[i], label, after)
 
-proc collectExceptState(n: PNode): PNode =
+proc newNullifyCurExc(ctx: var Ctx): PNode =
+  # :curEcx = nil
+  result = newNode(nkAsgn)
+  let curExc = ctx.newCurExcAccess()
+  result.add(curExc)
+
+  let nilnode = newNode(nkNilLit)
+  nilnode.typ = curExc.typ
+  result.add(nilnode)
+
+proc collectExceptState(ctx: var Ctx, n: PNode): PNode =
   var ifStmt = newNode(nkIfStmt)
   for c in n:
     if c.kind == nkExceptBranch:
@@ -208,8 +330,10 @@ proc collectExceptState(n: PNode): PNode =
         assert(c.len == 2)
         ifBranch = newNode(nkElifBranch)
         let expression = newNodeI(nkCall, n.info)
+        expression.add(newSymNode(getSysMagic("of", mOf)))
         expression.add(callCodegenProc("getCurrentException", emptyNode))
         expression.add(c[0])
+        expression.typ = getSysType(tyBool)
         ifBranch.add(expression)
         branchBody = c[1]
       else:
@@ -226,10 +350,37 @@ proc collectExceptState(n: PNode): PNode =
 
   if ifStmt.len != 0:
     result = newNode(nkStmtList)
+    result.add(ctx.newNullifyCurExc())
     result.add(ifStmt)
   else:
     result = emptyNode
 
+proc addElseToExcept(ctx: var Ctx, n: PNode) =
+  if n.kind == nkStmtList and n[1].kind == nkIfStmt and n[1][^1].kind != nkElse:
+    # Not all cases are covered
+    let elseBranch = newNode(nkElse)
+    let branchBody = newNode(nkStmtList)
+
+    block: # :unrollFinally = true
+      let asgn = newNode(nkAsgn)
+      asgn.add(ctx.newUnrollFinallyAccess())
+      asgn.add(newIntTypeNode(nkIntLit, 1, getSysType(tyBool)))
+      branchBody.add(asgn)
+
+    block: # :curExc = getCurrentException()
+      let asgn = newNode(nkAsgn)
+      asgn.add(ctx.newCurExcAccess)
+      asgn.add(callCodegenProc("getCurrentException", emptyNode))
+      branchBody.add(asgn)
+
+    block: # goto nearestFinally
+      let goto = newNode(nkGotoState)
+      goto.add(newIntLit(ctx.nearestFinally))
+      branchBody.add(goto)
+
+    elseBranch.add(branchBody)
+    n[1].add(elseBranch)
+
 proc getFinallyNode(n: PNode): PNode =
   result = n[^1]
   if result.kind == nkFinally:
@@ -273,6 +424,101 @@ proc lowerStmtListExpr(ctx: var Ctx, n: PNode): PNode =
     for i in 0 ..< n.len:
       n[i] = ctx.lowerStmtListExpr(n[i])
 
+proc newEndFinallyNode(ctx: var Ctx): PNode =
+  # Generate the following code:
+  #   if :unrollFinally:
+  #     when nearestFinally == 0: # Pseudocode. The `when` is not emitted in reality
+  #       if :curExc.isNil:
+  #         return :tmpResult
+  #       else:
+  #         raise
+  #     else:
+  #       goto nearestFinally
+  #       :state = nearestFinally
+  #       break :stateLoop
+
+  result = newNode(nkIfStmt)
+
+  let elifBranch = newNode(nkElifBranch)
+  elifBranch.add(ctx.newUnrollFinallyAccess())
+  result.add(elifBranch)
+
+  var ifBody: PNode
+
+  if ctx.nearestFinally == 0 or true:
+    ifBody = newNode(nkIfStmt)
+    let branch = newNode(nkElifBranch)
+
+    let cmp = newNode(nkCall)
+    cmp.add(getSysMagic("==", mEqRef).newSymNode)
+    let curExc = ctx.newCurExcAccess()
+    let nilnode = newNode(nkNilLit)
+    nilnode.typ = curExc.typ
+    cmp.add(curExc)
+    cmp.add(nilnode)
+    cmp.typ = getSysType(tyBool)
+    branch.add(cmp)
+
+    var retStmt = newNode(nkReturnStmt)
+    if true:
+      var a = newNode(nkAsgn)
+      addSon(a, newSymNode(getClosureIterResult(ctx.fn)))
+      addSon(a, ctx.newTmpResultAccess())
+      retStmt.add(a)
+    else:
+      retStmt.add(emptyNode)
+    branch.add(retStmt)
+
+    let elseBranch = newNode(nkElse)
+    let raiseStmt = newNode(nkRaiseStmt)
+
+    # The C++ backend requires `getCurrentException` here.
+    raiseStmt.add(callCodegenProc("getCurrentException", emptyNode))
+    elseBranch.add(raiseStmt)
+
+    ifBody.add(branch)
+    ifBody.add(elseBranch)
+  else:
+    ifBody = newNode(nkGotoState)
+    ifBody.add(newIntLit(ctx.nearestFinally))
+
+  elifBranch.add(ifBody)
+
+proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
+  result = n
+  # TODO: This is very inefficient. It traverses the node, looking for nkYieldStmt.
+  case n.kind
+  of nkReturnStmt:
+    # We're somewhere in try, transform to finally unrolling
+    assert(ctx.nearestFinally != 0)
+
+    result = newNodeI(nkStmtList, n.info)
+
+    block: # :unrollFinally = true
+      let asgn = newNodeI(nkAsgn, n.info)
+      asgn.add(ctx.newUnrollFinallyAccess())
+      asgn.add(newIntTypeNode(nkIntLit, 1, getSysType(tyBool)))
+      result.add(asgn)
+
+    if n[0].kind != nkEmpty: # TODO: And not void!
+      let asgnTmpResult = newNodeI(nkAsgn, n.info)
+      asgnTmpResult.add(ctx.newTmpResultAccess())
+      asgnTmpResult.add(n[0])
+      result.add(asgnTmpResult)
+
+    result.add(ctx.newNullifyCurExc())
+
+    let goto = newNodeI(nkGotoState, n.info)
+    goto.add(newIntLit(ctx.nearestFinally))
+    result.add(goto)
+
+  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
+      nkSym, nkIdent, procDefs, nkTemplateDef:
+    discard
+  else:
+    for i in 0 ..< n.len:
+      n[i] = ctx.transformReturnsInTry(n[i])
+
 proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode =
   result = n
   case n.kind:
@@ -310,7 +556,6 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
       assert(false, "nkStmtListExpr not lowered")
 
     of nkYieldStmt:
-      # echo "YIELD!"
       result = newNodeI(nkStmtList, n.info)
       result.add(n)
       result.add(gotoOut)
@@ -371,34 +616,64 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
       result[1] = ctx.transformClosureIteratorBody(result[1], gotoOut)
 
     of nkTryStmt:
+      # See explanation above about how this works
+      ctx.hasExceptions = true
+
+      result = newNode(nkGotoState)
       var tryBody = toStmtList(n[0])
-
-      # let popTry = newNode(nkPar)
-      # popTry.add(newIdentNode(getIdent("popTry"), n.info))
+      var exceptBody = ctx.collectExceptState(n)
       var finallyBody = newNode(nkStmtList)
-      # finallyBody.add(popTry)
       finallyBody.add(getFinallyNode(n))
+      finallyBody = ctx.transformReturnsInTry(finallyBody)
+      finallyBody.add(ctx.newEndFinallyNode())
 
-      var tryCatchOut = newNode(nkGotoState)
-
-      tryBody = ctx.transformClosureIteratorBody(tryBody, tryCatchOut)
-      var exceptBody = collectExceptState(n)
-
-      var exceptIdx = -1
+      # The following index calculation is based on the knowledge how state
+      # indexes are assigned
+      let tryIdx = ctx.states.len
+      var exceptIdx, finallyIdx: int
       if exceptBody.kind != nkEmpty:
-        exceptBody = ctx.transformClosureIteratorBody(exceptBody, tryCatchOut)
-        exceptIdx = ctx.newState(exceptBody, nil)
+        exceptIdx = -(tryIdx + 1)
+        finallyIdx = tryIdx + 2
+      else:
+        exceptIdx = tryIdx + 1
+        finallyIdx = tryIdx + 1
 
-      finallyBody = ctx.transformClosureIteratorBody(finallyBody, gotoOut)
-      let finallyIdx = ctx.newState(finallyBody, tryCatchOut)
+      let outToFinally = newNode(nkGotoState)
 
-      # let pushTry = newNode(nkPar) #newCall(newSym("pushTry"), newIntLit(exceptIdx))
-      # pushTry.add(newIdentNode(getIdent("pushTry"), n.info))
-      # pushTry.add(newIntLit(exceptIdx))
-      # pushTry.add(newIntLit(finallyIdx))
-      # tryBody.sons.insert(pushTry, 0)
+      block: # Create initial states.
+        let oldExcHandlingState = ctx.curExcHandlingState
+        ctx.curExcHandlingState = exceptIdx
+        let realTryIdx = ctx.newState(tryBody, result)
+        assert(realTryIdx == tryIdx)
 
-      result = tryBody
+        if exceptBody.kind != nkEmpty:
+          ctx.curExcHandlingState = finallyIdx
+          let realExceptIdx = ctx.newState(exceptBody, nil)
+          assert(realExceptIdx == -exceptIdx)
+
+        ctx.curExcHandlingState = oldExcHandlingState
+        let realFinallyIdx = ctx.newState(finallyBody, outToFinally)
+        assert(realFinallyIdx == finallyIdx)
+
+      block: # Subdivide the states
+        let oldNearestFinally = ctx.nearestFinally
+        ctx.nearestFinally = finallyIdx
+
+        let oldExcHandlingState = ctx.curExcHandlingState
+
+        ctx.curExcHandlingState = exceptIdx
+
+        discard ctx.transformReturnsInTry(tryBody)
+        discard ctx.transformClosureIteratorBody(tryBody, outToFinally)
+
+        ctx.curExcHandlingState = finallyIdx
+        ctx.addElseToExcept(exceptBody)
+        discard ctx.transformReturnsInTry(exceptBody)
+        discard ctx.transformClosureIteratorBody(exceptBody, outToFinally)
+
+        ctx.curExcHandlingState = oldExcHandlingState
+        ctx.nearestFinally = oldNearestFinally
+        discard ctx.transformClosureIteratorBody(finallyBody, gotoOut)
 
     of nkGotoState, nkForStmt:
       internalError("closure iter " & $n.kind)
@@ -460,7 +735,6 @@ proc tranformStateAssignments(ctx: var Ctx, n: PNode): PNode =
     discard
 
   of nkReturnStmt:
-
     result = newNodeI(nkStmtList, n.info)
     result.add(ctx.newStateAssgn(-1))
     result.add(n)
@@ -483,6 +757,29 @@ proc skipStmtList(n: PNode): PNode =
     if result.len == 0: return emptyNode
     result = result[0]
 
+proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
+  # Returns first non-empty state idx for `stateIdx`. Returns `stateIdx` if
+  # it is not empty
+  var maxJumps = ctx.states.len # maxJumps used only for debugging purposes.
+  var stateIdx = stateIdx
+  while true:
+    let label = stateIdx
+    if label == ctx.exitStateIdx: break
+    var newLabel = label
+    if label == -1:
+      newLabel = ctx.exitStateIdx
+    else:
+      let fs = ctx.states[label][1].skipStmtList()
+      if fs.kind == nkGotoState:
+        newLabel = fs[0].intVal.int
+    if label == newLabel: break
+    stateIdx = newLabel
+    dec maxJumps
+    if maxJumps == 0:
+      assert(false, "Internal error")
+
+  result = ctx.states[stateIdx][0].intVal.int
+
 proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
   result = n
   case n.kind
@@ -490,31 +787,143 @@ proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
       nkSym, nkIdent, procDefs, nkTemplateDef:
     discard
   of nkGotoState:
-    var maxJumps = ctx.states.len # maxJumps used only for debugging purposes.
     result = copyTree(n)
-    while true:
-      let label = result[0].intVal.int
-      if label == ctx.exitStateIdx: break
-      var newLabel = label
-      if label == -1:
-        newLabel = ctx.exitStateIdx
-      else:
-        let fs = ctx.states[label][1].skipStmtList()
-        if fs.kind == nkGotoState:
-          newLabel = fs[0].intVal.int
-      if label == newLabel: break
-      result[0].intVal = newLabel
-      dec maxJumps
-      if maxJumps == 0:
-        assert(false, "Internal error")
-
-    let label = result[0].intVal.int
-    result[0].intVal = ctx.states[label][0].intVal
+    result[0].intVal = ctx.skipEmptyStates(result[0].intVal.int)
   else:
     for i in 0 ..< n.len:
       n[i] = ctx.skipThroughEmptyStates(n[i])
 
+proc newArrayType(n: int, t: PType, owner: PSym): PType =
+  result = newType(tyArray, owner)
+
+  let rng = newType(tyRange, owner)
+  rng.n = newNode(nkRange)
+  rng.n.add(newIntLit(0))
+  rng.n.add(newIntLit(n))
+  rng.rawAddSon(t)
+
+  result.rawAddSon(rng)
+  result.rawAddSon(t)
+
+proc createExceptionTable(ctx: var Ctx): PNode =
+  result = newNode(nkBracket)
+  result.typ = newArrayType(ctx.exceptionTable.len, getSysType(tyInt16), ctx.fn)
+
+  for i in ctx.exceptionTable:
+    let elem = newIntNode(nkIntLit, i)
+    elem.typ = getSysType(tyInt16)
+    result.add(elem)
+
+proc newCatchBody(ctx: var Ctx): PNode {.inline.} =
+  # Generates the code:
+  # :state = exceptionTable[:state]
+  # if :state == 0: raise
+  # :unrollFinally = :state > 0
+  # if :state < 0:
+  #   :state = -:state
+  # :curExc = getCurrentException()
+
+  result = newNode(nkStmtList)
+
+  # :state = exceptionTable[:state]
+  block:
+
+    # exceptionTable[:state]
+    let getNextState = newNode(nkBracketExpr)
+    getNextState.add(ctx.createExceptionTable)
+    getNextState.add(ctx.newStateAccess())
+    getNextState.typ = getSysType(tyInt)
+
+    # :state = exceptionTable[:state]
+    result.add(ctx.newStateAssgn(getNextState))
+
+  # if :state == 0: raise
+  block:
+    let ifStmt = newNode(nkIfStmt)
+    let ifBranch = newNode(nkElifBranch)
+    let cond = newNode(nkCall)
+    cond.add(getSysMagic("==", mEqI).newSymNode)
+    cond.add(ctx.newStateAccess())
+    cond.add(newIntTypeNode(nkIntLit, 0, getSysType(tyInt)))
+    cond.typ = getSysType(tyBool)
+    ifBranch.add(cond)
+
+    let raiseStmt = newNode(nkRaiseStmt)
+    raiseStmt.add(emptyNode)
+
+    ifBranch.add(raiseStmt)
+    ifStmt.add(ifBranch)
+    result.add(ifStmt)
+
+  # :unrollFinally = :state > 0
+  block:
+    let asgn = newNode(nkAsgn)
+    asgn.add(ctx.newUnrollFinallyAccess())
+
+    let cond = newNode(nkCall)
+    cond.add(getSysMagic("<", mLtI).newSymNode)
+    cond.add(newIntTypeNode(nkIntLit, 0, getSysType(tyInt)))
+    cond.add(ctx.newStateAccess())
+    cond.typ = getSysType(tyBool)
+    asgn.add(cond)
+    result.add(asgn)
+
+  # if :state < 0: :state = -:state
+  block:
+    let ifStmt = newNode(nkIfStmt)
+    let ifBranch = newNode(nkElifBranch)
+    let cond = newNode(nkCall)
+    cond.add(getSysMagic("<", mLtI).newSymNode)
+    cond.add(ctx.newStateAccess())
+    cond.add(newIntTypeNode(nkIntLit, 0, getSysType(tyInt)))
+    cond.typ = getSysType(tyBool)
+    ifBranch.add(cond)
+
+    let negateState = newNode(nkCall)
+    negateState.add(getSysMagic("-", mUnaryMinusI).newSymNode)
+    negateState.add(ctx.newStateAccess())
+    negateState.typ = getSysType(tyInt)
+
+    ifBranch.add(ctx.newStateAssgn(negateState))
+    ifStmt.add(ifBranch)
+    result.add(ifStmt)
+
+  # :curExc = getCurrentException()
+  block:
+    let getCurExc = callCodegenProc("getCurrentException", emptyNode)
+    let asgn = newNode(nkAsgn)
+    asgn.add(ctx.newCurExcAccess())
+    asgn.add(getCurExc)
+    result.add(asgn)
+
+proc wrapIntoTryExcept(ctx: var Ctx, n: PNode): PNode =
+  result = newNode(nkTryStmt)
+
+  let tryBody = newNode(nkStmtList)
+
+  let setupExc = newNode(nkCall)
+  setupExc.add(newSymNode(getCompilerProc("closureIterSetupExc")))
+
+  tryBody.add(setupExc)
+
+  tryBody.add(n)
+  result.add(tryBody)
+
+  let catchNode = newNode(nkExceptBranch)
+  result.add(catchNode)
+
+  let catchBody = newNode(nkStmtList)
+  catchBody.add(ctx.newCatchBody())
+  catchNode.add(catchBody)
+
+  setupExc.add(ctx.newCurExcAccess())
+
 proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
+  # while true:
+  #   block :stateLoop:
+  #     gotoState :state
+  #     body # Might get wrapped in try-except
+
   result = newNode(nkWhileStmt)
   result.add(newSymNode(getSysSym("true")))
 
@@ -532,19 +941,19 @@ proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   let blockStmt = newNodeI(nkBlockStmt, n.info)
   blockStmt.add(newSymNode(ctx.stateLoopLabel))
 
-  let blockBody = newNodeI(nkStmtList, n.info)
-  blockStmt.add(blockBody)
+  var blockBody = newNodeI(nkStmtList, n.info)
 
   let gs = newNodeI(nkGotoState, n.info)
-  if ctx.stateVarSym.isNil:
-    gs.add(rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)), getStateField(ctx.fn), n.info))
-  else:
-    gs.add(newSymNode(ctx.stateVarSym))
-
+  gs.add(ctx.newStateAccess())
   gs.add(newIntLit(ctx.states.len - 1))
+
   blockBody.add(gs)
   blockBody.add(n)
-  # gs.add(rawIndirectAccess(newSymNode(ctx.fn.getHiddenParam), getStateField(ctx.fn), n.info))
+
+  if ctx.hasExceptions:
+    blockBody = ctx.wrapIntoTryExcept(blockBody)
+
+  blockStmt.add(blockBody)
 
   loopBody.add(blockStmt)
 
@@ -558,7 +967,7 @@ proc deleteEmptyStates(ctx: var Ctx) =
   var iValid = 0
   for i, s in ctx.states:
     let body = s[1].skipStmtList()
-    if body.kind == nkGotoState and i != ctx.states.len - 1:
+    if body.kind == nkGotoState and i != ctx.states.len - 1 and i != 0:
       # This is an empty state. Mark with -1.
       s[0].intVal = -1
     else:
@@ -567,14 +976,20 @@ proc deleteEmptyStates(ctx: var Ctx) =
 
   for i, s in ctx.states:
     let body = s[1].skipStmtList()
-    if body.kind != nkGotoState:
+    if body.kind != nkGotoState or i == 0:
       discard ctx.skipThroughEmptyStates(s)
+      let excHandlState = ctx.exceptionTable[i]
+      if excHandlState < 0:
+        ctx.exceptionTable[i] = -ctx.skipEmptyStates(-excHandlState)
+      elif excHandlState != 0:
+        ctx.exceptionTable[i] = ctx.skipEmptyStates(excHandlState)
 
   var i = 0
   while i < ctx.states.len - 1:
     let fs = ctx.states[i][1].skipStmtList()
-    if fs.kind == nkGotoState:
+    if fs.kind == nkGotoState and i != 0:
       ctx.states.delete(i)
+      ctx.exceptionTable.delete(i)
     else:
       inc i
 
@@ -591,6 +1006,7 @@ proc transformClosureIterator*(fn: PSym, n: PNode): PNode =
 
   ctx.states = @[]
   ctx.stateLoopLabel = newSym(skLabel, getIdent(":stateLoop"), fn, fn.info)
+  ctx.exceptionTable = @[]
   let n = n.toStmtList
 
   discard ctx.newState(n, nil)
@@ -613,19 +1029,11 @@ proc transformClosureIterator*(fn: PSym, n: PNode): PNode =
     result.add(body)
 
   result = ctx.tranformStateAssignments(result)
-
-  # Add excpetion handling
-  var hasExceptions = false
-  if hasExceptions:
-    discard # TODO:
-    # result = wrapIntoTryCatch(result)
-
-  # while true:
-  #   block :stateLoop:
-  #     gotoState
-  #     body
   result = ctx.wrapIntoStateLoop(result)
 
-  # echo "TRANSFORM TO STATES2: "
-  # debug(result)
+  # echo "TRANSFORM TO STATES: "
   # echo renderTree(result)
+
+  # echo "exception table:"
+  # for i, e in ctx.exceptionTable:
+  #   echo i, " -> ", e
diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim
index 1ef284a779..79010bfde9 100644
--- a/compiler/semexprs.nim
+++ b/compiler/semexprs.nim
@@ -1544,7 +1544,7 @@ proc semYield(c: PContext, n: PNode): PNode =
   checkSonsLen(n, 1)
   if c.p.owner == nil or c.p.owner.kind != skIterator:
     localError(n.info, errYieldNotAllowedHere)
-  elif c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline:
+  elif oldIterTransf in c.features and c.p.inTryStmt > 0 and c.p.owner.typ.callConv != ccInline:
     localError(n.info, errYieldNotAllowedInTryStmt)
   elif n.sons[0].kind != nkEmpty:
     n.sons[0] = semExprWithType(c, n.sons[0]) # check for type compatibility:
diff --git a/lib/system/embedded.nim b/lib/system/embedded.nim
index 46e84e0569..4d453fcca4 100644
--- a/lib/system/embedded.nim
+++ b/lib/system/embedded.nim
@@ -41,3 +41,6 @@ proc reraiseException() {.compilerRtl.} =
 proc writeStackTrace() = discard
 
 proc setControlCHook(hook: proc () {.noconv.}) = discard
+
+proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} =
+  sysFatal(ReraiseError, "exception handling is not available")
diff --git a/lib/system/excpt.nim b/lib/system/excpt.nim
index fb38948f7b..dabfe010ea 100644
--- a/lib/system/excpt.nim
+++ b/lib/system/excpt.nim
@@ -131,6 +131,10 @@ proc popCurrentExceptionEx(id: uint) {.compilerRtl.} =
       quitOrDebug()
     prev.up = cur.up
 
+proc closureIterSetupExc(e: ref Exception) {.compilerproc, inline.} =
+  if not e.isNil:
+    currException = e
+
 # some platforms have native support for stack traces:
 const
   nativeStackTraceSupported* = (defined(macosx) or defined(linux)) and
diff --git a/tests/async/tasynctry2.nim b/tests/async/tasynctry2.nim
index 444a058beb..f82b6cfe0d 100644
--- a/tests/async/tasynctry2.nim
+++ b/tests/async/tasynctry2.nim
@@ -1,10 +1,12 @@
 discard """
   file: "tasynctry2.nim"
   errormsg: "\'yield\' cannot be used within \'try\' in a non-inlined iterator"
-  line: 15
+  line: 17
 """
 import asyncdispatch
 
+{.experimental: "oldIterTransf".}
+
 proc foo(): Future[bool] {.async.} = discard
 
 proc test5(): Future[int] {.async.} =
diff --git a/tests/iter/tyieldintry.nim b/tests/iter/tyieldintry.nim
new file mode 100644
index 0000000000..9cb199c5b8
--- /dev/null
+++ b/tests/iter/tyieldintry.nim
@@ -0,0 +1,201 @@
+discard """
+targets: "c cpp"
+output: "ok"
+"""
+var closureIterResult = newSeq[int]()
+
+proc checkpoint(arg: int) =
+    closureIterResult.add(arg)
+
+type
+    TestException = object of Exception
+    AnotherException = object of Exception
+
+proc testClosureIterAux(it: iterator(): int, exceptionExpected: bool, expectedResults: varargs[int]) =
+    closureIterResult.setLen(0)
+
+    var exceptionCaught = false
+
+    try:
+        for i in it():
+            closureIterResult.add(i)
+    except TestException:
+        exceptionCaught = true
+
+    if closureIterResult != @expectedResults or exceptionCaught != exceptionExpected:
+        if closureIterResult != @expectedResults:
+            echo "Expected: ", @expectedResults
+            echo "Actual: ", closureIterResult
+        if exceptionCaught != exceptionExpected:
+            echo "Expected exception: ", exceptionExpected
+            echo "Got exception: ", exceptionCaught
+        doAssert(false)
+
+proc test(it: iterator(): int, expectedResults: varargs[int]) =
+    testClosureIterAux(it, false, expectedResults)
+
+proc testExc(it: iterator(): int, expectedResults: varargs[int]) =
+    testClosureIterAux(it, true, expectedResults)
+
+proc raiseException() =
+    raise newException(TestException, "Test exception!")
+
+block:
+    iterator it(): int {.closure.} =
+        var i = 5
+        while i != 0:
+            yield i
+            if i == 3:
+                yield 123
+            dec i
+
+    test(it, 5, 4, 3, 123, 2, 1)
+
+block:
+    iterator it(): int {.closure.} =
+        yield 0
+        try:
+            checkpoint(1)
+            raiseException()
+        except TestException:
+            checkpoint(2)
+            yield 3
+            checkpoint(4)
+        finally:
+            checkpoint(5)
+
+        checkpoint(6)
+
+    test(it, 0, 1, 2, 3, 4, 5, 6)
+
+block:
+    iterator it(): int {.closure.} =
+        yield 0
+        try:
+            yield 1
+            checkpoint(2)
+        finally:
+            checkpoint(3)
+            yield 4
+            checkpoint(5)
+            yield 6
+
+    test(it, 0, 1, 2, 3, 4, 5, 6)
+
+block:
+    iterator it(): int {.closure.} =
+        yield 0
+        try:
+            yield 1
+            raiseException()
+            yield 2
+        finally:
+            checkpoint(3)
+            yield 4
+            checkpoint(5)
+            yield 6
+
+    testExc(it, 0, 1, 3, 4, 5, 6)
+
+block:
+    iterator it(): int {.closure.} =
+        try:
+            try:
+                raiseException()
+            except AnotherException:
+                yield 123
+            finally:
+                checkpoint(3)
+        finally:
+            checkpoint(4)
+
+    testExc(it, 3, 4)
+
+block:
+    iterator it(): int {.closure.} =
+        try:
+            yield 1
+            raiseException()
+        except AnotherException:
+            checkpoint(123)
+        finally:
+            checkpoint(2)
+        checkpoint(3)
+
+    testExc(it, 1, 2)
+
+block:
+    iterator it(): int {.closure.} =
+        try:
+            yield 0
+            try:
+                yield 1
+                try:
+                    yield 2
+                    raiseException()
+                except AnotherException:
+                    yield 123
+                finally:
+                    yield 3
+            except AnotherException:
+                yield 124
+            finally:
+                yield 4
+            checkpoint(1234)
+        except:
+            yield 5
+            checkpoint(6)
+        finally:
+            checkpoint(7)
+            yield 8
+        checkpoint(9)
+
+    test(it, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9)
+
+block:
+    iterator it(): int {.closure.} =
+        try:
+            yield 0
+            return 2
+        finally:
+            checkpoint(1)
+        checkpoint(123)
+
+    test(it, 0, 1)
+
+block:
+    iterator it(): int {.closure.} =
+        try:
+            try:
+                yield 0
+                raiseException()
+            finally:
+                checkpoint(1)
+        except TestException:
+            yield 2
+            return
+        finally:
+            yield 3
+
+        checkpoint(123)
+
+    test(it, 0, 1, 2, 3)
+
+block:
+    iterator it(): int {.closure.} =
+        try:
+            try:
+                yield 0
+                raiseException()
+            finally:
+                return # Return in finally should stop exception propagation
+        except AnotherException:
+            yield 2
+            return
+        finally:
+            yield 3
+        checkpoint(123)
+
+    test(it, 0, 3)
+
+echo "ok"

From ac86b8ce615cbd55074dfd27f42ed0368d84b1fd Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Fri, 4 May 2018 16:39:59 +0300
Subject: [PATCH 07/51] Cleanup

---
 compiler/closureiters.nim | 72 +++++++++++++++------------------------
 1 file changed, 27 insertions(+), 45 deletions(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index 504f70347d..a8e7e02747 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -123,14 +123,10 @@
 # STATE2: # Finally
 #   yield 2
 #   if :unrollFinally: # This node is created by `newEndFinallyNode`
-#     when nearestFinally == 0: # Pseudocode. The `when` is not emitted in reality
-#       if :curExc.isNil:
-#         return :tmpResult
-#       else:
-#         raise
+#     if :curExc.isNil:
+#       return :tmpResult
 #     else:
-#       :state = nearestFinally
-#       break :stateLoop
+#       raise
 #   state = -1 # Goto next state. In this case we just exit
 #   break :stateLoop
 
@@ -427,15 +423,10 @@ proc lowerStmtListExpr(ctx: var Ctx, n: PNode): PNode =
 proc newEndFinallyNode(ctx: var Ctx): PNode =
   # Generate the following code:
   #   if :unrollFinally:
-  #     when nearestFinally == 0: # Pseudocode. The `when` is not emitted in reality
   #       if :curExc.isNil:
   #         return :tmpResult
   #       else:
   #         raise
-  #     else:
-  #       goto nearestFinally
-  #       :state = nearestFinally
-  #       break :stateLoop
 
   result = newNode(nkIfStmt)
 
@@ -443,44 +434,35 @@ proc newEndFinallyNode(ctx: var Ctx): PNode =
   elifBranch.add(ctx.newUnrollFinallyAccess())
   result.add(elifBranch)
 
-  var ifBody: PNode
+  let ifBody = newNode(nkIfStmt)
+  let branch = newNode(nkElifBranch)
 
-  if ctx.nearestFinally == 0 or true:
-    ifBody = newNode(nkIfStmt)
-    let branch = newNode(nkElifBranch)
+  let cmp = newNode(nkCall)
+  cmp.add(getSysMagic("==", mEqRef).newSymNode)
+  let curExc = ctx.newCurExcAccess()
+  let nilnode = newNode(nkNilLit)
+  nilnode.typ = curExc.typ
+  cmp.add(curExc)
+  cmp.add(nilnode)
+  cmp.typ = getSysType(tyBool)
+  branch.add(cmp)
 
-    let cmp = newNode(nkCall)
-    cmp.add(getSysMagic("==", mEqRef).newSymNode)
-    let curExc = ctx.newCurExcAccess()
-    let nilnode = newNode(nkNilLit)
-    nilnode.typ = curExc.typ
-    cmp.add(curExc)
-    cmp.add(nilnode)
-    cmp.typ = getSysType(tyBool)
-    branch.add(cmp)
+  let retStmt = newNode(nkReturnStmt)
+  let asgn = newNode(nkAsgn)
+  addSon(asgn, newSymNode(getClosureIterResult(ctx.fn)))
+  addSon(asgn, ctx.newTmpResultAccess())
+  retStmt.add(asgn)
+  branch.add(retStmt)
 
-    var retStmt = newNode(nkReturnStmt)
-    if true:
-      var a = newNode(nkAsgn)
-      addSon(a, newSymNode(getClosureIterResult(ctx.fn)))
-      addSon(a, ctx.newTmpResultAccess())
-      retStmt.add(a)
-    else:
-      retStmt.add(emptyNode)
-    branch.add(retStmt)
+  let elseBranch = newNode(nkElse)
+  let raiseStmt = newNode(nkRaiseStmt)
 
-    let elseBranch = newNode(nkElse)
-    let raiseStmt = newNode(nkRaiseStmt)
+  # The C++ backend requires `getCurrentException` here.
+  raiseStmt.add(callCodegenProc("getCurrentException", emptyNode))
+  elseBranch.add(raiseStmt)
 
-    # The C++ backend requires `getCurrentException` here.
-    raiseStmt.add(callCodegenProc("getCurrentException", emptyNode))
-    elseBranch.add(raiseStmt)
-
-    ifBody.add(branch)
-    ifBody.add(elseBranch)
-  else:
-    ifBody = newNode(nkGotoState)
-    ifBody.add(newIntLit(ctx.nearestFinally))
+  ifBody.add(branch)
+  ifBody.add(elseBranch)
 
   elifBranch.add(ifBody)
 

From 14ca79fe1f1fafb8e3aff2e4c27bcb94c0595792 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Tue, 8 May 2018 01:25:08 +0300
Subject: [PATCH 08/51] More elaborate nkStmtListExpr lowering

---
 compiler/closureiters.nim  | 404 ++++++++++++++++++++++++++++---
 tests/iter/tyieldintry.nim | 477 +++++++++++++++++++++++++------------
 2 files changed, 696 insertions(+), 185 deletions(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index a8e7e02747..a30b4e10ee 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -149,7 +149,6 @@ type
     exitStateIdx: int # index of the last state
     tempVarId: int # unique name counter
     tempVars: PNode # Temp var decls, nkVarSection
-    loweredStmtListExpr: PNode # Temporary used for nkStmtListExpr lowering
     exceptionTable: seq[int] # For state `i` jump to state `exceptionTable[i]` if exception is raised
     hasExceptions: bool # Does closure have yield in try?
     curExcHandlingState: int # Negative for except, positive for finally
@@ -177,6 +176,7 @@ proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
 proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
   result = newSym(skVar, getIdent(name), ctx.fn, ctx.fn.info)
   result.typ = typ
+  assert(not typ.isNil)
 
   if not ctx.stateVarSym.isNil:
     # We haven't gone through labmda lifting yet, so just create a local var,
@@ -197,7 +197,6 @@ proc newEnvVarAccess(ctx: Ctx, s: PSym): PNode =
 
 proc newTmpResultAccess(ctx: var Ctx): PNode =
   if ctx.tmpResultSym.isNil:
-    debug(ctx.fn.typ)
     ctx.tmpResultSym = ctx.newEnvVar(":tmpResult", ctx.fn.typ[0])
   ctx.newEnvVarAccess(ctx.tmpResultSym)
 
@@ -245,9 +244,8 @@ proc addGotoOut(n: PNode, gotoOut: PNode): PNode =
   if result.len != 0 and result.sons[^1].kind != nkGotoState:
     result.add(gotoOut)
 
-proc newTempVarAccess(ctx: var Ctx, typ: PType, i: TLineInfo): PNode =
-  let s = ctx.newEnvVar(":tmpSlLower" & $ctx.tempVarId, typ)
-  result = ctx.newEnvVarAccess(s)
+proc newTempVar(ctx: var Ctx, typ: PType): PSym =
+  result = ctx.newEnvVar(":tmpSlLower" & $ctx.tempVarId, typ)
   inc ctx.tempVarId
 
 proc hasYields(n: PNode): bool =
@@ -390,35 +388,382 @@ proc hasYieldsInExpressions(n: PNode): bool =
       nkSym, nkIdent, procDefs, nkTemplateDef:
     discard
   of nkStmtListExpr:
-    result = n.hasYields
-  of nkStmtList, nkWhileStmt, nkCaseStmt, nkIfStmt:
-    discard
+    if isEmptyType(n.typ):
+      for c in n:
+        if c.hasYieldsInExpressions:
+          return true
+    else:
+      result = n.hasYields
   else:
     for c in n:
       if c.hasYieldsInExpressions:
         return true
 
-proc lowerStmtListExpr(ctx: var Ctx, n: PNode): PNode =
+proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
+  assert(n.kind == nkStmtListExpr)
+
+  var parent = n
+  var lastSon = n[^1]
+
+  while lastSon.kind == nkStmtListExpr:
+    parent = lastSon
+    lastSon = lastSon[^1]
+
+  result.s = newNodeI(nkStmtList, n.info)
+  result.s.sons = parent.sons
+  result.s.sons.setLen(result.s.sons.len - 1) # delete last son
+  result.res = lastSon
+
+proc newEnvVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode =
+  result = newNode(nkFastAsgn)
+  result.add(ctx.newEnvVarAccess(s))
+  result.add(v)
+
+proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) =
+  if input.kind == nkStmtListExpr:
+    let (st, res) = exprToStmtList(input)
+    output.add(st)
+    output.add(ctx.newEnvVarAsgn(sym, res))
+  else:
+    output.add(ctx.newEnvVarAsgn(sym, input))
+
+proc convertExprBodyToAsgn(ctx: Ctx, exprBody: PNode, res: PSym): PNode =
+  result = newNode(nkStmtList)
+  ctx.addExprAssgn(result, exprBody, res)
+
+proc newNotCall(e: PNode): PNode =
+  result = newNode(nkCall)
+  result.add(newSymNode(getSysMagic("not", mNot)))
+  result.add(e)
+  result.typ = getSysType(tyBool)
+
+proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
   result = n
   case n.kind
   of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
       nkSym, nkIdent, procDefs, nkTemplateDef:
     discard
-  of nkStmtListExpr:
-    if n.hasYields:
-      for i in 0 .. n.len - 2:
-        ctx.loweredStmtListExpr.add(n[i])
 
-      let tv = ctx.newTempVarAccess(n.typ, n[^1].info)
-      let asgn = newNode(nkAsgn)
-      asgn.add(tv)
-      asgn.add(n[^1])
-      ctx.loweredStmtListExpr.add(asgn)
-      result = tv
+  of nkYieldStmt:
+    var ns = false
+    for i in 0 ..< n.len:
+      n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
+    if ns:
+      assert(n[0].kind == nkStmtListExpr)
+      result = newNodeI(nkStmtList, n.info)
+      let (st, ex) = exprToStmtList(n[0])
+      result.add(st)
+      n[0] = ex
+      result.add(n)
+
+    needsSplit = true
+
+  of nkPar, nkObjConstr, nkTupleConstr, nkBracket, nkArgList:
+    var ns = false
+    for i in 0 ..< n.len:
+      n[i] = ctx.lowerStmtListExprs(n[i], ns)
+
+    if ns:
+      needsSplit = true
+
+      result = newNodeI(nkStmtListExpr, n.info)
+      if n.typ.isNil: internalError("lowerStmtListExprs: constr typ.isNil")
+      result.typ = n.typ
+
+      for i in 0 ..< n.len:
+        if n[i].kind == nkStmtListExpr:
+          let (st, ex) = exprToStmtList(n[i])
+          result.add(st)
+          n[i] = ex
+      result.add(n)
+
+  of nkIfStmt, nkIfExpr:
+    var ns = false
+    for i in 0 ..< n.len:
+      n[i] = ctx.lowerStmtListExprs(n[i], ns)
+
+    if ns:
+      needsSplit = true
+      var tmp: PSym
+      var s: PNode
+      let isExpr = not isEmptyType(n.typ)
+      if isExpr:
+        tmp = ctx.newTempVar(n.typ)
+        result = newNode(nkStmtListExpr)
+        result.typ = n.typ
+      else:
+        result = newNode(nkStmtList)
+
+      var curS = result
+
+      for branch in n:
+        case branch.kind
+        of nkElseExpr, nkElse:
+          if isExpr:
+            var newBranch = newNodeI(nkElse, branch.info)
+            let branchBody = newNode(nkStmtList)
+            ctx.addExprAssgn(branchBody, branch[0], tmp)
+            newBranch.add(branchBody)
+            curS.add(newBranch)
+          else:
+            curS.add(branch)
+
+        of nkElifExpr, nkElifBranch:
+          var newBranch: PNode
+          if branch[0].kind == nkStmtListExpr:
+            let elseBody = newNode(nkStmtList)
+
+            let (st, res) = exprToStmtList(branch[0])
+            elseBody.add(st)
+
+            newBranch = newNodeI(nkElifBranch, branch.info)
+            newBranch.add(res)
+            newBranch.add(branch[1])
+
+            let newIf = newNodeI(nkIfStmt, branch.info)
+            newIf.add(newBranch)
+            elseBody.add(newIf)
+            if curS.kind == nkIfStmt:
+              let newElse = newNodeI(nkElse, branch.info)
+              newElse.add(elseBody)
+              curS.add(newElse)
+            else:
+              curS.add(elseBody)
+            curS = newIf
+          else:
+            newBranch = branch
+            if curS.kind == nkIfStmt:
+              curS.add(newBranch)
+            else:
+              let newIf = newNodeI(nkIfStmt, branch.info)
+              newIf.add(newBranch)
+              curS.add(newIf)
+              curS = newIf
+
+          if isExpr:
+            let branchBody = newNode(nkStmtList)
+            ctx.addExprAssgn(branchBody, branch[1], tmp)
+            newBranch[1] = branchBody
+
+        else:
+          internalError("lowerStmtListExpr(nkIf): " & $branch.kind)
+
+      if isExpr: result.add(ctx.newEnvVarAccess(tmp))
+
+  of nkTryStmt:
+    var ns = false
+    for i in 0 ..< n.len:
+      n[i] = ctx.lowerStmtListExprs(n[i], ns)
+
+    if ns:
+      needsSplit = true
+      let isExpr = not isEmptyType(n.typ)
+
+      if isExpr:
+        result = newNodeI(nkStmtListExpr, n.info)
+        result.typ = n.typ
+        let tmp = ctx.newTempVar(n.typ)
+
+        n[0] = ctx.convertExprBodyToAsgn(n[0], tmp)
+        for i in 1 ..< n.len:
+          let branch = n[i]
+          case branch.kind
+          of nkExceptBranch:
+            if branch[0].kind == nkType:
+              branch[1] = ctx.convertExprBodyToAsgn(branch[1], tmp)
+            else:
+              branch[0] = ctx.convertExprBodyToAsgn(branch[0], tmp)
+          of nkFinally:
+            discard
+          else:
+            internalError("lowerStmtListExpr(nkTryStmt): " & $branch.kind)
+        result.add(n)
+        result.add(ctx.newEnvVarAccess(tmp))
+
+  of nkCaseStmt:
+    var ns = false
+    for i in 0 ..< n.len:
+      n[i] = ctx.lowerStmtListExprs(n[i], ns)
+
+    if ns:
+      needsSplit = true
+
+      let isExpr = not isEmptyType(n.typ)
+
+      if isExpr:
+        let tmp = ctx.newTempVar(n.typ)
+        result = newNodeI(nkStmtListExpr, n.info)
+        result.typ = n.typ
+
+        if n[0].kind == nkStmtListExpr:
+          let (st, ex) = exprToStmtList(n[0])
+          result.add(st)
+          n[0] = ex
+
+        for i in 1 ..< n.len:
+          let branch = n[i]
+          case branch.kind
+          of nkOfBranch:
+            branch[1] = ctx.convertExprBodyToAsgn(branch[1], tmp)
+          of nkElse:
+            branch[0] = ctx.convertExprBodyToAsgn(branch[0], tmp)
+          else:
+            internalError("lowerStmtListExpr(nkCaseStmt): " & $branch.kind)
+        result.add(n)
+        result.add(ctx.newEnvVarAccess(tmp))
+
+  of nkCallKinds:
+    var ns = false
+    for i in 0 ..< n.len:
+      n[i] = ctx.lowerStmtListExprs(n[i], ns)
+
+    if ns:
+      needsSplit = true
+      let isExpr = not isEmptyType(n.typ)
+
+      if isExpr:
+        result = newNodeI(nkStmtListExpr, n.info)
+        result.typ = n.typ
+      else:
+        result = newNode(nkStmtList, n.info)
+
+      if n[0].kind == nkSym and n[0].sym.magic in {mAnd, mOr}: # `and`/`or` short cirquiting
+        var cond = n[1]
+        if cond.kind == nkStmtListExpr:
+          let (st, ex) = exprToStmtList(cond)
+          result.add(st)
+          cond = ex
+
+        let tmp = ctx.newTempVar(cond.typ)
+        result.add(ctx.newEnvVarAsgn(tmp, cond))
+
+        let ifNode = newNode(nkIfStmt)
+        let ifBranch = newNode(nkElifBranch)
+
+        var check = ctx.newEnvVarAccess(tmp)
+        if n[0].sym.magic == mOr:
+          check = newNotCall(check)
+        ifBranch.add(check)
+
+        cond = n[2]
+        let ifBody = newNode(nkStmtList)
+        if cond.kind == nkStmtListExpr:
+          let (st, ex) = exprToStmtList(cond)
+          ifBody.add(st)
+          cond = ex
+        ifBody.add(ctx.newEnvVarAsgn(tmp, cond))
+        ifBranch.add(ifBody)
+        ifNode.add(ifBranch)
+        result.add(ifNode)
+        result.add(ctx.newEnvVarAccess(tmp))
+      else:
+        for i in 0 ..< n.len:
+          if n[i].kind == nkStmtListExpr:
+            let (st, ex) = exprToStmtList(n[i])
+            result.add(st)
+            n[i] = ex
+
+          if n[i].kind in nkCallKinds: # XXX: This should better be some sort of side effect tracking
+            let tmp = ctx.newTempVar(n[i].typ)
+            result.add(ctx.newEnvVarAsgn(tmp, n[i]))
+            n[i] = ctx.newEnvVarAccess(tmp)
+
+        result.add(n)
+
+  of nkVarSection, nkLetSection:
+    result = newNodeI(nkStmtList, n.info)
+    for c in n:
+      let varSect = newNodeI(n.kind, n.info)
+      varSect.add(c)
+      var ns = false
+      c[^1] = ctx.lowerStmtListExprs(c[^1], ns)
+      if ns:
+        needsSplit = true
+        assert(c[^1].kind == nkStmtListExpr)
+        let (st, ex) = exprToStmtList(c[^1])
+        result.add(st)
+        c[^1] = ex
+      result.add(varSect)
+
+  of nkDiscardStmt, nkReturnStmt, nkRaiseStmt:
+    var ns = false
+    for i in 0 ..< n.len:
+      n[i] = ctx.lowerStmtListExprs(n[i], ns)
+
+    if ns:
+      needsSplit = true
+      result = newNodeI(nkStmtList, n.info)
+      let (st, ex) = exprToStmtList(n[0])
+      result.add(st)
+      n[0] = ex
+      result.add(n)
+
+  of nkCast:
+    var ns = false
+    for i in 0 ..< n.len:
+      n[i] = ctx.lowerStmtListExprs(n[i], ns)
+
+    if ns:
+      needsSplit = true
+      result = newNodeI(nkStmtListExpr, n.info)
+      result.typ = n.typ
+      let (st, ex) = exprToStmtList(n[1])
+      result.add(st)
+      n[1] = ex
+      result.add(n)
+
+  of nkAsgn, nkFastAsgn:
+    var ns = false
+    for i in 0 ..< n.len:
+      n[i] = ctx.lowerStmtListExprs(n[i], ns)
+
+    if ns:
+      needsSplit = true
+      result = newNodeI(nkStmtList, n.info)
+      if n[0].kind == nkStmtListExpr:
+        let (st, ex) = exprToStmtList(n[0])
+        result.add(st)
+        n[0] = ex
+
+      if n[1].kind == nkStmtListExpr:
+        let (st, ex) = exprToStmtList(n[1])
+        result.add(st)
+        n[1] = ex
+
+      result.add(n)
+
+  of nkWhileStmt:
+    var ns = false
+
+    var condNeedsSplit = false
+    n[0] = ctx.lowerStmtListExprs(n[0], condNeedsSplit)
+    var bodyNeedsSplit = false
+    n[1] = ctx.lowerStmtListExprs(n[1], bodyNeedsSplit)
+
+    if condNeedsSplit or bodyNeedsSplit:
+      needsSplit = true
+
+      if condNeedsSplit:
+        let newBody = newNode(nkStmtList)
+
+        let (st, ex) = exprToStmtList(n[0])
+        newBody.add(st)
+        let check = newNode(nkIfStmt)
+        let branch = newNode(nkElifBranch)
+        branch.add(newNotCall(ex))
+        let brk = newNode(nkBreakStmt)
+        brk.add(emptyNode)
+        branch.add(brk)
+        check.add(branch)
+        newBody.add(check)
+        newBody.add(n[1])
+
+        n[0] = newSymNode(getSysSym("true"))
+        n[1] = newBody
   else:
     for i in 0 ..< n.len:
-      n[i] = ctx.lowerStmtListExpr(n[i])
+      n[i] = ctx.lowerStmtListExprs(n[i], needsSplit)
 
 proc newEndFinallyNode(ctx: var Ctx): PNode =
   # Generate the following code:
@@ -448,7 +793,7 @@ proc newEndFinallyNode(ctx: var Ctx): PNode =
   branch.add(cmp)
 
   let retStmt = newNode(nkReturnStmt)
-  let asgn = newNode(nkAsgn)
+  let asgn = newNode(nkFastAsgn)
   addSon(asgn, newSymNode(getClosureIterResult(ctx.fn)))
   addSon(asgn, ctx.newTmpResultAccess())
   retStmt.add(asgn)
@@ -482,7 +827,7 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
       asgn.add(newIntTypeNode(nkIntLit, 1, getSysType(tyBool)))
       result.add(asgn)
 
-    if n[0].kind != nkEmpty: # TODO: And not void!
+    if n[0].kind != nkEmpty:
       let asgnTmpResult = newNodeI(nkAsgn, n.info)
       asgnTmpResult.add(ctx.newTmpResultAccess())
       asgnTmpResult.add(n[0])
@@ -508,17 +853,15 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
         nkSym, nkIdent, procDefs, nkTemplateDef:
       discard
 
-    of nkStmtList:
+    of nkStmtList, nkStmtListExpr:
+      assert(isEmptyType(n.typ), "nkStmtListExpr not lowered")
+
       result = addGotoOut(result, gotoOut)
       for i in 0 ..< n.len:
         if n[i].hasYieldsInExpressions:
           # Lower nkStmtListExpr nodes inside `n[i]` first
-          assert(ctx.loweredStmtListExpr.isNil)
-          ctx.loweredStmtListExpr = newNodeI(nkStmtList, n.info)
-          n[i] = ctx.lowerStmtListExpr(n[i])
-          ctx.loweredStmtListExpr.add(n[i])
-          n[i] = ctx.loweredStmtListExpr
-          ctx.loweredStmtListExpr = nil
+          var ns = false
+          n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
         if n[i].hasYields:
           # Create a new split
@@ -534,9 +877,6 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
           discard ctx.transformClosureIteratorBody(s, gotoOut)
           break
 
-    of nkStmtListExpr:
-      assert(false, "nkStmtListExpr not lowered")
-
     of nkYieldStmt:
       result = newNodeI(nkStmtList, n.info)
       result.add(n)
diff --git a/tests/iter/tyieldintry.nim b/tests/iter/tyieldintry.nim
index 9cb199c5b8..31ec65a830 100644
--- a/tests/iter/tyieldintry.nim
+++ b/tests/iter/tyieldintry.nim
@@ -5,197 +5,368 @@ output: "ok"
 var closureIterResult = newSeq[int]()
 
 proc checkpoint(arg: int) =
-    closureIterResult.add(arg)
+  closureIterResult.add(arg)
 
 type
-    TestException = object of Exception
-    AnotherException = object of Exception
+  TestException = object of Exception
+  AnotherException = object of Exception
 
 proc testClosureIterAux(it: iterator(): int, exceptionExpected: bool, expectedResults: varargs[int]) =
-    closureIterResult.setLen(0)
+  closureIterResult.setLen(0)
 
-    var exceptionCaught = false
+  var exceptionCaught = false
 
-    try:
-        for i in it():
-            closureIterResult.add(i)
-    except TestException:
-        exceptionCaught = true
+  try:
+    for i in it():
+      closureIterResult.add(i)
+  except TestException:
+    exceptionCaught = true
 
-    if closureIterResult != @expectedResults or exceptionCaught != exceptionExpected:
-        if closureIterResult != @expectedResults:
-            echo "Expected: ", @expectedResults
-            echo "Actual: ", closureIterResult
-        if exceptionCaught != exceptionExpected:
-            echo "Expected exception: ", exceptionExpected
-            echo "Got exception: ", exceptionCaught
-        doAssert(false)
+  if closureIterResult != @expectedResults or exceptionCaught != exceptionExpected:
+    if closureIterResult != @expectedResults:
+      echo "Expected: ", @expectedResults
+      echo "Actual: ", closureIterResult
+    if exceptionCaught != exceptionExpected:
+      echo "Expected exception: ", exceptionExpected
+      echo "Got exception: ", exceptionCaught
+    doAssert(false)
 
 proc test(it: iterator(): int, expectedResults: varargs[int]) =
-    testClosureIterAux(it, false, expectedResults)
+  testClosureIterAux(it, false, expectedResults)
 
 proc testExc(it: iterator(): int, expectedResults: varargs[int]) =
-    testClosureIterAux(it, true, expectedResults)
+  testClosureIterAux(it, true, expectedResults)
 
 proc raiseException() =
-    raise newException(TestException, "Test exception!")
+  raise newException(TestException, "Test exception!")
 
 block:
-    iterator it(): int {.closure.} =
-        var i = 5
-        while i != 0:
-            yield i
-            if i == 3:
-                yield 123
-            dec i
+  iterator it(): int {.closure.} =
+    var i = 5
+    while i != 0:
+      yield i
+      if i == 3:
+        yield 123
+      dec i
 
-    test(it, 5, 4, 3, 123, 2, 1)
+  test(it, 5, 4, 3, 123, 2, 1)
 
 block:
-    iterator it(): int {.closure.} =
-        yield 0
-        try:
-            checkpoint(1)
-            raiseException()
-        except TestException:
-            checkpoint(2)
-            yield 3
-            checkpoint(4)
-        finally:
-            checkpoint(5)
+  iterator it(): int {.closure.} =
+    yield 0
+    try:
+      checkpoint(1)
+      raiseException()
+    except TestException:
+      checkpoint(2)
+      yield 3
+      checkpoint(4)
+    finally:
+      checkpoint(5)
 
-        checkpoint(6)
+    checkpoint(6)
 
-    test(it, 0, 1, 2, 3, 4, 5, 6)
+  test(it, 0, 1, 2, 3, 4, 5, 6)
 
 block:
-    iterator it(): int {.closure.} =
-        yield 0
-        try:
-            yield 1
-            checkpoint(2)
-        finally:
-            checkpoint(3)
-            yield 4
-            checkpoint(5)
-            yield 6
+  iterator it(): int {.closure.} =
+    yield 0
+    try:
+      yield 1
+      checkpoint(2)
+    finally:
+      checkpoint(3)
+      yield 4
+      checkpoint(5)
+      yield 6
 
-    test(it, 0, 1, 2, 3, 4, 5, 6)
+  test(it, 0, 1, 2, 3, 4, 5, 6)
 
 block:
-    iterator it(): int {.closure.} =
-        yield 0
-        try:
-            yield 1
-            raiseException()
-            yield 2
-        finally:
-            checkpoint(3)
-            yield 4
-            checkpoint(5)
-            yield 6
+  iterator it(): int {.closure.} =
+    yield 0
+    try:
+      yield 1
+      raiseException()
+      yield 2
+    finally:
+      checkpoint(3)
+      yield 4
+      checkpoint(5)
+      yield 6
 
-    testExc(it, 0, 1, 3, 4, 5, 6)
+  testExc(it, 0, 1, 3, 4, 5, 6)
 
 block:
-    iterator it(): int {.closure.} =
-        try:
-            try:
-                raiseException()
-            except AnotherException:
-                yield 123
-            finally:
-                checkpoint(3)
-        finally:
-            checkpoint(4)
-
-    testExc(it, 3, 4)
-
-block:
-    iterator it(): int {.closure.} =
-        try:
-            yield 1
-            raiseException()
-        except AnotherException:
-            checkpoint(123)
-        finally:
-            checkpoint(2)
+  iterator it(): int {.closure.} =
+    try:
+      try:
+        raiseException()
+      except AnotherException:
+        yield 123
+      finally:
         checkpoint(3)
+    finally:
+      checkpoint(4)
 
-    testExc(it, 1, 2)
+  testExc(it, 3, 4)
 
 block:
-    iterator it(): int {.closure.} =
-        try:
-            yield 0
-            try:
-                yield 1
-                try:
-                    yield 2
-                    raiseException()
-                except AnotherException:
-                    yield 123
-                finally:
-                    yield 3
-            except AnotherException:
-                yield 124
-            finally:
-                yield 4
-            checkpoint(1234)
-        except:
-            yield 5
-            checkpoint(6)
-        finally:
-            checkpoint(7)
-            yield 8
-        checkpoint(9)
+  iterator it(): int {.closure.} =
+    try:
+      yield 1
+      raiseException()
+    except AnotherException:
+      checkpoint(123)
+    finally:
+      checkpoint(2)
+    checkpoint(3)
 
-    test(it, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9)
+  testExc(it, 1, 2)
 
 block:
-    iterator it(): int {.closure.} =
+  iterator it(): int {.closure.} =
+    try:
+      yield 0
+      try:
+        yield 1
         try:
-            yield 0
-            return 2
-        finally:
-            checkpoint(1)
-        checkpoint(123)
-
-    test(it, 0, 1)
-
-block:
-    iterator it(): int {.closure.} =
-        try:
-            try:
-                yield 0
-                raiseException()
-            finally:
-                checkpoint(1)
-        except TestException:
-            yield 2
-            return
-        finally:
-            yield 3
-
-        checkpoint(123)
-
-    test(it, 0, 1, 2, 3)
-
-block:
-    iterator it(): int {.closure.} =
-        try:
-            try:
-                yield 0
-                raiseException()
-            finally:
-                return # Return in finally should stop exception propagation
+          yield 2
+          raiseException()
         except AnotherException:
-            yield 2
-            return
+          yield 123
         finally:
-            yield 3
-        checkpoint(123)
+          yield 3
+      except AnotherException:
+        yield 124
+      finally:
+        yield 4
+      checkpoint(1234)
+    except:
+      yield 5
+      checkpoint(6)
+    finally:
+      checkpoint(7)
+      yield 8
+    checkpoint(9)
+
+  test(it, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9)
+
+block:
+  iterator it(): int {.closure.} =
+    try:
+      yield 0
+      return 2
+    finally:
+      checkpoint(1)
+    checkpoint(123)
+
+  test(it, 0, 1)
+
+block:
+  iterator it(): int {.closure.} =
+    try:
+      try:
+        yield 0
+        raiseException()
+      finally:
+        checkpoint(1)
+    except TestException:
+      yield 2
+      return
+    finally:
+      yield 3
+
+    checkpoint(123)
+
+  test(it, 0, 1, 2, 3)
+
+block:
+  iterator it(): int {.closure.} =
+    try:
+      try:
+        yield 0
+        raiseException()
+      finally:
+        return # Return in finally should stop exception propagation
+    except AnotherException:
+      yield 2
+      return
+    finally:
+      yield 3
+    checkpoint(123)
+
+  test(it, 0, 3)
+
+block: # Yield in yield
+  iterator it(): int {.closure.} =
+    template foo(): int =
+      yield 1
+      2
+
+    for i in 0 .. 2:
+      checkpoint(0)
+      yield foo()
+
+  test(it, 0, 1, 2, 0, 1, 2, 0, 1, 2)
+
+block:
+  iterator it(): int {.closure.} =
+    let i = if true:
+        yield 0
+        1
+      else:
+        2
+    yield i
+
+  test(it, 0, 1)
+
+block:
+  iterator it(): int {.closure.} =
+    var foo = 123
+    let i = try:
+        yield 0
+        raiseException()
+        1
+      except TestException as e:
+        assert(e.msg == "Test exception!")
+        case foo
+        of 1:
+          yield 123
+          2
+        of 123:
+          yield 5
+          6
+        else:
+          7
+    yield i
+
+  test(it, 0, 5, 6)
+
+block:
+  iterator it(): int {.closure.} =
+    proc voidFoo(i1, i2, i3: int) =
+      checkpoint(i1)
+      checkpoint(i2)
+      checkpoint(i3)
+
+    proc foo(i1, i2, i3: int): int =
+      voidFoo(i1, i2, i3)
+      i3
+
+    proc bar(i1: int): int =
+      checkpoint(i1)
+
+    template tryexcept: int =
+      try:
+        yield 1
+        raiseException()
+        123
+      except TestException:
+        yield 2
+        checkpoint(3)
+        4
+
+    let e1 = true
+
+    template ifelse1: int =
+      if e1:
+        yield 10
+        11
+      else:
+        12
+
+    template ifelse2: int =
+      if ifelse1() == 12:
+        yield 20
+        21
+      else:
+        yield 22
+        23
+
+    let i = foo(bar(0), tryexcept, ifelse2)
+    discard foo(bar(0), tryexcept, ifelse2)
+    voidFoo(bar(0), tryexcept, ifelse2)
+    yield i
+
+  test(it,
+
+    # let i = foo(bar(0), tryexcept, ifelse2)
+    0, # bar(0)
+    1, 2, 3, # tryexcept
+    10, # ifelse1
+    22, # ifelse22
+    0, 4, 23, # foo
+
+    # discard foo(bar(0), tryexcept, ifelse2)
+    0, # bar(0)
+    1, 2, 3, # tryexcept
+    10, # ifelse1
+    22, # ifelse22
+    0, 4, 23, # foo
+
+    # voidFoo(bar(0), tryexcept, ifelse2)
+    0, # bar(0)
+    1, 2, 3, # tryexcept
+    10, # ifelse1
+    22, # ifelse22
+    0, 4, 23, # foo
+
+    23 # i
+  )
+
+block:
+  iterator it(): int {.closure.} =
+    checkpoint(0)
+    for i in 0 .. 1:
+      try:
+        yield 1
+        raiseException()
+      except TestException as e:
+        doAssert(e.msg == "Test exception!")
+        yield 2
+      except AnotherException:
+        yield 123
+      except:
+        yield 1234
+      finally:
+        yield 3
+        checkpoint(4)
+        yield 5
+
+  test(it, 0, 1, 2, 3, 4, 5, 1, 2, 3, 4, 5)
+
+block:
+  iterator it(): int {.closure.} =
+    var i = 5
+    template foo(): bool =
+      yield i
+      true
+
+    while foo():
+      dec i
+      if i == 0:
+        break
+
+  test(it, 5, 4, 3, 2, 1)
+
+block: # Short cirquits
+  iterator it(): int {.closure.} =
+    template trueYield: bool =
+      yield 1
+      true
+
+    template falseYield: bool =
+      yield 0
+      false
+
+    if trueYield or falseYield:
+      discard falseYield and trueYield
+
+    if falseYield and trueYield:
+      checkpoint(123)
+
+  test(it, 1, 0, 0)
 
-    test(it, 0, 3)
 
 echo "ok"

From c854865d3ea24d40e0823791eda5ec6669375614 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Tue, 8 May 2018 01:47:19 +0300
Subject: [PATCH 09/51] Corrected nkExceptBranch transformation

---
 compiler/closureiters.nim | 37 ++++++++++++++++++++++++-------------
 1 file changed, 24 insertions(+), 13 deletions(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index a30b4e10ee..30c03bbe1e 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -313,33 +313,44 @@ proc newNullifyCurExc(ctx: var Ctx): PNode =
   nilnode.typ = curExc.typ
   result.add(nilnode)
 
+proc newOr(a, b: PNode): PNode =
+  result = newNode(nkCall)
+  result.add(newSymNode(getSysMagic("or", mOr)))
+  result.add(a)
+  result.add(b)
+  result.typ = getSysType(tyBool)
+
 proc collectExceptState(ctx: var Ctx, n: PNode): PNode =
   var ifStmt = newNode(nkIfStmt)
   for c in n:
     if c.kind == nkExceptBranch:
       var ifBranch: PNode
-      var branchBody: PNode
 
-      if c[0].kind == nkType:
-        assert(c.len == 2)
+      if c.len > 1:
+        var cond: PNode
+        for i in 0 .. c.len - 2:
+          assert(c[i].kind == nkType)
+          let nextCond = newNodeI(nkCall, n.info)
+          nextCond.add(newSymNode(getSysMagic("of", mOf)))
+          nextCond.add(callCodegenProc("getCurrentException", emptyNode))
+          nextCond.add(c[i])
+          nextCond.typ = getSysType(tyBool)
+
+          if cond.isNil:
+            cond = nextCond
+          else:
+            cond = newOr(cond, nextCond)
+
         ifBranch = newNode(nkElifBranch)
-        let expression = newNodeI(nkCall, n.info)
-        expression.add(newSymNode(getSysMagic("of", mOf)))
-        expression.add(callCodegenProc("getCurrentException", emptyNode))
-        expression.add(c[0])
-        expression.typ = getSysType(tyBool)
-        ifBranch.add(expression)
-        branchBody = c[1]
+        ifBranch.add(cond)
       else:
-        assert(c.len == 1)
         if ifStmt.len == 0:
           ifStmt = newNode(nkStmtList)
           ifBranch = newNode(nkStmtList)
         else:
           ifBranch = newNode(nkElse)
-        branchBody = c[0]
 
-      ifBranch.add(branchBody)
+      ifBranch.add(c[^1])
       ifStmt.add(ifBranch)
 
   if ifStmt.len != 0:

From fb965719a65809cbbfbcd9d83aabcc8216c6a54c Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Tue, 8 May 2018 03:52:24 +0300
Subject: [PATCH 10/51] Fixed codegen (added blockLeaveActions) to closure
 iters

---
 compiler/ccgstmts.nim | 73 ++++++++++++++++++++++---------------------
 1 file changed, 38 insertions(+), 35 deletions(-)

diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim
index 96f5b53a77..510dbfc185 100644
--- a/compiler/ccgstmts.nim
+++ b/compiler/ccgstmts.nim
@@ -157,6 +157,39 @@ proc genState(p: BProc, n: PNode) =
   elif n0.kind == nkStrLit:
     linefmt(p, cpsStmts, "$1: ;$n", n0.strVal.rope)
 
+proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
+  # Called by return and break stmts.
+  # Deals with issues faced when jumping out of try/except/finally stmts,
+
+  var stack = newSeq[tuple[n: PNode, inExcept: bool]](0)
+
+  for i in countup(1, howManyTrys):
+    let tryStmt = p.nestedTryStmts.pop
+    if not p.module.compileToCpp or optNoCppExceptions in gGlobalOptions:
+      # Pop safe points generated by try
+      if not tryStmt.inExcept:
+        linefmt(p, cpsStmts, "#popSafePoint();$n")
+
+    # Pop this try-stmt of the list of nested trys
+    # so we don't infinite recurse on it in the next step.
+    stack.add(tryStmt)
+
+    # Find finally-stmt for this try-stmt
+    # and generate a copy of its sons
+    var finallyStmt = lastSon(tryStmt.n)
+    if finallyStmt.kind == nkFinally:
+      genStmts(p, finallyStmt.sons[0])
+
+  # push old elements again:
+  for i in countdown(howManyTrys-1, 0):
+    p.nestedTryStmts.add(stack[i])
+
+  if not p.module.compileToCpp or optNoCppExceptions in gGlobalOptions:
+    # Pop exceptions that was handled by the
+    # except-blocks we are in
+    for i in countdown(howManyExcepts-1, 0):
+      linefmt(p, cpsStmts, "#popCurrentException();$n")
+
 proc genGotoState(p: BProc, n: PNode) =
   # we resist the temptation to translate it into duff's device as it later
   # will be translated into computed gotos anyway for GCC at least:
@@ -167,7 +200,11 @@ proc genGotoState(p: BProc, n: PNode) =
   initLocExpr(p, n.sons[0], a)
   lineF(p, cpsStmts, "switch ($1) {$n", [rdLoc(a)])
   p.beforeRetNeeded = true
-  lineF(p, cpsStmts, "case -1: goto BeforeRet_;$n", [])
+  lineF(p, cpsStmts, "case -1:$n", [])
+  blockLeaveActions(p,
+    howManyTrys    = p.nestedTryStmts.len,
+    howManyExcepts = p.inExceptBlockLen)
+  lineF(p, cpsStmts, " goto BeforeRet_;$n", [])
   var statesCounter = lastOrd(n.sons[0].typ)
   if n.len >= 2 and n[1].kind == nkIntLit:
     statesCounter = n[1].intVal
@@ -328,40 +365,6 @@ proc genIf(p: BProc, n: PNode, d: var TLoc) =
     else: internalError(n.info, "genIf()")
   if sonsLen(n) > 1: fixLabel(p, lend)
 
-
-proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
-  # Called by return and break stmts.
-  # Deals with issues faced when jumping out of try/except/finally stmts,
-
-  var stack = newSeq[tuple[n: PNode, inExcept: bool]](0)
-
-  for i in countup(1, howManyTrys):
-    let tryStmt = p.nestedTryStmts.pop
-    if not p.module.compileToCpp or optNoCppExceptions in gGlobalOptions:
-      # Pop safe points generated by try
-      if not tryStmt.inExcept:
-        linefmt(p, cpsStmts, "#popSafePoint();$n")
-
-    # Pop this try-stmt of the list of nested trys
-    # so we don't infinite recurse on it in the next step.
-    stack.add(tryStmt)
-
-    # Find finally-stmt for this try-stmt
-    # and generate a copy of its sons
-    var finallyStmt = lastSon(tryStmt.n)
-    if finallyStmt.kind == nkFinally:
-      genStmts(p, finallyStmt.sons[0])
-
-  # push old elements again:
-  for i in countdown(howManyTrys-1, 0):
-    p.nestedTryStmts.add(stack[i])
-
-  if not p.module.compileToCpp or optNoCppExceptions in gGlobalOptions:
-    # Pop exceptions that was handled by the
-    # except-blocks we are in
-    for i in countdown(howManyExcepts-1, 0):
-      linefmt(p, cpsStmts, "#popCurrentException();$n")
-
 proc genReturnStmt(p: BProc, t: PNode) =
   if nfPreventCg in t.flags: return
   p.beforeRetNeeded = true

From 0b5883c21ea0f99a6742fa37d7101c31bc34bcba Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Tue, 8 May 2018 09:39:58 +0300
Subject: [PATCH 11/51] Small fix and cosmetics

---
 compiler/closureiters.nim  | 7 +------
 compiler/lambdalifting.nim | 3 +--
 2 files changed, 2 insertions(+), 8 deletions(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index 30c03bbe1e..7172130baf 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -210,11 +210,6 @@ proc newCurExcAccess(ctx: var Ctx): PNode =
     ctx.curExcSym = ctx.newEnvVar(":curExc", callCodegenProc("getCurrentException", emptyNode).typ)
   ctx.newEnvVarAccess(ctx.curExcSym)
 
-proc setStateInAssgn(stateAssgn: PNode, stateNo: int) =
-  assert stateAssgn.kind == nkAsgn
-  assert stateAssgn[1].kind == nkIntLit
-  stateAssgn[1].intVal = stateNo
-
 proc newState(ctx: var Ctx, n, gotoOut: PNode): int =
   # Creates a new state, adds it to the context fills out `gotoOut` so that it
   # will goto this state.
@@ -710,7 +705,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       n[0] = ex
       result.add(n)
 
-  of nkCast:
+  of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv:
     var ns = false
     for i in 0 ..< n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim
index a118edf00f..3e4d09709b 100644
--- a/compiler/lambdalifting.nim
+++ b/compiler/lambdalifting.nim
@@ -881,9 +881,8 @@ proc liftForLoop*(body: PNode; owner: PSym): PNode =
       cl = createClosure()
       while true:
         let i = foo(cl)
-        if cl.state < 0:
+        if (nkBreakState(cl.state)):
           break
-        # nkBreakState(cl.state)
         ...
     """
   if liftingHarmful(owner): return body

From d99c82bc3b207952cdcce85fdd1f9033bfb7dbef Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Tue, 8 May 2018 12:32:55 +0300
Subject: [PATCH 12/51] Cosmetics

---
 compiler/closureiters.nim  | 237 ++++++++++++-------------------------
 compiler/lambdalifting.nim |   4 +-
 2 files changed, 77 insertions(+), 164 deletions(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index 7172130baf..f17cfbe25b 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -131,9 +131,8 @@
 #   break :stateLoop
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os, options,
-  idents, renderer, types, magicsys, rodread, lowerings, tables, sequtils,
-  lambdalifting
+  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, idents,
+  renderer, types, magicsys, rodread, lowerings, lambdalifting
 
 type
   Ctx = object
@@ -164,9 +163,7 @@ proc newStateAccess(ctx: var Ctx): PNode =
 proc newStateAssgn(ctx: var Ctx, toValue: PNode): PNode =
   # Creates state assignment:
   #   :state = toValue
-  result = newNode(nkAsgn)
-  result.add(ctx.newStateAccess())
-  result.add(toValue)
+  newTree(nkAsgn, ctx.newStateAccess(), toValue)
 
 proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
   # Creates state assignment:
@@ -300,22 +297,16 @@ proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode
 
 proc newNullifyCurExc(ctx: var Ctx): PNode =
   # :curEcx = nil
-  result = newNode(nkAsgn)
   let curExc = ctx.newCurExcAccess()
-  result.add(curExc)
-
   let nilnode = newNode(nkNilLit)
   nilnode.typ = curExc.typ
-  result.add(nilnode)
+  result = newTree(nkAsgn, curExc, nilnode)
 
-proc newOr(a, b: PNode): PNode =
-  result = newNode(nkCall)
-  result.add(newSymNode(getSysMagic("or", mOr)))
-  result.add(a)
-  result.add(b)
+proc newOr(a, b: PNode): PNode {.inline.} =
+  result = newTree(nkCall, newSymNode(getSysMagic("or", mOr)), a, b)
   result.typ = getSysType(tyBool)
 
-proc collectExceptState(ctx: var Ctx, n: PNode): PNode =
+proc collectExceptState(ctx: var Ctx, n: PNode): PNode {.inline.} =
   var ifStmt = newNode(nkIfStmt)
   for c in n:
     if c.kind == nkExceptBranch:
@@ -349,9 +340,7 @@ proc collectExceptState(ctx: var Ctx, n: PNode): PNode =
       ifStmt.add(ifBranch)
 
   if ifStmt.len != 0:
-    result = newNode(nkStmtList)
-    result.add(ctx.newNullifyCurExc())
-    result.add(ifStmt)
+    result = newTree(nkStmtList, ctx.newNullifyCurExc(), ifStmt)
   else:
     result = emptyNode
 
@@ -362,21 +351,17 @@ proc addElseToExcept(ctx: var Ctx, n: PNode) =
     let branchBody = newNode(nkStmtList)
 
     block: # :unrollFinally = true
-      let asgn = newNode(nkAsgn)
-      asgn.add(ctx.newUnrollFinallyAccess())
-      asgn.add(newIntTypeNode(nkIntLit, 1, getSysType(tyBool)))
-      branchBody.add(asgn)
+      branchBody.add(newTree(nkAsgn,
+        ctx.newUnrollFinallyAccess(),
+        newIntTypeNode(nkIntLit, 1, getSysType(tyBool))))
 
     block: # :curExc = getCurrentException()
-      let asgn = newNode(nkAsgn)
-      asgn.add(ctx.newCurExcAccess)
-      asgn.add(callCodegenProc("getCurrentException", emptyNode))
-      branchBody.add(asgn)
+      branchBody.add(newTree(nkAsgn,
+        ctx.newCurExcAccess(),
+        callCodegenProc("getCurrentException", emptyNode)))
 
     block: # goto nearestFinally
-      let goto = newNode(nkGotoState)
-      goto.add(newIntLit(ctx.nearestFinally))
-      branchBody.add(goto)
+      branchBody.add(newTree(nkGotoState, newIntLit(ctx.nearestFinally)))
 
     elseBranch.add(branchBody)
     n[1].add(elseBranch)
@@ -421,9 +406,7 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
   result.res = lastSon
 
 proc newEnvVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode =
-  result = newNode(nkFastAsgn)
-  result.add(ctx.newEnvVarAccess(s))
-  result.add(v)
+  newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
 
 proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) =
   if input.kind == nkStmtListExpr:
@@ -438,9 +421,7 @@ proc convertExprBodyToAsgn(ctx: Ctx, exprBody: PNode, res: PSym): PNode =
   ctx.addExprAssgn(result, exprBody, res)
 
 proc newNotCall(e: PNode): PNode =
-  result = newNode(nkCall)
-  result.add(newSymNode(getSysMagic("not", mNot)))
-  result.add(e)
+  result = newTree(nkCall, newSymNode(getSysMagic("not", mNot)), e)
   result.typ = getSysType(tyBool)
 
 proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
@@ -751,19 +732,11 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       needsSplit = true
 
       if condNeedsSplit:
-        let newBody = newNode(nkStmtList)
-
         let (st, ex) = exprToStmtList(n[0])
-        newBody.add(st)
-        let check = newNode(nkIfStmt)
-        let branch = newNode(nkElifBranch)
-        branch.add(newNotCall(ex))
-        let brk = newNode(nkBreakStmt)
-        brk.add(emptyNode)
-        branch.add(brk)
-        check.add(branch)
-        newBody.add(check)
-        newBody.add(n[1])
+        let brk = newTree(nkBreakStmt, emptyNode)
+        let branch = newTree(nkElifBranch, newNotCall(ex), brk)
+        let check = newTree(nkIfStmt, branch)
+        let newBody = newTree(nkStmtList, st, check, n[1])
 
         n[0] = newSymNode(getSysSym("true"))
         n[1] = newBody
@@ -778,44 +751,26 @@ proc newEndFinallyNode(ctx: var Ctx): PNode =
   #         return :tmpResult
   #       else:
   #         raise
-
-  result = newNode(nkIfStmt)
-
-  let elifBranch = newNode(nkElifBranch)
-  elifBranch.add(ctx.newUnrollFinallyAccess())
-  result.add(elifBranch)
-
-  let ifBody = newNode(nkIfStmt)
-  let branch = newNode(nkElifBranch)
-
-  let cmp = newNode(nkCall)
-  cmp.add(getSysMagic("==", mEqRef).newSymNode)
   let curExc = ctx.newCurExcAccess()
   let nilnode = newNode(nkNilLit)
   nilnode.typ = curExc.typ
-  cmp.add(curExc)
-  cmp.add(nilnode)
+  let cmp = newTree(nkCall, getSysMagic("==", mEqRef).newSymNode, curExc, nilnode)
   cmp.typ = getSysType(tyBool)
-  branch.add(cmp)
 
-  let retStmt = newNode(nkReturnStmt)
-  let asgn = newNode(nkFastAsgn)
-  addSon(asgn, newSymNode(getClosureIterResult(ctx.fn)))
-  addSon(asgn, ctx.newTmpResultAccess())
-  retStmt.add(asgn)
-  branch.add(retStmt)
+  let asgn = newTree(nkFastAsgn,
+    newSymNode(getClosureIterResult(ctx.fn)),
+    ctx.newTmpResultAccess())
 
-  let elseBranch = newNode(nkElse)
-  let raiseStmt = newNode(nkRaiseStmt)
+  let retStmt = newTree(nkReturnStmt, asgn)
+  let branch = newTree(nkElifBranch, cmp, retStmt)
 
   # The C++ backend requires `getCurrentException` here.
-  raiseStmt.add(callCodegenProc("getCurrentException", emptyNode))
-  elseBranch.add(raiseStmt)
+  let raiseStmt = newTree(nkRaiseStmt, callCodegenProc("getCurrentException", emptyNode))
+  let elseBranch = newTree(nkElse, raiseStmt)
 
-  ifBody.add(branch)
-  ifBody.add(elseBranch)
-
-  elifBranch.add(ifBody)
+  let ifBody = newTree(nkIfStmt, branch, elseBranch)
+  let elifBranch = newTree(nkElifBranch, ctx.newUnrollFinallyAccess(), ifBody)
+  result = newTree(nkIfStmt, elifBranch)
 
 proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
   result = n
@@ -950,8 +905,7 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
       result = newNode(nkGotoState)
       var tryBody = toStmtList(n[0])
       var exceptBody = ctx.collectExceptState(n)
-      var finallyBody = newNode(nkStmtList)
-      finallyBody.add(getFinallyNode(n))
+      var finallyBody = newTree(nkStmtList, getFinallyNode(n))
       finallyBody = ctx.transformReturnsInTry(finallyBody)
       finallyBody.add(ctx.newEndFinallyNode())
 
@@ -1125,15 +1079,13 @@ proc newArrayType(n: int, t: PType, owner: PSym): PType =
   result = newType(tyArray, owner)
 
   let rng = newType(tyRange, owner)
-  rng.n = newNode(nkRange)
-  rng.n.add(newIntLit(0))
-  rng.n.add(newIntLit(n))
+  rng.n = newTree(nkRange, newIntLit(0), newIntLit(n))
   rng.rawAddSon(t)
 
   result.rawAddSon(rng)
   result.rawAddSon(t)
 
-proc createExceptionTable(ctx: var Ctx): PNode =
+proc createExceptionTable(ctx: var Ctx): PNode {.inline.} =
   result = newNode(nkBracket)
   result.typ = newArrayType(ctx.exceptionTable.len, getSysType(tyInt16), ctx.fn)
 
@@ -1157,9 +1109,9 @@ proc newCatchBody(ctx: var Ctx): PNode {.inline.} =
   block:
 
     # exceptionTable[:state]
-    let getNextState = newNode(nkBracketExpr)
-    getNextState.add(ctx.createExceptionTable)
-    getNextState.add(ctx.newStateAccess())
+    let getNextState = newTree(nkBracketExpr,
+      ctx.createExceptionTable(),
+      ctx.newStateAccess())
     getNextState.typ = getSysType(tyInt)
 
     # :state = exceptionTable[:state]
@@ -1167,96 +1119,68 @@ proc newCatchBody(ctx: var Ctx): PNode {.inline.} =
 
   # if :state == 0: raise
   block:
-    let ifStmt = newNode(nkIfStmt)
-    let ifBranch = newNode(nkElifBranch)
-    let cond = newNode(nkCall)
-    cond.add(getSysMagic("==", mEqI).newSymNode)
-    cond.add(ctx.newStateAccess())
-    cond.add(newIntTypeNode(nkIntLit, 0, getSysType(tyInt)))
+    let cond = newTree(nkCall,
+      getSysMagic("==", mEqI).newSymNode(),
+      ctx.newStateAccess(),
+      newIntTypeNode(nkIntLit, 0, getSysType(tyInt)))
     cond.typ = getSysType(tyBool)
-    ifBranch.add(cond)
 
-    let raiseStmt = newNode(nkRaiseStmt)
-    raiseStmt.add(emptyNode)
-
-    ifBranch.add(raiseStmt)
-    ifStmt.add(ifBranch)
+    let raiseStmt = newTree(nkRaiseStmt, emptyNode)
+    let ifBranch = newTree(nkElifBranch, cond, raiseStmt)
+    let ifStmt = newTree(nkIfStmt, ifBranch)
     result.add(ifStmt)
 
   # :unrollFinally = :state > 0
   block:
-    let asgn = newNode(nkAsgn)
-    asgn.add(ctx.newUnrollFinallyAccess())
-
-    let cond = newNode(nkCall)
-    cond.add(getSysMagic("<", mLtI).newSymNode)
-    cond.add(newIntTypeNode(nkIntLit, 0, getSysType(tyInt)))
-    cond.add(ctx.newStateAccess())
+    let cond = newTree(nkCall,
+      getSysMagic("<", mLtI).newSymNode,
+      newIntTypeNode(nkIntLit, 0, getSysType(tyInt)),
+      ctx.newStateAccess())
     cond.typ = getSysType(tyBool)
-    asgn.add(cond)
+
+    let asgn = newTree(nkAsgn, ctx.newUnrollFinallyAccess(), cond)
     result.add(asgn)
 
   # if :state < 0: :state = -:state
   block:
-    let ifStmt = newNode(nkIfStmt)
-    let ifBranch = newNode(nkElifBranch)
-    let cond = newNode(nkCall)
-    cond.add(getSysMagic("<", mLtI).newSymNode)
-    cond.add(ctx.newStateAccess())
-    cond.add(newIntTypeNode(nkIntLit, 0, getSysType(tyInt)))
+    let cond = newTree(nkCall,
+      getSysMagic("<", mLtI).newSymNode,
+      ctx.newStateAccess(),
+      newIntTypeNode(nkIntLit, 0, getSysType(tyInt)))
     cond.typ = getSysType(tyBool)
-    ifBranch.add(cond)
 
-    let negateState = newNode(nkCall)
-    negateState.add(getSysMagic("-", mUnaryMinusI).newSymNode)
-    negateState.add(ctx.newStateAccess())
+    let negateState = newTree(nkCall,
+      getSysMagic("-", mUnaryMinusI).newSymNode,
+      ctx.newStateAccess())
     negateState.typ = getSysType(tyInt)
 
-    ifBranch.add(ctx.newStateAssgn(negateState))
-    ifStmt.add(ifBranch)
+    let ifBranch = newTree(nkElifBranch, cond, ctx.newStateAssgn(negateState))
+    let ifStmt = newTree(nkIfStmt, ifBranch)
     result.add(ifStmt)
 
   # :curExc = getCurrentException()
   block:
-    let getCurExc = callCodegenProc("getCurrentException", emptyNode)
-    let asgn = newNode(nkAsgn)
-    asgn.add(ctx.newCurExcAccess())
-    asgn.add(getCurExc)
-    result.add(asgn)
+    result.add(newTree(nkAsgn,
+      ctx.newCurExcAccess(),
+      callCodegenProc("getCurrentException", emptyNode)))
 
-proc wrapIntoTryExcept(ctx: var Ctx, n: PNode): PNode =
-  result = newNode(nkTryStmt)
+proc wrapIntoTryExcept(ctx: var Ctx, n: PNode): PNode {.inline.} =
+  let setupExc = newTree(nkCall,
+    newSymNode(getCompilerProc("closureIterSetupExc")),
+    ctx.newCurExcAccess())
 
-  let tryBody = newNode(nkStmtList)
+  let tryBody = newTree(nkStmtList, setupExc, n)
+  let exceptBranch = newTree(nkExceptBranch, ctx.newCatchBody())
 
-  let setupExc = newNode(nkCall)
-  setupExc.add(newSymNode(getCompilerProc("closureIterSetupExc")))
-
-  tryBody.add(setupExc)
-
-  tryBody.add(n)
-  result.add(tryBody)
-
-  let catchNode = newNode(nkExceptBranch)
-  result.add(catchNode)
-
-  let catchBody = newNode(nkStmtList)
-  catchBody.add(ctx.newCatchBody())
-  catchNode.add(catchBody)
-
-  setupExc.add(ctx.newCurExcAccess())
+  result = newTree(nkTryStmt, tryBody, exceptBranch)
 
 proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   # while true:
   #   block :stateLoop:
   #     gotoState :state
   #     body # Might get wrapped in try-except
-
-  result = newNode(nkWhileStmt)
-  result.add(newSymNode(getSysSym("true")))
-
   let loopBody = newNodeI(nkStmtList, n.info)
-  result.add(loopBody)
+  result = newTree(nkWhileStmt, newSymNode(getSysSym("true")), loopBody)
 
   if not ctx.stateVarSym.isNil:
     let varSect = newNodeI(nkVarSection, n.info)
@@ -1269,26 +1193,19 @@ proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   let blockStmt = newNodeI(nkBlockStmt, n.info)
   blockStmt.add(newSymNode(ctx.stateLoopLabel))
 
-  var blockBody = newNodeI(nkStmtList, n.info)
-
   let gs = newNodeI(nkGotoState, n.info)
   gs.add(ctx.newStateAccess())
   gs.add(newIntLit(ctx.states.len - 1))
 
-  blockBody.add(gs)
-  blockBody.add(n)
-
+  var blockBody = newTree(nkStmtList, gs, n)
   if ctx.hasExceptions:
     blockBody = ctx.wrapIntoTryExcept(blockBody)
 
   blockStmt.add(blockBody)
-
   loopBody.add(blockStmt)
 
 proc deleteEmptyStates(ctx: var Ctx) =
-  let goOut = newNode(nkGotoState)
-  goOut.add(newIntLit(-1))
-
+  let goOut = newTree(nkGotoState, newIntLit(-1))
   ctx.exitStateIdx = ctx.newState(goOut, nil)
 
   # Apply new state indexes and mark unused states with -1
@@ -1332,14 +1249,11 @@ proc transformClosureIterator*(fn: PSym, n: PNode): PNode =
     ctx.stateVarSym = newSym(skVar, getIdent(":state"), fn, fn.info)
     ctx.stateVarSym.typ = createClosureIterStateType(fn)
 
-  ctx.states = @[]
   ctx.stateLoopLabel = newSym(skLabel, getIdent(":stateLoop"), fn, fn.info)
-  ctx.exceptionTable = @[]
   let n = n.toStmtList
 
   discard ctx.newState(n, nil)
-  let gotoOut = newNode(nkGotoState)
-  gotoOut.add(newIntLit(-1))
+  let gotoOut = newTree(nkGotoState, newIntLit(-1))
 
   # Splitting transformation
   discard ctx.transformClosureIteratorBody(n, gotoOut)
@@ -1349,8 +1263,7 @@ proc transformClosureIterator*(fn: PSym, n: PNode): PNode =
 
   # Make new body by concating the list of states
   result = newNode(nkStmtList)
-  for i, s in ctx.states:
-    # result.add(s)
+  for s in ctx.states:
     let body = s[1]
     s.sons.del(1)
     result.add(s)
diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim
index 3e4d09709b..43ff50190d 100644
--- a/compiler/lambdalifting.nim
+++ b/compiler/lambdalifting.nim
@@ -10,8 +10,8 @@
 # This file implements lambda lifting for the transformator.
 
 import
-  intsets, strutils, options, ast, astalgo, trees, treetab, msgs, os, options,
-  idents, renderer, types, magicsys, rodread, lowerings, tables, sequtils
+  intsets, strutils, options, ast, astalgo, trees, treetab, msgs,
+  idents, renderer, types, magicsys, rodread, lowerings, tables
 
 discard """
   The basic approach is that captured vars need to be put on the heap and

From c1dde282d68fb4e21025c1d14077283ddc2330fa Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Tue, 8 May 2018 13:49:37 +0300
Subject: [PATCH 13/51] Fixed line info

---
 compiler/closureiters.nim | 1 +
 1 file changed, 1 insertion(+)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index f17cfbe25b..2328e2c55a 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -1181,6 +1181,7 @@ proc wrapIntoStateLoop(ctx: var Ctx, n: PNode): PNode =
   #     body # Might get wrapped in try-except
   let loopBody = newNodeI(nkStmtList, n.info)
   result = newTree(nkWhileStmt, newSymNode(getSysSym("true")), loopBody)
+  result.info = n.info
 
   if not ctx.stateVarSym.isNil:
     let varSect = newNodeI(nkVarSection, n.info)

From 5d166fcc0ae1ab203f965f0650fd4834542a46f6 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Thu, 10 May 2018 00:27:50 +0300
Subject: [PATCH 14/51] Review comments addressed. More thorough line info
 tracking.

---
 compiler/closureiters.nim        | 130 +++++++++++++++----------------
 tests/async/tasync_traceback.nim |   4 +-
 2 files changed, 67 insertions(+), 67 deletions(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index 2328e2c55a..193a812e1a 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -179,7 +179,7 @@ proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
     # We haven't gone through labmda lifting yet, so just create a local var,
     # it will be lifted later
     if ctx.tempVars.isNil:
-      ctx.tempVars = newNode(nkVarSection)
+      ctx.tempVars = newNodeI(nkVarSection, ctx.fn.info)
       addVar(ctx.tempVars, newSymNode(result))
   else:
     let envParam = getEnvParam(ctx.fn)
@@ -295,9 +295,10 @@ proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode
     for i in 0 ..< n.len:
       n[i] = ctx.transformBreaksInBlock(n[i], label, after)
 
-proc newNullifyCurExc(ctx: var Ctx): PNode =
+proc newNullifyCurExc(ctx: var Ctx, info: TLineInfo): PNode =
   # :curEcx = nil
   let curExc = ctx.newCurExcAccess()
+  curExc.info = info
   let nilnode = newNode(nkNilLit)
   nilnode.typ = curExc.typ
   result = newTree(nkAsgn, curExc, nilnode)
@@ -305,9 +306,10 @@ proc newNullifyCurExc(ctx: var Ctx): PNode =
 proc newOr(a, b: PNode): PNode {.inline.} =
   result = newTree(nkCall, newSymNode(getSysMagic("or", mOr)), a, b)
   result.typ = getSysType(tyBool)
+  result.info = a.info
 
 proc collectExceptState(ctx: var Ctx, n: PNode): PNode {.inline.} =
-  var ifStmt = newNode(nkIfStmt)
+  var ifStmt = newNodeI(nkIfStmt, n.info)
   for c in n:
     if c.kind == nkExceptBranch:
       var ifBranch: PNode
@@ -316,39 +318,39 @@ proc collectExceptState(ctx: var Ctx, n: PNode): PNode {.inline.} =
         var cond: PNode
         for i in 0 .. c.len - 2:
           assert(c[i].kind == nkType)
-          let nextCond = newNodeI(nkCall, n.info)
-          nextCond.add(newSymNode(getSysMagic("of", mOf)))
-          nextCond.add(callCodegenProc("getCurrentException", emptyNode))
-          nextCond.add(c[i])
+          let nextCond = newTree(nkCall,
+            newSymNode(getSysMagic("of", mOf)),
+            callCodegenProc("getCurrentException", emptyNode),
+            c[i])
           nextCond.typ = getSysType(tyBool)
+          nextCond.info = c.info
 
           if cond.isNil:
             cond = nextCond
           else:
             cond = newOr(cond, nextCond)
 
-        ifBranch = newNode(nkElifBranch)
+        ifBranch = newNodeI(nkElifBranch, c.info)
         ifBranch.add(cond)
       else:
         if ifStmt.len == 0:
-          ifStmt = newNode(nkStmtList)
-          ifBranch = newNode(nkStmtList)
+          ifStmt = newNodeI(nkStmtList, c.info)
+          ifBranch = newNodeI(nkStmtList, c.info)
         else:
-          ifBranch = newNode(nkElse)
+          ifBranch = newNodeI(nkElse, c.info)
 
       ifBranch.add(c[^1])
       ifStmt.add(ifBranch)
 
   if ifStmt.len != 0:
-    result = newTree(nkStmtList, ctx.newNullifyCurExc(), ifStmt)
+    result = newTree(nkStmtList, ctx.newNullifyCurExc(n.info), ifStmt)
   else:
     result = emptyNode
 
 proc addElseToExcept(ctx: var Ctx, n: PNode) =
   if n.kind == nkStmtList and n[1].kind == nkIfStmt and n[1][^1].kind != nkElse:
     # Not all cases are covered
-    let elseBranch = newNode(nkElse)
-    let branchBody = newNode(nkStmtList)
+    let branchBody = newNodeI(nkStmtList, n.info)
 
     block: # :unrollFinally = true
       branchBody.add(newTree(nkAsgn,
@@ -363,7 +365,7 @@ proc addElseToExcept(ctx: var Ctx, n: PNode) =
     block: # goto nearestFinally
       branchBody.add(newTree(nkGotoState, newIntLit(ctx.nearestFinally)))
 
-    elseBranch.add(branchBody)
+    let elseBranch = newTree(nkElse, branchBody)
     n[1].add(elseBranch)
 
 proc getFinallyNode(n: PNode): PNode =
@@ -406,7 +408,8 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
   result.res = lastSon
 
 proc newEnvVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode =
-  newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
+  result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
+  result.info = v.info
 
 proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) =
   if input.kind == nkStmtListExpr:
@@ -417,11 +420,11 @@ proc addExprAssgn(ctx: Ctx, output, input: PNode, sym: PSym) =
     output.add(ctx.newEnvVarAsgn(sym, input))
 
 proc convertExprBodyToAsgn(ctx: Ctx, exprBody: PNode, res: PSym): PNode =
-  result = newNode(nkStmtList)
+  result = newNodeI(nkStmtList, exprBody.info)
   ctx.addExprAssgn(result, exprBody, res)
 
 proc newNotCall(e: PNode): PNode =
-  result = newTree(nkCall, newSymNode(getSysMagic("not", mNot)), e)
+  result = newTree(nkCall, newSymNode(getSysMagic("not", mNot), e.info), e)
   result.typ = getSysType(tyBool)
 
 proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
@@ -446,7 +449,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
     needsSplit = true
 
-  of nkPar, nkObjConstr, nkTupleConstr, nkBracket, nkArgList:
+  of nkPar, nkObjConstr, nkTupleConstr, nkBracket:
     var ns = false
     for i in 0 ..< n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
@@ -477,10 +480,10 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       let isExpr = not isEmptyType(n.typ)
       if isExpr:
         tmp = ctx.newTempVar(n.typ)
-        result = newNode(nkStmtListExpr)
+        result = newNodeI(nkStmtListExpr, n.info)
         result.typ = n.typ
       else:
-        result = newNode(nkStmtList)
+        result = newNodeI(nkStmtList, n.info)
 
       var curS = result
 
@@ -488,10 +491,9 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         case branch.kind
         of nkElseExpr, nkElse:
           if isExpr:
-            var newBranch = newNodeI(nkElse, branch.info)
-            let branchBody = newNode(nkStmtList)
+            let branchBody = newNodeI(nkStmtList, branch.info)
             ctx.addExprAssgn(branchBody, branch[0], tmp)
-            newBranch.add(branchBody)
+            let newBranch = newTree(nkElse, branchBody)
             curS.add(newBranch)
           else:
             curS.add(branch)
@@ -499,17 +501,12 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         of nkElifExpr, nkElifBranch:
           var newBranch: PNode
           if branch[0].kind == nkStmtListExpr:
-            let elseBody = newNode(nkStmtList)
-
             let (st, res) = exprToStmtList(branch[0])
-            elseBody.add(st)
+            let elseBody = newTree(nkStmtList, st)
 
-            newBranch = newNodeI(nkElifBranch, branch.info)
-            newBranch.add(res)
-            newBranch.add(branch[1])
+            newBranch = newTree(nkElifBranch, res, branch[1])
 
-            let newIf = newNodeI(nkIfStmt, branch.info)
-            newIf.add(newBranch)
+            let newIf = newTree(nkIfStmt, newBranch)
             elseBody.add(newIf)
             if curS.kind == nkIfStmt:
               let newElse = newNodeI(nkElse, branch.info)
@@ -523,13 +520,12 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
             if curS.kind == nkIfStmt:
               curS.add(newBranch)
             else:
-              let newIf = newNodeI(nkIfStmt, branch.info)
-              newIf.add(newBranch)
+              let newIf = newTree(nkIfStmt, newBranch)
               curS.add(newIf)
               curS = newIf
 
           if isExpr:
-            let branchBody = newNode(nkStmtList)
+            let branchBody = newNodeI(nkStmtList, branch[1].info)
             ctx.addExprAssgn(branchBody, branch[1], tmp)
             newBranch[1] = branchBody
 
@@ -613,7 +609,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         result = newNodeI(nkStmtListExpr, n.info)
         result.typ = n.typ
       else:
-        result = newNode(nkStmtList, n.info)
+        result = newNodeI(nkStmtList, n.info)
 
       if n[0].kind == nkSym and n[0].sym.magic in {mAnd, mOr}: # `and`/`or` short cirquiting
         var cond = n[1]
@@ -625,23 +621,20 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         let tmp = ctx.newTempVar(cond.typ)
         result.add(ctx.newEnvVarAsgn(tmp, cond))
 
-        let ifNode = newNode(nkIfStmt)
-        let ifBranch = newNode(nkElifBranch)
-
         var check = ctx.newEnvVarAccess(tmp)
         if n[0].sym.magic == mOr:
           check = newNotCall(check)
-        ifBranch.add(check)
 
         cond = n[2]
-        let ifBody = newNode(nkStmtList)
+        let ifBody = newNodeI(nkStmtList, cond.info)
         if cond.kind == nkStmtListExpr:
           let (st, ex) = exprToStmtList(cond)
           ifBody.add(st)
           cond = ex
         ifBody.add(ctx.newEnvVarAsgn(tmp, cond))
-        ifBranch.add(ifBody)
-        ifNode.add(ifBranch)
+
+        let ifBranch = newTree(nkElifBranch, check, ifBody)
+        let ifNode = newTree(nkIfStmt, ifBranch)
         result.add(ifNode)
         result.add(ctx.newEnvVarAccess(tmp))
       else:
@@ -744,7 +737,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
     for i in 0 ..< n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], needsSplit)
 
-proc newEndFinallyNode(ctx: var Ctx): PNode =
+proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   # Generate the following code:
   #   if :unrollFinally:
   #       if :curExc.isNil:
@@ -754,11 +747,11 @@ proc newEndFinallyNode(ctx: var Ctx): PNode =
   let curExc = ctx.newCurExcAccess()
   let nilnode = newNode(nkNilLit)
   nilnode.typ = curExc.typ
-  let cmp = newTree(nkCall, getSysMagic("==", mEqRef).newSymNode, curExc, nilnode)
+  let cmp = newTree(nkCall, newSymNode(getSysMagic("==", mEqRef), info), curExc, nilnode)
   cmp.typ = getSysType(tyBool)
 
   let asgn = newTree(nkFastAsgn,
-    newSymNode(getClosureIterResult(ctx.fn)),
+    newSymNode(getClosureIterResult(ctx.fn), info),
     ctx.newTmpResultAccess())
 
   let retStmt = newTree(nkReturnStmt, asgn)
@@ -766,10 +759,12 @@ proc newEndFinallyNode(ctx: var Ctx): PNode =
 
   # The C++ backend requires `getCurrentException` here.
   let raiseStmt = newTree(nkRaiseStmt, callCodegenProc("getCurrentException", emptyNode))
+  raiseStmt.info = info
   let elseBranch = newTree(nkElse, raiseStmt)
 
   let ifBody = newTree(nkIfStmt, branch, elseBranch)
   let elifBranch = newTree(nkElifBranch, ctx.newUnrollFinallyAccess(), ifBody)
+  elifBranch.info = info
   result = newTree(nkIfStmt, elifBranch)
 
 proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
@@ -794,7 +789,7 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
       asgnTmpResult.add(n[0])
       result.add(asgnTmpResult)
 
-    result.add(ctx.newNullifyCurExc())
+    result.add(ctx.newNullifyCurExc(n.info))
 
     let goto = newNodeI(nkGotoState, n.info)
     goto.add(newIntLit(ctx.nearestFinally))
@@ -826,16 +821,17 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
 
         if n[i].hasYields:
           # Create a new split
-          let go = newNode(nkGotoState)
+          let go = newNodeI(nkGotoState, n[i].info)
           n[i] = ctx.transformClosureIteratorBody(n[i], go)
 
-          let s = newNode(nkStmtList)
+          let s = newNodeI(nkStmtList, n[i + 1].info)
           for j in i + 1 ..< n.len:
             s.add(n[j])
 
           n.sons.setLen(i + 1)
           discard ctx.newState(s, go)
-          discard ctx.transformClosureIteratorBody(s, gotoOut)
+          if ctx.transformClosureIteratorBody(s, gotoOut) != s:
+            internalError("transformClosureIteratorBody != s")
           break
 
     of nkYieldStmt:
@@ -857,8 +853,7 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
       if n[^1].kind != nkElse:
         # We don't have an else branch, but every possible branch has to end with
         # gotoOut, so add else here.
-        let elseBranch = newNode(nkElse)
-        elseBranch.add(gotoOut)
+        let elseBranch = newTree(nkElse, gotoOut)
         n.add(elseBranch)
 
     of nkWhileStmt:
@@ -888,8 +883,7 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
       elifBranch.add(body)
       ifNode.add(elifBranch)
 
-      let elseBranch = newNode(nkElse)
-      elseBranch.add(gotoOut)
+      let elseBranch = newTree(nkElse, gotoOut)
       ifNode.add(elseBranch)
       s.add(ifNode)
 
@@ -902,12 +896,12 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
       # See explanation above about how this works
       ctx.hasExceptions = true
 
-      result = newNode(nkGotoState)
+      result = newNodeI(nkGotoState, n.info)
       var tryBody = toStmtList(n[0])
       var exceptBody = ctx.collectExceptState(n)
       var finallyBody = newTree(nkStmtList, getFinallyNode(n))
       finallyBody = ctx.transformReturnsInTry(finallyBody)
-      finallyBody.add(ctx.newEndFinallyNode())
+      finallyBody.add(ctx.newEndFinallyNode(finallyBody.info))
 
       # The following index calculation is based on the knowledge how state
       # indexes are assigned
@@ -920,7 +914,7 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
         exceptIdx = tryIdx + 1
         finallyIdx = tryIdx + 1
 
-      let outToFinally = newNode(nkGotoState)
+      let outToFinally = newNodeI(nkGotoState, finallyBody.info)
 
       block: # Create initial states.
         let oldExcHandlingState = ctx.curExcHandlingState
@@ -945,17 +939,22 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
 
         ctx.curExcHandlingState = exceptIdx
 
-        discard ctx.transformReturnsInTry(tryBody)
-        discard ctx.transformClosureIteratorBody(tryBody, outToFinally)
+        if ctx.transformReturnsInTry(tryBody) != tryBody:
+          internalError("transformReturnsInTry != tryBody")
+        if ctx.transformClosureIteratorBody(tryBody, outToFinally) != tryBody:
+          internalError("transformClosureIteratorBody != tryBody")
 
         ctx.curExcHandlingState = finallyIdx
         ctx.addElseToExcept(exceptBody)
-        discard ctx.transformReturnsInTry(exceptBody)
-        discard ctx.transformClosureIteratorBody(exceptBody, outToFinally)
+        if ctx.transformReturnsInTry(exceptBody) != exceptBody:
+          internalError("transformReturnsInTry != exceptBody")
+        if ctx.transformClosureIteratorBody(exceptBody, outToFinally) != exceptBody:
+          internalError("transformClosureIteratorBody != exceptBody")
 
         ctx.curExcHandlingState = oldExcHandlingState
         ctx.nearestFinally = oldNearestFinally
-        discard ctx.transformClosureIteratorBody(finallyBody, gotoOut)
+        if ctx.transformClosureIteratorBody(finallyBody, gotoOut) != finallyBody:
+          internalError("transformClosureIteratorBody != finallyBody")
 
     of nkGotoState, nkForStmt:
       internalError("closure iter " & $n.kind)
@@ -1086,7 +1085,7 @@ proc newArrayType(n: int, t: PType, owner: PSym): PType =
   result.rawAddSon(t)
 
 proc createExceptionTable(ctx: var Ctx): PNode {.inline.} =
-  result = newNode(nkBracket)
+  result = newNodeI(nkBracket, ctx.fn.info)
   result.typ = newArrayType(ctx.exceptionTable.len, getSysType(tyInt16), ctx.fn)
 
   for i in ctx.exceptionTable:
@@ -1103,7 +1102,7 @@ proc newCatchBody(ctx: var Ctx): PNode {.inline.} =
   #   :state = -:state
   # :curExc = getCurrentException()
 
-  result = newNode(nkStmtList)
+  result = newNodeI(nkStmtList, ctx.fn.info)
 
   # :state = exceptionTable[:state]
   block:
@@ -1263,8 +1262,9 @@ proc transformClosureIterator*(fn: PSym, n: PNode): PNode =
   ctx.deleteEmptyStates()
 
   # Make new body by concating the list of states
-  result = newNode(nkStmtList)
+  result = newNodeI(nkStmtList, n.info)
   for s in ctx.states:
+    assert(s.len == 2)
     let body = s[1]
     s.sons.del(1)
     result.add(s)
diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim
index e4c8a67b34..618a1dc769 100644
--- a/tests/async/tasync_traceback.nim
+++ b/tests/async/tasync_traceback.nim
@@ -82,7 +82,7 @@ Async traceback:
     asyncmacro\.nim\(\d+?\)\s+?a
     asyncmacro\.nim\(\d+?\)\s+?a_continue
       ## Resumes an async procedure
-    asyncmacro\.nim\(\d+?\)\s+?aIter
+    tasync_traceback\.nim\(\d+?\)\s+?aIter
     asyncfutures\.nim\(\d+?\)\s+?read
   \]#
 Exception message: b failure
@@ -110,7 +110,7 @@ Async traceback:
       ## Executes pending callbacks
     asyncmacro\.nim\(\d+?\)\s+?foo_continue
       ## Resumes an async procedure
-    asyncmacro\.nim\(\d+?\)\s+?fooIter
+    tasync_traceback\.nim\(\d+?\)\s+?fooIter
     asyncfutures\.nim\(\d+?\)\s+?read
   \]#
 Exception message: bar failure

From 5e8faaf7103aed2a8098880e1f2410961546df21 Mon Sep 17 00:00:00 2001
From: data-man <datamanrb@gmail.com>
Date: Mon, 14 May 2018 19:13:11 +0300
Subject: [PATCH 15/51] Fixes #2753

---
 lib/pure/net.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index fc04ef1af4..ce769cb6a6 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -1148,7 +1148,7 @@ proc waitFor(socket: Socket, waited: var float, timeout, size: int,
           return 1
         let sslPending = SSLPending(socket.sslHandle)
         if sslPending != 0:
-          return sslPending
+          return min(sslPending, size)
 
     var startTime = epochTime()
     let selRet = select(socket, timeout - int(waited * 1000.0))

From 606d8b2f6d0ed4c26d44d2b07b8fef39cabc0b3c Mon Sep 17 00:00:00 2001
From: data-man <datamanrb@gmail.com>
Date: Tue, 15 May 2018 00:12:44 +0300
Subject: [PATCH 16/51] Added test

---
 tests/stdlib/thttpclient.nim | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tests/stdlib/thttpclient.nim b/tests/stdlib/thttpclient.nim
index fff02722a9..c28f091003 100644
--- a/tests/stdlib/thttpclient.nim
+++ b/tests/stdlib/thttpclient.nim
@@ -154,8 +154,20 @@ proc ipv6Test() =
   serverFd.closeSocket()
   client.close()
 
+proc longTimeoutTest() =
+# Issue #2753
+  try:
+    var client = newHttpClient(timeout = 1000)
+    var resp = client.request("https://au.yahoo.com")
+    client.close()
+  except AssertionError:
+    doAssert false, "Exceptions should not be raised"
+  except:
+    discard
+
 syncTest()
 waitFor(asyncTest())
 ipv6Test()
+longTimeoutTest()
 
 echo "OK"

From fd2823636820e995f0d1c4370542ee1dbfc55442 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= <oscarnihlgard@gmail.com>
Date: Wed, 30 May 2018 09:40:35 +0200
Subject: [PATCH 17/51] Multi byte characters should not be treated as part of
 an operator

---
 compiler/lexer.nim                 |  2 +-
 lib/packages/docutils/highlite.nim |  2 +-
 tests/parser/tunicodeidents.nim    | 11 +++++++++++
 3 files changed, 13 insertions(+), 2 deletions(-)
 create mode 100644 tests/parser/tunicodeidents.nim

diff --git a/compiler/lexer.nim b/compiler/lexer.nim
index 5915619877..d498cf4afa 100644
--- a/compiler/lexer.nim
+++ b/compiler/lexer.nim
@@ -25,7 +25,7 @@ const
   SymChars*: set[char] = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF'}
   SymStartChars*: set[char] = {'a'..'z', 'A'..'Z', '\x80'..'\xFF'}
   OpChars*: set[char] = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.',
-    '|', '=', '%', '&', '$', '@', '~', ':', '\x80'..'\xFF'}
+    '|', '=', '%', '&', '$', '@', '~', ':'}
 
 # don't forget to update the 'highlite' module if these charsets should change
 
diff --git a/lib/packages/docutils/highlite.nim b/lib/packages/docutils/highlite.nim
index 4f1264c9ed..fbd2d7ecac 100644
--- a/lib/packages/docutils/highlite.nim
+++ b/lib/packages/docutils/highlite.nim
@@ -130,7 +130,7 @@ proc nimNumber(g: var GeneralTokenizer, position: int): int =
 
 const
   OpChars  = {'+', '-', '*', '/', '\\', '<', '>', '!', '?', '^', '.',
-              '|', '=', '%', '&', '$', '@', '~', ':', '\x80'..'\xFF'}
+              '|', '=', '%', '&', '$', '@', '~', ':'}
 
 proc nimNextToken(g: var GeneralTokenizer) =
   const
diff --git a/tests/parser/tunicodeidents.nim b/tests/parser/tunicodeidents.nim
new file mode 100644
index 0000000000..3347eb7a95
--- /dev/null
+++ b/tests/parser/tunicodeidents.nim
@@ -0,0 +1,11 @@
+discard """
+    action: run
+"""
+
+# #7884
+
+type Obj = object
+    ö: int
+
+let o = Obj(ö: 1)
+doAssert o.ö == 1

From 59ba1e77afeddc172dbc09edc752c9725c8cfdf5 Mon Sep 17 00:00:00 2001
From: WhiteDuke <Wh1teDuke@users.noreply.github.com>
Date: Mon, 4 Jun 2018 13:31:22 +0200
Subject: [PATCH 18/51] Wait until the end to print hint Conf (#7931)

---
 compiler/nimconf.nim | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/compiler/nimconf.nim b/compiler/nimconf.nim
index 6cb5bab0fb..a455b4a442 100644
--- a/compiler/nimconf.nim
+++ b/compiler/nimconf.nim
@@ -201,7 +201,8 @@ proc parseAssignment(L: var TLexer, tok: var TToken;
   else:
     processSwitch(s, val, passPP, info, config)
 
-proc readConfigFile(filename: string; cache: IdentCache; config: ConfigRef) =
+proc readConfigFile(
+    filename: string; cache: IdentCache; config: ConfigRef): bool =
   var
     L: TLexer
     tok: TToken
@@ -216,7 +217,7 @@ proc readConfigFile(filename: string; cache: IdentCache; config: ConfigRef) =
     while tok.tokType != tkEof: parseAssignment(L, tok, config, condStack)
     if len(condStack) > 0: lexMessage(L, errGenerated, "expected @end")
     closeLexer(L)
-    rawMessage(config, hintConf, filename)
+    return true
 
 proc getUserConfigPath(filename: string): string =
   result = joinPath(getConfigDir(), filename)
@@ -232,27 +233,37 @@ proc getSystemConfigPath(conf: ConfigRef; filename: string): string =
 
 proc loadConfigs*(cfg: string; cache: IdentCache; conf: ConfigRef) =
   setDefaultLibpath(conf)
+  
+  var configFiles = newSeq[string]()
+
+  template readConfigFile(path: string) =
+    let configPath = path
+    if readConfigFile(configPath, cache, conf):
+      add(configFiles, configPath)
 
   if optSkipConfigFile notin conf.globalOptions:
-    readConfigFile(getSystemConfigPath(conf, cfg), cache, conf)
+    readConfigFile(getSystemConfigPath(conf, cfg))
 
   if optSkipUserConfigFile notin conf.globalOptions:
-    readConfigFile(getUserConfigPath(cfg), cache, conf)
+    readConfigFile(getUserConfigPath(cfg))
 
   let pd = if conf.projectPath.len > 0: conf.projectPath else: getCurrentDir()
   if optSkipParentConfigFiles notin conf.globalOptions:
     for dir in parentDirs(pd, fromRoot=true, inclusive=false):
-      readConfigFile(dir / cfg, cache, conf)
+      readConfigFile(dir / cfg)
 
   if optSkipProjConfigFile notin conf.globalOptions:
-    readConfigFile(pd / cfg, cache, conf)
+    readConfigFile(pd / cfg)
 
     if conf.projectName.len != 0:
       # new project wide config file:
       var projectConfig = changeFileExt(conf.projectFull, "nimcfg")
       if not fileExists(projectConfig):
         projectConfig = changeFileExt(conf.projectFull, "nim.cfg")
-      readConfigFile(projectConfig, cache, conf)
+      readConfigFile(projectConfig)
+
+  for filename in configFiles:
+    rawMessage(conf, hintConf, filename)
 
 proc loadConfigs*(cfg: string; conf: ConfigRef) =
   # for backwards compatibility only.

From 440212a154ba26a633fa1360ed1f7bb29b274026 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= <oscarnihlgard@gmail.com>
Date: Mon, 4 Jun 2018 13:38:26 +0200
Subject: [PATCH 19/51] Fix for newStringOfCap in VM (#7901)

---
 compiler/vmgen.nim   | 3 ++-
 tests/vm/tvmmisc.nim | 8 ++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/compiler/vmgen.nim b/compiler/vmgen.nim
index 7ac3b5cf75..8a3c7e2e6e 100644
--- a/compiler/vmgen.nim
+++ b/compiler/vmgen.nim
@@ -844,7 +844,8 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
   of mNewStringOfCap:
     # we ignore the 'cap' argument and translate it as 'newString(0)'.
     # eval n.sons[1] for possible side effects:
-    var tmp = c.genx(n.sons[1])
+    c.freeTemp(c.genx(n.sons[1]))
+    var tmp = c.getTemp(n.sons[1].typ)
     c.gABx(n, opcLdImmInt, tmp, 0)
     if dest < 0: dest = c.getTemp(n.typ)
     c.gABC(n, opcNewStr, dest, tmp)
diff --git a/tests/vm/tvmmisc.nim b/tests/vm/tvmmisc.nim
index 472660bc22..4af824cf47 100644
--- a/tests/vm/tvmmisc.nim
+++ b/tests/vm/tvmmisc.nim
@@ -82,3 +82,11 @@ block:
 
     assert fileExists("MISSINGFILE") == false
     assert dirExists("MISSINGDIR") == false
+
+# #7210
+block:
+  static:
+    proc f(size: int): int =
+      var some = newStringOfCap(size)
+      result = size
+    doAssert f(4) == 4
\ No newline at end of file

From 05b447374bb6c8d2d09cee46e4f1fd68f5a8067f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= <oscarnihlgard@gmail.com>
Date: Mon, 4 Jun 2018 14:56:56 +0200
Subject: [PATCH 20/51] Use higher time resolution when available in os.nim
 (#7709)

---
 lib/posix/posix.nim       | 22 ++++++++++++++++++++--
 lib/posix/posix_other.nim | 10 +++++-----
 lib/pure/os.nim           | 32 ++++++++++++++++++++------------
 tests/stdlib/tos.nim      | 10 ++++++++++
 4 files changed, 55 insertions(+), 19 deletions(-)

diff --git a/lib/posix/posix.nim b/lib/posix/posix.nim
index 3ff156bdf0..db5f575afb 100644
--- a/lib/posix/posix.nim
+++ b/lib/posix/posix.nim
@@ -82,6 +82,14 @@ const
 # Special types
 type Sighandler = proc (a: cint) {.noconv.}
 
+const StatHasNanoseconds* = defined(linux) or defined(freebsd) or
+    defined(openbsd) or defined(dragonfly) ## \
+  ## Boolean flag that indicates if the system supports nanosecond time
+  ## resolution in the fields of ``Stat``. Note that the nanosecond based fields
+  ## (``Stat.st_atim``, ``Stat.st_mtim`` and ``Stat.st_ctim``) can be accessed
+  ## without checking this flag, because this module defines fallback procs
+  ## when they are not available.
+
 # Platform specific stuff
 
 when defined(linux) and defined(amd64):
@@ -92,9 +100,9 @@ else:
 # There used to be this name in posix.nim a long time ago, not sure why!
 {.deprecated: [cSIG_HOLD: SIG_HOLD].}
 
-when not defined(macosx) and not defined(android):
+when StatHasNanoseconds:
   proc st_atime*(s: Stat): Time {.inline.} =
-    ## Second-granularity time of last access
+    ## Second-granularity time of last access.
     result = s.st_atim.tv_sec
   proc st_mtime*(s: Stat): Time {.inline.} =
     ## Second-granularity time of last data modification.
@@ -102,6 +110,16 @@ when not defined(macosx) and not defined(android):
   proc st_ctime*(s: Stat): Time {.inline.} =
     ## Second-granularity time of last status change.
     result = s.st_ctim.tv_sec
+else:
+  proc st_atim*(s: Stat): TimeSpec {.inline.} =
+    ## Nanosecond-granularity time of last access.
+    result.tv_sec = s.st_atime
+  proc st_mtim*(s: Stat): TimeSpec {.inline.} =
+    ## Nanosecond-granularity time of last data modification.
+    result.tv_sec = s.st_mtime
+  proc st_ctim*(s: Stat): TimeSpec {.inline.} =
+    ## Nanosecond-granularity time of last data modification.
+    result.tv_sec = s.st_ctime
 
 when hasAioH:
   proc aio_cancel*(a1: cint, a2: ptr Taiocb): cint {.importc, header: "<aio.h>".}
diff --git a/lib/posix/posix_other.nim b/lib/posix/posix_other.nim
index 004a4205b3..b7570bd159 100644
--- a/lib/posix/posix_other.nim
+++ b/lib/posix/posix_other.nim
@@ -215,14 +215,14 @@ type
                           ## For a typed memory object, the length in bytes.
                           ## For other file types, the use of this field is
                           ## unspecified.
-    when defined(macosx) or defined(android):
-      st_atime*: Time     ## Time of last access.
-      st_mtime*: Time     ## Time of last data modification.
-      st_ctime*: Time     ## Time of last status change.
-    else:
+    when StatHasNanoseconds:
       st_atim*: Timespec  ## Time of last access.
       st_mtim*: Timespec  ## Time of last data modification.
       st_ctim*: Timespec  ## Time of last status change.
+    else:
+      st_atime*: Time     ## Time of last access.
+      st_mtime*: Time     ## Time of last data modification.
+      st_ctime*: Time     ## Time of last status change.
     st_blksize*: Blksize  ## A file system-specific preferred I/O block size
                           ## for this object. In some file system types, this
                           ## may vary from file to file.
diff --git a/lib/pure/os.nim b/lib/pure/os.nim
index 3ff608cfc0..04afb1eff6 100644
--- a/lib/pure/os.nim
+++ b/lib/pure/os.nim
@@ -23,6 +23,10 @@ when defined(windows):
   import winlean
 elif defined(posix):
   import posix
+
+  proc toTime(ts: Timespec): times.Time {.inline.} =
+    result = initTime(ts.tv_sec.int64, ts.tv_nsec.int)
+
 else:
   {.error: "OS module not ported to your operating system!".}
 
@@ -186,7 +190,7 @@ proc getLastModificationTime*(file: string): times.Time {.rtl, extern: "nos$1".}
   when defined(posix):
     var res: Stat
     if stat(file, res) < 0'i32: raiseOSError(osLastError())
-    return fromUnix(res.st_mtime.int64)
+    result = res.st_mtim.toTime
   else:
     var f: WIN32_FIND_DATA
     var h = findFirstFile(file, f)
@@ -199,7 +203,7 @@ proc getLastAccessTime*(file: string): times.Time {.rtl, extern: "nos$1".} =
   when defined(posix):
     var res: Stat
     if stat(file, res) < 0'i32: raiseOSError(osLastError())
-    return fromUnix(res.st_atime.int64)
+    result = res.st_atim.toTime
   else:
     var f: WIN32_FIND_DATA
     var h = findFirstFile(file, f)
@@ -216,7 +220,7 @@ proc getCreationTime*(file: string): times.Time {.rtl, extern: "nos$1".} =
   when defined(posix):
     var res: Stat
     if stat(file, res) < 0'i32: raiseOSError(osLastError())
-    return fromUnix(res.st_ctime.int64)
+    result = res.st_ctim.toTime
   else:
     var f: WIN32_FIND_DATA
     var h = findFirstFile(file, f)
@@ -228,10 +232,13 @@ proc fileNewer*(a, b: string): bool {.rtl, extern: "nos$1".} =
   ## Returns true if the file `a` is newer than file `b`, i.e. if `a`'s
   ## modification time is later than `b`'s.
   when defined(posix):
-    result = getLastModificationTime(a) - getLastModificationTime(b) >= DurationZero
-    # Posix's resolution sucks so, we use '>=' for posix.
+    # If we don't have access to nanosecond resolution, use '>='
+    when not StatHasNanoseconds:  
+      result = getLastModificationTime(a) >= getLastModificationTime(b)
+    else:
+      result = getLastModificationTime(a) > getLastModificationTime(b)
   else:
-    result = getLastModificationTime(a) - getLastModificationTime(b) > DurationZero
+    result = getLastModificationTime(a) > getLastModificationTime(b)
 
 proc getCurrentDir*(): string {.rtl, extern: "nos$1", tags: [].} =
   ## Returns the `current working directory`:idx:.
@@ -1494,7 +1501,7 @@ type
 
 template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped =
   ## Transforms the native file info structure into the one nim uses.
-  ## 'rawInfo' is either a 'TBY_HANDLE_FILE_INFORMATION' structure on Windows,
+  ## 'rawInfo' is either a 'BY_HANDLE_FILE_INFORMATION' structure on Windows,
   ## or a 'Stat' structure on posix
   when defined(Windows):
     template merge(a, b): untyped = a or (b shl 32)
@@ -1520,7 +1527,6 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped =
     if (rawInfo.dwFileAttributes and FILE_ATTRIBUTE_REPARSE_POINT) != 0'i32:
       formalInfo.kind = succ(result.kind)
 
-
   else:
     template checkAndIncludeMode(rawMode, formalMode: untyped) =
       if (rawInfo.st_mode and rawMode) != 0'i32:
@@ -1528,9 +1534,9 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped =
     formalInfo.id = (rawInfo.st_dev, rawInfo.st_ino)
     formalInfo.size = rawInfo.st_size
     formalInfo.linkCount = rawInfo.st_Nlink.BiggestInt
-    formalInfo.lastAccessTime = fromUnix(rawInfo.st_atime.int64)
-    formalInfo.lastWriteTime = fromUnix(rawInfo.st_mtime.int64)
-    formalInfo.creationTime = fromUnix(rawInfo.st_ctime.int64)
+    formalInfo.lastAccessTime = rawInfo.st_atim.toTime
+    formalInfo.lastWriteTime = rawInfo.st_mtim.toTime
+    formalInfo.creationTime = rawInfo.st_ctim.toTime
 
     result.permissions = {}
     checkAndIncludeMode(S_IRUSR, fpUserRead)
@@ -1644,7 +1650,9 @@ proc setLastModificationTime*(file: string, t: times.Time) =
   ## an error.
   when defined(posix):
     let unixt = posix.Time(t.toUnix)
-    var timevals = [Timeval(tv_sec: unixt), Timeval(tv_sec: unixt)] # [last access, last modification]
+    let micro = convert(Nanoseconds, Microseconds, t.nanosecond)
+    var timevals = [Timeval(tv_sec: unixt, tv_usec: micro),
+      Timeval(tv_sec: unixt, tv_usec: micro)] # [last access, last modification]
     if utimes(file, timevals.addr) != 0: raiseOSError(osLastError())
   else:
     let h = openHandle(path = file, writeAccess = true)
diff --git a/tests/stdlib/tos.nim b/tests/stdlib/tos.nim
index 771dc24562..e6fbb0e512 100644
--- a/tests/stdlib/tos.nim
+++ b/tests/stdlib/tos.nim
@@ -42,6 +42,7 @@ Raises
 true
 true
 true
+true
 '''
 """
 # test os path creation, iteration, and deletion
@@ -129,3 +130,12 @@ echo fileExists("../dest/a/b/file.txt")
 
 echo fileExists("../dest/a/b/c/fileC.txt")
 removeDir("../dest")
+
+# Test get/set modification times
+# Should support at least microsecond resolution
+import times
+let tm = fromUnix(0) + 100.microseconds
+writeFile("a", "")
+setLastModificationTime("a", tm)
+echo getLastModificationTime("a") == tm
+removeFile("a")
\ No newline at end of file

From 069a53ad4bff8a3160794360227eceb1fc37d8d8 Mon Sep 17 00:00:00 2001
From: andri lim <jangko128@gmail.com>
Date: Mon, 4 Jun 2018 22:43:15 +0700
Subject: [PATCH 21/51] fixes #7906, array and openarray arg vs. ptr/ref
 generic (#7909)

* fixes #7906, array and openarray arg vs. ptr/ref generic

* add comment
---
 compiler/sem.nim      |   6 +-
 compiler/types.nim    |   4 +-
 tests/array/t7818.nim | 141 ++++++++++++++++++++++++++++++++++--------
 3 files changed, 120 insertions(+), 31 deletions(-)

diff --git a/compiler/sem.nim b/compiler/sem.nim
index 55e1f47dca..d56355f14b 100644
--- a/compiler/sem.nim
+++ b/compiler/sem.nim
@@ -158,11 +158,13 @@ proc commonType*(x, y: PType): PType =
       a = a.lastSon.skipTypes({tyGenericInst})
       b = b.lastSon.skipTypes({tyGenericInst})
     if a.kind == tyObject and b.kind == tyObject:
-      result = commonSuperclass(a, b, k)
+      result = commonSuperclass(a, b)
       # this will trigger an error later:
       if result.isNil or result == a: return x
       if result == b: return y
-      if k != tyNone:
+      # bug #7906, tyRef/tyPtr + tyGenericInst of ref/ptr object ->
+      # ill-formed AST, no need for additional tyRef/tyPtr
+      if k != tyNone and x.kind != tyGenericInst:
         let r = result
         result = newType(k, r.owner)
         result.addSonSkipIntLit(r)
diff --git a/compiler/types.nim b/compiler/types.nim
index 7f9b8239fc..1fab842cc2 100644
--- a/compiler/types.nim
+++ b/compiler/types.nim
@@ -1043,7 +1043,7 @@ proc inheritanceDiff*(a, b: PType): int =
     inc(result)
   result = high(int)
 
-proc commonSuperclass*(a, b: PType, k: TTypeKind): PType =
+proc commonSuperclass*(a, b: PType): PType =
   # quick check: are they the same?
   if sameObjectTypes(a, b): return a
 
@@ -1063,7 +1063,7 @@ proc commonSuperclass*(a, b: PType, k: TTypeKind): PType =
     y = skipTypes(y, skipPtrs)
     if ancestors.contains(y.id):
       # bug #7818, defer the previous skipTypes
-      if k in {tyRef, tyPtr}: t = y
+      if t.kind != tyGenericInst: t = y
       return t
     y = y.sons[0]
 
diff --git a/tests/array/t7818.nim b/tests/array/t7818.nim
index 5d73efec59..4e43bff85a 100644
--- a/tests/array/t7818.nim
+++ b/tests/array/t7818.nim
@@ -1,24 +1,5 @@
 discard """
-  msg: '''BracketExpr
-  Sym "array"
-  Infix
-    Ident ".."
-    IntLit 0
-    IntLit 2
-  BracketExpr
-    Sym "Vehicle"
-    Sym "int"
----------
-BracketExpr
-  Sym "array"
-  Infix
-    Ident ".."
-    IntLit 0
-    IntLit 2
-  BracketExpr
-    Sym "Vehicle"
-    Sym "int"
----------'''
+  output: "OK"
 """
 
 # bug #7818
@@ -34,12 +15,118 @@ type
   Bike[T] = object of Vehicle[T]
 
 macro peek(n: typed): untyped =
-  echo getTypeImpl(n).treeRepr
-  echo "---------"
+  let val = getTypeImpl(n).treeRepr
+  newLit(val)
+
+block test_t7818:
+  var v = Vehicle[int](tire: 3)
+  var c = Car[int](tire: 4)
+  var b = Bike[int](tire: 2)
+
+  let y = peek([c, b, v])
+  let z = peek([v, c, b])
+  doAssert(y == z)
+  
+block test_t7906_1:
+  proc init(x: typedesc, y: int): ref x =
+    result = new(ref x)
+    result.tire = y
+  
+  var v = init(Vehicle[int], 3)
+  var c = init(Car[int], 4)
+  var b = init(Bike[int], 2)
+
+  let y = peek([c, b, v])
+  let z = peek([v, c, b])
+  doAssert(y == z)
+  
+block test_t7906_2:
+  var v = Vehicle[int](tire: 3)
+  var c = Car[int](tire: 4)
+  var b = Bike[int](tire: 2)
+
+  let y = peek([c.addr, b.addr, v.addr])
+  let z = peek([v.addr, c.addr, b.addr])
+  doAssert(y == z)
+
+block test_t7906_3:
+  type
+    Animal[T] = object of RootObj
+      hair: T
+    Mammal[T] = object of Animal[T]
+    Monkey[T] = object of Mammal[T]
+
+  var v = Animal[int](hair: 3)
+  var c = Mammal[int](hair: 4)
+  var b = Monkey[int](hair: 2)
+
+  let z = peek([c.addr, b.addr, v.addr])
+  let y = peek([v.addr, c.addr, b.addr])
+  doAssert(y == z)
+  
+type
+  Fruit[T] = ref object of RootObj
+    color: T
+  Apple[T] = ref object of Fruit[T]
+  Banana[T] = ref object of Fruit[T]
+    
+proc testArray[T](x: array[3, Fruit[T]]): string =
+  result = ""
+  for c in x:
+    result.add $c.color
+
+proc testOpenArray[T](x: openArray[Fruit[T]]): string =
+  result = ""
+  for c in x:
+    result.add $c.color
+    
+block test_t7906_4:
+  var v = Fruit[int](color: 3)
+  var c = Apple[int](color: 4)
+  var b = Banana[int](color: 2)
+
+  let y = peek([c, b, v])
+  let z = peek([v, c, b])
+  doAssert(y == z)
+  
+block test_t7906_5:
+  var a = Fruit[int](color: 1)
+  var b = Apple[int](color: 2)
+  var c = Banana[int](color: 3)
+
+  doAssert(testArray([a, b, c]) == "123")
+  doAssert(testArray([b, c, a]) == "231")
+
+  doAssert(testOpenArray([a, b, c]) == "123")
+  doAssert(testOpenArray([b, c, a]) == "231")
+
+  doAssert(testOpenArray(@[a, b, c]) == "123")
+  doAssert(testOpenArray(@[b, c, a]) == "231")
+
+proc testArray[T](x: array[3, ptr Vehicle[T]]): string =
+  result = ""
+  for c in x:
+    result.add $c.tire
+
+proc testOpenArray[T](x: openArray[ptr Vehicle[T]]): string =
+  result = ""
+  for c in x:
+    result.add $c.tire
+
+block test_t7906_6:
+  var u = Vehicle[int](tire: 1)
+  var v = Bike[int](tire: 2)
+  var w = Car[int](tire: 3)
+  
+  doAssert(testArray([u.addr, v.addr, w.addr]) == "123")
+  doAssert(testArray([w.addr, u.addr, v.addr]) == "312")
+  
+  doAssert(testOpenArray([u.addr, v.addr, w.addr]) == "123")
+  doAssert(testOpenArray([w.addr, u.addr, v.addr]) == "312")
+  
+  doAssert(testOpenArray(@[u.addr, v.addr, w.addr]) == "123")
+  doAssert(testOpenArray(@[w.addr, u.addr, v.addr]) == "312")
+  
+echo "OK"
 
-var v = Vehicle[int](tire: 3)
-var c = Car[int](tire: 4)
-var b = Bike[int](tire: 2)
 
-peek([c, b, v])
-peek([v, c, b])

From a3e5242d31fa2ac86072534a7528468c7a6f257d Mon Sep 17 00:00:00 2001
From: Koki Fushimi <paalon1936@gmail.com>
Date: Tue, 5 Jun 2018 07:24:34 +0900
Subject: [PATCH 22/51] Add product proc (#7951)

* Add product proc

* Update changelog
---
 changelog.md      |  1 +
 lib/pure/math.nim | 14 +++++++++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/changelog.md b/changelog.md
index 514bfbaca1..b9d634c21b 100644
--- a/changelog.md
+++ b/changelog.md
@@ -65,6 +65,7 @@
 - Added cotangent, secant and cosecant procs ``math.cot``, ``math.sec`` and ``math.csc``; and their hyperbolic, inverse and inverse hyperbolic functions, ``math.coth``, ``math.sech``, ``math.csch``, ``math.arccot``, ``math.arcsec``, ``math.arccsc``, ``math.arccoth``, ``math.arcsech`` and ``math.arccsch`` procs.
 - Added the procs ``math.floorMod`` and ``math.floorDiv`` for floor based integer division.
 - Added the procs ``rationals.`div```, ``rationals.`mod```, ``rationals.floorDiv`` and ``rationals.floorMod`` for rationals.
+- Added the proc ``math.prod`` for product of elements in openArray.
 
 ### Library changes
 
diff --git a/lib/pure/math.nim b/lib/pure/math.nim
index 6be19a3395..6658b6307c 100644
--- a/lib/pure/math.nim
+++ b/lib/pure/math.nim
@@ -129,6 +129,12 @@ proc sum*[T](x: openArray[T]): T {.noSideEffect.} =
   ## If `x` is empty, 0 is returned.
   for i in items(x): result = result + i
 
+proc prod*[T](x: openArray[T]): T {.noSideEffect.} =
+  ## Computes the product of the elements in ``x``.
+  ## If ``x`` is empty, 1 is returned.
+  result = 1.T
+  for i in items(x): result = result * i
+
 {.push noSideEffect.}
 when not defined(JS): # C
   proc sqrt*(x: float32): float32 {.importc: "sqrtf", header: "<math.h>".}
@@ -372,7 +378,7 @@ when not defined(JS): # C
 
   proc `mod`*(x, y: float32): float32 {.importc: "fmodf", header: "<math.h>".}
   proc `mod`*(x, y: float64): float64 {.importc: "fmod", header: "<math.h>".}
-    ## Computes the modulo operation for float operators. 
+    ## Computes the modulo operation for float operators.
 else: # JS
   proc hypot*[T: float32|float64](x, y: T): T = return sqrt(x*x + y*y)
   proc pow*(x, y: float32): float32 {.importC: "Math.pow", nodecl.}
@@ -560,6 +566,12 @@ when isMainModule:
   # Function for approximate comparison of floats
   proc `==~`(x, y: float): bool = (abs(x-y) < 1e-9)
 
+  block: # prod
+    doAssert prod([1, 2, 3, 4]) == 24
+    doAssert prod([1.5, 3.4]) == 5.1
+    let x: seq[float] = @[]
+    doAssert prod(x) == 1.0
+
   block: # round() tests
     # Round to 0 decimal places
     doAssert round(54.652) ==~ 55.0

From fd102f39bb8c969d33015654422ff4541f211b51 Mon Sep 17 00:00:00 2001
From: skilchen <skilchen@users.noreply.github.com>
Date: Tue, 5 Jun 2018 00:26:16 +0200
Subject: [PATCH 23/51] Fix strformat precision handling for strings (#7941)

* fix strformat precision handling for strings

* add some limited unicode awareness to the precision handling for strings

* improvement suggested by Varriount: use setLen and runeOffset instead of runeSubstr
---
 lib/pure/strformat.nim      |  4 ++++
 tests/stdlib/tstrformat.nim | 26 ++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim
index 12a102c9f6..3e7b043ce3 100644
--- a/lib/pure/strformat.nim
+++ b/lib/pure/strformat.nim
@@ -558,12 +558,16 @@ proc format*(value: string; specifier: string; res: var string) =
   ## sense to call this directly, but it is required to exist
   ## by the ``&`` macro.
   let spec = parseStandardFormatSpecifier(specifier)
+  var value = value
   case spec.typ
   of 's', '\0': discard
   else:
     raise newException(ValueError,
       "invalid type in format string for string, expected 's', but got " &
       spec.typ)
+  if spec.precision != -1:
+    if spec.precision < runelen(value):
+      setLen(value, runeOffset(value, spec.precision))
   res.add alignString(value, spec.minimumWidth, spec.align, spec.fill)
 
 when isMainModule:
diff --git a/tests/stdlib/tstrformat.nim b/tests/stdlib/tstrformat.nim
index b4cbd41e05..919158ac40 100644
--- a/tests/stdlib/tstrformat.nim
+++ b/tests/stdlib/tstrformat.nim
@@ -12,6 +12,32 @@ var o: Obj
 doAssert fmt"{o}" == "foobar"
 doAssert fmt"{o:10}" == "foobar    "
 
+# see issue #7933
+var str = "abc"
+doAssert fmt">7.1 :: {str:>7.1}" == ">7.1 ::       a"
+doAssert fmt">7.2 :: {str:>7.2}" == ">7.2 ::      ab"
+doAssert fmt">7.3 :: {str:>7.3}" == ">7.3 ::     abc"
+doAssert fmt">7.9 :: {str:>7.9}" == ">7.9 ::     abc"
+doAssert fmt">7.0 :: {str:>7.0}" == ">7.0 ::        "
+doAssert fmt" 7.1 :: {str:7.1}" == " 7.1 :: a      "
+doAssert fmt" 7.2 :: {str:7.2}" == " 7.2 :: ab     "
+doAssert fmt" 7.3 :: {str:7.3}" == " 7.3 :: abc    "
+doAssert fmt" 7.9 :: {str:7.9}" == " 7.9 :: abc    "
+doAssert fmt" 7.0 :: {str:7.0}" == " 7.0 ::        "
+doAssert fmt"^7.1 :: {str:^7.1}" == "^7.1 ::    a   "
+doAssert fmt"^7.2 :: {str:^7.2}" == "^7.2 ::   ab   "
+doAssert fmt"^7.3 :: {str:^7.3}" == "^7.3 ::   abc  "
+doAssert fmt"^7.9 :: {str:^7.9}" == "^7.9 ::   abc  "
+doAssert fmt"^7.0 :: {str:^7.0}" == "^7.0 ::        "
+str = "äöüe\u0309\u0319o\u0307\u0359"
+doAssert fmt"^7.1 :: {str:^7.1}" == "^7.1 ::    ä   "
+doAssert fmt"^7.2 :: {str:^7.2}" == "^7.2 ::   äö   "
+doAssert fmt"^7.3 :: {str:^7.3}" == "^7.3 ::   äöü  "
+doAssert fmt"^7.0 :: {str:^7.0}" == "^7.0 ::        "
+# this is actually wrong, but the unicode module has no support for graphemes
+doAssert fmt"^7.4 :: {str:^7.4}" == "^7.4 ::  äöüe  "
+doAssert fmt"^7.9 :: {str:^7.9}" == "^7.9 :: äöüe\u0309\u0319o\u0307\u0359"
+
 # see issue #7932
 doAssert fmt"{15:08}" == "00000015" # int, works
 doAssert fmt"{1.5:08}" == "000001.5" # float, works

From 80107b360c8bb01efba39a9a4a0b9129b300a248 Mon Sep 17 00:00:00 2001
From: andri lim <jangko128@gmail.com>
Date: Tue, 5 Jun 2018 09:18:20 +0700
Subject: [PATCH 24/51] add more test to 4799

---
 compiler/ccgexprs.nim     |   4 +-
 compiler/sigmatch.nim     |   8 ++
 tests/typerel/t4799.nim   | 169 ++++++++++++++++++++++++++++++++++++++
 tests/typerel/t4799_1.nim |  20 +++++
 tests/typerel/t4799_2.nim |  24 ++++++
 tests/typerel/t4799_3.nim |  24 ++++++
 tests/typerel/t4799_4.nim |  23 ++++++
 tests/typerel/t4799_5.nim |  23 ++++++
 tests/typerel/t4799_6.nim |  20 +++++
 tests/typerel/t4799_7.nim |  20 +++++
 10 files changed, 334 insertions(+), 1 deletion(-)
 create mode 100644 tests/typerel/t4799.nim
 create mode 100644 tests/typerel/t4799_1.nim
 create mode 100644 tests/typerel/t4799_2.nim
 create mode 100644 tests/typerel/t4799_3.nim
 create mode 100644 tests/typerel/t4799_4.nim
 create mode 100644 tests/typerel/t4799_5.nim
 create mode 100644 tests/typerel/t4799_6.nim
 create mode 100644 tests/typerel/t4799_7.nim

diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim
index 335aa2f84a..ad36d3e921 100644
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -351,7 +351,9 @@ proc genAssignment(p: BProc, dest, src: TLoc, flags: TAssignmentFlags) =
     else:
       useStringh(p.module)
       linefmt(p, cpsStmts,
-           "memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($1[0])*$1Len_0);$n",
+           # bug #4799, keep the memcpy for a while
+           #"memcpy((void*)$1, (NIM_CONST void*)$2, sizeof($1[0])*$1Len_0);$n",
+           "$1 = $2;$n",
            rdLoc(dest), rdLoc(src))
   of tySet:
     if mapType(ty) == ctArray:
diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim
index 41cac2a4af..fcfdda8bbe 100644
--- a/compiler/sigmatch.nim
+++ b/compiler/sigmatch.nim
@@ -2013,6 +2013,14 @@ proc paramTypesMatchAux(m: var TCandidate, f, a: PType,
           if r == isGeneric:
             result.typ = getInstantiatedType(c, arg, m, base(f))
           m.baseTypeMatch = true
+        # bug #4799, varargs accepting subtype relation object
+        elif r == isSubtype:
+          inc(m.subtypeMatches)
+          if f.kind == tyTypeDesc:
+            result = arg
+          else:
+            result = implicitConv(nkHiddenSubConv, f, arg, m, c)
+          m.baseTypeMatch = true
         else:
           result = userConvMatch(c, m, base(f), a, arg)
           if result != nil: m.baseTypeMatch = true
diff --git a/tests/typerel/t4799.nim b/tests/typerel/t4799.nim
new file mode 100644
index 0000000000..f1c283165f
--- /dev/null
+++ b/tests/typerel/t4799.nim
@@ -0,0 +1,169 @@
+discard """
+  output: "OK"
+"""
+
+type
+  GRBase[T] = ref object of RootObj
+    val: T
+  GRC[T] = ref object of GRBase[T]
+  GRD[T] = ref object of GRBase[T]
+
+proc testGR[T](x: varargs[GRBase[T]]): string =
+  result = ""
+  for c in x:
+    result.add $c.val
+
+block test_t4799_1:
+  var rgv = GRBase[int](val: 3)
+  var rgc = GRC[int](val: 4)
+  var rgb = GRD[int](val: 2)
+  doAssert(testGR(rgb, rgc, rgv) == "243")
+  doAssert(testGR(rgc, rgv, rgb) == "432")
+  doAssert(testGR(rgv, rgb, rgc) == "324")
+  doAssert(testGR([rgb, rgc, rgv]) == "243")
+  doAssert(testGR([rgc, rgv, rgb]) == "432")
+  doAssert(testGR([rgv, rgb, rgc]) == "324")
+
+type
+  PRBase[T] = object of RootObj
+    val: T
+  PRC[T] = object of PRBase[T]
+  PRD[T] = object of PRBase[T]
+
+proc testPR[T](x: varargs[ptr PRBase[T]]): string =
+  result = ""
+  for c in x:
+    result.add $c.val
+
+block test_t4799_2:
+  var pgv = PRBase[int](val: 3)
+  var pgc = PRC[int](val: 4)
+  var pgb = PRD[int](val: 2)
+  doAssert(testPR(pgb.addr, pgc.addr, pgv.addr) == "243")
+  doAssert(testPR(pgc.addr, pgv.addr, pgb.addr) == "432")
+  doAssert(testPR(pgv.addr, pgb.addr, pgc.addr) == "324")
+  doAssert(testPR([pgb.addr, pgc.addr, pgv.addr]) == "243")
+  doAssert(testPR([pgc.addr, pgv.addr, pgb.addr]) == "432")
+  doAssert(testPR([pgv.addr, pgb.addr, pgc.addr]) == "324")
+
+type
+  RBase = ref object of RootObj
+    val: int
+  RC = ref object of RBase
+  RD = ref object of RBase
+
+proc testR(x: varargs[RBase]): string =
+  result = ""
+  for c in x:
+    result.add $c.val
+
+block test_t4799_3:
+  var rv = RBase(val: 3)
+  var rc = RC(val: 4)
+  var rb = RD(val: 2)
+  doAssert(testR(rb, rc, rv) == "243")
+  doAssert(testR(rc, rv, rb) == "432")
+  doAssert(testR(rv, rb, rc) == "324")
+  doAssert(testR([rb, rc, rv]) == "243")
+  doAssert(testR([rc, rv, rb]) == "432")
+  doAssert(testR([rv, rb, rc]) == "324")
+
+type
+  PBase = object of RootObj
+    val: int
+  PC = object of PBase
+  PD = object of PBase
+
+proc testP(x: varargs[ptr PBase]): string =
+  result = ""
+  for c in x:
+    result.add $c.val
+
+block test_t4799_4:
+  var pv = PBase(val: 3)
+  var pc = PC(val: 4)
+  var pb = PD(val: 2)
+  doAssert(testP(pb.addr, pc.addr, pv.addr) == "243")
+  doAssert(testP(pc.addr, pv.addr, pb.addr) == "432")
+  doAssert(testP(pv.addr, pb.addr, pc.addr) == "324")
+  doAssert(testP([pb.addr, pc.addr, pv.addr]) == "243")
+  doAssert(testP([pc.addr, pv.addr, pb.addr]) == "432")
+  doAssert(testP([pv.addr, pb.addr, pc.addr]) == "324")
+
+type
+  PSBase[T, V] = ref object of RootObj
+    val: T
+    color: V
+  PSRC[T] = ref object of PSBase[T, int]
+  PSRD[T] = ref object of PSBase[T, int]
+
+proc testPS[T, V](x: varargs[PSBase[T, V]]): string =
+  result = ""
+  for c in x:
+    result.add c.val
+    result.add $c.color
+
+block test_t4799_5:
+  var a = PSBase[string, int](val: "base", color: 1)
+  var b = PSRC[string](val: "rc", color: 2)
+  var c = PSRD[string](val: "rd", color: 3)
+
+  doAssert(testPS(a, b, c) == "base1rc2rd3")
+  doAssert(testPS(b, a, c) == "rc2base1rd3")
+  doAssert(testPS(c, b, a) == "rd3rc2base1")
+  doAssert(testPS([a, b, c]) == "base1rc2rd3")
+  doAssert(testPS([b, a, c]) == "rc2base1rd3")
+  doAssert(testPS([c, b, a]) == "rd3rc2base1")
+
+type
+  SBase[T, V] = ref object of RootObj
+    val: T
+    color: V
+  SRC = ref object of SBase[string, int]
+  SRD = ref object of SBase[string, int]
+
+proc testS[T, V](x: varargs[SBase[T, V]]): string =
+  result = ""
+  for c in x:
+    result.add c.val
+    result.add $c.color
+
+block test_t4799_6:
+  var a = SBase[string, int](val: "base", color: 1)
+  var b = SRC(val: "rc", color: 2)
+  var c = SRD(val: "rd", color: 3)
+
+  doAssert(testS(a, b, c) == "base1rc2rd3")
+  doAssert(testS(b, a, c) == "rc2base1rd3")
+  doAssert(testS(c, b, a) == "rd3rc2base1")
+  doAssert(testS([a, b, c]) == "base1rc2rd3")
+  # this is not varargs bug, but array construction bug
+  # see #7955
+  #doAssert(testS([b, c, a]) == "rc2rd3base1")
+  #doAssert(testS([c, b, a]) == "rd3rc2base1")
+
+proc test_inproc() =  
+  block test_inproc_1:
+    var rgv = GRBase[int](val: 3)
+    var rgc = GRC[int](val: 4)
+    var rgb = GRD[int](val: 2)
+    doAssert(testGR(rgb, rgc, rgv) == "243")
+    doAssert(testGR(rgc, rgv, rgb) == "432")
+    doAssert(testGR(rgv, rgb, rgc) == "324")
+    doAssert(testGR([rgb, rgc, rgv]) == "243")
+    doAssert(testGR([rgc, rgv, rgb]) == "432")
+    doAssert(testGR([rgv, rgb, rgc]) == "324")
+  
+  block test_inproc_2:
+    var pgv = PRBase[int](val: 3)
+    var pgc = PRC[int](val: 4)
+    var pgb = PRD[int](val: 2)
+    doAssert(testPR(pgb.addr, pgc.addr, pgv.addr) == "243")
+    doAssert(testPR(pgc.addr, pgv.addr, pgb.addr) == "432")
+    doAssert(testPR(pgv.addr, pgb.addr, pgc.addr) == "324")
+    doAssert(testPR([pgb.addr, pgc.addr, pgv.addr]) == "243")
+    doAssert(testPR([pgc.addr, pgv.addr, pgb.addr]) == "432")
+    doAssert(testPR([pgv.addr, pgb.addr, pgc.addr]) == "324")
+  
+test_inproc()
+echo "OK"
diff --git a/tests/typerel/t4799_1.nim b/tests/typerel/t4799_1.nim
new file mode 100644
index 0000000000..549b6bf3c5
--- /dev/null
+++ b/tests/typerel/t4799_1.nim
@@ -0,0 +1,20 @@
+discard """
+  outputsub: '''ObjectAssignmentError'''
+  exitcode: "1"
+"""
+
+type
+  Vehicle[T] = object of RootObj
+    tire: T
+  Car[T] = object of Vehicle[T]
+  Bike[T] = object of Vehicle[T]
+
+proc testVehicle[T](x: varargs[Vehicle[T]]): string =
+  result = ""
+  for c in x:
+    result.add $c.tire
+
+var v = Vehicle[int](tire: 3)
+var c = Car[int](tire: 4)
+var b = Bike[int](tire: 2)
+echo testVehicle b, c, v
diff --git a/tests/typerel/t4799_2.nim b/tests/typerel/t4799_2.nim
new file mode 100644
index 0000000000..191a40469e
--- /dev/null
+++ b/tests/typerel/t4799_2.nim
@@ -0,0 +1,24 @@
+discard """
+errormsg: "type mismatch: got <Bike[system.int]>"
+nimout: '''t4799_2.nim(24, 18) Error: type mismatch: got <Bike[system.int]>
+but expected one of:
+proc testVehicle[T](x: varargs[Vehicle[T]]): string
+
+expression: testVehicle b'''
+"""
+
+type
+  Vehicle[T] = ref object of RootObj
+    tire: T
+  Car[T] = object of Vehicle[T]
+  Bike[T] = object of Vehicle[T]
+
+proc testVehicle[T](x: varargs[Vehicle[T]]): string =
+  result = ""
+  for c in x:
+    result.add $c.tire
+
+var v = Vehicle[int](tire: 3)
+var c = Car[int](tire: 4)
+var b = Bike[int](tire: 2)
+echo testVehicle b, c, v
diff --git a/tests/typerel/t4799_3.nim b/tests/typerel/t4799_3.nim
new file mode 100644
index 0000000000..a447da6b24
--- /dev/null
+++ b/tests/typerel/t4799_3.nim
@@ -0,0 +1,24 @@
+discard """
+errormsg: "Error: type mismatch: got <Bike>"
+nimout: '''t4799_3.nim(24, 18) Error: type mismatch: got <Bike>
+but expected one of:
+proc testVehicle(x: varargs[Vehicle]): string
+
+expression: testVehicle b'''
+"""
+
+type
+  Vehicle = ref object of RootObj
+    tire: int
+  Car = object of Vehicle
+  Bike = object of Vehicle
+
+proc testVehicle(x: varargs[Vehicle]): string =
+  result = ""
+  for c in x:
+    result.add $c.tire
+
+var v = Vehicle(tire: 3)
+var c = Car(tire: 4)
+var b = Bike(tire: 2)
+echo testVehicle b, c, v
\ No newline at end of file
diff --git a/tests/typerel/t4799_4.nim b/tests/typerel/t4799_4.nim
new file mode 100644
index 0000000000..52c45793c8
--- /dev/null
+++ b/tests/typerel/t4799_4.nim
@@ -0,0 +1,23 @@
+discard """
+errormsg: "type mismatch: got <Bike[system.int]>"
+nimout: '''t4799_4.nim(23, 18) Error: type mismatch: got <Bike[system.int]>
+but expected one of:
+proc testVehicle[T](x: varargs[Vehicle[T]]): string
+
+expression: testVehicle b'''
+"""
+
+type
+  Vehicle[T] = ptr object of RootObj
+    tire: T
+  Car[T] = object of Vehicle[T]
+  Bike[T] = object of Vehicle[T]
+
+proc testVehicle[T](x: varargs[Vehicle[T]]): string =
+  result = ""
+  for c in x:
+    result.add $c.tire
+
+var c = Car[int](tire: 4)
+var b = Bike[int](tire: 2)
+echo testVehicle b, c
\ No newline at end of file
diff --git a/tests/typerel/t4799_5.nim b/tests/typerel/t4799_5.nim
new file mode 100644
index 0000000000..8c7fdc3135
--- /dev/null
+++ b/tests/typerel/t4799_5.nim
@@ -0,0 +1,23 @@
+discard """
+errormsg: "type mismatch: got <Bike>"
+nimout: '''t4799_5.nim(23, 18) Error: type mismatch: got <Bike>
+but expected one of:
+proc testVehicle(x: varargs[Vehicle]): string
+
+expression: testVehicle b'''
+"""
+
+type
+  Vehicle = ptr object of RootObj
+    tire: int
+  Car = object of Vehicle
+  Bike = object of Vehicle
+
+proc testVehicle(x: varargs[Vehicle]): string =
+  result = ""
+  for c in x:
+    result.add $c.tire
+
+var c = Car(tire: 4)
+var b = Bike(tire: 2)
+echo testVehicle b, c
\ No newline at end of file
diff --git a/tests/typerel/t4799_6.nim b/tests/typerel/t4799_6.nim
new file mode 100644
index 0000000000..cfd399a6e0
--- /dev/null
+++ b/tests/typerel/t4799_6.nim
@@ -0,0 +1,20 @@
+discard """
+  outputsub: '''ObjectAssignmentError'''
+  exitcode: "1"
+"""
+
+type
+  Vehicle[T] = object of RootObj
+    tire: T
+  Car[T] = object of Vehicle[T]
+  Bike[T] = object of Vehicle[T]
+
+proc testVehicle[T](x: varargs[Vehicle[T]]): string =
+  result = ""
+  for c in x:
+    result.add $c.tire
+
+var v = Vehicle[int](tire: 3)
+var c = Car[int](tire: 4)
+var b = Bike[int](tire: 2)
+echo testVehicle([b, c, v])
\ No newline at end of file
diff --git a/tests/typerel/t4799_7.nim b/tests/typerel/t4799_7.nim
new file mode 100644
index 0000000000..784eee8fc1
--- /dev/null
+++ b/tests/typerel/t4799_7.nim
@@ -0,0 +1,20 @@
+discard """
+  outputsub: '''ObjectAssignmentError'''
+  exitcode: "1"
+"""
+
+type
+  Vehicle = object of RootObj
+    tire: int
+  Car = object of Vehicle
+  Bike = object of Vehicle
+
+proc testVehicle(x: varargs[Vehicle]): string =
+  result = ""
+  for c in x:
+    result.add $c.tire
+
+var v = Vehicle(tire: 3)
+var c = Car(tire: 4)
+var b = Bike(tire: 2)
+echo testVehicle([b, c, v])
\ No newline at end of file

From 230692a22f92ed010e04ba8c1b2b95f86350f1a5 Mon Sep 17 00:00:00 2001
From: skilchen <skilchen@users.noreply.github.com>
Date: Tue, 5 Jun 2018 06:09:07 +0200
Subject: [PATCH 25/51] Fix strformat neg zero (#7954)

* fix strformat handling of neg zero with sign

* better tests for neg zero with sign

* use inplace insertion of the sign as suggested by Varriount
---
 lib/pure/strformat.nim      | 7 ++++++-
 tests/stdlib/tstrformat.nim | 8 ++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/lib/pure/strformat.nim b/lib/pure/strformat.nim
index 3e7b043ce3..36404cdf7c 100644
--- a/lib/pure/strformat.nim
+++ b/lib/pure/strformat.nim
@@ -527,8 +527,13 @@ proc format*(value: SomeFloat; specifier: string; res: var string) =
   var sign = false
   if value >= 0.0:
     if spec.sign != '-':
-      f = spec.sign & f
       sign = true
+      if  value == 0.0:
+        if 1.0 / value == Inf:
+          # only insert the sign if value != negZero
+          f.insert($spec.sign, 0)
+      else:
+        f.insert($spec.sign, 0)
   else:
     sign = true
 
diff --git a/tests/stdlib/tstrformat.nim b/tests/stdlib/tstrformat.nim
index 919158ac40..db76899d42 100644
--- a/tests/stdlib/tstrformat.nim
+++ b/tests/stdlib/tstrformat.nim
@@ -46,3 +46,11 @@ doAssert fmt"{-1.5:0>8}" == "0000-1.5" # even that does not work for negative fl
 doAssert fmt"{-1.5:08}" == "-00001.5" # works
 doAssert fmt"{1.5:+08}" == "+00001.5" # works
 doAssert fmt"{1.5: 08}" == " 00001.5" # works
+
+# only add explicitly requested sign if value != -0.0 (neg zero)
+doAssert fmt"{-0.0:g}" == "-0"
+doassert fmt"{-0.0:+g}" == "-0"
+doassert fmt"{-0.0: g}" == "-0"
+doAssert fmt"{0.0:g}" == "0"
+doAssert fmt"{0.0:+g}" == "+0"
+doAssert fmt"{0.0: g}" == " 0"

From 8063ecbb8fdd79f3a097789a72500a1663587bd5 Mon Sep 17 00:00:00 2001
From: andri lim <jangko128@gmail.com>
Date: Tue, 5 Jun 2018 16:54:01 +0700
Subject: [PATCH 26/51] fix test case output

---
 tests/typerel/t4799_3.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/typerel/t4799_3.nim b/tests/typerel/t4799_3.nim
index a447da6b24..aa2ddac56a 100644
--- a/tests/typerel/t4799_3.nim
+++ b/tests/typerel/t4799_3.nim
@@ -1,5 +1,5 @@
 discard """
-errormsg: "Error: type mismatch: got <Bike>"
+errormsg: "type mismatch: got <Bike>"
 nimout: '''t4799_3.nim(24, 18) Error: type mismatch: got <Bike>
 but expected one of:
 proc testVehicle(x: varargs[Vehicle]): string

From 959b6354c126159e5a59c3a021e472380d04e088 Mon Sep 17 00:00:00 2001
From: Koki Fushimi <paalon1936@gmail.com>
Date: Wed, 6 Jun 2018 00:15:04 +0900
Subject: [PATCH 27/51] Rename tgamma to gamma (#7929)

* Rename tgamma to gamma

* set the deprecating version  0.19.0

* update changelog and use description in deprecated pragma
---
 changelog.md      |  1 +
 lib/pure/math.nim | 13 ++++++++++---
 2 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/changelog.md b/changelog.md
index b9d634c21b..6fd12e62fe 100644
--- a/changelog.md
+++ b/changelog.md
@@ -87,6 +87,7 @@
 - Added the parameter ``val`` for the ``CritBitTree[int].inc`` proc.
 - An exception raised from ``test`` block of ``unittest`` now shows its type in
   the error message
+- The proc ``tgamma`` was renamed to ``gamma``. ``tgamma`` is deprecated.
 
 ### Language additions
 
diff --git a/lib/pure/math.nim b/lib/pure/math.nim
index 6658b6307c..8ea8ee2037 100644
--- a/lib/pure/math.nim
+++ b/lib/pure/math.nim
@@ -280,12 +280,18 @@ when not defined(JS): # C
   proc erfc*(x: float64): float64 {.importc: "erfc", header: "<math.h>".}
     ## The complementary error function
 
+  proc gamma*(x: float32): float32 {.importc: "tgammaf", header: "<math.h>".}
+  proc gamma*(x: float64): float64 {.importc: "tgamma", header: "<math.h>".}
+    ## The gamma function
+  proc tgamma*(x: float32): float32
+    {.deprecated: "use gamma instead", importc: "tgammaf", header: "<math.h>".}
+  proc tgamma*(x: float64): float64
+    {.deprecated: "use gamma instead", importc: "tgamma", header: "<math.h>".}
+    ## The gamma function
+    ## **Deprecated since version 0.19.0**: Use ``gamma`` instead.
   proc lgamma*(x: float32): float32 {.importc: "lgammaf", header: "<math.h>".}
   proc lgamma*(x: float64): float64 {.importc: "lgamma", header: "<math.h>".}
     ## Natural log of the gamma function
-  proc tgamma*(x: float32): float32 {.importc: "tgammaf", header: "<math.h>".}
-  proc tgamma*(x: float64): float64 {.importc: "tgamma", header: "<math.h>".}
-    ## The gamma function
 
   proc floor*(x: float32): float32 {.importc: "floorf", header: "<math.h>".}
   proc floor*(x: float64): float64 {.importc: "floor", header: "<math.h>".}
@@ -557,6 +563,7 @@ when isMainModule and not defined(JS):
     return sqrt(num)
 
   # check gamma function
+  assert(gamma(5.0) == 24.0) # 4!
   assert($tgamma(5.0) == $24.0) # 4!
   assert(lgamma(1.0) == 0.0) # ln(1.0) == 0.0
   assert(erf(6.0) > erf(5.0))

From 436c1229563c6c44c85cf9b26814b0a4a45ef3f4 Mon Sep 17 00:00:00 2001
From: andri lim <jangko128@gmail.com>
Date: Tue, 5 Jun 2018 22:16:53 +0700
Subject: [PATCH 28/51] combine/reduce test

---
 tests/typerel/t4799.nim   | 82 +++++++++++++++++++++++++++++++++++++--
 tests/typerel/t4799_2.nim | 12 ++----
 tests/typerel/t4799_3.nim | 12 ++----
 tests/typerel/t4799_4.nim | 23 -----------
 tests/typerel/t4799_5.nim | 23 -----------
 tests/typerel/t4799_6.nim | 20 ----------
 tests/typerel/t4799_7.nim | 20 ----------
 7 files changed, 87 insertions(+), 105 deletions(-)
 delete mode 100644 tests/typerel/t4799_4.nim
 delete mode 100644 tests/typerel/t4799_5.nim
 delete mode 100644 tests/typerel/t4799_6.nim
 delete mode 100644 tests/typerel/t4799_7.nim

diff --git a/tests/typerel/t4799.nim b/tests/typerel/t4799.nim
index f1c283165f..89312950fe 100644
--- a/tests/typerel/t4799.nim
+++ b/tests/typerel/t4799.nim
@@ -142,7 +142,7 @@ block test_t4799_6:
   #doAssert(testS([b, c, a]) == "rc2rd3base1")
   #doAssert(testS([c, b, a]) == "rd3rc2base1")
 
-proc test_inproc() =  
+proc test_inproc() =
   block test_inproc_1:
     var rgv = GRBase[int](val: 3)
     var rgc = GRC[int](val: 4)
@@ -153,7 +153,7 @@ proc test_inproc() =
     doAssert(testGR([rgb, rgc, rgv]) == "243")
     doAssert(testGR([rgc, rgv, rgb]) == "432")
     doAssert(testGR([rgv, rgb, rgc]) == "324")
-  
+
   block test_inproc_2:
     var pgv = PRBase[int](val: 3)
     var pgc = PRC[int](val: 4)
@@ -164,6 +164,82 @@ proc test_inproc() =
     doAssert(testPR([pgb.addr, pgc.addr, pgv.addr]) == "243")
     doAssert(testPR([pgc.addr, pgv.addr, pgb.addr]) == "432")
     doAssert(testPR([pgv.addr, pgb.addr, pgc.addr]) == "324")
-  
+
 test_inproc()
+
+template reject(x) =
+  static: assert(not compiles(x))
+
+block test_t4799_7:
+  type
+    Vehicle[T] = ref object of RootObj
+      tire: T
+    Car[T] = object of Vehicle[T]
+    Bike[T] = object of Vehicle[T]
+
+  proc testVehicle[T](x: varargs[Vehicle[T]]): string {.used.} =
+    result = ""
+    for c in x:
+      result.add $c.tire
+
+  var v = Vehicle[int](tire: 3)
+  var c = Car[int](tire: 4)
+  var b = Bike[int](tire: 2)
+
+  reject:
+    echo testVehicle b, c, v
+
+block test_t4799_8:
+  type
+    Vehicle = ref object of RootObj
+      tire: int
+    Car = object of Vehicle
+    Bike = object of Vehicle
+
+  proc testVehicle(x: varargs[Vehicle]): string {.used.} =
+    result = ""
+    for c in x:
+      result.add $c.tire
+
+  var v = Vehicle(tire: 3)
+  var c = Car(tire: 4)
+  var b = Bike(tire: 2)
+
+  reject:
+    echo testVehicle b, c, v
+
+type
+  PGVehicle[T] = ptr object of RootObj
+    tire: T
+  PGCar[T] = object of PGVehicle[T]
+  PGBike[T] = object of PGVehicle[T]
+
+proc testVehicle[T](x: varargs[PGVehicle[T]]): string {.used.} =
+  result = ""
+  for c in x:
+    result.add $c.tire
+
+var pgc = PGCar[int](tire: 4)
+var pgb = PGBike[int](tire: 2)
+
+reject:
+  echo testVehicle pgb, pgc
+
+type
+  RVehicle = ptr object of RootObj
+    tire: int
+  RCar = object of RVehicle
+  RBike = object of RVehicle
+
+proc testVehicle(x: varargs[RVehicle]): string {.used.} =
+  result = ""
+  for c in x:
+    result.add $c.tire
+
+var rc = RCar(tire: 4)
+var rb = RBike(tire: 2)
+
+reject:
+  echo testVehicle rb, rc
+
 echo "OK"
diff --git a/tests/typerel/t4799_2.nim b/tests/typerel/t4799_2.nim
index 191a40469e..cfd399a6e0 100644
--- a/tests/typerel/t4799_2.nim
+++ b/tests/typerel/t4799_2.nim
@@ -1,14 +1,10 @@
 discard """
-errormsg: "type mismatch: got <Bike[system.int]>"
-nimout: '''t4799_2.nim(24, 18) Error: type mismatch: got <Bike[system.int]>
-but expected one of:
-proc testVehicle[T](x: varargs[Vehicle[T]]): string
-
-expression: testVehicle b'''
+  outputsub: '''ObjectAssignmentError'''
+  exitcode: "1"
 """
 
 type
-  Vehicle[T] = ref object of RootObj
+  Vehicle[T] = object of RootObj
     tire: T
   Car[T] = object of Vehicle[T]
   Bike[T] = object of Vehicle[T]
@@ -21,4 +17,4 @@ proc testVehicle[T](x: varargs[Vehicle[T]]): string =
 var v = Vehicle[int](tire: 3)
 var c = Car[int](tire: 4)
 var b = Bike[int](tire: 2)
-echo testVehicle b, c, v
+echo testVehicle([b, c, v])
\ No newline at end of file
diff --git a/tests/typerel/t4799_3.nim b/tests/typerel/t4799_3.nim
index aa2ddac56a..784eee8fc1 100644
--- a/tests/typerel/t4799_3.nim
+++ b/tests/typerel/t4799_3.nim
@@ -1,14 +1,10 @@
 discard """
-errormsg: "type mismatch: got <Bike>"
-nimout: '''t4799_3.nim(24, 18) Error: type mismatch: got <Bike>
-but expected one of:
-proc testVehicle(x: varargs[Vehicle]): string
-
-expression: testVehicle b'''
+  outputsub: '''ObjectAssignmentError'''
+  exitcode: "1"
 """
 
 type
-  Vehicle = ref object of RootObj
+  Vehicle = object of RootObj
     tire: int
   Car = object of Vehicle
   Bike = object of Vehicle
@@ -21,4 +17,4 @@ proc testVehicle(x: varargs[Vehicle]): string =
 var v = Vehicle(tire: 3)
 var c = Car(tire: 4)
 var b = Bike(tire: 2)
-echo testVehicle b, c, v
\ No newline at end of file
+echo testVehicle([b, c, v])
\ No newline at end of file
diff --git a/tests/typerel/t4799_4.nim b/tests/typerel/t4799_4.nim
deleted file mode 100644
index 52c45793c8..0000000000
--- a/tests/typerel/t4799_4.nim
+++ /dev/null
@@ -1,23 +0,0 @@
-discard """
-errormsg: "type mismatch: got <Bike[system.int]>"
-nimout: '''t4799_4.nim(23, 18) Error: type mismatch: got <Bike[system.int]>
-but expected one of:
-proc testVehicle[T](x: varargs[Vehicle[T]]): string
-
-expression: testVehicle b'''
-"""
-
-type
-  Vehicle[T] = ptr object of RootObj
-    tire: T
-  Car[T] = object of Vehicle[T]
-  Bike[T] = object of Vehicle[T]
-
-proc testVehicle[T](x: varargs[Vehicle[T]]): string =
-  result = ""
-  for c in x:
-    result.add $c.tire
-
-var c = Car[int](tire: 4)
-var b = Bike[int](tire: 2)
-echo testVehicle b, c
\ No newline at end of file
diff --git a/tests/typerel/t4799_5.nim b/tests/typerel/t4799_5.nim
deleted file mode 100644
index 8c7fdc3135..0000000000
--- a/tests/typerel/t4799_5.nim
+++ /dev/null
@@ -1,23 +0,0 @@
-discard """
-errormsg: "type mismatch: got <Bike>"
-nimout: '''t4799_5.nim(23, 18) Error: type mismatch: got <Bike>
-but expected one of:
-proc testVehicle(x: varargs[Vehicle]): string
-
-expression: testVehicle b'''
-"""
-
-type
-  Vehicle = ptr object of RootObj
-    tire: int
-  Car = object of Vehicle
-  Bike = object of Vehicle
-
-proc testVehicle(x: varargs[Vehicle]): string =
-  result = ""
-  for c in x:
-    result.add $c.tire
-
-var c = Car(tire: 4)
-var b = Bike(tire: 2)
-echo testVehicle b, c
\ No newline at end of file
diff --git a/tests/typerel/t4799_6.nim b/tests/typerel/t4799_6.nim
deleted file mode 100644
index cfd399a6e0..0000000000
--- a/tests/typerel/t4799_6.nim
+++ /dev/null
@@ -1,20 +0,0 @@
-discard """
-  outputsub: '''ObjectAssignmentError'''
-  exitcode: "1"
-"""
-
-type
-  Vehicle[T] = object of RootObj
-    tire: T
-  Car[T] = object of Vehicle[T]
-  Bike[T] = object of Vehicle[T]
-
-proc testVehicle[T](x: varargs[Vehicle[T]]): string =
-  result = ""
-  for c in x:
-    result.add $c.tire
-
-var v = Vehicle[int](tire: 3)
-var c = Car[int](tire: 4)
-var b = Bike[int](tire: 2)
-echo testVehicle([b, c, v])
\ No newline at end of file
diff --git a/tests/typerel/t4799_7.nim b/tests/typerel/t4799_7.nim
deleted file mode 100644
index 784eee8fc1..0000000000
--- a/tests/typerel/t4799_7.nim
+++ /dev/null
@@ -1,20 +0,0 @@
-discard """
-  outputsub: '''ObjectAssignmentError'''
-  exitcode: "1"
-"""
-
-type
-  Vehicle = object of RootObj
-    tire: int
-  Car = object of Vehicle
-  Bike = object of Vehicle
-
-proc testVehicle(x: varargs[Vehicle]): string =
-  result = ""
-  for c in x:
-    result.add $c.tire
-
-var v = Vehicle(tire: 3)
-var c = Car(tire: 4)
-var b = Bike(tire: 2)
-echo testVehicle([b, c, v])
\ No newline at end of file

From 0321ea36c987c88a757879cfee0eb571ff908b73 Mon Sep 17 00:00:00 2001
From: Kaushal Modi <kaushal.modi@gmail.com>
Date: Tue, 5 Jun 2018 12:58:23 -0400
Subject: [PATCH 29/51] Fix typo: PRCE -> PCRE

---
 lib/impure/re.nim | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/lib/impure/re.nim b/lib/impure/re.nim
index 34d55b7b0d..201c490f31 100644
--- a/lib/impure/re.nim
+++ b/lib/impure/re.nim
@@ -10,11 +10,11 @@
 ## Regular expression support for Nim.
 ##
 ## This module is implemented by providing a wrapper around the
-## `PRCE (Perl-Compatible Regular Expressions) <http://www.pcre.org>`_
-## C library. This means that your application will depend on the PRCE
+## `PCRE (Perl-Compatible Regular Expressions) <http://www.pcre.org>`_
+## C library. This means that your application will depend on the PCRE
 ## library's licence when using this module, which should not be a problem
 ## though.
-## PRCE's licence follows:
+## PCRE's licence follows:
 ##
 ## .. include:: ../../doc/regexprs.txt
 ##

From c69b88688fe9a531a88d693e9815af9caaf21396 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Oscar=20Nihlg=C3=A5rd?= <oscarnihlgard@gmail.com>
Date: Tue, 5 Jun 2018 21:05:13 +0200
Subject: [PATCH 30/51] Combine some of the lexer tests into a single file

---
 tests/lexer/thexlit.nim         | 12 ------------
 tests/lexer/thexrange.nim       |  8 --------
 tests/lexer/tlexermisc.nim      | 27 +++++++++++++++++++++++++++
 tests/parser/tunicodeidents.nim | 11 -----------
 4 files changed, 27 insertions(+), 31 deletions(-)
 delete mode 100644 tests/lexer/thexlit.nim
 delete mode 100644 tests/lexer/thexrange.nim
 create mode 100644 tests/lexer/tlexermisc.nim
 delete mode 100644 tests/parser/tunicodeidents.nim

diff --git a/tests/lexer/thexlit.nim b/tests/lexer/thexlit.nim
deleted file mode 100644
index 2b7f0a40e0..0000000000
--- a/tests/lexer/thexlit.nim
+++ /dev/null
@@ -1,12 +0,0 @@
-discard """
-  file: "thexlit.nim"
-  output: "equal"
-"""
-
-var t=0x950412DE
-
-if t==0x950412DE:
-    echo "equal"
-else:
-    echo "not equal"
-
diff --git a/tests/lexer/thexrange.nim b/tests/lexer/thexrange.nim
deleted file mode 100644
index 461e41dfde..0000000000
--- a/tests/lexer/thexrange.nim
+++ /dev/null
@@ -1,8 +0,0 @@
-
-type
-  TArray = array[0x0012..0x0013, int]
-
-var a: TArray
-
-echo a[0x0012] #OUT 0
-
diff --git a/tests/lexer/tlexermisc.nim b/tests/lexer/tlexermisc.nim
new file mode 100644
index 0000000000..3e3993599c
--- /dev/null
+++ b/tests/lexer/tlexermisc.nim
@@ -0,0 +1,27 @@
+discard """
+  action: run
+  output: "equal"
+"""
+
+var t=0x950412DE
+
+if t==0x950412DE:
+    echo "equal"
+else:
+    echo "not equal"
+
+type
+  TArray = array[0x0012..0x0013, int]
+
+var a: TArray
+
+doAssert a[0x0012] == 0
+
+
+# #7884
+
+type Obj = object
+    ö: int
+
+let o = Obj(ö: 1)
+doAssert o.ö == 1
diff --git a/tests/parser/tunicodeidents.nim b/tests/parser/tunicodeidents.nim
deleted file mode 100644
index 3347eb7a95..0000000000
--- a/tests/parser/tunicodeidents.nim
+++ /dev/null
@@ -1,11 +0,0 @@
-discard """
-    action: run
-"""
-
-# #7884
-
-type Obj = object
-    ö: int
-
-let o = Obj(ö: 1)
-doAssert o.ö == 1

From 44589e9ca8b8fcb985426f73f83c7af76139291d Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Tue, 5 Jun 2018 21:02:37 +0300
Subject: [PATCH 31/51] Cosmetics

---
 compiler/closureiters.nim | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index 86b63e34be..75f0b92f64 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -155,6 +155,10 @@ type
     nearestFinally: int # Index of the nearest finally block. For try/except it
                     # is their finally. For finally it is parent finally. Otherwise -1
 
+const
+  nkSkip = { nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
+            nkCommentStmt } + procDefs
+
 proc newStateAccess(ctx: var Ctx): PNode =
   if ctx.stateVarSym.isNil:
     result = rawIndirectAccess(newSymNode(getEnvParam(ctx.fn)),
@@ -247,8 +251,7 @@ proc hasYields(n: PNode): bool =
   case n.kind
   of nkYieldStmt:
     result = true
-  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
-      nkSym, nkIdent, procDefs, nkTemplateDef:
+  of nkSkip:
     discard
   else:
     for c in n:
@@ -259,8 +262,7 @@ proc hasYields(n: PNode): bool =
 proc transformBreaksAndContinuesInWhile(ctx: var Ctx, n: PNode, before, after: PNode): PNode =
   result = n
   case n.kind
-  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
-      nkSym, nkIdent, procDefs, nkTemplateDef:
+  of nkSkip:
     discard
   of nkWhileStmt: discard # Do not recurse into nested whiles
   of nkContinueStmt:
@@ -279,8 +281,7 @@ proc transformBreaksAndContinuesInWhile(ctx: var Ctx, n: PNode, before, after: P
 proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode =
   result = n
   case n.kind
-  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
-      nkSym, nkIdent, procDefs, nkTemplateDef:
+  of nkSkip:
     discard
   of nkBlockStmt, nkWhileStmt:
     inc ctx.blockLevel
@@ -380,8 +381,7 @@ proc getFinallyNode(n: PNode): PNode =
 
 proc hasYieldsInExpressions(n: PNode): bool =
   case n.kind
-  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
-      nkSym, nkIdent, procDefs, nkTemplateDef:
+  of nkSkip:
     discard
   of nkStmtListExpr:
     if isEmptyType(n.typ):
@@ -433,8 +433,7 @@ proc newNotCall(g: ModuleGraph; e: PNode): PNode =
 proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
   result = n
   case n.kind
-  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
-      nkSym, nkIdent, procDefs, nkTemplateDef:
+  of nkSkip:
     discard
 
   of nkYieldStmt:
@@ -797,8 +796,7 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
     let goto = newTree(nkGotoState, ctx.g.newIntLit(n.info, ctx.nearestFinally))
     result.add(goto)
 
-  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
-      nkSym, nkIdent, procDefs, nkTemplateDef:
+  of nkSkip:
     discard
   else:
     for i in 0 ..< n.len:
@@ -807,8 +805,7 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
 proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode =
   result = n
   case n.kind:
-    of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
-        nkSym, nkIdent, procDefs, nkTemplateDef:
+    of nkSkip:
       discard
 
     of nkStmtList, nkStmtListExpr:
@@ -1013,8 +1010,7 @@ proc tranformStateAssignments(ctx: var Ctx, n: PNode): PNode =
       for i in 0 ..< n.len:
         n[i] = ctx.tranformStateAssignments(n[i])
 
-  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
-      nkSym, nkIdent, procDefs, nkTemplateDef:
+  of nkSkip:
     discard
 
   of nkReturnStmt:
@@ -1066,8 +1062,7 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
 proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
   result = n
   case n.kind
-  of nkCharLit..nkUInt64Lit, nkFloatLit..nkFloat128Lit, nkStrLit..nkTripleStrLit,
-      nkSym, nkIdent, procDefs, nkTemplateDef:
+  of nkSkip:
     discard
   of nkGotoState:
     result = copyTree(n)

From a0cb1a80dd61428de8046cee25e664b41156e6ab Mon Sep 17 00:00:00 2001
From: hlaaf <hlaaftana@users.noreply.github.com>
Date: Wed, 6 Jun 2018 00:36:56 +0300
Subject: [PATCH 32/51] Allow `%` overloading in `%*` macro in json (again)

---
 lib/pure/json.nim | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/lib/pure/json.nim b/lib/pure/json.nim
index e7ad5bd5ad..1bd53edb76 100644
--- a/lib/pure/json.nim
+++ b/lib/pure/json.nim
@@ -328,14 +328,14 @@ proc toJson(x: NimNode): NimNode {.compiletime.} =
     result = newNimNode(nnkBracket)
     for i in 0 ..< x.len:
       result.add(toJson(x[i]))
-    result = newCall(bindSym"%", result)
+    result = newCall(bindSym("%", brOpen), result)
   of nnkTableConstr: # object
     if x.len == 0: return newCall(bindSym"newJObject")
     result = newNimNode(nnkTableConstr)
     for i in 0 ..< x.len:
       x[i].expectKind nnkExprColonExpr
       result.add newTree(nnkExprColonExpr, x[i][0], toJson(x[i][1]))
-    result = newCall(bindSym"%", result)
+    result = newCall(bindSym("%", brOpen), result)
   of nnkCurly: # empty object
     x.expectLen(0)
     result = newCall(bindSym"newJObject")
@@ -343,9 +343,9 @@ proc toJson(x: NimNode): NimNode {.compiletime.} =
     result = newCall(bindSym"newJNull")
   of nnkPar:
     if x.len == 1: result = toJson(x[0])
-    else: result = newCall(bindSym"%", x)
+    else: result = newCall(bindSym("%", brOpen), x)
   else:
-    result = newCall(bindSym"%", x)
+    result = newCall(bindSym("%", brOpen), x)
 
 macro `%*`*(x: untyped): untyped =
   ## Convert an expression to a JsonNode directly, without having to specify

From 7c71e1b0583a253019712bca74609877d194bb79 Mon Sep 17 00:00:00 2001
From: hlaaf <hlaaftana@users.noreply.github.com>
Date: Wed, 6 Jun 2018 00:39:07 +0300
Subject: [PATCH 33/51] Fix GC_getStatistics calling itself
 GC_disableMarkAndSweep in JS (again)

---
 lib/system.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/system.nim b/lib/system.nim
index b8aa170ea2..fee9dc3141 100644
--- a/lib/system.nim
+++ b/lib/system.nim
@@ -2681,7 +2681,7 @@ when not defined(nimscript) and hasAlloc:
       {.warning: "GC_unref is a no-op in JavaScript".}
 
     template GC_getStatistics*(): string =
-      {.warning: "GC_disableMarkAndSweep is a no-op in JavaScript".}
+      {.warning: "GC_getStatistics is a no-op in JavaScript".}
       ""
 
 template accumulateResult*(iter: untyped) =

From ba39f359aa6b28eb30f7165f9e629373c730b3b2 Mon Sep 17 00:00:00 2001
From: nitely <ecastroborsani@gmail.com>
Date: Tue, 5 Jun 2018 20:22:27 -0300
Subject: [PATCH 34/51] check bounds instead of index

---
 compiler/ccgcalls.nim         |  5 ++---
 compiler/ccgexprs.nim         | 19 ++++++++++++-------
 tests/system/tsystem_misc.nim | 25 +++++++++++++++++++++++++
 3 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim
index 7d355db5fd..22733f6acf 100644
--- a/compiler/ccgcalls.nim
+++ b/compiler/ccgcalls.nim
@@ -83,7 +83,7 @@ proc isInCurrentFrame(p: BProc, n: PNode): bool =
     result = isInCurrentFrame(p, n.sons[0])
   else: discard
 
-proc genIndexCheck(p: BProc; arr, idx: TLoc)
+proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
 
 proc openArrayLoc(p: BProc, n: PNode): Rope =
   var a: TLoc
@@ -97,8 +97,7 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
     initLocExpr(p, q[3], c)
     # but first produce the required index checks:
     if optBoundsCheck in p.options:
-      genIndexCheck(p, a, b)
-      genIndexCheck(p, a, c)
+      genBoundsCheck(p, a, b, c)
     let ty = skipTypes(a.t, abstractVar+{tyPtr})
     case ty.kind
     of tyArray:
diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim
index 335aa2f84a..6748216668 100644
--- a/compiler/ccgexprs.nim
+++ b/compiler/ccgexprs.nim
@@ -873,21 +873,26 @@ proc genCStringElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   putIntoDest(p, d, n,
               ropecg(p.module, "$1[$2]", rdLoc(a), rdCharLoc(b)), a.storage)
 
-proc genIndexCheck(p: BProc; arr, idx: TLoc) =
+proc genBoundsCheck(p: BProc; arr, a, b: TLoc) =
   let ty = skipTypes(arr.t, abstractVarRange)
   case ty.kind
   of tyOpenArray, tyVarargs:
-    linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)) #raiseIndexError();$n",
-            rdLoc(idx), rdLoc(arr))
+    linefmt(p, cpsStmts,
+      "if ($2-$1 != -1 && " &
+      "((NU)($1) >= (NU)($3Len_0) || (NU)($2) >= (NU)($3Len_0))) #raiseIndexError();$n",
+      rdLoc(a), rdLoc(b), rdLoc(arr))
   of tyArray:
     let first = intLiteral(firstOrd(ty))
     if tfUncheckedArray notin ty.flags:
-      linefmt(p, cpsStmts, "if ($1 < $2 || $1 > $3) #raiseIndexError();$n",
-              rdCharLoc(idx), first, intLiteral(lastOrd(ty)))
+      linefmt(p, cpsStmts,
+        "if ($2-$1 != -1 && " &
+        "($2-$1 < -1 || $1 < $3 || $1 > $4 || $2 < $3 || $2 > $4)) #raiseIndexError();$n",
+        rdCharLoc(a), rdCharLoc(b), first, intLiteral(lastOrd(ty)))
   of tySequence, tyString:
     linefmt(p, cpsStmts,
-          "if (!$2 || (NU)($1) >= (NU)($2->$3)) #raiseIndexError();$n",
-          rdLoc(idx), rdLoc(arr), lenField(p))
+      "if ($2-$1 != -1 && " &
+      "(!$3 || (NU)($1) >= (NU)($3->$4) || (NU)($2) >= (NU)($3->$4))) #raiseIndexError();$n",
+      rdLoc(a), rdLoc(b), rdLoc(arr), lenField(p))
   else: discard
 
 proc genOpenArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) =
diff --git a/tests/system/tsystem_misc.nim b/tests/system/tsystem_misc.nim
index 85228e9e7e..460d94d56e 100644
--- a/tests/system/tsystem_misc.nim
+++ b/tests/system/tsystem_misc.nim
@@ -11,6 +11,7 @@ discard """
 2
 3
 4
+2
 '''
 """
 
@@ -47,3 +48,27 @@ foo(toOpenArray(arr, 8, 12))
 
 var seqq = @[1, 2, 3, 4, 5]
 foo(toOpenArray(seqq, 1, 3))
+
+# empty openArray issue #7904
+foo(toOpenArray(seqq, 0, -1))
+foo(toOpenArray(seqq, 1, 0))
+doAssertRaises(IndexError):
+  foo(toOpenArray(seqq, 0, -2))
+
+foo(toOpenArray(arr, 9, 8))
+foo(toOpenArray(arr, 0, -1))
+foo(toOpenArray(arr, 1, 0))
+doAssertRaises(IndexError):
+  foo(toOpenArray(arr, 10, 8))
+
+# test openArray of openArray
+proc oaEmpty(a: openArray[int]) =
+  foo(toOpenArray(a, 0, -1))
+
+proc oaFirstElm(a: openArray[int]) =
+  foo(toOpenArray(a, 0, 0))
+
+oaEmpty(toOpenArray(seqq, 0, -1))
+oaEmpty(toOpenArray(seqq, 1, 0))
+oaEmpty(toOpenArray(seqq, 1, 2))
+oaFirstElm(toOpenArray(seqq, 1, seqq.len-1))

From 7297195f9fc5aa47de4b64f1402cfc7af109badc Mon Sep 17 00:00:00 2001
From: nitely <ecastroborsani@gmail.com>
Date: Wed, 6 Jun 2018 02:22:33 -0300
Subject: [PATCH 35/51] test negative range array

---
 tests/system/tsystem_misc.nim | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/tests/system/tsystem_misc.nim b/tests/system/tsystem_misc.nim
index 460d94d56e..6d14aa68fd 100644
--- a/tests/system/tsystem_misc.nim
+++ b/tests/system/tsystem_misc.nim
@@ -12,6 +12,9 @@ discard """
 3
 4
 2
+1
+2
+3
 '''
 """
 
@@ -72,3 +75,14 @@ oaEmpty(toOpenArray(seqq, 0, -1))
 oaEmpty(toOpenArray(seqq, 1, 0))
 oaEmpty(toOpenArray(seqq, 1, 2))
 oaFirstElm(toOpenArray(seqq, 1, seqq.len-1))
+
+var arrNeg: array[-3 .. -1, int] = [1, 2, 3]
+foo(toOpenArray(arrNeg, -3, -1))
+foo(toOpenArray(arrNeg, 0, -1))
+foo(toOpenArray(arrNeg, -3, -4))
+doAssertRaises(IndexError):
+  foo(toOpenArray(arrNeg, -4, -1))
+doAssertRaises(IndexError):
+  foo(toOpenArray(arrNeg, -1, 0))
+doAssertRaises(IndexError):
+  foo(toOpenArray(arrNeg, -1, -3))

From b7a8eef87a720e1bd4727ca4d5a9c488a2ca0c69 Mon Sep 17 00:00:00 2001
From: Michael Voronin <m.voronin@ngenix.net>
Date: Thu, 10 May 2018 13:24:45 +0300
Subject: [PATCH 36/51] [change] Replace mutators with their more generic
 versions

---
 lib/pure/times.nim | 59 +++++++++++++++++-----------------------------
 1 file changed, 22 insertions(+), 37 deletions(-)

diff --git a/lib/pure/times.nim b/lib/pure/times.nim
index 60b3626656..7cecc31ab5 100644
--- a/lib/pure/times.nim
+++ b/lib/pure/times.nim
@@ -185,8 +185,7 @@ type
 
   DurationParts* = array[FixedTimeUnit, int64] # Array of Duration parts starts
   TimeIntervalParts* = array[TimeUnit, int] # Array of Duration parts starts
-
-
+  TimesMutableTypes = DateTime | Time | Duration | TimeInterval
 
 {.deprecated: [TMonth: Month, TWeekDay: WeekDay, TTime: Time,
     TTimeInterval: TimeInterval, TTimeInfo: DateTime, TimeInfo: DateTime].}
@@ -607,30 +606,12 @@ proc `+`*(a: Time, b: Duration): Time {.operator, extern: "ntAddTime".} =
     doAssert (fromUnix(0) + initDuration(seconds = 1)) == fromUnix(1)
   addImpl[Time](a, b)
 
-proc `+=`*(a: var Time, b: Duration) {.operator.} =
-  ## Modify ``a`` in place by subtracting ``b``.
-  runnableExamples:
-    var tm = fromUnix(0)
-    tm += initDuration(seconds = 1)
-    doAssert tm == fromUnix(1)
-
-  a = addImpl[Time](a, b)
-
 proc `-`*(a: Time, b: Duration): Time {.operator, extern: "ntSubTime".} =
   ## Subtracts a duration of time from a ``Time``.
   runnableExamples:
     doAssert (fromUnix(0) - initDuration(seconds = 1)) == fromUnix(-1)
   subImpl[Time](a, b)
 
-proc `-=`*(a: var Time, b: Duration) {.operator.} =
-  ## Modify ``a`` in place by adding ``b``.
-  runnableExamples:
-    var tm = fromUnix(0)
-    tm -= initDuration(seconds = 1)
-    doAssert tm == fromUnix(-1)
-
-  a = subImpl[Time](a, b)
-
 proc `<`*(a, b: Time): bool {.operator, extern: "ntLtTime".} =
   ## Returns true iff ``a < b``, that is iff a happened before b.
   ltImpl(a, b)
@@ -1377,17 +1358,6 @@ proc `+`*(time: Time, interval: TimeInterval): Time =
   else:
     toTime(time.local + interval)
 
-proc `+=`*(time: var Time, interval: TimeInterval) =
-  ## Modifies `time` by adding `interval`.
-  ## If `interval` contains any years, months, weeks or days the operation
-  ## is performed in the local timezone.
-  runnableExamples:
-    var tm = fromUnix(0)
-    tm += 5.seconds
-    doAssert tm == fromUnix(5)
-
-  time = time + interval
-
 proc `-`*(time: Time, interval: TimeInterval): Time =
   ## Subtracts `interval` from Time `time`.
   ## If `interval` contains any years, months, weeks or days the operation
@@ -1401,15 +1371,30 @@ proc `-`*(time: Time, interval: TimeInterval): Time =
   else:
     toTime(time.local - interval)
 
-proc `-=`*(time: var Time, interval: TimeInterval) =
-  ## Modifies `time` by subtracting `interval`.
-  ## If `interval` contains any years, months, weeks or days the operation
-  ## is performed in the local timezone.
+proc `+=`*[T, U: TimesMutableTypes](a: var T, b: U) =
+  ## Modify ``a`` in place by adding ``b``.
+  runnableExamples:
+    var tm = fromUnix(0)
+    tm += initDuration(seconds = 1)
+    doAssert tm == fromUnix(1)
+  a = a + b
+
+proc `-=`*[T, U: TimesMutableTypes](a: var T, b: U) =
+  ## Modify ``a`` in place by subtracting ``b``.
   runnableExamples:
     var tm = fromUnix(5)
-    tm -= 5.seconds
+    tm -= initDuration(seconds = 5)
     doAssert tm == fromUnix(0)
-  time = time - interval
+  a = a - b
+
+proc `*=`*[T: TimesMutableTypes, U](a: var T, b: U) =
+  # Mutable type is often multiplied by number
+  runnableExamples:
+    var dur = initDuration(seconds = 1)
+    dur *= 5
+    doAssert dur == initDuration(seconds = 5)
+
+  a = a * b
 
 proc formatToken(dt: DateTime, token: string, buf: var string) =
   ## Helper of the format proc to parse individual tokens.

From 0ec2b33c50bf96e2b9f164fd8d44fe04fa76de52 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Wed, 6 Jun 2018 16:17:24 +0300
Subject: [PATCH 37/51] Fixed yield in dotExpr and nkOfBranch lowering. Closes
 #7969.

---
 compiler/closureiters.nim  | 17 +++++++++++++++--
 tests/iter/tyieldintry.nim | 24 ++++++++++++++++++++++++
 2 files changed, 39 insertions(+), 2 deletions(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index 75f0b92f64..3d86954c25 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -735,6 +735,19 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
         n[0] = newSymNode(ctx.g.getSysSym(n[0].info, "true"))
         n[1] = newBody
+
+  of nkDotExpr:
+    var ns = false
+    n[0] = ctx.lowerStmtListExprs(n[0], ns)
+    if ns:
+      needsSplit = true
+      result = newNodeI(nkStmtListExpr, n.info)
+      result.typ = n.typ
+      let (st, ex) = exprToStmtList(n[0])
+      result.add(st)
+      n[0] = ex
+      result.add(n)
+
   else:
     for i in 0 ..< n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], needsSplit)
@@ -843,8 +856,8 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
       result[0] = ctx.transformClosureIteratorBody(result[0], gotoOut)
 
     of nkElifBranch, nkElifExpr, nkOfBranch:
-      result[1] = addGotoOut(result[1], gotoOut)
-      result[1] = ctx.transformClosureIteratorBody(result[1], gotoOut)
+      result[^1] = addGotoOut(result[^1], gotoOut)
+      result[^1] = ctx.transformClosureIteratorBody(result[^1], gotoOut)
 
     of nkIfStmt, nkCaseStmt:
       for i in 0 ..< n.len:
diff --git a/tests/iter/tyieldintry.nim b/tests/iter/tyieldintry.nim
index 31ec65a830..c9e5843b89 100644
--- a/tests/iter/tyieldintry.nim
+++ b/tests/iter/tyieldintry.nim
@@ -368,5 +368,29 @@ block: # Short cirquits
 
   test(it, 1, 0, 0)
 
+block: #7969
+  type
+    SomeObj = object
+      id: int
+
+  iterator it(): int {.closure.} =
+    template yieldAndSomeObj: SomeObj =
+      var s: SomeObj
+      s.id = 2
+      yield 1
+      s
+
+    checkpoint(yieldAndSomeObj().id)
+
+    var i = 5
+    case i
+    of 0:
+      checkpoint(123)
+    of 1, 2, 5:
+      checkpoint(3)
+    else:
+      checkpoint(123)
+
+  test(it, 1, 2, 3)
 
 echo "ok"

From 4262a85653e22e430824c92db35c5ee3bcecf954 Mon Sep 17 00:00:00 2001
From: andri lim <jangko128@gmail.com>
Date: Wed, 6 Jun 2018 22:29:31 +0700
Subject: [PATCH 38/51] fixed wrong test

---
 tests/typerel/t4799.nim | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tests/typerel/t4799.nim b/tests/typerel/t4799.nim
index 89312950fe..0758934763 100644
--- a/tests/typerel/t4799.nim
+++ b/tests/typerel/t4799.nim
@@ -187,7 +187,7 @@ block test_t4799_7:
   var b = Bike[int](tire: 2)
 
   reject:
-    echo testVehicle b, c, v
+    echo testVehicle(b, c, v)
 
 block test_t4799_8:
   type
@@ -206,7 +206,7 @@ block test_t4799_8:
   var b = Bike(tire: 2)
 
   reject:
-    echo testVehicle b, c, v
+    echo testVehicle(b, c, v)
 
 type
   PGVehicle[T] = ptr object of RootObj
@@ -223,7 +223,7 @@ var pgc = PGCar[int](tire: 4)
 var pgb = PGBike[int](tire: 2)
 
 reject:
-  echo testVehicle pgb, pgc
+  echo testVehicle(pgb, pgc)
 
 type
   RVehicle = ptr object of RootObj
@@ -240,6 +240,6 @@ var rc = RCar(tire: 4)
 var rb = RBike(tire: 2)
 
 reject:
-  echo testVehicle rb, rc
+  echo testVehicle(rb, rc)
 
 echo "OK"

From 6ee6f252d487ff22cdaa20eecf9a3975082863e9 Mon Sep 17 00:00:00 2001
From: Dominik Picheta <dominikpicheta@gmail.com>
Date: Tue, 8 May 2018 10:51:17 +0100
Subject: [PATCH 39/51] Rip out the `try` transformation in the async macro.

---
 lib/pure/asyncmacro.nim | 111 +++-------------------------------------
 1 file changed, 6 insertions(+), 105 deletions(-)

diff --git a/lib/pure/asyncmacro.nim b/lib/pure/asyncmacro.nim
index 96a6fa1582..4665ad25f1 100644
--- a/lib/pure/asyncmacro.nim
+++ b/lib/pure/asyncmacro.nim
@@ -62,52 +62,6 @@ template createCb(retFutureSym, iteratorNameSym,
 
   identName()
   #{.pop.}
-proc generateExceptionCheck(futSym,
-    tryStmt, rootReceiver, fromNode: NimNode): NimNode {.compileTime.} =
-  if tryStmt.kind == nnkNilLit:
-    result = rootReceiver
-  else:
-    var exceptionChecks: seq[tuple[cond, body: NimNode]] = @[]
-    let errorNode = newDotExpr(futSym, newIdentNode("error"))
-    for i in 1 ..< tryStmt.len:
-      let exceptBranch = tryStmt[i]
-      if exceptBranch[0].kind == nnkStmtList:
-        exceptionChecks.add((newIdentNode("true"), exceptBranch[0]))
-      else:
-        var exceptIdentCount = 0
-        var ifCond: NimNode
-        for i in 0 ..< exceptBranch.len:
-          let child = exceptBranch[i]
-          if child.kind == nnkIdent:
-            let cond = infix(errorNode, "of", child)
-            if exceptIdentCount == 0:
-              ifCond = cond
-            else:
-              ifCond = infix(ifCond, "or", cond)
-          else:
-            break
-          exceptIdentCount.inc
-
-        expectKind(exceptBranch[exceptIdentCount], nnkStmtList)
-        exceptionChecks.add((ifCond, exceptBranch[exceptIdentCount]))
-    # -> -> else: raise futSym.error
-    exceptionChecks.add((newIdentNode("true"),
-        newNimNode(nnkRaiseStmt).add(errorNode)))
-    # Read the future if there is no error.
-    # -> else: futSym.read
-    let elseNode = newNimNode(nnkElse, fromNode)
-    elseNode.add newNimNode(nnkStmtList, fromNode)
-    elseNode[0].add rootReceiver
-
-    let ifBody = newStmtList()
-    ifBody.add newCall(newIdentNode("setCurrentException"), errorNode)
-    ifBody.add newIfStmt(exceptionChecks)
-    ifBody.add newCall(newIdentNode("setCurrentException"), newNilLit())
-
-    result = newIfStmt(
-      (newDotExpr(futSym, newIdentNode("failed")), ifBody)
-    )
-    result.add elseNode
 
 template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver,
                 rootReceiver: untyped, fromNode: NimNode) =
@@ -123,8 +77,7 @@ template useVar(result: var NimNode, futureVarNode: NimNode, valueReceiver,
   result.add newNimNode(nnkYieldStmt, fromNode).add(futureVarNode)
   # -> future<x>.read
   valueReceiver = newDotExpr(futureVarNode, newIdentNode("read"))
-  result.add generateExceptionCheck(futureVarNode, tryStmt, rootReceiver,
-      fromNode)
+  result.add rootReceiver
 
 template createVar(result: var NimNode, futSymName: string,
                    asyncProc: NimNode,
@@ -154,8 +107,8 @@ proc createFutureVarCompletions(futureVarIdents: seq[NimNode],
     )
 
 proc processBody(node, retFutureSym: NimNode,
-                 subTypeIsVoid: bool, futureVarIdents: seq[NimNode],
-                 tryStmt: NimNode): NimNode {.compileTime.} =
+                 subTypeIsVoid: bool,
+                 futureVarIdents: seq[NimNode]): NimNode {.compileTime.} =
   #echo(node.treeRepr)
   result = node
   case node.kind
@@ -173,7 +126,7 @@ proc processBody(node, retFutureSym: NimNode,
         result.add newCall(newIdentNode("complete"), retFutureSym)
     else:
       let x = node[0].processBody(retFutureSym, subTypeIsVoid,
-                                  futureVarIdents, tryStmt)
+                                  futureVarIdents)
       if x.kind == nnkYieldStmt: result.add x
       else:
         result.add newCall(newIdentNode("complete"), retFutureSym, x)
@@ -224,63 +177,11 @@ proc processBody(node, retFutureSym: NimNode,
       var newDiscard = node
       result.createVar("futureDiscard_" & $toStrLit(node[0][1]), node[0][1],
                 newDiscard[0], newDiscard, node)
-  of nnkTryStmt:
-    # try: await x; except: ...
-    result = newNimNode(nnkStmtList, node)
-    template wrapInTry(n, tryBody: untyped) =
-      var temp = n
-      n[0] = tryBody
-      tryBody = temp
-
-      # Transform ``except`` body.
-      # TODO: Could we perform some ``await`` transformation here to get it
-      # working in ``except``?
-      tryBody[1] = processBody(n[1], retFutureSym, subTypeIsVoid,
-                               futureVarIdents, nil)
-
-    proc processForTry(n: NimNode, i: var int,
-                       res: NimNode): bool {.compileTime.} =
-      ## Transforms the body of the tryStmt. Does not transform the
-      ## body in ``except``.
-      ## Returns true if the tryStmt node was transformed into an ifStmt.
-      result = false
-      var skipped = n.skipStmtList()
-      while i < skipped.len:
-        var processed = processBody(skipped[i], retFutureSym,
-                                    subTypeIsVoid, futureVarIdents, n)
-
-        # Check if we transformed the node into an exception check.
-        # This suggests skipped[i] contains ``await``.
-        if processed.kind != skipped[i].kind or processed.len != skipped[i].len:
-          processed = processed.skipUntilStmtList()
-          expectKind(processed, nnkStmtList)
-          expectKind(processed[2][1], nnkElse)
-          i.inc
-
-          if not processForTry(n, i, processed[2][1][0]):
-            # We need to wrap the nnkElse nodes back into a tryStmt.
-            # As they are executed if an exception does not happen
-            # inside the awaited future.
-            # The following code will wrap the nodes inside the
-            # original tryStmt.
-            wrapInTry(n, processed[2][1][0])
-
-          res.add processed
-          result = true
-        else:
-          res.add skipped[i]
-          i.inc
-    var i = 0
-    if not processForTry(node, i, result):
-      # If the tryStmt hasn't been transformed we can just put the body
-      # back into it.
-      wrapInTry(node, result)
-    return
   else: discard
 
   for i in 0 ..< result.len:
     result[i] = processBody(result[i], retFutureSym, subTypeIsVoid,
-                            futureVarIdents, nil)
+                            futureVarIdents)
 
 proc getName(node: NimNode): string {.compileTime.} =
   case node.kind
@@ -362,7 +263,7 @@ proc asyncSingleProc(prc: NimNode): NimNode {.compileTime.} =
   # ->   complete(retFuture, result)
   var iteratorNameSym = genSym(nskIterator, $prcName & "Iter")
   var procBody = prc.body.processBody(retFutureSym, subtypeIsVoid,
-                                    futureVarIdents, nil)
+                                    futureVarIdents)
   # don't do anything with forward bodies (empty)
   if procBody.kind != nnkEmpty:
     procBody.add(createFutureVarCompletions(futureVarIdents, nil))

From 511d7079a1d5890c4add4b00f79606934e3e3414 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Wed, 6 Jun 2018 20:04:37 +0300
Subject: [PATCH 40/51] Fixed tests

---
 tests/async/tasync_traceback.nim | 26 +++++++++++++++++++++-----
 tests/async/tasynctry.nim        |  4 ++--
 tests/async/tasynctry2.nim       |  2 +-
 3 files changed, 24 insertions(+), 8 deletions(-)

diff --git a/tests/async/tasync_traceback.nim b/tests/async/tasync_traceback.nim
index 618a1dc769..b6c6a916bd 100644
--- a/tests/async/tasync_traceback.nim
+++ b/tests/async/tasync_traceback.nim
@@ -3,7 +3,7 @@ discard """
   disabled: "windows"
   output: "Matched"
 """
-import asyncdispatch
+import asyncdispatch, strutils
 
 # Tests to ensure our exception trace backs are friendly.
 
@@ -117,10 +117,26 @@ Exception message: bar failure
 Exception type:
 """
 
-if result.match(re(expected)):
-  echo("Matched")
-else:
-  echo("Not matched!")
+let resLines = splitLines(result.strip)
+let expLines = splitLines(expected.strip)
+
+if resLines.len != expLines.len:
+  echo("Not matched! Wrong number of lines!")
   echo()
   echo(result)
   quit(QuitFailure)
+
+var ok = true
+for i in 0 ..< resLines.len:
+  if not resLines[i].match(re(expLines[i])):
+    echo "Not matched! Line ", i + 1
+    echo "Expected:"
+    echo expLines[i]
+    echo "Actual:"
+    echo resLines[i]
+    ok = false
+
+if ok:
+  echo("Matched")
+else:
+  quit(QuitFailure)
diff --git a/tests/async/tasynctry.nim b/tests/async/tasynctry.nim
index 5930f296f4..6749aabbf0 100644
--- a/tests/async/tasynctry.nim
+++ b/tests/async/tasynctry.nim
@@ -9,7 +9,7 @@ Multiple except branches
 Multiple except branches 2
 '''
 """
-import asyncdispatch
+import asyncdispatch, strutils
 
 # Here we are testing the ability to catch exceptions.
 
@@ -22,7 +22,7 @@ proc catch() {.async.} =
   try:
     await foobar()
   except:
-    echo("Generic except: ", getCurrentExceptionMsg())
+    echo("Generic except: ", getCurrentExceptionMsg().splitLines[0])
 
   try:
     await foobar()
diff --git a/tests/async/tasynctry2.nim b/tests/async/tasynctry2.nim
index f82b6cfe0d..4b3f17cc54 100644
--- a/tests/async/tasynctry2.nim
+++ b/tests/async/tasynctry2.nim
@@ -1,7 +1,7 @@
 discard """
   file: "tasynctry2.nim"
   errormsg: "\'yield\' cannot be used within \'try\' in a non-inlined iterator"
-  line: 17
+  line: 14
 """
 import asyncdispatch
 

From fef60716bfbae0f0eda8cf976d5cd4b61f1c5fdd Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Thu, 7 Jun 2018 00:14:56 +0300
Subject: [PATCH 41/51] Fixed yield in nkBlockExpr

---
 compiler/closureiters.nim  | 16 ++++++++++++++--
 tests/iter/tyieldintry.nim | 15 +++++++++++++++
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index 3d86954c25..e8f4d62c15 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -442,7 +442,6 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
-      assert(n[0].kind == nkStmtListExpr)
       result = newNodeI(nkStmtList, n.info)
       let (st, ex) = exprToStmtList(n[0])
       result.add(st)
@@ -662,7 +661,6 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       c[^1] = ctx.lowerStmtListExprs(c[^1], ns)
       if ns:
         needsSplit = true
-        assert(c[^1].kind == nkStmtListExpr)
         let (st, ex) = exprToStmtList(c[^1])
         result.add(st)
         c[^1] = ex
@@ -748,6 +746,20 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       n[0] = ex
       result.add(n)
 
+  of nkBlockExpr:
+    var ns = false
+    n[1] = ctx.lowerStmtListExprs(n[1], ns)
+    if ns:
+      needsSplit = true
+      result = newNodeI(nkStmtListExpr, n.info)
+      result.typ = n.typ
+      let (st, ex) = exprToStmtList(n[1])
+      n.kind = nkBlockStmt
+      n.typ = nil
+      n[1] = st
+      result.add(n)
+      result.add(ex)
+
   else:
     for i in 0 ..< n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], needsSplit)
diff --git a/tests/iter/tyieldintry.nim b/tests/iter/tyieldintry.nim
index c9e5843b89..3c07736e16 100644
--- a/tests/iter/tyieldintry.nim
+++ b/tests/iter/tyieldintry.nim
@@ -393,4 +393,19 @@ block: #7969
 
   test(it, 1, 2, 3)
 
+block: # yield in blockexpr
+  type
+    SomeObj = object
+      id: int
+
+  iterator it(): int {.closure.} =
+    yield(block:
+      checkpoint(1)
+      yield 2
+      3
+    )
+
+  test(it, 1, 2, 3)
+
+
 echo "ok"

From 722462ae299775a65f5f8204fac7bc47611568db Mon Sep 17 00:00:00 2001
From: data-man <datamanrb@gmail.com>
Date: Thu, 7 Jun 2018 01:18:08 +0300
Subject: [PATCH 42/51] Removed test for live website

---
 tests/stdlib/thttpclient.nim | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/tests/stdlib/thttpclient.nim b/tests/stdlib/thttpclient.nim
index c28f091003..fff02722a9 100644
--- a/tests/stdlib/thttpclient.nim
+++ b/tests/stdlib/thttpclient.nim
@@ -154,20 +154,8 @@ proc ipv6Test() =
   serverFd.closeSocket()
   client.close()
 
-proc longTimeoutTest() =
-# Issue #2753
-  try:
-    var client = newHttpClient(timeout = 1000)
-    var resp = client.request("https://au.yahoo.com")
-    client.close()
-  except AssertionError:
-    doAssert false, "Exceptions should not be raised"
-  except:
-    discard
-
 syncTest()
 waitFor(asyncTest())
 ipv6Test()
-longTimeoutTest()
 
 echo "OK"

From 46a6fa53a92225637da776bae4fdba0be7aed94a Mon Sep 17 00:00:00 2001
From: cheatfate <ka@hardcore.kiev.ua>
Date: Thu, 7 Jun 2018 12:33:29 +0300
Subject: [PATCH 43/51] Fix fromSockAddrAux() to handle IPv6 addresses
 properly.

---
 lib/pure/net.nim | 25 ++++++++++++++++---------
 1 file changed, 16 insertions(+), 9 deletions(-)

diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index cac10d11c1..ddfb3460db 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -405,33 +405,40 @@ proc isIpAddress*(address_str: string): bool {.tags: [].} =
     return false
   return true
 
-proc toSockAddr*(address: IpAddress, port: Port, sa: var Sockaddr_storage, sl: var Socklen) =
+proc toSockAddr*(address: IpAddress, port: Port, sa: var Sockaddr_storage,
+                 sl: var Socklen) =
   ## Converts `IpAddress` and `Port` to `SockAddr` and `Socklen`
   let port = htons(uint16(port))
   case address.family
   of IpAddressFamily.IPv4:
     sl = sizeof(Sockaddr_in).Socklen
     let s = cast[ptr Sockaddr_in](addr sa)
-    s.sin_family = type(s.sin_family)(AF_INET)
+    s.sin_family = type(s.sin_family)(toInt(AF_INET))
     s.sin_port = port
-    copyMem(addr s.sin_addr, unsafeAddr address.address_v4[0], sizeof(s.sin_addr))
+    copyMem(addr s.sin_addr, unsafeAddr address.address_v4[0],
+            sizeof(s.sin_addr))
   of IpAddressFamily.IPv6:
     sl = sizeof(Sockaddr_in6).Socklen
     let s = cast[ptr Sockaddr_in6](addr sa)
-    s.sin6_family = type(s.sin6_family)(AF_INET6)
+    s.sin6_family = type(s.sin6_family)(toInt(AF_INET6))
     s.sin6_port = port
-    copyMem(addr s.sin6_addr, unsafeAddr address.address_v6[0], sizeof(s.sin6_addr))
+    copyMem(addr s.sin6_addr, unsafeAddr address.address_v6[0],
+            sizeof(s.sin6_addr))
 
-proc fromSockAddrAux(sa: ptr Sockaddr_storage, sl: Socklen, address: var IpAddress, port: var Port) =
+proc fromSockAddrAux(sa: ptr Sockaddr_storage, sl: Socklen,
+                     address: var IpAddress, port: var Port) =
   if sa.ss_family.int == AF_INET.int and sl == sizeof(Sockaddr_in).Socklen:
     address = IpAddress(family: IpAddressFamily.IPv4)
     let s = cast[ptr Sockaddr_in](sa)
-    copyMem(addr address.address_v4[0], addr s.sin_addr, sizeof(address.address_v4))
+    copyMem(addr address.address_v4[0], addr s.sin_addr,
+            sizeof(address.address_v4))
     port = ntohs(s.sin_port).Port
-  elif sa.ss_family.int == AF_INET6.int and sl == sizeof(Sockaddr_in6).Socklen:
+  elif sa.ss_family.int == toInt(AF_INET6) and
+       sl == sizeof(Sockaddr_in6).Socklen:
     address = IpAddress(family: IpAddressFamily.IPv6)
     let s = cast[ptr Sockaddr_in6](sa)
-    copyMem(addr address.address_v6[0], addr s.sin6_addr, sizeof(address.address_v6))
+    copyMem(addr address.address_v6[0], addr s.sin6_addr,
+            sizeof(address.address_v6))
     port = ntohs(s.sin6_port).Port
   else:
     raise newException(ValueError, "Neither IPv4 nor IPv6")

From 2902308a056d8568c79b02dc58e4e2d1d35fc924 Mon Sep 17 00:00:00 2001
From: cheatfate <ka@hardcore.kiev.ua>
Date: Thu, 7 Jun 2018 13:06:06 +0300
Subject: [PATCH 44/51] Fix one more place.

---
 lib/pure/net.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/pure/net.nim b/lib/pure/net.nim
index ddfb3460db..60817484a2 100644
--- a/lib/pure/net.nim
+++ b/lib/pure/net.nim
@@ -427,7 +427,7 @@ proc toSockAddr*(address: IpAddress, port: Port, sa: var Sockaddr_storage,
 
 proc fromSockAddrAux(sa: ptr Sockaddr_storage, sl: Socklen,
                      address: var IpAddress, port: var Port) =
-  if sa.ss_family.int == AF_INET.int and sl == sizeof(Sockaddr_in).Socklen:
+  if sa.ss_family.int == toInt(AF_INET) and sl == sizeof(Sockaddr_in).Socklen:
     address = IpAddress(family: IpAddressFamily.IPv4)
     let s = cast[ptr Sockaddr_in](sa)
     copyMem(addr address.address_v4[0], addr s.sin_addr,

From ec1d42b9f0846957d33f0fdfbdf0407a0227c6af Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Thu, 7 Jun 2018 11:19:52 +0300
Subject: [PATCH 45/51] Cleanup copypaste leftover

---
 tests/iter/tyieldintry.nim | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tests/iter/tyieldintry.nim b/tests/iter/tyieldintry.nim
index 3c07736e16..6f0acb1699 100644
--- a/tests/iter/tyieldintry.nim
+++ b/tests/iter/tyieldintry.nim
@@ -394,10 +394,6 @@ block: #7969
   test(it, 1, 2, 3)
 
 block: # yield in blockexpr
-  type
-    SomeObj = object
-      id: int
-
   iterator it(): int {.closure.} =
     yield(block:
       checkpoint(1)

From 29a01da90f395e32fdb5ae88949010700b2c427e Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Thu, 7 Jun 2018 16:38:47 +0300
Subject: [PATCH 46/51] Fixes #7982

---
 compiler/ccgstmts.nim | 13 +++++++++----
 compiler/cgendata.nim |  2 ++
 compiler/pragmas.nim  |  4 ++++
 3 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim
index 91a3add70c..f99ee92708 100644
--- a/compiler/ccgstmts.nim
+++ b/compiler/ccgstmts.nim
@@ -775,6 +775,13 @@ proc genCase(p: BProc, t: PNode, d: var TLoc) =
     else:
       genOrdinalCase(p, t, d)
 
+proc genRestoreFrameAfterException(p: BProc) =
+  if optStackTrace in p.module.config.options:
+    if not p.hasCurFramePointer:
+      p.hasCurFramePointer = true
+      p.procSec(cpsLocals).add(ropecg(p.module, "\tTFrame* _nimCurFrame;$n", []))
+      p.procSec(cpsInit).add(ropecg(p.module, "\t_nimCurFrame = #getFrame();$n", []))
+    linefmt(p, cpsStmts, "#setFrame(_nimCurFrame);$n")
 
 proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   # code to generate:
@@ -794,8 +801,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   #   finallyPart();
 
   template genExceptBranchBody(body: PNode) {.dirty.} =
-    if optStackTrace in p.options:
-      linefmt(p, cpsStmts, "#setFrame((TFrame*)&FR_);$n")
+    genRestoreFrameAfterException(p)
     expr(p, body, d)
 
   if not isEmptyType(t.typ) and d.k == locNone:
@@ -898,8 +904,7 @@ proc genTry(p: BProc, t: PNode, d: var TLoc) =
   endBlock(p)
   startBlock(p, "else {$n")
   linefmt(p, cpsStmts, "#popSafePoint();$n")
-  if optStackTrace in p.options:
-    linefmt(p, cpsStmts, "#setFrame((TFrame*)&FR_);$n")
+  genRestoreFrameAfterException(p)
   p.nestedTryStmts[^1].inExcept = true
   var i = 1
   while (i < length) and (t.sons[i].kind == nkExceptBranch):
diff --git a/compiler/cgendata.nim b/compiler/cgendata.nim
index ce3fc2f905..8436776547 100644
--- a/compiler/cgendata.nim
+++ b/compiler/cgendata.nim
@@ -69,6 +69,8 @@ type
     prc*: PSym                # the Nim proc that this C proc belongs to
     beforeRetNeeded*: bool    # true iff 'BeforeRet' label for proc is needed
     threadVarAccessed*: bool  # true if the proc already accessed some threadvar
+    hasCurFramePointer*: bool # true if _nimCurFrame var needed to recover after
+                              # exception is generated
     lastLineInfo*: TLineInfo  # to avoid generating excessive 'nimln' statements
     currLineInfo*: TLineInfo  # AST codegen will make this superfluous
     nestedTryStmts*: seq[tuple[n: PNode, inExcept: bool]]
diff --git a/compiler/pragmas.nim b/compiler/pragmas.nim
index de98a5e42a..d3fa506cb4 100644
--- a/compiler/pragmas.nim
+++ b/compiler/pragmas.nim
@@ -374,6 +374,10 @@ proc processPush(c: PContext, n: PNode, start: int) =
       x.otherPragmas.add n.sons[i]
     #localError(c.config, n.info, errOptionExpected)
 
+  # If stacktrace is disabled globally we should not enable it
+  if optStackTrace notin c.optionStack[0].options:
+    c.config.options.excl(optStackTrace)
+
 proc processPop(c: PContext, n: PNode) =
   if c.optionStack.len <= 1:
     localError(c.config, n.info, "{.pop.} without a corresponding {.push.}")

From aa7348b3565e9d63bda1c58b806b6d4f9cc522f9 Mon Sep 17 00:00:00 2001
From: data-man <datamanrb@gmail.com>
Date: Thu, 7 Jun 2018 18:39:46 +0300
Subject: [PATCH 47/51] Quote a keys for CritBitTree $ impl. Fixes #7987

---
 lib/pure/collections/critbits.nim | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim
index 5ae5e26b21..71615002e7 100644
--- a/lib/pure/collections/critbits.nim
+++ b/lib/pure/collections/critbits.nim
@@ -322,10 +322,14 @@ proc `$`*[T](c: CritBitTree[T]): string =
       const avgItemLen = 16
     result = newStringOfCap(c.count * avgItemLen)
     result.add("{")
-    for key, val in pairs(c):
-      if result.len > 1: result.add(", ")
-      result.add($key)
-      when T isnot void:
+    when T is void:
+      for key in keys(c):
+        if result.len > 1: result.add(", ")
+        result.addQuoted(key)
+    else:
+      for key, val in pairs(c):
+        if result.len > 1: result.add(", ")
+        result.addQuoted(key)
         result.add(": ")
         result.addQuoted(val)
     result.add("}")

From cc63351a5a44ff5793195b59d961bc93257d879d Mon Sep 17 00:00:00 2001
From: data-man <datamanrb@gmail.com>
Date: Thu, 7 Jun 2018 18:49:59 +0300
Subject: [PATCH 48/51] Updated tests for CritBitTree $

---
 tests/collections/tcollections_to_string.nim | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/collections/tcollections_to_string.nim b/tests/collections/tcollections_to_string.nim
index 48b06a6aa0..0c4f1e91c7 100644
--- a/tests/collections/tcollections_to_string.nim
+++ b/tests/collections/tcollections_to_string.nim
@@ -68,15 +68,15 @@ block:
 block:
   var t: CritBitTree[int]
   t["a"] = 1
-  doAssert $t == "{a: 1}"
+  doAssert $t == """{"a": 1}"""
 block:
   var t: CritBitTree[string]
   t["a"] = "1"
-  doAssert $t == """{a: "1"}"""
+  doAssert $t == """{"a": "1"}"""
 block:
   var t: CritBitTree[char]
   t["a"] = '1'
-  doAssert $t == "{a: '1'}"
+  doAssert $t == """{"a": '1'}"""
 
 
 # Test escaping behavior

From 60b9c9dc1f4a45afe2813abc4234766019862549 Mon Sep 17 00:00:00 2001
From: Yuriy Glukhov <yuriy.glukhov@gmail.com>
Date: Thu, 7 Jun 2018 19:14:14 +0300
Subject: [PATCH 49/51] Fixes #7985

---
 compiler/closureiters.nim | 21 ++++++++++-----------
 tests/async/t7985.nim     | 19 +++++++++++++++++++
 2 files changed, 29 insertions(+), 11 deletions(-)
 create mode 100644 tests/async/t7985.nim

diff --git a/compiler/closureiters.nim b/compiler/closureiters.nim
index e8f4d62c15..5568fd37b9 100644
--- a/compiler/closureiters.nim
+++ b/compiler/closureiters.nim
@@ -397,18 +397,17 @@ proc hasYieldsInExpressions(n: PNode): bool =
 
 proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
   assert(n.kind == nkStmtListExpr)
-
-  var parent = n
-  var lastSon = n[^1]
-
-  while lastSon.kind == nkStmtListExpr:
-    parent = lastSon
-    lastSon = lastSon[^1]
-
   result.s = newNodeI(nkStmtList, n.info)
-  result.s.sons = parent.sons
-  result.s.sons.setLen(result.s.sons.len - 1) # delete last son
-  result.res = lastSon
+  result.s.sons = @[]
+
+  var n = n
+  while n.kind == nkStmtListExpr:
+    result.s.sons.add(n.sons)
+    result.s.sons.setLen(result.s.sons.len - 1) # delete last son
+    n = n[^1]
+
+  result.res = n
+
 
 proc newEnvVarAsgn(ctx: Ctx, s: PSym, v: PNode): PNode =
   result = newTree(nkFastAsgn, ctx.newEnvVarAccess(s), v)
diff --git a/tests/async/t7985.nim b/tests/async/t7985.nim
new file mode 100644
index 0000000000..0365499d3b
--- /dev/null
+++ b/tests/async/t7985.nim
@@ -0,0 +1,19 @@
+discard """
+  file: "t7985.nim"
+  exitcode: 0
+  output: "(value: 1)"
+"""
+import json, asyncdispatch
+
+proc getData(): Future[JsonNode] {.async.} =
+  result = %*{"value": 1}
+
+type
+  MyData = object
+    value: BiggestInt
+
+proc main() {.async.} =
+  let data = to(await(getData()), MyData)
+  echo data
+
+waitFor(main())

From 12f929e5822beeab2e1d60af9b4ef53d8339e11e Mon Sep 17 00:00:00 2001
From: data-man <datamanrb@gmail.com>
Date: Thu, 7 Jun 2018 19:29:40 +0300
Subject: [PATCH 50/51] Fixed bug in CritBitTree.inc. Fixes #7990.

---
 lib/pure/collections/critbits.nim | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/lib/pure/collections/critbits.nim b/lib/pure/collections/critbits.nim
index 71615002e7..eaba257ae8 100644
--- a/lib/pure/collections/critbits.nim
+++ b/lib/pure/collections/critbits.nim
@@ -167,7 +167,7 @@ proc inc*(c: var CritBitTree[int]; key: string, val: int = 1) =
   ## increments `c[key]` by `val`.
   let oldCount = c.count
   var n = rawInsert(c, key)
-  if c.count == oldCount or oldCount == 0:
+  if c.count >= oldCount or oldCount == 0:
     # not a new key:
     inc n.val, val
 
@@ -366,3 +366,12 @@ when isMainModule:
 
   c.inc("a", -5)
   assert c["a"] == 0
+
+  c.inc("b", 2)
+  assert c["b"] == 2
+
+  c.inc("c", 3)
+  assert c["c"] == 3
+
+  c.inc("a", 1)
+  assert c["a"] == 1

From 3c7bbfebb1dea666413f6824ceef53c5badf430c Mon Sep 17 00:00:00 2001
From: Araq <rumpf_a@web.de>
Date: Thu, 7 Jun 2018 21:35:41 +0200
Subject: [PATCH 51/51] fixes seq copying in channels for --gc:regions

---
 lib/system/channels.nim | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/system/channels.nim b/lib/system/channels.nim
index 3c5bda4b11..254b87dfcc 100644
--- a/lib/system/channels.nim
+++ b/lib/system/channels.nim
@@ -116,7 +116,7 @@ proc storeAux(dest, src: pointer, mt: PNimType, t: PRawChannel,
       if mode == mStore:
         x[] = alloc0(t.region, seq.len *% mt.base.size +% GenericSeqSize)
       else:
-        unsureAsgnRef(x, newObj(mt, seq.len * mt.base.size + GenericSeqSize))
+        unsureAsgnRef(x, newSeq(mt, seq.len))
       var dst = cast[ByteAddress](cast[PPointer](dest)[])
       var dstseq = cast[PGenericSeq](dst)
       dstseq.len = seq.len