closureiters: fixes #15243 (#15454) [backport:1.2]

* fixes #15243 [backport:1.2]

compiler/ccgexprs.nim

@@ -3050,7 +3050,23 @@ proc genConstSeqV2(p: BProc, n: PNode, t: PType; isConst: bool): Rope =
 proc genBracedInit(p: BProc, n: PNode; isConst: bool): Rope =
   case n.kind
   of nkHiddenStdConv, nkHiddenSubConv:
-    result = genBracedInit(p, n[1], isConst)
+    when false:
+      # XXX The frontend doesn't keep conversions to openArray for us. :-(
+      # We need to change 'transformConv' first, but that is hard.
+      if n.typ.kind == tyOpenArray:
+        assert n[1].kind == nkBracket
+        let data = genBracedInit(p, n[1], isConst)
+
+        let payload = getTempName(p.module)
+        let ctype = getTypeDesc(p.module, n.typ.skipTypes(abstractInst)[0])
+        let arrLen = n[1].len
+        appcg(p.module, cfsData,
+          "static $5 $1 $3[$2] = $4;$n", [
+          ctype, arrLen, payload, data,
+          if isConst: "const" else: ""])
+        result = "{($1*)&$2, $3}" % [ctype, payload, rope arrLen]
+    else:
+      result = genBracedInit(p, n[1], isConst)
   else:
     var ty = tyNone
     if n.typ == nil:

compiler/closureiters.nim

@@ -130,7 +130,8 @@
 
 import
   ast, msgs, idents,
-  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
+  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos,
+  tables, options
 
 type
   Ctx = object
@@ -1103,7 +1104,7 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
 
   result = ctx.states[stateIdx][0].intVal.int
 
-proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
+proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
   result = n
   case n.kind
   of nkSkip:
@@ -1283,6 +1284,101 @@ proc deleteEmptyStates(ctx: var Ctx) =
     else:
       inc i
 
+type
+  PreprocessContext = object
+    finallys: seq[PNode]
+    config: ConfigRef
+    blocks: seq[(PNode, int)]
+  FreshVarsContext = object
+    tab: Table[int, PSym]
+    config: ConfigRef
+    info: TLineInfo
+
+proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
+  case n.kind
+  of nkSym:
+    let x = c.tab.getOrDefault(n.sym.id)
+    if x == nil:
+      result = n
+    else:
+      result = newSymNode(x, n.info)
+  of nkSkip - {nkSym}:
+    result = n
+  of nkLetSection, nkVarSection:
+    result = copyNode(n)
+    for it in n:
+      if it.kind in {nkIdentDefs, nkVarTuple}:
+        let idefs = copyNode(it)
+        for v in 0..it.len-3:
+          if it[v].kind == nkSym:
+            let x = copySym(it[v].sym)
+            c.tab[it[v].sym.id] = x
+            idefs.add newSymNode(x)
+          else:
+            idefs.add it[v]
+
+        for rest in it.len-2 ..< it.len: idefs.add it[rest]
+        result.add idefs
+      else:
+        result.add it
+  of nkRaiseStmt:
+    localError(c.config, c.info, "unsupported control flow: 'finally: ... raise' duplicated because of 'break'")
+  else:
+    result = n
+    for i in 0..<n.safeLen:
+      result[i] = freshVars(n[i], c)
+
+proc preprocess(c: var PreprocessContext; n: PNode): PNode =
+  # in order to fix bug #15243 without risking regressions, we preprocess
+  # the AST so that 'break' statements inside a 'try finally' also have the
+  # finally section. We need to duplicate local variables here and also
+  # detect: 'finally: raises X' which is currently not supported. We produce
+  # an error for this case for now. All this will be done properly with Yuriy's
+  # patch.
+  result = n
+  case n.kind
+  of nkTryStmt:
+    let f = n.lastSon
+    if f.kind == nkFinally:
+      c.finallys.add f.lastSon
+
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
+    if f.kind == nkFinally:
+      discard c.finallys.pop()
+
+  of nkWhileStmt, nkBlockStmt:
+    c.blocks.add((n, c.finallys.len))
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+    discard c.blocks.pop()
+
+  of nkBreakStmt:
+    if c.blocks.len == 0:
+      discard
+    else:
+      var fin = -1
+      if n[0].kind == nkEmpty:
+        fin = c.blocks[^1][1]
+      elif n[0].kind == nkSym:
+        for i in countdown(c.blocks.high, 0):
+          if c.blocks[i][0].kind == nkBlockStmt and c.blocks[i][0][0].kind == nkSym and
+              c.blocks[i][0][0].sym == n[0].sym:
+            fin = c.blocks[i][1]
+            break
+
+      if fin >= 0:
+        result = newNodeI(nkStmtList, n.info)
+        for i in countdown(c.finallys.high, fin):
+          var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info)
+          result.add freshVars(preprocess(c, c.finallys[i]), vars)
+        result.add n
+  of nkSkip: discard
+  else:
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
 proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   var ctx: Ctx
   ctx.g = g
@@ -1295,7 +1391,10 @@ proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
     ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), fn, fn.info)
     ctx.stateVarSym.typ = g.createClosureIterStateType(fn)
   ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), fn, fn.info)
-  var n = n.toStmtList
+  var pc = PreprocessContext(finallys: @[], config: g.config)
+  var n = preprocess(pc, n.toStmtList)
+  #echo "transformed into ", n
+  #var n = n.toStmtList
 
   discard ctx.newState(n, nil)
   let gotoOut = newTree(nkGotoState, g.newIntLit(n.info, -1))

compiler/nim.cfg

@@ -19,6 +19,13 @@ define:useStdoutAsStdmsg
   styleCheck:error
 @end
 
+# die when the Nim compiler uses more than 4GB:
+@if cpu32:
+  define:"nimMaxHeap=2000"
+@else:
+  define:"nimMaxHeap=4000"
+@end
+
 #define:useNodeIds
 #gc:markAndSweep
 

lib/pure/xmltree.nim

@@ -45,7 +45,7 @@ type
 
   XmlNodeKind* = enum ## Different kinds of XML nodes.
     xnText,           ## a text element
-    xnVerbatimText,   ## 
+    xnVerbatimText,   ##
     xnElement,        ## an element with 0 or more children
     xnCData,          ## a CDATA node
     xnEntity,         ## an entity (like ``&thing;``)
@@ -103,7 +103,7 @@ proc newText*(text: string): XmlNode =
   result = newXmlNode(xnText)
   result.fText = text
 
-proc newVerbatimText*(text: string): XmlNode {.since: (1, 3).} = 
+proc newVerbatimText*(text: string): XmlNode {.since: (1, 3).} =
   ## Creates a new ``XmlNode`` of kind ``xnVerbatimText`` with the text `text`.
   ## **Since**: Version 1.3.
   result = newXmlNode(xnVerbatimText)

tests/async/tasyncawait_cyclebreaker.nim

@@ -1,66 +0,0 @@
-discard """
-  output: "20000"
-  cmd: "nim c -d:nimTypeNames -d:nimCycleBreaker $file"
-"""
-import asyncdispatch, asyncnet, nativesockets, net, strutils
-from stdtest/netutils import bindAvailablePort
-
-var msgCount = 0
-
-const
-  swarmSize = 400
-  messagesToSend = 50
-
-var clientCount = 0
-
-proc sendMessages(client: AsyncFD) {.async.} =
-  for i in 0 ..< messagesToSend:
-    await send(client, "Message " & $i & "\c\L")
-
-proc launchSwarm(port: Port) {.async.} =
-  for i in 0 ..< swarmSize:
-    var sock = createAsyncNativeSocket()
-
-    await connect(sock, "localhost", port)
-    await sendMessages(sock)
-    closeSocket(sock)
-
-proc readMessages(client: AsyncFD) {.async.} =
-  # wrapping the AsyncFd into a AsyncSocket object
-  var sockObj = newAsyncSocket(client)
-  var (ipaddr, port) = sockObj.getPeerAddr()
-  doAssert ipaddr == "127.0.0.1"
-  (ipaddr, port) = sockObj.getLocalAddr()
-  doAssert ipaddr == "127.0.0.1"
-  while true:
-    var line = await recvLine(sockObj)
-    if line == "":
-      closeSocket(client)
-      clientCount.inc
-      break
-    else:
-      if line.startswith("Message "):
-        msgCount.inc
-      else:
-        doAssert false
-
-proc createServer(server: AsyncFD) {.async.} =
-  discard server.SocketHandle.listen()
-  while true:
-    asyncCheck readMessages(await accept(server))
-
-let server = createAsyncNativeSocket()
-let port = bindAvailablePort(server.SocketHandle)
-asyncCheck createServer(server)
-asyncCheck launchSwarm(port)
-while true:
-  poll()
-  GC_collectZct()
-
-  let (allocs, deallocs) = getMemCounters()
-  doAssert allocs < deallocs + 1000
-
-  if clientCount == swarmSize: break
-
-assert msgCount == swarmSize * messagesToSend
-echo msgCount

tests/async/tbreak_must_exec_finally.nim

@@ -0,0 +1,25 @@
+discard """
+  output: '''
+finally handler 8
+do not duplicate this one
+'''
+"""
+
+# bug #15243
+
+import asyncdispatch
+
+proc f() {.async.} =
+  try:
+    while true:
+      try:
+        await sleepAsync(400)
+        break
+      finally:
+        var localHere = 8
+        echo "finally handler ", localHere
+  finally:
+    echo "do not duplicate this one"
+
+when isMainModule:
+  waitFor f()