don't access void* out of alignment in refc GC to avoid UB (#21560)

(cherry picked from commit e8a70ff179)

.github/workflows/ci_docs.yml

@@ -27,7 +27,7 @@ jobs:
           - target: windows
             os: windows-2019
           - target: osx
-            os: macos-10.15
+            os: macos-11
 
     name: ${{ matrix.target }}
     runs-on: ${{ matrix.os }}

azure-pipelines.yml

@@ -21,10 +21,10 @@ jobs:
         vmImage: 'ubuntu-18.04'
         CPU: i386
       OSX_amd64:
-        vmImage: 'macOS-10.15'
+        vmImage: 'macOS-11'
         CPU: amd64
       OSX_amd64_cpp:
-        vmImage: 'macOS-10.15'
+        vmImage: 'macOS-11'
         CPU: amd64
         NIM_COMPILE_TO_CPP: true
       Windows_amd64:

compiler/ccgtypes.nim

@@ -220,7 +220,7 @@ proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool =
   # such a poor programming language.
   # We exclude records with refs too. This enhances efficiency and
   # is necessary for proper code generation of assignments.
-  if rettype == nil or getSize(conf, rettype) > conf.target.floatSize*3:
+  if rettype == nil or (tfByCopy notin rettype.flags and getSize(conf, rettype) > conf.target.floatSize*3):
     result = true
   else:
     case mapType(conf, rettype)

compiler/closureiters.nim

@@ -121,7 +121,10 @@
 #   yield 2
 #   if :unrollFinally: # This node is created by `newEndFinallyNode`
 #     if :curExc.isNil:
-#       return :tmpResult
+#       if nearestFinally == 0:
+#         return :tmpResult
+#       else:
+#         :state = nearestFinally # bubble up
 #     else:
 #       closureIterSetupExc(nil)
 #       raise
@@ -793,7 +796,10 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   # Generate the following code:
   #   if :unrollFinally:
   #       if :curExc.isNil:
-  #         return :tmpResult
+  #         if nearestFinally == 0:
+  #           return :tmpResult
+  #         else:
+  #           :state = nearestFinally # bubble up
   #       else:
   #         raise
   let curExc = ctx.newCurExcAccess()
@@ -802,11 +808,17 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   let cmp = newTree(nkCall, newSymNode(ctx.g.getSysMagic(info, "==", mEqRef), info), curExc, nilnode)
   cmp.typ = ctx.g.getSysType(info, tyBool)
 
-  let asgn = newTree(nkFastAsgn,
-    newSymNode(getClosureIterResult(ctx.g, ctx.fn), info),
-    ctx.newTmpResultAccess())
+  let retStmt =
+    if ctx.nearestFinally == 0:
+      # last finally, we can return
+      let asgn = newTree(nkFastAsgn,
+        newSymNode(getClosureIterResult(ctx.g, ctx.fn), info),
+        ctx.newTmpResultAccess())
+      newTree(nkReturnStmt, asgn)
+    else:
+      # bubble up to next finally
+      newTree(nkGotoState, ctx.g.newIntLit(info, ctx.nearestFinally))
 
-  let retStmt = newTree(nkReturnStmt, asgn)
   let branch = newTree(nkElifBranch, cmp, retStmt)
 
   let nullifyExc = newTree(nkCall, newSymNode(ctx.g.getCompilerProc("closureIterSetupExc")), nilnode)
@@ -850,6 +862,13 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
 
   of nkSkip:
     discard
+  of nkTryStmt:
+    if n.hasYields:
+      # the inner try will handle these transformations
+      discard
+    else:
+      for i in 0..<n.len:
+        n[i] = ctx.transformReturnsInTry(n[i])
   else:
     for i in 0..<n.len:
       n[i] = ctx.transformReturnsInTry(n[i])

compiler/commands.nim

@@ -225,7 +225,7 @@ const
   errNoneBoehmRefcExpectedButXFound = "'none', 'boehm' or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found"
-  errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found"
+  errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found"
 
 proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo): bool =
   case switch.normalize

compiler/renderer.nim

@@ -150,6 +150,7 @@ proc putNL(g: var TSrcGen) =
 proc optNL(g: var TSrcGen, indent: int) =
   g.pendingNL = indent
   g.lineLen = indent
+  g.col = g.indent
   when defined(nimpretty): g.pendingNewlineCount = 0
 
 proc optNL(g: var TSrcGen) =
@@ -158,6 +159,7 @@ proc optNL(g: var TSrcGen) =
 proc optNL(g: var TSrcGen; a, b: PNode) =
   g.pendingNL = g.indent
   g.lineLen = g.indent
+  g.col = g.indent
   when defined(nimpretty): g.pendingNewlineCount = lineDiff(a, b)
 
 proc indentNL(g: var TSrcGen) =

config/config.nims

@@ -3,3 +3,13 @@
 when defined(nimHasCppDefine):
   cppDefine "errno"
   cppDefine "unix"
+
+# mangle the macro names in nimbase.h
+cppDefine "NAN_INFINITY"
+cppDefine "INF"
+cppDefine "NAN"
+
+when defined(windows) and not defined(booting):
+  # Avoid some rare stack corruption while using exceptions with a SEH-enabled
+  # toolchain: https://github.com/nim-lang/Nim/pull/19197
+  switch("define", "nimRawSetjmp")

lib/pure/uri.nim

@@ -163,8 +163,6 @@ proc parseAuthority(authority: string, result: var Uri) =
       inIPv6 = true
     of ']':
       inIPv6 = false
-    of '\0':
-      break
     else:
       if inPort:
         result.port.add(authority[i])

lib/system.nim

@@ -2058,7 +2058,7 @@ const
   NimMinor* {.intdefine.}: int = 2
     ## is the minor number of Nim's version.
 
-  NimPatch* {.intdefine.}: int = 16
+  NimPatch* {.intdefine.}: int = 19
     ## is the patch number of Nim's version.
 
   NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch

lib/system/ansi_c.nim

@@ -113,8 +113,9 @@ elif defined(nimBuiltinSetjmp):
     proc c_builtin_setjmp(jmpb: ptr pointer): cint {.
       importc: "__builtin_setjmp", nodecl.}
     c_builtin_setjmp(unsafeAddr jmpb[0])
+
 elif defined(nimRawSetjmp) and not defined(nimStdSetjmp):
-  when defined(windows):
+  when defined(windows) and not defined(vcc):
     # No `_longjmp()` on Windows.
     proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {.
       header: "<setjmp.h>", importc: "longjmp".}
@@ -124,10 +125,16 @@ elif defined(nimRawSetjmp) and not defined(nimStdSetjmp):
     # prone to stack corruption during unwinding, so we disable that by setting
     # it to NULL.
     # More details: https://github.com/status-im/nimbus-eth2/issues/3121
+    when defined(nimHasStyleChecks):
+      {.push styleChecks: off.}
+
     proc c_setjmp*(jmpb: C_JmpBuf): cint =
       proc c_setjmp_win(jmpb: C_JmpBuf, ctx: pointer): cint {.
         header: "<setjmp.h>", importc: "_setjmp".}
       c_setjmp_win(jmpb, nil)
+
+    when defined(nimHasStyleChecks):
+      {.pop.}
   else:
     proc c_longjmp*(jmpb: C_JmpBuf, retval: cint) {.
       header: "<setjmp.h>", importc: "_longjmp".}

lib/system/gc_common.nim

@@ -381,7 +381,6 @@ else:
             let regEnd = sp +% sizeof(registers)
             while sp <% regEnd:
               gcMark(gch, cast[PPointer](sp)[])
-              gcMark(gch, cast[PPointer](sp +% sizeof(pointer) div 2)[])
               sp = sp +% sizeof(pointer)
         # Make sure sp is word-aligned
         sp = sp and not (sizeof(pointer) - 1)

testament/important_packages.nim

@@ -15,7 +15,7 @@ pkg "bncurve", true
 pkg "c2nim", false, "nim c testsuite/tester.nim"
 pkg "cascade"
 pkg "chroma"
-pkg "chronicles", true, "nim c -o:chr -r chronicles.nim"
+pkg "chronicles", true, "nim c --stylecheck:hint -o:chr -r chronicles.nim"
 pkg "chronos", true, "nim c -r -d:release tests/testall"
 pkg "cligen", false, "nim c -o:cligenn -r cligen.nim"
 pkg "coco", true
@@ -26,8 +26,8 @@ pkg "comprehension", false, "nimble test", "https://github.com/alehander42/compr
 pkg "dashing", false, "nim c tests/functional.nim"
 pkg "docopt"
 pkg "easygl", true, "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl"
-pkg "elvis"
-pkg "fragments", false, "nim c -r fragments/dsl.nim"
+# pkg "elvis"
+# pkg "fragments", false, "nim c -r fragments/dsl.nim"
 pkg "gara"
 # pkg "ggplotnim", true, "nimble testCI"
 pkg "glob"
@@ -68,7 +68,7 @@ pkg "ormin", true, "nim c -o:orminn ormin.nim"
 pkg "patty"
 pkg "plotly", true, "nim c --oldgensym:on examples/all.nim"
 pkg "pnm"
-pkg "polypbren"
+# pkg "polypbren"
 pkg "protobuf", true, "nim c -o:protobuff -r src/protobuf.nim"
 pkg "rbtree"
 pkg "react", false, "nimble example"
@@ -80,10 +80,10 @@ pkg "sdl2_nim", false, "nim c -r sdl2/sdl.nim"
 pkg "snip", false, "nimble test", "https://github.com/genotrance/snip"
 pkg "stint", false, "nim c -o:stintt -r stint.nim"
 pkg "strunicode", true, "nim c -r src/strunicode.nim"
-pkg "telebot", true, "nim c -o:tbot -r src/telebot.nim"
+# pkg "telebot", true, "nim c -o:tbot -r src/telebot.nim"
 pkg "tempdir"
 pkg "tensordsl", false, "nim c -r tests/tests.nim", "https://krux02@bitbucket.org/krux02/tensordslnim.git"
-pkg "tiny_sqlite"
+# pkg "tiny_sqlite"
 pkg "unicodedb"
 pkg "unicodeplus", true
 pkg "unpack"
@@ -91,4 +91,4 @@ pkg "unpack"
 pkg "with"
 # pkg "ws"
 pkg "yaml"
-pkg "zero_functional", false, "nim c -r test.nim"
+# pkg "zero_functional", false, "nim c -r test.nim"

tests/ccgbugs/tmangle.nim

@@ -0,0 +1,16 @@
+block:
+  proc hello() =
+    let NAN_INFINITY = 12
+    doAssert NAN_INFINITY == 12
+    let INF = "2.0"
+    doAssert INF == "2.0"
+    let NAN = 2.3
+    doAssert NAN == 2.3
+
+  hello()
+
+block:
+  proc hello(NAN: float) =
+    doAssert NAN == 2.0
+
+  hello(2.0)

tests/exception/texceptions2.nim

@@ -92,10 +92,10 @@ block:
       result.add(1212)
     try:
       try:
-        raise newException(AssertionDefect, "a")
+        raise newException(AssertionError, "a")
       finally:
         result.add(42)
-    except AssertionDefect:
+    except AssertionError:
       result.add(99)
     finally:
       result.add(10)
@@ -107,7 +107,7 @@ block:
       result.add(-1)
     except ValueError:
       result.add(-1)
-    except IndexDefect:
+    except IndexError:
       result.add(2)
     except:
       result.add(3)

tests/iter/titer_issues.nim

@@ -18,6 +18,14 @@ end
 1
 2
 7
+nested finally
+outer finally
+nested finally
+outer finally
+nested finally
+outer finally
+nested finally
+outer finally
 '''
 """
 
@@ -213,3 +221,71 @@ block t2023_objiter:
 
   var o = init()
   echo(o.iter())
+
+
+block:
+  # bug #19911 (return in nested try)
+
+  # try yield -> try
+  iterator p1: int {.closure.} =
+    try:
+      yield 0
+      try:
+        return
+      finally:
+        echo "nested finally"
+      echo "shouldn't run"
+    finally:
+      echo "outer finally"
+    echo "shouldn't run"
+
+  for _ in p1():
+    discard
+
+  # try -> try yield
+  iterator p2: int {.closure.} =
+    try:
+      try:
+        yield 0
+        return
+      finally:
+        echo "nested finally"
+      echo "shouldn't run"
+    finally:
+      echo "outer finally"
+    echo "shouldn't run"
+
+  for _ in p2():
+    discard
+
+  # try yield -> try yield
+  iterator p3: int {.closure.} =
+    try:
+      yield 0
+      try:
+        yield 0
+        return
+      finally:
+        echo "nested finally"
+      echo "shouldn't run"
+    finally:
+      echo "outer finally"
+    echo "shouldn't run"
+
+  for _ in p3():
+    discard
+
+  # try -> try
+  iterator p4: int {.closure.} =
+    try:
+      try:
+        return
+      finally:
+        echo "nested finally"
+      echo "shouldn't run"
+    finally:
+      echo "outer finally"
+    echo "shouldn't run"
+
+  for _ in p4():
+    discard

tests/macros/tmacros1.nim

@@ -60,3 +60,22 @@ macro foo(t: static Tuple): untyped =
   doAssert t.b == 12345
 
 foo((a: "foo", b: 12345))
+
+
+# bug #16307
+
+macro bug(x: untyped): string =
+  newLit repr(x)
+
+let res = bug:
+  block:
+    ## one
+    ## two
+    ## three
+
+doAssert res == """
+
+block:
+  ## one
+  ## two
+  ## three"""

tests/objects/m19342.c

@@ -0,0 +1,12 @@
+struct Node
+{
+  int data[25];
+};
+
+
+struct Node hello(int name) {
+  struct Node x = {999, 1, 2, 3, 4, 5, 6, 7, 8, 9,
+            0, 1, 2, 3, 4, 5, 6, 7 ,8, 9,
+            1, 2, 3, 4, 5};
+  return x;
+}

tests/objects/t19342.nim

@@ -0,0 +1,18 @@
+discard """
+  targets: "c cpp"
+"""
+
+{.compile: "m19342.c".}
+
+# bug #19342
+type
+  Node* {.bycopy.} = object
+    data: array[25, cint]
+
+proc myproc(name: cint): Node {.importc: "hello", cdecl.}
+
+proc parse =
+  let node = myproc(10)
+  doAssert node.data[0] == 999
+
+parse()