bump NimVersion to 1.6.10

* fix =#13790 ptr char (+friends) should not implicitly convert to cstring

* Apply suggestions from code review

* first round; compiles on windows

* nimPreviewSlimSystem

* conversion is unsafe, cast needed

* fixes more tests

* fixes asyncnet

* another try another error

* last one

* true

* one more

* why bugs didn't show at once

* add `nimPreviewCstringConversion` switch

* typo

* fixes ptr to cstring warnings[backport]

* add fixes

Co-authored-by: xflywind <43030857+xflywind@users.noreply.github.com>
(cherry picked from commit 06cd15663d)

.github/workflows/ci_docs.yml

@@ -40,7 +40,7 @@ jobs:
           - target: windows
             os: windows-2019
           - target: osx
-            os: macos-10.15
+            os: macos-11
 
     name: ${{ matrix.target }}
     runs-on: ${{ matrix.os }}

.github/workflows/ci_packages.yml

@@ -6,7 +6,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-20.04, macos-10.15]
+        os: [ubuntu-20.04, macos-11]
         cpu: [amd64]
         batch: ["allowed_failures", "0_3", "1_3", "2_3"] # list of `index_num`
     name: '${{ matrix.os }} (batch: ${{ matrix.batch }})'

azure-pipelines.yml

@@ -28,10 +28,10 @@ jobs:
         vmImage: 'ubuntu-18.04'
         CPU: i386
       OSX_amd64:
-        vmImage: 'macOS-10.15'
+        vmImage: 'macOS-11'
         CPU: amd64
       OSX_amd64_cpp:
-        vmImage: 'macOS-10.15'
+        vmImage: 'macOS-11'
         CPU: amd64
         NIM_COMPILE_TO_CPP: true
       Windows_amd64_batch0_3:

build_all.bat

@@ -24,6 +24,6 @@ if not exist %nim_csources% (
   cd ..
   copy /y bin\nim.exe  %nim_csources%
 )
- bin\nim.exe c --skipUserCfg --skipParentCfg --hints:off koch
+ bin\nim.exe c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch
  koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
  koch tools --skipUserCfg --skipParentCfg --hints:off

build_all.sh

@@ -11,7 +11,7 @@ set -e # exit on first error
 . ci/funs.sh
 nimBuildCsourcesIfNeeded "$@"
 
-echo_run bin/nim c --skipUserCfg --skipParentCfg --hints:off koch
+echo_run bin/nim c --noNimblePath --skipUserCfg --skipParentCfg --hints:off koch
 echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg --hints:off
 echo_run ./koch tools --skipUserCfg --skipParentCfg --hints:off
 

changelog.md

@@ -3,14 +3,33 @@
 
 ## Changes affecting backward compatibility
 
+- `addr` is now available for all addressable locations,
+  `unsafeAddr` is now deprecated and an alias for `addr`.
+
+- `io`, `assertions`, `formatfloat`, and `` dollars.`$` `` for objects are about to move out of the `system` module. You may instead import `std/syncio`, `std/assertions`, `std/formatfloat` and `std/objectdollar`.
+  The `-d:nimPreviewSlimSystem` option makes these imports required.
+
+- The `gc:v2` option is removed.
+
+- The `mainmodule` and `m` options are removed.
+
+- The `threads:on` option is now the default.
 
 - Optional parameters in combination with `: body` syntax (RFC #405) are now opt-in via
   `experimental:flexibleOptionalParams`.
 
 ## Standard library additions and changes
 
+- Pointer to `cstring` conversion now triggers a `[PtrToCstringConv]` warning.
+  This warning will become an error in future versions! Use a `cast` operation
+  like `cast[cstring](x)` instead.
+
 ## Standard library additions and changes
 
+- `macros.parseExpr` and `macros.parseStmt` now accept an optional
+  filename argument for more informative errors.
+- Module `colors` expanded with missing colors from the CSS color standard.
+- Fixed `lists.SinglyLinkedList` being broken after removing the last node ([#19353](https://github.com/nim-lang/Nim/pull/19353)).
 
 
 ## Language changes
@@ -54,6 +73,8 @@
 - `nim` can now compile version 1.4.0 as follows: `nim c --lib:lib --stylecheck:off compiler/nim`,
   without requiring `-d:nimVersion140` which is now a noop.
 
+- `--styleCheck` now only applies to the current package.
+
 
 ## Tool changes
 

compiler/ast.nim

@@ -501,7 +501,7 @@ type
     nfHasComment # node has a comment
 
   TNodeFlags* = set[TNodeFlag]
-  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 43)
+  TTypeFlag* = enum   # keep below 32 for efficiency reasons (now: 45)
     tfVarargs,        # procedure has C styled varargs
                       # tyArray type represeting a varargs list
     tfNoSideEffect,   # procedure type does not allow side effects
@@ -1105,21 +1105,6 @@ proc getPIdent*(a: PNode): PIdent {.inline.} =
   of nkIdent: a.ident
   else: nil
 
-proc getnimblePkg*(a: PSym): PSym =
-  result = a
-  while result != nil:
-    case result.kind
-    of skModule:
-      result = result.owner
-      assert result.kind == skPackage
-    of skPackage:
-      if result.owner == nil:
-        break
-      else:
-        result = result.owner
-    else:
-      assert false, $result.kind
-
 const
   moduleShift = when defined(cpu32): 20 else: 24
 
@@ -1164,13 +1149,7 @@ when false:
     assert dest.ItemId.item <= src.ItemId.item
     dest = src
 
-proc getnimblePkgId*(a: PSym): int =
-  let b = a.getnimblePkg
-  result = if b == nil: -1 else: b.id
-
 var ggDebug* {.deprecated.}: bool ## convenience switch for trying out things
-#var
-#  gMainPackageId*: int
 
 proc isCallExpr*(n: PNode): bool =
   result = n.kind in nkCallKinds

compiler/ccgcalls.nim

@@ -21,7 +21,7 @@ proc canRaiseDisp(p: BProc; n: PNode): bool =
     # we have to be *very* conservative:
     result = canRaiseConservative(n)
 
-proc preventNrvo(p: BProc; le, ri: PNode): bool =
+proc preventNrvo(p: BProc; dest, le, ri: PNode): bool =
   proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
     var n = le
     while true:
@@ -54,6 +54,11 @@ proc preventNrvo(p: BProc; le, ri: PNode): bool =
     if canRaise(ri[0]) and
         locationEscapes(p, le, p.nestedTryStmts.len > 0):
       message(p.config, le.info, warnObservableStores, $le)
+  # bug #19613 prevent dangerous aliasing too:
+  if dest != nil and dest != le:
+    for i in 1..<ri.len:
+      let r = ri[i]
+      if isPartOf(dest, r) != arNo: return true
 
 proc hasNoInit(call: PNode): bool {.inline.} =
   result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
@@ -76,10 +81,10 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
   # getUniqueType() is too expensive here:
   var typ = skipTypes(ri[0].typ, abstractInst)
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ[0]):
+    if isInvalidReturnType(p.config, typ):
       if params != nil: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, d.lode, le, ri):
         # Great, we can use 'd':
         if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
@@ -187,10 +192,12 @@ proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope,
         optSeqDestructors in p.config.globalOptions:
       linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
     if atyp.kind in {tyVar} and not compileToCpp(p.module):
-      result = ("($4*)(*$1)$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
+      result = ("(($5) ? (($4*)(*$1)$3+($2)) : NIM_NIL)" %
+                  [rdLoc(a), rdLoc(b), dataField(p), dest, dataFieldAccessor(p, "*" & rdLoc(a))],
                 lengthExpr)
     else:
-      result = ("($4*)$1$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
+      result = ("(($5) ? (($4*)$1$3+($2)) : NIM_NIL)" %
+                  [rdLoc(a), rdLoc(b), dataField(p), dest, dataFieldAccessor(p, rdLoc(a))],
                 lengthExpr)
   else:
     internalError(p.config, "openArrayLoc: " & typeToString(a.t))
@@ -231,9 +238,12 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
       if ntyp.kind in {tyVar} and not compileToCpp(p.module):
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
-        result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
+        result = "($4) ? ((*$1)$3) : NIM_NIL, $2" %
+                     [a.rdLoc, lenExpr(p, t), dataField(p),
+                      dataFieldAccessor(p, "*" & a.rdLoc)]
       else:
-        result = "$1$3, $2" % [a.rdLoc, lenExpr(p, a), dataField(p)]
+        result = "($4) ? ($1$3) : NIM_NIL, $2" %
+                     [a.rdLoc, lenExpr(p, a), dataField(p), dataFieldAccessor(p, a.rdLoc)]
     of tyArray:
       result = "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, a.t))]
     of tyPtr, tyRef:
@@ -241,7 +251,9 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
       of tyString, tySequence:
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
-        result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
+        result = "($4) ? ((*$1)$3) : NIM_NIL, $2" %
+                     [a.rdLoc, lenExpr(p, t), dataField(p),
+                      dataFieldAccessor(p, "*" & a.rdLoc)]
       of tyArray:
         result = "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, lastSon(a.t)))]
       else:
@@ -376,8 +388,8 @@ proc genParams(p: BProc, ri: PNode, typ: PType): Rope =
         if not needTmp[i - 1]:
           needTmp[i - 1] = potentialAlias(n, potentialWrites)
       getPotentialWrites(ri[i], false, potentialWrites)
-    if ri[i].kind == nkHiddenAddr:
-      # Optimization: don't use a temp, if we would only take the adress anyway
+    if ri[i].kind in {nkHiddenAddr, nkAddr}:
+      # Optimization: don't use a temp, if we would only take the address anyway
       needTmp[i - 1] = false
 
   for i in 1..<ri.len:
@@ -439,10 +451,10 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
   let rawProc = getClosureType(p.module, typ, clHalf)
   let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ[0]):
+    if isInvalidReturnType(p.config, typ):
       if ri.len > 1: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, d.lode, le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
           getTemp(p, typ[0], d, needsInit=true)
@@ -737,7 +749,7 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
     pl.add(~": ")
     pl.add(genArg(p, ri[i], param, ri))
   if typ[0] != nil:
-    if isInvalidReturnType(p.config, typ[0]):
+    if isInvalidReturnType(p.config, typ):
       if ri.len > 1: pl.add(~" ")
       # beware of 'result = p(result)'. We always allocate a temporary:
       if d.k in {locTemp, locNone}:

compiler/ccgexprs.nim

@@ -292,8 +292,8 @@ proc genOpenArrayConv(p: BProc; d: TLoc; a: TLoc) =
       linefmt(p, cpsStmts, "$1.Field0 = $2; $1.Field1 = $2Len_0;$n",
         [rdLoc(d), a.rdLoc])
   of tySequence:
-    linefmt(p, cpsStmts, "$1.Field0 = $2$3; $1.Field1 = $4;$n",
-      [rdLoc(d), a.rdLoc, dataField(p), lenExpr(p, a)])
+    linefmt(p, cpsStmts, "$1.Field0 = ($5) ? ($2$3) : NIM_NIL; $1.Field1 = $4;$n",
+      [rdLoc(d), a.rdLoc, dataField(p), lenExpr(p, a), dataFieldAccessor(p, a.rdLoc)])
   of tyArray:
     linefmt(p, cpsStmts, "$1.Field0 = $2; $1.Field1 = $3;$n",
       [rdLoc(d), rdLoc(a), rope(lengthOrd(p.config, a.t))])
@@ -302,8 +302,8 @@ proc genOpenArrayConv(p: BProc; d: TLoc; a: TLoc) =
     if etyp.kind in {tyVar} and optSeqDestructors in p.config.globalOptions:
       linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
 
-    linefmt(p, cpsStmts, "$1.Field0 = $2$3; $1.Field1 = $4;$n",
-      [rdLoc(d), a.rdLoc, dataField(p), lenExpr(p, a)])
+    linefmt(p, cpsStmts, "$1.Field0 = ($5) ? ($2$3) : NIM_NIL; $1.Field1 = $4;$n",
+      [rdLoc(d), a.rdLoc, dataField(p), lenExpr(p, a), dataFieldAccessor(p, a.rdLoc)])
   else:
     internalError(p.config, a.lode.info, "cannot handle " & $a.t.kind)
 
@@ -584,13 +584,23 @@ proc binaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
   else:
     # we handle div by zero here so that we know that the compilerproc's
     # result is only for overflows.
+    var needsOverflowCheck = true
     if m in {mDivI, mModI}:
-      linefmt(p, cpsStmts, "if ($1 == 0){ #raiseDivByZero(); $2}$n",
-              [rdLoc(b), raiseInstr(p)])
-
-    let res = binaryArithOverflowRaw(p, t, a, b,
-      if t.kind == tyInt64: prc64[m] else: prc[m])
-    putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res])
+      var canBeZero = true
+      if e[2].kind in {nkIntLit..nkUInt64Lit}:
+        canBeZero = e[2].intVal == 0
+      if e[2].kind in {nkIntLit..nkInt64Lit}:
+        needsOverflowCheck = e[2].intVal == -1
+      if canBeZero:
+        linefmt(p, cpsStmts, "if ($1 == 0){ #raiseDivByZero(); $2}$n",
+                [rdLoc(b), raiseInstr(p)])
+    if needsOverflowCheck:
+      let res = binaryArithOverflowRaw(p, t, a, b,
+        if t.kind == tyInt64: prc64[m] else: prc[m])
+      putIntoDest(p, d, e, "($#)($#)" % [getTypeDesc(p.module, e.typ), res])
+    else:
+      let res = "($1)($2 $3 $4)" % [getTypeDesc(p.module, e.typ), rdLoc(a), rope(opr[m]), rdLoc(b)]
+      putIntoDest(p, d, e, res)
 
 proc unaryArithOverflow(p: BProc, e: PNode, d: var TLoc, m: TMagic) =
   var
@@ -721,7 +731,22 @@ proc isCppRef(p: BProc; typ: PType): bool {.inline.} =
       skipTypes(typ, abstractInstOwned).kind in {tyVar} and
       tfVarIsPtr notin skipTypes(typ, abstractInstOwned).flags
 
+proc derefBlock(p: BProc, e: PNode, d: var TLoc) =
+  # We transform (block: x)[] to (block: x[])
+  let e0 = e[0]
+  var n = shallowCopy(e0)
+  n.typ = e.typ
+  for i in 0 ..< e0.len - 1:
+    n[i] = e0[i]
+  n[e0.len-1] = newTreeIT(nkHiddenDeref, e.info, e.typ, e0[e0.len-1])
+  expr p, n, d
+
 proc genDeref(p: BProc, e: PNode, d: var TLoc) =
+  if e.kind == nkHiddenDeref and e[0].kind in {nkBlockExpr, nkBlockStmt}:
+    # bug #20107. Watch out to not deref the pointer too late.
+    derefBlock(p, e, d)
+    return
+
   let mt = mapType(p.config, e[0].typ, mapTypeChooser(e[0]))
   if mt in {ctArray, ctPtrToArray} and lfEnforceDeref notin d.flags:
     # XXX the amount of hacks for C's arrays is incredible, maybe we should
@@ -978,12 +1003,12 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) =
     if reifiedOpenArray(arr.lode):
       linefmt(p, cpsStmts,
         "if ($2-$1 != -1 && " &
-        "((NU)($1) >= (NU)($3.Field1) || (NU)($2) >= (NU)($3.Field1))){ #raiseIndexError(); $4}$n",
+        "($1 < 0 || $1 >= $3.Field1 || $2 < 0 || $2 >= $3.Field1)){ #raiseIndexError(); $4}$n",
         [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)])
     else:
       linefmt(p, cpsStmts,
-        "if ($2-$1 != -1 && " &
-        "((NU)($1) >= (NU)($3Len_0) || (NU)($2) >= (NU)($3Len_0))){ #raiseIndexError(); $4}$n",
+        "if ($2-$1 != -1 && ($1 < 0 || $1 >= $3Len_0 || $2 < 0 || $2 >= $3Len_0))" &
+        "{ #raiseIndexError(); $4}$n",
         [rdLoc(a), rdLoc(b), rdLoc(arr), raiseInstr(p)])
   of tyArray:
     let first = intLiteral(firstOrd(p.config, ty))
@@ -994,7 +1019,7 @@ proc genBoundsCheck(p: BProc; arr, a, b: TLoc) =
   of tySequence, tyString:
     linefmt(p, cpsStmts,
       "if ($2-$1 != -1 && " &
-      "((NU)($1) >= (NU)$3 || (NU)($2) >= (NU)$3)){ #raiseIndexError(); $4}$n",
+      "($1 < 0 || $1 >= $3 || $2 < 0 || $2 >= $3)){ #raiseIndexError(); $4}$n",
       [rdLoc(a), rdLoc(b), lenExpr(p, arr), raiseInstr(p)])
   else: discard
 
@@ -1005,14 +1030,14 @@ proc genOpenArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   if not reifiedOpenArray(x):
     # emit range check:
     if optBoundsCheck in p.options:
-      linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2Len_0)){ #raiseIndexError2($1,$2Len_0-1); $3}$n",
+      linefmt(p, cpsStmts, "if ($1 < 0 || $1 >= $2Len_0){ #raiseIndexError2($1,$2Len_0-1); $3}$n",
               [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``!
     inheritLocation(d, a)
     putIntoDest(p, d, n,
                 ropecg(p.module, "$1[$2]", [rdLoc(a), rdCharLoc(b)]), a.storage)
   else:
     if optBoundsCheck in p.options:
-      linefmt(p, cpsStmts, "if ((NU)($1) >= (NU)($2.Field1)){ #raiseIndexError2($1,$2.Field1-1); $3}$n",
+      linefmt(p, cpsStmts, "if ($1 < 0 || $1 >= $2.Field1){ #raiseIndexError2($1,$2.Field1-1); $3}$n",
               [rdCharLoc(b), rdLoc(a), raiseInstr(p)]) # BUGFIX: ``>=`` and not ``>``!
     inheritLocation(d, a)
     putIntoDest(p, d, n,
@@ -1027,7 +1052,7 @@ proc genSeqElem(p: BProc, n, x, y: PNode, d: var TLoc) =
     ty = skipTypes(ty.lastSon, abstractVarRange) # emit range check:
   if optBoundsCheck in p.options:
     linefmt(p, cpsStmts,
-            "if ((NU)($1) >= (NU)$2){ #raiseIndexError2($1,$2-1); $3}$n",
+            "if ($1 < 0 || $1 >= $2){ #raiseIndexError2($1,$2-1); $3}$n",
             [rdCharLoc(b), lenExpr(p, a), raiseInstr(p)])
   if d.k == locNone: d.storage = OnHeap
   if skipTypes(a.t, abstractVar).kind in {tyRef, tyPtr}:
@@ -1605,8 +1630,11 @@ proc genNewFinalize(p: BProc, e: PNode) =
 
 proc genOfHelper(p: BProc; dest: PType; a: Rope; info: TLineInfo): Rope =
   if optTinyRtti in p.config.globalOptions:
-    result = ropecg(p.module, "#isObj($1.m_type, $2)",
-      [a, genTypeInfo2Name(p.module, dest)])
+    let ti = genTypeInfo2Name(p.module, dest)
+    inc p.module.labels
+    let cache = "Nim_OfCheck_CACHE" & p.module.labels.rope
+    p.module.s[cfsVars].addf("static TNimTypeV2* $#[2];$n", [cache])
+    result = ropecg(p.module, "#isObjWithCache($#.m_type, $#, $#)", [a, ti, cache])
   else:
     # unfortunately 'genTypeInfoV1' sets tfObjHasKids as a side effect, so we
     # have to call it here first:
@@ -1686,7 +1714,9 @@ proc genRepr(p: BProc, e: PNode, d: var TLoc) =
       putIntoDest(p, b, e, "$1, $1Len_0" % [rdLoc(a)], a.storage)
     of tyString, tySequence:
       putIntoDest(p, b, e,
-                  "$1$3, $2" % [rdLoc(a), lenExpr(p, a), dataField(p)], a.storage)
+                  "($4) ? ($1$3) : NIM_NIL, $2" %
+                    [rdLoc(a), lenExpr(p, a), dataField(p), dataFieldAccessor(p, a.rdLoc)],
+                  a.storage)
     of tyArray:
       putIntoDest(p, b, e,
                   "$1, $2" % [rdLoc(a), rope(lengthOrd(p.config, a.t))], a.storage)
@@ -2064,6 +2094,11 @@ proc genSomeCast(p: BProc, e: PNode, d: var TLoc) =
     elif etyp.kind == tyBool and srcTyp.kind in IntegralTypes:
       putIntoDest(p, d, e, "(($1) != 0)" % [rdCharLoc(a)], a.storage)
     else:
+      if etyp.kind == tyPtr:
+        # generates the definition of structs for casts like cast[ptr object](addr x)[]
+        let internalType = etyp.skipTypes({tyPtr})
+        if internalType.kind == tyObject:
+          discard getTypeDesc(p.module, internalType)
       putIntoDest(p, d, e, "(($1) ($2))" %
           [getTypeDesc(p.module, e.typ), rdCharLoc(a)], a.storage)
 

compiler/ccgreset.nim

@@ -87,7 +87,20 @@ proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
     lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
   of tyCstring, tyPointer, tyPtr, tyVar, tyLent:
     lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
-  else:
+  of tySet:
+    case mapSetType(p.config, typ)
+    of ctArray:
+      lineCg(p, cpsStmts, "#nimZeroMem($1, sizeof($2));$n",
+          [accessor, getTypeDesc(p.module, typ)])
+    of ctInt8, ctInt16, ctInt32, ctInt64:
+      lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
+    else:
+      doAssert false, "unexpected set type kind"
+  of {tyNone, tyEmpty, tyNil, tyUntyped, tyTyped, tyGenericInvocation,
+      tyGenericParam, tyOrdinal, tyRange, tyOpenArray, tyForward, tyVarargs,
+      tyUncheckedArray, tyProxy, tyBuiltInTypeClass, tyUserTypeClass,
+      tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot,
+      tyAnything, tyStatic, tyFromExpr, tyConcept, tyVoid, tyIterable}:
     discard
 
 proc specializeReset(p: BProc, a: TLoc) =

compiler/ccgstmts.nim

@@ -32,13 +32,20 @@ proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) =
       "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc])
 
 proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} =
-  if n.kind == nkEmpty: return false
-  if isInvalidReturnType(conf, n.typ):
-    # var v = f()
-    # is transformed into: var v;  f(addr v)
-    # where 'f' **does not** initialize the result!
-    return false
-  result = true
+  if n.kind == nkEmpty:
+    result = false
+  elif n.kind in nkCallKinds and n[0] != nil and n[0].typ != nil and n[0].typ.skipTypes(abstractInst).kind == tyProc:
+    if isInvalidReturnType(conf, n[0].typ, true):
+      # var v = f()
+      # is transformed into: var v;  f(addr v)
+      # where 'f' **does not** initialize the result!
+      result = false
+    else:
+      result = true
+  elif isInvalidReturnType(conf, n.typ, false):
+    result = false
+  else:
+    result = true
 
 proc inExceptBlockLen(p: BProc): int =
   for x in p.nestedTryStmts:
@@ -1360,13 +1367,18 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
       linefmt(p, cpsStmts, "$1.status = __builtin_setjmp($1.context);$n", [safePoint])
     elif isDefined(p.config, "nimRawSetjmp"):
       if isDefined(p.config, "mswindows"):
-        # The Windows `_setjmp()` takes two arguments, with the second being an
-        # undocumented buffer used by the SEH mechanism for stack unwinding.
-        # Mingw-w64 has been trying to get it right for years, but it's still
-        # prone to stack corruption during unwinding, so we disable that by setting
-        # it to NULL.
-        # More details: https://github.com/status-im/nimbus-eth2/issues/3121
-        linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint])
+        if isDefined(p.config, "vcc") or isDefined(p.config, "clangcl"):
+          # For the vcc compiler, use `setjmp()` with one argument.
+          # See https://docs.microsoft.com/en-us/cpp/c-runtime-library/reference/setjmp?view=msvc-170
+          linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])        
+        else:
+          # The Windows `_setjmp()` takes two arguments, with the second being an
+          # undocumented buffer used by the SEH mechanism for stack unwinding.
+          # Mingw-w64 has been trying to get it right for years, but it's still
+          # prone to stack corruption during unwinding, so we disable that by setting
+          # it to NULL.
+          # More details: https://github.com/status-im/nimbus-eth2/issues/3121
+          linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint])
       else:
         linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint])
     else:

compiler/ccgtypes.nim

@@ -215,12 +215,18 @@ proc isObjLackingTypeField(typ: PType): bool {.inline.} =
   result = (typ.kind == tyObject) and ((tfFinal in typ.flags) and
       (typ[0] == nil) or isPureObject(typ))
 
-proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool =
+proc isInvalidReturnType(conf: ConfigRef; typ: PType, isProc = true): bool =
   # Arrays and sets cannot be returned by a C procedure, because C is
   # such a poor programming language.
   # We exclude records with refs too. This enhances efficiency and
   # is necessary for proper code generation of assignments.
-  if rettype == nil or getSize(conf, rettype) > conf.target.floatSize*3:
+  var rettype = typ
+  var isAllowedCall = true
+  if isProc:
+    rettype = rettype[0]
+    isAllowedCall = typ.callConv in {ccClosure, ccInline, ccNimCall}
+  if rettype == nil or (isAllowedCall and
+                    getSize(conf, rettype) > conf.target.floatSize*3):
     result = true
   else:
     case mapType(conf, rettype, skResult)
@@ -257,11 +263,11 @@ proc addAbiCheck(m: BModule, t: PType, name: Rope) =
     # see `testCodegenABICheck` for example error message it generates
 
 
-proc fillResult(conf: ConfigRef; param: PNode) =
+proc fillResult(conf: ConfigRef; param: PNode, proctype: PType) =
   fillLoc(param.sym.loc, locParam, param, ~"Result",
           OnStack)
   let t = param.sym.typ
-  if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, t):
+  if mapReturnType(conf, t) != ctArray and isInvalidReturnType(conf, proctype):
     incl(param.sym.loc.flags, lfIndirect)
     param.sym.loc.storage = OnUnknown
 
@@ -426,7 +432,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
                    check: var IntSet, declareEnvironment=true;
                    weakDep=false) =
   params = nil
-  if t[0] == nil or isInvalidReturnType(m.config, t[0]):
+  if t[0] == nil or isInvalidReturnType(m.config, t):
     rettype = ~"void"
   else:
     rettype = getTypeDescAux(m, t[0], check, skResult)
@@ -461,12 +467,17 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
       params.addf(", NI $1Len_$2", [param.loc.r, j.rope])
       inc(j)
       arr = arr[0].skipTypes({tySink})
-  if t[0] != nil and isInvalidReturnType(m.config, t[0]):
+  if t[0] != nil and isInvalidReturnType(m.config, t):
     var arr = t[0]
     if params != nil: params.add(", ")
     if mapReturnType(m.config, t[0]) != ctArray:
-      params.add(getTypeDescWeak(m, arr, check, skResult))
-      params.add("*")
+      if isHeaderFile in m.flags:
+        # still generates types for `--header`
+        params.add(getTypeDescAux(m, arr, check, skResult))
+        params.add("*")
+      else:
+        params.add(getTypeDescWeak(m, arr, check, skResult))
+        params.add("*")
     else:
       params.add(getTypeDescAux(m, arr, check, skResult))
     params.addf(" Result", [])
@@ -1267,7 +1278,7 @@ proc genTypeInfo2Name(m: BModule; t: PType): Rope =
   var it = t
   while it != nil:
     it = it.skipTypes(skipPtrs)
-    if it.sym != nil:
+    if it.sym != nil and tfFromGeneric notin it.flags:
       var m = it.sym.owner
       while m != nil and m.kind != skModule: m = m.owner
       if m == nil or sfSystemModule in m.flags:

compiler/cgen.nim

@@ -15,7 +15,7 @@ import
   ccgutils, os, ropes, math, passes, wordrecg, treetab, cgmeth,
   rodutils, renderer, cgendata, aliases,
   lowerings, tables, sets, ndi, lineinfos, pathutils, transf,
-  injectdestructors, astmsgs
+  injectdestructors, astmsgs, modulepaths
 
 when not defined(leanCompiler):
   import spawn, semparallel
@@ -48,8 +48,13 @@ proc addForwardedProc(m: BModule, prc: PSym) =
   m.g.forwardedProcs.add(prc)
 
 proc findPendingModule(m: BModule, s: PSym): BModule =
-  let ms = s.itemId.module  #getModule(s)
-  result = m.g.modules[ms]
+  # TODO fixme
+  if m.config.symbolFiles == v2Sf:
+    let ms = s.itemId.module  #getModule(s)
+    result = m.g.modules[ms]
+  else:
+    var ms = getModule(s)
+    result = m.g.modules[ms.position]
 
 proc initLoc(result: var TLoc, k: TLocKind, lode: PNode, s: TStorageLoc) =
   result.k = k
@@ -302,6 +307,12 @@ proc lenExpr(p: BProc; a: TLoc): Rope =
   else:
     result = "($1 ? $1->$2 : 0)" % [rdLoc(a), lenField(p)]
 
+proc dataFieldAccessor(p: BProc, sym: Rope): Rope =
+  if optSeqDestructors in p.config.globalOptions:
+    result = "(" & sym & ").p"
+  else:
+    result = sym
+
 proc dataField(p: BProc): Rope =
   if optSeqDestructors in p.config.globalOptions:
     result = rope".p->data"
@@ -876,7 +887,7 @@ proc containsResult(n: PNode): bool =
       if containsResult(n[i]): return true
 
 const harmless = {nkConstSection, nkTypeSection, nkEmpty, nkCommentStmt, nkTemplateDef,
-                  nkMacroDef, nkMixinStmt, nkBindStmt} +
+                  nkMacroDef, nkMixinStmt, nkBindStmt, nkFormalParams} +
                   declarativeDefs
 
 proc easyResultAsgn(n: PNode): PNode =
@@ -1034,7 +1045,7 @@ proc genProcAux(m: BModule, prc: PSym) =
       internalError(m.config, prc.info, "proc has no result symbol")
     let resNode = prc.ast[resultPos]
     let res = resNode.sym # get result symbol
-    if not isInvalidReturnType(m.config, prc.typ[0]):
+    if not isInvalidReturnType(m.config, prc.typ):
       if sfNoInit in prc.flags: incl(res.flags, sfNoInit)
       if sfNoInit in prc.flags and p.module.compileToCpp and (let val = easyResultAsgn(procBody); val != nil):
         var decl = localVarDecl(p, resNode)
@@ -1048,7 +1059,7 @@ proc genProcAux(m: BModule, prc: PSym) =
         initLocalVar(p, res, immediateAsgn=false)
       returnStmt = ropecg(p.module, "\treturn $1;$n", [rdLoc(res.loc)])
     else:
-      fillResult(p.config, resNode)
+      fillResult(p.config, resNode, prc.typ)
       assignParam(p, res, prc.typ[0])
       # We simplify 'unsureAsgn(result, nil); unsureAsgn(result, x)'
       # to 'unsureAsgn(result, x)'
@@ -1297,17 +1308,19 @@ proc getFileHeader(conf: ConfigRef; cfile: Cfile): Rope =
   if conf.hcrOn: result.add("#define NIM_HOT_CODE_RELOADING\L")
   addNimDefines(result, conf)
 
-proc getSomeNameForModule(m: PSym): Rope =
-  assert m.kind == skModule
-  assert m.owner.kind == skPackage
-  if {sfSystemModule, sfMainModule} * m.flags == {}:
-    result = m.owner.name.s.mangle.rope
-    result.add "_"
-  result.add m.name.s.mangle
+proc getSomeNameForModule(conf: ConfigRef, filename: AbsoluteFile): Rope =
+  ## Returns a mangled module name.
+  result.add mangleModuleName(conf, filename).mangle
+
+proc getSomeNameForModule(m: BModule): Rope =
+  ## Returns a mangled module name.
+  assert m.module.kind == skModule
+  assert m.module.owner.kind == skPackage
+  result.add mangleModuleName(m.g.config, m.filename).mangle
 
 proc getSomeInitName(m: BModule, suffix: string): Rope =
   if not m.hcrOn:
-    result = getSomeNameForModule(m.module)
+    result = getSomeNameForModule(m)
   result.add suffix
 
 proc getInitName(m: BModule): Rope =
@@ -1358,16 +1371,25 @@ proc genMainProc(m: BModule) =
     # The use of a volatile function pointer to call Pre/NimMainInner
     # prevents inlining of the NimMainInner function and dependent
     # functions, which might otherwise merge their stack frames.
-    PreMainBody = "$N" &
+
+    PreMainVolatileBody =
+      "\tvoid (*volatile inner)(void);$N" &
+      "\tinner = PreMainInner;$N" &
+      "$1" &
+      "\t(*inner)();$N"
+
+    PreMainNonVolatileBody =
+      "$1" &
+      "\tPreMainInner();$N"
+
+    PreMainBodyStart = "$N" &
       "N_LIB_PRIVATE void PreMainInner(void) {$N" &
       "$2" &
       "}$N$N" &
       PosixCmdLine &
-      "N_LIB_PRIVATE void PreMain(void) {$N" &
-      "\tvoid (*volatile inner)(void);$N" &
-      "\tinner = PreMainInner;$N" &
-      "$1" &
-      "\t(*inner)();$N" &
+      "N_LIB_PRIVATE void PreMain(void) {$N"
+
+    PreMainBodyEnd =
       "}$N$N"
 
     MainProcs =
@@ -1380,17 +1402,32 @@ proc genMainProc(m: BModule) =
         "$1" &
       "}$N$N"
 
-    NimMainProc =
-      "N_CDECL(void, $5NimMain)(void) {$N" &
-        "\tvoid (*volatile inner)(void);$N" &
-        "$4" &
-        "\tinner = NimMainInner;$N" &
-        "$2" &
-        "\t(*inner)();$N" &
+    NimMainVolatileBody =
+      "\tvoid (*volatile inner)(void);$N" &
+      "$4" &
+      "\tinner = NimMainInner;$N" &
+      "$2" &
+      "\t(*inner)();$N"
+
+    NimMainNonVolatileBody =
+      "$4" &
+      "$2" &
+      "\tNimMainInner();$N"
+
+    NimMainProcStart =
+      "N_CDECL(void, $5NimMain)(void) {$N"
+
+    NimMainProcEnd =
       "}$N$N"
 
+    NimMainProc = NimMainProcStart & NimMainVolatileBody & NimMainProcEnd
+
+    NimSlimMainProc = NimMainProcStart & NimMainNonVolatileBody & NimMainProcEnd
+
     NimMainBody = NimMainInner & NimMainProc
 
+    NimSlimMainBody = NimMainInner & NimSlimMainProc
+
     PosixCMain =
       "int main(int argc, char** args, char** env) {$N" &
         "\tcmdLine = args;$N" &
@@ -1451,10 +1488,13 @@ proc genMainProc(m: BModule) =
     m.includeHeader("<libc/component.h>")
 
   let initStackBottomCall =
-    if m.config.target.targetOS == osStandalone or m.config.selectedGC == gcNone: "".rope
+    if m.config.target.targetOS == osStandalone or m.config.selectedGC in {gcNone, gcArc, gcOrc}: "".rope
     else: ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", [])
   inc(m.labels)
-  appcg(m, m.s[cfsProcs], PreMainBody, [m.g.mainDatInit, m.g.otherModsInit])
+  if m.config.selectedGC notin {gcNone, gcArc, gcOrc}:
+    appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit])
+  else:
+    appcg(m, m.s[cfsProcs], PreMainBodyStart & PreMainNonVolatileBody & PreMainBodyEnd, [m.g.mainDatInit, m.g.otherModsInit])
 
   if m.config.target.targetOS == osWindows and
       m.config.globalOptions * {optGenGuiApp, optGenDynLib} != {}:
@@ -1479,9 +1519,14 @@ proc genMainProc(m: BModule) =
     appcg(m, m.s[cfsProcs], nimMain,
         [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix])
   else:
-    const nimMain = NimMainBody
-    appcg(m, m.s[cfsProcs], nimMain,
-        [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix])
+    if m.config.selectedGC notin {gcNone, gcArc, gcOrc}:
+      const nimMain = NimMainBody
+      appcg(m, m.s[cfsProcs], nimMain,
+          [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix])
+    else:
+      const nimMain = NimSlimMainBody
+      appcg(m, m.s[cfsProcs], nimMain,
+          [m.g.mainModInit, initStackBottomCall, m.labels, preMainCode, m.config.nimMainPrefix])
 
   if optNoMain notin m.config.globalOptions:
     if m.config.cppCustomNamespace.len > 0:
@@ -1514,11 +1559,11 @@ proc genMainProc(m: BModule) =
 proc registerInitProcs*(g: BModuleList; m: PSym; flags: set[ModuleBackendFlag]) =
   ## Called from the IC backend.
   if HasDatInitProc in flags:
-    let datInit = getSomeNameForModule(m) & "DatInit000"
+    let datInit = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "DatInit000"
     g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [datInit])
     g.mainDatInit.addf("\t$1();$N", [datInit])
   if HasModuleInitProc in flags:
-    let init = getSomeNameForModule(m) & "Init000"
+    let init = getSomeNameForModule(g.config, g.config.toFullPath(m.info.fileIndex).AbsoluteFile) & "Init000"
     g.mainModProcs.addf("N_LIB_PRIVATE N_NIMCALL(void, $1)(void);$N", [init])
     let initCall = "\t$1();$N" % [init]
     if sfMainModule in m.flags:
@@ -1899,7 +1944,7 @@ proc getCFile(m: BModule): AbsoluteFile =
       if m.compileToCpp: ".nim.cpp"
       elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m"
       else: ".nim.c"
-  result = changeFileExt(completeCfilePath(m.config, withPackageName(m.config, m.cfilename)), ext)
+  result = changeFileExt(completeCfilePath(m.config, mangleModuleName(m.config, m.cfilename).AbsoluteFile), ext)
 
 when false:
   proc myOpenCached(graph: ModuleGraph; module: PSym, rd: PRodReader): PPassContext =
@@ -2052,7 +2097,7 @@ proc finalCodegenActions*(graph: ModuleGraph; m: BModule; n: PNode) =
         discard cgsym(m, "rawWrite")
 
       # raise dependencies on behalf of genMainProc
-      if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone:
+      if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcOrc}:
         discard cgsym(m, "initStackBottomWith")
       if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
         discard cgsym(m, "initThreadVarsEmulation")

compiler/cgmeth.nim

@@ -46,6 +46,7 @@ proc methodCall*(n: PNode; conf: ConfigRef): PNode =
   # replace ordinary method by dispatcher method:
   let disp = getDispatcher(result[0].sym)
   if disp != nil:
+    result[0].typ = disp.typ
     result[0].sym = disp
     # change the arguments to up/downcasts to fit the dispatcher's parameters:
     for i in 1..<result.len:

compiler/closureiters.nim

@@ -121,7 +121,10 @@
 #   yield 2
 #   if :unrollFinally: # This node is created by `newEndFinallyNode`
 #     if :curExc.isNil:
-#       return :tmpResult
+#       if nearestFinally == 0:
+#         return :tmpResult
+#       else:
+#         :state = nearestFinally # bubble up
 #     else:
 #       closureIterSetupExc(nil)
 #       raise
@@ -804,7 +807,10 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   # Generate the following code:
   #   if :unrollFinally:
   #       if :curExc.isNil:
-  #         return :tmpResult
+  #         if nearestFinally == 0:
+  #           return :tmpResult
+  #         else:
+  #           :state = nearestFinally # bubble up
   #       else:
   #         raise
   let curExc = ctx.newCurExcAccess()
@@ -813,11 +819,20 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   let cmp = newTree(nkCall, newSymNode(ctx.g.getSysMagic(info, "==", mEqRef), info), curExc, nilnode)
   cmp.typ = ctx.g.getSysType(info, tyBool)
 
-  let asgn = newTree(nkFastAsgn,
-    newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen), info),
-    ctx.newTmpResultAccess())
+  let retStmt =
+    if ctx.nearestFinally == 0:
+      # last finally, we can return
+      let retValue = if ctx.fn.typ[0].isNil:
+                   ctx.g.emptyNode
+                 else:
+                   newTree(nkFastAsgn,
+                           newSymNode(getClosureIterResult(ctx.g, ctx.fn, ctx.idgen), info),
+                           ctx.newTmpResultAccess())
+      newTree(nkReturnStmt, retValue)
+    else:
+      # bubble up to next finally
+      newTree(nkGotoState, ctx.g.newIntLit(info, ctx.nearestFinally))
 
-  let retStmt = newTree(nkReturnStmt, asgn)
   let branch = newTree(nkElifBranch, cmp, retStmt)
 
   let nullifyExc = newTree(nkCall, newSymNode(ctx.g.getCompilerProc("closureIterSetupExc")), nilnode)
@@ -861,6 +876,13 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
 
   of nkSkip:
     discard
+  of nkTryStmt:
+    if n.hasYields:
+      # the inner try will handle these transformations
+      discard
+    else:
+      for i in 0..<n.len:
+        n[i] = ctx.transformReturnsInTry(n[i])
   else:
     for i in 0..<n.len:
       n[i] = ctx.transformReturnsInTry(n[i])
@@ -1130,7 +1152,7 @@ proc newArrayType(g: ModuleGraph; n: int, t: PType; idgen: IdGenerator; owner: P
   result = newType(tyArray, nextTypeId(idgen), owner)
 
   let rng = newType(tyRange, nextTypeId(idgen), owner)
-  rng.n = newTree(nkRange, g.newIntLit(owner.info, 0), g.newIntLit(owner.info, n))
+  rng.n = newTree(nkRange, g.newIntLit(owner.info, 0), g.newIntLit(owner.info, n - 1))
   rng.rawAddSon(t)
 
   result.rawAddSon(rng)
@@ -1347,20 +1369,24 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode =
   # detect: 'finally: raises X' which is currently not supported. We produce
   # an error for this case for now. All this will be done properly with Yuriy's
   # patch.
+
   result = n
   case n.kind
   of nkTryStmt:
     let f = n.lastSon
+    var didAddSomething = false
     if f.kind == nkFinally:
       c.finallys.add f.lastSon
+      didAddSomething = true
 
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
 
-    if f.kind == nkFinally:
+    if didAddSomething:
       discard c.finallys.pop()
 
   of nkWhileStmt, nkBlockStmt:
+    if n.hasYields == false: return n
     c.blocks.add((n, c.finallys.len))
     for i in 0 ..< n.len:
       result[i] = preprocess(c, n[i])
@@ -1384,7 +1410,7 @@ proc preprocess(c: var PreprocessContext; n: PNode): PNode =
         result = newNodeI(nkStmtList, n.info)
         for i in countdown(c.finallys.high, fin):
           var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info, idgen: c.idgen)
-          result.add freshVars(preprocess(c, c.finallys[i]), vars)
+          result.add freshVars(copyTree(c.finallys[i]), vars)
           c.idgen = vars.idgen
         result.add n
   of nkSkip: discard

compiler/cmdlinehelper.nim

@@ -64,7 +64,8 @@ proc loadConfigsAndProcessCmdLine*(self: NimProg, cache: IdentCache; conf: Confi
       if conf.cmd == cmdNimscript: return false
   # now process command line arguments again, because some options in the
   # command line can overwrite the config file's settings
-  extccomp.initVars(conf)
+  if conf.backend != backendJs: # bug #19059
+    extccomp.initVars(conf)
   self.processCmdLine(passCmd2, "", conf)
   if conf.cmd == cmdNone:
     rawMessage(conf, errGenerated, "command missing")

compiler/commands.nim

@@ -113,7 +113,7 @@ const
   errInvalidCmdLineOption = "invalid command line option: '$1'"
   errOnOrOffExpectedButXFound = "'on' or 'off' expected, but '$1' found"
   errOnOffOrListExpectedButXFound = "'on', 'off' or 'list' expected, but '$1' found"
-  errOffHintsError = "'off', 'hint' or 'error' expected, but '$1' found"
+  errOffHintsError = "'off', 'hint', 'error' or 'usages' expected, but '$1' found"
 
 proc invalidCmdLineOption(conf: ConfigRef; pass: TCmdLinePass, switch: string, info: TLineInfo) =
   if switch == " ": localError(conf, info, errInvalidCmdLineOption % "-")
@@ -238,7 +238,7 @@ const
   errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found"
-  errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found"
+  errInvalidExceptionSystem = "'goto', 'setjmp', 'cpp' or 'quirky' expected, but '$1' found"
 
 template warningOptionNoop(switch: string) =
   warningDeprecated(conf, info, "'$#' is deprecated, now a noop" % switch)
@@ -885,8 +885,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     splitSwitch(conf, arg, key, val, pass, info)
     os.putEnv(key, val)
   of "cc":
-    expectArg(conf, switch, arg, pass, info)
-    setCC(conf, arg, info)
+    if conf.backend != backendJs: # bug #19330
+      expectArg(conf, switch, arg, pass, info)
+      setCC(conf, arg, info)
   of "track":
     expectArg(conf, switch, arg, pass, info)
     track(conf, arg, info)
@@ -1051,7 +1052,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     processOnOffSwitchG(conf, {optEnableDeepCopy}, arg, pass, info)
   of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -)
     handleStdinInput(conf)
-  of "nilseqs", "nilchecks", "mainmodule", "m", "symbol", "taintmode", "cs", "deadcodeelim": warningOptionNoop(switch)
+  of "nilseqs", "nilchecks", "symbol", "taintmode", "cs", "deadcodeelim": warningOptionNoop(switch)
   of "nimmainprefix": conf.nimMainPrefix = arg
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
@@ -1089,6 +1090,8 @@ proc processArgument*(pass: TCmdLinePass; p: OptParser;
   else:
     if pass == passCmd1: config.commandArgs.add p.key
     if argsCount == 1:
+      if p.key.endsWith(".nims"):
+        incl(config.globalOptions, optWasNimscript)
       # support UNIX style filenames everywhere for portable build scripts:
       if config.projectName.len == 0:
         config.projectName = unixToNativePath(p.key)

compiler/depends.nim

@@ -9,10 +9,12 @@
 
 # This module implements a dependency file generator.
 
-import
-  options, ast, ropes, idents, passes, modulepaths, pathutils
+import options, ast, ropes, passes, pathutils, msgs, lineinfos
 
-from modulegraphs import ModuleGraph, PPassContext
+import modulegraphs
+
+import std/[os, strutils, parseutils]
+import std/private/globs
 
 type
   TGen = object of PPassContext
@@ -28,6 +30,50 @@ proc addDependencyAux(b: Backend; importing, imported: string) =
   b.dotGraph.addf("\"$1\" -> \"$2\";$n", [rope(importing), rope(imported)])
   # s1 -> s2_4[label="[0-9]"];
 
+proc toNimblePath(s: string, isStdlib: bool): string =
+  const stdPrefix = "std/"
+  const pkgPrefix = "pkg/"
+  if isStdlib:
+    let sub = "lib/"
+    var start = s.find(sub)
+    if start < 0:
+      doAssert false
+    else:
+      start += sub.len
+      let base = s[start..^1]
+
+      if base.startsWith("system") or base.startsWith("std"):
+        result = base
+      else:
+        for dir in stdlibDirs:
+          if base.startsWith(dir):
+            return stdPrefix & base.splitFile.name
+
+        result = stdPrefix & base
+  else:
+    var sub = getEnv("NIMBLE_DIR")
+    if sub.len == 0:
+      sub = ".nimble/pkgs/"
+    else:
+      sub.add "/pkgs/"
+    var start = s.find(sub)
+    if start < 0:
+      result = s
+    else:
+      start += sub.len
+      start += skipUntil(s, '/', start)
+      start += 1
+      result = pkgPrefix & s[start..^1]
+
+proc addDependency(c: PPassContext, g: PGen, b: Backend, n: PNode) =
+  doAssert n.kind == nkSym, $n.kind
+
+  let path = splitFile(toProjPath(g.config, n.sym.position.FileIndex))
+  let modulePath = splitFile(toProjPath(g.config, g.module.position.FileIndex))
+  let parent = nativeToUnixPath(modulePath.dir / modulePath.name).toNimblePath(belongsToStdlib(g.graph, g.module))
+  let child = nativeToUnixPath(path.dir / path.name).toNimblePath(belongsToStdlib(g.graph, n.sym))
+  addDependencyAux(b, parent, child)
+
 proc addDotDependency(c: PPassContext, n: PNode): PNode =
   result = n
   let g = PGen(c)
@@ -35,11 +81,9 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode =
   case n.kind
   of nkImportStmt:
     for i in 0..<n.len:
-      var imported = getModuleName(g.config, n[i])
-      addDependencyAux(b, g.module.name.s, imported)
+      addDependency(c, g, b, n[i])
   of nkFromStmt, nkImportExceptStmt:
-    var imported = getModuleName(g.config, n[0])
-    addDependencyAux(b, g.module.name.s, imported)
+    addDependency(c, g, b, n[0])
   of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr:
     for i in 0..<n.len: discard addDotDependency(c, n[i])
   else:

compiler/docgen.nim

@@ -16,7 +16,7 @@ import
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, trees, types,
   typesrenderer, astalgo, lineinfos, intsets,
-  pathutils, tables, nimpaths, renderverbatim, osproc
+  pathutils, tables, nimpaths, renderverbatim, osproc, packages
 import packages/docutils/rstast except FileIndex, TLineInfo
 
 from uri import encodeUrl
@@ -221,6 +221,7 @@ template declareClosures =
     of meInvalidDirective: k = errRstInvalidDirectiveX
     of meInvalidField: k = errRstInvalidField
     of meFootnoteMismatch: k = errRstFootnoteMismatch
+    of meSandboxedDirective: k = errRstSandboxedDirective
     of mwRedefinitionOfLabel: k = warnRstRedefinitionOfLabel
     of mwUnknownSubstitution: k = warnRstUnknownSubstitutionX
     of mwBrokenLink: k = warnRstBrokenLink
@@ -265,7 +266,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   result.cache = cache
   result.outDir = conf.outDir.string
   result.isPureRst = isPureRst
-  var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown}
+  var options= {roSupportRawDirective, roSupportMarkdown, roPreferMarkdown, roSandboxDisabled}
   if not isPureRst: options.incl roNimFile
   result.sharedState = newRstSharedState(
       options, filename.string,
@@ -274,6 +275,10 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
                    conf.configVars, filename.string,
                    docgenFindFile, compilerMsgHandler)
 
+  if conf.configVars.hasKey("doc.googleAnalytics") and
+      conf.configVars.hasKey("doc.plausibleAnalytics"):
+    doAssert false, "Either use googleAnalytics or plausibleAnalytics"
+
   if conf.configVars.hasKey("doc.googleAnalytics"):
     result.analytics = """
 <script>
@@ -287,6 +292,10 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
 
 </script>
     """ % [conf.configVars.getOrDefault"doc.googleAnalytics"]
+  elif conf.configVars.hasKey("doc.plausibleAnalytics"):
+    result.analytics = """
+    <script defer data-domain="$1" src="https://plausible.io/js/plausible.js"></script>
+    """ % [conf.configVars.getOrDefault"doc.plausibleAnalytics"]
   else:
     result.analytics = ""
 
@@ -394,9 +403,6 @@ proc getPlainDocstring(n: PNode): string =
       result = getPlainDocstring(n[i])
       if result.len > 0: return
 
-proc belongsToPackage(conf: ConfigRef; module: PSym): bool =
-  result = module.kind == skModule and module.getnimblePkgId == conf.mainPackageId
-
 proc externalDep(d: PDoc; module: PSym): string =
   if optWholeProject in d.conf.globalOptions or d.conf.docRoot.len > 0:
     let full = AbsoluteFile toFullPath(d.conf, FileIndex module.position)
@@ -452,7 +458,7 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var string;
               "\\spanIdentifier{$1}", [escLit, procLink])
       elif s != nil and s.kind in {skType, skVar, skLet, skConst} and
            sfExported in s.flags and s.owner != nil and
-           belongsToPackage(d.conf, s.owner) and d.target == outHtml:
+           belongsToProjectPackage(d.conf, s.owner) and d.target == outHtml:
         let external = externalDep(d, s.owner)
         result.addf "<a href=\"$1#$2\"><span class=\"Identifier\">$3</span></a>",
           [changeFileExt(external, "html"), literal,
@@ -1040,7 +1046,7 @@ proc traceDeps(d: PDoc, it: PNode) =
     for x in it[2]:
       a[2] = x
       traceDeps(d, a)
-  elif it.kind == nkSym and belongsToPackage(d.conf, it.sym):
+  elif it.kind == nkSym and belongsToProjectPackage(d.conf, it.sym):
     let external = externalDep(d, it.sym)
     if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
     dispA(d.conf, d.section[k].finalMarkup,
@@ -1050,7 +1056,7 @@ proc traceDeps(d: PDoc, it: PNode) =
 
 proc exportSym(d: PDoc; s: PSym) =
   const k = exportSection
-  if s.kind == skModule and belongsToPackage(d.conf, s):
+  if s.kind == skModule and belongsToProjectPackage(d.conf, s):
     let external = externalDep(d, s)
     if d.section[k].finalMarkup != "": d.section[k].finalMarkup.add(", ")
     dispA(d.conf, d.section[k].finalMarkup,
@@ -1059,7 +1065,7 @@ proc exportSym(d: PDoc; s: PSym) =
                  changeFileExt(external, "html")])
   elif s.kind != skModule and s.owner != nil:
     let module = originatingModule(s)
-    if belongsToPackage(d.conf, module):
+    if belongsToProjectPackage(d.conf, module):
       let
         complexSymbol = complexName(s.kind, s.ast, s.name.s)
         symbolOrId = d.newUniquePlainSymbol(complexSymbol)
@@ -1226,9 +1232,10 @@ proc finishGenerateDoc*(d: var PDoc) =
       var str: string
       renderRstToOut(d[], resolved, str)
       entry.json[entry.rstField] = %str
-      d.jEntriesFinal.add entry.json
       d.jEntriesPre[i].rst = nil
 
+    d.jEntriesFinal.add entry.json # generates docs
+
 proc add(d: PDoc; j: JsonItem) =
   if j.json != nil or j.rst != nil: d.jEntriesPre.add j
 

compiler/docgen2.nim

@@ -11,7 +11,7 @@
 # semantic checking.
 
 import
-  options, ast, msgs, passes, docgen, lineinfos, pathutils
+  options, ast, msgs, passes, docgen, lineinfos, pathutils, packages
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -23,7 +23,7 @@ type
   PGen = ref TGen
 
 proc shouldProcess(g: PGen): bool =
-  (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or
+  (optWholeProject in g.doc.conf.globalOptions and g.doc.conf.belongsToProjectPackage(g.module)) or
       sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
 
 template closeImpl(body: untyped) {.dirty.} =

compiler/evalffi.nim

@@ -110,8 +110,8 @@ proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI
   else:
     globalError(conf, info, "cannot map calling convention to FFI")
 
-template rd(T, p: untyped): untyped = (cast[ptr T](p))[]
-template wr(T, p, v: untyped): untyped = (cast[ptr T](p))[] = v
+template rd(typ, p: untyped): untyped = (cast[ptr typ](p))[]
+template wr(typ, p, v: untyped): untyped = (cast[ptr typ](p))[] = v
 template `+!`(x, y: untyped): untyped =
   cast[pointer](cast[ByteAddress](x) + y)
 
@@ -177,8 +177,8 @@ const maxPackDepth = 20
 var packRecCheck = 0
 
 proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
-  template awr(T, v: untyped): untyped =
-    wr(T, res, v)
+  template awr(typ, v: untyped): untyped =
+    wr(typ, res, v)
 
   case typ.kind
   of tyBool: awr(bool, v.intVal != 0)

compiler/extccomp.nim

@@ -12,9 +12,9 @@
 # from a lineinfos file, to provide generalized procedures to compile
 # nim files.
 
-import ropes, platform, condsyms, options, msgs, lineinfos, pathutils
+import ropes, platform, condsyms, options, msgs, lineinfos, pathutils, modulepaths
 
-import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar]
+import std/[os, strutils, osproc, sha1, streams, sequtils, times, strtabs, json, jsonutils, sugar, parseutils]
 
 type
   TInfoCCProp* = enum         # properties of the C compiler:
@@ -367,6 +367,7 @@ proc initVars*(conf: ConfigRef) =
 
 proc completeCfilePath*(conf: ConfigRef; cfile: AbsoluteFile,
                         createSubDir: bool = true): AbsoluteFile =
+  ## Generate the absolute file path to the generated modules.
   result = completeGeneratedFilePath(conf, cfile, createSubDir)
 
 proc toObjFile*(conf: ConfigRef; filename: AbsoluteFile): AbsoluteFile =
@@ -377,7 +378,7 @@ proc addFileToCompile*(conf: ConfigRef; cf: Cfile) =
   conf.toCompile.add(cf)
 
 proc addLocalCompileOption*(conf: ConfigRef; option: string; nimfile: AbsoluteFile) =
-  let key = completeCfilePath(conf, withPackageName(conf, nimfile)).string
+  let key = completeCfilePath(conf, mangleModuleName(conf, nimfile).AbsoluteFile).string
   var value = conf.cfileSpecificOptions.getOrDefault(key)
   if strutils.find(value, option, 0) < 0:
     addOpt(value, option)
@@ -483,7 +484,10 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} =
            (conf.target.hostOS == osWindows)
 
 proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool =
-  conf.backend == backendCpp and not cfile.string.endsWith(".c")
+  # List of possible file extensions taken from gcc
+  for ext in [".C", ".cc", ".cpp", ".CPP", ".c++", ".cp", ".cxx"]:
+    if cfile.string.endsWith(ext): return true
+  false
 
 proc envFlags(conf: ConfigRef): string =
   result = if conf.backend == backendCpp:
@@ -491,14 +495,14 @@ proc envFlags(conf: ConfigRef): string =
           else:
             getEnv("CFLAGS")
 
-proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string =
+proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; isCpp: bool): string =
   if compiler == ccEnv:
-    result = if useCpp(conf, cfile):
+    result = if isCpp:
               getEnv("CXX")
             else:
               getEnv("CC")
   else:
-    result = if useCpp(conf, cfile):
+    result = if isCpp:
               CC[compiler].cppCompiler
             else:
               CC[compiler].compilerExe
@@ -512,47 +516,36 @@ proc ccHasSaneOverflow*(conf: ConfigRef): bool =
     result = false # assume an old or crappy GCC
     var exe = getConfigVar(conf, conf.cCompiler, ".exe")
     if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
-    let (s, exitCode) = try: execCmdEx(exe & " --version") except: ("", 1)
+    # NOTE: should we need the full version, use -dumpfullversion
+    let (s, exitCode) = try: execCmdEx(exe & " -dumpversion") except: ("", 1)
     if exitCode == 0:
-      var i = 0
-      var j = 0
-      # the version is the last part of the first line:
-      while i < s.len and s[i] != '\n':
-        if s[i] in {' ', '\t'}: j = i+1
-        inc i
-      if j > 0:
-        var major = 0
-        while j < s.len and s[j] in {'0'..'9'}:
-          major = major * 10 + (ord(s[j]) - ord('0'))
-          inc j
-        if i < s.len and s[j] == '.': inc j
-        while j < s.len and s[j] in {'0'..'9'}:
-          inc j
-        if j+1 < s.len and s[j] == '.' and s[j+1] in {'0'..'9'}:
-          # we found a third version number, chances are high
-          # we really parsed the version:
-          result = major >= 5
+      var major: int
+      discard parseInt(s, major)
+      result = major >= 5
   else:
     result = conf.cCompiler == ccCLang
 
 proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
-           elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler
-           else: getCompilerExe(conf, compiler, AbsoluteFile"")
+           else: getCompilerExe(conf, compiler, optMixedMode in conf.globalOptions or conf.backend == backendCpp)
 
 proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
                          isMainFile = false; produceOutput = false): string =
-  let c = conf.cCompiler
+  let
+    c = conf.cCompiler
+    isCpp = useCpp(conf, cfile.cname)
   # We produce files like module.nim.cpp, so the absolute Nim filename is not
   # cfile.name but `cfile.cname.changeFileExt("")`:
   var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string)
-  if useCpp(conf, cfile.cname):
+  if isCpp:
     # needs to be prepended so that --passc:-std=c++17 can override default.
     # we could avoid allocation by making cFileSpecificOptions inplace
     options = CC[c].cppXsupport & ' ' & options
+    # If any C++ file was compiled, we need to use C++ driver for linking as well
+    incl conf.globalOptions, optMixedMode
 
   var exe = getConfigVar(conf, c, ".exe")
-  if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname)
+  if exe.len == 0: exe = getCompilerExe(conf, c, isCpp)
 
   if needsExeExt(conf): exe = addFileExt(exe, "exe")
   if (optGenDynLib in conf.globalOptions or (conf.hcrOn and not isMainFile)) and
@@ -572,7 +565,7 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
 
     compilePattern = joinPath(conf.cCompilerPath, exe)
   else:
-    compilePattern = getCompilerExe(conf, c, cfile.cname)
+    compilePattern = getCompilerExe(conf, c, isCpp)
 
   includeCmd.add(join([CC[c].includeCmd, quoteShell(conf.projectPath.string)]))
 
@@ -634,7 +627,7 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
 proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
   if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good
 
-  let hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1")
+  let hashFile = toGeneratedFile(conf, conf.mangleModuleName(cfile.cname).AbsoluteFile, "sha1")
   let currentHash = footprint(conf, cfile)
   var f: File
   if open(f, hashFile.string, fmRead):
@@ -649,8 +642,10 @@ proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
       close(f)
 
 proc addExternalFileToCompile*(conf: ConfigRef; c: var Cfile) =
+  # we want to generate the hash file unconditionally
+  let extFileChanged = externalFileChanged(conf, c)
   if optForceFullMake notin conf.globalOptions and fileExists(c.obj) and
-      not externalFileChanged(conf, c):
+      not extFileChanged:
     c.flags.incl CfileFlag.Cached
   else:
     # make sure Nim keeps recompiling the external file on reruns
@@ -840,9 +835,9 @@ proc hcrLinkTargetName(conf: ConfigRef, objFile: string, isMain = false): Absolu
 proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string =
   if conf.hasHint(hintCC):
     if optListCmd in conf.globalOptions or conf.verbosity > 1:
-      result = MsgKindToStr[hintCC] % (demanglePackageName(path.splitFile.name) & ": " & compileCmd)
+      result = MsgKindToStr[hintCC] % (demangleModuleName(path.splitFile.name) & ": " & compileCmd)
     else:
-      result = MsgKindToStr[hintCC] % demanglePackageName(path.splitFile.name)
+      result = MsgKindToStr[hintCC] % demangleModuleName(path.splitFile.name)
 
 proc callCCompiler*(conf: ConfigRef) =
   var

compiler/ic/bitabs.nim

@@ -90,13 +90,13 @@ proc getOrIncl*[T](t: var BiTable[T]; v: T): LitId =
   t.vals.add v
 
 
-proc `[]`*[T](t: var BiTable[T]; LitId: LitId): var T {.inline.} =
-  let idx = idToIdx LitId
+proc `[]`*[T](t: var BiTable[T]; litId: LitId): var T {.inline.} =
+  let idx = idToIdx litId
   assert idx < t.vals.len
   result = t.vals[idx]
 
-proc `[]`*[T](t: BiTable[T]; LitId: LitId): lent T {.inline.} =
-  let idx = idToIdx LitId
+proc `[]`*[T](t: BiTable[T]; litId: LitId): lent T {.inline.} =
+  let idx = idToIdx litId
   assert idx < t.vals.len
   result = t.vals[idx]
 

compiler/ic/cbackend.nim

@@ -21,7 +21,7 @@
 import std/packedsets, algorithm, tables
 
 import ".."/[ast, options, lineinfos, modulegraphs, cgendata, cgen,
-  pathutils, extccomp, msgs]
+  pathutils, extccomp, msgs, modulepaths]
 
 import packed_ast, ic, dce, rodfiles
 
@@ -61,7 +61,8 @@ proc addFileToLink(config: ConfigRef; m: PSym) =
       if config.backend == backendCpp: ".nim.cpp"
       elif config.backend == backendObjc: ".nim.m"
       else: ".nim.c"
-  let cfile = changeFileExt(completeCfilePath(config, withPackageName(config, filename)), ext)
+  let cfile = changeFileExt(completeCfilePath(config,
+                            mangleModuleName(config, filename).AbsoluteFile), ext)
   let objFile = completeCfilePath(config, toObjFile(config, cfile))
   if fileExists(objFile):
     var cf = Cfile(nimname: m.name.s, cname: cfile,

compiler/ic/ic.nim

@@ -10,7 +10,7 @@
 import hashes, tables, intsets, std/sha1
 import packed_ast, bitabs, rodfiles
 import ".." / [ast, idents, lineinfos, msgs, ropes, options,
-  pathutils, condsyms]
+  pathutils, condsyms, packages, modulepaths]
 #import ".." / [renderer, astalgo]
 from os import removeFile, isAbsolute
 
@@ -548,6 +548,10 @@ proc loadError(err: RodFileError; filename: AbsoluteFile; config: ConfigRef;) =
     rawMessage(config, warnCannotOpenFile, filename.string & " reason: " & $err)
     #echo "Error: ", $err, " loading file: ", filename.string
 
+proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile =
+  result = changeFileExt(completeGeneratedFilePath(conf,
+    mangleModuleName(conf, f).AbsoluteFile), ext)
+
 proc loadRodFile*(filename: AbsoluteFile; m: var PackedModule; config: ConfigRef;
                   ignoreConfig = false): RodFileError =
   var f = rodfiles.open(filename.string)
@@ -927,17 +931,6 @@ proc loadType(c: var PackedDecoder; g: var PackedModuleGraph; thisModule: int; t
       result = g[si].types[t.item]
     assert result.itemId.item > 0
 
-proc newPackage(config: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym =
-  let filename = AbsoluteFile toFullPath(config, fileIdx)
-  let name = getIdent(cache, splitFile(filename).name)
-  let info = newLineInfo(fileIdx, 1, 1)
-  let
-    pck = getPackageName(config, filename.string)
-    pck2 = if pck.len > 0: pck else: "unknown"
-    pack = getIdent(cache, pck2)
-  result = newSym(skPackage, getIdent(cache, pck2),
-    ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info)
-
 proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;
                        fileIdx: FileIndex; m: var LoadedModule) =
   m.iface = initTable[PIdent, seq[PackedItemId]]()
@@ -965,7 +958,7 @@ proc setupLookupTables(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCa
                   name: getIdent(cache, splitFile(filename).name),
                   info: newLineInfo(fileIdx, 1, 1),
                   position: int(fileIdx))
-  m.module.owner = newPackage(conf, cache, fileIdx)
+  m.module.owner = getPackage(conf, cache, fileIdx)
   m.module.flags = m.fromDisk.moduleFlags
 
 proc loadToReplayNodes(g: var PackedModuleGraph; conf: ConfigRef; cache: IdentCache;

compiler/ic/rodfiles.nim

@@ -34,8 +34,7 @@ from typetraits import supportsCopyMem
 ##
 ## Now read the bits below to understand what's missing.
 ##
-## Issues with the Example
-## ```````````````````````
+## ### Issues with the Example
 ## Missing Sections:
 ## This is a low level API, so headers and sections need to be stored and
 ## loaded by the user, see `storeHeader` & `loadHeader` and `storeSection` &

compiler/importer.nim

@@ -199,7 +199,7 @@ template addUnnamedIt(c: PContext, fromMod: PSym; filter: untyped) {.dirty.} =
 
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
   c.addImport ImportedModule(m: fromMod, mode: importExcept, exceptSet: exceptSet)
-  addUnnamedIt(c, fromMod, it.sym.id notin exceptSet)
+  addUnnamedIt(c, fromMod, it.sym.name.id notin exceptSet)
 
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
   c.addImport ImportedModule(m: fromMod, mode: importAll)

compiler/injectdestructors.nim

@@ -573,7 +573,7 @@ proc processScope(c: var Con; s: var Scope; ret: PNode): PNode =
 
 template processScopeExpr(c: var Con; s: var Scope; ret: PNode, processCall: untyped): PNode =
   assert not ret.typ.isEmptyType
-  var result = newNodeI(nkStmtListExpr, ret.info)
+  var result = newNodeIT(nkStmtListExpr, ret.info, ret.typ)
   # There is a possibility to do this check: s.wasMoved.len > 0 or s.final.len > 0
   # later and use it to eliminate the temporary when theres no need for it, but its
   # tricky because you would have to intercept moveOrCopy at a certain point
@@ -1018,6 +1018,26 @@ proc sameLocation*(a, b: PNode): bool =
     of nkHiddenStdConv, nkHiddenSubConv: sameLocation(a[1], b)
     else: false
 
+proc genFieldAccessSideEffects(c: var Con; dest, ri: PNode, isDecl: bool): PNode =
+  # with side effects
+  var temp = newSym(skLet, getIdent(c.graph.cache, "bracketTmp"), nextSymId c.idgen, c.owner, ri[1].info)
+  temp.typ = ri[1].typ
+  var v = newNodeI(nkLetSection, ri[1].info)
+  let tempAsNode = newSymNode(temp)
+
+  var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
+  vpart[0] = tempAsNode
+  vpart[1] = newNodeI(nkEmpty, tempAsNode.info)
+  vpart[2] = ri[1]
+  v.add(vpart)
+
+  var newAccess = copyNode(ri)
+  newAccess.add ri[0]
+  newAccess.add tempAsNode
+
+  var snk = c.genSink(dest, newAccess, isDecl)
+  result = newTree(nkStmtList, v, snk, c.genWasMoved(newAccess))
+
 proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, isDecl = false): PNode =
   if sameLocation(dest, ri):
     # rule (self-assignment-removal):
@@ -1041,8 +1061,11 @@ proc moveOrCopy(dest, ri: PNode; c: var Con; s: var Scope, isDecl = false): PNod
       elif isAnalysableFieldAccess(ri, c.owner) and isLastRead(ri, c):
         if aliases(dest, ri) == no:
           # Rule 3: `=sink`(x, z); wasMoved(z)
-          var snk = c.genSink(dest, ri, isDecl)
-          result = newTree(nkStmtList, snk, c.genWasMoved(ri))
+          if isAtom(ri[1]):
+            var snk = c.genSink(dest, ri, isDecl)
+            result = newTree(nkStmtList, snk, c.genWasMoved(ri))
+          else:
+            result = genFieldAccessSideEffects(c, dest, ri, isDecl)
         else:
           result = c.genSink(dest, destructiveMoveVar(ri, c, s), isDecl)
       else:

compiler/jsgen.nim

@@ -178,7 +178,7 @@ const
 proc mapType(typ: PType): TJSTypeKind =
   let t = skipTypes(typ, abstractInst)
   case t.kind
-  of tyVar, tyRef, tyPtr, tyLent:
+  of tyVar, tyRef, tyPtr:
     if skipTypes(t.lastSon, abstractInst).kind in MappedToObject:
       result = etyObject
     else:
@@ -186,7 +186,8 @@ proc mapType(typ: PType): TJSTypeKind =
   of tyPointer:
     # treat a tyPointer like a typed pointer to an array of bytes
     result = etyBaseIndex
-  of tyRange, tyDistinct, tyOrdinal, tyProxy:
+  of tyRange, tyDistinct, tyOrdinal, tyProxy, tyLent:
+    # tyLent is no-op as JS has pass-by-reference semantics
     result = mapType(t[0])
   of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar: result = etyInt
   of tyBool: result = etyBool
@@ -1060,14 +1061,14 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
     xtyp = etySeq
   case xtyp
   of etySeq:
-    if (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
+    if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
       lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
       lineF(p, "$1 = nimCopy(null, $2, $3);$n",
                [a.rdLoc, b.res, genTypeInfo(p, y.typ)])
   of etyObject:
-    if x.typ.kind in {tyVar} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
+    if x.typ.kind in {tyVar, tyLent} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
       lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
@@ -1092,10 +1093,18 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
         lineF(p, "$# = [$#, $#];$n", [a.res, b.address, b.res])
         lineF(p, "$1 = $2;$n", [a.address, b.res])
         lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
+      elif a.typ == etyBaseIndex:
+        # array indexing may not map to var type
+        if b.address != nil:
+          lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res])
+        else:
+          lineF(p, "$1 = $2;$n", [a.address, b.res])
       else:
         internalError(p.config, x.info, $("genAsgn", b.typ, a.typ))
-    else:
+    elif b.address != nil:
       lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res])
+    else:
+      lineF(p, "$1 = $2;$n", [a.address, b.res])
   else:
     lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
 
@@ -1442,13 +1451,17 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
   else:
     if s.loc.r == nil:
       internalError(p.config, n.info, "symbol has no generated name: " & s.name.s)
-    r.res = s.loc.r
+    if mapType(p, s.typ) == etyBaseIndex:
+      r.address = s.loc.r
+      r.res = s.loc.r & "_Idx"
+    else:
+      r.res = s.loc.r
   r.kind = resVal
 
 proc genDeref(p: PProc, n: PNode, r: var TCompRes) =
   let it = n[0]
   let t = mapType(p, it.typ)
-  if t == etyObject:
+  if t == etyObject or it.typ.kind == tyLent:
     gen(p, it, r)
   else:
     var a: TCompRes
@@ -1689,7 +1702,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
       result = putToSeq("0", indirect)
   of tyFloat..tyFloat128:
     result = putToSeq("0.0", indirect)
-  of tyRange, tyGenericInst, tyAlias, tySink, tyOwned:
+  of tyRange, tyGenericInst, tyAlias, tySink, tyOwned, tyLent:
     result = createVar(p, lastSon(typ), indirect)
   of tySet:
     result = putToSeq("{}", indirect)
@@ -1731,7 +1744,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
     createObjInitList(p, t, initIntSet(), initList)
     result = ("({$1})") % [initList]
     if indirect: result = "[$1]" % [result]
-  of tyVar, tyPtr, tyLent, tyRef, tyPointer:
+  of tyVar, tyPtr, tyRef, tyPointer:
     if mapType(p, t) == etyBaseIndex:
       result = putToSeq("[null, 0]", indirect)
     else:
@@ -2380,16 +2393,17 @@ proc genProc(oldProc: PProc, prc: PSym): Rope =
   if prc.typ[0] != nil and sfPure notin prc.flags:
     resultSym = prc.ast[resultPos].sym
     let mname = mangleName(p.module, resultSym)
-    if not isIndirect(resultSym) and
+    let returnAddress = not isIndirect(resultSym) and
       resultSym.typ.kind in {tyVar, tyPtr, tyLent, tyRef, tyOwned} and
-        mapType(p, resultSym.typ) == etyBaseIndex:
+        mapType(p, resultSym.typ) == etyBaseIndex
+    if returnAddress:
       resultAsgn = p.indentLine(("var $# = null;$n") % [mname])
       resultAsgn.add p.indentLine("var $#_Idx = 0;$n" % [mname])
     else:
       let resVar = createVar(p, resultSym.typ, isIndirect(resultSym))
       resultAsgn = p.indentLine(("var $# = $#;$n") % [mname, resVar])
     gen(p, prc.ast[resultPos], a)
-    if mapType(p, resultSym.typ) == etyBaseIndex:
+    if returnAddress:
       returnStmt = "return [$#, $#];$n" % [a.address, a.res]
     else:
       returnStmt = "return $#;$n" % [a.res]
@@ -2565,8 +2579,15 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkObjConstr: genObjConstr(p, n, r)
   of nkHiddenStdConv, nkHiddenSubConv, nkConv: genConv(p, n, r)
   of nkAddr, nkHiddenAddr:
-    genAddr(p, n, r)
-  of nkDerefExpr, nkHiddenDeref: genDeref(p, n, r)
+    if n.typ.kind in {tyLent}:
+      gen(p, n[0], r)
+    else:
+      genAddr(p, n, r)
+  of nkDerefExpr, nkHiddenDeref:
+    if n.typ.kind in {tyLent}:
+      gen(p, n[0], r)
+    else:
+      genDeref(p, n, r)
   of nkBracketExpr: genArrayAccess(p, n, r)
   of nkDotExpr: genFieldAccess(p, n, r)
   of nkCheckedFieldExpr: genCheckedFieldOp(p, n, nil, r)

compiler/lambdalifting.nim

@@ -272,16 +272,11 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: PSym): PN
 proc freshVarForClosureIter*(g: ModuleGraph; s: PSym; idgen: IdGenerator; owner: PSym): PNode =
   let envParam = getHiddenParam(g, owner)
   let obj = envParam.typ.skipTypes({tyOwned, tyRef, tyPtr})
-  addField(obj, s, g.cache, idgen)
+  let field = addField(obj, s, g.cache, idgen)
 
   var access = newSymNode(envParam)
   assert obj.kind == tyObject
-  let field = getFieldFromObj(obj, s)
-  if field != nil:
-    result = rawIndirectAccess(access, field, s.info)
-  else:
-    localError(g.config, s.info, "internal error: cannot generate fresh variable")
-    result = access
+  result = rawIndirectAccess(access, field, s.info)
 
 # ------------------ new stuff -------------------------------------------
 
@@ -449,7 +444,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
             if s.name.id == getIdent(c.graph.cache, ":state").id:
               obj.n[0].sym.itemId = ItemId(module: s.itemId.module, item: -s.itemId.item)
             else:
-              addField(obj, s, c.graph.cache, c.idgen)
+              discard addField(obj, s, c.graph.cache, c.idgen)
     # direct or indirect dependency:
     elif (innerProc and s.typ.callConv == ccClosure) or interestingVar(s):
       discard """
@@ -471,7 +466,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id):
         let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef, tyPtr})
         #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
-        addField(obj, s, c.graph.cache, c.idgen)
+        discard addField(obj, s, c.graph.cache, c.idgen)
       # create required upFields:
       var w = owner.skipGenericOwner
       if isInnerProc(w) or owner.isIterator:

compiler/layouter.nim

@@ -551,11 +551,15 @@ proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
     if not preventComment:
       emitComment(em, tok, dontIndent = false)
   of tkIntLit..tkStrLit, tkRStrLit, tkTripleStrLit, tkGStrLit, tkGTripleStrLit, tkCharLit:
-    let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
-    if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
-    em.lineSpan = countNewlines(lit)
-    if em.lineSpan > 0: calcCol(em, lit)
-    wr em, lit, ltLit
+    if not em.inquote:
+      let lit = fileSection(em.config, em.fid, tok.offsetA, tok.offsetB)
+      if endsInAlpha(em) and tok.tokType notin {tkGStrLit, tkGTripleStrLit}: wrSpace(em)
+      em.lineSpan = countNewlines(lit)
+      if em.lineSpan > 0: calcCol(em, lit)
+      wr em, lit, ltLit
+    else:
+      if endsInAlpha(em): wrSpace(em)
+      wr em, tok.literal, ltLit
   of tkEof: discard
   else:
     let lit = if tok.ident != nil: tok.ident.s else: tok.literal

compiler/lexer.nim

@@ -912,7 +912,7 @@ proc getSymbol(L: var Lexer, tok: var Token) =
     else: break
   tokenEnd(tok, pos-1)
   h = !$h
-  tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
+  tok.ident = L.cache.getIdent(cast[cstring](addr(L.buf[L.bufpos])), pos - L.bufpos, h)
   if (tok.ident.id < ord(tokKeywordLow) - ord(tkSymbol)) or
       (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
     tok.tokType = tkSymbol
@@ -926,7 +926,7 @@ proc getSymbol(L: var Lexer, tok: var Token) =
 proc endOperator(L: var Lexer, tok: var Token, pos: int,
                  hash: Hash) {.inline.} =
   var h = !$hash
-  tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
+  tok.ident = L.cache.getIdent(cast[cstring](addr(L.buf[L.bufpos])), pos - L.bufpos, h)
   if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr
   else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon))
   L.bufpos = pos

compiler/liftdestructors.nim

@@ -162,9 +162,12 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       # the value needs to be destroyed before we assign the selector
       # or the value is lost
       let prevKind = c.kind
+      let prevAddMemReset = c.addMemReset
       c.kind = attachedDestructor
+      c.addMemReset = true
       fillBodyObj(c, n, body, x, y, enforceDefaultOp = false)
       c.kind = prevKind
+      c.addMemReset = prevAddMemReset
       localEnforceDefaultOp = true
 
     if c.kind != attachedDestructor:

compiler/lineinfos.nim

@@ -39,6 +39,7 @@ type
     errRstInvalidDirectiveX,
     errRstInvalidField,
     errRstFootnoteMismatch,
+    errRstSandboxedDirective,
     errProveInit, # deadcode
     errGenerated,
     errUser,
@@ -75,6 +76,7 @@ type
     warnAnyEnumConv = "AnyEnumConv",
     warnHoleEnumConv = "HoleEnumConv",
     warnCstringConv = "CStringConv",
+    warnPtrToCstringConv = "PtrToCstringConv",
     warnEffect = "Effect",
     warnUser = "User",
     # hints
@@ -109,6 +111,7 @@ const
     errRstInvalidDirectiveX: "invalid directive: '$1'",
     errRstInvalidField: "invalid field: $1",
     errRstFootnoteMismatch: "number of footnotes and their references don't match: $1",
+    errRstSandboxedDirective: "disabled directive: '$1'",
     errProveInit: "Cannot prove that '$1' is initialized.",  # deadcode
     errGenerated: "$1",
     errUser: "$1",
@@ -164,6 +167,7 @@ const
     warnAnyEnumConv: "$1",
     warnHoleEnumConv: "$1",
     warnCstringConv: "$1",
+    warnPtrToCstringConv: "unsafe conversion to 'cstring' from '$1'; this will become a compile time error in the future",
     warnEffect: "$1",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",

compiler/linter.nim

@@ -12,7 +12,8 @@
 import std/strutils
 from std/sugar import dup
 
-import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs
+import options, ast, msgs, idents, lineinfos, wordrecg, astmsgs, semdata, packages
+export packages
 
 const
   Letters* = {'a'..'z', 'A'..'Z', '0'..'9', '\x80'..'\xFF', '_'}
@@ -85,24 +86,33 @@ proc differ*(line: string, a, b: int, x: string): string =
       result = y
 
 proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
-  # operators stay as they are:
-  if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
-  if k in {skType, skGenericParam} and sfAnon in s.flags: return
-  if s.typ != nil and s.typ.kind == tyTypeDesc: return
-  if {sfImportc, sfExportc} * s.flags != {}: return
-  if optStyleCheck notin s.options: return
   let beau = beautifyName(s.name.s, k)
   if s.name.s != beau:
     lintReport(conf, info, beau, s.name.s)
 
-template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
-  if {optStyleHint, optStyleError} * conf.globalOptions != {} and optStyleUsages notin conf.globalOptions:
-    nep1CheckDefImpl(conf, info, s, k)
+template styleCheckDef*(ctx: PContext; info: TLineInfo; sym: PSym; k: TSymKind) =
+  ## Check symbol definitions adhere to NEP1 style rules.
+  if optStyleCheck in ctx.config.options and # ignore if styleChecks are off
+     {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # check only if hint/error is enabled
+     hintName in ctx.config.notes and # ignore if name checks are not requested
+     ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages
+     optStyleUsages notin ctx.config.globalOptions and # ignore if requested to only check name usage
+     sym.kind != skResult and # ignore `result`
+     sym.kind != skTemp and # ignore temporary variables created by the compiler
+     sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols???
+     k notin {skType, skGenericParam} and # ignore types and generic params
+     (sym.typ == nil or sym.typ.kind != tyTypeDesc) and # ignore `typedesc`
+     {sfImportc, sfExportc} * sym.flags == {} and # ignore FFI
+     sfAnon notin sym.flags: # ignore if created by compiler
+    nep1CheckDefImpl(ctx.config, info, sym, k)
 
-template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) =
-  styleCheckDef(conf, info, s, s.kind)
-template styleCheckDef*(conf: ConfigRef; s: PSym) =
-  styleCheckDef(conf, s.info, s, s.kind)
+template styleCheckDef*(ctx: PContext; info: TLineInfo; s: PSym) =
+  ## Check symbol definitions adhere to NEP1 style rules.
+  styleCheckDef(ctx, info, s, s.kind)
+
+template styleCheckDef*(ctx: PContext; s: PSym) =
+  ## Check symbol definitions adhere to NEP1 style rules.
+  styleCheckDef(ctx, s.info, s, s.kind)
 
 proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string =
   let line = sourceLine(conf, info)
@@ -116,23 +126,27 @@ proc differs(conf: ConfigRef; info: TLineInfo; newName: string): string =
   let last = first+identLen(line, first)-1
   result = differ(line, first, last, newName)
 
-proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) =
-  if info.fileIndex.int < 0: return
-  # we simply convert it to what it looks like in the definition
-  # for consistency
-
-  # operators stay as they are:
-  if s.kind == skTemp or s.name.s[0] notin Letters or sfAnon in s.flags:
-    return
-
+proc styleCheckUseImpl(conf: ConfigRef; info: TLineInfo; s: PSym) =
   let newName = s.name.s
   let badName = differs(conf, info, newName)
   if badName.len > 0:
-    # special rules for historical reasons
-    let forceHint = badName == "nnkArgList" and newName == "nnkArglist" or badName == "nnkArglist" and newName == "nnkArgList"
-    lintReport(conf, info, newName, badName, forceHint = forceHint, extraMsg = "".dup(addDeclaredLoc(conf, s)))
+    lintReport(conf, info, newName, badName, "".dup(addDeclaredLoc(conf, s)))
 
-proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
+template styleCheckUse*(ctx: PContext; info: TLineInfo; sym: PSym) =
+  ## Check symbol uses match their definition's style.
+  if {optStyleHint, optStyleError} * ctx.config.globalOptions != {} and # ignore if styleChecks are off
+     hintName in ctx.config.notes and # ignore if name checks are not requested
+     ctx.config.belongsToProjectPackage(ctx.module) and # ignore foreign packages
+     sym.kind != skTemp and # ignore temporary variables created by the compiler
+     sym.name.s[0] in Letters and # ignore operators TODO: what about unicode symbols???
+     sfAnon notin sym.flags: # ignore temporary variables created by the compiler
+    styleCheckUseImpl(ctx.config, info, sym)
+
+proc checkPragmaUseImpl(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
   let wanted = $w
   if pragmaName != wanted:
     lintReport(conf, info, wanted, pragmaName)
+
+template checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
+  if {optStyleHint, optStyleError} * conf.globalOptions != {}:
+    checkPragmaUseImpl(conf, info, w, pragmaName)

compiler/lookups.nim

@@ -44,6 +44,11 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
         case x.kind
         of nkIdent: id.add(x.ident.s)
         of nkSym: id.add(x.sym.name.s)
+        of nkSymChoices:
+          if x[0].kind == nkSym:
+            id.add(x[0].sym.name.s)
+          else:
+            handleError(n, origin)
         of nkLiterals - nkFloatLiterals: id.add(x.renderTree)
         else: handleError(n, origin)
       result = getIdent(c.cache, id)

compiler/lowerings.nim

@@ -71,14 +71,20 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; idgen: IdGenerator; owner: P
   let value = n.lastSon
   result = newNodeI(nkStmtList, n.info)
 
-  var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen),
-                    owner, value.info, g.config.options)
-  temp.typ = skipTypes(value.typ, abstractInst)
-  incl(temp.flags, sfFromGeneric)
+  var tempAsNode: PNode
+  let avoidTemp = value.kind == nkSym
+  if avoidTemp:
+    tempAsNode = value
+  else:
+    var temp = newSym(skTemp, getIdent(g.cache, genPrefix), nextSymId(idgen),
+                  owner, value.info, g.config.options)
+    temp.typ = skipTypes(value.typ, abstractInst)
+    incl(temp.flags, sfFromGeneric)
+    tempAsNode = newSymNode(temp)
 
   var v = newNodeI(nkVarSection, value.info)
-  let tempAsNode = newSymNode(temp)
-  v.addVar(tempAsNode, value)
+  if not avoidTemp:
+    v.addVar(tempAsNode, value)
   result.add(v)
 
   for i in 0..<n.len-2:
@@ -221,7 +227,7 @@ proc lookupInRecord(n: PNode, id: ItemId): PSym =
     if n.sym.itemId.module == id.module and n.sym.itemId.item == -abs(id.item): result = n.sym
   else: discard
 
-proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) =
+proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym =
   # because of 'gensym' support, we have to mangle the name with its ID.
   # This is hacky but the clean solution is much more complex than it looks.
   var field = newSym(skField, getIdent(cache, s.name.s & $obj.n.len),
@@ -235,6 +241,7 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator) =
   field.flags = s.flags * {sfCursor}
   obj.n.add newSymNode(field)
   fieldCheck()
+  result = field
 
 proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache; idgen: IdGenerator): PSym {.discardable.} =
   result = lookupInRecord(obj.n, s.itemId)

compiler/main.nim

@@ -58,6 +58,9 @@ proc commandCheck(graph: ModuleGraph) =
   let conf = graph.config
   conf.setErrorMaxHighMaybe
   defineSymbol(conf.symbols, "nimcheck")
+  if optWasNimscript in conf.globalOptions:
+    defineSymbol(conf.symbols, "nimscript")
+    defineSymbol(conf.symbols, "nimconfig")
   semanticPasses(graph)  # use an empty backend for semantic checking only
   compileProject(graph)
 
@@ -363,7 +366,8 @@ proc mainCommand*(graph: ModuleGraph) =
       msgWriteln(conf, "-- end of list --", {msgStdout, msgSkipHook})
 
       for it in conf.searchPaths: msgWriteln(conf, it.string)
-  of cmdCheck: commandCheck(graph)
+  of cmdCheck:
+    commandCheck(graph)
   of cmdParse:
     wantMainModule(conf)
     discard parseFile(conf.projectMainIdx, cache, conf)

compiler/modulegraphs.nim

@@ -11,8 +11,8 @@
 ## represents a complete Nim project. Single modules can either be kept in RAM
 ## or stored in a rod-file.
 
-import intsets, tables, hashes, md5
-import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils
+import intsets, tables, hashes, md5, sequtils
+import ast, astalgo, options, lineinfos,idents, btrees, ropes, msgs, pathutils, packages
 import ic / [packed_ast, ic]
 
 type
@@ -50,6 +50,10 @@ type
     concreteTypes*: seq[FullId]
     inst*: PInstantiation
 
+  SymInfoPair* = object
+    sym*: PSym
+    info*: TLineInfo
+
   ModuleGraph* {.acyclic.} = ref object
     ifaces*: seq[Iface]  ## indexed by int32 fileIdx
     packed*: PackedModuleGraph
@@ -64,7 +68,6 @@ type
 
     startupPackedConfig*: PackedConfig
     packageSyms*: TStrTable
-    modulesPerPackage*: Table[ItemId, TStrTable]
     deps*: IntSet # the dependency graph or potentially its transitive closure.
     importDeps*: Table[FileIndex, seq[FileIndex]] # explicit import module dependencies
     suggestMode*: bool # whether we are in nimsuggest mode or not.
@@ -81,6 +84,8 @@ type
     doStopCompile*: proc(): bool {.closure.}
     usageSym*: PSym # for nimsuggest
     owners*: seq[PSym]
+    suggestSymbols*: Table[FileIndex, seq[SymInfoPair]]
+    suggestErrors*: Table[FileIndex, seq[Suggest]]
     methods*: seq[tuple[methods: seq[PSym], dispatcher: PSym]] # needs serialization!
     systemModule*: PSym
     sysTypes*: array[TTypeKind, PType]
@@ -368,20 +373,9 @@ template getPContext(): untyped =
   when c is PContext: c
   else: c.c
 
-when defined(nimfind):
-  template onUse*(info: TLineInfo; s: PSym) =
-    let c = getPContext()
-    if c.graph.onUsage != nil: c.graph.onUsage(c.graph, s, info)
-
-  template onDef*(info: TLineInfo; s: PSym) =
-    let c = getPContext()
-    if c.graph.onDefinition != nil: c.graph.onDefinition(c.graph, s, info)
-
-  template onDefResolveForward*(info: TLineInfo; s: PSym) =
-    let c = getPContext()
-    if c.graph.onDefinitionResolveForward != nil:
-      c.graph.onDefinitionResolveForward(c.graph, s, info)
-
+when defined(nimsuggest):
+  template onUse*(info: TLineInfo; s: PSym) = discard
+  template onDefResolveForward*(info: TLineInfo; s: PSym) = discard
 else:
   template onUse*(info: TLineInfo; s: PSym) = discard
   template onDef*(info: TLineInfo; s: PSym) = discard
@@ -432,8 +426,7 @@ proc initOperators*(g: ModuleGraph): Operators =
   result.opNot = createMagic(g, "not", mNot)
   result.opContains = createMagic(g, "contains", mInSet)
 
-proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
-  result = ModuleGraph()
+proc initModuleGraphFields(result: ModuleGraph) =
   # A module ID of -1 means that the symbol is not attached to a module at all,
   # but to the module graph:
   result.idgen = IdGenerator(module: -1'i32, symId: 0'i32, typeId: 0'i32)
@@ -443,9 +436,9 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.ifaces = @[]
   result.importStack = @[]
   result.inclToMod = initTable[FileIndex, FileIndex]()
-  result.config = config
-  result.cache = cache
   result.owners = @[]
+  result.suggestSymbols = initTable[FileIndex, seq[SymInfoPair]]()
+  result.suggestErrors = initTable[FileIndex, seq[Suggest]]()
   result.methods = @[]
   initStrTable(result.compilerprocs)
   initStrTable(result.exposed)
@@ -459,6 +452,12 @@ proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result.operators = initOperators(result)
   result.emittedTypeInfo = initTable[string, FileIndex]()
 
+proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
+  result = ModuleGraph()
+  result.config = config
+  result.cache = cache
+  initModuleGraphFields(result)
+
 proc resetAllModules*(g: ModuleGraph) =
   initStrTable(g.packageSyms)
   g.deps = initIntSet()
@@ -470,6 +469,7 @@ proc resetAllModules*(g: ModuleGraph) =
   g.methods = @[]
   initStrTable(g.compilerprocs)
   initStrTable(g.exposed)
+  initModuleGraphFields(g)
 
 proc getModule*(g: ModuleGraph; fileIdx: FileIndex): PSym =
   if fileIdx.int32 >= 0:
@@ -548,7 +548,19 @@ proc transitiveClosure(g: var IntSet; n: int) =
 
 proc markDirty*(g: ModuleGraph; fileIdx: FileIndex) =
   let m = g.getModule fileIdx
-  if m != nil: incl m.flags, sfDirty
+  if m != nil:
+    g.suggestSymbols.del(fileIdx)
+    g.suggestErrors.del(fileIdx)
+    incl m.flags, sfDirty
+
+proc unmarkAllDirty*(g: ModuleGraph) =
+  for i in 0i32..<g.ifaces.len.int32:
+    let m = g.ifaces[i].module
+    if m != nil:
+      m.flags.excl sfDirty
+
+proc isDirty*(g: ModuleGraph; m: PSym): bool =
+  result = g.suggestMode and sfDirty in m.flags
 
 proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
   # we need to mark its dependent modules D as dirty right away because after
@@ -560,12 +572,31 @@ proc markClientsDirty*(g: ModuleGraph; fileIdx: FileIndex) =
 
   # every module that *depends* on this file is also dirty:
   for i in 0i32..<g.ifaces.len.int32:
-    let m = g.ifaces[i].module
-    if m != nil and g.deps.contains(i.dependsOn(fileIdx.int)):
-      incl m.flags, sfDirty
+    if g.deps.contains(i.dependsOn(fileIdx.int)):
+      let
+        fi = FileIndex(i)
+        module = g.getModule(fi)
+      if module != nil and not g.isDirty(module):
+        g.markDirty(fi)
+        g.markClientsDirty(fi)
 
-proc isDirty*(g: ModuleGraph; m: PSym): bool =
-  result = g.suggestMode and sfDirty in m.flags
+proc needsCompilation*(g: ModuleGraph): bool =
+  # every module that *depends* on this file is also dirty:
+  for i in 0i32..<g.ifaces.len.int32:
+    let m = g.ifaces[i].module
+    if m != nil:
+      if sfDirty in m.flags:
+        return true
+
+proc needsCompilation*(g: ModuleGraph, fileIdx: FileIndex): bool =
+  let module = g.getModule(fileIdx)
+  if module != nil and g.isDirty(module):
+    return true
+
+  for i in 0i32..<g.ifaces.len.int32:
+    let m = g.ifaces[i].module
+    if m != nil and g.isDirty(m) and g.deps.contains(fileIdx.int32.dependsOn(i)):
+      return true
 
 proc getBody*(g: ModuleGraph; s: PSym): PNode {.inline.} =
   result = s.ast[bodyPos]
@@ -594,3 +625,34 @@ proc onProcessing*(graph: ModuleGraph, fileIdx: FileIndex, moduleStatus: string,
     let fromModule2 = if fromModule != nil: $fromModule.name.s else: "(toplevel)"
     let mode = if isNimscript: "(nims) " else: ""
     rawMessage(conf, hintProcessing, "$#$# $#: $#: $#" % [mode, indent, fromModule2, moduleStatus, path])
+
+proc getPackage*(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+  ## Returns a package symbol for yet to be defined module for fileIdx.
+  ## The package symbol is added to the graph if it doesn't exist.
+  let pkgSym = getPackage(graph.config, graph.cache, fileIdx)
+  # check if the package is already in the graph
+  result = graph.packageSyms.strTableGet(pkgSym.name)
+  if result == nil:
+     # the package isn't in the graph, so create and add it
+    result = pkgSym
+    graph.packageSyms.strTableAdd(pkgSym)
+
+func belongsToStdlib*(graph: ModuleGraph, sym: PSym): bool =
+  ## Check if symbol belongs to the 'stdlib' package.
+  sym.getPackageSymbol.getPackageId == graph.systemModule.getPackageId
+
+proc `==`*(a, b: SymInfoPair): bool =
+  result = a.sym == b.sym and a.info.exactEquals(b.info)
+
+proc fileSymbols*(graph: ModuleGraph, fileIdx: FileIndex): seq[SymInfoPair] =
+  result = graph.suggestSymbols.getOrDefault(fileIdx, @[])
+
+iterator suggestSymbolsIter*(g: ModuleGraph): SymInfoPair =
+  for xs in g.suggestSymbols.values:
+    for x in xs:
+      yield x
+
+iterator suggestErrorsIter*(g: ModuleGraph): Suggest =
+  for xs in g.suggestErrors.values:
+    for x in xs:
+      yield x

compiler/modulepaths.nim

@@ -10,100 +10,6 @@
 import ast, renderer, strutils, msgs, options, idents, os, lineinfos,
   pathutils
 
-when false:
-  const
-    considerParentDirs = not defined(noParentProjects)
-    considerNimbleDirs = not defined(noNimbleDirs)
-
-  proc findInNimbleDir(pkg, subdir, dir: string): string =
-    var best = ""
-    var bestv = ""
-    for k, p in os.walkDir(dir, relative=true):
-      if k == pcDir and p.len > pkg.len+1 and
-          p[pkg.len] == '-' and p.startsWith(pkg):
-        let (_, a, _) = getPathVersionChecksum(p)
-        if bestv.len == 0 or bestv < a:
-          bestv = a
-          best = dir / p
-
-    if best.len > 0:
-      var f: File
-      if open(f, best / changeFileExt(pkg, ".nimble-link")):
-        # the second line contains what we're interested in, see:
-        # https://github.com/nim-lang/nimble#nimble-link
-        var override = ""
-        discard readLine(f, override)
-        discard readLine(f, override)
-        close(f)
-        if not override.isAbsolute():
-          best = best / override
-        else:
-          best = override
-    let f = if subdir.len == 0: pkg else: subdir
-    let res = addFileExt(best / f, "nim")
-    if best.len > 0 and fileExists(res):
-      result = res
-
-when false:
-  proc resolveDollar(project, source, pkg, subdir: string; info: TLineInfo): string =
-    template attempt(a) =
-      let x = addFileExt(a, "nim")
-      if fileExists(x): return x
-
-    case pkg
-    of "stdlib":
-      if subdir.len == 0:
-        return options.libpath
-      else:
-        for candidate in stdlibDirs:
-          attempt(options.libpath / candidate / subdir)
-    of "root":
-      let root = project.splitFile.dir
-      if subdir.len == 0:
-        return root
-      else:
-        attempt(root / subdir)
-    else:
-      when considerParentDirs:
-        var p = parentDir(source.splitFile.dir)
-        # support 'import $karax':
-        let f = if subdir.len == 0: pkg else: subdir
-
-        while p.len > 0:
-          let dir = p / pkg
-          if dirExists(dir):
-            attempt(dir / f)
-            # 2nd attempt: try to use 'karax/karax'
-            attempt(dir / pkg / f)
-            # 3rd attempt: try to use 'karax/src/karax'
-            attempt(dir / "src" / f)
-            attempt(dir / "src" / pkg / f)
-          p = parentDir(p)
-
-      when considerNimbleDirs:
-        if not options.gNoNimblePath:
-          var nimbleDir = getEnv("NIMBLE_DIR")
-          if nimbleDir.len == 0: nimbleDir = getHomeDir() / ".nimble"
-          result = findInNimbleDir(pkg, subdir, nimbleDir / "pkgs")
-          if result.len > 0: return result
-          when not defined(windows):
-            result = findInNimbleDir(pkg, subdir, "/opt/nimble/pkgs")
-            if result.len > 0: return result
-
-  proc scriptableImport(pkg, sub: string; info: TLineInfo): string =
-    resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info)
-
-  proc lookupPackage(pkg, subdir: PNode): string =
-    let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: ""
-    case pkg.kind
-    of nkStrLit, nkRStrLit, nkTripleStrLit:
-      result = scriptableImport(pkg.strVal, sub, pkg.info)
-    of nkIdent:
-      result = scriptableImport(pkg.ident.s, sub, pkg.info)
-    else:
-      localError(pkg.info, "package name must be an identifier or string literal")
-      result = ""
-
 proc getModuleName*(conf: ConfigRef; n: PNode): string =
   # This returns a short relative module name without the nim extension
   # e.g. like "system", "importer" or "somepath/module"
@@ -163,3 +69,18 @@ proc checkModuleName*(conf: ConfigRef; n: PNode; doLocalError=true): FileIndex =
     result = InvalidFileIdx
   else:
     result = fileInfoIdx(conf, fullPath)
+
+proc mangleModuleName*(conf: ConfigRef; path: AbsoluteFile): string =
+  ## Mangle a relative module path to avoid path and symbol collisions.
+  ##
+  ## Used by backends that need to generate intermediary files from Nim modules.
+  ## This is needed because the compiler uses a flat cache file hierarchy.
+  ##
+  ## Example:
+  ## `foo-#head/../bar` becomes `@foo-@hhead@s..@sbar`
+  "@m" & relativeTo(path, conf.projectPath).string.multiReplace(
+    {$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"})
+
+proc demangleModuleName*(path: string): string =
+  ## Demangle a relative module path.
+  result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"})

compiler/modules.nim

@@ -12,7 +12,7 @@
 import
   ast, astalgo, magicsys, msgs, options,
   idents, lexer, passes, syntaxes, llstream, modulegraphs,
-  lineinfos, pathutils, tables
+  lineinfos, pathutils, tables, packages
 
 import ic / replayer
 
@@ -22,56 +22,11 @@ proc resetSystemArtifacts*(g: ModuleGraph) =
 template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent =
   getIdent(graph.cache, splitFile(filename).name)
 
-template packageId(): untyped {.dirty.} = ItemId(module: PackageModuleId, item: int32(fileIdx))
-
-proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym =
-  ## returns package symbol (skPackage) for yet to be defined module for fileIdx
-  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-  let name = getModuleIdent(graph, filename)
-  let info = newLineInfo(fileIdx, 1, 1)
-  let
-    pck = getPackageName(graph.config, filename.string)
-    pck2 = if pck.len > 0: pck else: "unknown"
-    pack = getIdent(graph.cache, pck2)
-  result = graph.packageSyms.strTableGet(pack)
-  if result == nil:
-    result = newSym(skPackage, getIdent(graph.cache, pck2), packageId(), nil, info)
-    #initStrTable(packSym.tab)
-    graph.packageSyms.strTableAdd(result)
-  else:
-    let modules = graph.modulesPerPackage.getOrDefault(result.itemId)
-    let existing = if modules.data.len > 0: strTableGet(modules, name) else: nil
-    if existing != nil and existing.info.fileIndex != info.fileIndex:
-      when false:
-        # we used to produce an error:
-        localError(graph.config, info,
-          "module names need to be unique per Nimble package; module clashes with " &
-            toFullPath(graph.config, existing.info.fileIndex))
-      else:
-        # but starting with version 0.20 we now produce a fake Nimble package instead
-        # to resolve the conflicts:
-        let pck3 = fakePackageName(graph.config, filename)
-        # this makes the new `result`'s owner be the original `result`
-        result = newSym(skPackage, getIdent(graph.cache, pck3), packageId(), result, info)
-        #initStrTable(packSym.tab)
-        graph.packageSyms.strTableAdd(result)
-
 proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
   let packSym = getPackage(graph, fileIdx)
   result.owner = packSym
   result.position = int fileIdx
 
-  #initStrTable(result.tab(graph))
-  when false:
-    strTableAdd(result.tab, result) # a module knows itself
-    # This is now implemented via
-    #   c.moduleScope.addSym(module) # a module knows itself
-    # in sem.nim, around line 527
-
-  if graph.modulesPerPackage.getOrDefault(packSym.itemId).data.len == 0:
-    graph.modulesPerPackage[packSym.itemId] = newStrTable()
-  graph.modulesPerPackage[packSym.itemId].strTableAdd(result)
-
 proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
   let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
   # We cannot call ``newSym`` here, because we have to circumvent the ID
@@ -133,7 +88,7 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
   #  localError(result.info, errAttemptToRedefine, result.name.s)
   # restore the notes for outer module:
   graph.config.notes =
-    if s.getnimblePkgId == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
+    if graph.config.belongsToProjectPackage(s) or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
     else: graph.config.foreignPackageNotes
 
 proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode =
@@ -168,7 +123,7 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
   conf.projectMainIdx2 = projectFile
 
   let packSym = getPackage(graph, projectFile)
-  graph.config.mainPackageId = packSym.getnimblePkgId
+  graph.config.mainPackageId = packSym.getPackageId
   graph.importStack.add projectFile
 
   if projectFile == systemFileIdx:

compiler/msgs.nim

@@ -618,9 +618,9 @@ template internalAssert*(conf: ConfigRef, e: bool) =
     let arg = info2.toFileLineCol
     internalErrorImpl(conf, unknownLineInfo, arg, info2)
 
-template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, forceHint = false, extraMsg = "") =
+template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string, extraMsg = "") =
   let m = "'$1' should be: '$2'$3" % [got, beau, extraMsg]
-  let msg = if optStyleError in conf.globalOptions and not forceHint: errGenerated else: hintName
+  let msg = if optStyleError in conf.globalOptions: errGenerated else: hintName
   liMessage(conf, info, msg, m, doNothing, instLoc())
 
 proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =

compiler/nim.cfg

@@ -10,6 +10,7 @@ define:nimPreviewFloatRoundtrip
 
 @if windows:
   cincludes: "$lib/wrappers/libffi/common"
+  tlsEmulation:off
 @end
 
 define:useStdoutAsStdmsg

compiler/nimconf.nim

@@ -298,7 +298,7 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen:
   if conf.cmd == cmdNimscript:
     showHintConf()
     conf.configFiles.setLen 0
-  if conf.cmd != cmdIdeTools:
+  if conf.cmd notin {cmdIdeTools, cmdCheck, cmdDump}:
     if conf.cmd == cmdNimscript:
       runNimScriptIfExists(conf.projectFull, isMain = true)
     else:
@@ -308,5 +308,6 @@ proc loadConfigs*(cfg: RelativeFile; cache: IdentCache; conf: ConfigRef; idgen:
       runNimScriptIfExists(scriptFile, isMain = true)
     else:
       # 'nimsuggest foo.nims' means to just auto-complete the NimScript file
+      # `nim check foo.nims' means to check the syntax of the NimScript file
       discard
   showHintConf()

compiler/options.nim

@@ -183,8 +183,9 @@ type
     # as far as usesWriteBarrier() is concerned
 
   IdeCmd* = enum
-    ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideMod,
-    ideHighlight, ideOutline, ideKnown, ideMsg, ideProject
+    ideNone, ideSug, ideCon, ideDef, ideUse, ideDus, ideChk, ideChkFile, ideMod,
+    ideHighlight, ideOutline, ideKnown, ideMsg, ideProject, ideGlobalSymbols,
+    ideRecompile, ideChanged, ideType, ideDeclaration
 
   Feature* = enum  ## experimental features; DO NOT RENAME THESE!
     implicitDeref,
@@ -597,7 +598,7 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool =
                             osQnx, osAtari, osAix,
                             osHaiku, osVxWorks, osSolaris, osNetbsd,
                             osFreebsd, osOpenbsd, osDragonfly, osMacosx, osIos,
-                            osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos}
+                            osAndroid, osNintendoSwitch, osFreeRTOS, osCrossos, osZephyr}
     of "linux":
       result = conf.target.targetOS in {osLinux, osAndroid}
     of "bsd":
@@ -617,6 +618,8 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool =
       result = conf.target.targetOS == osNintendoSwitch
     of "freertos", "lwip":
       result = conf.target.targetOS == osFreeRTOS
+    of "zephyr":
+      result = conf.target.targetOS == osZephyr
     of "littleendian": result = CPU[conf.target.targetCPU].endian == littleEndian
     of "bigendian": result = CPU[conf.target.targetCPU].endian == bigEndian
     of "cpu8": result = CPU[conf.target.targetCPU].bit == 8
@@ -799,6 +802,8 @@ proc toGeneratedFile*(conf: ConfigRef; path: AbsoluteFile,
 
 proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile,
                                 createSubDir: bool = true): AbsoluteFile =
+  ## Return an absolute path of a generated intermediary file.
+  ## Optionally creates the cache directory if `createSubDir` is `true`.
   let subdir = getNimcacheDir(conf)
   if createSubDir:
     try:
@@ -806,11 +811,6 @@ proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile,
     except OSError:
       conf.quitOrRaise "cannot create directory: " & subdir.string
   result = subdir / RelativeFile f.string.splitPath.tail
-  #echo "completeGeneratedFilePath(", f, ") = ", result
-
-proc toRodFile*(conf: ConfigRef; f: AbsoluteFile; ext = RodExt): AbsoluteFile =
-  result = changeFileExt(completeGeneratedFilePath(conf,
-    withPackageName(conf, f)), ext)
 
 proc rawFindFile(conf: ConfigRef; f: RelativeFile; suppressStdlib: bool): AbsoluteFile =
   for it in conf.searchPaths:
@@ -847,7 +847,7 @@ when (NimMajor, NimMinor) < (1, 1) or not declared(isRelativeTo):
     let ret = relativePath(path, base)
     result = path.len > 0 and not ret.startsWith ".."
 
-const stdlibDirs = [
+const stdlibDirs* = [
   "pure", "core", "arch",
   "pure/collections",
   "pure/concurrency",
@@ -989,12 +989,17 @@ proc parseIdeCmd*(s: string): IdeCmd =
   of "use": ideUse
   of "dus": ideDus
   of "chk": ideChk
+  of "chkFile": ideChkFile
   of "mod": ideMod
   of "highlight": ideHighlight
   of "outline": ideOutline
   of "known": ideKnown
   of "msg": ideMsg
   of "project": ideProject
+  of "globalSymbols": ideGlobalSymbols
+  of "recompile": ideRecompile
+  of "changed": ideChanged
+  of "type": ideType
   else: ideNone
 
 proc `$`*(c: IdeCmd): string =
@@ -1005,6 +1010,7 @@ proc `$`*(c: IdeCmd): string =
   of ideUse: "use"
   of ideDus: "dus"
   of ideChk: "chk"
+  of ideChkFile: "chkFile"
   of ideMod: "mod"
   of ideNone: "none"
   of ideHighlight: "highlight"
@@ -1012,6 +1018,11 @@ proc `$`*(c: IdeCmd): string =
   of ideKnown: "known"
   of ideMsg: "msg"
   of ideProject: "project"
+  of ideGlobalSymbols: "globalSymbols"
+  of ideDeclaration: "declaration"
+  of ideRecompile: "recompile"
+  of ideChanged: "changed"
+  of ideType: "type"
 
 proc floatInt64Align*(conf: ConfigRef): int16 =
   ## Returns either 4 or 8 depending on reasons.

compiler/packagehandling.nim

@@ -37,24 +37,7 @@ proc getNimbleFile*(conf: ConfigRef; path: string): string =
 proc getPackageName*(conf: ConfigRef; path: string): string =
   ## returns nimble package name, e.g.: `cligen`
   let path = getNimbleFile(conf, path)
-  result = path.splitFile.name
-
-proc fakePackageName*(conf: ConfigRef; path: AbsoluteFile): string =
-  # Convert `path` so that 2 modules with same name
-  # in different directory get different name and they can be
-  # placed in a directory.
-  # foo-#head/../bar becomes @foo-@hhead@s..@sbar
-  result = "@m" & relativeTo(path, conf.projectPath).string.multiReplace(
-    {$os.DirSep: "@s", $os.AltSep: "@s", "#": "@h", "@": "@@", ":": "@c"})
-
-proc demanglePackageName*(path: string): string =
-  result = path.multiReplace({"@@": "@", "@h": "#", "@s": "/", "@m": "", "@c": ":"})
-
-proc withPackageName*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile =
-  let x = getPackageName(conf, path.string)
-  let (p, file, ext) = path.splitFile
-  if x == "stdlib":
-    # Hot code reloading now relies on 'stdlib_system' names etc.
-    result = p / RelativeFile((x & '_' & file) & ext)
+  if path.len > 0:
+    return path.splitFile.name
   else:
-    result = p / RelativeFile(fakePackageName(conf, path))
+    return "unknown"

compiler/packages.nim

@@ -0,0 +1,49 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2022 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Package related procs.
+## 
+## See Also:
+## * `packagehandling` for package path handling
+## * `modulegraphs.getPackage`
+## * `modulegraphs.belongsToStdlib`
+
+import "." / [options, ast, lineinfos, idents, pathutils, msgs]
+
+proc getPackage*(conf: ConfigRef; cache: IdentCache; fileIdx: FileIndex): PSym =
+  ## Return a new package symbol.
+  ## 
+  ## See Also:
+  ## * `modulegraphs.getPackage`
+  let
+    filename = AbsoluteFile toFullPath(conf, fileIdx)
+    name = getIdent(cache, splitFile(filename).name)
+    info = newLineInfo(fileIdx, 1, 1)
+    pkgName = getPackageName(conf, filename.string)
+    pkgIdent = getIdent(cache, pkgName)
+  newSym(skPackage, pkgIdent, ItemId(module: PackageModuleId, item: int32(fileIdx)), nil, info)
+
+func getPackageSymbol*(sym: PSym): PSym =
+  ## Return the owning package symbol.
+  assert sym != nil
+  result = sym
+  while result.kind != skPackage:
+    result = result.owner
+    assert result != nil, repr(sym.info)
+
+func getPackageId*(sym: PSym): int =
+  ## Return the owning package ID.
+  sym.getPackageSymbol.id
+
+func belongsToProjectPackage*(conf: ConfigRef, sym: PSym): bool =
+  ## Return whether the symbol belongs to the project's package.
+  ## 
+  ## See Also:
+  ## * `modulegraphs.belongsToStdlib`
+  conf.mainPackageId == sym.getPackageId

compiler/parser.nim

@@ -582,10 +582,10 @@ proc parsePar(p: var Parser): PNode =
   #|         | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let'
   #|         | 'when' | 'var' | 'mixin'
   #| par = '(' optInd
-  #|           ( &parKeyw (ifExpr \ complexOrSimpleStmt) ^+ ';'
-  #|           | ';' (ifExpr \ complexOrSimpleStmt) ^+ ';'
+  #|           ( &parKeyw (ifExpr / complexOrSimpleStmt) ^+ ';'
+  #|           | ';' (ifExpr / complexOrSimpleStmt) ^+ ';'
   #|           | pragmaStmt
-  #|           | simpleExpr ( ('=' expr (';' (ifExpr \ complexOrSimpleStmt) ^+ ';' )? )
+  #|           | simpleExpr ( ('=' expr (';' (ifExpr / complexOrSimpleStmt) ^+ ';' )? )
   #|                        | (':' expr (',' exprColonEqExpr     ^+ ',' )? ) ) )
   #|           optPar ')'
   #

compiler/passes.nim

@@ -14,7 +14,8 @@ import
   options, ast, llstream, msgs,
   idents,
   syntaxes, modulegraphs, reorder,
-  lineinfos, pathutils
+  lineinfos, pathutils, std/sha1, packages
+
 
 type
   TPassData* = tuple[input: PNode, closeOutput: PNode]
@@ -101,7 +102,7 @@ const
 
 proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
   # don't be verbose unless the module belongs to the main package:
-  if module.getnimblePkgId == graph.config.mainPackageId:
+  if graph.config.belongsToProjectPackage(module):
     graph.config.notes = graph.config.mainPackageNotes
   else:
     if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes
@@ -111,12 +112,6 @@ proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
   result = true
   #module.id >= 0 or isDefined(graph.config, "nimBackendAssumesChange")
 
-proc partOfStdlib(x: PSym): bool =
-  var it = x.owner
-  while it != nil and it.kind == skPackage and it.owner != nil:
-    it = it.owner
-  result = it != nil and it.name.s == "stdlib"
-
 proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
                     stream: PLLStream): bool {.discardable.} =
   if graph.stopCompile(): return true
@@ -135,10 +130,15 @@ proc processModule*(graph: ModuleGraph; module: PSym; idgen: IdGenerator;
       return false
   else:
     s = stream
+
+  when defined(nimsuggest):
+    let filename = toFullPathConsiderDirty(graph.config, fileIdx).string
+    msgs.setHash(graph.config, fileIdx, $sha1.secureHashFile(filename))
+
   while true:
     openParser(p, fileIdx, s, graph.cache, graph.config)
 
-    if not partOfStdlib(module) or module.name.s == "distros":
+    if not belongsToStdlib(graph, module) or (belongsToStdlib(graph, module) and module.name.s == "distros"):
       # XXX what about caching? no processing then? what if I change the
       # modules to include between compilation runs? we'd need to track that
       # in ROD files. I think we should enable this feature only

compiler/platform.nim

@@ -22,7 +22,7 @@ type
     osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris,
     osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osCrossos, osAix, osPalmos, osQnx,
     osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks
-    osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osAny
+    osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osZephyr, osAny
 
 type
   TInfoOSProp* = enum
@@ -185,6 +185,10 @@ const
       objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
       scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
       props: {ospPosix}),
+     (name: "Zephyr", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
+      objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
+      scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
+      props: {ospPosix}),
      (name: "Any", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
       objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
       scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",

compiler/plugins/locals.nim

@@ -15,7 +15,7 @@ import ".." / [ast, astalgo,
 proc semLocals*(c: PContext, n: PNode): PNode =
   var counter = 0
   var tupleType = newTypeS(tyTuple, c)
-  result = newNodeIT(nkPar, n.info, tupleType)
+  result = newNodeIT(nkTupleConstr, n.info, tupleType)
   tupleType.n = newNodeI(nkRecList, n.info)
   let owner = getCurrOwner(c)
   # for now we skip openarrays ...

compiler/pragmas.nim

@@ -635,12 +635,13 @@ proc pragmaLine(c: PContext, n: PNode) =
     n.info = getInfoContext(c.config, -1)
 
 proc processPragma(c: PContext, n: PNode, i: int) =
+  ## Create and add a new custom pragma `{.pragma: name.}` node to the module's context.
   let it = n[i]
   if it.kind notin nkPragmaCallKinds and it.safeLen == 2: invalidPragma(c, n)
   elif it.safeLen != 2 or it[0].kind != nkIdent or it[1].kind != nkIdent:
     invalidPragma(c, n)
 
-  var userPragma = newSym(skTemplate, it[1].ident, nextSymId(c.idgen), nil, it.info, c.config.options)
+  var userPragma = newSym(skTemplate, it[1].ident, nextSymId(c.idgen), c.module, it.info, c.config.options)
   userPragma.ast = newTreeI(nkPragma, n.info, n.sons[i+1..^1])
   strTableAdd(c.userPragmas, userPragma)
 
@@ -822,8 +823,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
   let ident = considerQuotedIdent(c, key)
   var userPragma = strTableGet(c.userPragmas, ident)
   if userPragma != nil:
-    if {optStyleHint, optStyleError} * c.config.globalOptions != {}:
-      styleCheckUse(c.config, key.info, userPragma)
+    styleCheckUse(c, key.info, userPragma)
 
     # number of pragmas increase/decrease with user pragma expansion
     inc c.instCounter
@@ -837,8 +837,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
   else:
     let k = whichKeyword(ident)
     if k in validPragmas:
-      if {optStyleHint, optStyleError} * c.config.globalOptions != {}:
-        checkPragmaUse(c.config, key.info, k, ident.s)
+      checkPragmaUse(c.config, key.info, k, ident.s)
       case k
       of wExportc, wExportCpp:
         makeExternExport(c, sym, getOptionalStr(c, it, "$1"), it.info)
@@ -1243,7 +1242,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
     elif comesFromPush and whichKeyword(ident) != wInvalid:
       discard "ignore the .push pragma; it doesn't apply"
     else:
-      if sym == nil or (sym.kind in {skVar, skLet, skParam,
+      if sym == nil or (sym.kind in {skVar, skLet, skParam, skIterator,
                         skField, skProc, skFunc, skConverter, skMethod, skType}):
         n[i] = semCustomPragma(c, it)
       elif sym != nil:

compiler/renderer.nim

@@ -162,6 +162,7 @@ proc putNL(g: var TSrcGen) =
 proc optNL(g: var TSrcGen, indent: int) =
   g.pendingNL = indent
   g.lineLen = indent
+  g.col = g.indent
   when defined(nimpretty): g.pendingNewlineCount = 0
 
 proc optNL(g: var TSrcGen) =
@@ -170,6 +171,7 @@ proc optNL(g: var TSrcGen) =
 proc optNL(g: var TSrcGen; a, b: PNode) =
   g.pendingNL = g.indent
   g.lineLen = g.indent
+  g.col = g.indent
   when defined(nimpretty): g.pendingNewlineCount = lineDiff(a, b)
 
 proc indentNL(g: var TSrcGen) =

compiler/semexprs.nim

@@ -645,7 +645,7 @@ proc fixAbstractType(c: PContext, n: PNode) =
         skipTypes(it.typ, abstractVar).kind notin {tyOpenArray, tyVarargs}:
       if skipTypes(it[1].typ, abstractVar).kind in
             {tyNil, tyTuple, tySet} or it[1].isArrayConstr:
-        var s = skipTypes(it.typ, abstractVar)
+        var s = skipTypes(it.typ, abstractVar + tyUserTypeClasses)
         if s.kind != tyUntyped:
           changeType(c, it[1], s, check=true)
         n[i] = it[1]
@@ -1188,7 +1188,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
   let s = getGenSym(c, sym)
   case s.kind
   of skConst:
-    markUsed(c, n.info, s)
+    if n.kind != nkDotExpr: # dotExpr is already checked by builtinFieldAccess
+      markUsed(c, n.info, s)
     onUse(n.info, s)
     let typ = skipTypes(s.typ, abstractInst-{tyTypeDesc})
     case typ.kind
@@ -1249,7 +1250,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     if s.magic == mNimvm:
       localError(c.config, n.info, "illegal context for 'nimvm' magic")
 
-    markUsed(c, n.info, s)
+    if n.kind != nkDotExpr: # dotExpr is already checked by builtinFieldAccess
+      markUsed(c, n.info, s)
     onUse(n.info, s)
     result = newSymNode(s, n.info)
     # We cannot check for access to outer vars for example because it's still
@@ -1270,7 +1272,8 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
       n.typ = s.typ
       return n
   of skType:
-    markUsed(c, n.info, s)
+    if n.kind != nkDotExpr: # dotExpr is already checked by builtinFieldAccess
+      markUsed(c, n.info, s)
     onUse(n.info, s)
     if s.typ.kind == tyStatic and s.typ.base.kind != tyNone and s.typ.n != nil:
       return s.typ.n
@@ -1414,7 +1417,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
 
   if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass:
     ty = ty.lastSon
-  ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink})
+  ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink, tyStatic})
   while tfBorrowDot in ty.flags: ty = ty.skipTypes({tyDistinct, tyGenericInst, tyAlias})
   var check: PNode = nil
   if ty.kind == tyObject:
@@ -1433,8 +1436,10 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
         # is the access to a public field or in the same module or in a friend?
         markUsed(c, n[1].info, f)
         onUse(n[1].info, f)
+        let info = n[1].info
         n[0] = makeDeref(n[0])
         n[1] = newSymNode(f) # we now have the correct field
+        n[1].info = info # preserve the original info
         n.typ = f.typ
         if check == nil:
           result = n
@@ -2579,7 +2584,7 @@ proc semBlock(c: PContext, n: PNode; flags: TExprFlags): PNode =
       labl.owner = c.p.owner
     n[0] = newSymNode(labl, n[0].info)
     suggestSym(c.graph, n[0].info, labl, c.graph.usageSym)
-    styleCheckDef(c.config, labl)
+    styleCheckDef(c, labl)
     onDef(n[0].info, labl)
   n[1] = semExpr(c, n[1], flags)
   n.typ = n[1].typ

compiler/semfold.nim

@@ -412,7 +412,7 @@ proc foldConv(n, a: PNode; idgen: IdGenerator; g: ModuleGraph; check = false): P
       rangeCheck(n, getInt(result), g)
   of tyFloat..tyFloat64:
     case srcTyp.kind
-    of tyInt..tyInt64, tyEnum, tyBool, tyChar:
+    of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyBool, tyChar:
       result = newFloatNodeT(toFloat64(getOrdValue(a)), n, g)
     else:
       result = a

compiler/semgnrc.nim

@@ -179,7 +179,7 @@ proc fuzzyLookup(c: PContext, n: PNode, flags: TSemGenericFlags,
 proc addTempDecl(c: PContext; n: PNode; kind: TSymKind) =
   let s = newSymS(skUnknown, getIdentNode(c, n), c)
   addPrelimDecl(c, s)
-  styleCheckDef(c.config, n.info, s, kind)
+  styleCheckDef(c, n.info, s, kind)
   onDef(n.info, s)
 
 proc semGenericStmt(c: PContext, n: PNode,

compiler/semmagic.nim

@@ -433,7 +433,7 @@ proc semQuantifier(c: PContext; n: PNode): PNode =
       let op = considerQuotedIdent(c, it[0])
       if op.id == ord(wIn):
         let v = newSymS(skForVar, it[1], c)
-        styleCheckDef(c.config, v)
+        styleCheckDef(c, v)
         onDef(it[1].info, v)
         let domain = semExprWithType(c, it[2], {efWantIterator})
         v.typ = domain.typ

compiler/sempass2.nim

@@ -1051,7 +1051,7 @@ proc track(tracked: PEffects, n: PNode) =
     addAsgnFact(tracked.guards, n[0], n[1])
     notNilCheck(tracked, n[1], n[0].typ)
     when false: cstringCheck(tracked, n)
-    if tracked.owner.kind != skMacro:
+    if tracked.owner.kind != skMacro and n[0].typ.kind notin {tyOpenArray, tyVarargs}:
       createTypeBoundOps(tracked, n[0].typ, n.info)
     if n[0].kind != nkSym or not isLocalVar(tracked, n[0].sym):
       checkForSink(tracked.config, tracked.c.idgen, tracked.owner, n[1])
@@ -1216,6 +1216,11 @@ proc track(tracked: PEffects, n: PNode) =
       message(tracked.config, n.info, warnCstringConv,
         "implicit conversion to 'cstring' from a non-const location: $1; this will become a compile time error in the future" %
           $n[1])
+    if n.typ.skipTypes(abstractInst).kind == tyCstring and
+        isCharArrayPtr(n[1].typ, true):
+      message(tracked.config, n.info, warnPtrToCstringConv,
+          $n[1].typ)
+
 
     let t = n.typ.skipTypes(abstractInst)
     if t.kind == tyEnum:

compiler/semstmts.nim

@@ -306,7 +306,7 @@ proc identWithin(n: PNode, s: PIdent): bool =
     if identWithin(n[i], s): return true
   result = n.kind == nkSym and n.sym.name.id == s.id
 
-proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym =
+proc semIdentDef(c: PContext, n: PNode, kind: TSymKind, reportToNimsuggest = true): PSym =
   if isTopLevel(c):
     result = semIdentWithPragma(c, kind, n, {sfExported})
     incl(result.flags, sfGlobal)
@@ -330,7 +330,8 @@ proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym =
       discard
     result = n.info
   let info = getLineInfo(n)
-  suggestSym(c.graph, info, result, c.graph.usageSym)
+  if reportToNimsuggest:
+    suggestSym(c.graph, info, result, c.graph.usageSym)
 
 proc checkNilable(c: PContext; v: PSym) =
   if {sfGlobal, sfImportc} * v.flags == {sfGlobal} and v.typ.requiresInit:
@@ -368,7 +369,7 @@ proc semUsing(c: PContext; n: PNode): PNode =
       let typ = semTypeNode(c, a[^2], nil)
       for j in 0..<a.len-2:
         let v = semIdentDef(c, a[j], skParam)
-        styleCheckDef(c.config, v)
+        styleCheckDef(c, v)
         onDef(a[j].info, v)
         v.typ = typ
         strTableIncl(c.signatures, v)
@@ -595,8 +596,8 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
           if a.kind != nkVarTuple: typ else: tup[j])
         addToVarSection(c, result, n, a)
         continue
-      var v = semIdentDef(c, a[j], symkind)
-      styleCheckDef(c.config, v)
+      var v = semIdentDef(c, a[j], symkind, false)
+      styleCheckDef(c, v)
       onDef(a[j].info, v)
       if sfGenSym notin v.flags:
         if not isDiscardUnderscore(v): addInterfaceDecl(c, v)
@@ -660,6 +661,8 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode =
       if v.flags * {sfGlobal, sfThread} == {sfGlobal}:
         message(c.config, v.info, hintGlobalVar)
 
+      suggestSym(c.graph, v.info, v, c.graph.usageSym)
+
 proc semConst(c: PContext, n: PNode): PNode =
   result = copyNode(n)
   inc c.inStaticContext
@@ -719,7 +722,7 @@ proc semConst(c: PContext, n: PNode): PNode =
       var v = semIdentDef(c, a[j], skConst)
       if sfGenSym notin v.flags: addInterfaceDecl(c, v)
       elif v.owner == nil: v.owner = getCurrOwner(c)
-      styleCheckDef(c.config, v)
+      styleCheckDef(c, v)
       onDef(a[j].info, v)
 
       if a.kind != nkVarTuple:
@@ -744,7 +747,7 @@ include semfields
 proc symForVar(c: PContext, n: PNode): PSym =
   let m = if n.kind == nkPragmaExpr: n[0] else: n
   result = newSymG(skForVar, m, c)
-  styleCheckDef(c.config, result)
+  styleCheckDef(c, result)
   onDef(n.info, result)
   if n.kind == nkPragmaExpr:
     pragma(c, result, n[1], forVarPragmas)
@@ -975,7 +978,7 @@ proc semCase(c: PContext, n: PNode; flags: TExprFlags): PNode =
   var typ = commonTypeBegin
   var hasElse = false
   let caseTyp = skipTypes(n[0].typ, abstractVar-{tyTypeDesc})
-  const shouldChckCovered = {tyInt..tyInt64, tyChar, tyEnum, tyUInt..tyUInt32, tyBool}
+  const shouldChckCovered = {tyInt..tyInt64, tyChar, tyEnum, tyUInt..tyUInt64, tyBool}
   case caseTyp.kind
   of shouldChckCovered:
     chckCovered = true
@@ -1362,7 +1365,7 @@ proc typeSectionFinalPass(c: PContext, n: PNode) =
     let name = typeSectionTypeName(c, a[0])
     var s = name.sym
     # check the style here after the pragmas have been processed:
-    styleCheckDef(c.config, s)
+    styleCheckDef(c, s)
     # compute the type's size and check for illegal recursions:
     if a[1].kind == nkEmpty:
       var x = a[2]
@@ -1976,7 +1979,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
       ("'" & proto.name.s & "' from " & c.config$proto.info &
         " '" & s.name.s & "' from " & c.config$s.info))
 
-  styleCheckDef(c.config, s)
+  styleCheckDef(c, s)
   if hasProto:
     onDefResolveForward(n[namePos].info, proto)
   else:
@@ -2053,7 +2056,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
         trackProc(c, s, s.ast[bodyPos])
       else:
         if (s.typ[0] != nil and s.kind != skIterator):
-          addDecl(c, newSym(skUnknown, getIdent(c.cache, "result"), nextSymId c.idgen, nil, n.info))
+          addDecl(c, newSym(skUnknown, getIdent(c.cache, "result"), nextSymId c.idgen, s, n.info))
 
         openScope(c)
         n[bodyPos] = semGenericStmt(c, n[bodyPos])
@@ -2075,6 +2078,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind,
       incl(s.flags, sfWasForwarded)
     elif sfBorrow in s.flags: semBorrow(c, n, s)
   sideEffectsCheck(c, s)
+
   closeScope(c)           # close scope for parameters
   # c.currentScope = oldScope
   popOwner(c)

compiler/semtempl.nim

@@ -230,7 +230,7 @@ proc addLocalDecl(c: var TemplCtx, n: var PNode, k: TSymKind) =
       if n.kind != nkSym:
         let local = newGenSym(k, ident, c)
         addPrelimDecl(c.c, local)
-        styleCheckDef(c.c.config, n.info, local)
+        styleCheckDef(c.c, n.info, local)
         onDef(n.info, local)
         replaceIdentBySym(c.c, n, newSymNode(local, n.info))
         if k == skParam and c.inTemplateHeader > 0:
@@ -250,8 +250,14 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode =
   of skUnknown:
     # Introduced in this pass! Leave it as an identifier.
     result = n
-  of OverloadableSyms-{skEnumField}:
+  of OverloadableSyms-{skEnumField, skTemplate, skMacro}:
     result = symChoice(c, n, s, scOpen, isField)
+  of skTemplate, skMacro:
+    result = symChoice(c, n, s, scOpen, isField)
+    if result.kind == nkSym:
+      # template/macro symbols might need to be semchecked again
+      # prepareOperand etc don't do this without setting the type to nil
+      result.typ = nil
   of skGenericParam:
     if isField and sfGenSym in s.flags: result = n
     else: result = newSymNodeTypeDesc(s, c.idgen, n.info)
@@ -270,8 +276,9 @@ proc semTemplSymbol(c: PContext, n: PNode, s: PSym; isField: bool): PNode =
     # Issue #12832
     when defined(nimsuggest):
       suggestSym(c.graph, n.info, s, c.graph.usageSym, false)
-    if {optStyleHint, optStyleError} * c.config.globalOptions != {}:
-      styleCheckUse(c.config, n.info, s)
+    # field access (dot expr) will be handled by builtinFieldAccess
+    if not isField:
+      styleCheckUse(c, n.info, s)
 
 proc semRoutineInTemplName(c: var TemplCtx, n: PNode): PNode =
   result = n
@@ -296,7 +303,7 @@ proc semRoutineInTemplBody(c: var TemplCtx, n: PNode, k: TSymKind): PNode =
       var s = newGenSym(k, ident, c)
       s.ast = n
       addPrelimDecl(c.c, s)
-      styleCheckDef(c.c.config, n.info, s)
+      styleCheckDef(c.c, n.info, s)
       onDef(n.info, s)
       n[namePos] = newSymNode(s, n[namePos].info)
     else:
@@ -430,7 +437,7 @@ proc semTemplBody(c: var TemplCtx, n: PNode): PNode =
         # labels are always 'gensym'ed:
         let s = newGenSym(skLabel, n[0], c)
         addPrelimDecl(c.c, s)
-        styleCheckDef(c.c.config, s)
+        styleCheckDef(c.c, s)
         onDef(n[0].info, s)
         n[0] = newSymNode(s, n[0].info)
     n[1] = semTemplBody(c, n[1])
@@ -618,7 +625,7 @@ proc semTemplateDef(c: PContext, n: PNode): PNode =
        s.owner.name.s == "vm" and s.name.s == "stackTrace":
       incl(s.flags, sfCallsite)
 
-  styleCheckDef(c.config, s)
+  styleCheckDef(c, s)
   onDef(n[namePos].info, s)
   # check parameter list:
   #s.scope = c.currentScope

compiler/semtypes.nim

@@ -139,7 +139,7 @@ proc semEnum(c: PContext, n: PNode, prev: PType): PType =
       e.flags.incl {sfUsed, sfExported}
 
     result.n.add symNode
-    styleCheckDef(c.config, e)
+    styleCheckDef(c, e)
     onDef(e.info, e)
     if sfGenSym notin e.flags:
       if not isPure:
@@ -476,7 +476,7 @@ proc semTuple(c: PContext, n: PNode, prev: PType): PType =
       else:
         result.n.add newSymNode(field)
         addSonSkipIntLit(result, typ, c.idgen)
-      styleCheckDef(c.config, a[j].info, field)
+      styleCheckDef(c, a[j].info, field)
       onDef(field.info, field)
   if result.n.len == 0: result.n = nil
   if isTupleRecursive(result):
@@ -808,7 +808,7 @@ proc semRecordNodeAux(c: PContext, n: PNode, check: var IntSet, pos: var int,
         localError(c.config, info, "attempt to redefine: '" & f.name.s & "'")
       if a.kind == nkEmpty: father.add newSymNode(f)
       else: a.add newSymNode(f)
-      styleCheckDef(c.config, f)
+      styleCheckDef(c, f)
       onDef(f.info, f)
     if a.kind != nkEmpty: father.add a
   of nkSym:
@@ -1248,7 +1248,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
     if hasDefault:
       def = a[^1]
       block determineType:
-        if genericParams.isGenericParams:
+        if genericParams != nil and genericParams.len > 0:
           def = semGenericStmt(c, def)
           if hasUnresolvedArgs(c, def):
             def.typ = makeTypeFromExpr(c, def.copyTree)
@@ -1315,7 +1315,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
       result.n.add newSymNode(arg)
       rawAddSon(result, finalType)
       addParamOrResult(c, arg, kind)
-      styleCheckDef(c.config, a[j].info, arg)
+      styleCheckDef(c, a[j].info, arg)
       onDef(a[j].info, arg)
       if {optNimV1Emulation, optNimV12Emulation} * c.config.globalOptions == {}:
         a[j] = newSymNode(arg)
@@ -1376,7 +1376,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode,
           result.flags.excl tfHasMeta
       result.n.typ = r
 
-  if genericParams.isGenericParams:
+  if genericParams != nil and genericParams.len > 0:
     for n in genericParams:
       if {sfUsed, sfAnon} * n.sym.flags == {}:
         result.flags.incl tfUnresolved

compiler/sighashes.nim

@@ -16,7 +16,7 @@ import types
 proc `&=`(c: var MD5Context, s: string) = md5Update(c, s, s.len)
 proc `&=`(c: var MD5Context, ch: char) =
   # XXX suspicious code here; relies on ch being zero terminated?
-  md5Update(c, unsafeAddr ch, 1)
+  md5Update(c, cast[cstring](unsafeAddr ch), 1)
 proc `&=`(c: var MD5Context, r: Rope) =
   for l in leaves(r): md5Update(c, l.cstring, l.len)
 proc `&=`(c: var MD5Context, i: BiggestInt) =

compiler/sizealignoffsetimpl.nim

@@ -40,15 +40,15 @@ proc inc(arg: var OffsetAccum; value: int) =
   else:
     arg.offset += value
 
-proc alignmentMax(a,b: int): int =
+proc alignmentMax(a, b: int): int =
   if unlikely(a == szIllegalRecursion or b == szIllegalRecursion): raiseIllegalTypeRecursion()
   if a == szUnknownSize or b == szUnknownSize:
     szUnknownSize
   else:
-    max(a,b)
+    max(a, b)
 
 proc align(arg: var OffsetAccum; value: int) =
-  if unlikely(value == szIllegalRecursion):  raiseIllegalTypeRecursion()
+  if unlikely(value == szIllegalRecursion): raiseIllegalTypeRecursion()
   if value == szUnknownSize or arg.maxAlign == szUnknownSize or arg.offset == szUnknownSize:
     arg.maxAlign = szUnknownSize
     arg.offset = szUnknownSize
@@ -112,7 +112,7 @@ proc setOffsetsToUnknown(n: PNode) =
     for i in 0..<n.safeLen:
       setOffsetsToUnknown(n[i])
 
-proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode, packed: bool, accum: var OffsetAccum) =
+proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode; packed: bool; accum: var OffsetAccum) =
   ## ``offset`` is the offset within the object, after the node has been written, no padding bytes added
   ## ``align`` maximum alignment from all sub nodes
   assert n != nil
@@ -380,10 +380,8 @@ proc computeSizeAlign(conf: ConfigRef; typ: PType) =
           let info = if typ.sym != nil: typ.sym.info else: unknownLineInfo
           localError(conf, info, "union type may not have an object header")
           accum = OffsetAccum(offset: szUnknownSize, maxAlign: szUnknownSize)
-        elif tfPacked in typ.flags:
-          computeUnionObjectOffsetsFoldFunction(conf, typ.n, true, accum)
         else:
-          computeUnionObjectOffsetsFoldFunction(conf, typ.n, false, accum)
+          computeUnionObjectOffsetsFoldFunction(conf, typ.n, tfPacked in typ.flags, accum)
       elif tfPacked in typ.flags:
         accum.maxAlign = 1
         computeObjectOffsetsFoldFunction(conf, typ.n, true, accum)
@@ -497,7 +495,7 @@ template foldOffsetOf*(conf: ConfigRef; n: PNode; fallback: PNode): PNode =
   ## Returns an int literal node of the given offsetof expression in `n`.
   ## Falls back to `fallback`, if the `offsetof` expression can't be processed.
   let config = conf
-  let node : PNode = n
+  let node = n
   var dotExpr: PNode
   block findDotExpr:
     if node[1].kind == nkDotExpr:

compiler/spawn.nim

@@ -221,7 +221,7 @@ proc setupArgsForConcurrency(g: ModuleGraph; n: PNode; objType: PType;
     let fieldname = if i < formals.len: formals[i].sym.name else: tmpName
     var field = newSym(skField, fieldname, nextSymId idgen, objType.owner, n.info, g.config.options)
     field.typ = argType
-    objType.addField(field, g.cache, idgen)
+    discard objType.addField(field, g.cache, idgen)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[i])
 
     let temp = addLocalVar(g, varSection, varInit, idgen, owner, argType,
@@ -260,17 +260,17 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType;
       slice[0].typ = getSysType(g, n.info, tyInt) # fake type
       var fieldB = newSym(skField, tmpName, nextSymId idgen, objType.owner, n.info, g.config.options)
       fieldB.typ = getSysType(g, n.info, tyInt)
-      objType.addField(fieldB, g.cache, idgen)
+      discard objType.addField(fieldB, g.cache, idgen)
 
       if getMagic(n) == mSlice:
         let a = genAddrOf(n[1], idgen)
         field.typ = a.typ
-        objType.addField(field, g.cache, idgen)
+        discard objType.addField(field, g.cache, idgen)
         result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
 
         var fieldA = newSym(skField, tmpName, nextSymId idgen, objType.owner, n.info, g.config.options)
         fieldA.typ = getSysType(g, n.info, tyInt)
-        objType.addField(fieldA, g.cache, idgen)
+        discard objType.addField(fieldA, g.cache, idgen)
         result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldA), n[2])
         result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), n[3])
 
@@ -281,7 +281,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType;
       else:
         let a = genAddrOf(n, idgen)
         field.typ = a.typ
-        objType.addField(field, g.cache, idgen)
+        discard objType.addField(field, g.cache, idgen)
         result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
         result.add newFastAsgnStmt(newDotExpr(scratchObj, fieldB), genHigh(g, n))
 
@@ -299,7 +299,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType;
       # it is more efficient to pass a pointer instead:
       let a = genAddrOf(n, idgen)
       field.typ = a.typ
-      objType.addField(field, g.cache, idgen)
+      discard objType.addField(field, g.cache, idgen)
       result.add newFastAsgnStmt(newDotExpr(scratchObj, field), a)
       let threadLocal = addLocalVar(g, varSection, nil, idgen, owner, field.typ,
                                     indirectAccess(castExpr, field, n.info),
@@ -308,7 +308,7 @@ proc setupArgsForParallelism(g: ModuleGraph; n: PNode; objType: PType;
     else:
       # boring case
       field.typ = argType
-      objType.addField(field, g.cache, idgen)
+      discard objType.addField(field, g.cache, idgen)
       result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n)
       let threadLocal = addLocalVar(g, varSection, varInit,
                                     idgen, owner, field.typ,
@@ -377,7 +377,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp
     var argType = n[0].typ.skipTypes(abstractInst)
     var field = newSym(skField, getIdent(g.cache, "fn"), nextSymId idgen, owner, n.info, g.config.options)
     field.typ = argType
-    objType.addField(field, g.cache, idgen)
+    discard objType.addField(field, g.cache, idgen)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), n[0])
     fn = indirectAccess(castExpr, field, n.info)
   elif fn.kind == nkSym and fn.sym.kind == skIterator:
@@ -399,7 +399,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp
     typ.rawAddSon(magicsys.getCompilerProc(g, "Barrier").typ)
     var field = newSym(skField, getIdent(g.cache, "barrier"), nextSymId idgen, owner, n.info, g.config.options)
     field.typ = typ
-    objType.addField(field, g.cache, idgen)
+    discard objType.addField(field, g.cache, idgen)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), barrier)
     barrierAsExpr = indirectAccess(castExpr, field, n.info)
 
@@ -407,7 +407,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp
   if spawnKind == srFlowVar:
     var field = newSym(skField, getIdent(g.cache, "fv"), nextSymId idgen, owner, n.info, g.config.options)
     field.typ = retType
-    objType.addField(field, g.cache, idgen)
+    discard objType.addField(field, g.cache, idgen)
     fvField = newDotExpr(scratchObj, field)
     fvAsExpr = indirectAccess(castExpr, field, n.info)
     # create flowVar:
@@ -419,7 +419,7 @@ proc wrapProcForSpawn*(g: ModuleGraph; idgen: IdGenerator; owner: PSym; spawnExp
     var field = newSym(skField, getIdent(g.cache, "fv"), nextSymId idgen, owner, n.info, g.config.options)
     field.typ = newType(tyPtr, nextTypeId idgen, objType.owner)
     field.typ.rawAddSon(retType)
-    objType.addField(field, g.cache, idgen)
+    discard objType.addField(field, g.cache, idgen)
     fvAsExpr = indirectAccess(castExpr, field, n.info)
     result.add newFastAsgnStmt(newDotExpr(scratchObj, field), genAddrOf(dest, idgen))
 

compiler/suggest.nim

@@ -117,7 +117,7 @@ proc getTokenLenFromSource(conf: ConfigRef; ident: string; info: TLineInfo): int
     elif sourceIdent != ident:
       result = 0
 
-proc symToSuggest(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
+proc symToSuggest*(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info: TLineInfo;
                   quality: range[0..100]; prefix: PrefixMatch;
                   inTypeContext: bool; scope: int;
                   useSuppliedInfo = false): Suggest =
@@ -157,19 +157,25 @@ proc symToSuggest(g: ModuleGraph; s: PSym, isLocal: bool, section: IdeCmd, info:
       result.forth = ""
     when defined(nimsuggest) and not defined(noDocgen) and not defined(leanCompiler):
       result.doc = extractDocComment(g, s)
-  let infox =
-    if useSuppliedInfo or section in {ideUse, ideHighlight, ideOutline}:
-      info
-    else:
-      s.info
-  result.filePath = toFullPath(g.config, infox)
-  result.line = toLinenumber(infox)
-  result.column = toColumn(infox)
+  if s.kind == skModule and s.ast.len != 0 and section != ideHighlight:
+    result.filePath = toFullPath(g.config, s.ast[0].info)
+    result.line = 1
+    result.column = 0
+    result.tokenLen = 0
+  else:
+    let infox =
+      if useSuppliedInfo or section in {ideUse, ideHighlight, ideOutline, ideDeclaration}:
+        info
+      else:
+        s.info
+    result.filePath = toFullPath(g.config, infox)
+    result.line = toLinenumber(infox)
+    result.column = toColumn(infox)
+    result.tokenLen = if section != ideHighlight:
+                        s.name.s.len
+                      else:
+                        getTokenLenFromSource(g.config, s.name.s, infox)
   result.version = g.config.suggestVersion
-  result.tokenLen = if section != ideHighlight:
-                      s.name.s.len
-                    else:
-                      getTokenLenFromSource(g.config, s.name.s, infox)
 
 proc `$`*(suggest: Suggest): string =
   result = $suggest.section
@@ -203,14 +209,14 @@ proc `$`*(suggest: Suggest): string =
     result.add(sep)
     when defined(nimsuggest) and not defined(noDocgen) and not defined(leanCompiler):
       result.add(suggest.doc.escape)
-    if suggest.version == 0:
+    if suggest.version in {0, 3}:
       result.add(sep)
       result.add($suggest.quality)
       if suggest.section == ideSug:
         result.add(sep)
         result.add($suggest.prefix)
 
-proc suggestResult(conf: ConfigRef; s: Suggest) =
+proc suggestResult*(conf: ConfigRef; s: Suggest) =
   if not isNil(conf.suggestionResultHook):
     conf.suggestionResultHook(s)
   else:
@@ -424,7 +430,7 @@ proc suggestFieldAccess(c: PContext, n, field: PNode, outputs: var Suggestions)
         t = skipTypes(t[0], skipPtrs)
     elif typ.kind == tyTuple and typ.n != nil:
       suggestSymList(c, typ.n, field, n.info, outputs)
-    
+
     suggestOperations(c, n, field, orig, outputs)
     if typ != orig:
       suggestOperations(c, n, field, typ, outputs)
@@ -482,7 +488,7 @@ proc findDefinition(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym
   if s.isNil: return
   if isTracked(info, g.config.m.trackPos, s.name.s.len) or (s == usageSym and sfForward notin s.flags):
     suggestResult(g.config, symToSuggest(g, s, isLocal=false, ideDef, info, 100, PrefixMatch.None, false, 0, useSuppliedInfo = s == usageSym))
-    if sfForward notin s.flags:
+    if sfForward notin s.flags and g.config.suggestVersion != 3:
       suggestQuit()
     else:
       usageSym = s
@@ -497,6 +503,8 @@ proc suggestSym*(g: ModuleGraph; info: TLineInfo; s: PSym; usageSym: var PSym; i
   ## misnamed: should be 'symDeclared'
   let conf = g.config
   when defined(nimsuggest):
+    g.suggestSymbols.mgetOrPut(info.fileIndex, @[]).add SymInfoPair(sym: s, info: info)
+
     if conf.suggestVersion == 0:
       if s.allUsages.len == 0:
         s.allUsages = @[info]
@@ -596,8 +604,7 @@ proc markUsed(c: PContext; info: TLineInfo; s: PSym) =
     if sfError in s.flags: userError(conf, info, s)
   when defined(nimsuggest):
     suggestSym(c.graph, info, s, c.graph.usageSym, false)
-  if {optStyleHint, optStyleError} * conf.globalOptions != {}:
-    styleCheckUse(conf, info, s)
+  styleCheckUse(c, info, s)
   markOwnerModuleAsUsed(c, s)
 
 proc safeSemExpr*(c: PContext, n: PNode): PNode =
@@ -692,3 +699,16 @@ proc suggestSentinel*(c: PContext) =
 
   dec(c.compilesContextId)
   produceOutput(outputs, c.config)
+
+when defined(nimsuggest):
+  proc onDef(graph: ModuleGraph, s: PSym, info: TLineInfo) =
+    if graph.config.suggestVersion == 3 and info.exactEquals(s.info):
+       suggestSym(graph, info, s, graph.usageSym)
+
+  template getPContext(): untyped =
+    when c is PContext: c
+    else: c.c
+
+  template onDef*(info: TLineInfo; s: PSym) =
+    let c = getPContext()
+    onDef(c.graph, s, info)

compiler/typeallowed.nim

@@ -57,6 +57,8 @@ proc typeAllowedAux(marker: var IntSet, typ: PType, kind: TSymKind,
   of tyVar, tyLent:
     if kind in {skProc, skFunc, skConst} and (views notin c.features):
       result = t
+    elif taIsOpenArray in flags:
+      result = t
     elif t.kind == tyLent and ((kind != skResult and views notin c.features) or
                               kind == skParam): # lent can't be used as parameters.
       result = t
@@ -231,7 +233,7 @@ proc classifyViewTypeAux(marker: var IntSet, t: PType): ViewTypeKind =
   case t.kind
   of tyVar:
     result = mutableView
-  of tyLent, tyOpenArray:
+  of tyLent, tyOpenArray, tyVarargs:
     result = immutableView
   of tyGenericInst, tyDistinct, tyAlias, tyInferred, tySink, tyOwned,
      tyUncheckedArray, tySequence, tyArray, tyRef, tyStatic:

compiler/types.nim

@@ -1145,13 +1145,14 @@ proc sameTypeAux(x, y: PType, c: var TSameTypeClosure): bool =
   of tyEmpty, tyChar, tyBool, tyNil, tyPointer, tyString, tyCstring,
      tyInt..tyUInt64, tyTyped, tyUntyped, tyVoid:
     result = sameFlags(a, b)
-    if result and PickyCAliases in c.flags:
+    if result and {PickyCAliases, ExactTypeDescValues} <= c.flags:
       # additional requirement for the caching of generics for importc'ed types:
       # the symbols must be identical too:
       let symFlagsA = if a.sym != nil: a.sym.flags else: {}
       let symFlagsB = if b.sym != nil: b.sym.flags else: {}
       if (symFlagsA+symFlagsB) * {sfImportc, sfExportc} != {}:
         result = symFlagsA == symFlagsB
+
   of tyStatic, tyFromExpr:
     result = exprStructuralEquivalent(a.n, b.n) and sameFlags(a, b)
     if result and a.len == b.len and a.len == 1:

compiler/varpartitions.nim

@@ -848,7 +848,7 @@ proc computeLiveRanges(c: var Partitions; n: PNode) =
     #     connect(graph, cursorVar)
     inc c.inLoop
     for child in n: computeLiveRanges(c, child)
-    inc c.inLoop
+    dec c.inLoop
   of nkElifBranch, nkElifExpr, nkElse, nkOfBranch:
     inc c.inConditional
     for child in n: computeLiveRanges(c, child)

compiler/vm.nim

@@ -85,9 +85,9 @@ proc bailOut(c: PCtx; tos: PStackFrame) =
 when not defined(nimComputedGoto):
   {.pragma: computedGoto.}
 
-proc ensureKind(n: var TFullReg, kind: TRegisterKind) =
-  if n.kind != kind:
-    n = TFullReg(kind: kind)
+proc ensureKind(n: var TFullReg, k: TRegisterKind) {.inline.} =
+  if n.kind != k:
+    n = TFullReg(kind: k)
 
 template ensureKind(k: untyped) {.dirty.} =
   ensureKind(regs[ra], k)
@@ -119,13 +119,13 @@ template move(a, b: untyped) {.dirty.} = system.shallowCopy(a, b)
 
 proc derefPtrToReg(address: BiggestInt, typ: PType, r: var TFullReg, isAssign: bool): bool =
   # nim bug: `isAssign: static bool` doesn't work, giving odd compiler error
-  template fun(field, T, rkind) =
+  template fun(field, typ, rkind) =
     if isAssign:
-      cast[ptr T](address)[] = T(r.field)
+      cast[ptr typ](address)[] = typ(r.field)
     else:
       r.ensureKind(rkind)
-      let val = cast[ptr T](address)[]
-      when T is SomeInteger | char:
+      let val = cast[ptr typ](address)[]
+      when typ is SomeInteger | char:
         r.field = BiggestInt(val)
       else:
         r.field = val
@@ -521,6 +521,19 @@ template maybeHandlePtr(node2: PNode, reg: TFullReg, isAssign2: bool): bool =
 when not defined(nimHasSinkInference):
   {.pragma: nosinks.}
 
+template takeAddress(reg, source) =
+  reg.nodeAddr = addr source
+  GC_ref source
+
+proc takeCharAddress(c: PCtx, src: PNode, index: BiggestInt, pc: int): TFullReg =
+  let typ = newType(tyPtr, nextTypeId c.idgen, c.module.owner)
+  typ.add getSysType(c.graph, c.debug[pc], tyChar)
+  var node = newNodeIT(nkIntLit, c.debug[pc], typ) # xxx nkPtrLit
+  node.intVal = cast[int](src.strVal[index].addr)
+  node.flags.incl nfIsPtr
+  TFullReg(kind: rkNode, node: node)
+
+
 proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
   var pc = start
   var tos = tos
@@ -637,7 +650,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcNodeToReg:
       let ra = instr.regA
       let rb = instr.regB
-      # opcDeref might already have loaded it into a register. XXX Let's hope
+      # opcLdDeref might already have loaded it into a register. XXX Let's hope
       # this is still correct this way:
       if regs[rb].kind != rkNode:
         regs[ra] = regs[rb]
@@ -654,6 +667,48 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         else:
           ensureKind(rkNode)
           regs[ra].node = nb
+    of opcSlice:
+      # A bodge, but this takes in `toOpenArray(rb, rc, rc)` and emits
+      # nkTupleConstr(x, y, z) into the `regs[ra]`. These can later be used for calculating the slice we have taken.
+      decodeBC(rkNode)
+      let
+        collection = regs[ra].node
+        leftInd = regs[rb].intVal
+        rightInd = regs[rc].intVal
+
+      proc rangeCheck(left, right: BiggestInt, safeLen: BiggestInt) =
+        if left < 0:
+          stackTrace(c, tos, pc, formatErrorIndexBound(left, safeLen))
+
+        if right > safeLen:
+          stackTrace(c, tos, pc, formatErrorIndexBound(right, safeLen))
+
+      case collection.kind
+      of nkTupleConstr: # slice of a slice
+        let safeLen = collection[2].intVal - collection[1].intVal
+        rangeCheck(leftInd, rightInd, safeLen)
+        let
+          leftInd = leftInd + collection[1].intVal # Slice is from the start of the old
+          rightInd = rightInd + collection[1].intVal
+
+        regs[ra].node = newTree(
+          nkTupleConstr,
+          collection[0],
+          newIntNode(nkIntLit, BiggestInt leftInd),
+          newIntNode(nkIntLit, BiggestInt rightInd)
+        )
+
+      else:
+        let safeLen = safeArrLen(collection) - 1
+        rangeCheck(leftInd, rightInd, safeLen)
+        regs[ra].node = newTree(
+          nkTupleConstr,
+          collection,
+          newIntNode(nkIntLit, BiggestInt leftInd),
+          newIntNode(nkIntLit, BiggestInt rightInd)
+        )
+
+
     of opcLdArr:
       # a = b[c]
       decodeBC(rkNode)
@@ -661,7 +716,24 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         stackTrace(c, tos, pc, formatErrorIndexBound(regs[rc].intVal, high(int)))
       let idx = regs[rc].intVal.int
       let src = regs[rb].node
-      if src.kind in {nkStrLit..nkTripleStrLit}:
+      case src.kind
+      of nkTupleConstr: # refer to `of opcSlice`
+        let
+          left = src[1].intVal
+          right = src[2].intVal
+          realIndex = left + idx
+        if idx in 0..(right - left):
+          case src[0].kind
+          of nkStrKinds:
+            regs[ra].node =  newIntNode(nkCharLit, ord src[0].strVal[int realIndex])
+          of nkBracket:
+            regs[ra].node = src[0][int realIndex]
+          else:
+            stackTrace(c, tos, pc, "opcLdArr internal error")
+        else:
+          stackTrace(c, tos, pc, formatErrorIndexBound(idx, int right))
+
+      of nkStrLit..nkTripleStrLit:
         if idx <% src.strVal.len:
           regs[ra].node = newNodeI(nkCharLit, c.debug[pc])
           regs[ra].node.intVal = src.strVal[idx].ord
@@ -678,10 +750,27 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         stackTrace(c, tos, pc, formatErrorIndexBound(regs[rc].intVal, high(int)))
       let idx = regs[rc].intVal.int
       let src = if regs[rb].kind == rkNode: regs[rb].node else: regs[rb].nodeAddr[]
-      if src.kind notin {nkEmpty..nkTripleStrLit} and idx <% src.len:
-        regs[ra].nodeAddr = addr src.sons[idx]
+      case src.kind
+      of nkTupleConstr:
+        let
+          left = src[1].intVal
+          right = src[2].intVal
+          realIndex = left + idx
+        if idx in 0..(right - left): # Refer to `opcSlice`
+          case src[0].kind
+          of nkStrKinds:
+            regs[ra] = takeCharAddress(c, src[0], realIndex, pc)
+          of nkBracket:
+            takeAddress regs[ra], src.sons[0].sons[realIndex]
+          else:
+            stackTrace(c, tos, pc, "opcLdArrAddr internal error")
+        else:
+          stackTrace(c, tos, pc, formatErrorIndexBound(idx, int right))
       else:
-        stackTrace(c, tos, pc, formatErrorIndexBound(idx, src.safeLen-1))
+        if src.kind notin {nkEmpty..nkTripleStrLit} and idx <% src.len:
+          takeAddress regs[ra], src.sons[idx]
+        else:
+          stackTrace(c, tos, pc, formatErrorIndexBound(idx, src.safeLen-1))
     of opcLdStrIdx:
       decodeBC(rkInt)
       let idx = regs[rc].intVal.int
@@ -698,13 +787,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       let idx = regs[rc].intVal.int
       let s = regs[rb].node.strVal.addr # or `byaddr`
       if idx <% s[].len:
-         # `makePtrType` not accessible from vm.nim
-        let typ = newType(tyPtr, nextTypeId c.idgen, c.module.owner)
-        typ.add getSysType(c.graph, c.debug[pc], tyChar)
-        let node = newNodeIT(nkIntLit, c.debug[pc], typ) # xxx nkPtrLit
-        node.intVal = cast[int](s[][idx].addr)
-        node.flags.incl nfIsPtr
-        regs[ra].node = node
+        regs[ra] = takeCharAddress(c, regs[rb].node, idx, pc)
       else:
         stackTrace(c, tos, pc, formatErrorIndexBound(idx, s[].len-1))
     of opcWrArr:
@@ -712,7 +795,24 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       decodeBC(rkNode)
       let idx = regs[rb].intVal.int
       let arr = regs[ra].node
-      if arr.kind in {nkStrLit..nkTripleStrLit}:
+      case arr.kind
+      of nkTupleConstr: # refer to `opcSlice`
+        let
+          src = arr[0]
+          left = arr[1].intVal
+          right = arr[2].intVal
+          realIndex = left + idx
+        if idx in 0..(right - left):
+          case src.kind
+          of nkStrKinds:
+            src.strVal[int(realIndex)] = char(regs[rc].intVal)
+          of nkBracket:
+            src[int(realIndex)] = regs[rc].node
+          else:
+            stackTrace(c, tos, pc, "opcWrArr internal error")
+        else:
+          stackTrace(c, tos, pc, formatErrorIndexBound(idx, int right))
+      of {nkStrLit..nkTripleStrLit}:
         if idx <% arr.strVal.len:
           arr.strVal[idx] = chr(regs[rc].intVal)
         else:
@@ -747,11 +847,11 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       of nkObjConstr:
         let n = src.sons[rc + 1]
         if n.kind == nkExprColonExpr:
-          regs[ra].nodeAddr = addr n.sons[1]
+          takeAddress regs[ra], n.sons[1]
         else:
-          regs[ra].nodeAddr = addr src.sons[rc + 1]
+          takeAddress regs[ra], src.sons[rc + 1]
       else:
-        regs[ra].nodeAddr = addr src.sons[rc]
+        takeAddress regs[ra], src.sons[rc]
     of opcWrObj:
       # a.b = c
       decodeBC(rkNode)
@@ -778,7 +878,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       decodeB(rkNodeAddr)
       case regs[rb].kind
       of rkNode:
-        regs[ra].nodeAddr = addr(regs[rb].node)
+        takeAddress regs[ra], regs[rb].node
       of rkNodeAddr: # bug #14339
         regs[ra].nodeAddr = regs[rb].nodeAddr
       else:
@@ -870,14 +970,21 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
     of opcLenSeq:
       decodeBImm(rkInt)
       #assert regs[rb].kind == nkBracket
-      let high = (imm and 1) # discard flags
+      let
+        high = (imm and 1) # discard flags
+        node = regs[rb].node
       if (imm and nimNodeFlag) != 0:
         # used by mNLen (NimNode.len)
         regs[ra].intVal = regs[rb].node.safeLen - high
       else:
-        # safeArrLen also return string node len
-        # used when string is passed as openArray in VM
-        regs[ra].intVal = regs[rb].node.safeArrLen - high
+        case node.kind
+        of nkTupleConstr: # refer to `of opcSlice`
+          regs[ra].intVal = node[2].intVal - node[1].intVal + 1 - high
+        else:
+          # safeArrLen also return string node len
+          # used when string is passed as openArray in VM
+          regs[ra].intVal = node.safeArrLen - high
+
     of opcLenStr:
       decodeBImm(rkInt)
       assert regs[rb].kind == rkNode
@@ -1006,6 +1113,12 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
       decodeBC(rkInt)
       template getTyp(n): untyped =
         n.typ.skipTypes(abstractInst)
+      template skipRegisterAddr(n: TFullReg): TFullReg =
+        var tmp = n
+        while tmp.kind == rkRegisterAddr:
+          tmp = tmp.regAddr[]
+        tmp
+
       proc ptrEquality(n1: ptr PNode, n2: PNode): bool =
         ## true if n2.intVal represents a ptr equal to n1
         let p1 = cast[int](n1)
@@ -1019,16 +1132,19 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
           return t2.kind in PtrLikeKinds and n2.intVal == p1
         else: return false
 
-      if regs[rb].kind == rkNodeAddr:
-        if regs[rc].kind == rkNodeAddr:
-          ret = regs[rb].nodeAddr == regs[rc].nodeAddr
+      let rbReg = skipRegisterAddr(regs[rb])
+      let rcReg = skipRegisterAddr(regs[rc])
+
+      if rbReg.kind == rkNodeAddr:
+        if rcReg.kind == rkNodeAddr:
+          ret = rbReg.nodeAddr == rcReg.nodeAddr
         else:
-          ret = ptrEquality(regs[rb].nodeAddr, regs[rc].node)
-      elif regs[rc].kind == rkNodeAddr:
-        ret = ptrEquality(regs[rc].nodeAddr, regs[rb].node)
+          ret = ptrEquality(rbReg.nodeAddr, rcReg.node)
+      elif rcReg.kind == rkNodeAddr:
+        ret = ptrEquality(rcReg.nodeAddr, rbReg.node)
       else:
-        let nb = regs[rb].node
-        let nc = regs[rc].node
+        let nb = rbReg.node
+        let nc = rcReg.node
         if nb.kind != nc.kind: discard
         elif (nb == nc) or (nb.kind == nkNilLit): ret = true # intentional
         elif nb.kind in {nkSym, nkTupleConstr, nkClosure} and nb.typ != nil and nb.typ.kind == tyProc and sameConstant(nb, nc):
@@ -2266,7 +2382,6 @@ proc setupMacroParam(x: PNode, typ: PType): TFullReg =
   else:
     var n = x
     if n.kind in {nkHiddenSubConv, nkHiddenStdConv}: n = n[1]
-    n = n.canonValue
     n.flags.incl nfIsRef
     n.typ = x.typ
     result = TFullReg(kind: rkNode, node: n)

compiler/vmdef.nim

@@ -81,6 +81,7 @@ type
     opcWrStrIdx,
     opcLdStrIdx, # a = b[c]
     opcLdStrIdxAddr,  # a = addr(b[c])
+    opcSlice, # toOpenArray(collection, left, right)
 
     opcAddInt,
     opcAddImmInt,

compiler/vmgen.nim

@@ -441,14 +441,11 @@ proc genAndOr(c: PCtx; n: PNode; opc: TOpcode; dest: var TDest) =
     c.gABC(n, opcAsgnInt, dest, tmp)
     freeTemp(c, tmp)
 
-proc canonValue*(n: PNode): PNode =
-  result = n
-
 proc rawGenLiteral(c: PCtx; n: PNode): int =
   result = c.constants.len
   #assert(n.kind != nkCall)
   n.flags.incl nfAllConst
-  c.constants.add n.canonValue
+  c.constants.add n
   internalAssert c.config, result < regBxMax
 
 proc sameConstant*(a, b: PNode): bool =
@@ -646,9 +643,19 @@ proc genCheckedObjAccessAux(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags
 proc genAsgnPatch(c: PCtx; le: PNode, value: TRegister) =
   case le.kind
   of nkBracketExpr:
-    let dest = c.genx(le[0], {gfNode})
-    let idx = c.genIndex(le[1], le[0].typ)
-    c.gABC(le, opcWrArr, dest, idx, value)
+    let
+      dest = c.genx(le[0], {gfNode})
+      idx = c.genIndex(le[1], le[0].typ)
+      collTyp = le[0].typ.skipTypes(abstractVarRange-{tyTypeDesc})
+
+    case collTyp.kind
+    of tyString, tyCstring:
+      c.gABC(le, opcWrStrIdx, dest, idx, value)
+    of tyTuple:
+      c.gABC(le, opcWrObj, dest, int le[1].intVal, value)
+    else:
+      c.gABC(le, opcWrArr, dest, idx, value)
+
     c.freeTemp(dest)
     c.freeTemp(idx)
   of nkCheckedFieldExpr:
@@ -1060,6 +1067,18 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     of tyString: genUnaryABI(c, n, dest, opcLenStr)
     of tyCstring: genUnaryABI(c, n, dest, opcLenCstring)
     else: doAssert false, $n[1].typ.kind
+  of mSlice:
+    var
+      d = c.genx(n[1])
+      left = c.genIndex(n[2], n[1].typ)
+      right = c.genIndex(n[3], n[1].typ)
+    if dest < 0: dest = c.getTemp(n.typ)
+    c.gABC(n, opcNodeToReg, dest, d)
+    c.gABC(n, opcSlice, dest, left, right)
+    c.freeTemp(left)
+    c.freeTemp(right)
+    c.freeTemp(d)
+
   of mIncl, mExcl:
     unused(c, n, dest)
     var d = c.genx(n[1])
@@ -1185,7 +1204,7 @@ proc genMagic(c: PCtx; n: PNode; dest: var TDest; m: TMagic) =
     var d = c.genx(n[1])
     # XXX use ldNullOpcode() here?
     c.gABx(n, opcLdNull, d, c.genType(n[1].typ))
-    c.gABx(n, opcNodeToReg, d, d)
+    c.gABC(n, opcNodeToReg, d, d)
     c.genAsgnPatch(n[1], d)
   of mDefault:
     if dest < 0: dest = c.getTemp(n.typ)
@@ -1403,9 +1422,6 @@ proc unneededIndirection(n: PNode): bool =
   n.typ.skipTypes(abstractInstOwned-{tyTypeDesc}).kind == tyRef
 
 proc canElimAddr(n: PNode): PNode =
-  if n[0].typ.skipTypes(abstractInst).kind in {tyObject, tyTuple, tyArray}:
-    # objects are reference types in the VM
-    return n[0]
   case n[0].kind
   of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64:
     var m = n[0][0]
@@ -1497,7 +1513,7 @@ proc checkCanEval(c: PCtx; n: PNode) =
   # proc foo() = var x ...
   let s = n.sym
   if {sfCompileTime, sfGlobal} <= s.flags: return
-  if s.importcCondVar: return
+  if compiletimeFFI in c.config.features and s.importcCondVar: return
   if s.kind in {skVar, skTemp, skLet, skParam, skResult} and
       not s.isOwnedBy(c.prc.sym) and s.owner != c.module and c.mode != emRepl:
     # little hack ahead for bug #12612: assume gensym'ed variables
@@ -1532,12 +1548,16 @@ proc preventFalseAlias(c: PCtx; n: PNode; opc: TOpcode;
 proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) =
   case le.kind
   of nkBracketExpr:
-    let dest = c.genx(le[0], {gfNode})
-    let idx = c.genIndex(le[1], le[0].typ)
-    let tmp = c.genx(ri)
-    if le[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind in {
-        tyString, tyCstring}:
+    let
+      dest = c.genx(le[0], {gfNode})
+      idx = c.genIndex(le[1], le[0].typ)
+      tmp = c.genx(ri)
+      collTyp = le[0].typ.skipTypes(abstractVarRange-{tyTypeDesc})
+    case collTyp.kind
+    of tyString, tyCstring:
       c.preventFalseAlias(le, opcWrStrIdx, dest, idx, tmp)
+    of tyTuple:
+      c.preventFalseAlias(le, opcWrObj, dest, int le[1].intVal, tmp)
     else:
       c.preventFalseAlias(le, opcWrArr, dest, idx, tmp)
     c.freeTemp(tmp)
@@ -1701,9 +1721,7 @@ proc genArrAccessOpcode(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
   c.freeTemp(a)
   c.freeTemp(b)
 
-proc genObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
-  let a = c.genx(n[0], flags)
-  let b = genField(c, n[1])
+proc genObjAccessAux(c: PCtx; n: PNode; a, b: int, dest: var TDest; flags: TGenFlags) =
   if dest < 0: dest = c.getTemp(n.typ)
   if {gfNodeAddr} * flags != {}:
     c.gABC(n, opcLdObjAddr, dest, a, b)
@@ -1716,6 +1734,11 @@ proc genObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
     c.gABC(n, opcLdObj, dest, a, b)
   c.freeTemp(a)
 
+proc genObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
+  genObjAccessAux(c, n, c.genx(n[0], flags), genField(c, n[1]), dest, flags)
+
+
+
 proc genCheckedObjAccessAux(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
   internalAssert c.config, n.kind == nkCheckedFieldExpr
   # nkDotExpr to access the requested field
@@ -1783,10 +1806,13 @@ proc genCheckedObjAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
 
 proc genArrAccess(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags) =
   let arrayType = n[0].typ.skipTypes(abstractVarRange-{tyTypeDesc}).kind
-  if arrayType in {tyString, tyCstring}:
+  case arrayType
+  of tyString, tyCstring:
     let opc = if gfNodeAddr in flags: opcLdStrIdxAddr else: opcLdStrIdx
     genArrAccessOpcode(c, n, dest, opc, flags)
-  elif arrayType == tyTypeDesc:
+  of tyTuple:
+    c.genObjAccessAux(n, c.genx(n[0], flags), int n[1].intVal, dest, flags)
+  of tyTypeDesc:
     c.genTypeLit(n.typ, dest)
   else:
     let opc = if gfNodeAddr in flags: opcLdArrAddr else: opcLdArr
@@ -1875,7 +1901,7 @@ proc genVarSection(c: PCtx; n: PNode) =
           else:
             let sa = getNullValue(s.typ, a.info, c.config)
             #if s.ast.isNil: getNullValue(s.typ, a.info)
-            #else: canonValue(s.ast)
+            #else: s.ast
             assert sa.kind != nkCall
             c.globals.add(sa)
             s.position = c.globals.len

compiler/vmops.nim

@@ -56,13 +56,13 @@ template macrosop(op) {.dirty.} =
 template md5op(op) {.dirty.} =
   registerCallback(c, "stdlib.md5." & astToStr(op), `op Wrapper`)
 
-template wrap1f_math(op) {.dirty.} =
+template wrap1fMath(op) {.dirty.} =
   proc `op Wrapper`(a: VmArgs) {.nimcall.} =
     doAssert a.numArgs == 1
     setResult(a, op(getFloat(a, 0)))
   mathop op
 
-template wrap2f_math(op) {.dirty.} =
+template wrap2fMath(op) {.dirty.} =
   proc `op Wrapper`(a: VmArgs) {.nimcall.} =
     setResult(a, op(getFloat(a, 0), getFloat(a, 1)))
   mathop op
@@ -170,40 +170,40 @@ proc registerAdditionalOps*(c: PCtx) =
   proc getProjectPathWrapper(a: VmArgs) =
     setResult a, c.config.projectPath.string
 
-  wrap1f_math(sqrt)
-  wrap1f_math(cbrt)
-  wrap1f_math(ln)
-  wrap1f_math(log10)
-  wrap1f_math(log2)
-  wrap1f_math(exp)
-  wrap1f_math(arccos)
-  wrap1f_math(arcsin)
-  wrap1f_math(arctan)
-  wrap1f_math(arcsinh)
-  wrap1f_math(arccosh)
-  wrap1f_math(arctanh)
-  wrap2f_math(arctan2)
-  wrap1f_math(cos)
-  wrap1f_math(cosh)
-  wrap2f_math(hypot)
-  wrap1f_math(sinh)
-  wrap1f_math(sin)
-  wrap1f_math(tan)
-  wrap1f_math(tanh)
-  wrap2f_math(pow)
-  wrap1f_math(trunc)
-  wrap1f_math(floor)
-  wrap1f_math(ceil)
-  wrap1f_math(erf)
-  wrap1f_math(erfc)
-  wrap1f_math(gamma)
-  wrap1f_math(lgamma)
+  wrap1fMath(sqrt)
+  wrap1fMath(cbrt)
+  wrap1fMath(ln)
+  wrap1fMath(log10)
+  wrap1fMath(log2)
+  wrap1fMath(exp)
+  wrap1fMath(arccos)
+  wrap1fMath(arcsin)
+  wrap1fMath(arctan)
+  wrap1fMath(arcsinh)
+  wrap1fMath(arccosh)
+  wrap1fMath(arctanh)
+  wrap2fMath(arctan2)
+  wrap1fMath(cos)
+  wrap1fMath(cosh)
+  wrap2fMath(hypot)
+  wrap1fMath(sinh)
+  wrap1fMath(sin)
+  wrap1fMath(tan)
+  wrap1fMath(tanh)
+  wrap2fMath(pow)
+  wrap1fMath(trunc)
+  wrap1fMath(floor)
+  wrap1fMath(ceil)
+  wrap1fMath(erf)
+  wrap1fMath(erfc)
+  wrap1fMath(gamma)
+  wrap1fMath(lgamma)
 
   when declared(copySign):
-    wrap2f_math(copySign)
+    wrap2fMath(copySign)
 
   when declared(signbit):
-    wrap1f_math(signbit)
+    wrap1fMath(signbit)
 
   registerCallback c, "stdlib.math.round", proc (a: VmArgs) {.nimcall.} =
     let n = a.numArgs

compiler/wordrecg.nim

@@ -41,7 +41,7 @@ type
     wImmediate = "immediate", wConstructor = "constructor", wDestructor = "destructor",
     wDelegator = "delegator", wOverride = "override", wImportCpp = "importcpp",
     wCppNonPod = "cppNonPod",
-    wImportObjC = "importobjc", wImportCompilerProc = "importcompilerproc",
+    wImportObjC = "importobjc", wImportCompilerProc = "importCompilerProc",
     wImportc = "importc", wImportJs = "importjs", wExportc = "exportc", wExportCpp = "exportcpp",
     wExportNims = "exportnims",
     wIncompleteStruct = "incompleteStruct", # deprecated
@@ -89,23 +89,23 @@ type
     wLiftLocals = "liftlocals", wEnforceNoRaises = "enforceNoRaises",
 
     wAuto = "auto", wBool = "bool", wCatch = "catch", wChar = "char",
-    wClass = "class", wCompl = "compl", wConst_cast = "const_cast", wDefault = "default",
-    wDelete = "delete", wDouble = "double", wDynamic_cast = "dynamic_cast",
+    wClass = "class", wCompl = "compl", wConstCast = "const_cast", wDefault = "default",
+    wDelete = "delete", wDouble = "double", wDynamicCast = "dynamic_cast",
     wExplicit = "explicit", wExtern = "extern", wFalse = "false", wFloat = "float",
     wFriend = "friend", wGoto = "goto", wInt = "int", wLong = "long", wMutable = "mutable",
     wNamespace = "namespace", wNew = "new", wOperator = "operator", wPrivate = "private",
     wProtected = "protected", wPublic = "public", wRegister = "register",
-    wReinterpret_cast = "reinterpret_cast", wRestrict = "restrict", wShort = "short",
-    wSigned = "signed", wSizeof = "sizeof", wStatic_cast = "static_cast", wStruct = "struct",
+    wReinterpretCast = "reinterpret_cast", wRestrict = "restrict", wShort = "short",
+    wSigned = "signed", wSizeof = "sizeof", wStaticCast = "static_cast", wStruct = "struct",
     wSwitch = "switch", wThis = "this", wThrow = "throw", wTrue = "true", wTypedef = "typedef",
     wTypeid = "typeid", wTypeof = "typeof",  wTypename = "typename",
     wUnion = "union", wPacked = "packed", wUnsigned = "unsigned", wVirtual = "virtual",
-    wVoid = "void", wVolatile = "volatile", wWchar_t = "wchar_t",
+    wVoid = "void", wVolatile = "volatile", wWchar = "wchar_t",
 
     wAlignas = "alignas", wAlignof = "alignof", wConstexpr = "constexpr", wDecltype = "decltype",
     wNullptr = "nullptr", wNoexcept = "noexcept",
-    wThread_local = "thread_local", wStatic_assert = "static_assert",
-    wChar16_t = "char16_t", wChar32_t = "char32_t",
+    wThreadLocal = "thread_local", wStaticAssert = "static_assert",
+    wChar16 = "char16_t", wChar32 = "char32_t",
 
     wStdIn = "stdin", wStdOut = "stdout", wStdErr = "stderr",
 

config/config.nims

@@ -3,6 +3,11 @@
 cppDefine "errno"
 cppDefine "unix"
 
+# mangle the macro names in nimbase.h
+cppDefine "NAN_INFINITY"
+cppDefine "INF"
+cppDefine "NAN"
+
 when defined(nimStrictMode):
   # xxx add more flags here, and use `-d:nimStrictMode` in more contexts in CI.
 

config/nim.cfg

@@ -156,6 +156,9 @@ nimblepath="$home/.nimble/pkgs/"
 # Configuration for the GNU C/C++ compiler:
 @if windows:
   #gcc.path = r"$nim\dist\mingw\bin"
+  @if gcc or tcc:
+    tlsEmulation:on
+  @end
 @end
 
 gcc.maxerrorsimpl = "-fmax-errors=3"

doc/advopt.txt

@@ -57,7 +57,8 @@ Advanced options:
   -w:on|off|list, --warnings:on|off|list
                             same as `--hints` but for warnings.
   --warning:X:on|off        ditto
-  --warningAsError:X:on|off ditto
+  --warningAsError:X:on|off
+                            ditto
   --styleCheck:off|hint|error
                             produce hints or errors for Nim identifiers that
                             do not adhere to Nim's official style guide

doc/grammar.txt

@@ -37,10 +37,10 @@ parKeyw = 'discard' | 'include' | 'if' | 'while' | 'case' | 'try'
         | 'finally' | 'except' | 'for' | 'block' | 'const' | 'let'
         | 'when' | 'var' | 'mixin'
 par = '(' optInd
-          ( &parKeyw (ifExpr \ complexOrSimpleStmt) ^+ ';'
-          | ';' (ifExpr \ complexOrSimpleStmt) ^+ ';'
+          ( &parKeyw (ifExpr / complexOrSimpleStmt) ^+ ';'
+          | ';' (ifExpr / complexOrSimpleStmt) ^+ ';'
           | pragmaStmt
-          | simpleExpr ( ('=' expr (';' (ifExpr \ complexOrSimpleStmt) ^+ ';' )? )
+          | simpleExpr ( ('=' expr (';' (ifExpr / complexOrSimpleStmt) ^+ ';' )? )
                        | (':' expr (',' exprColonEqExpr     ^+ ',' )? ) ) )
           optPar ')'
 literal = | INT_LIT | INT8_LIT | INT16_LIT | INT32_LIT | INT64_LIT

doc/manual.rst

@@ -1616,7 +1616,7 @@ type conversions in this context:
 
   myWriteln(stdout, 123, "abc", 4.0)
   # is transformed to:
-  myWriteln(stdout, [$123, $"def", $4.0])
+  myWriteln(stdout, [$123, $"abc", $4.0])
 
 In this example `$` is applied to any argument that is passed to the
 parameter `a`. (Note that `$` applied to strings is a nop.)
@@ -1899,7 +1899,7 @@ A small example:
 cast uncheckedAssign
 --------------------
 
-Some restrictions for case objects can be disabled via a `{.cast(unsafeAssign).}` section:
+Some restrictions for case objects can be disabled via a `{.cast(uncheckedAssign).}` section:
 
 .. code-block:: nim
     :test: "nim c $1"
@@ -5002,7 +5002,7 @@ be used:
 
 See also:
 
-- `Shared heap memory management <gc.html>`_.
+- `Shared heap memory management <mm.html>`_.
 
 
 
@@ -6488,6 +6488,19 @@ iterator in which case the overloading resolution takes place:
   write(stdout, x) # not ambiguous: uses the module C's x
 
 
+Packages
+--------
+A collection of modules in a file tree with an ``identifier.nimble`` file in the
+root of the tree is called a Nimble package. A valid package name can only be a
+valid Nim identifier and thus its filename is ``identifier.nimble`` where
+``identifier`` is the desired package name. A module without a ``.nimble`` file
+is assigned the package identifier: `unknown`.
+
+The distinction between packages allows diagnostic compiler messages to be
+scoped to the current project's package vs foreign packages.
+
+
+
 Compiler Messages
 =================
 
@@ -7123,7 +7136,7 @@ The `link` pragma can be used to link an additional file with the project:
   {.link: "myfile.o".}
 
 
-PassC pragma
+passc pragma
 ------------
 The `passc` pragma can be used to pass additional parameters to the C
 compiler like one would using the command-line switch `--passc`:option:\:
@@ -7151,20 +7164,20 @@ the pragma resides in:
   {.localPassc: "-Wall -Werror".} # Passed when compiling A.nim.cpp
 
 
-PassL pragma
+passl pragma
 ------------
-The `passL` pragma can be used to pass additional parameters to the linker
-like one would be using the command-line switch `--passL`:option:\:
+The `passl` pragma can be used to pass additional parameters to the linker
+like one would be using the command-line switch `--passl`:option:\:
 
 .. code-block:: Nim
-  {.passL: "-lSDLmain -lSDL".}
+  {.passl: "-lSDLmain -lSDL".}
 
 Note that one can use `gorge` from the `system module <system.html>`_ to
 embed parameters from an external command that will be executed
 during semantic analysis:
 
 .. code-block:: Nim
-  {.passL: gorge("pkg-config --libs sdl").}
+  {.passl: gorge("pkg-config --libs sdl").}
 
 
 Emit pragma
@@ -7478,7 +7491,7 @@ allows *sloppy* interfacing with libraries written in Objective C:
 .. code-block:: Nim
   # horrible example of how to interface with GNUStep ...
 
-  {.passL: "-lobjc".}
+  {.passl: "-lobjc".}
   {.emit: """
   #include <objc/Object.h>
   @interface Greeter:Object

doc/nimc.rst

@@ -679,6 +679,39 @@ is not available but C's `malloc` is. You can use the `nimAllocPagesViaMalloc`
 define to use `malloc` instead of `mmap`. `nimAllocPagesViaMalloc` is currently
 only supported with `--mm:arc` or `--mm:orc`. (Since version 1.6)
 
+nimPage256 / nimPage512 / nimPage1k
+===================================
+
+Adjust the page size for Nim's GC allocator. This enables using
+`nimAllocPagesViaMalloc` on devices with less RAM. The default
+page size requires too much RAM to work.
+
+Recommended settings:
+
+- < 32 kB of RAM use `nimPage256`
+
+- < 512 kB of RAM use `nimPage512`
+
+- < 2 MB of RAM use `nimPage1k`
+
+Initial testing hasn't shown much difference between 512B or 1kB page sizes
+in terms of performance or latency. Using `nimPages256` will limit the
+total amount of allocatable RAM.
+
+nimMemAlignTiny
+===============
+
+Sets `MemAlign` to `4` bytes which reduces the memory alignment
+to better match some embedded devices.
+
+Thread stack size 
+=================
+
+Nim's thread API provides a simple wrapper around more advanced
+RTOS task features. Customizing the stack size and stack guard size can
+be done by setting `-d:nimThreadStackSize=16384` or `-d:nimThreadStackGuard=32`.
+
+Currently only Zephyr and FreeRTOS support these configurations. 
 
 Nim for realtime systems
 ========================

doc/testament.rst

@@ -36,7 +36,7 @@ Options
                           (for debugging)
 --failing                 Only show failing/ignored tests
 --targets:"c cpp js objc"
-                          Run tests for specified targets (default: all)
+                          Run tests for specified targets (default: c)
 --nim:path                Use a particular nim executable (default: $PATH/nim)
 --directory:dir           Change to directory dir before reading the tests
                           or doing anything else.
@@ -164,7 +164,7 @@ Example "template" **to edit** and write a Testament unittest:
     # Timeout seconds to run the test. Fractional values are supported.
     timeout: 1.5
 
-    # Targets to run the test into (c, cpp, objc, js).
+    # Targets to run the test into (c, cpp, objc, js). Defaults to c.
     targets: "c js"
 
     # flags with which to run the test, delimited by `;`

koch.nim

@@ -89,9 +89,6 @@ Commands for core developers:
   tests [options]          run the testsuite (run a subset of tests by
                            specifying a category, e.g. `tests cat async`)
   temp options             creates a temporary compiler for testing
-Web options:
-  --googleAnalytics:UA-... add the given google analytics code to the docs. To
-                           build the official docs, use UA-48159761-1
 """
 
 let kochExe* = when isMainModule: os.getAppFilename() # always correct when koch is main program, even if `koch` exe renamed e.g.: `nim c -o:koch_debug koch.nim`
@@ -332,9 +329,9 @@ proc boot(args: string) =
     # in order to use less memory, we split the build into two steps:
     # --compileOnly produces a $project.json file and does not run GCC/Clang.
     # jsonbuild then uses the $project.json file to build the Nim binary.
-    exec "$# $# $# --nimcache:$# $# --compileOnly compiler" / "nim.nim" %
+    exec "$# $# $# --nimcache:$# $# --noNimblePath --compileOnly compiler" / "nim.nim" %
       [nimi, bootOptions, extraOption, smartNimcache, args]
-    exec "$# jsonscript --nimcache:$# $# compiler" / "nim.nim" %
+    exec "$# jsonscript --noNimblePath --nimcache:$# $# compiler" / "nim.nim" %
       [nimi, smartNimcache, args]
 
     if sameFileContent(output, i.thVersion):
@@ -559,7 +556,8 @@ proc runCI(cmd: string) =
 
   let batchParam = "--batch:$1" % "NIM_TESTAMENT_BATCH".getEnv("_")
   if getEnv("NIM_TEST_PACKAGES", "0") == "1":
-    execFold("Test selected Nimble packages", "nim r testament/testament $# pcat nimble-packages" % batchParam)
+    nimCompileFold("Compile testament", "testament/testament.nim", options = "-d:release")
+    execFold("Test selected Nimble packages", "testament $# pcat nimble-packages" % batchParam)
   else:
     buildTools()
 
@@ -693,7 +691,7 @@ when isMainModule:
       case normalize(op.key)
       of "boot": boot(op.cmdLineRest)
       of "clean": clean(op.cmdLineRest)
-      of "doc", "docs": buildDocs(op.cmdLineRest, localDocsOnly, localDocsOut)
+      of "doc", "docs": buildDocs(op.cmdLineRest & paCode, localDocsOnly, localDocsOut)
       of "doc0", "docs0":
         # undocumented command for Araq-the-merciful:
         buildDocs(op.cmdLineRest & gaCode)

lib/core/macros.nim

@@ -1352,7 +1352,9 @@ proc `$`*(node: NimNode): string =
   of nnkOpenSymChoice, nnkClosedSymChoice:
     result = $node[0]
   of nnkAccQuoted:
-    result = $node[0]
+    result = ""
+    for i in 0 ..< node.len:
+      result.add(repr(node[i]))
   else:
     badNodeKind node, "$"
 
@@ -1718,8 +1720,8 @@ proc extractDocCommentsAndRunnables*(n: NimNode): NimNode =
     case ni.kind
     of nnkCommentStmt:
       result.add ni
-    of nnkCall:
-      if ni[0].kind == nnkIdent and ni[0].strVal == "runnableExamples":
+    of nnkCall, nnkCommand:
+      if ni[0].kind == nnkIdent and ni[0].eqIdent "runnableExamples":
         result.add ni
       else: break
     else: break

lib/impure/db_mysql.nim

@@ -117,7 +117,7 @@ when false:
     discard mysql_stmt_close(stmt)
 
 proc dbQuote*(s: string): string =
-  ## DB quotes the string.
+  ## DB quotes the string. Note that this doesn't escape `%` and `_`.
   result = newStringOfCap(s.len + 2)
   result.add "'"
   for c in items(s):
@@ -132,7 +132,6 @@ proc dbQuote*(s: string): string =
     of '"': result.add "\\\""
     of '\'': result.add "\\'"
     of '\\': result.add "\\\\"
-    of '_': result.add "\\_"
     else: result.add c
   add(result, '\'')
 
@@ -150,17 +149,17 @@ proc tryExec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]): bool {.
   tags: [ReadDbEffect, WriteDbEffect].} =
   ## tries to execute the query and returns true if successful, false otherwise.
   var q = dbFormat(query, args)
-  return mysql.realQuery(PMySQL db, q, q.len) == 0'i32
+  return mysql.real_query(PMySQL db, q, q.len) == 0'i32
 
 proc rawExec(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) =
   var q = dbFormat(query, args)
-  if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db)
+  if mysql.real_query(PMySQL db, q, q.len) != 0'i32: dbError(db)
 
 proc exec*(db: DbConn, query: SqlQuery, args: varargs[string, `$`]) {.
   tags: [ReadDbEffect, WriteDbEffect].} =
   ## executes the query and raises EDB if not successful.
   var q = dbFormat(query, args)
-  if mysql.realQuery(PMySQL db, q, q.len) != 0'i32: dbError(db)
+  if mysql.real_query(PMySQL db, q, q.len) != 0'i32: dbError(db)
 
 proc newRow(L: int): Row =
   newSeq(result, L)
@@ -168,7 +167,7 @@ proc newRow(L: int): Row =
 
 proc properFreeResult(sqlres: mysql.PRES, row: cstringArray) =
   if row != nil:
-    while mysql.fetchRow(sqlres) != nil: discard
+    while mysql.fetch_row(sqlres) != nil: discard
   mysql.freeResult(sqlres)
 
 iterator fastRows*(db: DbConn, query: SqlQuery,
@@ -190,7 +189,7 @@ iterator fastRows*(db: DbConn, query: SqlQuery,
       backup: Row
     newSeq(result, L)
     while true:
-      row = mysql.fetchRow(sqlres)
+      row = mysql.fetch_row(sqlres)
       if row == nil: break
       for i in 0..L-1:
         setLen(result[i], 0)
@@ -209,7 +208,7 @@ iterator instantRows*(db: DbConn, query: SqlQuery,
     let L = int(mysql.numFields(sqlres))
     var row: cstringArray
     while true:
-      row = mysql.fetchRow(sqlres)
+      row = mysql.fetch_row(sqlres)
       if row == nil: break
       yield InstantRow(row: row, len: L)
     properFreeResult(sqlres, row)
@@ -290,7 +289,7 @@ iterator instantRows*(db: DbConn; columns: var DbColumns; query: SqlQuery;
     setColumnInfo(columns, sqlres, L)
     var row: cstringArray
     while true:
-      row = mysql.fetchRow(sqlres)
+      row = mysql.fetch_row(sqlres)
       if row == nil: break
       yield InstantRow(row: row, len: L)
     properFreeResult(sqlres, row)
@@ -317,7 +316,7 @@ proc getRow*(db: DbConn, query: SqlQuery,
   if sqlres != nil:
     var L = int(mysql.numFields(sqlres))
     result = newRow(L)
-    var row = mysql.fetchRow(sqlres)
+    var row = mysql.fetch_row(sqlres)
     if row != nil:
       for i in 0..L-1:
         setLen(result[i], 0)
@@ -335,7 +334,7 @@ proc getAllRows*(db: DbConn, query: SqlQuery,
     var row: cstringArray
     var j = 0
     while true:
-      row = mysql.fetchRow(sqlres)
+      row = mysql.fetch_row(sqlres)
       if row == nil: break
       setLen(result, j+1)
       newSeq(result[j], L)
@@ -361,7 +360,7 @@ proc tryInsertId*(db: DbConn, query: SqlQuery,
   ## executes the query (typically "INSERT") and returns the
   ## generated ID for the row or -1 in case of an error.
   var q = dbFormat(query, args)
-  if mysql.realQuery(PMySQL db, q, q.len) != 0'i32:
+  if mysql.real_query(PMySQL db, q, q.len) != 0'i32:
     result = -1'i64
   else:
     result = mysql.insertId(PMySQL db)

lib/impure/db_odbc.nim

@@ -133,7 +133,7 @@ proc getErrInfo(db: var DbConn): tuple[res: int, ss, ne, msg: string] {.
               511.TSqlSmallInt, retSz.addr)
   except:
     discard
-  return (res.int, $(addr sqlState), $(addr nativeErr), $(addr errMsg))
+  return (res.int, $(cast[cstring](addr sqlState)), $cast[cstring](addr nativeErr), $cast[cstring](addr errMsg))
 
 proc dbError*(db: var DbConn) {.
           tags: [ReadDbEffect, WriteDbEffect], raises: [DbError] .} =
@@ -183,7 +183,7 @@ proc sqlGetDBMS(db: var DbConn): string {.
     db.sqlCheck(SQLGetInfo(db.hDb, SQL_DBMS_NAME, cast[SqlPointer](buf.addr),
                         4095.TSqlSmallInt, sz.addr))
   except: discard
-  return $(addr buf)
+  return $(cast[cstring](addr buf))
 
 proc dbQuote*(s: string): string {.noSideEffect.} =
   ## DB quotes the string.
@@ -294,7 +294,7 @@ iterator fastRows*(db: var DbConn, query: SqlQuery,
         buf[0] = '\0'
         db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
                                  cast[cstring](buf.addr), 4095, sz.addr))
-        rowRes[colId-1] = $(addr buf)
+        rowRes[colId-1] = $cast[cstring]((addr buf))
       yield rowRes
       res = SQLFetch(db.stmt)
   properFreeResult(SQL_HANDLE_STMT, db.stmt)
@@ -322,7 +322,7 @@ iterator instantRows*(db: var DbConn, query: SqlQuery,
         buf[0] = '\0'
         db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
                                  cast[cstring](buf.addr), 4095, sz.addr))
-        rowRes[colId-1] = $(addr buf)
+        rowRes[colId-1] = $cast[cstring](addr buf)
       yield (row: rowRes, len: cCnt.int)
       res = SQLFetch(db.stmt)
   properFreeResult(SQL_HANDLE_STMT, db.stmt)
@@ -361,7 +361,7 @@ proc getRow*(db: var DbConn, query: SqlQuery,
       buf[0] = '\0'
       db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
                                cast[cstring](buf.addr), 4095, sz.addr))
-      rowRes[colId-1] = $(addr buf)
+      rowRes[colId-1] = $cast[cstring](addr buf)
     res = SQLFetch(db.stmt)
     result = rowRes
   properFreeResult(SQL_HANDLE_STMT, db.stmt)
@@ -389,7 +389,7 @@ proc getAllRows*(db: var DbConn, query: SqlQuery,
         buf[0] = '\0'
         db.sqlCheck(SQLGetData(db.stmt, colId.SqlUSmallInt, SQL_C_CHAR,
                                  cast[cstring](buf.addr), 4095, sz.addr))
-        rowRes[colId-1] = $(addr buf)
+        rowRes[colId-1] = $cast[cstring](addr buf)
       rows.add(rowRes)
       res = SQLFetch(db.stmt)
     result = rows

lib/impure/nre.nim

@@ -281,7 +281,7 @@ proc matchesCrLf(pattern: Regex): bool =
   let newlineFlags = flags and (pcre.NEWLINE_CRLF or
                                 pcre.NEWLINE_ANY or
                                 pcre.NEWLINE_ANYCRLF)
-  if newLineFlags > 0u32:
+  if newlineFlags > 0u32:
     return true
 
   # get flags from build config

lib/impure/rdstdin.nim

@@ -27,6 +27,7 @@ when defined(windows):
                           tags: [ReadIOEffect, WriteIOEffect].} =
     ## Reads a line from stdin.
     stdout.write(prompt)
+    stdout.flushFile()
     result = readLine(stdin)
 
   proc readLineFromStdin*(prompt: string, line: var string): bool {.

lib/impure/re.nim

@@ -522,19 +522,22 @@ iterator split*(s: string, sep: Regex; maxsplit = -1): string =
       @["", "this", "is", "an", "example", ""]
   var last = 0
   var splits = maxsplit
-  var x: int
+  var x = -1
+  if len(s) == 0:
+    last = 1
+  if matchLen(s, sep, 0) == 0:
+    x = 0
   while last <= len(s):
     var first = last
     var sepLen = 1
+    if x == 0:
+      inc(last)
     while last < len(s):
       x = matchLen(s, sep, last)
       if x >= 0:
         sepLen = x
         break
       inc(last)
-    if x == 0:
-      if last >= len(s): break
-      inc last
     if splits == 0: last = len(s)
     yield substr(s, first, last-1)
     if splits == 0: break

lib/packages/docutils/rst.nim

@@ -210,6 +210,12 @@ type
                               ## to Markdown) -- implies `roSupportMarkdown`
     roNimFile                 ## set for Nim files where default interpreted
                               ## text role should be :nim:
+    roSandboxDisabled         ## this option enables certain options
+                              ## (e.g. raw, include)
+                              ## which are disabled by default as they can
+                              ## enable users to read arbitrary data and
+                              ## perform XSS if the parser is used in a web
+                              ## app.
 
   RstParseOptions* = set[RstParseOption]
 
@@ -234,7 +240,8 @@ type
     mwBrokenLink = "broken link '$1'",
     mwUnsupportedLanguage = "language '$1' not supported",
     mwUnsupportedField = "field '$1' not supported",
-    mwRstStyle = "RST style: $1"
+    mwRstStyle = "RST style: $1",
+    meSandboxedDirective = "disabled directive: '$1'",
 
   MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind,
                        arg: string) {.closure, gcsafe.} ## what to do in case of an error
@@ -289,6 +296,7 @@ const
     ":geek:": "icon_e_geek",
     ":ugeek:": "icon_e_ugeek"
   }
+  SandboxDirAllowlist = ["image", "code", "code-block"]
 
 type
   TokType = enum
@@ -1856,11 +1864,16 @@ proc countTitles(s: PRstSharedState, n: PRstNode) =
         if s.hTitleCnt >= 2:
           break
 
+proc isMarkdownCodeBlock(p: RstParser, idx: int): bool =
+  roSupportMarkdown in p.s.options and p.tok[idx].symbol == "```"
+
 proc isAdornmentHeadline(p: RstParser, adornmentIdx: int): bool =
   ## check that underline/overline length is enough for the heading.
   ## No support for Unicode.
   if p.tok[adornmentIdx].symbol in ["::", "..", "|"]:
     return false
+  if isMarkdownCodeBlock(p, adornmentIdx):
+    return false
   var headlineLen = 0
   var failure = ""
   if p.idx < adornmentIdx:  # check for underline
@@ -1938,7 +1951,7 @@ proc findPipe(p: RstParser, start: int): bool =
 proc whichSection(p: RstParser): RstNodeKind =
   if currentTok(p).kind in {tkAdornment, tkPunct}:
     # for punctuation sequences that can be both tkAdornment and tkPunct
-    if roSupportMarkdown in p.s.options and currentTok(p).symbol == "```":
+    if isMarkdownCodeBlock(p, p.idx):
       return rnCodeBlock
     elif currentTok(p).symbol == "::":
       return rnLiteralBlock
@@ -2668,10 +2681,14 @@ proc dirCodeBlock(p: var RstParser, nimExtension = false): PRstNode =
   ##
   ## As an extension this proc will process the ``file`` extension field and if
   ## present will replace the code block with the contents of the referenced
-  ## file.
+  ## file. This behaviour is disabled in sandboxed mode and can be re-enabled
+  ## with the `roSandboxDisabled` flag.
   result = parseDirective(p, rnCodeBlock, {hasArg, hasOptions}, parseLiteralBlock)
   var filename = strip(getFieldValue(result, "file"))
   if filename != "":
+    if roSandboxDisabled notin p.s.options:
+      let tok = p.tok[p.idx-2]
+      rstMessage(p, meSandboxedDirective, "file", tok.line, tok.col)
     var path = p.findRelativeFile(filename)
     if path == "": rstMessage(p, meCannotOpenFile, filename)
     var n = newRstNode(rnLiteralBlock)
@@ -2767,6 +2784,11 @@ proc dirRaw(p: var RstParser): PRstNode =
 
 proc selectDir(p: var RstParser, d: string): PRstNode =
   result = nil
+  let tok = p.tok[p.idx-2] # report on directive in ".. directive::"
+  if roSandboxDisabled notin p.s.options:
+    if d notin SandboxDirAllowlist:
+      rstMessage(p, meSandboxedDirective, d, tok.line, tok.col)
+
   case d
   of "admonition", "attention", "caution": result = dirAdmonition(p, d)
   of "code": result = dirCodeBlock(p)
@@ -2793,7 +2815,6 @@ proc selectDir(p: var RstParser, d: string): PRstNode =
   of "title": result = dirTitle(p)
   of "warning": result = dirAdmonition(p, d)
   else:
-    let tok = p.tok[p.idx-2]  # report on directive in ".. directive::"
     rstMessage(p, meInvalidDirective, d, tok.line, tok.col)
 
 proc prefix(ftnType: FootnoteType): string =

lib/packages/docutils/rstgen.nim

@@ -927,7 +927,8 @@ proc getField1Int(d: PDoc, n: PRstNode, fieldName: string): int =
   let nChars = parseInt(value, number)
   if nChars == 0:
     if value.len == 0:
-      err("field $1 requires an argument" % [fieldName])
+      # use a good default value:
+      result = 1
     else:
       err("field $1 requires an integer, but '$2' was given" %
           [fieldName, value])

lib/posix/epoll.nim

@@ -33,19 +33,28 @@ const
   EPOLL_CTL_DEL* = 2          # Remove a file descriptor from the interface.
   EPOLL_CTL_MOD* = 3          # Change file descriptor epoll_event structure.
 
+# https://github.com/torvalds/linux/blob/ff6992735ade75aae3e35d16b17da1008d753d28/include/uapi/linux/eventpoll.h#L77
+when defined(linux) and defined(amd64):
+  {.pragma: epollPacked, packed.}
+else:
+  {.pragma: epollPacked.}
+
 type
-  EpollData* {.importc: "union epoll_data",
-      header: "<sys/epoll.h>", pure, final.} = object # TODO: This is actually a union.
+  EpollData* {.importc: "epoll_data_t",
+      header: "<sys/epoll.h>", pure, final, union.} = object
+    `ptr`* {.importc: "ptr".}: pointer
+    fd* {.importc: "fd".}: cint
+    u32* {.importc: "u32".}: uint32
     u64* {.importc: "u64".}: uint64
 
-  EpollEvent* {.importc: "struct epoll_event", header: "<sys/epoll.h>", pure, final.} = object
+  EpollEvent* {.importc: "struct epoll_event", header: "<sys/epoll.h>", pure, final, epollPacked.} = object
     events*: uint32 # Epoll events
     data*: EpollData # User data variable
 
 proc epoll_create*(size: cint): cint {.importc: "epoll_create",
     header: "<sys/epoll.h>".}
   ## Creates an epoll instance.  Returns an fd for the new instance.
-  ## 
+  ##
   ## The "size" parameter is a hint specifying the number of file
   ## descriptors to be associated with the new instance.  The fd
   ## returned by epoll_create() should be closed with close().
@@ -59,7 +68,7 @@ proc epoll_ctl*(epfd: cint; op: cint; fd: cint | SocketHandle; event: ptr EpollE
     importc: "epoll_ctl", header: "<sys/epoll.h>".}
   ## Manipulate an epoll instance "epfd". Returns `0` in case of success,
   ## `-1` in case of error (the "errno" variable will contain the specific error code).
-  ## 
+  ##
   ## The "op" parameter is one of the `EPOLL_CTL_*`
   ## constants defined above. The "fd" parameter is the target of the
   ## operation. The "event" parameter describes which events the caller

lib/posix/posix.nim

@@ -151,12 +151,20 @@ proc htons*(a1: uint16): uint16 {.importc, header: "<arpa/inet.h>".}
 proc ntohl*(a1: uint32): uint32 {.importc, header: "<arpa/inet.h>".}
 proc ntohs*(a1: uint16): uint16 {.importc, header: "<arpa/inet.h>".}
 
-proc inet_addr*(a1: cstring): InAddrT {.importc, header: "<arpa/inet.h>".}
-proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "<arpa/inet.h>".}
-proc inet_ntop*(a1: cint, a2: pointer, a3: cstring, a4: int32): cstring {.
-  importc:"(char *)$1", header: "<arpa/inet.h>".}
-proc inet_pton*(a1: cint, a2: cstring, a3: pointer): cint {.
-  importc, header: "<arpa/inet.h>".}
+when not defined(zephyr):
+  proc inet_addr*(a1: cstring): InAddrT {.importc, header: "<arpa/inet.h>".}
+  proc inet_ntoa*(a1: InAddr): cstring {.importc, header: "<arpa/inet.h>".}
+
+when defined(zephyr): # this switch is done to keep Nim 1.6 API backward compatible
+  proc inet_ntop*(a1: cint, a2: pointer | ptr InAddr | ptr In6Addr, a3: cstring, a4: int32): cstring {.
+    importc:"(char *)$1", header: "<arpa/inet.h>".}
+  proc inet_pton*(a1: cint, a2: cstring, a3: pointer | ptr InAddr | ptr In6Addr): cint {.
+    importc, header: "<arpa/inet.h>".}
+else:
+  proc inet_ntop*(a1: cint, a2: pointer, a3: cstring, a4: int32): cstring {.
+    importc:"(char *)$1", header: "<arpa/inet.h>".}
+  proc inet_pton*(a1: cint, a2: cstring, a3: pointer): cint {.
+    importc, header: "<arpa/inet.h>".}
 
 var
   in6addr_any* {.importc, header: "<netinet/in.h>".}: In6Addr
@@ -1004,7 +1012,7 @@ proc endhostent*() {.importc, header: "<netdb.h>".}
 proc endnetent*() {.importc, header: "<netdb.h>".}
 proc endprotoent*() {.importc, header: "<netdb.h>".}
 proc endservent*() {.importc, header: "<netdb.h>".}
-proc freeaddrinfo*(a1: ptr AddrInfo) {.importc, header: "<netdb.h>".}
+proc freeAddrInfo*(a1: ptr AddrInfo) {.importc: "freeaddrinfo", header: "<netdb.h>".}
 
 proc gai_strerror*(a1: cint): cstring {.importc:"(char *)$1", header: "<netdb.h>".}
 

lib/posix/posix_freertos_consts.nim

@@ -363,6 +363,8 @@ var SEM_FAILED* {.importc: "SEM_FAILED", header: "<semaphore.h>".}: pointer
 # <sys/resource.h>
 # var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "<sys/resource.h>".}: cint
 
+var FD_MAX* {.importc: "CONFIG_LWIP_MAX_SOCKETS", header: "<lwipopts.h>".}: cint
+
 # <sys/select.h>
 var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "<sys/select.h>".}: cint
 

lib/posix/posix_linux_amd64.nim

@@ -168,7 +168,7 @@ type
   Pthread_key* {.importc: "pthread_key_t", header: "<sys/types.h>".} = cuint
   Pthread_mutex* {.importc: "pthread_mutex_t", header: "<sys/types.h>",
                    pure, final.} = object
-    abi: array[48 div sizeof(clong), clong]
+    abi: array[40 div sizeof(clong), clong]
   Pthread_mutexattr* {.importc: "pthread_mutexattr_t",
                         header: "<sys/types.h>", pure, final.} = object
     abi: array[4 div sizeof(cint), cint]

lib/posix/posix_other.nim

@@ -10,7 +10,7 @@
 when defined(nimHasStyleChecks):
   {.push styleChecks: off.}
 
-when defined(freertos):
+when defined(freertos) or defined(zephyr):
   const
     hasSpawnH = false # should exist for every Posix system nowadays
     hasAioH = false
@@ -391,9 +391,33 @@ when hasSpawnH:
                                  header: "<spawn.h>", final, pure.} = object
 
 when defined(linux):
+  const Sockaddr_max_length* = 255
   # from sys/un.h
   const Sockaddr_un_path_length* = 108
+elif defined(zephyr):
+  when defined(net_ipv6):
+    const Sockaddr_max_length* = 24
+  elif defined(net_raw):
+    const Sockaddr_max_length* = 20
+  elif defined(net_ipv4):
+    const Sockaddr_max_length* = 8
+  else:
+    const Sockaddr_max_length* = 255 # just for compilation purposes
+
+  const Sockaddr_un_path_length* = Sockaddr_max_length
+  # Zephyr is heavily customizable so it's easy to get to a state  
+  # where Nim & Zephyr IPv6 settings are out of sync, causing painful runtime failures. 
+  when defined(net_ipv4) or defined(net_ipv6) or defined(net_raw):
+    {.emit: ["NIM_STATIC_ASSERT(NET_SOCKADDR_MAX_SIZE == ",
+        Sockaddr_max_length,
+        ",\"NET_SOCKADDR_MAX_SIZE and Sockaddr_max_length size mismatch!",
+        " Check that Nim and Zephyr IPv4/IPv6 settings match.",
+        " Try adding -d:net_ipv6 to enable IPv6 for Nim on Zephyr.\" );"].}
+elif defined(freertos) or defined(lwip):
+  const Sockaddr_max_length* = 14
+  const Sockaddr_un_path_length* = 108
 else:
+  const Sockaddr_max_length* = 255
   # according to http://pubs.opengroup.org/onlinepubs/009604499/basedefs/sys/un.h.html
   # this is >=92
   const Sockaddr_un_path_length* = 92
@@ -408,48 +432,50 @@ when defined(lwip):
                 pure, final.} = object ## struct sockaddr
       sa_len*: uint8         ## Address family.
       sa_family*: TSa_Family         ## Address family.
-      sa_data*: array[0..255, char] ## Socket address (variable-length data).
+      sa_data*: array[0..Sockaddr_max_length-sizeof(uint8)-sizeof(TSa_Family), char] ## Socket address (variable-length data).
+
+    Sockaddr_storage* {.importc: "struct sockaddr_storage",
+                        header: "<sys/socket.h>",
+                        pure, final.} = object ## struct sockaddr_storage
+      s2_len*: uint8 ## Address family.
+      ss_family*: TSa_Family ## Address family.
+      s2_data1*: array[2, char] ## Address family.
+      s2_data2*: array[3, uint32] ## Address family.
+      when defined(lwip6) or defined(net_ipv6):
+        s2_data3*: array[3, uint32] ## Address family.
+elif defined(zephyr):
+  type
+    SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>",
+                pure, final.} = object ## struct sockaddr
+      sa_family*: TSa_Family         ## Address family.
+      data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data).
+
+    Sockaddr_storage* {.importc: "struct sockaddr_storage",
+                        header: "<sys/socket.h>",
+                        pure, final.} = object ## struct sockaddr_storage
+      ss_family*: TSa_Family ## Address family.
+      data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data).
+  {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr) == ", sizeof(Sockaddr), ",\"struct size mismatch\" );"].}
+  {.emit: ["NIM_STATIC_ASSERT(sizeof(struct sockaddr_storage) == ", sizeof(Sockaddr_storage), ",\"struct size mismatch\" );"].}
 else:
   type
     SockAddr* {.importc: "struct sockaddr", header: "<sys/socket.h>",
                 pure, final.} = object ## struct sockaddr
       sa_family*: TSa_Family         ## Address family.
-      sa_data*: array[0..255, char] ## Socket address (variable-length data).
+      sa_data*: array[0..Sockaddr_max_length-sizeof(TSa_Family), char] ## Socket address (variable-length data).
+
+    Sockaddr_storage* {.importc: "struct sockaddr_storage",
+                        header: "<sys/socket.h>",
+                        pure, final.} = object ## struct sockaddr_storage
+      ss_family*: TSa_Family ## Address family.
 
 type
   Sockaddr_un* {.importc: "struct sockaddr_un", header: "<sys/un.h>",
               pure, final.} = object ## struct sockaddr_un
     sun_family*: TSa_Family         ## Address family.
-    sun_path*: array[0..Sockaddr_un_path_length-1, char] ## Socket path
+    sun_path*: array[0..Sockaddr_un_path_length-sizeof(TSa_Family), char] ## Socket path
 
 
-when defined(lwip):
-  when not defined(lwip6):
-    type
-      Sockaddr_storage* {.importc: "struct sockaddr_storage",
-                          header: "<sys/socket.h>",
-                          pure, final.} = object ## struct sockaddr_storage
-        s2_len*: uint8 ## Address family.
-        ss_family*: TSa_Family ## Address family.
-        s2_data1*: array[2, char] ## Address family.
-        s2_data2*: array[3, uint32] ## Address family.
-  else:
-    type
-      Sockaddr_storage* {.importc: "struct sockaddr_storage",
-                          header: "<sys/socket.h>",
-                          pure, final.} = object ## struct sockaddr_storage
-        s2_len*: uint8 ## Address family.
-        ss_family*: TSa_Family ## Address family.
-        s2_data1*: array[2, char] ## Address family.
-        s2_data2*: array[3, uint32] ## Address family.
-        s2_data3*: array[3, uint32] ## Address family.
-else:
-  type
-    Sockaddr_storage* {.importc: "struct sockaddr_storage",
-                        header: "<sys/socket.h>",
-                        pure, final.} = object ## struct sockaddr_storage
-      ss_family*: TSa_Family ## Address family.
-
 type
   Tif_nameindex* {.importc: "struct if_nameindex", final,
                    pure, header: "<net/if.h>".} = object ## struct if_nameindex
@@ -494,6 +520,7 @@ type
              header: "<netinet/in.h>".} = object ## struct in_addr
     s_addr*: InAddrScalar
 
+  # TODO: Fixme for FreeRTOS/LwIP, these are incorrect
   Sockaddr_in* {.importc: "struct sockaddr_in", pure, final,
                   header: "<netinet/in.h>".} = object ## struct sockaddr_in
     sin_family*: TSa_Family ## AF_INET.
@@ -577,7 +604,12 @@ when not defined(lwip):
       events*: cshort  ## The input event flags (see below).
       revents*: cshort ## The output event flags (see below).
 
-    Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = cint
+  when defined(zephyr):
+    type
+      Tnfds* = distinct cint
+  else:
+    type
+      Tnfds* {.importc: "nfds_t", header: "<poll.h>".} = cint
 
 var
   errno* {.importc, header: "<errno.h>".}: cint ## error variable
@@ -625,7 +657,7 @@ elif defined(solaris):
   # Solaris doesn't have MSG_NOSIGNAL
   const
     MSG_NOSIGNAL* = 0'i32
-elif defined(freertos) or defined(lwip):
+elif defined(zephyr) or defined(freertos) or defined(lwip):
   # LwIP/FreeRTOS doesn't have MSG_NOSIGNAL
   const
     MSG_NOSIGNAL* = 0x20'i32

lib/posix/posix_other_consts.nim

@@ -462,6 +462,9 @@ var RLIMIT_NOFILE* {.importc: "RLIMIT_NOFILE", header: "<sys/resource.h>".}: cin
 
 # <sys/select.h>
 var FD_SETSIZE* {.importc: "FD_SETSIZE", header: "<sys/select.h>".}: cint
+when defined(zephyr):
+  # Zephyr specific hardcoded value
+  var FD_MAX* {.importc: "CONFIG_POSIX_MAX_FDS ", header: "<sys/select.h>".}: cint
 
 # <sys/socket.h>
 var MSG_CTRUNC* {.importc: "MSG_CTRUNC", header: "<sys/socket.h>".}: cint
@@ -487,10 +490,15 @@ var SO_SNDTIMEO* {.importc: "SO_SNDTIMEO", header: "<sys/socket.h>".}: cint
 var SO_TYPE* {.importc: "SO_TYPE", header: "<sys/socket.h>".}: cint
 var SOCK_DGRAM* {.importc: "SOCK_DGRAM", header: "<sys/socket.h>".}: cint
 var SOCK_RAW* {.importc: "SOCK_RAW", header: "<sys/socket.h>".}: cint
-var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "<sys/socket.h>".}: cint
+when defined(zephyr):
+  const SOCK_SEQPACKET* = cint(5)
+  var SOMAXCONN* {.importc: "CONFIG_NET_SOCKETS_POLL_MAX", header: "<sys/socket.h>".}: cint
+else:
+  var SOCK_SEQPACKET* {.importc: "SOCK_SEQPACKET", header: "<sys/socket.h>".}: cint
+  var SOMAXCONN* {.importc: "SOMAXCONN", header: "<sys/socket.h>".}: cint
+
 var SOCK_STREAM* {.importc: "SOCK_STREAM", header: "<sys/socket.h>".}: cint
 var SOL_SOCKET* {.importc: "SOL_SOCKET", header: "<sys/socket.h>".}: cint
-var SOMAXCONN* {.importc: "SOMAXCONN", header: "<sys/socket.h>".}: cint
 var MSG_PEEK* {.importc: "MSG_PEEK", header: "<sys/socket.h>".}: cint
 var MSG_TRUNC* {.importc: "MSG_TRUNC", header: "<sys/socket.h>".}: cint
 var MSG_WAITALL* {.importc: "MSG_WAITALL", header: "<sys/socket.h>".}: cint

lib/posix/posix_utils.nim

@@ -18,7 +18,7 @@ type Uname* = object
   sysname*, nodename*, release*, version*, machine*: string
 
 template charArrayToString(input: typed): string =
-  $cstring(addr input)
+  $cast[cstring](addr input)
 
 proc uname*(): Uname =
   ## Provides system information in a `Uname` struct with sysname, nodename,

lib/pure/asyncdispatch.nim

@@ -187,9 +187,9 @@
 ##
 ## Known `async` backends include:
 ##
-## * `none` - ``-d:asyncBackend=none`` - disable ``async`` support completely
-## * `asyncdispatch <https://nim-lang.org/docs/asyncdispatch.html> -``-d:asyncBackend=asyncdispatch``
-## * `chronos <https://github.com/status-im/nim-chronos/>` - ``-d:asyncBackend=chronos``
+## * `-d:asyncBackend=none`: disable `async` support completely
+## * `-d:asyncBackend=asyncdispatch`: https://nim-lang.org/docs/asyncdispatch.html
+## * `-d:asyncBackend=chronos`: https://github.com/status-im/nim-chronos/
 ##
 ## ``none`` can be used when a library supports both a synchronous and
 ## asynchronous API, to disable the latter.
@@ -1791,7 +1791,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped,
         curAddrInfo = curAddrInfo.ai_next
 
       if curAddrInfo == nil:
-        freeaddrinfo(addrInfo)
+        freeAddrInfo(addrInfo)
         when shouldCreateFd:
           closeUnusedFds()
         if lastException != nil:
@@ -1807,7 +1807,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped,
           try:
             curFd = createAsyncNativeSocket(domain, sockType, protocol)
           except:
-            freeaddrinfo(addrInfo)
+            freeAddrInfo(addrInfo)
             closeUnusedFds()
             raise getCurrentException()
           when defined(windows):
@@ -1817,7 +1817,7 @@ template asyncAddrInfoLoop(addrInfo: ptr AddrInfo, fd: untyped,
       doConnect(curFd, curAddrInfo).callback = tryNextAddrInfo
       curAddrInfo = curAddrInfo.ai_next
     else:
-      freeaddrinfo(addrInfo)
+      freeAddrInfo(addrInfo)
       when shouldCreateFd:
         closeUnusedFds(ord(domain))
         retFuture.complete(curFd)
@@ -1973,15 +1973,18 @@ proc activeDescriptors*(): int {.inline.} =
 when defined(posix):
   import posix
 
-when defined(linux) or defined(windows) or defined(macosx) or defined(bsd):
+when defined(linux) or defined(windows) or defined(macosx) or defined(bsd) or
+       defined(zephyr) or defined(freertos):
   proc maxDescriptors*(): int {.raises: OSError.} =
     ## Returns the maximum number of active file descriptors for the current
     ## process. This involves a system call. For now `maxDescriptors` is
     ## supported on the following OSes: Windows, Linux, OSX, BSD.
     when defined(windows):
       result = 16_700_000
+    elif defined(zephyr) or defined(freertos):
+      result = FD_MAX
     else:
       var fdLim: RLimit
       if getrlimit(RLIMIT_NOFILE, fdLim) < 0:
         raiseOSError(osLastError())
-      result = int(fdLim.rlim_cur) - 1
+      result = int(fdLim.rlim_cur) - 1

lib/pure/asyncnet.nim

@@ -103,6 +103,7 @@ export SOBool
 # TODO: Remove duplication introduced by PR #4683.
 
 const defineSsl = defined(ssl) or defined(nimdoc)
+const useNimNetLite = defined(nimNetLite) or defined(freertos) or defined(zephyr)
 
 when defineSsl:
   import openssl
@@ -178,11 +179,12 @@ proc getLocalAddr*(socket: AsyncSocket): (string, Port) =
   ## This is high-level interface for `getsockname`:idx:.
   getLocalAddr(socket.fd, socket.domain)
 
-proc getPeerAddr*(socket: AsyncSocket): (string, Port) =
-  ## Get the socket's peer address and port number.
-  ##
-  ## This is high-level interface for `getpeername`:idx:.
-  getPeerAddr(socket.fd, socket.domain)
+when not useNimNetLite:
+  proc getPeerAddr*(socket: AsyncSocket): (string, Port) =
+    ## Get the socket's peer address and port number.
+    ##
+    ## This is high-level interface for `getpeername`:idx:.
+    getPeerAddr(socket.fd, socket.domain)
 
 proc newAsyncSocket*(domain, sockType, protocol: cint,
                      buffered = true,
@@ -224,7 +226,7 @@ when defineSsl:
     let len = bioCtrlPending(socket.bioOut)
     if len > 0:
       var data = newString(len)
-      let read = bioRead(socket.bioOut, addr data[0], len)
+      let read = bioRead(socket.bioOut, cast[cstring](addr data[0]), len)
       assert read != 0
       if read < 0:
         raiseSSLError()
@@ -242,7 +244,7 @@ when defineSsl:
       var data = await recv(socket.fd.AsyncFD, BufferSize, flags)
       let length = len(data)
       if length > 0:
-        let ret = bioWrite(socket.bioIn, addr data[0], length.cint)
+        let ret = bioWrite(socket.bioIn, cast[cstring](addr data[0]), length.cint)
         if ret < 0:
           raiseSSLError()
       elif length == 0:
@@ -453,7 +455,7 @@ proc send*(socket: AsyncSocket, data: string,
     when defineSsl:
       var copy = data
       sslLoop(socket, flags,
-        sslWrite(socket.sslHandle, addr copy[0], copy.len.cint))
+        sslWrite(socket.sslHandle, cast[cstring](addr copy[0]), copy.len.cint))
       await sendPendingSslData(socket, flags)
   else:
     await send(socket.fd.AsyncFD, data, flags)
@@ -646,16 +648,16 @@ proc bindAddr*(socket: AsyncSocket, port = Port(0), address = "") {.
 
   var aiList = getAddrInfo(realaddr, port, socket.domain)
   if bindAddr(socket.fd, aiList.ai_addr, aiList.ai_addrlen.SockLen) < 0'i32:
-    freeaddrinfo(aiList)
+    freeAddrInfo(aiList)
     raiseOSError(osLastError())
-  freeaddrinfo(aiList)
+  freeAddrInfo(aiList)
 
 proc hasDataBuffered*(s: AsyncSocket): bool {.since: (1, 5).} =
   ## Determines whether an AsyncSocket has data buffered.
   # xxx dedup with std/net
   s.isBuffered and s.bufLen > 0 and s.currPos != s.bufLen
 
-when defined(posix):
+when defined(posix) and not useNimNetLite:
 
   proc connectUnix*(socket: AsyncSocket, path: string): owned(Future[void]) =
     ## Binds Unix socket to `path`.
@@ -857,7 +859,7 @@ proc sendTo*(socket: AsyncSocket, address: string, port: Port, data: string,
 
     it = it.ai_next
 
-  freeaddrinfo(aiList)
+  freeAddrInfo(aiList)
 
   if not success:
     if lastException != nil:

lib/pure/collections/lists.nim

@@ -531,11 +531,12 @@ proc addMoved*[T](a, b: var SinglyLinkedList[T]) {.since: (1, 5, 1).} =
         ci
     assert s == [0, 1, 0, 1, 0, 1]
 
-  if a.tail != nil:
-    a.tail.next = b.head
-  a.tail = b.tail
-  if a.head == nil:
-    a.head = b.head
+  if b.head != nil:
+    if a.head == nil:
+      a.head = b.head
+    else:
+      a.tail.next = b.head
+    a.tail = b.tail
   if a.addr != b.addr:
     b.head = nil
     b.tail = nil
@@ -675,12 +676,12 @@ proc addMoved*[T](a, b: var DoublyLinkedList[T]) {.since: (1, 5, 1).} =
     assert s == [0, 1, 0, 1, 0, 1]
 
   if b.head != nil:
-    b.head.prev = a.tail
-  if a.tail != nil:
-    a.tail.next = b.head
-  a.tail = b.tail
-  if a.head == nil:
-    a.head = b.head
+    if a.head == nil:
+      a.head = b.head
+    else:
+      b.head.prev = a.tail
+      a.tail.next = b.head
+    a.tail = b.tail
   if a.addr != b.addr:
     b.head = nil
     b.tail = nil
@@ -739,6 +740,8 @@ proc remove*[T](L: var SinglyLinkedList[T], n: SinglyLinkedNode[T]): bool {.disc
     if prev.next == nil:
       return false
     prev.next = n.next
+    if L.tail == n:
+      L.tail = prev # update tail if we removed the last node
   true
 
 proc remove*[T](L: var DoublyLinkedList[T], n: DoublyLinkedNode[T]) =

lib/pure/collections/sequtils.nim

@@ -910,7 +910,7 @@ template foldl*(sequence, operation, first): untyped =
   ##
   ## The `operation` parameter should be an expression which uses the variables
   ## `a` and `b` for each step of the fold. The `first` parameter is the
-  ## start value (the first `a`) and therefor defines the type of the result.
+  ## start value (the first `a`) and therefore defines the type of the result.
   ##
   ## **See also:**
   ## * `foldr template<#foldr.t,untyped,untyped>`_