Attempt to explain better why delImplIdx is the way it is. Maybe this can (#15108)

avoid future implementation mischief.  (Maybe not.  Sometimes, general
distrust of theory leads people to distrust simple reasoning over times
from CPUs trying as hard as possible to mask DRAM latency via pre-fetch.)

(cherry picked from commit 196e747df1)

.gitattributes

@@ -0,0 +1,6 @@
+# Avoids changelog conflicts by assuming additions-only, which is by far the common case.
+# In the rare case where branch b1 rebases against branch b2 and both branches
+# modified the same changelog entry, you'll end up with that changelog entry
+# duplicated, which is easily identifiable and fixable.
+/changelog.md merge=union
+

azure-pipelines.yml

@@ -30,8 +30,12 @@ jobs:
       Windows_amd64:
         vmImage: 'windows-2019'
         CPU: amd64
-      Windows_amd64_pkg:
-        vmImage: 'windows-2019'
+      Linux_amd64_pkg:
+        vmImage: 'ubuntu-16.04'
+        CPU: amd64
+        NIM_TEST_PACKAGES: true
+      OSX_amd64_pkg:
+        vmImage: 'macOS-10.15'
         CPU: amd64
         NIM_TEST_PACKAGES: true
 
@@ -53,8 +57,8 @@ jobs:
 
     - task: NodeTool@0
       inputs:
-        versionSpec: '8.x'
-      displayName: 'Install node.js 8.x'
+        versionSpec: '12.x'
+      displayName: 'Install node.js 12.x'
 
     - bash: |
         sudo apt-fast update -qq
@@ -107,7 +111,7 @@ jobs:
       displayName: 'Install dependencies (i386 Linux)'
       condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
 
-    - bash: brew install boehmgc make sfml
+    - bash: brew install boehmgc make sfml gtk+3
       displayName: 'Install dependencies (OSX)'
       condition: eq(variables['Agent.OS'], 'Darwin')
 

changelog.md

@@ -4,9 +4,158 @@
 
 ## Standard library additions and changes
 
+  For `net` and `nativesockets`, an `inheritable` flag has been added to all
+  `proc`s that create sockets, allowing the user to control whether the
+  resulting socket is inheritable. This flag is provided to ease the writing of
+  multi-process servers, where sockets inheritance is desired.
+
+  For a transistion period, define `nimInheritHandles` to enable file handle
+  inheritance by default. This flag does **not** affect the `selectors` module
+  due to the differing semantics between operating systems.
+
+  `system.setInheritable` and `nativesockets.setInheritable` is also introduced
+  for setting file handle or socket inheritance. Not all platform have these
+  `proc`s defined.
+
+- The file descriptors created for internal bookkeeping by `ioselector_kqueue`
+  and `ioselector_epoll` will no longer be leaked to child processes.
+
+- `strutils.formatFloat` with `precision = 0` has been restored to the version
+  1 behaviour that produces a trailing dot, e.g. `formatFloat(3.14159, precision = 0)`
+  is now `3.`, not `3`.
+- `critbits` adds `commonPrefixLen`.
+
+- `relativePath(rel, abs)` and `relativePath(abs, rel)` used to silently give wrong results
+  (see #13222); instead they now use `getCurrentDir` to resolve those cases,
+  and this can now throw in edge cases where `getCurrentDir` throws.
+  `relativePath` also now works for js with `-d:nodejs`.
+
+- JavaScript and NimScript standard library changes: `streams.StringStream` is
+  now supported in JavaScript, with the limitation that any buffer `pointer`s
+  used must be castable to `ptr string`, any incompatible pointer type will not
+  work. The `lexbase` and `streams` modules used to fail to compile on
+  NimScript due to a bug, but this has been fixed.
+  
+  The following modules now compile on both JS and NimScript: `parsecsv`,
+  `parsecfg`, `parsesql`, `xmlparser`, `htmlparser` and `ropes`. Additionally
+  supported for JS is `cstrutils.startsWith` and `cstrutils.endsWith`, for
+  NimScript: `json`, `parsejson`, `strtabs` and `unidecode`. 
+
+- Added `streams.readStr` and `streams.peekStr` overloads to
+  accept an existing string to modify, which avoids memory
+  allocations, similar to `streams.readLine` (#13857).
+
+- Added high-level `asyncnet.sendTo` and `asyncnet.recvFrom`. UDP functionality.
+
+- `paramCount` & `paramStr` are now defined in os.nim instead of nimscript.nim for nimscript/nimble.
+- `dollars.$` now works for unsigned ints with `nim js`
+
+- Improvements to the `bitops` module, including bitslices, non-mutating versions
+  of the original masking functions, `mask`/`masked`, and varargs support for
+  `bitand`, `bitor`, and `bitxor`.
+
+- `sugar.=>` and `sugar.->` changes: Previously `(x, y: int)` was transformed
+  into `(x: auto, y: int)`, it now becomes `(x: int, y: int)` in consistency
+  with regular proc definitions (although you cannot use semicolons).
+  
+  Pragmas and using a name are now allowed on the lefthand side of `=>`. Here
+  is an aggregate example of these changes:
+  ```nim
+  import sugar
+
+  foo(x, y: int) {.noSideEffect.} => x + y
+
+  # is transformed into
+
+  proc foo(x: int, y: int): auto {.noSideEffect.} = x + y
+  ```
+- The fields of `times.DateTime` are now private, and are accessed with getters and deprecated setters.
+
+- The `times` module now handles the default value for `DateTime` more consistently. Most procs raise an assertion error when given
+  an uninitialized `DateTime`, the exceptions are `==` and `$` (which returns `"Uninitialized DateTime"`). The proc `times.isInitialized`
+  has been added which can be used to check if a `DateTime` has been initialized.
+
+- Fix a bug where calling `close` on io streams in osproc.startProcess was a noop and led to
+  hangs if a process had both reads from stdin and writes (eg to stdout).
+
+- The callback that is passed to `system.onThreadDestruction` must now be `.raises: []`.
+- The callback that is assigned to `system.onUnhandledException` must now be `.gcsafe`.
+
+- `osproc.execCmdEx` now takes an optional `input` for stdin, `workingDir` and `env`
+  parameters.
+
+- Added a `ssl_config` module containing lists of secure ciphers as recommended by
+  [Mozilla OpSec](https://wiki.mozilla.org/Security/Server_Side_TLS)
+
+- `net.newContext` now defaults to the list of ciphers targeting
+  ["Intermediate compatibility"](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29)
+  per Mozilla's recommendation instead of `ALL`. This change should protect
+  users from the use of weak and insecure ciphers while still provides
+  adequate compatibility with the majority of the Internet.
+
+- A new module `std/jsonutils` with hookable `jsonTo,toJson,fromJson` operations for json
+  serialization/deserialization of custom types was added.
+
+- A new proc `heapqueue.find[T](heap: HeapQueue[T], x: T): int` to get index of element ``x``
+  was added.
+- Added `rstgen.rstToLatex` convenience proc for `renderRstToOut` and `initRstGenerator`
+  with `outLatex` output.
+- Added `os.normalizeExe`, e.g.: `koch` => `./koch`.
+- `macros.newLit` now preserves named vs unnamed tuples; use `-d:nimHasWorkaround14720`
+  to keep old behavior.
+- Added `random.gauss`, that uses the ratio of uniforms method of sampling from a Gaussian distribution.
+- Added `typetraits.elementType` to get element type of an iterable.
+- `typetraits.$` changes: `$(int,)` is now `"(int,)"` instead of `"(int)"`;
+  `$tuple[]` is now `"tuple[]"` instead of `"tuple"`;
+  `$((int, float), int)` is now `"((int, float), int)"` instead of `"(tuple of (int, float), int)"`
+- Added `macros.extractDocCommentsAndRunnables` helper
+
+- `strformat.fmt` and `strformat.&` support `= specifier`. `fmt"{expr=}"` now
+  expands to `fmt"expr={expr}"`.
+- deprecations: `os.existsDir` => `dirExists`, `os.existsFile` => `fileExists`
+
+- Added `jsre` module, [Regular Expressions for the JavaScript target.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions)
+- Made `maxLines` argument `Positive` in `logging.newRollingFileLogger`,
+  because negative values will result in a new file being created for each logged
+  line which doesn't make sense.
+- Changed `log` in `logging` to use proper log level on JavaScript target,
+  e.g. `debug` uses `console.debug`, `info` uses `console.info`, `warn` uses `console.warn`, etc.
+- Tables, HashSets, SharedTables and deques don't require anymore that the passed
+  initial size must be a power of two - this is done internally.
+  Proc `rightSize` for Tables and HashSets is deprecated, as it is not needed anymore.
+  `CountTable.inc` takes `val: int` again not `val: Positive`; I.e. it can "count down" again.
+- Removed deprecated symbols from `macros` module, deprecated as far back as `0.15`.
+
 
 ## Language changes
+- In newruntime it is now allowed to assign discriminator field without restrictions as long as case object doesn't have custom destructor. Discriminator value doesn't have to be a constant either. If you have custom destructor for case object and you do want to freely assign discriminator fields, it is recommended to refactor object into 2 objects like this: 
+  ```nim
+  type
+    MyObj = object
+      case kind: bool
+        of true: y: ptr UncheckedArray[float]
+        of false: z: seq[int]
 
+  proc `=destroy`(x: MyObj) =
+    if x.kind and x.y != nil:
+      deallocShared(x.y)
+      x.y = nil
+  ```
+  Refactor into:
+  ```nim
+  type
+    MySubObj = object
+      val: ptr UncheckedArray[float]
+    MyObj = object
+      case kind: bool
+      of true: y: MySubObj
+      of false: z: seq[int]
+
+  proc `=destroy`(x: MySubObj) =
+    if x.val != nil:
+      deallocShared(x.val)
+      x.val = nil
+  ```
 
 ## Compiler changes
 

compiler/ast.nim

@@ -292,6 +292,7 @@ type
     sfInjectDestructors # whether the proc needs the 'injectdestructors' transformation
     sfNeverRaises     # proc can never raise an exception, not even OverflowError
                       # or out-of-memory
+    sfUsedInFinallyOrExcept  # symbol is used inside an 'except' or 'finally'
 
   TSymFlags* = set[TSymFlag]
 

compiler/astalgo.nim

@@ -1048,3 +1048,7 @@ proc listSymbolNames*(symbols: openArray[PSym]): string =
       result.add ", "
     result.add sym.name.s
 
+proc isDiscriminantField*(n: PNode): bool =
+  if n.kind == nkCheckedFieldExpr: sfDiscriminant in n[0][1].sym.flags
+  elif n.kind == nkDotExpr: sfDiscriminant in n[1].sym.flags
+  else: false

compiler/ccgcalls.nim

@@ -21,11 +21,39 @@ proc canRaiseDisp(p: BProc; n: PNode): bool =
     # we have to be *very* conservative:
     result = canRaiseConservative(n)
 
-proc leftAppearsOnRightSide(le, ri: PNode): bool =
+proc preventNrvo(p: BProc; le, ri: PNode): bool =
+  proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
+    var n = le
+    while true:
+      # do NOT follow nkHiddenDeref here!
+      case n.kind
+      of nkSym:
+        # we don't own the location so it escapes:
+        if n.sym.owner != p.prc:
+          return true
+        elif inTryStmt and sfUsedInFinallyOrExcept in n.sym.flags:
+          # it is also an observable store if the location is used
+          # in 'except' or 'finally'
+          return true
+        return false
+      of nkDotExpr, nkBracketExpr, nkObjUpConv, nkObjDownConv,
+          nkCheckedFieldExpr:
+        n = n[0]
+      of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+        n = n[1]
+      else:
+        # cannot analyse the location; assume the worst
+        return true
+
   if le != nil:
     for i in 1..<ri.len:
       let r = ri[i]
       if isPartOf(le, r) != arNo: return true
+    # we use the weaker 'canRaise' here in order to prevent too many
+    # annoying warnings, see #14514
+    if canRaise(ri[0]) and
+        locationEscapes(p, le, p.nestedTryStmts.len > 0):
+      message(p.config, le.info, warnObservableStores, $le)
 
 proc hasNoInit(call: PNode): bool {.inline.} =
   result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
@@ -51,7 +79,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
     if isInvalidReturnType(p.config, typ[0]):
       if params != nil: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
@@ -113,7 +141,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
 
 proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
 
-proc openArrayLoc(p: BProc, n: PNode): Rope =
+proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
   var a: TLoc
 
   var q = skipConv(n)
@@ -149,8 +177,11 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
     of tyOpenArray, tyVarargs, tyUncheckedArray, tyCString:
       result = "($4*)($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dest]
     of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-          not compileToCpp(p.module):
+      let atyp = skipTypes(a.t, abstractInst)
+      if formalType.skipTypes(abstractInst).kind == tyVar and atyp.kind == tyString and
+          optSeqDestructors in p.config.globalOptions:
+        linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+      if atyp.kind == tyVar and not compileToCpp(p.module):
         result = "($5*)(*$1)$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
       else:
         result = "($5*)$1$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
@@ -162,8 +193,11 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
     of tyOpenArray, tyVarargs:
       result = "$1, $1Len_0" % [rdLoc(a)]
     of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-            not compileToCpp(p.module):
+      let ntyp = skipTypes(n.typ, abstractInst)
+      if formalType.skipTypes(abstractInst).kind == tyVar and ntyp.kind == tyString and
+          optSeqDestructors in p.config.globalOptions:
+        linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+      if ntyp.kind == tyVar and not compileToCpp(p.module):
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
         result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
@@ -194,7 +228,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
     result = genArgStringToCString(p, n)
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
     var n = if n.kind != nkHiddenAddr: n else: n[0]
-    result = openArrayLoc(p, n)
+    result = openArrayLoc(p, param.typ, n)
   elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
     result = addrLoc(p.config, a)
@@ -286,7 +320,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
     if isInvalidReturnType(p.config, typ[0]):
       if ri.len > 1: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
           getTemp(p, typ[0], d, needsInit=true)

compiler/ccgexprs.nim

@@ -1194,9 +1194,11 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) =
 proc genReset(p: BProc, n: PNode) =
   var a: TLoc
   initLocExpr(p, n[1], a)
-  linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
-          [addrLoc(p.config, a),
-          genTypeInfo(p.module, skipTypes(a.t, {tyVar}), n.info)])
+  specializeReset(p, a)
+  when false:
+    linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
+            [addrLoc(p.config, a),
+            genTypeInfo(p.module, skipTypes(a.t, {tyVar}), n.info)])
 
 proc genDefault(p: BProc; n: PNode; d: var TLoc) =
   if d.k == locNone: getTemp(p, n.typ, d, needsInit=true)
@@ -2156,7 +2158,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     const opr: array[mInc..mDec, string] = ["+=", "-="]
     const fun64: array[mInc..mDec, string] = ["nimAddInt64", "nimSubInt64"]
     const fun: array[mInc..mDec, string] = ["nimAddInt","nimSubInt"]
-    let underlying = skipTypes(e[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent, tyRange})
+    let underlying = skipTypes(e[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent, tyRange, tyDistinct})
     if optOverflowCheck notin p.options or underlying.kind in {tyUInt..tyUInt64}:
       binaryStmt(p, e, d, opr[op])
     else:
@@ -2787,6 +2789,21 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo): Rope =
   else:
     globalError(p.config, info, "cannot create null element for: " & $t.kind)
 
+proc caseObjDefaultBranch(obj: PNode; branch: Int128): int =
+  for i in 1 ..< obj.len:
+    for j in 0 .. obj[i].len - 2:
+      if obj[i][j].kind == nkRange:
+        let x = getOrdValue(obj[i][j][0])
+        let y = getOrdValue(obj[i][j][1])
+        if branch >= x and branch <= y:
+          return i
+      elif getOrdValue(obj[i][j]) == branch:
+        return i
+    if obj[i].len == 1:
+      # else branch
+      return i
+  assert(false, "unreachable")
+
 proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode,
                      result: var Rope; count: var int;
                      isConst: bool, info: TLineInfo) =
@@ -2809,31 +2826,14 @@ proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode,
           branch = getOrdValue(constOrNil[i])
           break
 
-    var selectedBranch = -1
-    block branchSelection:
-      for i in 1 ..< obj.len:
-        for j in 0 .. obj[i].len - 2:
-          if obj[i][j].kind == nkRange:
-              let x = getOrdValue(obj[i][j][0])
-              let y = getOrdValue(obj[i][j][1])
-              if branch >= x and branch <= y:
-                selectedBranch = i
-                break branchSelection
-          elif getOrdValue(obj[i][j]) == branch:
-            selectedBranch = i
-            break branchSelection
-        if obj[i].len == 1:
-          # else branch
-          selectedBranch = i
-    assert(selectedBranch >= 1)
-
+    let selectedBranch = caseObjDefaultBranch(obj, branch)
     result.add "{"
     var countB = 0
     let b = lastSon(obj[selectedBranch])
     # designated initilization is the only way to init non first element of unions
     # branches are allowed to have no members (b.len == 0), in this case they don't need initializer
-    if  b.kind == nkRecList and b.len > 0:
-      result.add "._" &  mangleRecFieldName(p.module, obj[0].sym) & "_" & $selectedBranch & " = {"
+    if b.kind == nkRecList and b.len > 0:
+      result.add "._" & mangleRecFieldName(p.module, obj[0].sym) & "_" & $selectedBranch & " = {"
       getNullValueAux(p, t,  b, constOrNil, result, countB, isConst, info)
       result.add "}"
     elif b.kind == nkSym:

compiler/ccgreset.nim

@@ -0,0 +1,94 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# included from cgen.nim
+
+## Code specialization instead of the old, incredibly slow 'genericReset'
+## implementation.
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType)
+
+proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
+                     typ: PType) =
+  if n == nil: return
+  case n.kind
+  of nkRecList:
+    for i in 0..<n.len:
+      specializeResetN(p, accessor, n[i], typ)
+  of nkRecCase:
+    if (n[0].kind != nkSym): internalError(p.config, n.info, "specializeResetN")
+    let disc = n[0].sym
+    if disc.loc.r == nil: fillObjectFields(p.module, typ)
+    if disc.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
+    for i in 1..<n.len:
+      let branch = n[i]
+      assert branch.kind in {nkOfBranch, nkElse}
+      if branch.kind == nkOfBranch:
+        genCaseRange(p, branch)
+      else:
+        lineF(p, cpsStmts, "default:$n", [])
+      specializeResetN(p, accessor, lastSon(branch), typ)
+      lineF(p, cpsStmts, "break;$n", [])
+    lineF(p, cpsStmts, "} $n", [])
+    specializeResetT(p, "$1.$2" % [accessor, disc.loc.r], disc.loc.t)
+  of nkSym:
+    let field = n.sym
+    if field.typ.kind == tyVoid: return
+    if field.loc.r == nil: fillObjectFields(p.module, typ)
+    if field.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    specializeResetT(p, "$1.$2" % [accessor, field.loc.r], field.loc.t)
+  else: internalError(p.config, n.info, "specializeResetN()")
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
+  if typ == nil: return
+
+  case typ.kind
+  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred,
+     tySink, tyOwned:
+    specializeResetT(p, accessor, lastSon(typ))
+  of tyArray:
+    let arraySize = lengthOrd(p.config, typ[0])
+    var i: TLoc
+    getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i)
+    linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
+            [i.r, arraySize])
+    specializeResetT(p, ropecg(p.module, "$1[$2]", [accessor, i.r]), typ[1])
+    lineF(p, cpsStmts, "}$n", [])
+  of tyObject:
+    for i in 0..<typ.len:
+      var x = typ[i]
+      if x != nil: x = x.skipTypes(skipPtrs)
+      specializeResetT(p, accessor.parentObj(p.module), x)
+    if typ.n != nil: specializeResetN(p, accessor, typ.n, typ)
+  of tyTuple:
+    let typ = getUniqueType(typ)
+    for i in 0..<typ.len:
+      specializeResetT(p, ropecg(p.module, "$1.Field$2", [accessor, i]), typ[i])
+
+  of tyString, tyRef, tySequence:
+    lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1, NIM_NIL);$n", [accessor])
+
+  of tyProc:
+    if typ.callConv == ccClosure:
+      lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1.ClE_0, NIM_NIL);$n", [accessor])
+      lineCg(p, cpsStmts, "$1.ClP_0 = NIM_NIL;$n", [accessor])
+    else:
+      lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  of tyChar, tyBool, tyEnum, tyInt..tyUInt64:
+    lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
+  of tyCString, tyPointer, tyPtr, tyVar, tyLent:
+    lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  else:
+    discard
+
+proc specializeReset(p: BProc, a: TLoc) =
+  specializeResetT(p, rdLoc(a), a.t)

compiler/ccgstmts.nim

@@ -688,14 +688,15 @@ proc genBreakStmt(p: BProc, t: PNode) =
 
 proc raiseExit(p: BProc) =
   assert p.config.exc == excGoto
-  p.flags.incl nimErrorFlagAccessed
-  if p.nestedTryStmts.len == 0:
-    p.flags.incl beforeRetNeeded
-    # easy case, simply goto 'ret':
-    lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto BeforeRet_;$n", [])
-  else:
-    lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto LA$1_;$n",
-      [p.nestedTryStmts[^1].label])
+  if nimErrorFlagDisabled notin p.flags:
+    p.flags.incl nimErrorFlagAccessed
+    if p.nestedTryStmts.len == 0:
+      p.flags.incl beforeRetNeeded
+      # easy case, simply goto 'ret':
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto BeforeRet_;$n", [])
+    else:
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto LA$1_;$n",
+        [p.nestedTryStmts[^1].label])
 
 proc finallyActions(p: BProc) =
   if p.config.exc != excGoto and p.nestedTryStmts.len > 0 and p.nestedTryStmts[^1].inExcept:
@@ -981,7 +982,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   inc(p.labels, 2)
   let etmp = p.labels
 
-  lineCg(p, cpsStmts, "std::exception_ptr T$1_ = nullptr;", [etmp])
+  p.procSec(cpsInit).add(ropecg(p.module, "\tstd::exception_ptr T$1_ = nullptr;", [etmp]))
 
   let fin = if t[^1].kind == nkFinally: t[^1] else: nil
   p.nestedTryStmts.add((fin, false, 0.Natural))
@@ -1489,15 +1490,6 @@ proc genPragma(p: BProc, n: PNode) =
       p.module.injectStmt = p.s(cpsStmts)
     else: discard
 
-proc fieldDiscriminantCheckNeeded(p: BProc, asgn: PNode): bool =
-  if optFieldCheck in p.options:
-    var le = asgn[0]
-    if le.kind == nkCheckedFieldExpr:
-      var field = le[0][1].sym
-      result = sfDiscriminant in field.flags
-    elif le.kind == nkDotExpr:
-      var field = le[1].sym
-      result = sfDiscriminant in field.flags
 
 proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
                           field: PSym) =
@@ -1534,24 +1526,20 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   initLocExpr(p, e[0], a)
   getTemp(p, a.t, tmp)
   expr(p, e[1], tmp)
-  let field = dotExpr[1].sym
-  if optTinyRtti in p.config.globalOptions:
-    let t = dotExpr[0].typ.skipTypes(abstractInst)
-    var oldVal, newVal: TLoc
-    genCaseObjDiscMapping(p, e[0], t, field, oldVal)
-    genCaseObjDiscMapping(p, e[1], t, field, newVal)
-    lineCg(p, cpsStmts,
-          "if ($1 != $2) { #raiseObjectCaseTransition(); $3}$n",
-          [rdLoc(oldVal), rdLoc(newVal), raiseInstr(p)])
-  else:
+  if optTinyRtti notin p.config.globalOptions:
+    let field = dotExpr[1].sym
     genDiscriminantCheck(p, a, tmp, dotExpr[0].typ, field)
+    message(p.config, e.info, warnCaseTransition)
   genAssignment(p, a, tmp, {})
 
 proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
   if e[0].kind == nkSym and sfGoto in e[0].sym.flags:
     genLineDir(p, e)
     genGotoVar(p, e[1])
-  elif not fieldDiscriminantCheckNeeded(p, e):
+  elif optFieldCheck in p.options and isDiscriminantField(e[0]):
+    genLineDir(p, e)
+    asgnFieldDiscriminant(p, e)
+  else:
     let le = e[0]
     let ri = e[1]
     var a: TLoc
@@ -1565,10 +1553,6 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
     assert(a.t != nil)
     genLineDir(p, ri)
     loadInto(p, le, ri, a)
-  else:
-    genLineDir(p, e)
-    asgnFieldDiscriminant(p, e)
-    message(p.config, e.info, warnCaseTransition)
 
 proc genStmts(p: BProc, t: PNode) =
   var a: TLoc

compiler/ccgtypes.nim

@@ -1308,6 +1308,7 @@ proc genTypeInfoV2(m: BModule, t, origType: PType, name: Rope; info: TLineInfo)
   else:
     typeName = rope("NIM_NIL")
 
+  discard cgsym(m, "TNimType")
   m.s[cfsData].addf("N_LIB_PRIVATE TNimType $1;$n", [name])
   let destroyImpl = genHook(m, t, info, attachedDestructor)
   let traceImpl = genHook(m, t, info, attachedTrace)

compiler/cgen.nim

@@ -390,6 +390,8 @@ proc isComplexValueType(t: PType): bool {.inline.} =
   result = t.kind in {tyArray, tySet, tyTuple, tyObject} or
     (t.kind == tyProc and t.callConv == ccClosure)
 
+include ccgreset
+
 proc resetLoc(p: BProc, loc: var TLoc) =
   let containsGcRef = optSeqDestructors notin p.config.globalOptions and containsGarbageCollectedRef(loc.t)
   let typ = skipTypes(loc.t, abstractVarRange)
@@ -409,8 +411,10 @@ proc resetLoc(p: BProc, loc: var TLoc) =
     if optNilCheck in p.options:
       linefmt(p, cpsStmts, "#chckNil((void*)$1);$n", [addrLoc(p.config, loc)])
     if loc.storage != OnStack and containsGcRef:
-      linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
-              [addrLoc(p.config, loc), genTypeInfo(p.module, loc.t, loc.lode.info)])
+      specializeReset(p, loc)
+      when false:
+        linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
+                [addrLoc(p.config, loc), genTypeInfo(p.module, loc.t, loc.lode.info)])
       # XXX: generated reset procs should not touch the m_type
       # field, so disabling this should be safe:
       genObjectInit(p, cpsStmts, loc.t, loc, constructObj)
@@ -680,6 +684,7 @@ proc loadDynamicLib(m: BModule, lib: PLib) =
     else:
       var p = newProc(nil, m)
       p.options = p.options - {optStackTrace}
+      p.flags.incl nimErrorFlagDisabled
       var dest: TLoc
       initLoc(dest, locTemp, lib.path, OnStack)
       dest.r = getTempName(m)
@@ -973,6 +978,9 @@ proc genProcBody(p: BProc; procBody: PNode) =
     p.blocks[0].sections[cpsLocals].add(ropecg(p.module, "NIM_BOOL* nimErr_;$n", []))
     p.blocks[0].sections[cpsInit].add(ropecg(p.module, "nimErr_ = #nimErrorFlag();$n", []))
 
+proc isNoReturn(m: BModule; s: PSym): bool {.inline.} =
+  sfNoReturn in s.flags and m.config.exc != excGoto
+
 proc genProcAux(m: BModule, prc: PSym) =
   var p = newProc(prc, m)
   var header = genProcHeader(m, prc)
@@ -1027,7 +1035,7 @@ proc genProcAux(m: BModule, prc: PSym) =
 
   var generatedProc: Rope
   generatedProc.genCLineDir prc.info, m.config
-  if sfNoReturn in prc.flags:
+  if isNoReturn(p.module, prc):
     if hasDeclspec in extccomp.CC[p.config.cCompiler].props:
       header = "__declspec(noreturn) " & header
   if sfPure in prc.flags:
@@ -1090,13 +1098,13 @@ proc genProcPrototype(m: BModule, sym: PSym) =
     let asPtr = isReloadable(m, sym)
     var header = genProcHeader(m, sym, asPtr)
     if not asPtr:
-      if sfNoReturn in sym.flags and hasDeclspec in extccomp.CC[m.config.cCompiler].props:
+      if isNoReturn(m, sym) and hasDeclspec in extccomp.CC[m.config.cCompiler].props:
         header = "__declspec(noreturn) " & header
       if sym.typ.callConv != ccInline and requiresExternC(m, sym):
         header = "extern \"C\" " & header
       if sfPure in sym.flags and hasAttribute in CC[m.config.cCompiler].props:
         header.add(" __attribute__((naked))")
-      if sfNoReturn in sym.flags and hasAttribute in CC[m.config.cCompiler].props:
+      if isNoReturn(m, sym) and hasAttribute in CC[m.config.cCompiler].props:
         header.add(" __attribute__((noreturn))")
     m.s[cfsProcHeaders].add(ropecg(m, "$1;$N", [header]))
 
@@ -1999,8 +2007,9 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
     if m.config.exc == excGoto and getCompilerProc(graph, "nimTestErrorFlag") != nil:
       discard cgsym(m, "nimTestErrorFlag")
 
-    for i in countdown(high(graph.globalDestructors), 0):
-      n.add graph.globalDestructors[i]
+    if {optGenStaticLib, optGenDynLib} * m.config.globalOptions == {}:
+      for i in countdown(high(graph.globalDestructors), 0):
+        n.add graph.globalDestructors[i]
   if passes.skipCodegen(m.config, n): return
   if moduleHasChanged(graph, m.module):
     # if the module is cached, we don't regenerate the main proc

compiler/cgendata.nim

@@ -70,7 +70,8 @@ type
     hasCurFramePointer,
     noSafePoints,
     nimErrorFlagAccessed,
-    nimErrorFlagDeclared
+    nimErrorFlagDeclared,
+    nimErrorFlagDisabled
 
   TCProc = object             # represents C proc that is currently generated
     prc*: PSym                # the Nim proc that this C proc belongs to

compiler/commands.nim

@@ -478,6 +478,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       of "boehm":
         conf.selectedGC = gcBoehm
         defineSymbol(conf.symbols, "boehmgc")
+        incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS
       of "refc":
         conf.selectedGC = gcRefc
       of "v2":

compiler/dfa.nim

@@ -35,26 +35,30 @@ from patterns import sameTrees
 
 type
   InstrKind* = enum
-    goto, fork, join, def, use
+    goto, fork, def, use
   Instr* = object
     n*: PNode # contains the def/use location.
     case kind*: InstrKind
-    of goto, fork, join: dest*: int
+    of goto, fork: dest*: int
     else: discard
 
   ControlFlowGraph* = seq[Instr]
 
   TPosition = distinct int
+
   TBlock = object
-    label: PSym
-    fixups: seq[TPosition]
+    case isTryBlock: bool
+    of false:
+      label: PSym
+      breakFixups: seq[(TPosition, seq[PNode])] #Contains the gotos for the breaks along with their pending finales
+    of true:
+      finale: PNode
+      raiseFixups: seq[TPosition] #Contains the gotos for the raises
 
   Con = object
     code: ControlFlowGraph
     inCall, inTryStmt: int
     blocks: seq[TBlock]
-    tryStmtFixups: seq[TPosition]
-    forks: seq[TPosition]
     owner: PSym
 
 proc debugInfo(info: TLineInfo): string =
@@ -66,7 +70,7 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
   var jumpTargets = initIntSet()
   let last = if last < 0: c.len-1 else: min(last, c.len-1)
   for i in start..last:
-    if c[i].kind in {goto, fork, join}:
+    if c[i].kind in {goto, fork}:
       jumpTargets.incl(i+c[i].dest)
   var i = start
   while i <= last:
@@ -77,7 +81,7 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
     case c[i].kind
     of def, use:
       result.add renderTree(c[i].n)
-    of goto, fork, join:
+    of goto, fork:
       result.add "L"
       result.addInt c[i].dest+i
     result.add("\t#")
@@ -96,7 +100,6 @@ proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
 proc forkI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: fork, dest: 0)
-  c.forks.add result
 
 proc gotoI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
@@ -104,6 +107,24 @@ proc gotoI(c: var Con; n: PNode): TPosition =
 
 #[
 
+Join is no more
+===============
+Instead of generating join instructions we adapt our traversal of the CFG.
+
+When encountering a fork we split into two paths, we follow the path
+starting at "pc + 1" until it encounters the joinpoint: "pc + forkInstr.dest".
+If we encounter gotos that would jump further than the current joinpoint,
+as can happen with gotos generated by unstructured controlflow such as break, raise or return,
+we simply suspend following the current path, and follow the other path until the new joinpoint
+which is simply the instruction pointer returned to us by the now suspended path.
+If the path we are following now, also encounters a goto that exceeds the joinpoint
+we repeat the process; suspending the current path and evaluating the other one with a new joinpoint.
+If we eventually reach a common joinpoint we join the two paths.
+This new "ping-pong" approach has the obvious advantage of not requiring join instructions, as such
+cutting down on the CFG size but is also mandatory for correctly handling complicated cases
+of unstructured controlflow.
+
+
 Design of join
 ==============
 
@@ -252,10 +273,6 @@ duplicate the 'join' instructions on breaks and return exits!
 
 ]#
 
-proc joinI(c: var Con; fromFork: TPosition; n: PNode) =
-  let dist = fromFork.int - c.code.len
-  c.code.add Instr(n: n, kind: join, dest: dist)
-
 proc genLabel(c: Con): TPosition =
   result = TPosition(c.code.len)
 
@@ -271,14 +288,23 @@ proc patch(c: var Con, p: TPosition) =
   doAssert(low(int) div 2 + 1 < diff and diff < high(int) div 2)
   c.code[p].dest = diff
 
+proc gen(c: var Con; n: PNode) # {.noSideEffect.}
+
 proc popBlock(c: var Con; oldLen: int) =
-  for f in c.blocks[oldLen].fixups:
-    c.patch(f)
+  var exits: seq[TPosition]
+  exits.add c.gotoI(newNode(nkEmpty))
+  for f in c.blocks[oldLen].breakFixups:
+    c.patch(f[0])
+    for finale in f[1]:
+      c.gen(finale)
+    exits.add c.gotoI(newNode(nkEmpty))
+  for e in exits:
+    c.patch e
   c.blocks.setLen(oldLen)
 
 template withBlock(labl: PSym; body: untyped) {.dirty.} =
   var oldLen {.gensym.} = c.blocks.len
-  c.blocks.add TBlock(label: labl, fixups: @[])
+  c.blocks.add TBlock(isTryBlock: false, label: labl)
   body
   popBlock(c, oldLen)
 
@@ -286,8 +312,6 @@ proc isTrue(n: PNode): bool =
   n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or
     n.kind == nkIntLit and n.intVal != 0
 
-proc gen(c: var Con; n: PNode) # {.noSideEffect.}
-
 when true:
   proc genWhile(c: var Con; n: PNode) =
     # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
@@ -299,12 +323,13 @@ when true:
 
     Becomes:
 
-    if cond:
-      body
+    block:
       if cond:
         body
         if cond:
           body
+          if cond:
+            body
 
     We still need to ensure 'break' resolves properly, so an AST to AST
     translation is impossible.
@@ -330,22 +355,19 @@ when true:
     if isTrue(n[0]):
       # 'while true' is an idiom in Nim and so we produce
       # better code for it:
-      for i in 0..2:
-        withBlock(nil):
+      withBlock(nil):
+        for i in 0..2:
           c.gen(n[1])
     else:
-      let oldForksLen = c.forks.len
-      var endings: array[3, TPosition]
-      for i in 0..2:
-        withBlock(nil):
+      withBlock(nil):
+        var endings: array[3, TPosition]
+        for i in 0..2:
           c.gen(n[0])
           endings[i] = c.forkI(n)
           c.gen(n[1])
-      for i in countdown(endings.high, 0):
-        let endPos = endings[i]
-        c.patch(endPos)
-        c.joinI(c.forks.pop(), n)
-      doAssert(c.forks.len == oldForksLen)
+        for i in countdown(endings.high, 0):
+          let endPos = endings[i]
+          c.patch(endPos)
 
 else:
 
@@ -356,7 +378,6 @@ else:
     #   body
     #   jmp lab1
     # lab2:
-    let oldForksLen = c.forks.len
     let lab1 = c.genLabel
     withBlock(nil):
       if isTrue(n[0]):
@@ -368,35 +389,11 @@ else:
         c.gen(n[1])
         c.jmpBack(n, lab1)
         c.patch(lab2)
-    setLen(c.forks, oldForksLen)
-
-proc genBlock(c: var Con; n: PNode) =
-  withBlock(n[0].sym):
-    c.gen(n[1])
-
-proc genJoins(c: var Con; n: PNode) =
-  for i in countdown(c.forks.high, 0): joinI(c, c.forks[i], n)
-
-proc genBreak(c: var Con; n: PNode) =
-  genJoins(c, n)
-  let lab1 = c.gotoI(n)
-  if n[0].kind == nkSym:
-    #echo cast[int](n[0].sym)
-    for i in countdown(c.blocks.len-1, 0):
-      if c.blocks[i].label == n[0].sym:
-        c.blocks[i].fixups.add lab1
-        return
-    #globalError(n.info, "VM problem: cannot find 'break' target")
-  else:
-    c.blocks[c.blocks.high].fixups.add lab1
 
 template forkT(n, body) =
-  let oldLen = c.forks.len
   let lab1 = c.forkI(n)
   body
   c.patch(lab1)
-  c.joinI(lab1, n)
-  setLen(c.forks, oldLen)
 
 proc genIf(c: var Con, n: PNode) =
   #[
@@ -435,7 +432,6 @@ proc genIf(c: var Con, n: PNode) =
     join F1
 
   ]#
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
   for i in 0..<n.len:
     var it = n[i]
@@ -448,8 +444,6 @@ proc genIf(c: var Con, n: PNode) =
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
 
 proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, a
@@ -476,7 +470,6 @@ proc genCase(c: var Con; n: PNode) =
     abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
 
   var endings: seq[TPosition] = @[]
-  let oldLen = c.forks.len
   c.gen(n[0])
   for i in 1..<n.len:
     let it = n[i]
@@ -493,27 +486,53 @@ proc genCase(c: var Con; n: PNode) =
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
+
+proc genBlock(c: var Con; n: PNode) =
+  withBlock(n[0].sym):
+    c.gen(n[1])
+
+proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) =
+  let lab1 = c.gotoI(n)
+  if c.blocks[i].isTryBlock:
+    c.blocks[i].raiseFixups.add lab1
+  else:
+    var trailingFinales: seq[PNode]
+    if c.inTryStmt > 0: #Ok, we are in a try, lets see which (if any) try's we break out from:
+      for b in countdown(c.blocks.high, i):
+        if c.blocks[b].isTryBlock:
+          trailingFinales.add c.blocks[b].finale
+
+    c.blocks[i].breakFixups.add (lab1, trailingFinales)
+
+proc genBreak(c: var Con; n: PNode) =
+  if n[0].kind == nkSym:
+    #echo cast[int](n[0].sym)
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock and c.blocks[i].label == n[0].sym:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    #globalError(n.info, "VM problem: cannot find 'break' target")
+  else:
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
 
 proc genTry(c: var Con; n: PNode) =
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
-  inc c.inTryStmt
-  let oldFixups = c.tryStmtFixups.len
 
+  let oldLen = c.blocks.len
+  c.blocks.add TBlock(isTryBlock: true, finale: if n[^1].kind == nkFinally: n[^1] else: newNode(nkEmpty))
+
+  inc c.inTryStmt
   #let elsePos = c.forkI(n)
   c.gen(n[0])
   dec c.inTryStmt
-  for i in oldFixups..c.tryStmtFixups.high:
-    let f = c.tryStmtFixups[i]
-    c.patch(f)
-    # we also need to produce join instructions
-    # for the 'fork' that might precede the goto instruction
-    if f.int-1 >= 0 and c.code[f.int-1].kind == fork:
-      c.joinI(TPosition(f.int-1), n)
 
-  setLen(c.tryStmtFixups, oldFixups)
+  for f in c.blocks[oldLen].raiseFixups:
+    c.patch(f)
+
+  c.blocks.setLen oldLen
 
   #c.patch(elsePos)
   for i in 1..<n.len:
@@ -526,25 +545,26 @@ proc genTry(c: var Con; n: PNode) =
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
 
   # join the 'elsePos' forkI instruction:
-  #c.joinI(c.forks.pop(), n)
+  #c.joinI(c.blocks[^1].forks.pop(), n)
 
   let fin = lastSon(n)
   if fin.kind == nkFinally:
     c.gen(fin[0])
-  doAssert(c.forks.len == oldLen)
 
 template genNoReturn(c: var Con; n: PNode) =
   # leave the graph
   c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 proc genRaise(c: var Con; n: PNode) =
-  genJoins(c, n)
   gen(c, n[0])
   if c.inTryStmt > 0:
-    c.tryStmtFixups.add c.gotoI(n)
+    for i in countdown(c.blocks.high, 0):
+      if c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    assert false #Unreachable
   else:
     genNoReturn(c, n)
 
@@ -553,12 +573,11 @@ proc genImplicitReturn(c: var Con) =
     gen(c, c.owner.ast[resultPos])
 
 proc genReturn(c: var Con; n: PNode) =
-  genJoins(c, n)
   if n[0].kind != nkEmpty:
     gen(c, n[0])
   else:
     genImplicitReturn(c)
-  genNoReturn(c, n)
+  genBreakOrRaiseAux(c, 0, n)
 
 const
   InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp}
@@ -598,7 +617,8 @@ proc genUse(c: var Con; orig: PNode) =
 proc aliases*(obj, field: PNode): bool =
   var n = field
   var obj = obj
-  while obj.kind in {nkHiddenSubConv, nkHiddenStdConv, nkObjDownConv, nkObjUpConv}:
+  while obj.kind in {nkHiddenSubConv, nkHiddenStdConv, nkObjDownConv, nkObjUpConv,
+                     nkAddr, nkHiddenAddr, nkDerefExpr, nkHiddenDeref}:
     obj = obj[0]
   while true:
     if sameTrees(obj, n): return true
@@ -639,8 +659,10 @@ proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
     of nkHiddenDeref, nkDerefExpr:
       # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
       # pointer indirection.
+      # bug #14159, we cannot reason about sinkParam[].location as it can
+      # still be shared for tyRef.
       n = n[0]
-      return n.kind == nkSym and n.sym.owner == owner and (isSinkParam(n.sym) or
+      return n.kind == nkSym and n.sym.owner == owner and (
           n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
     else:
       break
@@ -706,9 +728,11 @@ proc genCall(c: var Con; n: PNode) =
     # lab1:
     # join F1
     let endGoto = c.forkI(n)
-    c.tryStmtFixups.add c.gotoI(n)
+    for i in countdown(c.blocks.high, 0):
+      if c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        break
     c.patch(endGoto)
-    c.joinI(c.forks.pop(), n)
   dec c.inCall
 
 proc genMagic(c: var Con; n: PNode; m: TMagic) =
@@ -782,6 +806,7 @@ proc gen(c: var Con; n: PNode) =
 proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
   ## constructs a control flow graph for ``body``.
   var c = Con(code: @[], blocks: @[], owner: s)
-  gen(c, body)
-  genImplicitReturn(c)
+  withBlock(s):
+    gen(c, body)
+    genImplicitReturn(c)
   shallowCopy(result, c.code)

compiler/docgen.nim

@@ -86,11 +86,14 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
     result = relativeTo(file, conf.docRoot.AbsoluteDir)
   else:
     bail()
+  if isAbsolute(result.string):
+    result = file.string.splitPath()[1].RelativeFile
   if isTitle:
     result = result.string.nativeToUnix.RelativeFile
   else:
     result = result.string.replace("..", "@@").RelativeFile ## refs #13223
   doAssert not result.isEmpty
+  doAssert not isAbsolute(result.string)
 
 proc whichType(d: PDoc; n: PNode): PSym =
   if n.kind == nkSym:
@@ -1140,7 +1143,10 @@ proc generateIndex*(d: PDoc) =
 proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
   if d.module == nil or sfMainModule in d.module.flags: # nil for eg for commandRst2Html
     if d.conf.outDir.isEmpty: d.conf.outDir = d.conf.docOutDir
-    if d.conf.outFile.isEmpty: d.conf.outFile = outfile.relativeTo(d.conf.outDir)
+    if d.conf.outFile.isEmpty:
+      d.conf.outFile = outfile.relativeTo(d.conf.outDir)
+      if isAbsolute(d.conf.outFile.string):
+        d.conf.outFile = splitPath(d.conf.outFile.string)[1].RelativeFile
 
 proc writeOutput*(d: PDoc, useWarning = false) =
   runAllExamples(d)

compiler/extccomp.nim

@@ -601,6 +601,34 @@ proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile):
       "Compiler '$1' doesn't support the requested target" %
       CC[compiler].name)
 
+proc ccHasSaneOverflow*(conf: ConfigRef): bool =
+  if conf.cCompiler == ccGcc:
+    result = false # assume an old or crappy GCC
+    var exe = getConfigVar(conf, conf.cCompiler, ".exe")
+    if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
+    let (s, exitCode) = try: execCmdEx(exe & " --version") except: ("", 1)
+    if exitCode == 0:
+      var i = 0
+      var j = 0
+      # the version is the last part of the first line:
+      while i < s.len and s[i] != '\n':
+        if s[i] in {' ', '\t'}: j = i+1
+        inc i
+      if j > 0:
+        var major = 0
+        while j < s.len and s[j] in {'0'..'9'}:
+          major = major * 10 + (ord(s[j]) - ord('0'))
+          inc j
+        if i < s.len and s[j] == '.': inc j
+        while j < s.len and s[j] in {'0'..'9'}:
+          inc j
+        if j+1 < s.len and s[j] == '.' and s[j+1] in {'0'..'9'}:
+          # we found a third version number, chances are high
+          # we really parsed the version:
+          result = major >= 5
+  else:
+    result = conf.cCompiler == ccCLang
+
 proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
            elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler
@@ -836,12 +864,12 @@ proc execLinkCmd(conf: ConfigRef; linkCmd: string) =
 
 proc maybeRunDsymutil(conf: ConfigRef; exe: AbsoluteFile) =
   when defined(osx):
-    if optCDebug notin conf.globalOptions: return
-    # if needed, add an option to skip or override location
-    let cmd = "dsymutil " & $(exe).quoteShell
-    conf.extraCmds.add cmd
-    tryExceptOSErrorMessage(conf, "invocation of dsymutil failed."):
-      execExternalProgram(conf, cmd, hintExecuting)
+    if optCDebug in conf.globalOptions and optGenStaticLib notin conf.globalOptions:
+      # if needed, add an option to skip or override location
+      let cmd = "dsymutil " & $(exe).quoteShell
+      conf.extraCmds.add cmd
+      tryExceptOSErrorMessage(conf, "invocation of dsymutil failed."):
+        execExternalProgram(conf, cmd, hintExecuting)
 
 proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx: int)) =
   let runCb = proc (idx: int, p: Process) =

compiler/injectdestructors.nim

@@ -22,9 +22,9 @@
 # - eliminate 'wasMoved(x); destroy(x)' pairs as a post processing step.
 
 import
-  intsets, ast, msgs, renderer, magicsys, types, idents,
+  intsets, ast, astalgo, msgs, renderer, magicsys, types, idents,
   strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
-  lineinfos, parampatterns, sighashes
+  lineinfos, parampatterns, sighashes, liftdestructors
 
 from trees import exprStructuralEquivalent
 from algorithm import reverse
@@ -49,6 +49,12 @@ type
     uninit: IntSet # set of uninit'ed vars
     uninitComputed: bool
 
+  ProcessMode = enum
+    normal
+    consumed
+    sinkArg
+
+
 const toDebug {.strdefine.} = ""
 
 template dbg(body) =
@@ -56,9 +62,12 @@ template dbg(body) =
     if c.owner.name.s == toDebug or toDebug == "always":
       body
 
-proc isLastRead(location: PNode; c: var Con; pc, comesFrom: int): int =
+proc p(n: PNode; c: var Con; mode: ProcessMode): PNode
+proc moveOrCopy(dest, ri: PNode; c: var Con): PNode
+
+proc isLastRead(location: PNode; c: var Con; pc, until: int): int =
   var pc = pc
-  while pc < c.g.len:
+  while pc < c.g.len and pc < until:
     case c.g[pc].kind
     of def:
       if defInstrTargets(c.g[pc], location):
@@ -74,17 +83,17 @@ proc isLastRead(location: PNode; c: var Con; pc, comesFrom: int): int =
       pc = pc + c.g[pc].dest
     of fork:
       # every branch must lead to the last read of the location:
-      let variantA = isLastRead(location, c, pc+1, pc)
-      if variantA < 0: return -1
-      var variantB = isLastRead(location, c, pc + c.g[pc].dest, pc)
-      if variantB < 0: return -1
-      elif variantB == high(int):
-        variantB = variantA
-      pc = variantB
-    of InstrKind.join:
-      let dest = pc + c.g[pc].dest
-      if dest == comesFrom: return pc + 1
-      inc pc
+      var variantA = pc + 1
+      var variantB = pc + c.g[pc].dest
+      while variantA != variantB:
+        if min(variantA, variantB) < 0: return -1
+        if max(variantA, variantB) >= c.g.len or min(variantA, variantB) >= until:
+          break
+        if variantA < variantB:
+          variantA = isLastRead(location, c, variantA, min(variantB, until))
+        else:
+          variantB = isLastRead(location, c, variantB, min(variantA, until))
+      pc = min(variantA, variantB)
   return pc
 
 proc isLastRead(n: PNode; c: var Con): bool =
@@ -107,12 +116,12 @@ proc isLastRead(n: PNode; c: var Con): bool =
   # ensure that we don't find another 'use X' instruction.
   if instr+1 >= c.g.len: return true
 
-  result = isLastRead(n, c, instr+1, -1) >= 0
+  result = isLastRead(n, c, instr+1, int.high) >= 0
   dbg: echo "ugh ", c.otherRead.isNil, " ", result
 
-proc isFirstWrite(location: PNode; c: var Con; pc, comesFrom: int; instr: int): int =
+proc isFirstWrite(location: PNode; c: var Con; pc, until: int): int =
   var pc = pc
-  while pc < instr:
+  while pc < until:
     case c.g[pc].kind
     of def:
       if defInstrTargets(c.g[pc], location):
@@ -127,17 +136,17 @@ proc isFirstWrite(location: PNode; c: var Con; pc, comesFrom: int; instr: int):
       pc = pc + c.g[pc].dest
     of fork:
       # every branch must not contain a def/use of our location:
-      let variantA = isFirstWrite(location, c, pc+1, pc, instr)
-      if variantA < 0: return -1
-      var variantB = isFirstWrite(location, c, pc + c.g[pc].dest, pc, instr + c.g[pc].dest)
-      if variantB < 0: return -1
-      elif variantB == high(int):
-        variantB = variantA
-      pc = variantB
-    of InstrKind.join:
-      let dest = pc + c.g[pc].dest
-      if dest == comesFrom: return pc + 1
-      inc pc
+      var variantA = pc + 1
+      var variantB = pc + c.g[pc].dest
+      while variantA != variantB:
+        if min(variantA, variantB) < 0: return -1
+        if max(variantA, variantB) > until:
+          break
+        if variantA < variantB:
+          variantA = isFirstWrite(location, c, variantA, min(variantB, until))
+        else:
+          variantB = isFirstWrite(location, c, variantB, min(variantA, until))
+      pc = min(variantA, variantB)
   return pc
 
 proc isFirstWrite(n: PNode; c: var Con): bool =
@@ -156,10 +165,10 @@ proc isFirstWrite(n: PNode; c: var Con): bool =
   # ensure that we don't find another 'def/use X' instruction.
   if instr == 0: return true
 
-  result = isFirstWrite(n, c, 0, -1, instr) >= 0
+  result = isFirstWrite(n, c, 0, instr) >= 0
 
 proc initialized(code: ControlFlowGraph; pc: int,
-                 init, uninit: var IntSet; comesFrom: int): int =
+                 init, uninit: var IntSet; until: int): int =
   ## Computes the set of definitely initialized variables across all code paths
   ## as an IntSet of IDs.
   var pc = pc
@@ -168,20 +177,22 @@ proc initialized(code: ControlFlowGraph; pc: int,
     of goto:
       pc = pc + code[pc].dest
     of fork:
-      let target = pc + code[pc].dest
       var initA = initIntSet()
       var initB = initIntSet()
-      let pcA = initialized(code, pc+1, initA, uninit, pc)
-      discard initialized(code, target, initB, uninit, pc)
+      var variantA = pc + 1
+      var variantB = pc + code[pc].dest
+      while variantA != variantB:
+        if max(variantA, variantB) > until:
+          break
+        if variantA < variantB:
+          variantA = initialized(code, variantA, initA, uninit, min(variantB, until))
+        else:
+          variantB = initialized(code, variantB, initB, uninit, min(variantA, until))
+      pc = min(variantA, variantB)
       # we add vars if they are in both branches:
       for v in initA:
         if v in initB:
           init.incl v
-      pc = pcA+1
-    of InstrKind.join:
-      let target = pc + code[pc].dest
-      if comesFrom == target: return pc
-      inc pc
     of use:
       let v = code[pc].n.sym
       if v.kind != skParam and v.id notin init:
@@ -220,16 +231,19 @@ proc makePtrType(c: Con, baseType: PType): PType =
   result = newType(tyPtr, c.owner)
   addSonSkipIntLit(result, baseType)
 
+proc genOp(c: Con; op: PSym; dest: PNode): PNode =
+  let addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
+  addrExp.add(dest)
+  result = newTree(nkCall, newSymNode(op), addrExp)
+
 proc genOp(c: Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode =
   var op = t.attachedOps[kind]
-
   if op == nil or op.ast[genericParamsPos].kind != nkEmpty:
     # give up and find the canonical type instead:
     let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
     let canon = c.graph.canonTypes.getOrDefault(h)
     if canon != nil:
       op = canon.attachedOps[kind]
-
   if op == nil:
     #echo dest.typ.id
     globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &
@@ -241,9 +255,7 @@ proc genOp(c: Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode =
     if kind == attachedDestructor:
       echo "destructor is ", op.id, " ", op.ast
   if sfError in op.flags: checkForErrorPragma(c, t, ri, AttachedOpToStr[kind])
-  let addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
-  addrExp.add(dest)
-  result = newTree(nkCall, newSymNode(op), addrExp)
+  genOp(c, op, dest)
 
 proc genDestroy(c: Con; dest: PNode): PNode =
   let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink})
@@ -300,6 +312,44 @@ proc getTemp(c: var Con; typ: PType; info: TLineInfo): PNode =
   sym.typ = typ
   result = newSymNode(sym)
 
+proc genDiscriminantAsgn(c: var Con; n: PNode): PNode =
+  # discriminator is ordinal value that doesn't need sink destroy
+  # but fields within active case branch might need destruction
+
+  # tmp to support self assignments 
+  let tmp = getTemp(c, n[1].typ, n.info)
+  c.addTopVar(tmp)
+
+  result = newTree(nkStmtList)
+  result.add newTree(nkFastAsgn, tmp, p(n[1], c, consumed))
+  result.add p(n[0], c, normal)
+
+  let le = p(n[0], c, normal)
+  let leDotExpr = if le.kind == nkCheckedFieldExpr: le[0] else: le
+  let objType = leDotExpr[0].typ
+
+  if hasDestructor(objType):
+    if objType.attachedOps[attachedDestructor] != nil and
+        sfOverriden in objType.attachedOps[attachedDestructor].flags:
+      localError(c.graph.config, n.info, errGenerated, """Assignment to discriminant for object's with user defined destructor is not supported, object must have default destructor. 
+It is best to factor out piece of object that needs custom destructor into separate object or not use discriminator assignment""")
+      result.add newTree(nkFastAsgn, le, tmp)
+      return
+
+    # generate: if le != tmp: `=destroy`(le)
+    let branchDestructor = produceDestructorForDiscriminator(c.graph, objType, leDotExpr[1].sym, n.info)
+    let cond = newNodeIT(nkInfix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
+    cond.add newSymNode(getMagicEqSymForType(c.graph, le.typ, n.info))
+    cond.add le
+    cond.add tmp
+    let notExpr = newNodeIT(nkPrefix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
+    notExpr.add newSymNode(createMagic(c.graph, "not", mNot))
+    notExpr.add cond
+    result.add newTree(nkIfStmt, newTree(nkElifBranch, notExpr, genOp(c, branchDestructor, le)))
+    result.add newTree(nkFastAsgn, le, tmp)
+  else:
+    result.add newTree(nkFastAsgn, le, tmp)
+
 proc genWasMoved(n: PNode; c: var Con): PNode =
   result = newNodeI(nkCall, n.info)
   result.add(newSymNode(createMagic(c.graph, "wasMoved", mWasMoved)))
@@ -313,6 +363,7 @@ proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode =
 proc destructiveMoveVar(n: PNode; c: var Con): PNode =
   # generate: (let tmp = v; reset(v); tmp)
   if not hasDestructor(n.typ):
+    assert n.kind != nkSym or not hasDestructor(n.sym.typ)
     result = copyTree(n)
   else:
     result = newNodeIT(nkStmtListExpr, n.info, n.typ)
@@ -338,16 +389,8 @@ proc sinkParamIsLastReadCheck(c: var Con, s: PNode) =
     localError(c.graph.config, c.otherRead.info, "sink parameter `" & $s.sym.name.s &
         "` is already consumed at " & toFileLineCol(c. graph.config, s.info))
 
-type
-  ProcessMode = enum
-    normal
-    consumed
-    sinkArg
-
-proc p(n: PNode; c: var Con; mode: ProcessMode): PNode
-proc moveOrCopy(dest, ri: PNode; c: var Con): PNode
-
-proc isClosureEnv(n: PNode): bool = n.kind == nkSym and n.sym.name.s[0] == ':'
+proc isClosureEnv(n: PNode): bool =
+  n.kind == nkDotExpr and n[0].kind == nkHiddenDeref and n[0][0].typ.kind == tyRef
 
 proc passCopyToSink(n: PNode; c: var Con): PNode =
   result = newNodeIT(nkStmtListExpr, n.info, n.typ)
@@ -362,7 +405,7 @@ proc passCopyToSink(n: PNode; c: var Con): PNode =
     if isLValue(n) and not isClosureEnv(n) and n.typ.skipTypes(abstractInst).kind != tyRef:
       message(c.graph.config, n.info, hintPerformance,
         ("passing '$1' to a sink parameter introduces an implicit copy; " &
-        "use 'move($1)' to prevent it") % $n)
+        "if possible, rearrange your program's control flow to prevent it") % $n)
   else:
     if c.graph.config.selectedGC in {gcArc, gcOrc}:
       assert(not containsGarbageCollectedRef(n.typ))
@@ -867,6 +910,8 @@ proc p(n: PNode; c: var Con; mode: ProcessMode): PNode =
             cycleCheck(n, c)
           assert n[1].kind notin {nkAsgn, nkFastAsgn}
           result = moveOrCopy(p(n[0], c, mode), n[1], c)
+      elif isDiscriminantField(n[0]):
+        result = genDiscriminantAsgn(c, n)
       else:
         result = copyNode(n)
         result.add p(n[0], c, mode)
@@ -910,6 +955,10 @@ proc p(n: PNode; c: var Con; mode: ProcessMode): PNode =
       for i in 0..<n.len:
         result[i] = p(n[i], c, mode)
       inc c.hasUnstructuredCf
+    of nkCast:
+      result = shallowCopy(n)
+      result[0] = n[0]
+      result[1] = p(n[1], c, mode)
     else:
       result = shallowCopy(n)
       for i in 0..<n.len:
@@ -991,7 +1040,7 @@ proc computeUninit(c: var Con) =
     c.uninitComputed = true
     c.uninit = initIntSet()
     var init = initIntSet()
-    discard initialized(c.g, pc = 0, init, c.uninit, comesFrom = -1)
+    discard initialized(c.g, pc = 0, init, c.uninit, int.high)
 
 proc injectDefaultCalls(n: PNode, c: var Con) =
   case n.kind
@@ -1015,7 +1064,7 @@ proc injectDefaultCalls(n: PNode, c: var Con) =
 proc extractDestroysForTemporaries(c: Con, destroys: PNode): PNode =
   result = newNodeI(nkStmtList, destroys.info)
   for i in 0..<destroys.len:
-    if destroys[i][1][0].sym.kind == skTemp:
+    if destroys[i][1][0].sym.kind in {skTemp, skForVar}:
       result.add destroys[i]
       destroys[i] = c.emptyNode
 

compiler/installer.ini

@@ -89,6 +89,7 @@ Files: "bin/vccexe.exe"
 Files: "bin/nimgrab.exe"
 Files: "bin/nimpretty.exe"
 Files: "bin/testament.exe"
+Files: "bin/nim-gdb.bat"
 
 Files: "koch.exe"
 Files: "finish.exe"
@@ -117,6 +118,8 @@ Files: "bin/nim"
 [Unix]
 InstallScript: "yes"
 UninstallScript: "yes"
+Files: "bin/nim-gdb"
+Files: "bin/nim-gdb.bash"
 
 
 [InnoSetup]

compiler/jsgen.nim

@@ -1634,7 +1634,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   of tyObject:
     var initList: Rope
     createObjInitList(p, t, initIntSet(), initList)
-    result = ("{$1}") % [initList]
+    result = ("({$1})") % [initList]
     if indirect: result = "[$1]" % [result]
   of tyVar, tyPtr, tyLent, tyRef, tyPointer:
     if mapType(p, t) == etyBaseIndex:

compiler/liftdestructors.nim

@@ -26,6 +26,8 @@ type
     fn: PSym
     asgnForType: PType
     recurse: bool
+    filterDiscriminator: PSym  # we generating destructor for case branch
+    addMemReset: bool    # add wasMoved() call after destructor call
     c: PContext # c can be nil, then we are called from lambdalifting!
 
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode)
@@ -72,14 +74,62 @@ proc genAddr(g: ModuleGraph; x: PNode): PNode =
     result = newNodeIT(nkHiddenAddr, x.info, makeVarType(x.typ.owner, x.typ))
     result.add x
 
-proc destructorCall(g: ModuleGraph; op: PSym; x: PNode): PNode =
-  result = newNodeIT(nkCall, x.info, op.typ[0])
-  result.add(newSymNode(op))
-  result.add genAddr(g, x)
+
+proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode =
+  result = newNodeI(nkCall, i.info)
+  result.add createMagic(g, name, magic).newSymNode
+  result.add i
+
+proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
+  result = newNodeI(nkWhileStmt, c.info, 2)
+  let cmp = genBuiltin(c.g, mLtI, "<", i)
+  cmp.add genLen(c.g, dest)
+  cmp.typ = getSysType(c.g, c.info, tyBool)
+  result[0] = cmp
+  result[1] = newNodeI(nkStmtList, c.info)
+
+proc genIf(c: var TLiftCtx; cond, action: PNode): PNode =
+  result = newTree(nkIfStmt, newTree(nkElifBranch, cond, action))
+
+proc genContainerOf(c: TLiftCtx; objType: PType, field, x: PSym): PNode = 
+  # generate: cast[ptr ObjType](cast[int](addr(x)) - offsetOf(objType.field))
+  let intType = getSysType(c.g, unknownLineInfo, tyInt)
+
+  let addrOf = newNodeIT(nkAddr, c.info, makePtrType(x.owner, x.typ))
+  addrOf.add newDeref(newSymNode(x))
+  let castExpr1 = newNodeIT(nkCast, c.info, intType)
+  castExpr1.add newNodeIT(nkType, c.info, intType)
+  castExpr1.add addrOf
+
+  let dotExpr = newNodeIT(nkDotExpr, c.info, x.typ)
+  dotExpr.add newNodeIT(nkType, c.info, objType)
+  dotExpr.add newSymNode(field)
+  
+  let offsetOf = genBuiltin(c.g, mOffsetOf, "offsetof", dotExpr)
+  offsetOf.typ = intType
+
+  let minusExpr = genBuiltin(c.g, mSubI, "-", castExpr1)
+  minusExpr.typ = intType
+  minusExpr.add offsetOf
+
+  let objPtr = makePtrType(objType.owner, objType)
+  result = newNodeIT(nkCast, c.info, objPtr)
+  result.add newNodeIT(nkType, c.info, objPtr)
+  result.add minusExpr
+
+proc destructorCall(c: TLiftCtx; op: PSym; x: PNode): PNode =
+  var destroy = newNodeIT(nkCall, x.info, op.typ[0])
+  destroy.add(newSymNode(op))
+  destroy.add genAddr(c.g, x)
+  if c.addMemReset:
+    result = newTree(nkStmtList, destroy, genBuiltin(c.g, mWasMoved,  "wasMoved", x))
+  else:
+    result = destroy
 
 proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool) =
   case n.kind
   of nkSym:
+    if c.filterDiscriminator != nil: return
     let f = n.sym
     let b = if c.kind == attachedTrace: y else: y.dotField(f)
     if (sfCursor in f.flags and f.typ.skipTypes(abstractInst).kind in {tyRef, tyProc} and
@@ -90,6 +140,9 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       fillBody(c, f.typ, body, x.dotField(f), b)
   of nkNilLit: discard
   of nkRecCase:
+    let oldfilterDiscriminator = c.filterDiscriminator
+    if c.filterDiscriminator == n[0].sym:
+      c.filterDiscriminator = nil # we have found the case part, proceed as normal
     # XXX This is only correct for 'attachedSink'!
     var localEnforceDefaultOp = enforceDefaultOp
     if c.kind == attachedSink:
@@ -121,6 +174,7 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       caseStmt.add(branch)
     if emptyBranches != n.len-1:
       body.add(caseStmt)
+    c.filterDiscriminator = oldfilterDiscriminator 
   of nkRecList:
     for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp)
   else:
@@ -275,7 +329,7 @@ proc addDestructorCall(c: var TLiftCtx; orig: PType; body, x: PNode) =
   if op != nil:
     #markUsed(c.g.config, c.info, op, c.g.usageSym)
     onUse(c.info, op)
-    body.add destructorCall(c.g, op, x)
+    body.add destructorCall(c, op, x)
   elif useNoGc(c, t):
     internalError(c.g.config, c.info,
       "type-bound operator could not be resolved")
@@ -293,7 +347,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
 
       #markUsed(c.g.config, c.info, op, c.g.usageSym)
       onUse(c.info, op)
-      body.add destructorCall(c.g, op, x)
+      body.add destructorCall(c, op, x)
       result = true
     #result = addDestructorCall(c, t, body, x)
   of attachedAsgn, attachedSink, attachedTrace:
@@ -318,22 +372,6 @@ proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
   v.addVar(result, lowerings.newIntLit(c.g, body.info, first))
   body.add v
 
-proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode =
-  result = newNodeI(nkCall, i.info)
-  result.add createMagic(g, name, magic).newSymNode
-  result.add i
-
-proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
-  result = newNodeI(nkWhileStmt, c.info, 2)
-  let cmp = genBuiltin(c.g, mLtI, "<", i)
-  cmp.add genLen(c.g, dest)
-  cmp.typ = getSysType(c.g, c.info, tyBool)
-  result[0] = cmp
-  result[1] = newNodeI(nkStmtList, c.info)
-
-proc genIf(c: var TLiftCtx; cond, action: PNode): PNode =
-  result = newTree(nkIfStmt, newTree(nkElifBranch, cond, action))
-
 proc addIncStmt(c: var TLiftCtx; body, i: PNode) =
   let incCall = genBuiltin(c.g, mInc, "inc", i)
   incCall.add lowerings.newIntLit(c.g, c.info, 1)
@@ -382,7 +420,7 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
+    moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
   of attachedDestructor:
     # destroy all elements:
@@ -413,7 +451,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
+    moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
     # alternatively we could do this:
     when false:
@@ -421,7 +459,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.add newHookCall(c.g, t.asink, x, y)
   of attachedDestructor:
     doAssert t.destructor != nil
-    body.add destructorCall(c.g, t.destructor, x)
+    body.add destructorCall(c, t.destructor, x)
   of attachedTrace:
     body.add newHookCall(c.g, t.attachedOps[c.kind], x, y)
   of attachedDispose:
@@ -435,7 +473,7 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
+    moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
   of attachedDestructor, attachedDispose:
     body.add genBuiltin(c.g, mDestroy, "destroy", x)
@@ -477,8 +515,8 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, cond, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
     body.add genIf(c, cond, actions)
+    body.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace:
     if isFinal(elemType):
@@ -526,8 +564,8 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, cond, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(xenv, newNodeIT(nkNilLit, body.info, xenv.typ))
     body.add genIf(c, cond, actions)
+    body.add newAsgnStmt(xenv, newNodeIT(nkNilLit, body.info, xenv.typ))
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace:
     body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv), y)
@@ -584,8 +622,8 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, x, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
     body.add genIf(c, x, actions)
+    body.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace, attachedDispose: discard
 
@@ -637,8 +675,8 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, xx, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(xx, newNodeIT(nkNilLit, body.info, xx.typ))
     body.add genIf(c, xx, actions)
+    body.add newAsgnStmt(xx, newNodeIT(nkNilLit, body.info, xx.typ))
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace, attachedDispose: discard
 
@@ -738,33 +776,14 @@ proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType;
   typ.attachedOps[kind] = baseType.attachedOps[kind]
   result = typ.attachedOps[kind]
 
-proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
+proc symPrototype(g: ModuleGraph; typ: PType; kind: TTypeAttachedOp;
               info: TLineInfo): PSym =
-  if typ.kind == tyDistinct:
-    return produceSymDistinctType(g, c, typ, kind, info)
 
-  var a: TLiftCtx
-  a.info = info
-  a.g = g
-  a.kind = kind
-  a.c = c
-  let body = newNodeI(nkStmtList, info)
   let procname = getIdent(g.cache, AttachedOpToStr[kind])
-
   result = newSym(skProc, procname, typ.owner, info)
-  a.fn = result
-  a.asgnForType = typ
-
   let dest = newSym(skParam, getIdent(g.cache, "dest"), result, info)
   let src = newSym(skParam, getIdent(g.cache, if kind == attachedTrace: "env" else: "src"), result, info)
-  var d: PNode
-  #if kind notin {attachedTrace, attachedDispose}:
   dest.typ = makeVarType(typ.owner, typ)
-  d = newDeref(newSymNode(dest))
-  #else:
-  #  dest.typ = typ
-  #  d = newSymNode(dest)
-
   if kind == attachedTrace:
     src.typ = getSysType(g, info, tyPointer)
   else:
@@ -775,6 +794,30 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
   if kind notin {attachedDestructor, attachedDispose}:
     result.typ.addParam src
 
+  var n = newNodeI(nkProcDef, info, bodyPos+1)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = newNodeI(nkStmtList, info)
+  result.ast = n
+  incl result.flags, sfFromGeneric
+  incl result.flags, sfGeneratedOp
+
+
+proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
+              info: TLineInfo): PSym =
+  if typ.kind == tyDistinct:
+    return produceSymDistinctType(g, c, typ, kind, info)
+
+  result = symPrototype(g, typ, kind, info)
+  var a = TLiftCtx(info: info, g: g, kind: kind, c: c, asgnForType:typ)
+  a.fn = result
+
+  let dest = result.typ.n[1].sym
+  let d = newDeref(newSymNode(dest))
+  let src = if kind in {attachedDestructor, attachedDispose}: newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
+            else: newSymNode(result.typ.n[2].sym)
+
   # register this operation already:
   typ.attachedOps[kind] = result
 
@@ -782,8 +825,8 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
        sfOverriden in typ.attachedOps[attachedDestructor].flags:
     ## compiler can use a combination of `=destroy` and memCopy for sink op
     dest.flags.incl sfCursor
-    body.add newOpCall(typ.attachedOps[attachedDestructor], d[0])
-    body.add newAsgnStmt(d, newSymNode(src))
+    result.ast[bodyPos].add newOpCall(typ.attachedOps[attachedDestructor], d[0])
+    result.ast[bodyPos].add newAsgnStmt(d, src)
   else:
     var tk: TTypeKind
     if g.config.selectedGC in {gcArc, gcOrc, gcHooks}:
@@ -792,20 +835,33 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
       tk = tyNone # no special casing for strings and seqs
     case tk
     of tySequence:
-      fillSeqOp(a, typ, body, d, newSymNode(src))
+      fillSeqOp(a, typ, result.ast[bodyPos], d, src)
     of tyString:
-      fillStrOp(a, typ, body, d, newSymNode(src))
+      fillStrOp(a, typ, result.ast[bodyPos], d, src)
     else:
-      fillBody(a, typ, body, d, newSymNode(src))
+      fillBody(a, typ, result.ast[bodyPos], d, src)
+
+
+proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym, info: TLineInfo): PSym =
+  assert(typ.kind == tyObject)
+  result = symPrototype(g, field.typ, attachedDestructor, info)
+  var a = TLiftCtx(info: info, g: g, kind: attachedDestructor, asgnForType: typ)
+  a.fn = result
+  a.asgnForType = typ
+  a.filterDiscriminator = field
+  a.addMemReset = true
+  let discrimantDest = result.typ.n[1].sym
+
+  let dst = newSym(skVar, getIdent(g.cache, "dest"), result, info)
+  dst.typ = makePtrType(typ.owner, typ)
+  let dstSym = newSymNode(dst)
+  let d = newDeref(dstSym)
+  let v = newNodeI(nkVarSection, info)
+  v.addVar(dstSym, genContainerOf(a, typ, field, discrimantDest))
+  result.ast[bodyPos].add v
+  let placeHolder = newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
+  fillBody(a, typ, result.ast[bodyPos], d, placeHolder)
 
-  var n = newNodeI(nkProcDef, info, bodyPos+1)
-  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
-  n[namePos] = newSymNode(result)
-  n[paramsPos] = result.typ.n
-  n[bodyPos] = body
-  result.ast = n
-  incl result.flags, sfFromGeneric
-  incl result.flags, sfGeneratedOp
 
 template liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
   discard "now a nop"

compiler/lineinfos.nim

@@ -42,7 +42,9 @@ type
     warnProveInit, warnProveField, warnProveIndex,
     warnStaticIndexCheck, warnGcUnsafe, warnGcUnsafe2,
     warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed,
-    warnInconsistentSpacing, warnCaseTransition, warnCycleCreated, warnUser,
+    warnInconsistentSpacing, warnCaseTransition, warnCycleCreated,
+    warnObservableStores,
+    warnUser,
     hintSuccess, hintSuccessX, hintCC,
     hintLineTooLong, hintXDeclaredButNotUsed,
     hintConvToBaseNotNeeded,
@@ -87,7 +89,8 @@ const
     warnWriteToForeignHeap: "write to foreign heap",
     warnUnsafeCode: "unsafe code: '$1'",
     warnUnusedImportX: "imported and not used: '$1'",
-    warnInheritFromException: "inherit from a more precise exception type like ValueError, IOError or OSError",
+    warnInheritFromException: "inherit from a more precise exception type like ValueError, " &
+      "IOError or OSError. If these don't suit, inherit from CatchableError or Defect.",
     warnEachIdentIsTuple: "each identifier is a tuple",
     warnUnsafeSetLen: "setLen can potentially expand the sequence, " &
                       "but the element type '$1' doesn't have a valid default value",
@@ -106,6 +109,7 @@ const
     warnInconsistentSpacing: "Number of spaces around '$#' is not consistent",
     warnCaseTransition: "Potential object case transition, instantiate new object instead",
     warnCycleCreated: "$1",
+    warnObservableStores: "observable stores to '$1'",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
@@ -155,7 +159,8 @@ const
     "ProveInit", "ProveField", "ProveIndex",
     "IndexCheck", "GcUnsafe", "GcUnsafe2", "Uninit",
     "GcMem", "Destructor", "LockLevel", "ResultShadowed",
-    "Spacing", "CaseTransition", "CycleCreated", "User"]
+    "Spacing", "CaseTransition", "CycleCreated",
+    "ObservableStores", "User"]
 
   HintsToStr* = [
     "Success", "SuccessX", "CC", "LineTooLong",

compiler/magicsys.nim

@@ -177,3 +177,28 @@ proc getNimScriptSymbol*(g: ModuleGraph; name: string): PSym =
   strTableGet(g.exposed, getIdent(g.cache, name))
 
 proc resetNimScriptSymbols*(g: ModuleGraph) = initStrTable(g.exposed)
+
+proc getMagicEqSymForType*(g: ModuleGraph; t: PType; info: TLineInfo): PSym =
+  case t.kind
+  of tyInt,  tyInt8, tyInt16, tyInt32, tyInt64,
+     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64: 
+    result = getSysMagic(g, info, "==", mEqI)
+  of tyEnum: 
+    result = getSysMagic(g, info, "==", mEqEnum)
+  of tyBool: 
+    result = getSysMagic(g, info, "==", mEqB)
+  of tyRef, tyPtr, tyPointer: 
+    result = getSysMagic(g, info, "==", mEqRef)
+  of tyString:
+    result = getSysMagic(g, info, "==", mEqStr)
+  of tyChar:
+    result = getSysMagic(g, info, "==", mEqCh)
+  of tySet:
+    result = getSysMagic(g, info, "==", mEqSet)
+  of tyProc:
+    result = getSysMagic(g, info, "==", mEqProc)
+  else:
+    globalError(g.config, info,
+      "can't find magic equals operator for type kind " & $t.kind)
+
+

compiler/main.nim

@@ -13,7 +13,7 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  llstream, strutils, ast, lexer, syntaxes, options, msgs,
+  llstream, strutils, os, ast, lexer, syntaxes, options, msgs,
   condsyms, times,
   sem, idents, passes, extccomp,
   cgen, json, nversion,
@@ -89,6 +89,9 @@ proc commandCompileToC(graph: ModuleGraph) =
       graph.config.notes = graph.config.mainPackageNotes
       return
 
+  if not extccomp.ccHasSaneOverflow(conf):
+    conf.symbols.defineSymbol("nimEmulateOverflowChecks")
+
   compileProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933
@@ -315,6 +318,7 @@ proc mainCommand*(graph: ModuleGraph) =
 
       var dumpdata = %[
         (key: "version", val: %VersionAsString),
+        (key: "nimExe", val: %(getAppFilename())),
         (key: "prefixdir", val: %conf.getPrefixDir().string),
         (key: "libpath", val: %conf.libpath.string),
         (key: "project_path", val: %conf.projectFull.string),

compiler/msgs.nim

@@ -379,7 +379,7 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
     if context.info != lastinfo and context.info != info:
       if conf.structuredErrorHook != nil:
         conf.structuredErrorHook(conf, context.info, instantiationFrom,
-                                 Severity.Error)
+                                 Severity.Hint)
       else:
         let message = if context.detail == "":
           instantiationFrom

compiler/parampatterns.nim

@@ -176,6 +176,7 @@ type
     arLocalLValue,            # is an l-value, but local var; must not escape
                               # its stack frame!
     arDiscriminant,           # is a discriminant
+    arLentValue,              # lent value
     arStrange                 # it is a strange beast like 'typedesc[var T]'
 
 proc exprRoot*(n: PNode): PSym =
@@ -260,9 +261,14 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
       # types that are equal modulo distinction preserve l-value:
       result = isAssignable(owner, n[1], isUnsafeAddr)
   of nkHiddenDeref:
-    if isUnsafeAddr and n[0].typ.kind == tyLent: result = arLValue
-    elif n[0].typ.kind == tyLent: result = arDiscriminant
-    else: result = arLValue
+    let n0 = n[0]
+    if n0.typ.kind == tyLent:
+      if isUnsafeAddr or (n0.kind == nkSym and n0.sym.kind == skResult):
+        result = arLValue
+      else:
+        result = arLentValue
+    else:
+      result = arLValue
   of nkDerefExpr, nkHiddenAddr:
     result = arLValue
   of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:

compiler/pragmas.nim

@@ -910,7 +910,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wNoreturn:
         noVal(c, it)
         # Disable the 'noreturn' annotation when in the "Quirky Exceptions" mode!
-        if c.config.exc notin {excQuirky, excGoto}:
+        if c.config.exc != excQuirky:
           incl(sym.flags, sfNoReturn)
         if sym.typ[0] != nil:
           localError(c.config, sym.ast[paramsPos][0].info,

compiler/semcall.nim

@@ -219,7 +219,11 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
       candidates.add("  first type mismatch at position: " & $err.firstMismatch.arg)
       # candidates.add "\n  reason: " & $err.firstMismatch.kind # for debugging
       case err.firstMismatch.kind
-      of kUnknownNamedParam: candidates.add("\n  unknown named parameter: " & $nArg[0])
+      of kUnknownNamedParam:
+        if nArg == nil:
+          candidates.add("\n  unknown named parameter")
+        else:
+          candidates.add("\n  unknown named parameter: " & $nArg[0])
       of kAlreadyGiven: candidates.add("\n  named param already provided: " & $nArg[0])
       of kPositionalAlreadyGiven: candidates.add("\n  positional param was already given as named param")
       of kExtraArg: candidates.add("\n  extra argument given")
@@ -318,7 +322,7 @@ proc getMsgDiagnostic(c: PContext, flags: TExprFlags, n, f: PNode): string =
       sym = nextOverloadIter(o, c, f)
 
   let ident = considerQuotedIdent(c, f, n).s
-  if nfDotField in n.flags and nfExplicitCall notin n.flags:
+  if {nfDotField, nfExplicitCall} * n.flags == {nfDotField}:
     let sym = n[1].typ.sym
     var typeHint = ""
     if sym == nil:

compiler/semdata.nim

@@ -274,10 +274,13 @@ proc addToLib*(lib: PLib, sym: PSym) =
 proc newTypeS*(kind: TTypeKind, c: PContext): PType =
   result = newType(kind, getCurrOwner(c))
 
-proc makePtrType*(c: PContext, baseType: PType): PType =
-  result = newTypeS(tyPtr, c)
+proc makePtrType*(owner: PSym, baseType: PType): PType =
+  result = newType(tyPtr, owner)
   addSonSkipIntLit(result, baseType)
 
+proc makePtrType*(c: PContext, baseType: PType): PType =
+  makePtrType(getCurrOwner(c), baseType)
+
 proc makeTypeWithModifier*(c: PContext,
                            modifier: TTypeKind,
                            baseType: PType): PType =

compiler/semexprs.nim

@@ -1703,7 +1703,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
   if le == nil:
     localError(c.config, a.info, "expression has no type")
   elif (skipTypes(le, {tyGenericInst, tyAlias, tySink}).kind != tyVar and
-        isAssignable(c, a) == arNone) or
+        isAssignable(c, a) in {arNone, arLentValue}) or
       skipTypes(le, abstractVar).kind in {tyOpenArray, tyVarargs}:
     # Direct assignment to a discriminant is allowed!
     localError(c.config, a.info, errXCannotBeAssignedTo %
@@ -2415,7 +2415,7 @@ proc checkPar(c: PContext; n: PNode): TParKind =
     for i in 0..<n.len:
       if result == paTupleFields:
         if (n[i].kind != nkExprColonExpr) or
-            n[i][0].kind notin {nkSym, nkIdent}:
+            n[i][0].kind notin {nkSym, nkIdent, nkAccQuoted}:
           localError(c.config, n[i].info, errNamedExprExpected)
           return paNone
       else:

compiler/semfold.nim

@@ -716,7 +716,11 @@ proc getConstExpr(m: PSym, n: PNode; g: ModuleGraph): PNode =
   of nkBracketExpr: result = foldArrayAccess(m, n, g)
   of nkDotExpr: result = foldFieldAccess(m, n, g)
   of nkStmtListExpr:
-    if n.len == 2 and n[0].kind == nkComesFrom:
-      result = getConstExpr(m, n[1], g)
+    var i = 0
+    while i <= n.len - 2:
+      if n[i].kind in {nkComesFrom, nkCommentStmt, nkEmpty}: i.inc
+      else: break
+    if i == n.len - 1:
+      result = getConstExpr(m, n[i], g)
   else:
     discard

compiler/semmagic.nim

@@ -512,12 +512,14 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     if n[^1].kind == nkSym and n[^1].sym.kind notin {skProc, skFunc}:
       localError(c.config, n.info, "finalizer must be a direct reference to a proc")
     elif optTinyRtti in c.config.globalOptions:
+      let fin = if n[^1].kind == nkLambda: n[^1][namePos].sym
+                else: n[^1].sym
       # check if we converted this finalizer into a destructor already:
-      let t = whereToBindTypeHook(c, n[^1].sym.typ[1].skipTypes(abstractInst+{tyRef}))
-      if t != nil and t.attachedOps[attachedDestructor] != nil and t.attachedOps[attachedDestructor].owner == n[^1].sym:
+      let t = whereToBindTypeHook(c, fin.typ[1].skipTypes(abstractInst+{tyRef}))
+      if t != nil and t.attachedOps[attachedDestructor] != nil and t.attachedOps[attachedDestructor].owner == fin:
         discard "already turned this one into a finalizer"
       else:
-        bindTypeHook(c, turnFinalizerIntoDestructor(c, n[^1].sym, n.info), n, attachedDestructor)
+        bindTypeHook(c, turnFinalizerIntoDestructor(c, fin, n.info), n, attachedDestructor)
     result = n
   of mDestroy:
     result = n
@@ -543,6 +545,10 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     let constructed = result[1].typ.base
     if constructed.requiresInit:
       message(c.config, n.info, warnUnsafeDefault, typeToString(constructed))
+  of mPred:
+    if n[1].typ.skipTypes(abstractInst).kind in {tyUInt..tyUInt64}:
+      n[0].sym.magic = mSubU
+    result = n
   else:
     result = n
 

compiler/sempass2.nim

@@ -66,7 +66,7 @@ type
   TEffects = object
     exc: PNode  # stack of exceptions
     tags: PNode # list of tags
-    bottom, inTryStmt: int
+    bottom, inTryStmt, inExceptOrFinallyStmt: int
     owner: PSym
     ownerModule: PSym
     init: seq[int] # list of initialized variables
@@ -248,6 +248,8 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool; conf: ConfigRef) =
 
 proc useVar(a: PEffects, n: PNode) =
   let s = n.sym
+  if a.inExceptOrFinallyStmt > 0:
+    incl s.flags, sfUsedInFinallyOrExcept
   if isLocalVar(a, s):
     if sfNoInit in s.flags:
       # If the variable is explicitly marked as .noinit. do not emit any error
@@ -379,6 +381,7 @@ proc trackTryStmt(tracked: PEffects, n: PNode) =
 
   var branches = 1
   var hasFinally = false
+  inc tracked.inExceptOrFinallyStmt
 
   # Collect the exceptions caught by the except branches
   for i in 1..<n.len:
@@ -411,6 +414,7 @@ proc trackTryStmt(tracked: PEffects, n: PNode) =
       hasFinally = true
 
   tracked.bottom = oldBottom
+  dec tracked.inExceptOrFinallyStmt
   if not hasFinally:
     setLen(tracked.init, oldState)
   for id, count in items(inter):
@@ -780,6 +784,8 @@ proc track(tracked: PEffects, n: PNode) =
     useVar(tracked, n)
     if n.sym.typ != nil and tfHasAsgn in n.sym.typ.flags:
       tracked.owner.flags.incl sfInjectDestructors
+      # bug #15038: ensure consistency
+      if not hasDestructor(n.typ): n.typ = n.sym.typ
   of nkRaiseStmt:
     if n[0].kind != nkEmpty:
       n[0].info = n.info
@@ -893,7 +899,8 @@ proc track(tracked: PEffects, n: PNode) =
     when false: cstringCheck(tracked, n)
     if tracked.owner.kind != skMacro:
       createTypeBoundOps(tracked, n[0].typ, n.info)
-    checkForSink(tracked.config, tracked.owner, n[1])
+    if n[0].kind != nkSym or not isLocalVar(tracked, n[0].sym):
+      checkForSink(tracked.config, tracked.owner, n[1])
   of nkVarSection, nkLetSection:
     for child in n:
       let last = lastSon(child)

compiler/semstmts.nim

@@ -1472,9 +1472,14 @@ proc semProcAnnotation(c: PContext, prc: PNode;
     if whichPragma(it) != wInvalid:
       # Not a custom pragma
       continue
-    elif strTableGet(c.userPragmas, considerQuotedIdent(c, key)) != nil:
-      # User-defined pragma
-      continue
+    else:
+      let ident = considerQuotedIdent(c, key)
+      if strTableGet(c.userPragmas, ident) != nil:
+        continue # User defined pragma
+      else: 
+        let sym = searchInScopes(c, ident)
+        if sym != nil and sfCustomPragma in sym.flags: 
+          continue # User custom pragma
 
     # we transform ``proc p {.m, rest.}`` into ``m(do: proc p {.rest.})`` and
     # let the semantic checker deal with it:

compiler/semtypes.nim

@@ -660,7 +660,7 @@ proc semRecordCase(c: PContext, n: PNode, check: var IntSet, pos: var int,
   case typ.kind
   of shouldChckCovered:
     chckCovered = true
-  of tyFloat..tyFloat128, tyString, tyError:
+  of tyFloat..tyFloat128, tyError:
     discard
   of tyRange:
     if skipTypes(typ[0], abstractInst).kind in shouldChckCovered:
@@ -668,7 +668,7 @@ proc semRecordCase(c: PContext, n: PNode, check: var IntSet, pos: var int,
   of tyForward:
     errorUndeclaredIdentifier(c, n[0].info, typ.sym.name.s)
   elif not isOrdinalType(typ):
-    localError(c.config, n[0].info, "selector must be of an ordinal type, float or string")
+    localError(c.config, n[0].info, "selector must be of an ordinal type, float")
   if firstOrd(c.config, typ) != 0:
     localError(c.config, n.info, "low(" & $a[0].sym.name.s &
                                      ") must be 0 for discriminant")
@@ -1576,28 +1576,34 @@ proc applyTypeSectionPragmas(c: PContext; pragmas, operand: PNode): PNode =
 
     if p.kind == nkEmpty or whichPragma(p) != wInvalid:
       discard "builtin pragma"
-    elif strTableGet(c.userPragmas, considerQuotedIdent(c, key)) != nil:
-      discard "User-defined pragma"
     else:
-      # we transform ``(arg1, arg2: T) {.m, rest.}`` into ``m((arg1, arg2: T) {.rest.})`` and
-      # let the semantic checker deal with it:
-      var x = newNodeI(nkCall, key.info)
-      x.add(key)
-      if p.kind in nkPragmaCallKinds and p.len > 1:
-        # pass pragma arguments to the macro too:
-        for i in 1 ..< p.len:
-          x.add(p[i])
-      # Also pass the node the pragma has been applied to
-      x.add(operand.copyTreeWithoutNode(p))
-      # recursion assures that this works for multiple macro annotations too:
-      var r = semOverloadedCall(c, x, x, {skMacro, skTemplate}, {efNoUndeclared})
-      if r != nil:
-        doAssert r[0].kind == nkSym
-        let m = r[0].sym
-        case m.kind
-        of skMacro: return semMacroExpr(c, r, r, m, {efNoSemCheck})
-        of skTemplate: return semTemplateExpr(c, r, m, {efNoSemCheck})
-        else: doAssert(false, "cannot happen")
+      let ident = considerQuotedIdent(c, key)
+      if strTableGet(c.userPragmas, ident) != nil:
+        discard "User-defined pragma"
+      else:
+        let sym = searchInScopes(c, ident)
+        if sym != nil and sfCustomPragma in sym.flags: 
+          discard "Custom user pragma"
+        else:
+          # we transform ``(arg1, arg2: T) {.m, rest.}`` into ``m((arg1, arg2: T) {.rest.})`` and
+          # let the semantic checker deal with it:
+          var x = newNodeI(nkCall, key.info)
+          x.add(key)
+          if p.kind in nkPragmaCallKinds and p.len > 1:
+            # pass pragma arguments to the macro too:
+            for i in 1 ..< p.len:
+              x.add(p[i])
+          # Also pass the node the pragma has been applied to
+          x.add(operand.copyTreeWithoutNode(p))
+          # recursion assures that this works for multiple macro annotations too:
+          var r = semOverloadedCall(c, x, x, {skMacro, skTemplate}, {efNoUndeclared})
+          if r != nil:
+            doAssert r[0].kind == nkSym
+            let m = r[0].sym
+            case m.kind
+            of skMacro: return semMacroExpr(c, r, r, m, {efNoSemCheck})
+            of skTemplate: return semTemplateExpr(c, r, m, {efNoSemCheck})
+            else: doAssert(false, "cannot happen")
 
 proc semProcTypeWithScope(c: PContext, n: PNode,
                           prev: PType, kind: TSymKind): PType =

compiler/sigmatch.nim

@@ -548,7 +548,8 @@ proc allowsNilDeprecated(c: TCandidate, f: PType): TTypeRelation =
     result = isNone
 
 proc inconsistentVarTypes(f, a: PType): bool {.inline.} =
-  result = f.kind != a.kind and (f.kind in {tyVar, tyLent} or a.kind in {tyVar, tyLent})
+  result = f.kind != a.kind and
+    (f.kind in {tyVar, tyLent, tySink} or a.kind in {tyVar, tyLent, tySink})
 
 proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
   ## For example we have:

compiler/sinkparameter_inference.nim

@@ -48,8 +48,7 @@ proc checkForSink*(config: ConfigRef; owner: PSym; arg: PNode) =
         # we only report every potential 'sink' parameter only once:
         incl arg.sym.flags, sfWasForwarded
         message(config, arg.info, hintPerformance,
-          ("could not turn '$1' to a sink parameter " &
-          "because '$2' was forward declared") % [arg.sym.name.s, owner.name.s])
+          "could not turn '$1' to a sink parameter" % [arg.sym.name.s])
       #echo config $ arg.info, " candidate for a sink parameter here"
   of nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr:
     if not isEmptyType(arg.typ):

compiler/vm.nim

@@ -510,6 +510,23 @@ const
     "if you are sure this is not a bug in your code, compile with `--maxLoopIterationsVM:number` (current value: $1)"
   errFieldXNotFound = "node lacks field: "
 
+
+template maybeHandlePtr(node2: PNode, reg: TFullReg, isAssign2: bool): bool =
+  let node = node2 # prevent double evaluation
+  if node.kind == nkNilLit:
+    stackTrace(c, tos, pc, errNilAccess)
+  let typ = node.typ
+  if nfIsPtr in node.flags or (typ != nil and typ.kind == tyPtr):
+    assert node.kind == nkIntLit, $(node.kind)
+    assert typ != nil
+    let typ2 = if typ.kind == tyPtr: typ[0] else: typ
+    if not derefPtrToReg(node.intVal, typ2, reg, isAssign = isAssign2):
+      # tyObject not supported in this context
+      stackTrace(c, tos, pc, "deref unsupported ptr type: " & $(typeToString(typ), typ.kind))
+    true
+  else:
+    false
+
 proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
   var pc = start
   var tos = tos
@@ -755,26 +772,12 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         ensureKind(regs[rb].regAddr.kind)
         regs[ra] = regs[rb].regAddr[]
       of rkNode:
-        if regs[rb].node.kind == nkNilLit:
-          stackTrace(c, tos, pc, errNilAccess)
         if regs[rb].node.kind == nkRefTy:
           regs[ra].node = regs[rb].node[0]
-        else:
-          let node = regs[rb].node
-          let typ = node.typ
-          # see also `nfIsPtr`
-          if node.kind == nkIntLit:
-            var typ2 = typ
-            doAssert typ != nil
-            if typ.kind == tyPtr:
-              typ2 = typ2[0]
-            if not derefPtrToReg(node.intVal, typ2, regs[ra], isAssign = false):
-              # tyObject not supported in this context
-              stackTrace(c, tos, pc, "opcLdDeref unsupported ptr type: " & $(typeToString(typ), typ.kind))
-          else:
-            ## eg: typ.kind = tyObject
-            ensureKind(rkNode)
-            regs[ra].node = regs[rb].node
+        elif not maybeHandlePtr(regs[rb].node, regs[ra], false):
+          ## eg: typ.kind = tyObject
+          ensureKind(rkNode)
+          regs[ra].node = regs[rb].node
       else:
         stackTrace(c, tos, pc, errNilAccess & " kind: " & $regs[rb].kind)
     of opcWrDeref:
@@ -795,18 +798,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
           regs[ra].nodeAddr[] = n
       of rkRegisterAddr: regs[ra].regAddr[] = regs[rc]
       of rkNode:
-        if regs[ra].node.kind == nkNilLit:
-          stackTrace(c, tos, pc, errNilAccess)
-        let node = regs[ra].node
-        let typ = node.typ
-        if nfIsPtr in node.flags or (typ != nil and typ.kind == tyPtr):
-          assert node.kind == nkIntLit, $(node.kind)
-          var typ2 = typ
-          if typ.kind == tyPtr:
-            typ2 = typ2[0]
-          if not derefPtrToReg(node.intVal, typ2, regs[rc], isAssign = true):
-            stackTrace(c, tos, pc, "opcWrDeref unsupported ptr type: " & $(typeToString(typ), typ.kind))
-        else:
+         # xxx: also check for nkRefTy as in opcLdDeref?
+        if not maybeHandlePtr(regs[ra].node, regs[rc], true):
           regs[ra].node[] = regs[rc].regToNode[]
           regs[ra].node.flags.incl nfIsRef
       else: stackTrace(c, tos, pc, errNilAccess)

compiler/vmops.nim

@@ -110,7 +110,7 @@ when defined(nimHasInvariant):
     of arguments: result = conf.arguments
     of outFile: result = conf.outFile.string
     of outDir: result = conf.outDir.string
-    of nimcacheDir: result = conf.nimcacheDir.string
+    of nimcacheDir: result = conf.getNimcacheDir().string
     of projectName: result = conf.projectName
     of projectPath: result = conf.projectPath.string
     of projectFull: result = conf.projectFull.string

doc/manual.rst

@@ -148,7 +148,7 @@ It is mostly accurate to picture semantic analysis proceeding top to bottom and
 left to right in the source code, with compile-time code execution interleaved
 when necessary to compute values that are required for subsequent semantic
 analysis. We will see much later in this document that macro invocation not only
-requires this interleaving, but also creates a situation where semantic analyis
+requires this interleaving, but also creates a situation where semantic analysis
 does not entirely proceed top to bottom and left to right.
 
 
@@ -792,7 +792,7 @@ features supported at compile time (as detailed in the next section below).
 Within such a code block, it is possible to declare variables and then later
 read and update them, or declare variables and pass them to procedures that
 modify them. However, the code in such a block must still adhere to the
-retrictions listed above for referencing values and operations outside the
+restrictions listed above for referencing values and operations outside the
 block.
 
 The ability to access and modify compile-time variables adds flexibility to
@@ -1296,7 +1296,7 @@ A parameter ``A`` may be an *open array*, in which case it is indexed by
 integers from 0 to ``len(A)-1``. An array expression may be constructed by the
 array constructor ``[]``. The element type of this array expression is
 inferred from the type of the first element. All other elements need to be
-implicitly convertable to this type.
+implicitly convertible to this type.
 
 Sequences are similar to arrays but of dynamic length which may change
 during runtime (like strings). Sequences are implemented as growable arrays,
@@ -1802,7 +1802,7 @@ Mixing GC'ed memory with ``ptr``
 
 Special care has to be taken if an untraced object contains traced objects like
 traced references, strings or sequences: in order to free everything properly,
-the built-in procedure ``GCunref`` has to be called before freeing the untraced
+the built-in procedure ``reset`` has to be called before freeing the untraced
 memory manually:
 
 .. code-block:: nim
@@ -1816,12 +1816,12 @@ memory manually:
   d.s = "abc"
 
   # tell the GC that the string is not needed anymore:
-  GCunref(d.s)
+  reset(d.s)
 
   # free the memory:
   dealloc(d)
 
-Without the ``GCunref`` call the memory allocated for the ``d.s`` string would
+Without the ``reset`` call the memory allocated for the ``d.s`` string would
 never be freed. The example also demonstrates two important features for low
 level programming: the ``sizeof`` proc returns the size of a type or value
 in bytes. The ``cast`` operator can circumvent the type system: the compiler
@@ -2362,7 +2362,7 @@ The convertible relation can be relaxed by a user-defined type
   # implicit conversion magic happens here
   x = chr
   echo x # => 97
-  # you can use the explicit form too
+  # one can use the explicit form too
   x = chr.toInt
   echo x # => 97
 
@@ -3856,7 +3856,7 @@ as there are components in the tuple. The i'th iteration variable's type is
 the type of the i'th component. In other words, implicit tuple unpacking in a
 for loop context is supported.
 
-Implict items/pairs invocations
+Implicit items/pairs invocations
 -------------------------------
 
 If the for loop expression ``e`` does not denote an iterator and the for loop
@@ -4250,7 +4250,7 @@ Exception hierarchy
 The exception tree is defined in the `system <system.html>`_ module.
 Every exception inherits from ``system.Exception``. Exceptions that indicate
 programming bugs inherit from ``system.Defect`` (which is a subtype of ``Exception``)
-and are stricly speaking not catchable as they can also be mapped to an operation
+and are strictly speaking not catchable as they can also be mapped to an operation
 that terminates the whole process. If panics are turned into exceptions, these
 exceptions inherit from `Defect`.
 
@@ -4539,8 +4539,8 @@ more complex type classes:
 
 Whilst the syntax of type classes appears to resemble that of ADTs/algebraic data
 types in ML-like languages, it should be understood that type classes are static
-constraints to be enforced at type instantations. Type classes are not really
-types in themsleves, but are instead a system of providing generic "checks" that
+constraints to be enforced at type instantiations. Type classes are not really
+types in themselves, but are instead a system of providing generic "checks" that
 ultimately *resolve* to some singular type. Type classes do not allow for
 runtime type dynamism, unlike object variants or methods.
 
@@ -4702,7 +4702,7 @@ instantiation. The following is not allowed:
   # not valid: 'T' is not inferred to be of type 'var int'
   g(v, i)
 
-  # also not allowed: explict instantiation via 'var int'
+  # also not allowed: explicit instantiation via 'var int'
   g[var int](v, i)
 
 
@@ -4858,7 +4858,7 @@ compiler. Explicit immediate templates are now deprecated.
 Passing a code block to a template
 ----------------------------------
 
-You can pass a block of statements as the last argument to a template
+One can pass a block of statements as the last argument to a template
 following the special ``:`` syntax:
 
 .. code-block:: nim
@@ -5192,7 +5192,7 @@ tree (AST) of the code that is passed to it. The macro can then do
 transformations on it and return the transformed AST. This can be used to
 add custom language features and implement `domain specific languages`:idx:.
 
-Macro invocation is a case where semantic analyis does **not** entirely proceed
+Macro invocation is a case where semantic analysis does **not** entirely proceed
 top to bottom and left to right. Instead, semantic analysis happens at least
 twice:
 
@@ -5444,7 +5444,7 @@ generic type ``static[T]``. The type param can be omitted to obtain the type
 class of all constant expressions. A more specific type class can be created by
 instantiating ``static`` with another type class.
 
-You can force an expression to be evaluated at compile time as a constant
+One can force an expression to be evaluated at compile time as a constant
 expression by coercing it to a corresponding ``static`` type:
 
 .. code-block:: nim
@@ -5458,14 +5458,14 @@ possible type mismatch error.
 typedesc[T]
 -----------
 
-In many contexts, Nim allows you to treat the names of types as regular
+In many contexts, Nim allows to treat the names of types as regular
 values. These values exists only during the compilation phase, but since
 all values must have a type, ``typedesc`` is considered their special type.
 
 ``typedesc`` acts like a generic type. For instance, the type of the symbol
 ``int`` is ``typedesc[int]``. Just like with regular generic types, when the
 generic param is omitted, ``typedesc`` denotes the type class of all types.
-As a syntactic convenience, you can also use ``typedesc`` as a modifier.
+As a syntactic convenience, one can also use ``typedesc`` as a modifier.
 
 Procs featuring ``typedesc`` params are considered implicitly generic.
 They will be instantiated for each unique combination of supplied types
@@ -5543,7 +5543,7 @@ typeof operator
 **Note**: ``typeof(x)`` can for historical reasons also be written as
 ``type(x)`` but ``type(x)`` is discouraged.
 
-You can obtain the type of a given expression by constructing a ``typeof``
+One can obtain the type of a given expression by constructing a ``typeof``
 value from it (in many other languages this is known as the `typeof`:idx:
 operator):
 
@@ -5770,9 +5770,9 @@ modules don't need to import a module's dependencies:
   echo $x
 
 When the exported symbol is another module, all of its definitions will
-be forwarded. You can use an ``except`` list to exclude some of the symbols.
+be forwarded. One can use an ``except`` list to exclude some of the symbols.
 
-Notice that when exporting, you need to specify only the module name:
+Notice that when exporting, one needs to specify only the module name:
 
 .. code-block:: nim
   import foo/bar/baz
@@ -6576,7 +6576,7 @@ and instead the generated code should contain an ``#include``:
     PFile {.importc: "FILE*", header: "<stdio.h>".} = distinct pointer
       # import C's FILE* type; Nim will treat it as a new pointer type
 
-The ``header`` pragma always expects a string constant. The string contant
+The ``header`` pragma always expects a string constant. The string constant
 contains the header file: As usual for C, a system header file is enclosed
 in angle brackets: ``<>``. If no angle brackets are given, Nim
 encloses the header file in ``""`` in the generated C code.
@@ -6604,7 +6604,7 @@ with the project:
   {.compile: "myfile.cpp".}
 
 **Note**: Nim computes a SHA1 checksum and only recompiles the file if it
-has changed. You can use the ``-f`` command line option to force recompilation
+has changed. One can use the ``-f`` command line option to force recompilation
 of the file.
 
 
@@ -6619,12 +6619,12 @@ The ``link`` pragma can be used to link an additional file with the project:
 PassC pragma
 ------------
 The ``passc`` pragma can be used to pass additional parameters to the C
-compiler like you would using the commandline switch ``--passc``:
+compiler like one would using the commandline switch ``--passc``:
 
 .. code-block:: Nim
   {.passc: "-Wall -Werror".}
 
-Note that you can use ``gorge`` from the `system module <system.html>`_ to
+Note that one can use ``gorge`` from the `system module <system.html>`_ to
 embed parameters from an external command that will be executed
 during semantic analysis:
 
@@ -6647,12 +6647,12 @@ the pragma resides in:
 PassL pragma
 ------------
 The ``passL`` pragma can be used to pass additional parameters to the linker
-like you would using the commandline switch ``--passL``:
+like one would using the commandline switch ``--passL``:
 
 .. code-block:: Nim
   {.passL: "-lSDLmain -lSDL".}
 
-Note that you can use ``gorge`` from the `system module <system.html>`_ to
+Note that one can use ``gorge`` from the `system module <system.html>`_ to
 embed parameters from an external command that will be executed
 during semantic analysis:
 
@@ -6907,7 +6907,7 @@ Importcpp for objects
 ~~~~~~~~~~~~~~~~~~~~~
 
 Generic ``importcpp``'ed objects are mapped to C++ templates. This means that
-you can import C++'s templates rather easily without the need for a pattern
+one can import C++'s templates rather easily without the need for a pattern
 language for object types:
 
 .. code-block:: nim
@@ -7156,9 +7156,9 @@ specified. It is possible to annotate procs, templates, type and variable
 definitions, statements, etc.
 
 Macros module includes helpers which can be used to simplify custom pragma
-access `hasCustomPragma`, `getCustomPragmaVal`. Please consult macros module
-documentation for details. These macros are no magic, they don't do anything
-you cannot do yourself by walking AST object representation.
+access `hasCustomPragma`, `getCustomPragmaVal`. Please consult the macros module
+documentation for details. These macros are not magic, everything they do can
+also be achieved by walking the AST of the object representation.
 
 More examples with custom pragmas:
 

koch.nim

@@ -10,7 +10,7 @@
 #
 
 const
-  NimbleStableCommit = "4007b2a778429a978e12307bf13a038029b4c4d9" # master
+  NimbleStableCommit = "63695f490728e3935692c29f3d71944d83bb1e83" # master
 
 when not defined(windows):
   const

lib/impure/db_odbc.nim

@@ -242,7 +242,7 @@ proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool
   try:
     db.prepareFetchDirect(query, args)
     var
-      rCnt = -1
+      rCnt:TSqlLen = -1
     res = SQLRowCount(db.stmt, rCnt)
     properFreeResult(SQL_HANDLE_STMT, db.stmt)
     if res != SQL_SUCCESS: dbError(db)
@@ -461,10 +461,10 @@ proc execAffectedRows*(db: var DbConn, query: SqlQuery,
   var q = dbFormat(query, args)
   db.sqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt))
   rawExec(db, query, args)
-  var rCnt = -1
+  var rCnt:TSqlLen = -1
   db.sqlCheck(SQLRowCount(db.hDb, rCnt))
   properFreeResult(SQL_HANDLE_STMT, db.stmt)
-  result = rCnt
+  result = rCnt.int64
 
 proc close*(db: var DbConn) {.
       tags: [WriteDbEffect], raises: [].} =
@@ -489,7 +489,7 @@ proc open*(connection, user, password, database: string): DbConn {.
   ## Currently the database parameter is ignored,
   ## but included to match ``open()`` in the other db_xxxxx library modules.
   var
-    val: TSqlInteger = SQL_OV_ODBC3
+    val = SQL_OV_ODBC3
     resLen = 0
   result = (hDb: nil, env: nil, stmt: nil)
   # allocate environment handle
@@ -497,7 +497,7 @@ proc open*(connection, user, password, database: string): DbConn {.
   if res != SQL_SUCCESS: dbError("Error: unable to initialise ODBC environment.")
   res = SQLSetEnvAttr(result.env,
                       SQL_ATTR_ODBC_VERSION.TSqlInteger,
-                      val, resLen.TSqlInteger)
+                      cast[SqlPointer](val.addr), resLen.TSqlInteger)
   if res != SQL_SUCCESS: dbError("Error: unable to set ODBC driver version.")
   # allocate hDb handle
   res = SQLAllocHandle(SQL_HANDLE_DBC, result.env, result.hDb)

lib/impure/nre.nim

@@ -6,6 +6,9 @@
 #    distribution, for details about the copyright.
 #
 
+when defined(js):
+  {.error: "This library needs to be compiled with a c-like backend, and depends on PCRE.".}
+
 ## What is NRE?
 ## ============
 ##

lib/impure/re.nim

@@ -7,6 +7,9 @@
 #    distribution, for details about the copyright.
 #
 
+when defined(js):
+  {.error: "This library needs to be compiled with a c-like backend, and depends on PCRE.".}
+
 ## Regular expression support for Nim.
 ##
 ## This module is implemented by providing a wrapper around the
@@ -678,4 +681,3 @@ when isMainModule:
     doAssert replace("bar", re"^", "foo") == "foobar"
     doAssert replace("foo", re"", "-") == "-foo"
     doAssert replace("foo", re"$", "bar") == "foobar"
-

lib/js/jsffi.nim

@@ -22,7 +22,7 @@
 ##  var document {.importc, nodecl.}: JsObject
 ##  var console {.importc, nodecl.}: JsObject
 ##  # import the "$" function
-##  proc jq(selector: JsObject): JsObject {.importcpp: "$(#)".}
+##  proc jq(selector: JsObject): JsObject {.importcpp: "$$(#)".}
 ##
 ##  # Use jQuery to make the following code run, after the document is ready.
 ##  # This uses an experimental ``.()`` operator for ``JsObject``, to emit

lib/posix/posix_other.nim

@@ -32,7 +32,12 @@ type
   SocketHandle* = distinct cint # The type used to represent socket descriptors
 
 type
-  Time* {.importc: "time_t", header: "<time.h>".} = distinct clong
+  Time* {.importc: "time_t", header: "<time.h>".} = distinct (
+    when defined(nimUse64BitCTime):
+      int64
+    else:
+      clong
+  )
 
   Timespec* {.importc: "struct timespec",
                header: "<time.h>", final, pure.} = object ## struct timespec

lib/pure/algorithm.nim

@@ -504,7 +504,7 @@ template sortedByIt*(seq1, op: untyped): untyped =
     # Nested sort
     assert people.sortedByIt((it.age, it.name)) == @[(name: "p2", age: 20),
        (name: "p3", age: 30), (name: "p4", age: 30), (name: "p1", age: 60)]
-  var result = sorted(seq1, proc(x, y: type(seq1[0])): int =
+  var result = sorted(seq1, proc(x, y: typeof(seq1[0])): int =
     var it {.inject.} = x
     let a = op
     it = y

lib/pure/asyncdispatch.nim

@@ -267,7 +267,7 @@ when defined(windows) or defined(nimdoc):
       pcd: PostCallbackDataPtr
     AsyncEvent* = ptr AsyncEventImpl
 
-    Callback = proc (fd: AsyncFD): bool {.closure, gcsafe.}
+    Callback* = proc (fd: AsyncFD): bool {.closure, gcsafe.}
 
   proc hash(x: AsyncFD): Hash {.borrow.}
   proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow.}
@@ -1002,10 +1002,11 @@ when defined(windows) or defined(nimdoc):
   proc addProcess*(pid: int, cb: Callback) =
     ## Registers callback ``cb`` to be called when process with process ID
     ## ``pid`` exited.
+    const NULL = Handle(0)
     let p = getGlobalDispatcher()
     let procFlags = SYNCHRONIZE
     var hProcess = openProcess(procFlags, 0, pid.DWORD)
-    if hProcess == INVALID_HANDLE_VALUE:
+    if hProcess == NULL:
       raiseOSError(osLastError())
 
     var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData)))
@@ -1098,7 +1099,7 @@ else:
                                      # queue.
   type
     AsyncFD* = distinct cint
-    Callback = proc (fd: AsyncFD): bool {.closure, gcsafe.}
+    Callback* = proc (fd: AsyncFD): bool {.closure, gcsafe.}
 
     AsyncData = object
       readList: seq[Callback]
@@ -1210,14 +1211,19 @@ else:
     let newLength = max(len(curList), InitCallbackListSize)
     var newList = newSeqOfCap[Callback](newLength)
 
+    var eventsExtinguished = false
     for cb in curList:
+      if eventsExtinguished:
+        newList.add(cb)
+        continue
       if not cb(fd):
         # Callback wants to be called again.
         newList.add(cb)
         # This callback has returned with EAGAIN, so we don't need to
         # call any other callbacks as they are all waiting for the same event
         # on the same fd.
-        break
+        # We do need to ensure they are called again though.
+        eventsExtinguished = true
 
     withData(selector, fd.int, fdData) do:
       # Descriptor is still present in the queue.

lib/pure/asyncftpclient.nim

@@ -134,7 +134,7 @@ proc expectReply(ftp: AsyncFtpClient): Future[TaintedString] {.async.} =
   var line = await ftp.csock.recvLine()
   result = TaintedString(line)
   var count = 0
-  while line[3] == '-':
+  while line.len > 3 and line[3] == '-':
     ## Multi-line reply.
     line = await ftp.csock.recvLine()
     string(result).add("\n" & line)
@@ -146,7 +146,11 @@ proc send*(ftp: AsyncFtpClient, m: string): Future[TaintedString] {.async.} =
   ## Send a message to the server, and wait for a primary reply.
   ## ``\c\L`` is added for you.
   ##
+  ## You need to make sure that the message ``m`` doesn't contain any newline
+  ## characters. Failing to do so will raise ``AssertionDefect``.
+  ##
   ## **Note:** The server may return multiple lines of coded replies.
+  doAssert(not m.contains({'\c', '\L'}), "message shouldn't contain any newline characters")
   await ftp.csock.send(m & "\c\L")
   return await ftp.expectReply()
 

lib/pure/asynchttpserver.nim

@@ -99,13 +99,15 @@ proc respond*(req: Request, code: HttpCode, content: string,
 
   if headers != nil:
     msg.addHeaders(headers)
-    
-  if not headers.hasKey("Content-Length"):
+
+  # If the headers did not contain a Content-Length use our own
+  if headers.isNil() or not headers.hasKey("Content-Length"):
     msg.add("Content-Length: ")
     # this particular way saves allocations:
     msg.addInt content.len
-  
-  msg.add "\c\L\c\L"
+    msg.add "\c\L"
+
+  msg.add "\c\L"
   msg.add(content)
   result = req.client.send(msg)
 

lib/pure/asyncmacro.nim

@@ -46,7 +46,7 @@ template createCb(retFutureSym, iteratorNameSym,
         else:
           {.gcsafe.}:
             {.push hint[ConvFromXtoItselfNotNeeded]: off.}
-            next.callback = cast[proc() {.closure, gcsafe.}](identName)
+            next.addCallback cast[proc() {.closure, gcsafe.}](identName)
             {.pop.}
     except:
       futureVarCompletions

lib/pure/browsers.nim

@@ -17,24 +17,30 @@ include "system/inclrtl"
 
 when defined(windows):
   import winlean
+  from os import absolutePath
 else:
   import os, osproc
 
 const osOpenCmd* =
-  when defined(macos) or defined(windows): "open" else: "xdg-open" ## \
+  when defined(macos) or defined(macosx) or defined(windows): "open" else: "xdg-open" ## \
   ## Alias for the operating system specific *"open"* command,
-  ## ``"open"`` on MacOS and Windows, ``"xdg-open"`` on Linux, BSD, etc.
+  ## ``"open"`` on OSX, MacOS and Windows, ``"xdg-open"`` on Linux, BSD, etc.
 
+proc prepare(s: string): string =
+  if s.contains("://"):
+    result = s
+  else:
+    result = "file://" & absolutePath(s)
 
-template openDefaultBrowserImpl(url: string) =
+proc openDefaultBrowserImpl(url: string) =
   when defined(windows):
     var o = newWideCString(osOpenCmd)
-    var u = newWideCString(url)
+    var u = newWideCString(prepare url)
     discard shellExecuteW(0'i32, o, u, nil, nil, SW_SHOWNORMAL)
   elif defined(macosx):
-    discard execShellCmd(osOpenCmd & " " & quoteShell(url))
+    discard execShellCmd(osOpenCmd & " " & quoteShell(prepare url))
   else:
-    var u = quoteShell(url)
+    var u = quoteShell(prepare url)
     if execShellCmd(osOpenCmd & " " & u) == 0: return
     for b in getEnv("BROWSER").string.split(PathSep):
       try:

lib/pure/cgi.nim

@@ -32,12 +32,7 @@
 import strutils, os, strtabs, cookies, uri
 export uri.encodeUrl, uri.decodeUrl
 
-proc handleHexChar(c: char, x: var int) {.inline.} =
-  case c
-  of '0'..'9': x = (x shl 4) or (ord(c) - ord('0'))
-  of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10)
-  of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
-  else: assert(false)
+include includes/decode_helpers
 
 proc addXmlChar(dest: var string, c: char) {.inline.} =
   case c
@@ -93,40 +88,27 @@ proc getEncodedData(allowedMethods: set[RequestMethod]): string =
 iterator decodeData*(data: string): tuple[key, value: TaintedString] =
   ## Reads and decodes CGI data and yields the (name, value) pairs the
   ## data consists of.
+  proc parseData(data: string, i: int, field: var string): int =
+    result = i
+    while result < data.len:
+      case data[result]
+      of '%': add(field, decodePercent(data, result))
+      of '+': add(field, ' ')
+      of '=', '&': break
+      else: add(field, data[result])
+      inc(result)
+
   var i = 0
   var name = ""
   var value = ""
   # decode everything in one pass:
   while i < data.len:
     setLen(name, 0) # reuse memory
-    while i < data.len:
-      case data[i]
-      of '%':
-        var x = 0
-        handleHexChar(data[i+1], x)
-        handleHexChar(data[i+2], x)
-        inc(i, 2)
-        add(name, chr(x))
-      of '+': add(name, ' ')
-      of '=', '&': break
-      else: add(name, data[i])
-      inc(i)
+    i = parseData(data, i, name)
     if i >= data.len or data[i] != '=': cgiError("'=' expected")
     inc(i) # skip '='
     setLen(value, 0) # reuse memory
-    while i < data.len:
-      case data[i]
-      of '%':
-        var x = 0
-        if i+2 < data.len:
-          handleHexChar(data[i+1], x)
-          handleHexChar(data[i+2], x)
-        inc(i, 2)
-        add(value, chr(x))
-      of '+': add(value, ' ')
-      of '&', '\0': break
-      else: add(value, data[i])
-      inc(i)
+    i = parseData(data, i, value)
     yield (name.TaintedString, value.TaintedString)
     if i < data.len:
       if data[i] == '&': inc(i)

lib/pure/collections/sharedtables.nim

@@ -140,6 +140,10 @@ proc hasKeyOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): bool =
   withLock t:
     hasKeyOrPutImpl(enlarge)
 
+template tabMakeEmpty(i) = t.data[i].hcode = 0
+template tabCellEmpty(i) = isEmpty(t.data[i].hcode)
+template tabCellHash(i)  = t.data[i].hcode
+
 proc withKey*[A, B](t: var SharedTable[A, B], key: A,
                     mapper: proc(key: A, val: var B, pairExists: var bool)) =
   ## Computes a new mapping for the ``key`` with the specified ``mapper``
@@ -181,7 +185,7 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A,
     if pairExists:
       mapper(t.data[index].key, t.data[index].val, pairExists)
       if not pairExists:
-        delImplIdx(t, index)
+        delImplIdx(t, index, tabMakeEmpty, tabCellEmpty, tabCellHash)
     else:
       var val: B
       mapper(key, val, pairExists)
@@ -202,7 +206,7 @@ proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) =
 proc del*[A, B](t: var SharedTable[A, B], key: A) =
   ## deletes `key` from hash table `t`.
   withLock t:
-    delImpl()
+    delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash)
 
 proc len*[A, B](t: var SharedTable[A, B]): int =
   ## number of elements in `t`

lib/pure/collections/tableimpl.nim

@@ -78,22 +78,52 @@ template hasKeyOrPutImpl(enlarge) {.dirty.} =
     maybeRehashPutImpl(enlarge)
   else: result = true
 
-template delImplIdx(t, i) =
+# delImplIdx is KnuthV3 Algo6.4R adapted to i=i+1 (from i=i-1) which has come to
+# be called "back shift delete".  It shifts elements in the collision cluster of
+# a victim backward to make things as-if the victim were never inserted in the
+# first place.  This is desirable to keep things "ageless" after many deletes.
+# It is trickier than you might guess since initial probe (aka "home") locations
+# of keys in a cluster may collide and since table addresses wrap around.
+#
+# A before-after diagram might look like ('.' means empty):
+#   slot:   0   1   2   3   4   5   6   7
+# before(1)
+#   hash1:  6   7   .   3   .   5   5   6  ; Really hash() and msk
+#   data1:  E   F   .   A   .   B   C   D  ; About to delete C @index 6
+# after(2)
+#   hash2:  7   .   .   3   .   5   6   6  ; Really hash() and msk
+#   data2:  F   .   .   A   .   B   D   E  ; After deletion of C
+#
+# This lowers total search depth over the whole table from 1+1+2+2+2+2=10 to 7.
+# Had the victim been B@5, C would need back shifting to slot 5.  Total depth is
+# always lowered by at least 1, e.g. victim A@3.  This is all quite fast when
+# empty slots are frequent (also needed to keep insert/miss searches fast) and
+# hash() is either fast or avoided (via `.hcode`).  It need not compare keys.
+#
+# delImplIdx realizes the above transformation, but only works for dense Linear
+# Probing, nextTry(h)=h+1.  This is not an important limitation since that's the
+# fastest sequence on any CPU made since the 1980s. { Performance analysis often
+# overweights "key cmp" neglecting cache behavior, giving bad ideas how big/slow
+# tables behave (when perf matters most!).  Comparing hcode first means usually
+# only 1 key cmp is needed for *any* seq.  Timing only predictable activity,
+# small tables, and/or integer keys often perpetuates such bad ideas. }
+
+template delImplIdx(t, i, makeEmpty, cellEmpty, cellHash) =
   let msk = maxHash(t)
   if i >= 0:
     dec(t.counter)
     block outer:
       while true:         # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1
-        var j = i         # The correctness of this depends on (h+1) in nextTry,
+        var j = i         # The correctness of this depends on (h+1) in nextTry
         var r = j         # though may be adaptable to other simple sequences.
-        t.data[i].hcode = 0              # mark current EMPTY
+        makeEmpty(i)                     # mark current EMPTY
         t.data[i].key = default(type(t.data[i].key))
         t.data[i].val = default(type(t.data[i].val))
         while true:
           i = (i + 1) and msk            # increment mod table size
-          if isEmpty(t.data[i].hcode):   # end of collision cluster; So all done
+          if cellEmpty(i):               # end of collision cluster; So all done
             break outer
-          r = t.data[i].hcode and msk    # "home" location of key@i
+          r = cellHash(i) and msk        # initial probe index for key@slot i
           if not ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)):
             break
         when defined(js):
@@ -101,10 +131,19 @@ template delImplIdx(t, i) =
         else:
           t.data[j] = move(t.data[i]) # data[j] will be marked EMPTY next loop
 
-template delImpl() {.dirty.} =
+template delImpl(makeEmpty, cellEmpty, cellHash) {.dirty.} =
   var hc: Hash
   var i = rawGet(t, key, hc)
-  delImplIdx(t, i)
+  delImplIdx(t, i, makeEmpty, cellEmpty, cellHash)
+
+template delImplNoHCode(makeEmpty, cellEmpty, cellHash) {.dirty.} =
+  if t.dataLen > 0:
+    var i: Hash = hash(key) and maxHash(t)
+    while not cellEmpty(i):
+      if t.data[i].key == key:
+        delImplIdx(t, i, makeEmpty, cellEmpty, cellHash)
+        break
+      i = nextTry(i, maxHash(t))
 
 template clearImpl() {.dirty.} =
   for i in 0 ..< t.dataLen:

lib/pure/collections/tables.nim

@@ -499,6 +499,10 @@ proc add*[A, B](t: var Table[A, B], key: A, val: B) =
   ## (key, value) pair in the table without introducing duplicates.
   addImpl(enlarge)
 
+template tabMakeEmpty(i) = t.data[i].hcode = 0
+template tabCellEmpty(i) = isEmpty(t.data[i].hcode)
+template tabCellHash(i)  = t.data[i].hcode
+
 proc del*[A, B](t: var Table[A, B], key: A) =
   ## Deletes ``key`` from hash table ``t``. Does nothing if the key does not exist.
   ##
@@ -512,7 +516,7 @@ proc del*[A, B](t: var Table[A, B], key: A) =
     a.del('z')
     doAssert a == {'b': 9, 'c': 13}.toTable
 
-  delImpl()
+  delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash)
 
 proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool =
   ## Deletes the ``key`` from the table.
@@ -540,7 +544,7 @@ proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool =
   result = index >= 0
   if result:
     val = move(t.data[index].val)
-    delImplIdx(t, index)
+    delImplIdx(t, index, tabMakeEmpty, tabCellEmpty, tabCellHash)
 
 proc take*[A, B](t: var Table[A, B], key: A, val: var B): bool {.inline.} =
   ## Alias for:
@@ -2218,19 +2222,6 @@ proc enlarge[A](t: var CountTable[A]) =
     if t.data[i].val != 0: ctRawInsert(t, n, move t.data[i].key, move t.data[i].val)
   swap(t.data, n)
 
-proc remove[A](t: var CountTable[A], key: A) =
-  var n: seq[tuple[key: A, val: int]]
-  newSeq(n, len(t.data))
-  var removed: bool
-  for i in countup(0, high(t.data)):
-    if t.data[i].val != 0:
-      if t.data[i].key != key:
-        ctRawInsert(t, n, move t.data[i].key, move t.data[i].val)
-      else:
-        removed = true
-  swap(t.data, n)
-  if removed: dec(t.counter)
-
 proc rawGet[A](t: CountTable[A], key: A): int =
   if t.data.len == 0:
     return -1
@@ -2244,7 +2235,7 @@ template ctget(t, key, default: untyped): untyped =
   var index = rawGet(t, key)
   result = if index >= 0: t.data[index].val else: default
 
-proc inc*[A](t: var CountTable[A], key: A, val: Positive = 1)
+proc inc*[A](t: var CountTable[A], key: A, val = 1)
 
 # ----------------------------------------------------------------------
 
@@ -2287,6 +2278,10 @@ proc `[]`*[A](t: CountTable[A], key: A): int =
   assert(not t.isSorted, "CountTable must not be used after sorting")
   ctget(t, key, 0)
 
+template cntMakeEmpty(i) = t.data[i].val = 0
+template cntCellEmpty(i) = t.data[i].val == 0
+template cntCellHash(i)  = hash(t.data[i].key)
+
 proc `[]=`*[A](t: var CountTable[A], key: A, val: int) =
   ## Inserts a ``(key, value)`` pair into ``t``.
   ##
@@ -2297,7 +2292,7 @@ proc `[]=`*[A](t: var CountTable[A], key: A, val: int) =
   assert(not t.isSorted, "CountTable must not be used after sorting")
   assert val >= 0
   if val == 0:
-    t.remove(key)
+    delImplNoHCode(cntMakeEmpty, cntCellEmpty, cntCellHash)
   else:
     let h = rawGet(t, key)
     if h >= 0:
@@ -2305,11 +2300,8 @@ proc `[]=`*[A](t: var CountTable[A], key: A, val: int) =
     else:
       insertImpl()
 
-proc inc*[A](t: var CountTable[A], key: A, val: Positive = 1) =
+proc inc*[A](t: var CountTable[A], key: A, val = 1) =
   ## Increments ``t[key]`` by ``val`` (default: 1).
-  ##
-  ## ``val`` must be a positive number. If you need to decrement a value,
-  ## use a regular ``Table`` instead.
   runnableExamples:
     var a = toCountTable("aab")
     a.inc('a')
@@ -2320,9 +2312,11 @@ proc inc*[A](t: var CountTable[A], key: A, val: Positive = 1) =
   var index = rawGet(t, key)
   if index >= 0:
     inc(t.data[index].val, val)
-    if t.data[index].val == 0: dec(t.counter)
+    if t.data[index].val == 0:
+      delImplIdx(t, index, cntMakeEmpty, cntCellEmpty, cntCellHash)
   else:
-    insertImpl()
+    if val != 0:
+      insertImpl()
 
 proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] =
   ## Returns the ``(key, value)`` pair with the smallest ``val``. Efficiency: O(n)
@@ -2383,8 +2377,6 @@ proc len*[A](t: CountTable[A]): int =
 proc del*[A](t: var CountTable[A], key: A) {.since: (1, 1).} =
   ## Deletes ``key`` from table ``t``. Does nothing if the key does not exist.
   ##
-  ## O(n) complexity.
-  ##
   ## See also:
   ## * `pop proc<#pop,CountTable[A],A,int>`_
   ## * `clear proc<#clear,CountTable[A]>`_ to empty the whole table
@@ -2397,7 +2389,7 @@ proc del*[A](t: var CountTable[A], key: A) {.since: (1, 1).} =
     a.del('c')
     assert a == toCountTable("aa")
 
-  remove(t, key)
+  delImplNoHCode(cntMakeEmpty, cntCellEmpty, cntCellHash)
 
 proc pop*[A](t: var CountTable[A], key: A, val: var int): bool {.since: (1, 1).} =
   ## Deletes the ``key`` from the table.
@@ -2405,8 +2397,6 @@ proc pop*[A](t: var CountTable[A], key: A, val: var int): bool {.since: (1, 1).}
   ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is
   ## unchanged.
   ##
-  ## O(n) complexity.
-  ##
   ## See also:
   ## * `del proc<#del,CountTable[A],A>`_
   ## * `clear proc<#clear,CountTable[A]>`_ to empty the whole table
@@ -2423,7 +2413,7 @@ proc pop*[A](t: var CountTable[A], key: A, val: var int): bool {.since: (1, 1).}
   result = index >= 0
   if result:
     val = move(t.data[index].val)
-    remove(t, key)
+    delImplIdx(t, index, cntMakeEmpty, cntCellEmpty, cntCellHash)
 
 proc clear*[A](t: var CountTable[A]) =
   ## Resets the table so that it is empty.
@@ -2717,8 +2707,6 @@ proc len*[A](t: CountTableRef[A]): int =
 proc del*[A](t: CountTableRef[A], key: A) {.since: (1, 1).} =
   ## Deletes ``key`` from table ``t``. Does nothing if the key does not exist.
   ##
-  ## O(n) complexity.
-  ##
   ## See also:
   ## * `pop proc<#pop,CountTableRef[A],A,int>`_
   ## * `clear proc<#clear,CountTableRef[A]>`_ to empty the whole table
@@ -2730,8 +2718,6 @@ proc pop*[A](t: CountTableRef[A], key: A, val: var int): bool {.since: (1, 1).}
   ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is
   ## unchanged.
   ##
-  ## O(n) complexity.
-  ##
   ## See also:
   ## * `del proc<#del,CountTableRef[A],A>`_
   ## * `clear proc<#clear,CountTableRef[A]>`_ to empty the whole table

lib/pure/encodings.nim

@@ -290,11 +290,8 @@ else:
     importc: "iconv_open", importIconv.}
   proc iconvClose(c: EncodingConverter) {.
     importc: "iconv_close", importIconv.}
-  proc iconv(c: EncodingConverter, inbuf: var cstring, inbytesLeft: var int,
-             outbuf: var cstring, outbytesLeft: var int): int {.
-    importc: "iconv", importIconv.}
-  proc iconv(c: EncodingConverter, inbuf: pointer, inbytesLeft: pointer,
-             outbuf: var cstring, outbytesLeft: var int): int {.
+  proc iconv(c: EncodingConverter, inbuf: ptr cstring, inbytesLeft: ptr csize_t,
+             outbuf: ptr cstring, outbytesLeft: ptr csize_t): csize_t {.
     importc: "iconv", importIconv.}
 
 proc getCurrentEncoding*(uiApp = false): string =
@@ -428,14 +425,14 @@ when defined(windows):
 else:
   proc convert*(c: EncodingConverter, s: string): string =
     result = newString(s.len)
-    var inLen = len(s)
-    var outLen = len(result)
+    var inLen = csize_t len(s)
+    var outLen = csize_t len(result)
     var src = cstring(s)
     var dst = cstring(result)
-    var iconvres: int
+    var iconvres: csize_t
     while inLen > 0:
-      iconvres = iconv(c, src, inLen, dst, outLen)
-      if iconvres == -1:
+      iconvres = iconv(c, addr src, addr inLen, addr dst, addr outLen)
+      if iconvres == high(csize_t):
         var lerr = errno
         if lerr == EILSEQ or lerr == EINVAL:
           # unknown char, skip
@@ -446,24 +443,24 @@ else:
           dec(outLen)
         elif lerr == E2BIG:
           var offset = cast[int](dst) - cast[int](cstring(result))
-          setLen(result, len(result)+inLen*2+5)
+          setLen(result, len(result) + inLen.int * 2 + 5)
           # 5 is minimally one utf-8 char
           dst = cast[cstring](cast[int](cstring(result)) + offset)
-          outLen = len(result) - offset
+          outLen = csize_t(len(result) - offset)
         else:
           raiseOSError(lerr.OSErrorCode)
     # iconv has a buffer that needs flushing, specially if the last char is
     # not '\0'
-    discard iconv(c, nil, nil, dst, outLen)
-    if iconvres == cint(-1) and errno == E2BIG:
+    discard iconv(c, nil, nil, addr dst, addr outLen)
+    if iconvres == high(csize_t) and errno == E2BIG:
       var offset = cast[int](dst) - cast[int](cstring(result))
-      setLen(result, len(result)+inLen*2+5)
+      setLen(result, len(result) + inLen.int * 2 + 5)
       # 5 is minimally one utf-8 char
       dst = cast[cstring](cast[int](cstring(result)) + offset)
-      outLen = len(result) - offset
-      discard iconv(c, nil, nil, dst, outLen)
+      outLen = csize_t(len(result) - offset)
+      discard iconv(c, nil, nil, addr dst, addr outLen)
     # trim output buffer
-    setLen(result, len(result) - outLen)
+    setLen(result, len(result) - outLen.int)
 
 proc convert*(s: string, destEncoding = "UTF-8",
                          srcEncoding = "CP1252"): string =

lib/pure/httpclient.nim

@@ -241,7 +241,8 @@ proc contentLength*(response: Response | AsyncResponse): int =
   ##
   ## A ``ValueError`` exception will be raised if the value is not an integer.
   var contentLengthHeader = response.headers.getOrDefault("Content-Length")
-  return contentLengthHeader.parseInt()
+  result = contentLengthHeader.parseInt()
+  doAssert(result >= 0 and result <= high(int32))
 
 proc lastModified*(response: Response | AsyncResponse): DateTime =
   ## Retrieves the specified response's last modified time.
@@ -1018,6 +1019,11 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string,
   ##
   ## This procedure will follow redirects up to a maximum number of redirects
   ## specified in ``client.maxRedirects``.
+  ##
+  ## You need to make sure that the ``url`` doesn't contain any newline
+  ## characters. Failing to do so will raise ``AssertionDefect``.
+  doAssert(not url.contains({'\c', '\L'}), "url shouldn't contain any newline characters")
+
   result = await client.requestAux(url, httpMethod, body, headers, multipart)
 
   var lastURL = url

lib/pure/includes/decode_helpers.nim

@@ -0,0 +1,24 @@
+# Include file that implements 'decodePercent' and friends. Do not import it!
+
+proc handleHexChar(c: char, x: var int, f: var bool) {.inline.} =
+  case c
+  of '0'..'9': x = (x shl 4) or (ord(c) - ord('0'))
+  of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10)
+  of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
+  else: f = true
+
+proc decodePercent(s: string, i: var int): char =
+  ## Converts `%xx` hexadecimal to the charracter with ordinal number `xx`.
+  ##
+  ## If `xx` is not a valid hexadecimal value, it is left intact: only the
+  ## leading `%` is returned as-is, and `xx` characters will be processed in the
+  ## next step (e.g. in `uri.decodeUrl`) as regular characters.
+  result = '%'
+  if i+2 < s.len:
+    var x = 0
+    var failed = false
+    handleHexChar(s[i+1], x, failed)
+    handleHexChar(s[i+2], x, failed)
+    if not failed:
+      result = chr(x)
+      inc(i, 2)

lib/pure/net.nim

@@ -180,11 +180,16 @@ proc isDisconnectionError*(flags: set[SocketFlag],
   ## if flags contains ``SafeDisconn``.
   when useWinVersion:
     SocketFlag.SafeDisconn in flags and
-      lastError.int32 in {WSAECONNRESET, WSAECONNABORTED, WSAENETRESET,
-                          WSAEDISCON, ERROR_NETNAME_DELETED}
+      (lastError.int32 == WSAECONNRESET or
+       lastError.int32 == WSAECONNABORTED or
+       lastError.int32 == WSAENETRESET or
+       lastError.int32 == WSAEDISCON or
+       lastError.int32 == ERROR_NETNAME_DELETED)
   else:
     SocketFlag.SafeDisconn in flags and
-      lastError.int32 in {ECONNRESET, EPIPE, ENETRESET}
+      (lastError.int32 == ECONNRESET or
+       lastError.int32 == EPIPE or
+       lastError.int32 == ENETRESET)
 
 proc toOSFlags*(socketFlags: set[SocketFlag]): cint =
   ## Converts the flags into the underlying OS representation.
@@ -586,7 +591,7 @@ when defineSsl:
       if verifyMode != CVerifyNone:
         # Use the caDir and caFile parameters if set
         if caDir != "" or caFile != "":
-          if newCTX.SSL_CTX_load_verify_locations(caDir, caFile) != 0:
+          if newCTX.SSL_CTX_load_verify_locations(caFile, caDir) != 0:
             raise newException(IOError, "Failed to load SSL/TLS CA certificate(s).")
 
         else:

lib/pure/options.nim

@@ -77,7 +77,7 @@ type
   UnpackError* = object of Defect
 
 
-proc option*[T](val: T): Option[T] =
+proc option*[T](val: T): Option[T] {.inline.} =
   ## Can be used to convert a pointer type (`ptr` or `ref` or `proc`) to an option type.
   ## It converts `nil` to `None`.
   ##
@@ -98,7 +98,7 @@ proc option*[T](val: T): Option[T] =
   when T isnot SomePointer:
     result.has = true
 
-proc some*[T](val: T): Option[T] =
+proc some*[T](val: T): Option[T] {.inline.} =
   ## Returns an `Option` that has the value `val`.
   ##
   ## See also:
@@ -121,7 +121,7 @@ proc some*[T](val: T): Option[T] =
     result.has = true
     result.val = val
 
-proc none*(T: typedesc): Option[T] =
+proc none*(T: typedesc): Option[T] {.inline.} =
   ## Returns an `Option` for this type that has no value.
   ##
   ## See also:
@@ -136,7 +136,7 @@ proc none*(T: typedesc): Option[T] =
   # the default is the none type
   discard
 
-proc none*[T]: Option[T] =
+proc none*[T]: Option[T] {.inline.} =
   ## Alias for `none(T) proc <#none,typedesc>`_.
   none(T)
 
@@ -167,7 +167,7 @@ proc isNone*[T](self: Option[T]): bool {.inline.} =
   else:
     not self.has
 
-proc get*[T](self: Option[T]): T =
+proc get*[T](self: Option[T]): T {.inline.} =
   ## Returns contents of an `Option`. If it is `None`, then an exception is
   ## thrown.
   ##
@@ -185,7 +185,7 @@ proc get*[T](self: Option[T]): T =
     raise newException(UnpackError, "Can't obtain a value from a `none`")
   self.val
 
-proc get*[T](self: Option[T], otherwise: T): T =
+proc get*[T](self: Option[T], otherwise: T): T {.inline.} =
   ## Returns the contents of the `Option` or an `otherwise` value if
   ## the `Option` is `None`.
   runnableExamples:
@@ -200,7 +200,7 @@ proc get*[T](self: Option[T], otherwise: T): T =
   else:
     otherwise
 
-proc get*[T](self: var Option[T]): var T =
+proc get*[T](self: var Option[T]): var T {.inline.} =
   ## Returns contents of the `var Option`. If it is `None`, then an exception
   ## is thrown.
   runnableExamples:
@@ -215,7 +215,7 @@ proc get*[T](self: var Option[T]): var T =
     raise newException(UnpackError, "Can't obtain a value from a `none`")
   return self.val
 
-proc map*[T](self: Option[T], callback: proc (input: T)) =
+proc map*[T](self: Option[T], callback: proc (input: T)) {.inline.} =
   ## Applies a `callback` function to the value of the `Option`, if it has one.
   ##
   ## See also:
@@ -239,7 +239,7 @@ proc map*[T](self: Option[T], callback: proc (input: T)) =
   if self.isSome:
     callback(self.val)
 
-proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] =
+proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] {.inline.} =
   ## Applies a `callback` function to the value of the `Option` and returns an
   ## `Option` containing the new value.
   ##
@@ -266,7 +266,7 @@ proc map*[T, R](self: Option[T], callback: proc (input: T): R): Option[R] =
   else:
     none(R)
 
-proc flatten*[A](self: Option[Option[A]]): Option[A] =
+proc flatten*[A](self: Option[Option[A]]): Option[A] {.inline.} =
   ## Remove one level of structure in a nested `Option`.
   runnableExamples:
     let a = some(some(42))
@@ -278,7 +278,7 @@ proc flatten*[A](self: Option[Option[A]]): Option[A] =
     none(A)
 
 proc flatMap*[A, B](self: Option[A],
-                    callback: proc (input: A): Option[B]): Option[B] =
+                    callback: proc (input: A): Option[B]): Option[B] {.inline.} =
   ## Applies a `callback` function to the value of the `Option` and returns an
   ## `Option` containing the new value.
   ##
@@ -308,7 +308,7 @@ proc flatMap*[A, B](self: Option[A],
 
   map(self, callback).flatten()
 
-proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] =
+proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] {.inline.} =
   ## Applies a `callback` to the value of the `Option`.
   ##
   ## If the `callback` returns `true`, the option is returned as `Some`.
@@ -333,7 +333,7 @@ proc filter*[T](self: Option[T], callback: proc (input: T): bool): Option[T] =
   else:
     self
 
-proc `==`*(a, b: Option): bool =
+proc `==`*(a, b: Option): bool {.inline.} =
   ## Returns `true` if both `Option`s are `None`,
   ## or if they are both `Some` and have equal values.
   runnableExamples:
@@ -363,7 +363,7 @@ proc `$`*[T](self: Option[T]): string =
   else:
     result = "None[" & name(T) & "]"
 
-proc unsafeGet*[T](self: Option[T]): T =
+proc unsafeGet*[T](self: Option[T]): T {.inline.}=
   ## Returns the value of a `some`. Behavior is undefined for `none`.
   ##
   ## **Note:** Use it only when you are **absolutely sure** the value is present
@@ -513,4 +513,3 @@ when isMainModule:
     test "Ref type with overloaded `==`":
       let p = some(RefPerson.new())
       check p.isSome
-

lib/pure/os.nim

@@ -1778,9 +1778,9 @@ proc tryMoveFSObject(source, dest: string): bool {.noNimScript.} =
     when useWinUnicode:
       let s = newWideCString(source)
       let d = newWideCString(dest)
-      if moveFileExW(s, d, MOVEFILE_COPY_ALLOWED) == 0'i32: raiseOSError(osLastError(), $(source, dest))
+      if moveFileExW(s, d, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) == 0'i32: raiseOSError(osLastError(), $(source, dest))
     else:
-      if moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED) == 0'i32: raiseOSError(osLastError(), $(source, dest))
+      if moveFileExA(source, dest, MOVEFILE_COPY_ALLOWED or MOVEFILE_REPLACE_EXISTING) == 0'i32: raiseOSError(osLastError(), $(source, dest))
   else:
     if c_rename(source, dest) != 0'i32:
       let err = osLastError()

lib/pure/osproc.nim

@@ -441,7 +441,11 @@ when defined(Windows) and not defined(useNimRtl):
       handle: Handle
       atTheEnd: bool
 
-  proc hsClose(s: Stream) = discard # nothing to do here
+  proc hsClose(s: Stream) =
+    # xxx here + elsewhere: check instead of discard; ignoring errors leads to
+    # hard to track bugs
+    discard FileHandleStream(s).handle.closeHandle
+
   proc hsAtEnd(s: Stream): bool = return FileHandleStream(s).atTheEnd
 
   proc hsReadData(s: Stream, buffer: pointer, bufLen: int): int =

lib/pure/parsecsv.nim

@@ -108,10 +108,10 @@ proc open*(my: var CsvParser, input: Stream, filename: string,
   ## the diverse optional parameters:
   ## - `separator`: character used to separate fields
   ## - `quote`: Used to quote fields containing special characters like
-  ##   `separator`, `quote` or new-line characters. '\0' disables the parsing
+  ##   `separator`, `quote` or new-line characters. '\\0' disables the parsing
   ##   of quotes.
   ## - `escape`: removes any special meaning from the following character;
-  ##   '\0' disables escaping; if escaping is disabled and `quote` is not '\0',
+  ##   '\\0' disables escaping; if escaping is disabled and `quote` is not '\\0',
   ##   two `quote` characters are parsed one literal `quote` character.
   ## - `skipInitialSpace`: If true, whitespace immediately following the
   ##   `separator` is ignored.

lib/pure/pegs.nim

@@ -870,26 +870,26 @@ macro mkHandlerTplts(handlers: untyped): untyped =
   # The AST structure of *handlers[0]*:
   #
   # .. code-block::
-  # StmtList
-  #   Call
-  #     Ident "pkNonTerminal"
-  #     StmtList
-  #       Call
-  #         Ident "enter"
-  #         StmtList
-  #           <handler code block>
-  #       Call
-  #         Ident "leave"
-  #         StmtList
-  #           <handler code block>
-  #   Call
-  #     Ident "pkChar"
-  #     StmtList
-  #       Call
-  #         Ident "leave"
-  #         StmtList
-  #           <handler code block>
-  #   ...
+  #   StmtList
+  #     Call
+  #       Ident "pkNonTerminal"
+  #       StmtList
+  #         Call
+  #           Ident "enter"
+  #           StmtList
+  #             <handler code block>
+  #         Call
+  #           Ident "leave"
+  #           StmtList
+  #             <handler code block>
+  #     Call
+  #       Ident "pkChar"
+  #       StmtList
+  #         Call
+  #           Ident "leave"
+  #           StmtList
+  #             <handler code block>
+  #     ...
   proc mkEnter(hdName, body: NimNode): NimNode =
     template helper(hdName, body) {.dirty.} =
       template hdName(s, p, start) =
@@ -907,16 +907,16 @@ macro mkHandlerTplts(handlers: untyped): untyped =
 
   result = newStmtList()
   for topCall in handlers[0]:
-    if nnkCall != topCall.kind:
+    if topCall.kind notin nnkCallKinds:
       error("Call syntax expected.", topCall)
     let pegKind = topCall[0]
-    if nnkIdent != pegKind.kind:
+    if pegKind.kind notin {nnkIdent, nnkSym}:
       error("PegKind expected.", pegKind)
     if 2 == topCall.len:
       for hdDef in topCall[1]:
-        if nnkCall != hdDef.kind:
+        if hdDef.kind notin nnkCallKinds:
           error("Call syntax expected.", hdDef)
-        if nnkIdent != hdDef[0].kind:
+        if hdDef[0].kind notin {nnkIdent, nnkSym}:
           error("Handler identifier expected.", hdDef[0])
         if 2 == hdDef.len:
           let hdPostf = substr(pegKind.strVal, 2)

lib/pure/smtp.nim

@@ -43,7 +43,7 @@
 ## For SSL support this module relies on OpenSSL. If you want to
 ## enable SSL, compile with ``-d:ssl``.
 
-import net, strutils, strtabs, base64, os
+import net, strutils, strtabs, base64, os, strutils
 import asyncnet, asyncdispatch
 
 export Port
@@ -65,7 +65,22 @@ type
   Smtp* = SmtpBase[Socket]
   AsyncSmtp* = SmtpBase[AsyncSocket]
 
-proc debugSend(smtp: Smtp | AsyncSmtp, cmd: string) {.multisync.} =
+proc containsNewline(xs: seq[string]): bool =
+  for x in xs:
+    if x.contains({'\c', '\L'}):
+      return true
+
+proc debugSend*(smtp: Smtp | AsyncSmtp, cmd: string) {.multisync.} =
+  ## Sends ``cmd`` on the socket connected to the SMTP server.
+  ##
+  ## If the ``smtp`` object was created with ``debug`` enabled,
+  ## debugSend will invoke ``echo("C:" & cmd)`` before sending.
+  ##
+  ## This is a lower level proc and not something that you typically
+  ## would need to call when using this module. One exception to
+  ## this is if you are implementing any
+  ## `SMTP extensions<https://en.wikipedia.org/wiki/Extended_SMTP>`_.
+
   if smtp.debug:
     echo("C:" & cmd)
   await smtp.sock.send(cmd)
@@ -95,6 +110,14 @@ else:
 proc createMessage*(mSubject, mBody: string, mTo, mCc: seq[string],
                 otherHeaders: openarray[tuple[name, value: string]]): Message =
   ## Creates a new MIME compliant message.
+  ##
+  ## You need to make sure that ``mSubject``, ``mTo`` and ``mCc`` don't contain
+  ## any newline characters. Failing to do so will raise ``AssertionDefect``.
+  doAssert(not mSubject.contains({'\c', '\L'}),
+           "'mSubject' shouldn't contain any newline characters")
+  doAssert(not (mTo.containsNewline() or mCc.containsNewline()),
+           "'mTo' and 'mCc' shouldn't contain any newline characters")
+
   result.msgTo = mTo
   result.msgCc = mCc
   result.msgSubject = mSubject
@@ -106,6 +129,13 @@ proc createMessage*(mSubject, mBody: string, mTo, mCc: seq[string],
 proc createMessage*(mSubject, mBody: string, mTo,
                     mCc: seq[string] = @[]): Message =
   ## Alternate version of the above.
+  ##
+  ## You need to make sure that ``mSubject``, ``mTo`` and ``mCc`` don't contain
+  ## any newline characters. Failing to do so will raise ``AssertionDefect``.
+  doAssert(not mSubject.contains({'\c', '\L'}),
+           "'mSubject' shouldn't contain any newline characters")
+  doAssert(not (mTo.containsNewline() or mCc.containsNewline()),
+           "'mTo' and 'mCc' shouldn't contain any newline characters")
   result.msgTo = mTo
   result.msgCc = mCc
   result.msgSubject = mSubject
@@ -213,6 +243,11 @@ proc sendMail*(smtp: Smtp | AsyncSmtp, fromAddr: string,
   ## Sends ``msg`` from ``fromAddr`` to the addresses specified in ``toAddrs``.
   ## Messages may be formed using ``createMessage`` by converting the
   ## Message into a string.
+  ##
+  ## You need to make sure that ``fromAddr`` and ``toAddrs`` don't contain
+  ## any newline characters. Failing to do so will raise ``AssertionDefect``.
+  doAssert(not (toAddrs.containsNewline() or fromAddr.contains({'\c', '\L'})),
+           "'toAddrs' and 'fromAddr' shouldn't contain any newline characters")
 
   await smtp.debugSend("MAIL FROM:<" & fromAddr & ">\c\L")
   await smtp.checkReply("250")

lib/pure/strformat.nim

@@ -692,11 +692,11 @@ when isMainModule:
   check &"{-123.456:.3f}", "-123.456"
   check &"{123.456:1g}", "123.456"
   check &"{123.456:.1f}", "123.5"
-  check &"{123.456:.0f}", "123"
+  check &"{123.456:.0f}", "123."
   check &"{123.456:>9.3f}", "  123.456"
   check &"{123.456:9.3f}", "  123.456"
   check &"{123.456:>9.4f}", " 123.4560"
-  check &"{123.456:>9.0f}", "      123"
+  check &"{123.456:>9.0f}", "     123."
   check &"{123.456:<9.4f}", "123.4560 "
 
   # Float (scientific) tests

lib/pure/strutils.nim

@@ -2357,10 +2357,6 @@ proc formatBiggestFloat*(f: BiggestFloat, format: FloatFormatMode = ffDefault,
       # but nothing else is possible:
       if buf[i] in {'.', ','}: result[i] = decimalSep
       else: result[i] = buf[i]
-    since (1, 1):
-      # remove trailing dot, compatible with Python's formatter and JS backend
-      if result[^1] == decimalSep:
-        result.setLen(len(result)-1)
     when defined(windows):
       # VS pre 2015 violates the C standard: "The exponent always contains at
       # least two digits, and only as many more digits as necessary to
@@ -2863,7 +2859,8 @@ proc isNilOrWhitespace*(s: string): bool {.noSideEffect, procvar, rtl,
 when isMainModule:
   proc nonStaticTests =
     doAssert formatBiggestFloat(1234.567, ffDecimal, -1) == "1234.567000"
-    doAssert formatBiggestFloat(1234.567, ffDecimal, 0) == "1235" # bugs 8242, 12586
+    when not defined(js):
+      doAssert formatBiggestFloat(1234.567, ffDecimal, 0) == "1235." # bugs 8242, 12586
     doAssert formatBiggestFloat(1234.567, ffDecimal, 1) == "1234.6"
     doAssert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001"
     doAssert formatBiggestFloat(0.00000000001, ffScientific, 1, ',') in

lib/pure/times.nim

@@ -260,6 +260,7 @@ type
   Month* = enum ## Represents a month. Note that the enum starts at ``1``,
                 ## so ``ord(month)`` will give the month number in the
                 ## range ``1..12``.
+    # mInvalid = (0, "Invalid") # intentionally left out so `items` works
     mJan = (1, "January")
     mFeb = "February"
     mMar = "March"
@@ -296,7 +297,8 @@ when defined(nimHasStyleChecks):
   {.pop.}
 
 type
-  MonthdayRange* = range[1..31]
+  MonthdayRange* = range[0..31]
+    ## 0 represents an invalid day of the month
   HourRange* = range[0..23]
   MinuteRange* = range[0..59]
   SecondRange* = range[0..60]
@@ -670,7 +672,7 @@ proc getDaysInYear*(year: int): int =
 
 proc assertValidDate(monthday: MonthdayRange, month: Month, year: int)
     {.inline.} =
-  assert monthday <= getDaysInMonth(month, year),
+  assert monthday > 0 and monthday <= getDaysInMonth(month, year),
     $year & "-" & intToStr(ord(month), 2) & "-" & $monthday &
       " is not a valid date"
 
@@ -1576,9 +1578,14 @@ proc `<=`*(a, b: DateTime): bool =
   ## Returns true if ``a`` happened before or at the same time as ``b``.
   return a.toTime <= b.toTime
 
+proc isDefault[T](a: T): bool =
+  system.`==`(a, default(T))
+
 proc `==`*(a, b: DateTime): bool =
   ## Returns true if ``a`` and ``b`` represent the same point in time.
-  return a.toTime == b.toTime
+  if a.isDefault: b.isDefault
+  elif b.isDefault: false
+  else: a.toTime == b.toTime
 
 proc isStaticInterval(interval: TimeInterval): bool =
   interval.years == 0 and interval.months == 0 and

lib/pure/typetraits.nim

@@ -89,7 +89,7 @@ since (1, 1):
 
 since (1, 1):
   template get*(T: typedesc[tuple], i: static int): untyped =
-    ## Return `i`th element of `T`
+    ## Return `i`\th element of `T`
     # Note: `[]` currently gives: `Error: no generic parameters allowed for ...`
     type(default(T)[i])
 

lib/pure/uri.nim

@@ -37,7 +37,9 @@
 ##    else:
 ##      echo "Wrong format"
 
-import strutils, parseutils
+import strutils, parseutils, base64
+include includes/decode_helpers
+
 type
   Url* = distinct string
 
@@ -81,6 +83,7 @@ proc decodeUrl*(s: string, decodePlus = true): string =
   ## This means that any ``%xx`` (where ``xx`` denotes a hexadecimal
   ## value) are converted to the character with ordinal number ``xx``,
   ## and every other character is carried over.
+  ## If ``xx`` is not a valid hexadecimal value, it is left intact.
   ##
   ## As a special rule, when the value of ``decodePlus`` is true, ``+``
   ## characters are converted to a space.
@@ -92,12 +95,7 @@ proc decodeUrl*(s: string, decodePlus = true): string =
     assert decodeUrl("https%3A%2F%2Fnim-lang.org%2Fthis+is+a+test") == "https://nim-lang.org/this is a test"
     assert decodeUrl("https%3A%2F%2Fnim-lang.org%2Fthis%20is%20a%20test",
         false) == "https://nim-lang.org/this is a test"
-  proc handleHexChar(c: char, x: var int) {.inline.} =
-    case c
-    of '0'..'9': x = (x shl 4) or (ord(c) - ord('0'))
-    of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10)
-    of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
-    else: assert(false)
+    assert decodeUrl("abc%xyz") == "abc%xyz"
 
   result = newString(s.len)
   var i = 0
@@ -105,11 +103,7 @@ proc decodeUrl*(s: string, decodePlus = true): string =
   while i < s.len:
     case s[i]
     of '%':
-      var x = 0
-      handleHexChar(s[i+1], x)
-      handleHexChar(s[i+2], x)
-      inc(i, 2)
-      result[j] = chr(x)
+      result[j] = decodePercent(s, i)
     of '+':
       if decodePlus:
         result[j] = ' '

lib/std/wordwrap.nim

@@ -11,10 +11,10 @@
 
 import strutils, unicode
 
-proc olen(s: string): int =
-  var i = 0
+proc olen(s: string; start, lastExclusive: int): int =
+  var i = start
   result = 0
-  while i < s.len:
+  while i < lastExclusive:
     inc result
     let L = graphemeLen(s, i)
     inc i, L
@@ -32,31 +32,46 @@ proc wrapWords*(s: string, maxLineWidth = 80,
   result = newStringOfCap(s.len + s.len shr 6)
   var spaceLeft = maxLineWidth
   var lastSep = ""
-  for word, isSep in tokenize(s, seps):
-    let wlen = olen(word)
+
+  var i = 0
+  while true:
+    var j = i
+    let isSep = j < s.len and s[j] in seps
+    while j < s.len and (s[j] in seps) == isSep: inc(j)
+    if j <= i: break
+    #yield (substr(s, i, j-1), isSep)
     if isSep:
-      lastSep = word
-      spaceLeft = spaceLeft - wlen
-    elif wlen > spaceLeft:
-      if splitLongWords and wlen > maxLineWidth:
-        var i = 0
-        while i < word.len:
-          if spaceLeft <= 0:
-            spaceLeft = maxLineWidth
-            result.add newLine
-          dec spaceLeft
-          let L = graphemeLen(word, i)
-          for j in 0 ..< L: result.add word[i+j]
-          inc i, L
+      lastSep.setLen 0
+      for k in i..<j:
+        if s[k] notin {'\L', '\C'}: lastSep.add s[k]
+      if lastSep.len == 0:
+        lastSep.add ' '
+        dec spaceLeft
       else:
-        spaceLeft = maxLineWidth - wlen
-        result.add(newLine)
-        result.add(word)
+        spaceLeft = spaceLeft - olen(lastSep, 0, lastSep.len)
     else:
-      spaceLeft = spaceLeft - wlen
-      result.add(lastSep)
-      result.add(word)
-      lastSep.setLen(0)
+      let wlen = olen(s, i, j)
+      if wlen > spaceLeft:
+        if splitLongWords and wlen > maxLineWidth:
+          var k = 0
+          while k < j - i:
+            if spaceLeft <= 0:
+              spaceLeft = maxLineWidth
+              result.add newLine
+            dec spaceLeft
+            let L = graphemeLen(s, k+i)
+            for m in 0 ..< L: result.add s[i+k+m]
+            inc k, L
+        else:
+          spaceLeft = maxLineWidth - wlen
+          result.add(newLine)
+          for k in i..<j: result.add(s[k])
+      else:
+        spaceLeft = spaceLeft - wlen
+        result.add(lastSep)
+        for k in i..<j: result.add(s[k])
+        #lastSep.setLen(0)
+    i = j
 
 when isMainModule:
 
@@ -93,3 +108,11 @@ tnaetdriaoenvlcyfglwckßqfgvwkßqgfvlwkßqfgvlwckßqvlwkgfUIαοιαοιαχολ
 ξχλωωχαοεοιαεοαεοιαεοαεοιαοεσναοεκνρκψγκψφϵιηαααοε"""
   doAssert wrapWords(longlongword) == longlongwordRes
 
+  # bug #14579
+  const input60 = """
+This is a long string. It is manually wrapped to 60
+characters. I would not expect it to be changed by
+wordwrap if wordwrap is set to wrap at 80 characters"""
+  const input60Res = """This is a long string. It is manually wrapped to 60 characters. I would not
+expect it to be changed by wordwrap if wordwrap is set to wrap at 80 characters"""
+  doAssert wrapWords(input60) == input60Res

lib/system.nim

@@ -2049,10 +2049,10 @@ const
   NimMajor* {.intdefine.}: int = 1
     ## is the major number of Nim's version.
 
-  NimMinor* {.intdefine.}: int = 1
+  NimMinor* {.intdefine.}: int = 2
     ## is the minor number of Nim's version.
 
-  NimPatch* {.intdefine.}: int = 1
+  NimPatch* {.intdefine.}: int = 6
     ## is the patch number of Nim's version.
 
   NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch

lib/system/atomics.nim

@@ -281,10 +281,7 @@ static int __tcc_cas(int *ptr, int oldVal, int newVal)
             : "r" (newVal), "m" (*ptr), "a" (oldVal)
             : "memory");
 
-    if (ret)
-      return 0;
-    else
-      return 1;
+    return ret;
 }
 """.}
   else:
@@ -301,10 +298,7 @@ static int __tcc_cas(int *ptr, int oldVal, int newVal)
             : "r" (newVal), "m" (*ptr), "a" (oldVal)
             : "memory");
 
-    if (ret)
-      return 0;
-    else
-      return 1;
+    return ret;
 }
 """.}
 

lib/system/gc_ms.nim

@@ -104,7 +104,7 @@ proc extGetCellType(c: pointer): PNimType {.compilerproc.} =
   # used for code generation concerning debugging
   result = usrToCell(c).typ
 
-proc unsureAsgnRef(dest: PPointer, src: pointer) {.inline.} =
+proc unsureAsgnRef(dest: PPointer, src: pointer) {.inline, compilerproc.} =
   dest[] = src
 
 proc internRefcount(p: pointer): int {.exportc: "getRefcount".} =

lib/system/integerops.nim

@@ -19,7 +19,7 @@ proc raiseDivByZero {.compilerproc, noinline.} =
 
 {.pragma: nimbaseH, importc, nodecl, noSideEffect, compilerproc.}
 
-when (defined(gcc) or defined(clang)) and not defined(nimEmulateOverflowChecks):
+when not defined(nimEmulateOverflowChecks):
   # take the #define from nimbase.h
 
   proc nimAddInt(a, b: int, res: ptr int): bool {.nimbaseH.}

lib/system/memory.nim

@@ -45,9 +45,9 @@ proc nimCmpMem*(a, b: pointer, size: Natural): cint {.compilerproc, nonReloadabl
       if d != 0: return d
       inc i
 
-proc nimCStrLen*(a: cstring): csize_t {.compilerproc, nonReloadable, inline.} =
+proc nimCStrLen*(a: cstring): int {.compilerproc, nonReloadable, inline.} =
   when useLibC:
-    c_strlen(a)
+    cast[int](c_strlen(a))
   else:
     var a = cast[ptr byte](a)
     while a[] != 0:

lib/system/nimscript.nim

@@ -384,11 +384,26 @@ when not defined(nimble):
   template `==?`(a, b: string): bool = cmpIgnoreStyle(a, b) == 0
   template task*(name: untyped; description: string; body: untyped): untyped =
     ## Defines a task. Hidden tasks are supported via an empty description.
+    ##
     ## Example:
     ##
     ## .. code-block:: nim
     ##  task build, "default build is via the C backend":
     ##    setCommand "c"
+    ##
+    ## For a task named ``foo``, this template generates a ``proc`` named
+    ## ``fooTask``.  This is useful if you need to call one task in
+    ## another in your Nimscript.
+    ##
+    ## Example:
+    ##
+    ## .. code-block:: nim
+    ##  task foo, "foo":        # > nim foo
+    ##    echo "Running foo"    # Running foo
+    ##
+    ##  task bar, "bar":        # > nim bar
+    ##    echo "Running bar"    # Running bar
+    ##    fooTask()             # Running foo
     proc `name Task`*() =
       setCommand "nop"
       body

lib/windows/winlean.nim

@@ -31,6 +31,8 @@ type
   ULONG* = int32
   PULONG* = ptr int
   WINBOOL* = int32
+    ## `WINBOOL` uses opposite convention as posix, !=0 meaning success.
+    # xxx this should be distinct int32, distinct would make code less error prone
   DWORD* = int32
   PDWORD* = ptr DWORD
   LPINT* = ptr int32

lib/wrappers/odbcsql.nim

@@ -45,8 +45,10 @@ type
   SqlHDBC* = SqlHandle
   SqlHStmt* = SqlHandle
   SqlHDesc* = SqlHandle
-  TSqlInteger* = int
-  SqlUInteger* = int
+  TSqlInteger* = int32
+  SqlUInteger* = int32
+  TSqlLen* = int
+  TSqlULen* = uint
   SqlPointer* = pointer
   TSqlReal* = cfloat
   TSqlDouble* = cdouble
@@ -666,7 +668,7 @@ proc SQLAllocHandle*(HandleType: TSqlSmallInt, InputHandle: SqlHandle,
                      OutputHandlePtr: var SqlHandle): TSqlSmallInt{.
     dynlib: odbclib, importc.}
 proc SQLSetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger,
-                    Value: TSqlInteger, StringLength: TSqlInteger): TSqlSmallInt{.
+                    Value: SqlPointer, StringLength: TSqlInteger): TSqlSmallInt{.
     dynlib: odbclib, importc.}
 proc SQLGetEnvAttr*(EnvironmentHandle: SqlHEnv, Attribute: TSqlInteger,
                     Value: SqlPointer, BufferLength: TSqlInteger,
@@ -716,14 +718,14 @@ proc SQLNumResultCols*(StatementHandle: SqlHStmt, ColumnCount: var TSqlSmallInt)
 proc SQLDescribeCol*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt,
                      ColumnName: PSQLCHAR, BufferLength: TSqlSmallInt,
                      NameLength: var TSqlSmallInt, DataType: var TSqlSmallInt,
-                     ColumnSize: var SqlUInteger,
+                     ColumnSize: var TSQLULEN,
                      DecimalDigits: var TSqlSmallInt, Nullable: var TSqlSmallInt): TSqlSmallInt{.
     dynlib: odbclib, importc.}
 proc SQLFetchScroll*(StatementHandle: SqlHStmt, FetchOrientation: TSqlSmallInt,
                      FetchOffset: TSqlInteger): TSqlSmallInt{.dynlib: odbclib,
     importc.}
 proc SQLExtendedFetch*(hstmt: SqlHStmt, fFetchType: SqlUSmallInt,
-                       irow: TSqlInteger, pcrow: PSQLUINTEGER,
+                       irow: TSqlInteger, pcrow: var TSQLULEN,
                        rgfRowStatus: PSQLUSMALLINT): TSqlSmallInt{.dynlib: odbclib,
     importc.}
 proc SQLGetData*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt,
@@ -741,13 +743,13 @@ proc SQLGetInfo*(ConnectionHandle: SqlHDBC, InfoType: SqlUSmallInt,
                  InfoValue: SqlPointer, BufferLength: TSqlSmallInt,
                  StringLength: PSQLSMALLINT): TSqlSmallInt{.dynlib: odbclib,
     importc.}
-proc SQLBulkOperations*(StatementHandle: SqlHStmt, Operation: TSqlSmallInt): TSqlSmallInt{.
+proc SQLBulkOperations*(StatementHandle: SqlHStmt, Operation: SqlUSmallInt): TSqlSmallInt{.
     dynlib: odbclib, importc.}
 proc SQLPutData*(StatementHandle: SqlHStmt, Data: SqlPointer,
-                 StrLen_or_Ind: TSqlInteger): TSqlSmallInt{.dynlib: odbclib, importc.}
+                 StrLen_or_Ind: TSQLLEN): TSqlSmallInt{.dynlib: odbclib, importc.}
 proc SQLBindCol*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt,
                  TargetType: TSqlSmallInt, TargetValue: SqlPointer,
-                 BufferLength: TSqlInteger, StrLen_or_Ind: PSQLINTEGER): TSqlSmallInt{.
+                 BufferLength: TSqlLEN, StrLen_or_Ind: PSQLINTEGER): TSqlSmallInt{.
     dynlib: odbclib, importc.}
 proc SQLSetPos*(hstmt: SqlHStmt, irow: SqlUSmallInt, fOption: SqlUSmallInt,
                 fLock: SqlUSmallInt): TSqlSmallInt{.dynlib: odbclib, importc.}
@@ -770,13 +772,13 @@ proc SQLGetCursorName*(StatementHandle: SqlHStmt, CursorName: PSQLCHAR,
 proc SQLSetCursorName*(StatementHandle: SqlHStmt, CursorName: PSQLCHAR,
                        NameLength: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib,
     importc.}
-proc SQLRowCount*(StatementHandle: SqlHStmt, RowCount: var TSqlInteger): TSqlSmallInt{.
+proc SQLRowCount*(StatementHandle: SqlHStmt, RowCount: var TSQLLEN): TSqlSmallInt{.
     dynlib: odbclib, importc.}
 proc SQLBindParameter*(hstmt: SqlHStmt, ipar: SqlUSmallInt,
                        fParamType: TSqlSmallInt, fCType: TSqlSmallInt,
-                       fSqlType: TSqlSmallInt, cbColDef: SqlUInteger,
+                       fSqlType: TSqlSmallInt, cbColDef: TSQLULEN,
                        ibScale: TSqlSmallInt, rgbValue: SqlPointer,
-                       cbValueMax: TSqlInteger, pcbValue: PSQLINTEGER): TSqlSmallInt{.
+                       cbValueMax: TSQLLEN, pcbValue: var TSQLLEN): TSqlSmallInt{.
     dynlib: odbclib, importc.}
 proc SQLFreeStmt*(StatementHandle: SqlHStmt, Option: SqlUSmallInt): TSqlSmallInt{.
     dynlib: odbclib, importc.}
@@ -784,7 +786,7 @@ proc SQLColAttribute*(StatementHandle: SqlHStmt, ColumnNumber: SqlUSmallInt,
                       FieldIdentifier: SqlUSmallInt,
                       CharacterAttribute: PSQLCHAR, BufferLength: TSqlSmallInt,
                       StringLength: PSQLSMALLINT,
-                      NumericAttribute: SqlPointer): TSqlSmallInt{.
+                      NumericAttribute: TSQLLEN): TSqlSmallInt{.
     dynlib: odbclib, importc.}
 proc SQLEndTran*(HandleType: TSqlSmallInt, Handle: SqlHandle,
                  CompletionType: TSqlSmallInt): TSqlSmallInt{.dynlib: odbclib,

lib/wrappers/openssl.nim

@@ -244,7 +244,7 @@ proc TLSv1_method*(): PSSL_METHOD{.cdecl, dynlib: DLLSSLName, importc.}
 # and support SSLv3, TLSv1, TLSv1.1 and TLSv1.2
 # SSLv23_method(), SSLv23_server_method(), SSLv23_client_method() are removed in 1.1.0
 
-when compileOption("dynlibOverride", "ssl"):
+when compileOption("dynlibOverride", "ssl") or defined(noOpenSSLHacks):
   # Static linking
 
   when defined(openssl10):
@@ -285,40 +285,59 @@ else:
   proc thisModule(): LibHandle {.inline.} =
     var thisMod {.global.}: LibHandle
     if thisMod.isNil: thisMod = loadLib()
+
     result = thisMod
 
   proc sslModule(): LibHandle {.inline.} =
     var sslMod {.global.}: LibHandle
     if sslMod.isNil: sslMod = loadLibPattern(DLLSSLName)
+
     result = sslMod
 
-  proc sslSym(name: string): pointer =
-    var dl = thisModule()
-    if not dl.isNil:
-      result = symAddr(dl, name)
+  proc sslSymNullable(name: string, alternativeName = ""): pointer =
+    # Load from DLL.
+    var sslDynlib = sslModule()
+    if not sslDynlib.isNil:
+      result = symAddr(sslDynlib, name)
+      if result.isNil and alternativeName.len > 0:
+        result = symAddr(sslDynlib, alternativeName)
+
+    # Attempt to load from current exe.
     if result.isNil:
-      dl = sslModule()
-      if not dl.isNil:
-        result = symAddr(dl, name)
+      let thisDynlib = thisModule()
+      if thisDynlib.isNil: return nil
+      result = symAddr(thisDynlib, name)
+      if result.isNil and alternativeName.len > 0:
+        result = symAddr(sslDynlib, alternativeName)
+
+  proc sslSymThrows(name: string, alternativeName = ""): pointer =
+    result = sslSymNullable(name, alternativeName)
+    if result.isNil: raiseInvalidLibrary(name)
 
   proc loadPSSLMethod(method1, method2: string): PSSL_METHOD =
     ## Load <method1> from OpenSSL if available, otherwise <method2>
-    let m1 = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym(method1))
-    if not m1.isNil:
-      return m1()
-    cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](sslSym(method2))()
+    ##
+    let methodSym = sslSymNullable(method1, method2)
+    if methodSym.isNil:
+      raise newException(LibraryError, "Could not load " & method1 & " nor " & method2)
+
+    let method2Proc = cast[proc(): PSSL_METHOD {.cdecl, gcsafe.}](methodSym)
+    return method2Proc()
 
   proc SSL_library_init*(): cint {.discardable.} =
     ## Initialize SSL using OPENSSL_init_ssl for OpenSSL >= 1.1.0 otherwise
     ## SSL_library_init
-    let theProc = cast[proc(opts: uint64, settings: uint8): cint {.cdecl.}](sslSym("OPENSSL_init_ssl"))
-    if not theProc.isNil:
-      return theProc(0, 0)
-    let olderProc = cast[proc(): cint {.cdecl.}](sslSym("SSL_library_init"))
+    let newInitSym = sslSymNullable("OPENSSL_init_ssl")
+    if not newInitSym.isNil:
+      let newInitProc =
+        cast[proc(opts: uint64, settings: uint8): cint {.cdecl.}](newInitSym)
+      return newInitProc(0, 0)
+    let olderProc = cast[proc(): cint {.cdecl.}](sslSymThrows("SSL_library_init"))
     if not olderProc.isNil: result = olderProc()
 
   proc SSL_load_error_strings*() =
-    let theProc = cast[proc() {.cdecl.}](sslSym("SSL_load_error_strings"))
+    # TODO: Are we ignoring this on purpose? SSL GitHub CI fails otherwise.
+    let theProc = cast[proc() {.cdecl.}](sslSymNullable("SSL_load_error_strings"))
     if not theProc.isNil: theProc()
 
   proc SSLv23_client_method*(): PSSL_METHOD =
@@ -343,12 +362,13 @@ else:
     loadPSSLMethod("TLS_server_method", "SSLv23_server_method")
 
   proc OpenSSL_add_all_algorithms*() =
-    let theProc = cast[proc() {.cdecl.}](sslSym("OPENSSL_add_all_algorithms_conf"))
+    # TODO: Are we ignoring this on purpose? SSL GitHub CI fails otherwise.
+    let theProc = cast[proc() {.cdecl.}](sslSymNullable("OPENSSL_add_all_algorithms_conf"))
     if not theProc.isNil: theProc()
 
   proc getOpenSSLVersion*(): culong =
     ## Return OpenSSL version as unsigned long or 0 if not available
-    let theProc = cast[proc(): culong {.cdecl.}](sslSym("OpenSSL_version_num"))
+    let theProc = cast[proc(): culong {.cdecl.}](sslSymNullable("OpenSSL_version_num"))
     result =
       if theProc.isNil: 0.culong
       else: theProc()

nimsuggest/tests/ttempl_inst.nim

@@ -0,0 +1,13 @@
+template foo() =
+  {.warning: "foo".}
+  
+foo()
+
+#[!]#
+discard """
+$nimsuggest --tester $file
+>chk $1
+chk;;skUnknown;;;;Hint;;???;;0;;-1;;"ttempl_inst [Processing]";;0
+chk;;skUnknown;;;;Hint;;$file;;4;;3;;"template/generic instantiation from here";;0
+chk;;skUnknown;;;;Warning;;$file;;2;;11;;"foo [User]";;0
+"""

testament/categories.nim

@@ -260,9 +260,10 @@ proc asyncTests(r: var TResults, cat: Category, options: string) =
 # ------------------------- debugger tests ------------------------------------
 
 proc debuggerTests(r: var TResults, cat: Category, options: string) =
-  var t = makeTest("tools/nimgrep", options & " --debugger:on", cat)
-  t.spec.action = actionCompile
-  testSpec r, t
+  if fileExists("tools/nimgrep.nim"):
+    var t = makeTest("tools/nimgrep", options & " --debugger:on", cat)
+    t.spec.action = actionCompile
+    testSpec r, t
 
 # ------------------------- JS tests ------------------------------------------
 

testament/important_packages.nim

@@ -36,7 +36,7 @@ pkg "glob"
 pkg "gnuplot"
 pkg "hts", false, "nim c -o:htss src/hts.nim"
 pkg "illwill", false, "nimble examples"
-pkg "inim"
+# pkg "inim", true # pending https://github.com/inim-repl/INim/issues/74
 pkg "itertools", false, "nim doc src/itertools.nim"
 pkg "iterutils"
 pkg "jstin"
@@ -62,8 +62,8 @@ pkg "nimquery"
 pkg "nimsl", true
 pkg "nimsvg"
 # pkg "nimterop", true
-pkg "nimx", true, "nim c --threads:on test/main.nim"
-pkg "norm", true, "nim c -r tests/tsqlite.nim"
+# pkg "nimx", true, "nim c --threads:on test/main.nim"
+pkg "norm", true, "nim c -r tests/tsqliterows.nim"
 pkg "npeg"
 pkg "ormin", true, "nim c -o:orminn ormin.nim"
 pkg "parsetoml"

testament/testament.nim

@@ -679,7 +679,7 @@ proc main() =
   of "all":
     #processCategory(r, Category"megatest", p.cmdLineRest.string, testsDir, runJoinableTests = false)
 
-    var myself = quoteShell(findExe("testament" / "testament"))
+    var myself = quoteShell(getAppFilename())
     if targetsStr.len > 0:
       myself &= " " & quoteShell("--targets:" & targetsStr)
 

tests/arc/amodule.nim

@@ -0,0 +1,10 @@
+var vectors = @["a", "b", "c", "d", "e"]
+
+iterator testVectors(): string =
+  for vector in vectors:
+    yield vector
+
+var r = ""
+for item in testVectors():
+  r.add item
+echo r

tests/arc/tamodule.nim

@@ -0,0 +1,6 @@
+discard """
+  output: "abcde"
+  cmd: "nim c --gc:arc $file"
+"""
+
+import amodule

tests/arc/tarcmisc.nim

@@ -1,13 +1,38 @@
 discard """
   output: '''
+123xyzabc
 destroyed: false
 destroyed: false
+(x: "0")
+(x: "1")
+(x: "2")
+(x: "3")
+(x: "4")
+(x: "5")
+(x: "6")
+(x: "7")
+(x: "8")
+(x: "9")
+(x: "10")
 closed
 destroying variable
 '''
   cmd: "nim c --gc:arc $file"
 """
 
+proc takeSink(x: sink string): bool = true
+
+proc b(x: sink string): string =
+  if takeSink(x):
+    return x & "abc"
+
+proc bbb(inp: string) =
+  let y = inp & "xyz"
+  echo b(y)
+
+bbb("123")
+
+
 # bug #13691
 type Variable = ref object
   value: int
@@ -46,5 +71,60 @@ type
 proc `=destroy`(x: var AObj) =
   close(x.io)
   echo "closed"
-  
+
 var x = B(io: newStringStream("thestream"))
+
+
+#------------------------------------------------------------------------------
+# issue #14003
+
+proc cryptCTR*(nonce: var openArray[char]) =
+  nonce[1] = 'A'
+
+proc main() =
+  var nonce1 = "0123456701234567"
+  cryptCTR(nonce1)
+  doAssert(nonce1 == "0A23456701234567")
+  var nonce2 = "01234567"
+  cryptCTR(nonce2.toOpenArray(0, nonce2.len-1))
+  doAssert(nonce2 == "0A234567")
+
+main()
+
+# bug #14079
+import std/algorithm
+
+let
+  n = @["c", "b"]
+  q = @[("c", "2"), ("b", "1")]
+
+assert n.sortedByIt(it) == @["b", "c"], "fine"
+assert q.sortedByIt(it[0]) == @[("b", "1"), ("c", "2")], "fails under arc"
+
+
+# bug #14495
+type
+  Gah = ref object
+    x: string
+
+proc bug14495 =
+  var owners: seq[Gah]
+  for i in 0..10:
+    owners.add Gah(x: $i)
+
+  var x: seq[Gah]
+  for i in 0..10:
+    x.add owners[i]
+
+  for i in 0..100:
+    setLen(x, 0)
+    setLen(x, 10)
+
+  for i in 0..x.len-1:
+    if x[i] != nil:
+      echo x[i][]
+
+  for o in owners:
+    echo o[]
+
+bug14495()

tests/arc/tcaseobj.nim

@@ -8,6 +8,7 @@ A
 B
 begin
 end
+prevented
 myobj destroyed
 '''
 """
@@ -144,3 +145,51 @@ when true:
     let x = sequence([charSet({'a'..'z', 'A'..'Z', '_'})])
     echo "end"
   testSubObjAssignment()
+
+
+#------------------------------------------------
+
+type
+  MyObject = object
+    x1: string
+    case kind1: bool
+      of false: y1: string
+      of true: 
+          y2: seq[string]
+          case kind2: bool
+              of true: z1: string
+              of false: 
+                z2: seq[string]
+                flag: bool
+    x2: string
+        
+proc test_myobject = 
+  var x: MyObject
+  x.x1 = "x1"
+  x.x2 = "x2"
+  x.y1 = "ljhkjhkjh"
+  x.kind1 = true
+  x.y2 = @["1", "2"]
+  x.kind2 = true
+  x.z1 = "yes"
+  x.kind2 = false
+  x.z2 = @["1", "2"]
+  x.kind2 = true
+  x.z1 = "yes"
+  x.kind2 = true # should be no effect
+  doAssert(x.z1 == "yes")
+  x.kind2 = false
+  x.kind1 = x.kind2 # support self assignment with effect
+
+  try:
+    x.kind1 = x.flag # flag is not accesible
+  except FieldError:
+    echo "prevented"
+
+  doAssert(x.x1 == "x1")
+  doAssert(x.x2 == "x2")
+
+
+test_myobject()
+
+

tests/arc/tcontrolflow.nim

@@ -10,6 +10,7 @@ end false
 begin true
 if
 end true
+7
 '''
   cmd: "nim c --gc:arc -d:danger $file"
   disabled: "true"
@@ -53,3 +54,26 @@ proc orIsHard(cond: bool) =
 
 orIsHard(false)
 orIsHard(true)
+
+type
+  Control = ref object
+    x: int
+
+  MouseEvent = ref object
+    control: Control
+    button: int
+
+proc run(data: Control) =
+  var evt = MouseEvent(button: 1)
+  evt.control = data
+  if evt.button == 1:
+    discard
+  else: 
+    return
+  
+  echo data.x
+
+var c = Control(x: 7)
+
+run(c)
+

tests/arc/tmovebug.nim

@@ -96,4 +96,4 @@ proc procStat() =
     let cols = line.splitWhitespace(maxSplit=1)
     let x = cols[0]
     let (nm, rest) = (cols[0], cols[1])
-procStat()
+procStat()

tests/arc/twrong_sinkinference.nim

@@ -0,0 +1,18 @@
+discard """
+  cmd: "nim c --gc:arc $file"
+  errormsg: "type mismatch: got <proc (a: string, b: sink string){.noSideEffect, gcsafe, locks: 0.}>"
+  line: 18
+"""
+
+type
+  Foo = proc (a, b: string)
+
+proc take(x: Foo) =
+  x("a", "b")
+
+proc willSink(a, b: string) = # {.nosinks.} =
+  var arr: array[3, string]
+  var x = a
+  arr[0] = b
+
+take willSink

tests/async/t13889.nim

@@ -0,0 +1,27 @@
+discard """
+  output: '''
+believer Foo is saved:true
+believer Bar is saved:true
+believer Baz is saved:true
+'''
+"""
+
+import asyncdispatch
+
+var
+  promise = newFuture[bool]()
+
+proc believers(name: string) {.async.} =
+  let v = await promise
+  echo "believer " & name & " is saved:" & $v
+
+asyncCheck believers("Foo")
+asyncCheck believers("Bar")
+asyncCheck believers("Baz")
+
+proc savior() {.async.} =
+  await sleepAsync(50)
+  complete(promise, true)
+  await sleepAsync(50) # give enough time to see who was saved
+
+waitFor(savior())

tests/async/tasynceagain.nim

@@ -0,0 +1,67 @@
+discard """
+  disabled: "windows"
+  exitcode: 0
+"""
+# AsyncSocketBug.nim
+# Jens Alfke (@snej) -- 16 July 2020
+# Demonstrates data loss by Nim's AsyncSocket.
+# Just run it, and it will raise an assertion failure within a minute.
+
+import asyncdispatch, asyncnet, strformat, strutils, sugar
+
+const FrameSize = 9999   # Exact size not important, but larger sizes fail quicker
+
+proc runServer() {.async.} =
+  # Server side:
+  var server = newAsyncSocket()
+  server.bindAddr(Port(9001))
+  server.listen()
+  let client = await server.accept()
+  echo "Server got client connection"
+  var lastN = 0
+  while true:
+    let frame = await client.recv(FrameSize)
+    assert frame.len == FrameSize
+    let n = frame[0..<6].parseInt()
+    echo "RCVD #", n, ":  ", frame[0..80], "..."
+    if n != lastN + 1:
+      echo &"******** ERROR: Server received #{n}, but last was #{lastN}!"
+    assert n == lastN + 1
+    lastN = n
+    await sleepAsync 100
+
+
+proc main() {.async.} =
+  asyncCheck runServer()
+
+  # Client side:
+  let socket = newAsyncSocket(buffered = false)
+  await socket.connect("localhost", Port(9001))
+  echo "Client socket connected"
+
+  var sentCount = 0
+  var completedCount = 0
+
+  while sentCount < 2000:
+    sentCount += 1
+    let n = sentCount
+
+    var message = &"{n:06} This is message #{n} of ∞. Please stay tuned for more. "
+    #echo ">>> ", message
+    while message.len < FrameSize:
+      message = message & message
+    let frame = message[0..<FrameSize]
+
+    capture n:
+      socket.send(frame).addCallback proc(f: Future[void]) =
+        # Callback when the send completes:
+        assert not f.failed
+        echo "SENT #", n
+        if n != completedCount + 1:
+          echo &"******** ERROR: Client completed #{n}, but last completed was #{completedCount}!"
+        # If this assert is enabled, it will trigger earlier than the server-side assert above:
+        assert n == completedCount + 1
+        completedCount = n
+    await sleepAsync 1
+
+waitFor main()

tests/ccgbugs/t13902.nim

@@ -0,0 +1,12 @@
+
+#issue #13902
+block:
+  type Slot = distinct uint64
+  var s = Slot(1)
+  proc `$`(x: Slot): string {.borrow.}
+  proc `+=`(x: var Slot, y: uint64) {.borrow.}
+  # test was failing with either 0 or 2 echos but not with 1 echo
+  # echo "s = ", s
+  s += 1
+  # echo "s = ", s
+  doAssert s.uint64 == 2, $s # was failing, showing 18419607611339964418

tests/ccgbugs/tcodegenbug1.nim

@@ -3,7 +3,10 @@ discard """
 obj.inner.id = 7
 id = 7
 obj = (inner: (kind: Just, id: 7))
-2'''
+2
+(a: "a", b: "b", c: "")
+caught
+(a: "a", b: "b", c: "")'''
 """
 
 # bug #6960
@@ -129,12 +132,53 @@ import macros
 func myfunc(obj: MyObject): MyResult {.raises: [].} =
   template index: auto =
     case obj.kind:
-      of Float: $obj.index 
+      of Float: $obj.index
       of Fixed: "Fixed"
   macro to_str(a: untyped): string =
-    result = newStrLitNode(a.repr)  
+    result = newStrLitNode(a.repr)
   result.val[0] = index
   result.val[1] = to_str(obj.kind + Ola)
 
 let x = MyObject(someInt: 10, kind: Fixed)
 echo myfunc(x).val.len
+
+# bug #14126
+
+type X = object
+  a, b, c: string
+
+proc f(): X =
+  result.a = "a"
+  result.b = "b"
+  raise (ref ValueError)()
+
+proc ohmanNoNRVO =
+  var x: X
+  x.a = "1"
+  x.b = "2"
+  x.c = "3"
+
+  try:
+    x = f()
+  except:
+    discard
+
+  echo x
+  # once NVRO is sorted out, x.c == "3"
+  doAssert x.c == "", "shouldn't modify x if f raises"
+
+ohmanNoNRVO()
+
+proc ohmanNoNRVO2(x: var X) =
+  x.a = "1"
+  x.c = "3"
+  x = f()
+
+var xgg: X
+try:
+  ohmanNoNRVO2(xgg)
+except:
+  echo "caught"
+echo xgg
+# once NVRO is sorted out, xgg.c == "3"
+doAssert xgg.c == "", "this assert will fail"

tests/destructor/tcycle3.nim

@@ -2,6 +2,8 @@ discard """
   output: '''BEGIN
 END
 END 2
+cpu.nes false
+cpu step nes is nil? - false
 0'''
   cmd: '''nim c --gc:orc $file'''
 """
@@ -59,6 +61,38 @@ proc main =
   c.run
   echo "END 2"
 
+# bug #14159
+type
+  NES = ref object
+    cpu: CPU
+    apu: APU
+
+  CPU = ref object
+    nes: NES
+
+  APU = object
+    nes: NES
+    cpu: CPU
+
+proc initAPU(nes: sink NES): APU {.nosinks.} =
+  result.nes = nes
+  result.cpu = nes.cpu
+
+proc step(cpu: CPU): int =
+  echo "cpu.nes ", cpu.isNil
+  echo "cpu step nes is nil? - ", cpu.nes.isNil()
+
+proc newNES(): NES =
+  new result
+  result.cpu = CPU(nes: result)
+  result.apu = initAPU(result)
+
+proc bug14159 =
+  var nesConsole = newNES()
+  discard nesConsole.cpu.step()
+
 let mem = getOccupiedMem()
 main()
+bug14159()
+GC_fullCollect()
 echo getOccupiedMem() - mem

tests/destructor/tgotoexceptions7.nim

@@ -1,7 +1,6 @@
 discard """
   cmd: "nim c --gc:arc --exceptions:goto --panics:off $file"
-  output: '''field error prevented
-prevented!
+  output: '''prevented!
 caught
 AssertionError
 900'''
@@ -26,11 +25,8 @@ proc helper = doAssert(false)
 
 proc main(i: int) =
   var obj = Obj(kind: kindA, s: "abc")
-  try:
-    obj.kind = kindB
-  except FieldError:
-    echo "field error prevented"
-
+  obj.kind = kindB
+  obj.i = 2
   try:
     var objA = ObjA()
     bplease(ObjB(objA))

tests/destructor/tnewruntime_strutils.nim

@@ -37,7 +37,7 @@ bug12899()
 
 proc nonStaticTests =
   doAssert formatBiggestFloat(1234.567, ffDecimal, -1) == "1234.567000"
-  doAssert formatBiggestFloat(1234.567, ffDecimal, 0) == "1235" # bugs 8242, 12586
+  doAssert formatBiggestFloat(1234.567, ffDecimal, 0) == "1235." # bugs 8242, 12586
   doAssert formatBiggestFloat(1234.567, ffDecimal, 1) == "1234.6"
   doAssert formatBiggestFloat(0.00000000001, ffDecimal, 11) == "0.00000000001"
   doAssert formatBiggestFloat(0.00000000001, ffScientific, 1, ',') in

tests/lent/tnot_allowed_lent.nim

@@ -7,10 +7,10 @@ type
 
 proc mytest1(s: MyObject, i: int): lent string =
   ## works fine
-  if i < s.x.len - 1 and s.x[i] != "": 
+  if i < s.x.len - 1 and s.x[i] != "":
     result = s.x[i]
   else: raise newException(KeyError, "err1")
-  
+
 proc mytest2(s: MyObject, i: int): lent string =
   ## reject due to if expr
   if i < s.x.len - 1 and s.x[i] != "": s.x[i]