bump NimVersion to 1.2.14

When assigning constant output to a seq, and then passing that static
seq to other functions that take `openArray`, the compiler may end up
producing errors, as it does not know how to convert `static[seq[T]]`
to `openArray[T]`. By ignoring the `static` wrapper on the type for
the purpose of determining data memory location and length, this gets
resolved cleanly. Unfortunately, it is relatively tricky to come up
with a minimal example, as there are followup problems from the failing
conversion, e.g., this may lead to `internal error: inconsistent
environment type`, instead of the relevant `openArrayLoc` error message.

(cherry picked from commit 490c4226a5)

.builds/freebsd.yml

@@ -1,27 +1,21 @@
 # see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd
 image: freebsd/latest
-
-# packages:
-# - databases/sqlite3
-# - devel/boehm-gc-threaded
-# - devel/pcre
-# - devel/sdl20
-# - devel/sfml
-# - www/node
-# - devel/gmake
-# - devel/git
+packages:
+- databases/sqlite3
+- devel/boehm-gc-threaded
+- devel/pcre
+- devel/sdl20
+- devel/sfml
+- www/node
+- devel/gmake
 sources:
 - https://github.com/nim-lang/Nim
 environment:
   CC: /usr/bin/clang
 tasks:
 - setup: |
-    # workaround https://github.com/timotheecour/Nim/issues/76
-    sudo pkg update -q -f
-    sudo pkg install -y -q databases/sqlite3 devel/boehm-gc-threaded devel/pcre \
-      devel/sdl20 devel/sfml www/node devel/gmake devel/git
     cd Nim
-    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
     gmake -C csources -j $(sysctl -n hw.ncpu)
     bin/nim c --skipUserCfg --skipParentCfg koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv

.github/workflows/ci.yml.disabled

@@ -85,11 +85,11 @@ jobs:
           curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
           7z x dist/mingw64.7z -odist
           7z x dist/dlls.zip -obin
-          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
-        run: echo "::add-path::${{ github.workspace }}/bin"
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
       - name: 'Build csources'
         shell: bash

.github/workflows/ci_docs.yml

@@ -49,17 +49,17 @@ jobs:
           curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
           7z x dist/mingw64.7z -odist
           7z x dist/dlls.zip -obin
-          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
-        run: echo "::add-path::${{ github.workspace }}/bin"
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
       - name: 'Get current csources version'
         id: csources-version
         shell: bash
         run: |
-          sha=$(git ls-remote https://github.com/nim-lang/csources master | cut -f 1)
+          sha=$(git ls-remote https://github.com/nim-lang/csources_v1 master | cut -f 1)
           echo "::set-output name=sha::$sha"
 
       - name: 'Get prebuilt csources from cache'
@@ -73,7 +73,7 @@ jobs:
         if: steps.csources-cache.outputs.cache-hit != 'true'
         uses: actions/checkout@v2
         with:
-          repository: nim-lang/csources
+          repository: nim-lang/csources_v1
           path: csources
 
       - name: 'Build 1-stage compiler from csources'

.github/workflows/ci_ssl.yml

@@ -47,11 +47,11 @@ jobs:
           curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
           7z x dist/mingw64.7z -odist
           7z x dist/dlls.zip -obin
-          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
-        run: echo "::add-path::${{ github.workspace }}/bin"
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
       - name: 'Build 1-stage compiler from csources'
         shell: bash

azure-pipelines.yml

@@ -51,9 +51,10 @@ jobs:
       condition: eq(variables['Agent.OS'], 'Windows_NT')
 
     - checkout: self
+      fetchDepth: 1
 
-    - bash: git clone --depth 1 https://github.com/nim-lang/csources.git
-      displayName: 'Checkout csources'
+    - bash: git clone --depth 1 https://github.com/nim-lang/csources_v1 csources
+      displayName: 'Checkout Nim csources'
 
     - task: NodeTool@0
       inputs:
@@ -71,19 +72,12 @@ jobs:
     - bash: |
         sudo dpkg --add-architecture i386
 
-        # Downgrade llvm, libgcc and libstdc++:
+        # Downgrade llvm:
         # - llvm has to be downgraded to have 32bit version installed for sfml.
-        # - libgcc and libstdc++ have to be downgraded as an optimization to
-        #   prevent the use of the toolchain ppa, which has a terrible download
-        #   speed.
         cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
-        Package: libllvm6.0 libgcc1 libstdc++6
+        Package: libllvm6.0
         Pin: origin "azure.archive.ubuntu.com"
         Pin-Priority: 1001
-
-        Package: *
-        Pin: release o=LP-PPA-ubuntu-toolchain-r-test
-        Pin-Priority: 100
         EOF
 
         sudo apt-fast update -qq
@@ -139,8 +133,18 @@ jobs:
         make -v
       displayName: 'System information'
 
+    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
+      displayName: 'Get csources version'
+
+    - task: Cache@2
+      inputs:
+        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+        path: csources/bin
+      displayName: 'Restore built csources'
+
     - bash: |
         ncpu=
+        ext=
         case '$(Agent.OS)' in
         'Linux')
           ncpu=$(nproc)
@@ -150,19 +154,25 @@ jobs:
           ;;
         'Windows_NT')
           ncpu=$NUMBER_OF_PROCESSORS
+          ext=.exe
           ;;
         esac
         [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
 
-        make -C csources -j $ncpu CC=gcc ucpu=$(CPU)
-      displayName: 'Build csources'
+        if [[ -x csources/bin/nim$ext ]]; then
+          echo "Found cached compiler, skipping build"
+        else
+          make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
+        fi
+
+        cp csources/bin/nim$ext bin
+      displayName: 'Build 1-stage compiler from csources'
 
     - bash: nim c koch
       displayName: 'Build koch'
 
       # set result to omit the "bash exited with error code '1'" message
-    - bash: |
-        ./koch runCI || echo '##vso[task.complete result=Failed]'
+    - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
       displayName: 'Run CI'
       env:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)

build_all.bat

@@ -1,7 +1,7 @@
 @echo off
 rem build development version of the compiler; can be rerun safely
 if not exist csources (
-  git clone --depth 1 https://github.com/nim-lang/csources.git
+  git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 )
 if not exist bin\nim.exe (
   cd csources

build_all.sh

@@ -11,7 +11,7 @@ echo_run(){
   "$@"
 }
 
-[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources.git
+[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources_v1.git csources
 
 nim_csources=bin/nim_csources
 

changelog.md

@@ -3,6 +3,13 @@
 
 
 ## Standard library additions and changes
+- Added support for parenthesized expressions in `strformat`
+
+- Fixed buffer overflow bugs in `net`
+
+- Added `sections` iterator in `parsecfg`.
+
+- Make custom op in macros.quote work for all statements.
 
   For `net` and `nativesockets`, an `inheritable` flag has been added to all
   `proc`s that create sockets, allowing the user to control whether the
@@ -125,10 +132,192 @@
   Proc `rightSize` for Tables and HashSets is deprecated, as it is not needed anymore.
   `CountTable.inc` takes `val: int` again not `val: Positive`; I.e. it can "count down" again.
 - Removed deprecated symbols from `macros` module, deprecated as far back as `0.15`.
+- On Windows the SSL library now checks for valid certificates.
+  It uses the `cacert.pem` file for this purpose which was extracted
+  from `https://curl.se/ca/cacert.pem`. Besides
+  the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
+  now also need to ship `cacert.pem` with your `.exe` file.
 
 
+- Make `{.requiresInit.}` pragma to work for `distinct` types.
+
+
+- Added `asyncdispatch.activeDescriptors` that returns the number of currently
+  active async event handles/file descriptors
+- Added `asyncdispatch.maxDescriptors` that returns the maximum number of
+  active async event handles/file descriptors.
+
+- Added `getPort` to `asynchttpserver`.
+
+- `--gc:orc` is now 10% faster than previously for common workloads. If
+  you have trouble with its changed behavior, compile with `-d:nimOldOrc`.
+
+
+- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`,
+  determining preferred I/O block size for this file object.
+
+- Added a simpler to use `io.readChars` overload.
+
+- Added `**` to jsffi.
+
+- `writeStackTrace` is available in JS backend now.
+
+- Added `decodeQuery` to `std/uri`.
+
+- `strscans.scanf` now supports parsing single characters.
+
+- `strscans.scanTuple` added which uses `strscans.scanf` internally,
+  returning a tuple which can be unpacked for easier usage of `scanf`.
+
+- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`,
+  if the iterable yields a built-in settable type.
+
+- Added `setutils.fullSet` which returns a full built-in `set` for a valid type.
+
+- Added `setutils.complement` which returns the complement of a built-in `set`.
+
+- Added `setutils.[]=`.
+
+- Added `math.isNaN`.
+
+- Added `jsbigints` module, arbitrary precision integers for JavaScript target.
+
+- Added `math.copySign`.
+
+- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList`
+  and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements
+  shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes
+  its argument, `addMoved`, is also supplied.
+
+- Added `euclDiv` and `euclMod` to `math`.
+
+- Added `httpcore.is1xx` and missing HTTP codes.
+
+- Added `jsconsole.jsAssert` for JavaScript target.
+
+- Added `posix_utils.osReleaseFile` to get system identification from `os-release` file on Linux and the BSDs.
+  https://www.freedesktop.org/software/systemd/man/os-release.html
+
+- Added `socketstream` module that wraps sockets in the stream interface
+
+- Added `sugar.dumpToString` which improves on `sugar.dump`.
+
+- Added `math.signbit`.
+
+- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix` iterators (it never worked reliably)
+
+- In `lists`: renamed `append` to `add` and retained `append` as an alias;
+  added `prepend` and `prependMoved` analogously to `add` and `addMoved`;
+  added `remove` for `SinglyLinkedList`s.
+
+- Deprecated `any`. See https://github.com/nim-lang/RFCs/issues/281
+
+- Added optional `options` argument to `copyFile`, `copyFileToDir`, and
+  `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are
+  followed (copy files symlinks point to); on Windows, `options` argument is
+  ignored and symlinks are skipped.
+
+- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as
+  symlinks (instead of skipping them as it was before); on Windows symlinks are
+  skipped.
+
+- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks
+  (instead of skipping them sometimes as it was before).
+
+- Added optional `followSymlinks` argument to `setFilePermissions`.
+
+- Added `os.isAdmin` to tell whether the caller's process is a member of the
+  Administrators local group (on Windows) or a root (on POSIX).
+
+- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C).
+
+- Added `compilesettings.SingleValueSetting.libPath`.
+
+- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding
+  issues like https://github.com/nim-lang/Nim/issues/13063 (which affected error messages)
+  for modules importing `std/wrapnils`.
+  Added `??.` macro which returns an `Option`.
+
+- Added `math.frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead.
+
+- `parseopt.initOptParser` has been made available and `parseopt` has been
+  added back to `prelude` for all backends. Previously `initOptParser` was
+  unavailable if the `os` module did not have `paramCount` or `paramStr`,
+  but the use of these in `initOptParser` were conditionally to the runtime
+  arguments passed to it, so `initOptParser` has been changed to raise
+  `ValueError` when the real command line is not available. `parseopt` was
+  previously excluded from `prelude` for JS, as it could not be imported.
+
+- Added `system.prepareStrMutation` for better support of low
+  level `moveMem`, `copyMem` operations for Orc's copy-on-write string
+  implementation.
+
+- Added `std/strbasics` for high performance string operations.
+  Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`.
+
+
+- Added to `wrapnils` an option-like API via `??.`, `isSome`, `get`.
+
+- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"`
+  and `$none(int)` to `"none(int)"` instead of `"None[int]"`.
+
+- Added `algorithm.merge`.
+
+
+- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) wrapper for JavaScript target.
+
+- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) wrapper for JavaScript target.
+
+- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData) wrapper for JavaScript target.
+
+- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible
+  with most other languages.
+
+- Removed support for named procs in `sugar.=>`.
+
+- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger).
+
+- Added `std/channels`.
+
+- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals).
+
+- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time
+  zone offsets without colons, e.g. `UTC+7 -> +0700`.
+
+- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`.
+
+- Added dollar `$` and `len` for `jsre.RegExp`.
+
+- Added `std/tasks`.
+
+- Added `hasDataBuffered` to `asyncnet`.
+
+- Added `hasClosure` to `std/typetraits`.
+
+- Added `std/tempfiles`.
+
+- Added `genasts.genAst` that avoids the problems inherent with `quote do` and can
+  be used as a replacement.
+
+- Added `copyWithin` [for `seq` and `array` for JavaScript targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin).
+
+- Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place.
+
 ## Language changes
 - In newruntime it is now allowed to assign discriminator field without restrictions as long as case object doesn't have custom destructor. Discriminator value doesn't have to be a constant either. If you have custom destructor for case object and you do want to freely assign discriminator fields, it is recommended to refactor object into 2 objects like this: 
+
+- The `=destroy` hook no longer has to reset its target, as the compiler now automatically inserts
+  `wasMoved` calls where needed.
+- The `=` hook is now called `=copy` for clarity. The old name `=` is still available so there
+  is no need to update your code. This change was backported to 1.2 too so you can use the
+  more readability `=copy` without loss of compatibility.
+
+- In the newruntime it is now allowed to assign to the discriminator field
+  without restrictions as long as case object doesn't have custom destructor.
+  The discriminator value doesn't have to be a constant either. If you have a
+  custom destructor for a case object and you do want to freely assign discriminator
+  fields, it is recommended to refactor object into 2 objects like this:
+
   ```nim
   type
     MyObj = object
@@ -160,5 +349,10 @@
 ## Compiler changes
 
 
+- The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch
+  enforces that every symbol is written as it was declared, not enforcing
+  the official Nim style guide. To be enabled, this has to be combined either
+  with `--styleCheck:error` or `--styleCheck:hint`.
+
 ## Tool changes
 

ci/build.bat

@@ -2,7 +2,7 @@ REM Some debug info
 echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%."
 gcc -v
 
-git clone --depth 1 https://github.com/nim-lang/csources.git
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 cd csources
 call build64.bat
 cd ..

ci/build.sh

@@ -1,7 +1,7 @@
 sh ci/deps.sh
 
 # Build from C sources.
-git clone --depth 1 https://github.com/nim-lang/csources.git
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 cd csources
 sh build.sh
 cd ..

ci/nsis_build.bat

@@ -25,7 +25,7 @@ Rem Build csources
 koch csources -d:release || exit /b
 
 rem Grab C sources and nimsuggest
-git clone --depth 1 https://github.com/nim-lang/csources.git
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 
 set PATH=%CD%\bin;%PATH%
 

compiler/ast.nim

@@ -1085,7 +1085,7 @@ proc safeLen*(n: PNode): int {.inline.} =
 
 proc safeArrLen*(n: PNode): int {.inline.} =
   ## works for array-like objects (strings passed as openArray in VM).
-  if n.kind in {nkStrLit..nkTripleStrLit}:result = n.strVal.len
+  if n.kind in {nkStrLit..nkTripleStrLit}: result = n.strVal.len
   elif n.kind in {nkNone..nkFloat128Lit}: result = 0
   else: result = n.len
 
@@ -1327,7 +1327,7 @@ const
   MaxLockLevel* = 1000'i16
   UnknownLockLevel* = TLockLevel(1001'i16)
   AttachedOpToStr*: array[TTypeAttachedOp, string] = [
-    "=destroy", "=", "=sink", "=trace", "=dispose", "=deepcopy"]
+    "=destroy", "=copy", "=sink", "=trace", "=dispose", "=deepcopy"]
 
 proc `$`*(x: TLockLevel): string =
   if x.ord == UnspecifiedLockLevel.ord: result = "<unspecified>"

compiler/ccgcalls.nim

@@ -189,7 +189,7 @@ proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
       internalError(p.config, "openArrayLoc: " & typeToString(a.t))
   else:
     initLocExpr(p, n, a)
-    case skipTypes(a.t, abstractVar).kind
+    case skipTypes(a.t, abstractVar+{tyStatic}).kind
     of tyOpenArray, tyVarargs:
       result = "$1, $1Len_0" % [rdLoc(a)]
     of tyString, tySequence:

compiler/ccgexprs.nim

@@ -833,6 +833,7 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) =
               [rdLoc(test), strLit, raiseInstr(p)])
 
 proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) =
+  assert e[0].kind == nkDotExpr
   if optFieldCheck in p.options:
     var a: TLoc
     genRecordFieldAux(p, e[0], d, a)
@@ -867,7 +868,7 @@ proc genArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   if optBoundsCheck in p.options and ty.kind != tyUncheckedArray:
     if not isConstExpr(y):
       # semantic pass has already checked for const index expressions
-      if firstOrd(p.config, ty) == 0:
+      if firstOrd(p.config, ty) == 0 and lastOrd(p.config, ty) >= 0:
         if (firstOrd(p.config, b.t) < firstOrd(p.config, ty)) or (lastOrd(p.config, b.t) > lastOrd(p.config, ty)):
           linefmt(p, cpsStmts, "if ((NU)($1) > (NU)($2)){ #raiseIndexError2($1, $2); $3}$n",
                   [rdCharLoc(b), intLiteral(lastOrd(p.config, ty)), raiseInstr(p)])
@@ -1781,9 +1782,9 @@ template binaryExprIn(p: BProc, e: PNode, a, b, d: var TLoc, frmt: string) =
 
 proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc) =
   case int(getSize(p.config, skipTypes(e[1].typ, abstractVar)))
-  of 1: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&7U)))!=0)")
-  of 2: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&15U)))!=0)")
-  of 4: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&31U)))!=0)")
+  of 1: binaryExprIn(p, e, a, b, d, "(($1 &((NU8)1<<((NU)($2)&7U)))!=0)")
+  of 2: binaryExprIn(p, e, a, b, d, "(($1 &((NU16)1<<((NU)($2)&15U)))!=0)")
+  of 4: binaryExprIn(p, e, a, b, d, "(($1 &((NU32)1<<((NU)($2)&31U)))!=0)")
   of 8: binaryExprIn(p, e, a, b, d, "(($1 &((NU64)1<<((NU)($2)&63U)))!=0)")
   else: binaryExprIn(p, e, a, b, d, "(($1[(NU)($2)>>3] &(1U<<((NU)($2)&7U)))!=0)")
 
@@ -2941,7 +2942,23 @@ proc genConstSeqV2(p: BProc, n: PNode, t: PType; isConst: bool): Rope =
 proc genBracedInit(p: BProc, n: PNode; isConst: bool): Rope =
   case n.kind
   of nkHiddenStdConv, nkHiddenSubConv:
-    result = genBracedInit(p, n[1], isConst)
+    when false:
+      # XXX The frontend doesn't keep conversions to openArray for us. :-(
+      # We need to change 'transformConv' first, but that is hard.
+      if n.typ.kind == tyOpenArray:
+        assert n[1].kind == nkBracket
+        let data = genBracedInit(p, n[1], isConst)
+
+        let payload = getTempName(p.module)
+        let ctype = getTypeDesc(p.module, n.typ.skipTypes(abstractInst)[0])
+        let arrLen = n[1].len
+        appcg(p.module, cfsData,
+          "static $5 $1 $3[$2] = $4;$n", [
+          ctype, arrLen, payload, data,
+          if isConst: "const" else: ""])
+        result = "{($1*)&$2, $3}" % [ctype, payload, rope arrLen]
+    else:
+      result = genBracedInit(p, n[1], isConst)
   else:
     var ty = tyNone
     if n.typ == nil:

compiler/ccgstmts.nim

@@ -25,10 +25,10 @@ proc getTraverseProc(p: BProc, v: PSym): Rope =
 
 proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) =
   if sfThread in v.flags:
-    appcg(p.module, p.module.initProc.procSec(cpsInit),
+    appcg(p.module, p.module.preInitProc.procSec(cpsInit),
       "$n\t#nimRegisterThreadLocalMarker($1);$n$n", [traverseProc])
   else:
-    appcg(p.module, p.module.initProc.procSec(cpsInit),
+    appcg(p.module, p.module.preInitProc.procSec(cpsInit),
       "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc])
 
 proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} =

compiler/cgen.nim

@@ -244,7 +244,7 @@ proc safeLineNm(info: TLineInfo): int =
 
 proc genCLineDir(r: var Rope, filename: string, line: int; conf: ConfigRef) =
   assert line >= 0
-  if optLineDir in conf.options:
+  if optLineDir in conf.options and line > 0:
     r.addf("$N#line $2 $1$N",
         [rope(makeSingleLineCString(filename)), rope(line)])
 
@@ -1645,7 +1645,7 @@ proc genInitCode(m: BModule) =
     writeSection(preInitProc, cpsLocals)
     writeSection(preInitProc, cpsInit, m.hcrOn)
     writeSection(preInitProc, cpsStmts)
-    prc.addf("}$N", [])
+    prc.addf("}/* preInitProc end */$N", [])
 
   # add new scope for following code, because old vcc compiler need variable
   # be defined at the top of the block

compiler/closureiters.nim

@@ -130,7 +130,8 @@
 
 import
   ast, msgs, idents,
-  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
+  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos,
+  tables, options
 
 type
   Ctx = object
@@ -1102,7 +1103,7 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
 
   result = ctx.states[stateIdx][0].intVal.int
 
-proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
+proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
   result = n
   case n.kind
   of nkSkip:
@@ -1282,6 +1283,101 @@ proc deleteEmptyStates(ctx: var Ctx) =
     else:
       inc i
 
+type
+  PreprocessContext = object
+    finallys: seq[PNode]
+    config: ConfigRef
+    blocks: seq[(PNode, int)]
+  FreshVarsContext = object
+    tab: Table[int, PSym]
+    config: ConfigRef
+    info: TLineInfo
+
+proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
+  case n.kind
+  of nkSym:
+    let x = c.tab.getOrDefault(n.sym.id)
+    if x == nil:
+      result = n
+    else:
+      result = newSymNode(x, n.info)
+  of nkSkip - {nkSym}:
+    result = n
+  of nkLetSection, nkVarSection:
+    result = copyNode(n)
+    for it in n:
+      if it.kind in {nkIdentDefs, nkVarTuple}:
+        let idefs = copyNode(it)
+        for v in 0..it.len-3:
+          if it[v].kind == nkSym:
+            let x = copySym(it[v].sym)
+            c.tab[it[v].sym.id] = x
+            idefs.add newSymNode(x)
+          else:
+            idefs.add it[v]
+
+        for rest in it.len-2 ..< it.len: idefs.add it[rest]
+        result.add idefs
+      else:
+        result.add it
+  of nkRaiseStmt:
+    localError(c.config, c.info, "unsupported control flow: 'finally: ... raise' duplicated because of 'break'")
+  else:
+    result = n
+    for i in 0..<n.safeLen:
+      result[i] = freshVars(n[i], c)
+
+proc preprocess(c: var PreprocessContext; n: PNode): PNode =
+  # in order to fix bug #15243 without risking regressions, we preprocess
+  # the AST so that 'break' statements inside a 'try finally' also have the
+  # finally section. We need to duplicate local variables here and also
+  # detect: 'finally: raises X' which is currently not supported. We produce
+  # an error for this case for now. All this will be done properly with Yuriy's
+  # patch.
+  result = n
+  case n.kind
+  of nkTryStmt:
+    let f = n.lastSon
+    if f.kind == nkFinally:
+      c.finallys.add f.lastSon
+
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
+    if f.kind == nkFinally:
+      discard c.finallys.pop()
+
+  of nkWhileStmt, nkBlockStmt:
+    c.blocks.add((n, c.finallys.len))
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+    discard c.blocks.pop()
+
+  of nkBreakStmt:
+    if c.blocks.len == 0:
+      discard
+    else:
+      var fin = -1
+      if n[0].kind == nkEmpty:
+        fin = c.blocks[^1][1]
+      elif n[0].kind == nkSym:
+        for i in countdown(c.blocks.high, 0):
+          if c.blocks[i][0].kind == nkBlockStmt and c.blocks[i][0][0].kind == nkSym and
+              c.blocks[i][0][0].sym == n[0].sym:
+            fin = c.blocks[i][1]
+            break
+
+      if fin >= 0:
+        result = newNodeI(nkStmtList, n.info)
+        for i in countdown(c.finallys.high, fin):
+          var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info)
+          result.add freshVars(preprocess(c, c.finallys[i]), vars)
+        result.add n
+  of nkSkip: discard
+  else:
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
 proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   var ctx: Ctx
   ctx.g = g
@@ -1294,7 +1390,10 @@ proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
     ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), fn, fn.info)
     ctx.stateVarSym.typ = g.createClosureIterStateType(fn)
   ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), fn, fn.info)
-  var n = n.toStmtList
+  var pc = PreprocessContext(finallys: @[], config: g.config)
+  var n = preprocess(pc, n.toStmtList)
+  #echo "transformed into ", n
+  #var n = n.toStmtList
 
   discard ctx.newState(n, nil)
   let gotoOut = newTree(nkGotoState, g.newIntLit(n.info, -1))

compiler/commands.nim

@@ -851,6 +851,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "off": conf.globalOptions = conf.globalOptions - {optStyleHint, optStyleError}
     of "hint": conf.globalOptions = conf.globalOptions + {optStyleHint} - {optStyleError}
     of "error": conf.globalOptions = conf.globalOptions + {optStyleError}
+    of "usages": conf.globalOptions.incl optStyleUsages
     else: localError(conf, info, errOffHintsError % arg)
   of "showallmismatches":
     processOnOffSwitchG(conf, {optShowAllMismatches}, arg, pass, info)

compiler/condsyms.nim

@@ -116,3 +116,5 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimNewIntegerOps")
   defineSymbol("nimHasInvariant")
   defineSymbol("nimHasStacktraceMsgs")
+  defineSymbol("nimHasStacktracesModule")
+  defineSymbol("nimHasJsBigIntBackend")

compiler/injectdestructors.nim

@@ -212,7 +212,7 @@ template isUnpackedTuple(n: PNode): bool =
 
 proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
   var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">"
-  if opname == "=" and ri != nil:
+  if (opname == "=" or opname == "=copy") and ri != nil:
     m.add "; requires a copy because it's not the last read of '"
     m.add renderTree(ri)
     m.add '\''
@@ -301,7 +301,7 @@ proc genCopy(c: var Con; dest, ri: PNode): PNode =
   if tfHasOwned in t.flags and ri.kind != nkNilLit:
     # try to improve the error message here:
     if c.otherRead == nil: discard isLastRead(ri, c)
-    checkForErrorPragma(c, t, ri, "=")
+    checkForErrorPragma(c, t, ri, "=copy")
   result = genCopyNoCheck(c, dest, ri)
 
 proc addTopVar(c: var Con; v: PNode) =

compiler/installer.ini

@@ -6,12 +6,12 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv64
-  macosx: i386;amd64;powerpc64
+  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv32;riscv64
+  macosx: i386;amd64;powerpc64;arm64
   solaris: i386;amd64;sparc;sparc64
-  freebsd: i386;amd64;powerpc64;arm;arm64
+  freebsd: i386;amd64;powerpc64;arm;arm64;riscv64;sparc64;mips;mipsel;mips64;mips64el;powerpc;powerpc64el
   netbsd: i386;amd64
-  openbsd: i386;amd64
+  openbsd: i386;amd64;arm;arm64
   dragonfly: i386;amd64
   haiku: i386;amd64
   android: i386;arm;arm64
@@ -77,6 +77,7 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/fusion"
 
 Files: "tests"
 
@@ -110,6 +111,7 @@ Download: r"Aporia Text Editor|dist|aporia.zip|97997|https://nim-lang.org/downlo
 Files: "bin/makelink.exe"
 Files: "bin/7zG.exe"
 Files: "bin/*.dll"
+Files: "bin/cacert.pem"
 
 [UnixBin]
 Files: "bin/nim"

compiler/jsgen.nim

@@ -1591,7 +1591,10 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   var t = skipTypes(typ, abstractInst)
   case t.kind
   of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar:
-    result = putToSeq("0", indirect)
+    if $t.sym.loc.r == "bigint":
+      result = putToSeq("0n", indirect)
+    else:
+      result = putToSeq("0", indirect)
   of tyFloat..tyFloat128:
     result = putToSeq("0.0", indirect)
   of tyRange, tyGenericInst, tyAlias, tySink, tyOwned:
@@ -2124,11 +2127,17 @@ proc genConv(p: PProc, n: PNode, r: var TCompRes) =
   if dest.kind == src.kind:
     # no-op conversion
     return
-  case dest.kind:
-  of tyBool:
+  let toInt = (dest.kind in tyInt..tyInt32)
+  let fromInt = (src.kind in tyInt..tyInt32)
+  let toUint = (dest.kind in tyUInt..tyUInt32)
+  let fromUint = (src.kind in tyUInt..tyUInt32)
+  if toUint and (fromInt or fromUint):
+    let trimmer = unsignedTrimmer(dest.size)
+    r.res = "($1 $2)" % [r.res, trimmer]
+  elif dest.kind == tyBool:
     r.res = "(!!($1))" % [r.res]
     r.kind = resExpr
-  of tyInt:
+  elif toInt:
     r.res = "(($1)|0)" % [r.res]
   else:
     # TODO: What types must we handle here?

compiler/lexer.nim

@@ -62,6 +62,7 @@ type
     tkComma, tkSemiColon,
     tkColon, tkColonColon, tkEquals, tkDot, tkDotDot, tkBracketLeColon,
     tkOpr, tkComment, tkAccent,
+    # these are fake tokens used by renderer.nim
     tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
 
   TTokTypes* = set[TTokType]

compiler/lineinfos.nim

@@ -44,6 +44,7 @@ type
     warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed,
     warnInconsistentSpacing, warnCaseTransition, warnCycleCreated,
     warnObservableStores,
+    warnResultUsed,
     warnUser,
     hintSuccess, hintSuccessX, hintCC,
     hintLineTooLong, hintXDeclaredButNotUsed,
@@ -110,6 +111,7 @@ const
     warnCaseTransition: "Potential object case transition, instantiate new object instead",
     warnCycleCreated: "$1",
     warnObservableStores: "observable stores to '$1'",
+    warnResultUsed: "used 'result' variable",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
@@ -160,7 +162,7 @@ const
     "IndexCheck", "GcUnsafe", "GcUnsafe2", "Uninit",
     "GcMem", "Destructor", "LockLevel", "ResultShadowed",
     "Spacing", "CaseTransition", "CycleCreated",
-    "ObservableStores", "User"]
+    "ObservableStores", "ResultUsed", "User"]
 
   HintsToStr* = [
     "Success", "SuccessX", "CC", "LineTooLong",
@@ -191,7 +193,7 @@ type
   TNoteKinds* = set[TNoteKind]
 
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
-  result[3] = {low(TNoteKind)..high(TNoteKind)} - {}
+  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnResultUsed}
   result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext}
   result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,

compiler/linter.nim

@@ -101,7 +101,7 @@ proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
     lintReport(conf, info, beau, s.name.s)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
-  if {optStyleHint, optStyleError} * conf.globalOptions != {}:
+  if {optStyleHint, optStyleError} * conf.globalOptions != {} and optStyleUsages notin conf.globalOptions:
     nep1CheckDefImpl(conf, info, s, k)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) =

compiler/nim.cfg

@@ -19,5 +19,6 @@ define:useStdoutAsStdmsg
   styleCheck:error
 @end
 
+
 #define:useNodeIds
 #gc:markAndSweep

compiler/options.nim

@@ -59,6 +59,7 @@ type                          # please make sure we have under 32 options
     optRun,                   # run the compiled project
     optStyleHint,             # check that the names adhere to NEP-1
     optStyleError,            # enforce that the names adhere to NEP-1
+    optStyleUsages,           # only enforce consistent **usages** of the symbol
     optSkipSystemConfigFile,  # skip the system's cfg/nims config file
     optSkipProjConfigFile,    # skip the project's cfg/nims config file
     optSkipUserConfigFile,    # skip the users's cfg/nims config file
@@ -309,7 +310,7 @@ proc setNote*(conf: ConfigRef, note: TNoteKind, enabled = true) =
 proc hasHint*(conf: ConfigRef, note: TNoteKind): bool =
   optHints in conf.options and note in conf.notes
 
-proc hasWarn*(conf: ConfigRef, note: TNoteKind): bool =
+proc hasWarn*(conf: ConfigRef, note: TNoteKind): bool {.inline.} =
   optWarns in conf.options and note in conf.notes
 
 proc hcrOn*(conf: ConfigRef): bool = return optHotCodeReloading in conf.globalOptions

compiler/platform.nim

@@ -22,7 +22,7 @@ type
     osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris,
     osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osAix, osPalmos, osQnx,
     osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks
-    osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osAny
+    osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osAny
 
 type
   TInfoOSProp* = enum
@@ -177,6 +177,10 @@ const
       objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
       scriptExt: ".sh", curDir: ".", exeExt: ".elf", extSep: ".",
       props: {ospNeedsPIC, ospPosix}),
+     (name: "FreeRTOS", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
+      objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
+      scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
+      props: {ospPosix}),
      (name: "Any", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
       objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
       scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
@@ -189,7 +193,7 @@ type
     cpuNone, cpuI386, cpuM68k, cpuAlpha, cpuPowerpc, cpuPowerpc64,
     cpuPowerpc64el, cpuSparc, cpuVm, cpuHppa, cpuIa64, cpuAmd64, cpuMips,
     cpuMipsel, cpuArm, cpuArm64, cpuJS, cpuNimVM, cpuAVR, cpuMSP430,
-    cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV64, cpuWasm32
+    cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64, cpuEsp, cpuWasm32
 
 type
   TEndian* = enum
@@ -222,7 +226,9 @@ const
     (name: "sparc64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64),
     (name: "mips64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64),
     (name: "mips64el", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
+    (name: "riscv32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
     (name: "riscv64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
+    (name: "esp", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
     (name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32)]
 
 type

compiler/pragmas.nim

@@ -20,7 +20,7 @@ const
 
 const
   declPragmas = {wImportc, wImportObjC, wImportCpp, wImportJs, wExportc, wExportCpp,
-    wExportNims, wExtern, wDeprecated, wNodecl, wError, wUsed, wAlign}
+    wExportNims, wExtern, wDeprecated, wNodecl, wError, wUsed}
     ## common pragmas for declarations, to a good approximation
   procPragmas* = declPragmas + {FirstCallConv..LastCallConv,
     wMagic, wNoSideEffect, wSideEffect, wNoreturn, wNosinks, wDynlib, wHeader,
@@ -54,22 +54,22 @@ const
     wFloatChecks, wInfChecks, wNanChecks, wPragma, wEmit, wUnroll,
     wLinearScanEnd, wPatterns, wTrMacros, wEffects, wNoForward, wReorder, wComputedGoto,
     wInjectStmt, wExperimental, wThis, wUsed, wInvariant, wAssume}
-  lambdaPragmas* = declPragmas + {FirstCallConv..LastCallConv,
+  lambdaPragmas* = {FirstCallConv..LastCallConv,
     wNoSideEffect, wSideEffect, wNoreturn, wNosinks, wDynlib, wHeader,
     wThread, wAsmNoStackFrame,
     wRaises, wLocks, wTags, wRequires, wEnsures,
-    wGcSafe, wCodegenDecl} - {wExportNims, wError, wUsed}  # why exclude these?
+    wGcSafe, wCodegenDecl, wNoInit}
   typePragmas* = declPragmas + {wMagic, wAcyclic,
     wPure, wHeader, wCompilerProc, wCore, wFinal, wSize, wShallow,
     wIncompleteStruct, wByCopy, wByRef,
     wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked,
     wBorrow, wGcSafe, wPartial, wExplain, wPackage}
-  fieldPragmas* = declPragmas + {
-    wGuard, wBitsize, wCursor, wRequiresInit} - {wExportNims, wNodecl} # why exclude these?
+  fieldPragmas* = declPragmas + {wGuard, wBitsize, wCursor,
+    wRequiresInit, wAlign} - {wExportNims, wNodecl} # why exclude these?
   varPragmas* = declPragmas + {wVolatile, wRegister, wThreadVar,
     wMagic, wHeader, wCompilerProc, wCore, wDynlib,
     wNoInit, wCompileTime, wGlobal,
-    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wCursor}
+    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wCursor, wAlign}
   constPragmas* = declPragmas + {wHeader, wMagic,
     wGensym, wInject,
     wIntDefine, wStrDefine, wBoolDefine, wCompilerProc, wCore}

compiler/renderer.nim

@@ -31,6 +31,7 @@ type
   TSrcGen* = object
     indent*: int
     lineLen*: int
+    col: int
     pos*: int              # current position for iteration over the buffer
     idx*: int              # current token index for iteration over the buffer
     tokens*: TRenderTokSeq
@@ -109,11 +110,10 @@ proc initSrcGen(g: var TSrcGen, renderFlags: TRenderFlags; config: ConfigRef) =
   g.config = config
 
 proc addTok(g: var TSrcGen, kind: TTokType, s: string; sym: PSym = nil) =
-  setLen(g.tokens, g.tokens.len + 1)
-  g.tokens[^1].kind = kind
-  g.tokens[^1].length = int16(s.len)
-  g.tokens[^1].sym = sym
+  g.tokens.add TRenderTok(kind: kind, length: int16(s.len), sym: sym)
   g.buf.add(s)
+  if kind != tkSpaces:
+    inc g.col, s.len
 
 proc addPendingNL(g: var TSrcGen) =
   if g.pendingNL >= 0:
@@ -123,6 +123,7 @@ proc addPendingNL(g: var TSrcGen) =
       const newlines = "\n"
     addTok(g, tkSpaces, newlines & spaces(g.pendingNL))
     g.lineLen = g.pendingNL
+    g.col = g.pendingNL
     g.pendingNL = - 1
     g.pendingWhitespace = -1
   elif g.pendingWhitespace >= 0:
@@ -131,7 +132,10 @@ proc addPendingNL(g: var TSrcGen) =
 
 proc putNL(g: var TSrcGen, indent: int) =
   if g.pendingNL >= 0: addPendingNL(g)
-  else: addTok(g, tkSpaces, "\n")
+  else:
+    addTok(g, tkSpaces, "\n")
+    g.col = 0
+
   g.pendingNL = indent
   g.lineLen = indent
   g.pendingWhitespace = -1
@@ -173,16 +177,17 @@ proc put(g: var TSrcGen, kind: TTokType, s: string; sym: PSym = nil) =
     addPendingNL(g)
     if s.len > 0:
       addTok(g, kind, s, sym)
-      inc(g.lineLen, s.len)
   else:
     g.pendingWhitespace = s.len
+    inc g.col, s.len
+  inc(g.lineLen, s.len)
 
 proc putComment(g: var TSrcGen, s: string) =
   if s.len == 0: return
   var i = 0
   let hi = s.len - 1
-  var isCode = (s.len >= 2) and (s[1] != ' ')
-  var ind = g.lineLen
+  let isCode = (s.len >= 2) and (s[1] != ' ')
+  let ind = g.col
   var com = "## "
   while i <= hi:
     case s[i]
@@ -208,7 +213,7 @@ proc putComment(g: var TSrcGen, s: string) =
       # compute length of the following word:
       var j = i
       while j <= hi and s[j] > ' ': inc(j)
-      if not isCode and (g.lineLen + (j - i) > MaxLineLen):
+      if not isCode and (g.col + (j - i) > MaxLineLen):
         put(g, tkComment, com)
         optNL(g, ind)
         com = "## "
@@ -290,15 +295,18 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool =
 proc gcom(g: var TSrcGen, n: PNode) =
   assert(n != nil)
   if shouldRenderComment(g, n):
+    var oneSpaceAdded = 0
     if (g.pendingNL < 0) and (g.buf.len > 0) and (g.buf[^1] != ' '):
       put(g, tkSpaces, Space)
+      oneSpaceAdded = 1
       # Before long comments we cannot make sure that a newline is generated,
       # because this might be wrong. But it is no problem in practice.
     if (g.pendingNL < 0) and (g.buf.len > 0) and
-        (g.lineLen < LineCommentColumn):
+        (g.col < LineCommentColumn):
       var ml = maxLineLength(n.comment)
       if ml + LineCommentColumn <= MaxLineLen:
-        put(g, tkSpaces, spaces(LineCommentColumn - g.lineLen))
+        put(g, tkSpaces, spaces(LineCommentColumn - g.col))
+        dec g.col, oneSpaceAdded
     putComment(g, n.comment)  #assert(g.comStack[high(g.comStack)] = n);
 
 proc gcoms(g: var TSrcGen) =

compiler/semexprs.nim

@@ -1207,6 +1207,9 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     # not sure the symbol really ends up being used:
     # var len = 0 # but won't be called
     # genericThatUsesLen(x) # marked as taking a closure?
+    if hasWarn(c.config, warnResultUsed):
+      message(c.config, n.info, warnResultUsed)
+
   of skGenericParam:
     onUse(n.info, s)
     if s.typ.kind == tyStatic:

compiler/semfields.nim

@@ -23,7 +23,7 @@ proc instFieldLoopBody(c: TFieldInstCtx, n: PNode, forLoop: PNode): PNode =
     result = newNode(nkEmpty)
     return
   case n.kind
-  of nkEmpty..pred(nkIdent), succ(nkSym)..nkNilLit: result = n
+  of nkEmpty..pred(nkIdent), succ(nkSym)..nkNilLit: result = copyNode(n)
   of nkIdent, nkSym:
     result = n
     let ident = considerQuotedIdent(c.c, n)
@@ -52,8 +52,7 @@ proc instFieldLoopBody(c: TFieldInstCtx, n: PNode, forLoop: PNode): PNode =
     if n.kind == nkContinueStmt:
       localError(c.c.config, n.info,
                  "'continue' not supported in a 'fields' loop")
-    result = copyNode(n)
-    newSons(result, n.len)
+    result = shallowCopy(n)
     for i in 0..<n.len:
       result[i] = instFieldLoopBody(c, n[i], forLoop)
 

compiler/semfold.nim

@@ -715,6 +715,9 @@ proc getConstExpr(m: PSym, n: PNode; g: ModuleGraph): PNode =
       result.typ = n.typ
   of nkBracketExpr: result = foldArrayAccess(m, n, g)
   of nkDotExpr: result = foldFieldAccess(m, n, g)
+  of nkCheckedFieldExpr:
+    assert n[0].kind == nkDotExpr
+    result = foldFieldAccess(m, n[0], g)
   of nkStmtListExpr:
     var i = 0
     while i <= n.len - 2:

compiler/sempass2.nim

@@ -785,7 +785,7 @@ proc track(tracked: PEffects, n: PNode) =
     if n.sym.typ != nil and tfHasAsgn in n.sym.typ.flags:
       tracked.owner.flags.incl sfInjectDestructors
       # bug #15038: ensure consistency
-      if not hasDestructor(n.typ): n.typ = n.sym.typ
+      if not hasDestructor(n.typ) and sameType(n.typ, n.sym.typ): n.typ = n.sym.typ
   of nkRaiseStmt:
     if n[0].kind != nkEmpty:
       n[0].info = n.info

compiler/semstmts.nim

@@ -161,7 +161,7 @@ proc semIf(c: PContext, n: PNode; flags: TExprFlags): PNode =
     if it.len == 2:
       openScope(c)
       it[0] = forceBool(c, semExprWithType(c, it[0]))
-      it[1] = semExprBranch(c, it[1])
+      it[1] = semExprBranch(c, it[1], flags)
       typ = commonType(typ, it[1])
       closeScope(c)
     elif it.len == 1:
@@ -722,7 +722,7 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode =
   result = n
   let iterBase = n[^2].typ
   var iter = skipTypes(iterBase, {tyGenericInst, tyAlias, tySink, tyOwned})
-  var iterAfterVarLent = iter.skipTypes({tyLent, tyVar})
+  var iterAfterVarLent = iter.skipTypes({tyGenericInst, tyAlias, tyLent, tyVar})
   # n.len == 3 means that there is one for loop variable
   # and thus no tuple unpacking:
   if iterAfterVarLent.kind != tyTuple or n.len == 3:
@@ -1748,7 +1748,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
                  "signature for 'deepCopy' must be proc[T: ptr|ref](x: T): T")
     incl(s.flags, sfUsed)
     incl(s.flags, sfOverriden)
-  of "=", "=sink":
+  of "=", "=copy", "=sink":
     if s.magic == mAsgn: return
     incl(s.flags, sfUsed)
     incl(s.flags, sfOverriden)
@@ -1770,7 +1770,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
         # attach these ops to the canonical tySequence
         obj = canonType(c, obj)
         #echo "ATTACHING TO ", obj.id, " ", s.name.s, " ", cast[int](obj)
-        let k = if name == "=": attachedAsgn else: attachedSink
+        let k = if name == "=" or name == "=copy": attachedAsgn else: attachedSink
         if obj.attachedOps[k] == s:
           discard "forward declared op"
         elif obj.attachedOps[k].isNil and tfCheckedForDestructor notin obj.flags:

compiler/sigmatch.nim

@@ -1086,7 +1086,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
       let x = typeRel(c, f, branch, flags + {trDontBind})
       if x == isNone: return isNone
       if x < result: result = x
-    return
+    return result
 
   of tyAnd:
     # XXX: deal with the current dual meaning of tyGenericParam
@@ -1421,7 +1421,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
           let ff = rootf[i]
           let aa = roota[i]
           let res = typeRel(c, ff, aa, nextFlags)
-          if res != isEqual: result = isGeneric
+          if res != isNone and res != isEqual: result = isGeneric
           if res notin {isEqual, isGeneric}:
             if trNoCovariance notin flags and ff.kind == aa.kind:
               let paramFlags = rootf.base[i-1].flags

compiler/sizealignoffsetimpl.nim

@@ -138,12 +138,13 @@ proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode, packed: bool, a
       accum.offset  = szUnknownSize
       accum.maxAlign = szUnknownSize
     else:
-      # the union neds to be aligned first, before the offsets can be assigned
+      # the union needs to be aligned first, before the offsets can be assigned
       accum.align(maxChildAlign)
       let accumRoot = accum # copy, because each branch should start af the same offset
       for i in 1..<n.len:
-        var branchAccum = accumRoot
+        var branchAccum = OffsetAccum(offset: accumRoot.offset, maxAlign: 1)
         computeObjectOffsetsFoldFunction(conf, n[i].lastSon, packed, branchAccum)
+        discard finish(branchAccum)
         accum.mergeBranch(branchAccum)
   of nkRecList:
     for i, child in n.sons:
@@ -173,9 +174,10 @@ proc computeUnionObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode; accum: var
     localError(conf, n.info, "Illegal use of ``case`` in union type.")
   of nkRecList:
     let accumRoot = accum # copy, because each branch should start af the same offset
-    for i, child in n.sons:
-      var branchAccum = accumRoot
+    for child in n.sons:
+      var branchAccum = OffsetAccum(offset: accumRoot.offset, maxAlign: 1)
       computeUnionObjectOffsetsFoldFunction(conf, child, branchAccum)
+      discard finish(branchAccum)
       accum.mergeBranch(branchAccum)
   of nkSym:
     var size = szUnknownSize

compiler/transf.nim

@@ -830,13 +830,6 @@ proc transformExceptBranch(c: PTransf, n: PNode): PNode =
   else:
     result = transformSons(c, n)
 
-proc dontInlineConstant(orig, cnst: PNode): bool {.inline.} =
-  # symbols that expand to a complex constant (array, etc.) should not be
-  # inlined, unless it's the empty array:
-  result = orig.kind == nkSym and
-           cnst.kind in {nkCurly, nkPar, nkTupleConstr, nkBracket} and
-           cnst.len != 0
-
 proc commonOptimizations*(g: ModuleGraph; c: PSym, n: PNode): PNode =
   result = n
   for i in 0..<n.safeLen:
@@ -1035,6 +1028,11 @@ proc transform(c: PTransf, n: PNode): PNode =
     return n
   of nkExceptBranch:
     result = transformExceptBranch(c, n)
+  of nkCheckedFieldExpr:
+    result = transformSons(c, n)
+    if result[0].kind != nkDotExpr:
+      # simplfied beyond a dot expression --> simplify further.
+      result = result[0]
   else:
     result = transformSons(c, n)
   when false:
@@ -1090,15 +1088,15 @@ proc liftDeferAux(n: PNode) =
         if n[i].kind == nkDefer:
           let deferPart = newNodeI(nkFinally, n[i].info)
           deferPart.add n[i][0]
-          var tryStmt = newNodeI(nkTryStmt, n[i].info)
-          var body = newNodeI(n.kind, n[i].info)
+          var tryStmt = newNodeIT(nkTryStmt, n[i].info, n.typ)
+          var body = newNodeIT(n.kind, n[i].info, n.typ)
           if i < last:
             body.sons = n.sons[(i+1)..last]
           tryStmt.add body
           tryStmt.add deferPart
           n[i] = tryStmt
           n.sons.setLen(i+1)
-          n.typ = n[i].typ
+          n.typ = tryStmt.typ
           goOn = true
           break
   for i in 0..n.safeLen-1:

compiler/trees.nim

@@ -169,3 +169,10 @@ proc flattenStmts*(n: PNode): PNode =
 proc extractRange*(k: TNodeKind, n: PNode, a, b: int): PNode =
   result = newNodeI(k, n.info, b-a+1)
   for i in 0..b-a: result[i] = n[i+a]
+
+proc dontInlineConstant*(orig, cnst: PNode): bool {.inline.} =
+  # symbols that expand to a complex constant (array, etc.) should not be
+  # inlined, unless it's the empty array:
+  result = orig.kind != cnst.kind and
+           cnst.kind in {nkCurly, nkPar, nkTupleConstr, nkBracket, nkObjConstr} and
+           cnst.len > ord(cnst.kind == nkObjConstr)

compiler/types.nim

@@ -71,7 +71,7 @@ const
                tyInferred, tySink, tyLent, tyOwned}
   # typedescX is used if we're sure tyTypeDesc should be included (or skipped)
   typedescPtrs* = abstractPtrs + {tyTypeDesc}
-  typedescInst* = abstractInst + {tyTypeDesc, tyOwned}
+  typedescInst* = abstractInst + {tyTypeDesc, tyOwned, tyUserTypeClass}
 
 proc invalidGenericInst*(f: PType): bool =
   result = f.kind == tyGenericInst and lastSon(f) == nil

compiler/vm.nim

@@ -2086,6 +2086,7 @@ proc execute(c: PCtx, start: int): PNode =
   result = rawExecute(c, start, tos).regToNode
 
 proc execProc*(c: PCtx; sym: PSym; args: openArray[PNode]): PNode =
+  c.loopIterations = c.config.maxLoopIterationsVM
   if sym.kind in routineKinds:
     if sym.typ.len-1 != args.len:
       localError(c.config, sym.info,

compiler/vmgen.nim

@@ -16,7 +16,7 @@
 #   types that use the 'node' field; the reason is that slots are
 #   re-used in a register based VM. Example:
 #
-#..code-block:: nim
+#.. code-block:: nim
 #   let s = a & b  # no matter what, create fresh node
 #   s = a & b  # no matter what, keep the node
 #
@@ -247,6 +247,8 @@ proc freeTemp(c: PCtx; r: TRegister) =
 proc getTempRange(cc: PCtx; n: int; kind: TSlotKind): TRegister =
   # if register pressure is high, we re-use more aggressively:
   let c = cc.prc
+  # we could also customize via the following (with proper caching in ConfigRef):
+  # let highRegisterPressure = cc.config.getConfigVar("vm.highRegisterPressure", "40").parseInt
   if c.maxSlots >= HighRegisterPressure or c.maxSlots+n >= high(TRegister):
     for i in 0..c.maxSlots-n:
       if not c.slots[i].inUse:
@@ -1521,14 +1523,14 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) =
     let tmp = c.genx(ri)
     c.preventFalseAlias(le[0], opcWrObj, objR, idx, tmp)
     c.freeTemp(tmp)
-    c.freeTemp(idx)
+    # c.freeTemp(idx) # BUGFIX, see nkDotExpr
     c.freeTemp(objR)
   of nkDotExpr:
     let dest = c.genx(le[0], {gfNode})
     let idx = genField(c, le[1])
     let tmp = c.genx(ri)
     c.preventFalseAlias(le, opcWrObj, dest, idx, tmp)
-    c.freeTemp(idx)
+    # c.freeTemp(idx) # BUGFIX: idx is an immediate (field position), not a register
     c.freeTemp(tmp)
     c.freeTemp(dest)
   of nkDerefExpr, nkHiddenDeref:
@@ -1957,14 +1959,6 @@ proc genProc*(c: PCtx; s: PSym): int
 
 proc matches(s: PSym; x: string): bool =
   let y = x.split('.')
-  var s = s
-  for i in 1..y.len:
-    if s == nil or (y[^i].cmpIgnoreStyle(s.name.s) != 0 and y[^i] != "*"):
-      return false
-    s = s.owner
-  result = true
-
-proc matches(s: PSym; y: varargs[string]): bool =
   var s = s
   for i in 1..y.len:
     if s == nil or (y[^i].cmpIgnoreStyle(s.name.s) != 0 and y[^i] != "*"):
@@ -2025,11 +2019,11 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
       elif s.kind == skMethod:
         localError(c.config, n.info, "cannot call method " & s.name.s &
           " at compile time")
-      elif matches(s, "stdlib", "marshal", "to"):
+      elif matches(s, "stdlib.marshal.to"):
         # XXX marshal load&store should not be opcodes, but use the
         # general callback mechanisms.
         genMarshalLoad(c, n, dest)
-      elif matches(s, "stdlib", "marshal", "$$"):
+      elif matches(s, "stdlib.marshal.$$"):
         genMarshalStore(c, n, dest)
       else:
         genCall(c, n, dest)

config/nimdoc.cfg

@@ -136,15 +136,10 @@ doc.body_toc_group = """
   </div>
   <div id="global-links">
     <ul class="simple-boot">
-      <li>
-        <a href="manual.html">Manual</a>
-      </li>
-      <li>
-        <a href="lib.html">Standard library</a>
-      </li>
-      <li>
-        <a href="theindex.html">Index</a>
-      </li>
+      <li><a href="manual.html">Manual</a></li>
+      <li><a href="lib.html">Standard library</a></li>
+      <li><a href="theindex.html">Index</a></li>
+      <li><a href="https://nim-lang.github.io/Nim/">devel</a>, <a href="https://nim-lang.org/documentation.html">stable</a></li>
     </ul>
   </div>
   <div id="searchInputDiv">

doc/advopt.txt

@@ -42,6 +42,8 @@ Advanced options:
                             produce hints or errors for Nim identifiers that
                             do not adhere to Nim's official style guide
                             https://nim-lang.org/docs/nep1.html
+  --styleCheck:usages       only enforce consistent spellings of identifiers,
+                            do not enforce the style on declarations
   --showAllMismatches:on|off
                             show all mismatching candidates in overloading
                             resolution

doc/destructors.rst

@@ -41,7 +41,7 @@ written as:
       dealloc(x.data)
       x.data = nil
 
-  proc `=`*[T](a: var myseq[T]; b: myseq[T]) =
+  proc `=copy`*[T](a: var myseq[T]; b: myseq[T]) =
     # do nothing for self-assignments:
     if a.data == b.data: return
     `=destroy`(a)
@@ -134,7 +134,7 @@ not free the resources afterwards by setting the object to its default value
 default value is written as ``wasMoved(x)``. When not provided the compiler
 is using a combination of `=destroy` and `copyMem` instead. This is efficient
 hence users rarely need to implement their own `=sink` operator, it is enough to
-provide `=destroy` and `=`, compiler will take care about the rest.
+provide `=destroy` and `=copy`, compiler will take care about the rest.
 
 The prototype of this hook for a type ``T`` needs to be:
 
@@ -156,10 +156,10 @@ The general pattern in ``=sink`` looks like:
 How self-assignments are handled is explained later in this document.
 
 
-`=` (copy) hook
+`=copy` hook
 ---------------
 
-The ordinary assignment in Nim conceptually copies the values. The ``=`` hook
+The ordinary assignment in Nim conceptually copies the values. The ``=copy`` hook
 is called for assignments that couldn't be transformed into ``=sink``
 operations.
 
@@ -167,21 +167,21 @@ The prototype of this hook for a type ``T`` needs to be:
 
 .. code-block:: nim
 
-  proc `=`(dest: var T; source: T)
+  proc `=copy`(dest: var T; source: T)
 
 
-The general pattern in ``=`` looks like:
+The general pattern in ``=copy`` looks like:
 
 .. code-block:: nim
 
-  proc `=`(dest: var T; source: T) =
+  proc `=copy`(dest: var T; source: T) =
     # protect against self-assignments:
     if dest.field != source.field:
       `=destroy`(dest)
       dest.field = duplicateResource(source.field)
 
 
-The ``=`` proc can be marked with the ``{.error.}`` pragma. Then any assignment
+The ``=copy`` proc can be marked with the ``{.error.}`` pragma. Then any assignment
 that otherwise would lead to a copy is prevented at compile-time.
 
 
@@ -199,7 +199,7 @@ Swap
 ====
 
 The need to check for self-assignments and also the need to destroy previous
-objects inside ``=`` and ``=sink`` is a strong indicator to treat
+objects inside ``=copy`` and ``=sink`` is a strong indicator to treat
 ``system.swap`` as a builtin primitive of its own that simply swaps every
 field in the involved objects via ``copyMem`` or a comparable mechanism.
 In other words, ``swap(a, b)`` is **not** implemented
@@ -320,7 +320,7 @@ not destroyed at the scope exit, but at the proc exit.
 
   x = y
   ------------------          (copy)
-  `=`(x, y)
+  `=copy`(x, y)
 
 
   f_sink(g())
@@ -330,7 +330,7 @@ not destroyed at the scope exit, but at the proc exit.
 
   f_sink(notLastReadOf y)
   --------------------------     (copy-to-sink)
-  (let tmp; `=`(tmp, y);
+  (let tmp; `=copy`(tmp, y);
   f_sink(tmp))
 
 

doc/manual.rst

@@ -2483,9 +2483,10 @@ matches) is preferred:
 Overloading based on 'var T'
 ----------------------------
 
-If the formal parameter ``f`` is of type ``var T`` in addition to the ordinary
-type checking, the argument is checked to be an `l-value`:idx:. ``var T``
-matches better than just ``T`` then.
+If the formal parameter ``f`` is of type ``var T``
+in addition to the ordinary type checking,
+the argument is checked to be an `l-value`:idx:.
+``var T`` matches better than just ``T`` then.
 
 .. code-block:: nim
   proc sayHi(x: int): string =
@@ -5714,12 +5715,12 @@ avoid ambiguity when there are multiple modules with the same path.
 There are two pseudo directories:
 
 1. ``std``: The ``std`` pseudo directory is the abstract location of Nim's standard
-library. For example, the syntax ``import std / strutils`` is used to unambiguously
-refer to the standard library's ``strutils`` module.
+   library. For example, the syntax ``import std / strutils`` is used to unambiguously
+   refer to the standard library's ``strutils`` module.
 2. ``pkg``: The ``pkg`` pseudo directory is used to unambiguously refer to a Nimble
-package. However, for technical details that lie outside of the scope of this document
-its semantics are: *Use the search path to look for module name but ignore the standard
-library locations*. In other words, it is the opposite of ``std``.
+   package. However, for technical details that lie outside of the scope of this document
+   its semantics are: *Use the search path to look for module name but ignore the standard
+   library locations*. In other words, it is the opposite of ``std``.
 
 
 From import statement
@@ -5981,10 +5982,10 @@ noSideEffect pragma
 The ``noSideEffect`` pragma is used to mark a proc/iterator to have no side
 effects. This means that the proc/iterator only changes locations that are
 reachable from its parameters and the return value only depends on the
-arguments. If none of its parameters have the type ``var T``
-or ``ref T`` or ``ptr T`` this means no locations are modified. It is a static
-error to mark a proc/iterator to have no side effect if the compiler cannot
-verify this.
+arguments. If none of its parameters have the type ``var T`` or ``ref T``
+or ``ptr T`` this means no locations are modified. It is a static error to
+mark a proc/iterator to have no side effect if the compiler cannot verify
+this.
 
 As a special semantic rule, the built-in `debugEcho
 <system.html#debugEcho,varargs[typed,]>`_ pretends to be free of side effects,

koch.nim

@@ -10,7 +10,7 @@
 #
 
 const
-  NimbleStableCommit = "63695f490728e3935692c29f3d71944d83bb1e83" # master
+  NimbleStableCommit = "d13f3b8ce288b4dc8c34c219a4e050aaeaf43fc9" # master
 
 when not defined(windows):
   const
@@ -136,35 +136,11 @@ proc bundleC2nim(args: string) =
 
 proc bundleNimbleExe(latest: bool, args: string) =
   let commit = if latest: "HEAD" else: NimbleStableCommit
-  cloneDependency(distDir, "https://github.com/nim-lang/nimble.git", commit = commit)
+  cloneDependency(distDir, "https://github.com/nim-lang/nimble.git",
+                  commit = commit, allowBundled = true)
   # installer.ini expects it under $nim/bin
   nimCompile("dist/nimble/src/nimble.nim",
-             options = "-d:release --nilseqs:on " & args)
-
-proc buildNimble(latest: bool, args: string) =
-  # if koch is used for a tar.xz, build the dist/nimble we shipped
-  # with the tarball:
-  var installDir = "dist/nimble"
-  if not latest and dirExists(installDir) and not dirExists("dist/nimble/.git"):
-    discard "don't do the git dance"
-  else:
-    if not dirExists("dist/nimble/.git"):
-      if dirExists(installDir):
-        var id = 0
-        while dirExists("dist/nimble" & $id):
-          inc id
-        installDir = "dist/nimble" & $id
-      # consider using/adapting cloneDependency
-      exec("git clone -q https://github.com/nim-lang/nimble.git " & installDir)
-    withDir(installDir):
-      if latest:
-        exec("git checkout -f master")
-        exec("git pull")
-      else:
-        exec("git fetch")
-        exec("git checkout " & NimbleStableCommit)
-  nimCompile(installDir / "src/nimble.nim",
-             options = "--noNimblePath --nilseqs:on -d:release " & args)
+             options = "-d:release --noNimblePath " & args)
 
 proc bundleNimsuggest(args: string) =
   nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim",
@@ -664,13 +640,13 @@ when isMainModule:
       of "temp": temp(op.cmdLineRest)
       of "xtemp": xtemp(op.cmdLineRest)
       of "wintools": bundleWinTools(op.cmdLineRest)
-      of "nimble": buildNimble(latest, op.cmdLineRest)
+      of "nimble": bundleNimbleExe(latest, op.cmdLineRest)
       of "nimsuggest": bundleNimsuggest(op.cmdLineRest)
       of "toolsnonimble":
         buildTools(op.cmdLineRest)
       of "tools":
         buildTools(op.cmdLineRest)
-        buildNimble(latest, op.cmdLineRest)
+        bundleNimbleExe(latest, op.cmdLineRest)
       of "pushcsource", "pushcsources": pushCsources()
       of "valgrind": valgrind(op.cmdLineRest)
       of "c2nim": bundleC2nim(op.cmdLineRest)

lib/core/macros.nim

@@ -1262,7 +1262,15 @@ proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} =
   else:
     badNodeKind someProc, "body="
 
-proc basename*(a: NimNode): NimNode {.compileTime, benign.}
+proc basename*(a: NimNode): NimNode {.raises: [].} =
+  ## Pull an identifier from prefix/postfix expressions.
+  case a.kind
+  of nnkIdent: result = a
+  of nnkPostfix, nnkPrefix: result = a[1]
+  of nnkPragmaExpr: result = basename(a[0])
+  else:
+    error("Do not know how to get basename of (" & treeRepr(a) & ")\n" &
+      repr(a), a)
 
 proc `$`*(node: NimNode): string {.compileTime.} =
   ## Get the string of an identifier node.
@@ -1325,16 +1333,6 @@ proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} =
       a[i + 1] = a[i]
     a[pos] = b
 
-proc basename*(a: NimNode): NimNode =
-  ## Pull an identifier from prefix/postfix expressions.
-  case a.kind
-  of nnkIdent: result = a
-  of nnkPostfix, nnkPrefix: result = a[1]
-  of nnkPragmaExpr: result = basename(a[0])
-  else:
-    error("Do not know how to get basename of (" & treeRepr(a) & ")\n" &
-      repr(a), a)
-
 proc `basename=`*(a: NimNode; val: string) {.compileTime.}=
   case a.kind
   of nnkIdent:

lib/core/typeinfo.nim

@@ -271,7 +271,7 @@ proc setPointer*(x: Any, y: pointer) =
   ## ``akPointer``, ``akSequence``.
   assert x.rawType.kind in {tyString, tyCString, tyRef, tyPtr, tyPointer,
                             tySequence, tyProc}
-  cast[ppointer](x.value)[] = y
+  genericAssign(x.value, y, x.rawType)
 
 proc fieldsAux(p: pointer, n: ptr TNimNode,
                ret: var seq[tuple[name: cstring, any: Any]]) =

lib/impure/nre.nim

@@ -62,7 +62,6 @@ from pcre import nil
 import nre/private/util
 import tables
 from strutils import `%`
-from math import ceil
 import options
 from unicode import runeLenAt
 
@@ -531,7 +530,7 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt
       raise RegexInternalError(msg : "Unknown internal error: " & $execRet)
 
 proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] =
-  ## Like ` ``find(...)`` <#proc-find>`_, but anchored to the start of the
+  ## Like `find(...)<#find,string,Regex,int>`_, but anchored to the start of the
   ## string.
   ##
   runnableExamples:
@@ -541,11 +540,11 @@ proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[R
   return str.matchImpl(pattern, start, endpos, pcre.ANCHORED)
 
 iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): RegexMatch =
-  ## Works the same as ` ``find(...)`` <#proc-find>`_, but finds every
+  ## Works the same as `find(...)<#find,string,Regex,int>`_, but finds every
   ## non-overlapping match. ``"2222".find(re"22")`` is ``"22", "22"``, not
   ## ``"22", "22", "22"``.
   ##
-  ## Arguments are the same as ` ``find(...)`` <#proc-find>`_
+  ## Arguments are the same as `find(...)<#find,string,Regex,int>`_
   ##
   ## Variants:
   ##
@@ -624,7 +623,7 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string]
   ## Splits the string with the given regex. This works according to the
   ## rules that Perl and Javascript use.
   ##
-  ## ``start`` behaves the same as in ` ``find(...)`` <#proc-find>`_.
+  ## ``start`` behaves the same as in `find(...)<#find,string,Regex,int>`_.
   ##
   runnableExamples:
     # -  If the match is zero-width, then the string is still split:
@@ -734,8 +733,25 @@ proc replace*(str: string, pattern: Regex, sub: string): string =
   replaceImpl(str, pattern,
     formatStr(sub, match.captures[name], match.captures[id - 1]))
 
-let SpecialCharMatcher = re"([\\+*?[^\]$(){}=!<>|:-])"
-proc escapeRe*(str: string): string =
-  ## Escapes the string so it doesn’t match any special characters.
+proc escapeRe*(str: string): string {.gcsafe.} =
+  ## Escapes the string so it doesn't match any special characters.
   ## Incompatible with the Extra flag (``X``).
-  str.replace(SpecialCharMatcher, "\\$1")
+  ##
+  ## Escaped char: `\ + * ? [ ^ ] $ ( ) { } = ! < > | : -`
+  runnableExamples:
+    doAssert escapeRe("fly+wind") == "fly\\+wind"
+    doAssert escapeRe("!") == "\\!"
+    doAssert escapeRe("nim*") == "nim\\*"
+
+  #([\\+*?[^\]$(){}=!<>|:-])
+  const SpecialCharMatcher = {'\\', '+', '*', '?', '[', '^', ']', '$', '(',
+                              ')', '{', '}', '=', '!', '<', '>', '|', ':',
+                              '-'}
+
+  for c in items(str):
+    case c
+    of SpecialCharMatcher:
+      result.add("\\")
+      result.add(c)
+    else:
+      result.add(c)

lib/pure/asyncdispatch.nim

@@ -1892,3 +1892,25 @@ proc waitFor*[T](fut: Future[T]): T =
     poll()
 
   fut.read
+
+proc activeDescriptors*(): int {.inline.} =
+  ## Returns the current number of active file descriptors for the current
+  ## event loop. This is a cheap operation that does not involve a system call.
+  when defined(windows):
+    result = getGlobalDispatcher().handles.len
+  elif not defined(nimdoc):
+    result = getGlobalDispatcher().selector.count
+
+when defined(posix):
+  import posix
+
+proc maxDescriptors*(): int {.raises: OSError.} =
+  ## Returns the maximum number of active file descriptors for the current
+  ## process. This involves a system call.
+  when defined(windows):
+    result = 16_700_000
+  else:
+    var fdLim: RLimit
+    if getrlimit(RLIMIT_NOFILE, fdLim) < 0:
+      raiseOSError(osLastError())
+    result = int(fdLim.rlim_cur) - 1

lib/pure/asyncfutures.nim

@@ -9,6 +9,8 @@
 
 import os, tables, strutils, times, heapqueue, options, deques, cstrutils
 
+import "system/stacktraces"
+
 # TODO: This shouldn't need to be included, but should ideally be exported.
 type
   CallbackFunc = proc () {.closure, gcsafe.}
@@ -296,49 +298,72 @@ proc `callback=`*[T](future: Future[T],
   ## If future has already completed then ``cb`` will be called immediately.
   future.callback = proc () = cb(future)
 
+template getFilenameProcname(entry: StackTraceEntry): (string, string) =
+  when compiles(entry.filenameStr) and compiles(entry.procnameStr):
+    # We can't rely on "entry.filename" and "entry.procname" still being valid
+    # cstring pointers, because the "string.data" buffers they pointed to might
+    # be already garbage collected (this entry being a non-shallow copy,
+    # "entry.filename" no longer points to "entry.filenameStr.data", but to the
+    # buffer of the original object).
+    (entry.filenameStr, entry.procnameStr)
+  else:
+    ($entry.filename, $entry.procname)
+
 proc getHint(entry: StackTraceEntry): string =
   ## We try to provide some hints about stack trace entries that the user
   ## may not be familiar with, in particular calls inside the stdlib.
+
+  let (filename, procname) = getFilenameProcname(entry)
+
   result = ""
-  if entry.procname == cstring"processPendingCallbacks":
-    if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0:
+  if procname == "processPendingCallbacks":
+    if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0:
       return "Executes pending callbacks"
-  elif entry.procname == cstring"poll":
-    if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0:
+  elif procname == "poll":
+    if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0:
       return "Processes asynchronous completion events"
 
-  if entry.procname.endsWith(NimAsyncContinueSuffix):
-    if cmpIgnoreStyle(entry.filename, "asyncmacro.nim") == 0:
+  if procname.endsWith(NimAsyncContinueSuffix):
+    if cmpIgnoreStyle(filename, "asyncmacro.nim") == 0:
       return "Resumes an async procedure"
 
-proc `$`*(entries: seq[StackTraceEntry]): string =
+proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string =
+  when defined(nimStackTraceOverride):
+    let entries = addDebuggingInfo(stackTraceEntries)
+  else:
+    let entries = stackTraceEntries
+
   result = ""
   # Find longest filename & line number combo for alignment purposes.
   var longestLeft = 0
   for entry in entries:
-    if entry.procname.isNil: continue
+    let (filename, procname) = getFilenameProcname(entry)
 
-    let left = $entry.filename & $entry.line
-    if left.len > longestLeft:
-      longestLeft = left.len
+    if procname == "": continue
+
+    let leftLen = filename.len + len($entry.line)
+    if leftLen > longestLeft:
+      longestLeft = leftLen
 
   var indent = 2
   # Format the entries.
   for entry in entries:
-    if entry.procname.isNil:
-      if entry.line == -10:
+    let (filename, procname) = getFilenameProcname(entry)
+
+    if procname == "":
+      if entry.line == reraisedFromBegin:
         result.add(spaces(indent) & "#[\n")
         indent.inc(2)
-      else:
+      elif entry.line == reraisedFromEnd:
         indent.dec(2)
         result.add(spaces(indent) & "]#\n")
       continue
 
-    let left = "$#($#)" % [$entry.filename, $entry.line]
+    let left = "$#($#)" % [filename, $entry.line]
     result.add((spaces(indent) & "$#$# $#\n") % [
       left,
       spaces(longestLeft - left.len + 2),
-      $entry.procname
+      procname
     ])
     let hint = getHint(entry)
     if hint.len > 0:
@@ -362,9 +387,9 @@ proc injectStacktrace[T](future: Future[T]) =
     newMsg.add($entries)
 
     newMsg.add("Exception message: " & exceptionMsg & "\n")
-    newMsg.add("Exception type:")
 
     # # For debugging purposes
+    # newMsg.add("Exception type:")
     # for entry in getStackTraceEntries(future.error):
     #   newMsg.add "\n" & $entry
     future.error.msg = newMsg

lib/pure/asynchttpserver.nim

@@ -11,24 +11,36 @@
 ##
 ## This HTTP server has not been designed to be used in production, but
 ## for testing applications locally. Because of this, when deploying your
-## application you should use a reverse proxy (for example nginx) instead of
-## allowing users to connect directly to this server.
+## application in production you should use a reverse proxy (for example nginx)
+## instead of allowing users to connect directly to this server.
 ##
-## Basic usage
-## ===========
+## Example
+## =======
 ##
 ## This example will create an HTTP server on port 8080. The server will
 ## respond to all requests with a ``200 OK`` response code and "Hello World"
 ## as the response body.
 ##
-## .. code-block::nim
-##    import asynchttpserver, asyncdispatch
+## .. code-block:: Nim
 ##
-##    var server = newAsyncHttpServer()
-##    proc cb(req: Request) {.async.} =
-##      await req.respond(Http200, "Hello World")
+##   import asynchttpserver, asyncdispatch
 ##
-##    waitFor server.serve(Port(8080), cb)
+##   proc main {.async.} =
+##     var server = newAsyncHttpServer()
+##     proc cb(req: Request) {.async.} =
+##       let headers = {"Date": "Tue, 29 Apr 2014 23:40:08 GMT",
+##           "Content-type": "text/plain; charset=utf-8"}
+##       await req.respond(Http200, "Hello World", headers.newHttpHeaders())
+##
+##     server.listen Port(8080)
+##     while true:
+##       if server.shouldAcceptRequest():
+##         asyncCheck server.acceptRequest(cb)
+##       else:
+##         poll()
+##
+##   asyncCheck main()
+##   runForever()
 
 import asyncnet, asyncdispatch, parseutils, uri, strutils
 import httpcore
@@ -58,14 +70,12 @@ type
     reuseAddr: bool
     reusePort: bool
     maxBody: int ## The maximum content-length that will be read for the body.
+    maxFDs: int
 
 proc newAsyncHttpServer*(reuseAddr = true, reusePort = false,
                          maxBody = 8388608): AsyncHttpServer =
   ## Creates a new ``AsyncHttpServer`` instance.
-  new result
-  result.reuseAddr = reuseAddr
-  result.reusePort = reusePort
-  result.maxBody = maxBody
+  result = AsyncHttpServer(reuseAddr: reuseAddr, reusePort: reusePort, maxBody: maxBody)
 
 proc addHeaders(msg: var string, headers: HttpHeaders) =
   for k, v in headers:
@@ -294,13 +304,9 @@ proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string
     )
     if not retry: break
 
-proc serve*(server: AsyncHttpServer, port: Port,
-            callback: proc (request: Request): Future[void] {.closure, gcsafe.},
-            address = "") {.async.} =
-  ## Starts the process of listening for incoming HTTP connections on the
-  ## specified address and port.
-  ##
-  ## When a request is made by a client the specified callback will be called.
+proc listen*(server: AsyncHttpServer; port: Port; address = "") =
+  ## Listen to the given port and address.
+  server.maxFDs = maxDescriptors()
   server.socket = newAsyncSocket()
   if server.reuseAddr:
     server.socket.setSockOpt(OptReuseAddr, true)
@@ -309,9 +315,44 @@ proc serve*(server: AsyncHttpServer, port: Port,
   server.socket.bindAddr(port, address)
   server.socket.listen()
 
+proc shouldAcceptRequest*(server: AsyncHttpServer;
+                          assumedDescriptorsPerRequest = 5): bool {.inline.} =
+  ## Returns true if the process's current number of opened file
+  ## descriptors is still within the maximum limit and so it's reasonable to
+  ## accept yet another request.
+  result = assumedDescriptorsPerRequest < 0 or
+    (activeDescriptors() + assumedDescriptorsPerRequest < server.maxFDs)
+
+proc acceptRequest*(server: AsyncHttpServer,
+            callback: proc (request: Request): Future[void] {.closure, gcsafe.}) {.async.} =
+  ## Accepts a single request. Write an explicit loop around this proc so that
+  ## errors can be handled properly.
+  var (address, client) = await server.socket.acceptAddr()
+  asyncCheck processClient(server, client, address, callback)
+
+proc serve*(server: AsyncHttpServer, port: Port,
+            callback: proc (request: Request): Future[void] {.closure, gcsafe.},
+            address = "";
+            assumedDescriptorsPerRequest = -1) {.async.} =
+  ## Starts the process of listening for incoming HTTP connections on the
+  ## specified address and port.
+  ##
+  ## When a request is made by a client the specified callback will be called.
+  ##
+  ## If `assumedDescriptorsPerRequest` is 0 or greater the server cares about
+  ## the process's maximum file descriptor limit. It then ensures that the
+  ## process still has the resources for `assumedDescriptorsPerRequest`
+  ## file descriptors before accepting a connection.
+  ##
+  ## You should prefer to call `acceptRequest` instead with a custom server
+  ## loop so that you're in control over the error handling and logging.
+  listen server, port, address
   while true:
-    var (address, client) = await server.socket.acceptAddr()
-    asyncCheck processClient(server, client, address, callback)
+    if shouldAcceptRequest(server, assumedDescriptorsPerRequest):
+      var (address, client) = await server.socket.acceptAddr()
+      asyncCheck processClient(server, client, address, callback)
+    else:
+      poll()
     #echo(f.isNil)
     #echo(f.repr)
 
@@ -320,7 +361,7 @@ proc close*(server: AsyncHttpServer) =
   server.socket.close()
 
 when not defined(testing) and isMainModule:
-  proc main =
+  proc main {.async.} =
     var server = newAsyncHttpServer()
     proc cb(req: Request) {.async.} =
       #echo(req.reqMethod, " ", req.url)
@@ -329,6 +370,12 @@ when not defined(testing) and isMainModule:
           "Content-type": "text/plain; charset=utf-8"}
       await req.respond(Http200, "Hello World", headers.newHttpHeaders())
 
-    asyncCheck server.serve(Port(5555), cb)
-    runForever()
-  main()
+    server.listen Port(5555)
+    while true:
+      if server.shouldAcceptRequest():
+        asyncCheck server.acceptRequest(cb)
+      else:
+        poll()
+
+  asyncCheck main()
+  runForever()

lib/pure/asyncmacro.nim

@@ -198,7 +198,8 @@ proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} =
   for i in 1 ..< len(params):
     expectKind(params[i], nnkIdentDefs)
     if params[i][1].kind == nnkBracketExpr and
-       params[i][1][0].eqIdent("futurevar"):
+       params[i][1][0].eqIdent(FutureVar.astToStr):
+      ## eqIdent: first char is case sensitive!!!
       result.add(params[i][0])
 
 proc isInvalidReturnType(typeName: string): bool =

lib/pure/asyncnet.nim

@@ -691,8 +691,12 @@ elif defined(nimdoc):
 
 proc close*(socket: AsyncSocket) =
   ## Closes the socket.
+  if socket.closed: return
+
   defer:
     socket.fd.AsyncFD.closeSocket()
+    socket.closed = true # TODO: Add extra debugging checks for this.
+
   when defineSsl:
     if socket.isSsl:
       let res = SSL_shutdown(socket.sslHandle)
@@ -701,7 +705,6 @@ proc close*(socket: AsyncSocket) =
         discard
       elif res != 1:
         raiseSSLError()
-  socket.closed = true # TODO: Add extra debugging checks for this.
 
 when defineSsl:
   proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) =

lib/pure/asyncstreams.nim

@@ -110,7 +110,11 @@ proc read*[T](future: FutureStream[T]): owned(Future[(bool, T)]) =
       resFut.complete(res)
 
       # If the saved callback isn't nil then let's call it.
-      if not savedCb.isNil: savedCb()
+      if not savedCb.isNil:
+        if fs.queue.len > 0:
+          savedCb()
+        else:
+          future.cb = savedCb
 
   if future.queue.len > 0 or future.finished:
     newCb(future)

lib/pure/base64.nim

@@ -230,11 +230,11 @@ proc decode*(s: string): string =
 
   template inputChar(x: untyped) =
     let x = int decodeTable[ord(s[inputIndex])]
-    inc inputIndex
     if x == invalidChar:
       raise newException(ValueError,
         "Invalid base64 format character `" & s[inputIndex] &
         "` (ord " & $s[inputIndex].ord & ") at location " & $inputIndex & ".")
+    inc inputIndex
 
   template outputChar(x: untyped) =
     result[outputIndex] = char(x and 255)

lib/pure/collections/tables.nim

@@ -1808,7 +1808,7 @@ proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): <//>OrderedTableRef[A, B]
     assert b == {'a': 5, 'b': 9}.newOrderedTable
 
   result = newOrderedTable[A, B](rightSize(pairs.len))
-  for key, val in items(pairs): result.add(key, val)
+  for key, val in items(pairs): result[key] = val
 
 
 proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B =
@@ -2634,7 +2634,7 @@ proc `[]`*[A](t: CountTableRef[A], key: A): int =
   ## See also:
   ## * `getOrDefault<#getOrDefault,CountTableRef[A],A,int>`_ to return
   ##   a custom value if the key doesn't exist
-  ## * `mget proc<#mget,CountTableRef[A],A>`_
+  ## * `inc proc<#inc,CountTableRef[A],A>`_ to inc even if missing
   ## * `[]= proc<#[]%3D,CountTableRef[A],A,int>`_ for inserting a new
   ##   (key, value) pair in the table
   ## * `hasKey proc<#hasKey,CountTableRef[A],A>`_ for checking if a key
@@ -3002,37 +3002,37 @@ when isMainModule:
   block: #5482
     var a = [("wrong?", "foo"), ("wrong?", "foo2")].newOrderedTable()
     var b = newOrderedTable[string, string](initialSize = 2)
-    b.add("wrong?", "foo")
-    b.add("wrong?", "foo2")
+    b["wrong?"] = "foo"
+    b["wrong?"] = "foo2"
     assert a == b
 
   block: #5482
     var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable()
     var b = newOrderedTable[string, string](initialSize = 2)
-    b.add("wrong?", "foo")
-    b.add("wrong?", "foo2")
+    b["wrong?"] = "foo"
+    b["wrong?"] = "foo2"
     assert a == b
 
   block: #5487
     var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable()
     var b = newOrderedTable[string, string]()         # notice, default size!
-    b.add("wrong?", "foo")
-    b.add("wrong?", "foo2")
+    b["wrong?"] = "foo"
+    b["wrong?"] = "foo2"
     assert a == b
 
   block: #5487
     var a = [("wrong?", "foo"), ("wrong?", "foo2")].newOrderedTable()
     var b = newOrderedTable[string, string]()         # notice, default size!
-    b.add("wrong?", "foo")
-    b.add("wrong?", "foo2")
+    b["wrong?"] = "foo"
+    b["wrong?"] = "foo2"
     assert a == b
 
   block:
     var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable()
     var b = [("wrong?", "foo"), ("wrong?", "foo2")].newOrderedTable()
     var c = newOrderedTable[string, string]()         # notice, default size!
-    c.add("wrong?", "foo")
-    c.add("wrong?", "foo2")
+    c["wrong?"] = "foo"
+    c["wrong?"] = "foo2"
     assert a == b
     assert a == c
 

lib/pure/httpclient.nim

@@ -317,7 +317,7 @@ proc getDefaultSSL(): SslContext =
   result = defaultSslContext
   when defined(ssl):
     if result == nil:
-      defaultSslContext = newContext(verifyMode = CVerifyNone)
+      defaultSslContext = newContext(verifyMode = CVerifyPeer)
       result = defaultSslContext
       doAssert result != nil, "failure to initialize the SSL context"
 
@@ -1140,15 +1140,15 @@ proc downloadFile*(client: HttpClient, url: string, filename: string) =
     client.getBody = true
   let resp = client.get(url)
 
+  if resp.code.is4xx or resp.code.is5xx:
+    raise newException(HttpRequestError, resp.status)
+
   client.bodyStream = newFileStream(filename, fmWrite)
   if client.bodyStream.isNil:
     fileError("Unable to open file")
   parseBody(client, resp.headers, resp.version)
   client.bodyStream.close()
 
-  if resp.code.is4xx or resp.code.is5xx:
-    raise newException(HttpRequestError, resp.status)
-
 proc downloadFile*(client: AsyncHttpClient, url: string,
                    filename: string): Future[void] =
   proc downloadFileEx(client: AsyncHttpClient,

lib/pure/includes/oserr.nim

@@ -114,7 +114,7 @@ proc osLastError*(): OSErrorCode {.sideEffect.} =
   when defined(nimscript):
     discard
   elif defined(windows):
-    result = OSErrorCode(getLastError())
+    result = cast[OSErrorCode](getLastError())
   else:
     result = OSErrorCode(errno)
 {.pop.}

lib/pure/ioselects/ioselectors_epoll.nim

@@ -55,7 +55,7 @@ when hasThreadSupport:
       maxFD: int
       numFD: int
       fds: ptr SharedArray[SelectorKey[T]]
-      count: int
+      count*: int
     Selector*[T] = ptr SelectorImpl[T]
 else:
   type
@@ -64,7 +64,7 @@ else:
       maxFD: int
       numFD: int
       fds: seq[SelectorKey[T]]
-      count: int
+      count*: int
     Selector*[T] = ref SelectorImpl[T]
 type
   SelectEventImpl = object

lib/pure/ioselects/ioselectors_kqueue.nim

@@ -30,7 +30,7 @@ when defined(macosx) or defined(freebsd) or defined(dragonfly):
   proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t,
               newp: pointer, newplen: csize_t): cint
        {.importc: "sysctl",header: """#include <sys/types.h>
-                                      #include <sys/sysctl.h>"""}
+                                      #include <sys/sysctl.h>""".}
 elif defined(netbsd) or defined(openbsd):
   # OpenBSD and NetBSD don't have KERN_MAXFILESPERPROC, so we are using
   # KERN_MAXFILES, because KERN_MAXFILES is always bigger,
@@ -39,7 +39,7 @@ elif defined(netbsd) or defined(openbsd):
   proc sysctl(name: ptr cint, namelen: cuint, oldp: pointer, oldplen: ptr csize_t,
               newp: pointer, newplen: csize_t): cint
        {.importc: "sysctl",header: """#include <sys/param.h>
-                                      #include <sys/sysctl.h>"""}
+                                      #include <sys/sysctl.h>""".}
 
 when hasThreadSupport:
   type
@@ -48,7 +48,7 @@ when hasThreadSupport:
       maxFD: int
       changes: ptr SharedArray[KEvent]
       fds: ptr SharedArray[SelectorKey[T]]
-      count: int
+      count*: int
       changesLock: Lock
       changesSize: int
       changesLength: int
@@ -61,7 +61,7 @@ else:
       maxFD: int
       changes: seq[KEvent]
       fds: seq[SelectorKey[T]]
-      count: int
+      count*: int
       sock: cint
     Selector*[T] = ref SelectorImpl[T]
 

lib/pure/ioselects/ioselectors_poll.nim

@@ -21,7 +21,7 @@ when hasThreadSupport:
       pollcnt: int
       fds: ptr SharedArray[SelectorKey[T]]
       pollfds: ptr SharedArray[TPollFd]
-      count: int
+      count*: int
       lock: Lock
     Selector*[T] = ptr SelectorImpl[T]
 else:
@@ -31,7 +31,7 @@ else:
       pollcnt: int
       fds: seq[SelectorKey[T]]
       pollfds: seq[TPollFd]
-      count: int
+      count*: int
     Selector*[T] = ref SelectorImpl[T]
 
 type

lib/pure/ioselects/ioselectors_select.nim

@@ -58,7 +58,7 @@ when hasThreadSupport:
       eSet: FdSet
       maxFD: int
       fds: ptr SharedArray[SelectorKey[T]]
-      count: int
+      count*: int
       lock: Lock
     Selector*[T] = ptr SelectorImpl[T]
 else:
@@ -69,7 +69,7 @@ else:
       eSet: FdSet
       maxFD: int
       fds: seq[SelectorKey[T]]
-      count: int
+      count*: int
     Selector*[T] = ref SelectorImpl[T]
 
 type

lib/pure/nativesockets.nim

@@ -68,11 +68,11 @@ type
   Protocol* = enum    ## third argument to `socket` proc
     IPPROTO_TCP = 6,  ## Transmission control protocol.
     IPPROTO_UDP = 17, ## User datagram protocol.
-    IPPROTO_IP,       ## Internet protocol. Unsupported on Windows.
-    IPPROTO_IPV6,     ## Internet Protocol Version 6. Unsupported on Windows.
+    IPPROTO_IP,       ## Internet protocol.
+    IPPROTO_IPV6,     ## Internet Protocol Version 6.
     IPPROTO_RAW,      ## Raw IP Packets Protocol. Unsupported on Windows.
-    IPPROTO_ICMP      ## Control message protocol. Unsupported on Windows.
-    IPPROTO_ICMPV6    ## Control message protocol for IPv6. Unsupported on Windows.
+    IPPROTO_ICMP      ## Internet Control message protocol.
+    IPPROTO_ICMPV6    ## Internet Control message protocol for IPv6.
 
   Servent* = object ## information about a service
     name*: string
@@ -113,7 +113,7 @@ proc `==`*(a, b: Port): bool {.borrow.}
   ## ``==`` for ports.
 
 proc `$`*(p: Port): string {.borrow.}
-  ## returns the port number as a string
+  ## Returns the port number as a string
 
 proc toInt*(domain: Domain): cint
   ## Converts the Domain enum to a platform-dependent ``cint``.
@@ -174,7 +174,21 @@ else:
     result = cint(ord(typ))
 
   proc toInt(p: Protocol): cint =
-    result = cint(ord(p))
+    case p
+    of IPPROTO_IP:
+      result = 0.cint
+    of IPPROTO_ICMP:
+      result = 1.cint
+    of IPPROTO_TCP:
+      result = 6.cint
+    of IPPROTO_UDP:
+      result = 17.cint
+    of IPPROTO_IPV6:
+      result = 41.cint
+    of IPPROTO_ICMPV6:
+      result = 58.cint
+    else:
+      result = cint(ord(p))
 
 proc toSockType*(protocol: Protocol): SockType =
   result = case protocol
@@ -414,7 +428,7 @@ proc getHostname*(): string {.tags: [ReadIOEffect].} =
   result.setLen(x)
 
 proc getSockDomain*(socket: SocketHandle): Domain =
-  ## returns the socket's domain (AF_INET or AF_INET6).
+  ## Returns the socket's domain (AF_INET or AF_INET6).
   var name: Sockaddr_in6
   var namelen = sizeof(name).SockLen
   if getsockname(socket, cast[ptr SockAddr](addr(name)),
@@ -426,7 +440,7 @@ proc getSockDomain*(socket: SocketHandle): Domain =
     raise newException(IOError, "Unknown socket family in getSockDomain")
 
 proc getAddrString*(sockAddr: ptr SockAddr): string =
-  ## return the string representation of address within sockAddr
+  ## Returns the string representation of address within sockAddr
   if sockAddr.sa_family.cint == nativeAfInet:
     result = $inet_ntoa(cast[ptr Sockaddr_in](sockAddr).sin_addr)
   elif sockAddr.sa_family.cint == nativeAfInet6:
@@ -459,7 +473,7 @@ when defined(posix) and not defined(nimdoc):
     copyMem(addr result.sun_path, path.cstring, path.len + 1)
 
 proc getSockName*(socket: SocketHandle): Port =
-  ## returns the socket's associated port number.
+  ## Returns the socket's associated port number.
   var name: Sockaddr_in
   when useWinVersion:
     name.sin_family = uint16(ord(AF_INET))
@@ -474,7 +488,7 @@ proc getSockName*(socket: SocketHandle): Port =
   result = Port(nativesockets.ntohs(name.sin_port))
 
 proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) =
-  ## returns the socket's local address and port number.
+  ## Returns the socket's local address and port number.
   ##
   ## Similar to POSIX's `getsockname`:idx:.
   case domain
@@ -511,7 +525,7 @@ proc getLocalAddr*(socket: SocketHandle, domain: Domain): (string, Port) =
     raiseOSError(OSErrorCode(-1), "invalid socket family in getLocalAddr")
 
 proc getPeerAddr*(socket: SocketHandle, domain: Domain): (string, Port) =
-  ## returns the socket's peer address and port number.
+  ## Returns the socket's peer address and port number.
   ##
   ## Similar to POSIX's `getpeername`:idx:
   case domain

lib/pure/net.nim

@@ -22,6 +22,17 @@
 ## In order to use the SSL procedures defined in this module, you will need to
 ## compile your application with the ``-d:ssl`` flag.
 ##
+##
+## SSL on Windows
+## ==============
+##
+## On Windows the SSL library checks for valid certificates.
+## It uses the `cacert.pem` file for this purpose which was extracted
+## from `https://curl.se/ca/cacert.pem`. Besides
+## the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
+## also need to ship `cacert.pem` with your `.exe` file.
+##
+##
 ## Examples
 ## ========
 ##
@@ -572,7 +583,7 @@ when defineSsl:
     if newCTX.SSL_CTX_set_cipher_list(cipherList) != 1:
       raiseSSLError()
 
-    when defined(nimDisableCertificateValidation) or defined(windows):
+    when defined(nimDisableCertificateValidation):
       newCTX.SSL_CTX_set_verify(SSL_VERIFY_NONE, nil)
     else:
       case verifyMode
@@ -587,11 +598,13 @@ when defineSsl:
     discard newCTX.SSLCTXSetMode(SSL_MODE_AUTO_RETRY)
     newCTX.loadCertificates(certFile, keyFile)
 
-    when not defined(nimDisableCertificateValidation) and not defined(windows):
+    const VerifySuccess = 1 # SSL_CTX_load_verify_locations returns 1 on success.
+
+    when not defined(nimDisableCertificateValidation):
       if verifyMode != CVerifyNone:
         # Use the caDir and caFile parameters if set
         if caDir != "" or caFile != "":
-          if newCTX.SSL_CTX_load_verify_locations(caFile, caDir) != 0:
+          if newCTX.SSL_CTX_load_verify_locations(caFile, caDir) != VerifySuccess:
             raise newException(IOError, "Failed to load SSL/TLS CA certificate(s).")
 
         else:
@@ -599,7 +612,7 @@ when defineSsl:
           # the SSL_CERT_FILE and SSL_CERT_DIR env vars
           var found = false
           for fn in scanSSLCertificates():
-            if newCTX.SSL_CTX_load_verify_locations(fn, "") == 0:
+            if newCTX.SSL_CTX_load_verify_locations(fn, nil) == VerifySuccess:
               found = true
               break
           if not found:
@@ -637,12 +650,11 @@ when defineSsl:
     let ctx = SslContext(context: ssl.SSL_get_SSL_CTX)
     let hintString = if hint == nil: "" else: $hint
     let (identityString, pskString) = (ctx.clientGetPskFunc)(hintString)
-    if psk.len.cuint > max_psk_len:
+    if pskString.len.cuint > max_psk_len:
       return 0
     if identityString.len.cuint >= max_identity_len:
       return 0
-
-    copyMem(identity, identityString.cstring, pskString.len + 1) # with the last zero byte
+    copyMem(identity, identityString.cstring, identityString.len + 1) # with the last zero byte
     copyMem(psk, pskString.cstring, pskString.len)
 
     return pskString.len.cuint
@@ -663,7 +675,7 @@ when defineSsl:
       max_psk_len: cint): cuint {.cdecl.} =
     let ctx = SslContext(context: ssl.SSL_get_SSL_CTX)
     let pskString = (ctx.serverGetPskFunc)($identity)
-    if psk.len.cint > max_psk_len:
+    if pskString.len.cint > max_psk_len:
       return 0
     copyMem(psk, pskString.cstring, pskString.len)
 
@@ -716,10 +728,11 @@ when defineSsl:
         raiseSSLError("No SSL certificate found.")
 
       const X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT = 0x1.cuint
-      const size = 1024
-      var peername: string = newString(size)
+      # https://www.openssl.org/docs/man1.1.1/man3/X509_check_host.html
       let match = certificate.X509_check_host(hostname.cstring, hostname.len.cint,
-        X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, peername)
+        X509_CHECK_FLAG_ALWAYS_CHECK_SUBJECT, nil)
+      # https://www.openssl.org/docs/man1.1.1/man3/SSL_get_peer_certificate.html
+      X509_free(certificate)
       if match != 1:
         raiseSSLError("SSL Certificate check failed.")
 
@@ -1816,4 +1829,5 @@ proc getPrimaryIPAddr*(dest = parseIpAddress("8.8.8.8")): IpAddress =
     else:
       newSocket(AF_INET6, SOCK_DGRAM, IPPROTO_UDP)
   socket.connect($dest, 80.Port)
-  socket.getLocalAddr()[0].parseIpAddress()
+  result = socket.getLocalAddr()[0].parseIpAddress()
+  socket.close()

lib/pure/os.nim

@@ -943,6 +943,8 @@ proc expandTilde*(path: string): string {.
   ## Windows: this is still supported despite Windows platform not having this
   ## convention; also, both ``~/`` and ``~\`` are handled.
   ##
+  ## **Warning**: `~bob` and `~bob/` are not yet handled correctly.
+  ##
   ## See also:
   ## * `getHomeDir proc <#getHomeDir>`_
   ## * `getConfigDir proc <#getConfigDir>`_
@@ -1156,6 +1158,7 @@ when not defined(windows) and not weirdTarget:
     else: result = S_ISLNK(rawInfo.st_mode)
 
 const
+  maxSymlinkLen = 1024
   ExeExts* = ## Platform specific file extension for executables.
     ## On Windows ``["exe", "cmd", "bat"]``, on Posix ``[""]``.
     when defined(windows): ["exe", "cmd", "bat"] else: [""]
@@ -1197,11 +1200,11 @@ proc findExe*(exe: string, followSymlinks: bool = true;
         when not defined(windows):
           while followSymlinks: # doubles as if here
             if x.checkSymlink:
-              var r = newString(256)
-              var len = readlink(x, r, 256)
+              var r = newString(maxSymlinkLen)
+              var len = readlink(x, r, maxSymlinkLen)
               if len < 0:
                 raiseOSError(osLastError(), exe)
-              if len > 256:
+              if len > maxSymlinkLen:
                 r = newString(len+1)
                 len = readlink(x, r, len)
               setLen(r, len)
@@ -2517,11 +2520,11 @@ proc expandSymlink*(symlinkPath: string): string {.noNimScript.} =
   when defined(windows):
     result = symlinkPath
   else:
-    result = newString(256)
-    var len = readlink(symlinkPath, result, 256)
+    result = newString(maxSymlinkLen)
+    var len = readlink(symlinkPath, result, maxSymlinkLen)
     if len < 0:
       raiseOSError(osLastError(), symlinkPath)
-    if len > 256:
+    if len > maxSymlinkLen:
       result = newString(len+1)
       len = readlink(symlinkPath, result, len)
     setLen(result, len)
@@ -2811,9 +2814,9 @@ when not weirdTarget and (defined(freebsd) or defined(dragonfly)):
 
 when not weirdTarget and (defined(linux) or defined(solaris) or defined(bsd) or defined(aix)):
   proc getApplAux(procPath: string): string =
-    result = newString(256)
-    var len = readlink(procPath, result, 256)
-    if len > 256:
+    result = newString(maxSymlinkLen)
+    var len = readlink(procPath, result, maxSymlinkLen)
+    if len > maxSymlinkLen:
       result = newString(len+1)
       len = readlink(procPath, result, len)
     setLen(result, len)

lib/pure/osproc.nim

@@ -381,6 +381,8 @@ proc execProcesses*(cmds: openArray[string],
             raiseOSError(err)
 
       if rexit >= 0:
+        when defined(windows):
+          let processHandle = q[rexit].fProcessHandle
         result = max(result, abs(q[rexit].peekExitCode()))
         if afterRunEvent != nil: afterRunEvent(idxs[rexit], q[rexit])
         close(q[rexit])
@@ -395,7 +397,7 @@ proc execProcesses*(cmds: openArray[string],
         else:
           when defined(windows):
             for k in 0..wcount - 1:
-              if w[k] == q[rexit].fProcessHandle:
+              if w[k] == processHandle:
                 w[k] = w[wcount - 1]
                 w[wcount - 1] = 0
                 dec(wcount)
@@ -441,10 +443,17 @@ when defined(Windows) and not defined(useNimRtl):
       handle: Handle
       atTheEnd: bool
 
+  proc closeHandleCheck(handle: Handle) {.inline.} =
+    if handle.closeHandle() == 0:
+      raiseOSError(osLastError())
+
+  proc fileClose[T: Handle | FileHandle](h: var T) {.inline.} =
+    if h > 4:
+      closeHandleCheck(h)
+      h = INVALID_HANDLE_VALUE.T
+
   proc hsClose(s: Stream) =
-    # xxx here + elsewhere: check instead of discard; ignoring errors leads to
-    # hard to track bugs
-    discard FileHandleStream(s).handle.closeHandle
+    FileHandleStream(s).handle.fileClose()
 
   proc hsAtEnd(s: Stream): bool = return FileHandleStream(s).atTheEnd
 
@@ -549,8 +558,8 @@ when defined(Windows) and not defined(useNimRtl):
 
     stdin = myDup(pipeIn, 0)
     stdout = myDup(pipeOut, 0)
-    discard closeHandle(pipeIn)
-    discard closeHandle(pipeOut)
+    closeHandleCheck(pipeIn)
+    closeHandleCheck(pipeOut)
     stderr = stdout
 
   proc createPipeHandles(rdHandle, wrHandle: var Handle) =
@@ -561,9 +570,6 @@ when defined(Windows) and not defined(useNimRtl):
     if createPipe(rdHandle, wrHandle, sa, 0) == 0'i32:
       raiseOSError(osLastError())
 
-  proc fileClose(h: Handle) {.inline.} =
-    if h > 4: discard closeHandle(h)
-
   proc startProcess(command: string, workingDir: string = "",
       args: openArray[string] = [], env: StringTableRef = nil,
       options: set[ProcessOption] = {poStdErrToStdOut}):
@@ -630,8 +636,11 @@ when defined(Windows) and not defined(useNimRtl):
       success = winlean.createProcessW(nil, tmp, nil, nil, 1, flags,
         ee, wwd, si, procInfo)
     else:
+      var ee =
+        if e.str.isNil: cstring(nil)
+        else: cstring(e.str)
       success = winlean.createProcessA(nil,
-        cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, e, wd, si, procInfo)
+        cmdl, nil, nil, 1, NORMAL_PRIORITY_CLASS, ee, wd, si, procInfo)
     let lastError = osLastError()
 
     if poParentStreams notin options:
@@ -655,13 +664,31 @@ when defined(Windows) and not defined(useNimRtl):
     result.id = procInfo.dwProcessId
     result.exitFlag = false
 
+  proc closeThreadAndProcessHandle(p: Process) =
+    if p.fThreadHandle != 0:
+      closeHandleCheck(p.fThreadHandle)
+      p.fThreadHandle = 0
+
+    if p.fProcessHandle != 0:
+      closeHandleCheck(p.fProcessHandle)
+      p.fProcessHandle = 0
+
   proc close(p: Process) =
     if poParentStreams notin p.options:
-      discard closeHandle(p.inHandle)
-      discard closeHandle(p.outHandle)
-      discard closeHandle(p.errHandle)
-    discard closeHandle(p.fThreadHandle)
-    discard closeHandle(p.fProcessHandle)
+      if p.inStream == nil:
+        p.inHandle.fileClose()
+      else:
+        # p.inHandle can be already closed via inputStream.
+        p.inStream.close
+
+      # You may NOT close outputStream and errorStream.
+      assert p.outStream == nil or FileHandleStream(p.outStream).handle != INVALID_HANDLE_VALUE
+      assert p.errStream == nil or FileHandleStream(p.errStream).handle != INVALID_HANDLE_VALUE
+
+      if p.outHandle != p.errHandle:
+        p.errHandle.fileClose()
+      p.outHandle.fileClose()
+    p.closeThreadAndProcessHandle()
 
   proc suspend(p: Process) =
     discard suspendThread(p.fThreadHandle)
@@ -695,8 +722,7 @@ when defined(Windows) and not defined(useNimRtl):
     if status != STILL_ACTIVE:
       p.exitFlag = true
       p.exitStatus = status
-      discard closeHandle(p.fThreadHandle)
-      discard closeHandle(p.fProcessHandle)
+      p.closeThreadAndProcessHandle()
       result = status
     else:
       result = -1
@@ -712,8 +738,7 @@ when defined(Windows) and not defined(useNimRtl):
       discard getExitCodeProcess(p.fProcessHandle, status)
       p.exitFlag = true
       p.exitStatus = status
-      discard closeHandle(p.fThreadHandle)
-      discard closeHandle(p.fProcessHandle)
+      p.closeThreadAndProcessHandle()
       result = status
 
   proc inputStream(p: Process): Stream =

lib/pure/parsesql.nim

@@ -672,8 +672,8 @@ proc getPrecedence(p: SqlParser): int =
   else:
     result = - 1
 
-proc parseExpr(p: var SqlParser): SqlNode
-proc parseSelect(p: var SqlParser): SqlNode
+proc parseExpr(p: var SqlParser): SqlNode {.gcsafe.}
+proc parseSelect(p: var SqlParser): SqlNode {.gcsafe.}
 
 proc identOrLiteral(p: var SqlParser): SqlNode =
   case p.tok.kind
@@ -1201,7 +1201,7 @@ proc addIden(s: var SqlWriter, thing: string) =
     iden = '"' & iden & '"'
   s.add(iden)
 
-proc ra(n: SqlNode, s: var SqlWriter)
+proc ra(n: SqlNode, s: var SqlWriter) {.gcsafe.}
 
 proc rs(n: SqlNode, s: var SqlWriter, prefix = "(", suffix = ")", sep = ", ") =
   if n.len > 0:

lib/pure/parsexml.nim

@@ -672,6 +672,9 @@ proc parseAttribute(my: var XmlParser) =
         parseEntity(my, my.b)
         my.kind = xmlAttribute # parseEntity overwrites my.kind!
         pos = my.bufpos
+      elif c == '/':
+        pos = lexbase.handleRefillChar(my, pos)
+        add(my.b, '/')
       else:
         add(my.b, c)
         inc(pos)

lib/pure/ropes.nim

@@ -204,8 +204,8 @@ proc `[]`*(r: Rope, i: int): char {.rtl, extern: "nroCharAt".} =
       if x.left.length > j:
         x = x.left
       else:
+        dec(j, x.left.length)
         x = x.right
-        dec(j, x.len)
 
 iterator leaves*(r: Rope): string =
   ## iterates over any leaf string in the rope `r`.
@@ -262,7 +262,7 @@ proc `%`*(frmt: string, args: openArray[Rope]): Rope {.
         while true:
           j = j * 10 + ord(frmt[i]) - ord('0')
           inc(i)
-          if frmt[i] notin {'0'..'9'}: break
+          if i >= frmt.len or frmt[i] notin {'0'..'9'}: break
         add(result, args[j-1])
       of '{':
         inc(i)

lib/pure/ssl_certs.nim

@@ -11,16 +11,48 @@
 ## SSL_CERT_DIR environment variables.
 
 import os, strutils
-from ospaths import existsEnv, getEnv
-import strutils
-
-# SECURITY: this unnecessarily scans through dirs/files regardless of the
-# actual host OS/distribution. Hopefully all the paths are writeble only by
-# root.
 
 # FWIW look for files before scanning entire dirs.
 
-const certificate_paths = [
+when defined(macosx):
+  const certificatePaths = [
+    "/etc/ssl/cert.pem",
+    "/System/Library/OpenSSL/certs/cert.pem"
+  ]
+elif defined(linux):
+  const certificatePaths = [
+    # Debian, Ubuntu, Arch: maintained by update-ca-certificates, SUSE, Gentoo
+    # NetBSD (security/mozilla-rootcerts)
+    # SLES10/SLES11, https://golang.org/issue/12139
+    "/etc/ssl/certs/ca-certificates.crt",
+    # OpenSUSE
+    "/etc/ssl/ca-bundle.pem",
+    # Red Hat 5+, Fedora, Centos
+    "/etc/pki/tls/certs/ca-bundle.crt",
+    # Red Hat 4
+    "/usr/share/ssl/certs/ca-bundle.crt",
+    # Fedora/RHEL
+    "/etc/pki/tls/certs",
+    # Android
+    "/system/etc/security/cacerts",
+  ]
+elif defined(bsd):
+  const certificatePaths = [
+    # Debian, Ubuntu, Arch: maintained by update-ca-certificates, SUSE, Gentoo
+    # NetBSD (security/mozilla-rootcerts)
+    # SLES10/SLES11, https://golang.org/issue/12139
+    "/etc/ssl/certs/ca-certificates.crt",
+    # FreeBSD (security/ca-root-nss package)
+    "/usr/local/share/certs/ca-root-nss.crt",
+    # OpenBSD, FreeBSD (optional symlink)
+    "/etc/ssl/cert.pem",
+    # FreeBSD
+    "/usr/local/share/certs",
+    # NetBSD
+    "/etc/openssl/certs",
+  ]
+else:
+  const certificatePaths = [
     # Debian, Ubuntu, Arch: maintained by update-ca-certificates, SUSE, Gentoo
     # NetBSD (security/mozilla-rootcerts)
     # SLES10/SLES11, https://golang.org/issue/12139
@@ -37,8 +69,6 @@ const certificate_paths = [
     "/etc/pki/ca-trust/extracted/pem/tls-ca-bundle.pem",
     # OpenBSD, FreeBSD (optional symlink)
     "/etc/ssl/cert.pem",
-    # Mac OS X
-    "/System/Library/OpenSSL/certs/cert.pem",
     # Fedora/RHEL
     "/etc/pki/tls/certs",
     # Android
@@ -47,7 +77,7 @@ const certificate_paths = [
     "/usr/local/share/certs",
     # NetBSD
     "/etc/openssl/certs",
-]
+  ]
 
 when defined(haiku):
   const
@@ -67,17 +97,30 @@ iterator scanSSLCertificates*(useEnvVars = false): string =
   ## if `useEnvVars` is true, the SSL_CERT_FILE and SSL_CERT_DIR
   ## environment variables can be used to override the certificate
   ## directories to scan or specify a CA certificate file.
-  if existsEnv("SSL_CERT_FILE"):
+  if useEnvVars and existsEnv("SSL_CERT_FILE"):
     yield getEnv("SSL_CERT_FILE")
 
-  elif existsEnv("SSL_CERT_DIR"):
+  elif useEnvVars and existsEnv("SSL_CERT_DIR"):
     let p = getEnv("SSL_CERT_DIR")
     for fn in joinPath(p, "*").walkFiles():
       yield fn
 
   else:
-    when not defined(haiku):
-      for p in certificate_paths:
+    when defined(windows):
+      const cacert = "cacert.pem"
+      let pem = getAppDir() / cacert
+      if fileExists(pem):
+        yield pem
+      else:
+        let path = getEnv("PATH")
+        for candidate in split(path, PathSep):
+          if candidate.len != 0:
+            let x = (if candidate[0] == '"' and candidate[^1] == '"':
+                      substr(candidate, 1, candidate.len-2) else: candidate) / cacert
+            if fileExists(x):
+              yield x
+    elif not defined(haiku):
+      for p in certificatePaths:
         if p.endsWith(".pem") or p.endsWith(".crt"):
           if existsFile(p):
             yield p

lib/pure/strscans.nim

@@ -396,7 +396,7 @@ macro scanf*(input: string; pattern: static[string]; results: varargs[typed]): b
           var resLen = genSym(nskLet, "resLen")
           conds.add newLetStmt(resLen, newCall(bindSym"parseUntil", inp,
               results[i], newLit(token), idx))
-          conds.add newCall(bindSym"!=", resLen, newLit min)
+          conds.add newCall(bindSym">=", resLen, newLit min)
           conds.add resLen
         else:
           matchError

lib/pure/strutils.nim

@@ -917,35 +917,39 @@ proc toOct*(x: BiggestInt, len: Positive): string {.noSideEffect,
     inc shift, 3
     mask = mask shl BiggestUInt(3)
 
-proc toHex*(x: BiggestInt, len: Positive): string {.noSideEffect,
-  rtl, extern: "nsuToHex".} =
+proc toHexImpl(x: BiggestUInt, len: Positive, handleNegative: bool): string {.noSideEffect.} =
+  const
+    HexChars = "0123456789ABCDEF"
+  var n = x
+  result = newString(len)
+  for j in countdown(len-1, 0):
+    result[j] = HexChars[int(n and 0xF)]
+    n = n shr 4
+    # handle negative overflow
+    if n == 0 and handleNegative: n = not(BiggestUInt 0)
+
+proc toHex*[T: SomeInteger](x: T, len: Positive): string {.noSideEffect.} =
   ## Converts `x` to its hexadecimal representation.
   ##
   ## The resulting string will be exactly `len` characters long. No prefix like
   ## ``0x`` is generated. `x` is treated as an unsigned value.
   runnableExamples:
     let
-      a = 62
-      b = 4097
+      a = 62'u64
+      b = 4097'u64
     doAssert a.toHex(3) == "03E"
     doAssert b.toHex(3) == "001"
     doAssert b.toHex(4) == "1001"
-  const
-    HexChars = "0123456789ABCDEF"
-  var
-    n = x
-  result = newString(len)
-  for j in countdown(len-1, 0):
-    result[j] = HexChars[int(n and 0xF)]
-    n = n shr 4
-    # handle negative overflow
-    if n == 0 and x < 0: n = -1
+    doAssert toHex(62, 3) == "03E"
+    doAssert toHex(-8, 6) == "FFFFF8"
+  toHexImpl(cast[BiggestUInt](x), len, x < 0)
 
-proc toHex*[T: SomeInteger](x: T): string =
+proc toHex*[T: SomeInteger](x: T): string {.noSideEffect.} =
   ## Shortcut for ``toHex(x, T.sizeof * 2)``
   runnableExamples:
     doAssert toHex(1984'i64) == "00000000000007C0"
-  toHex(BiggestInt(x), T.sizeof * 2)
+    doAssert toHex(1984'i16) == "07C0"
+  toHexImpl(cast[BiggestUInt](x), 2*sizeof(T), x < 0)
 
 proc toHex*(s: string): string {.noSideEffect, rtl.} =
   ## Converts a bytes string to its hexadecimal representation.
@@ -1806,7 +1810,7 @@ proc initSkipTable*(a: var SkipTable, sub: string)
 
 proc find*(a: SkipTable, s, sub: string, start: Natural = 0, last = 0): int
   {.noSideEffect, rtl, extern: "nsuFindStrA".} =
-  ## Searches for `sub` in `s` inside range `start`..`last` using preprocessed
+  ## Searches for `sub` in `s` inside range `start..last` using preprocessed
   ## table `a`. If `last` is unspecified, it defaults to `s.high` (the last
   ## element).
   ##

lib/pure/sugar.nim

@@ -188,7 +188,7 @@ macro capture*(locals: varargs[typed], body: untyped): untyped {.since: (1, 1).}
   for arg in locals:
     params.add(newIdentDefs(ident(arg.strVal), freshIdentNodes getTypeInst arg))
   result = newNimNode(nnkCall)
-  result.add(newProc(newEmptyNode(), params, body, nnkProcDef))
+  result.add(newProc(newEmptyNode(), params, body, nnkLambda))
   for arg in locals: result.add(arg)
 
 when (NimMajor, NimMinor) >= (1, 1):
@@ -369,3 +369,13 @@ when isMainModule:
         of "bird": "word"
         else: d
     assert z == @["word", "word"]
+
+
+    proc tforum =
+      let ans = collect(newSeq):
+        for y in 0..10:
+          if y mod 5 == 2:
+            for x in 0..y:
+              x
+
+    tforum()

lib/pure/terminal.nim

@@ -689,10 +689,9 @@ template styledEchoProcessArg(f: File, color: Color) =
 template styledEchoProcessArg(f: File, cmd: TerminalCmd) =
   when cmd == resetStyle:
     resetAttributes(f)
-  when cmd == fgColor:
-    fgSetColor = true
-  when cmd == bgColor:
-    fgSetColor = false
+  elif cmd in {fgColor, bgColor}:
+    let term = getTerminal()
+    term.fgSetColor = cmd == fgColor
 
 macro styledWrite*(f: File, m: varargs[typed]): untyped =
   ## Similar to ``write``, but treating terminal style arguments specially.
@@ -776,25 +775,20 @@ when defined(windows):
     ## ``true`` otherwise.
     password.string.setLen(0)
     stdout.write(prompt)
-    while true:
-      let c = getch()
-      case c.char
-      of '\r', chr(0xA):
-        break
-      of '\b':
-        # ensure we delete the whole UTF-8 character:
-        var i = 0
-        var x = 1
-        while i < password.len:
-          x = runeLenAt(password.string, i)
-          inc i, x
-        password.string.setLen(max(password.len - x, 0))
-      of chr(0x0):
-        # modifier key - ignore - for details see
-        # https://github.com/nim-lang/Nim/issues/7764
-        continue
-      else:
-        password.string.add(toUTF8(c.Rune))
+    let hi = createFileA("CONIN$",
+      GENERIC_READ or GENERIC_WRITE, 0, nil, OPEN_EXISTING, 0, 0)
+    var mode = DWORD 0
+    discard getConsoleMode(hi, addr mode)
+    let origMode = mode
+    const
+      ENABLE_PROCESSED_INPUT = 1
+      ENABLE_ECHO_INPUT = 4
+    mode = (mode or ENABLE_PROCESSED_INPUT) and not ENABLE_ECHO_INPUT
+
+    discard setConsoleMode(hi, mode)
+    result = readLine(stdin, password)
+    discard setConsoleMode(hi, origMode)
+    discard closeHandle(hi)
     stdout.write "\n"
 
 else:

lib/pure/uri.nim

@@ -284,7 +284,7 @@ proc removeDotSegments(path: string): string =
           discard collection.pop()
           i.inc 3
           continue
-      elif path[i+1] == '/':
+      elif i + 1 < path.len and path[i+1] == '/':
         i.inc 2
         continue
       currentSegment.add path[i]

lib/std/monotimes.nim

@@ -74,7 +74,7 @@ when defined(js):
     system.`+`(a, b)
   {.pop.}
 
-elif defined(posix):
+elif defined(posix) and not defined(osx):
   import posix
 
 elif defined(windows):

lib/std/sha1.nim

@@ -213,7 +213,22 @@ proc secureHashFile*(filename: string): SecureHash =
   ## **See also:**
   ## * `secureHash proc <#secureHash,openArray[char]>`_ for generating a ``SecureHash`` from a string
   ## * `parseSecureHash proc <#parseSecureHash,string>`_ for converting a string ``hash`` to ``SecureHash``
-  secureHash(readFile(filename))
+  const BufferLength = 8192
+
+  let f = open(filename)
+  var state = newSha1State()
+  var buffer = newString(BufferLength)
+  while true:
+    let length = readChars(f, buffer, 0, BufferLength)
+    if length == 0:
+      break
+    buffer.setLen(length)
+    state.update(buffer)
+    if length != BufferLength:
+      break
+  close(f)
+
+  SecureHash(state.finalize())
 
 proc `$`*(self: SecureHash): string =
   ## Returns the string representation of a ``SecureHash``.

lib/std/wrapnils.nim

@@ -1,6 +1,6 @@
 ## This module allows chains of field-access and indexing where the LHS can be nil.
 ## This simplifies code by reducing need for if-else branches around intermediate values
-## that maybe be nil.
+## that may be nil.
 ##
 ## Note: experimental module and relies on {.experimental: "dotOperators".}
 ## Unstable API.
@@ -82,7 +82,7 @@ template `[]`*(a: Wrapnil): untyped =
   else:
     default(T)
 
-import std/macros
+import macros
 
 proc replace(n: NimNode): NimNode =
   if n.kind == nnkPar:

lib/system.nim

@@ -1357,6 +1357,12 @@ type # these work for most platforms:
   culonglong* {.importc: "unsigned long long", nodecl.} = uint64
     ## This is the same as the type ``unsigned long long`` in *C*.
 
+  # There is a disparity on macOS where Nim's `uint` is `unsigned long long` and
+  # `uintptr_t` is `unsigned long`. Even though both data types are the same
+  # size (64 bits), clang++ refuses to do automatic conversion between them.
+  cuintptr_t* {.importc: "uintptr_t", nodecl.} = uint
+    ## This is the same as the type ``uintptr_t`` in *C*.
+
   cstringArray* {.importc: "char**", nodecl.} = ptr UncheckedArray[cstring]
     ## This is binary compatible to the type ``char**`` in *C*. The array's
     ## high value is large enough to disable bounds checking in practice.
@@ -2052,7 +2058,7 @@ const
   NimMinor* {.intdefine.}: int = 2
     ## is the minor number of Nim's version.
 
-  NimPatch* {.intdefine.}: int = 6
+  NimPatch* {.intdefine.}: int = 14
     ## is the patch number of Nim's version.
 
   NimVersion*: string = $NimMajor & "." & $NimMinor & "." & $NimPatch

lib/system/alloc.nim

@@ -250,7 +250,7 @@ proc llAlloc(a: var MemRegion, size: int): pointer =
     sysAssert roundup(size+sizeof(LLChunk), PageSize) == PageSize, "roundup 6"
     var old = a.llmem # can be nil and is correct with nil
     a.llmem = cast[PLLChunk](osAllocPages(PageSize))
-    when defined(avlcorruption):
+    when defined(nimAvlcorruption):
       trackLocation(a.llmem, PageSize)
     incCurrMem(a, PageSize)
     a.llmem.size = PageSize - sizeof(LLChunk)
@@ -273,7 +273,7 @@ proc allocAvlNode(a: var MemRegion, key, upperBound: int): PAvlNode =
     a.freeAvlNodes = a.freeAvlNodes.link[0]
   else:
     result = cast[PAvlNode](llAlloc(a, sizeof(AvlNode)))
-    when defined(avlcorruption):
+    when defined(nimAvlcorruption):
       cprintf("tracking location: %p\n", result)
   result.key = key
   result.upperBound = upperBound
@@ -281,7 +281,7 @@ proc allocAvlNode(a: var MemRegion, key, upperBound: int): PAvlNode =
   result.link[0] = bottom
   result.link[1] = bottom
   result.level = 1
-  #when defined(avlcorruption):
+  #when defined(nimAvlcorruption):
   #  track("allocAvlNode", result, sizeof(AvlNode))
   sysAssert(bottom == addr(a.bottomData), "bottom data")
   sysAssert(bottom.link[0] == bottom, "bottom link[0]")
@@ -1013,7 +1013,7 @@ when defined(nimTypeNames):
 template instantiateForRegion(allocator: untyped) {.dirty.} =
   {.push stackTrace: off.}
 
-  when defined(fulldebug):
+  when defined(nimFulldebug):
     proc interiorAllocatedPtr*(p: pointer): pointer =
       result = interiorAllocatedPtr(allocator, p)
 

lib/system/exceptions.nim

@@ -25,6 +25,11 @@ type
                             ## rendered at a later time, we should ensure the stacktrace
                             ## data isn't invalidated; any pointer into PFrame is
                             ## subject to being invalidated so shouldn't be stored.
+    when defined(nimStackTraceOverride):
+      programCounter*: uint ## Program counter - will be used to get the rest of the info,
+                            ## when `$` is called on this type. We can't use
+                            ## "cuintptr_t" in here.
+      procnameStr*, filenameStr*: string ## GC-ed alternatives to "procname" and "filename"
 
   Exception* {.compilerproc, magic: "Exception".} = object of RootObj ## \
     ## Base exception class.

lib/system/excpt.nim

@@ -10,6 +10,8 @@
 # Exception handling code. Carefully coded so that tiny programs which do not
 # use the heap (and nor exceptions) do not include the GC or memory allocator.
 
+import stacktraces
+
 var
   errorMessageWriter*: (proc(msg: string) {.tags: [WriteIOEffect], benign,
                                             nimcall.})
@@ -133,20 +135,6 @@ const
   hasSomeStackTrace = NimStackTrace or defined(nimStackTraceOverride) or
     (defined(nativeStackTrace) and nativeStackTraceSupported)
 
-when defined(nimStackTraceOverride):
-  type StackTraceOverrideProc* = proc (): string {.nimcall, noinline, benign, raises: [], tags: [].}
-    ## Procedure type for overriding the default stack trace.
-
-  var stackTraceOverrideGetTraceback: StackTraceOverrideProc = proc(): string {.noinline.} =
-    result = "Stack trace override procedure not registered.\n"
-
-  proc registerStackTraceOverride*(overrideProc: StackTraceOverrideProc) =
-    ## Override the default stack trace inside rawWriteStackTrace() with your
-    ## own procedure.
-    stackTraceOverrideGetTraceback = overrideProc
-
-  proc auxWriteStackTraceWithOverride(s: var string) =
-    add(s, stackTraceOverrideGetTraceback())
 
 when defined(nativeStacktrace) and nativeStackTraceSupported:
   type
@@ -164,13 +152,13 @@ when defined(nativeStacktrace) and nativeStackTraceSupported:
 
   when not hasThreadSupport:
     var
-      tempAddresses: array[0..127, pointer] # should not be alloc'd on stack
+      tempAddresses: array[maxStackTraceLines, pointer] # should not be alloc'd on stack
       tempDlInfo: TDl_info
 
   proc auxWriteStackTraceWithBacktrace(s: var string) =
     when hasThreadSupport:
       var
-        tempAddresses: array[0..127, pointer] # but better than a threadvar
+        tempAddresses: array[maxStackTraceLines, pointer] # but better than a threadvar
         tempDlInfo: TDl_info
     # This is allowed to be expensive since it only happens during crashes
     # (but this way you don't need manual stack tracing)
@@ -198,11 +186,7 @@ when defined(nativeStacktrace) and nativeStackTraceSupported:
 
 when hasSomeStackTrace and not hasThreadSupport:
   var
-    tempFrames: array[0..127, PFrame] # should not be alloc'd on stack
-
-const
-  reraisedFromBegin = -10
-  reraisedFromEnd = -100
+    tempFrames: array[maxStackTraceLines, PFrame] # should not be alloc'd on stack
 
 template reraisedFrom(z): untyped =
   StackTraceEntry(procname: nil, line: z, filename: nil)
@@ -253,7 +237,12 @@ template addFrameEntry(s: var string, f: StackTraceEntry|PFrame) =
       for i in first..<f.frameMsgLen: add(s, frameMsgBuf[i])
   add(s, "\n")
 
-proc `$`(s: seq[StackTraceEntry]): string =
+proc `$`(stackTraceEntries: seq[StackTraceEntry]): string =
+  when defined(nimStackTraceOverride):
+    let s = addDebuggingInfo(stackTraceEntries)
+  else:
+    let s = stackTraceEntries
+
   result = newStringOfCap(2000)
   for i in 0 .. s.len-1:
     if s[i].line == reraisedFromBegin: result.add "[[reraised from:\n"
@@ -265,7 +254,7 @@ when hasSomeStackTrace:
   proc auxWriteStackTrace(f: PFrame, s: var string) =
     when hasThreadSupport:
       var
-        tempFrames: array[0..127, PFrame] # but better than a threadvar
+        tempFrames: array[maxStackTraceLines, PFrame] # but better than a threadvar
     const
       firstCalls = 32
     var
@@ -324,7 +313,9 @@ when hasSomeStackTrace:
       add(s, "No stack traceback available\n")
 
   proc rawWriteStackTrace(s: var seq[StackTraceEntry]) =
-    when NimStackTrace:
+    when defined(nimStackTraceOverride):
+      auxWriteStackTraceWithOverride(s)
+    elif NimStackTrace:
       auxWriteStackTrace(framePtr, s)
     else:
       s = @[]
@@ -463,7 +454,12 @@ proc raiseExceptionEx(e: sink(ref Exception), ename, procname, filename: cstring
   if e.name.isNil: e.name = ename
   when hasSomeStackTrace:
     when defined(nimStackTraceOverride):
-      e.trace = @[]
+      if e.trace.len == 0:
+        rawWriteStackTrace(e.trace)
+      else:
+        e.trace.add reraisedFrom(reraisedFromBegin)
+        auxWriteStackTraceWithOverride(e.trace)
+        e.trace.add reraisedFrom(reraisedFromEnd)
     elif NimStackTrace:
       if e.trace.len == 0:
         rawWriteStackTrace(e.trace)

lib/system/gc.nim

@@ -436,7 +436,7 @@ proc newObjNoInit(typ: PNimType, size: int): pointer {.compilerRtl.} =
   result = rawNewObj(typ, size, gch)
   when defined(memProfiler): nimProfile(size)
 
-proc newObj(typ: PNimType, size: int): pointer {.compilerRtl.} =
+proc newObj(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} =
   result = rawNewObj(typ, size, gch)
   zeroMem(result, size)
   when defined(memProfiler): nimProfile(size)
@@ -451,7 +451,7 @@ proc newSeq(typ: PNimType, len: int): pointer {.compilerRtl.} =
   when defined(memProfiler): nimProfile(size)
 {.pop.}
 
-proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl.} =
+proc newObjRC1(typ: PNimType, size: int): pointer {.compilerRtl, noinline.} =
   # generates a new object and sets its reference counter to 1
   incTypeSize typ, size
   sysAssert(allocInv(gch.region), "newObjRC1 begin")
@@ -662,16 +662,16 @@ proc collectCycles(gch: var GcHeap) =
 proc gcMark(gch: var GcHeap, p: pointer) {.inline.} =
   # the addresses are not as cells on the stack, so turn them to cells:
   sysAssert(allocInv(gch.region), "gcMark begin")
-  var cell = usrToCell(p)
-  var c = cast[ByteAddress](cell)
+  var c = cast[ByteAddress](p)
   if c >% PageSize:
     # fast check: does it look like a cell?
-    var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell))
+    var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p))
     if objStart != nil:
       # mark the cell:
       incRef(objStart)
       add(gch.decStack, objStart)
     when false:
+      let cell = usrToCell(p)
       if isAllocatedPtr(gch.region, cell):
         sysAssert false, "allocated pointer but not interior?"
         # mark the cell:

lib/system/gc_ms.nim

@@ -454,11 +454,10 @@ proc markGlobals(gch: var GcHeap) =
 
 proc gcMark(gch: var GcHeap, p: pointer) {.inline.} =
   # the addresses are not as cells on the stack, so turn them to cells:
-  var cell = usrToCell(p)
-  var c = cast[ByteAddress](cell)
+  var c = cast[ByteAddress](p)
   if c >% PageSize:
     # fast check: does it look like a cell?
-    var objStart = cast[PCell](interiorAllocatedPtr(gch.region, cell))
+    var objStart = cast[PCell](interiorAllocatedPtr(gch.region, p))
     if objStart != nil:
       mark(gch, objStart)
 

lib/system/mmdisp.nim

@@ -17,10 +17,10 @@ const
   debugGC = false # we wish to debug the GC...
   logGC = false
   traceGC = false # extensive debugging
-  alwaysCycleGC = defined(smokeCycles)
-  alwaysGC = defined(fulldebug) # collect after every memory
+  alwaysCycleGC = defined(nimSmokeCycles)
+  alwaysGC = defined(nimFulldebug) # collect after every memory
                                 # allocation (for debugging)
-  leakDetector = defined(leakDetector)
+  leakDetector = defined(nimLeakDetector)
   overwriteFree = defined(nimBurnFree) # overwrite memory with 0xFF before free
   trackAllocationSource = leakDetector
 
@@ -30,7 +30,7 @@ const
   coalescRight = true
   coalescLeft = true
   logAlloc = false
-  useCellIds = defined(corruption)
+  useCellIds = defined(nimCorruption)
 
 type
   PPointer = ptr pointer

lib/system/stacktraces.nim

@@ -0,0 +1,83 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2015 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# Additional code for customizable stack traces. Unstable API, for internal
+# usage only.
+
+const
+  reraisedFromBegin* = -10
+  reraisedFromEnd* = -100
+  maxStackTraceLines* = 128
+
+when defined(nimStackTraceOverride):
+  ## Procedure types for overriding the default stack trace.
+  type
+    cuintptr_t {.importc: "uintptr_t", nodecl.} = uint
+      ## This is the same as the type ``uintptr_t`` in C.
+
+    StackTraceOverrideGetTracebackProc* = proc (): string {.
+      nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.}
+    StackTraceOverrideGetProgramCountersProc* = proc (maxLength: cint): seq[cuintptr_t] {.
+      nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.}
+    StackTraceOverrideGetDebuggingInfoProc* =
+      proc (programCounters: seq[cuintptr_t], maxLength: cint): seq[StackTraceEntry] {.
+        nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.}
+
+  # Default procedures (not normally used, because people opting in on this
+  # override are supposed to register their own versions).
+  var
+    stackTraceOverrideGetTraceback: StackTraceOverrideGetTracebackProc =
+      proc (): string {.nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.} =
+        discard
+        #result = "Stack trace override procedure not registered.\n"
+    stackTraceOverrideGetProgramCounters: StackTraceOverrideGetProgramCountersProc =
+      proc (maxLength: cint): seq[cuintptr_t] {.nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.} =
+        discard
+    stackTraceOverrideGetDebuggingInfo: StackTraceOverrideGetDebuggingInfoProc =
+      proc (programCounters: seq[cuintptr_t], maxLength: cint): seq[StackTraceEntry] {.
+        nimcall, gcsafe, locks: 0, raises: [], tags: [], noinline.} =
+          discard
+
+  # Custom procedure registration.
+  proc registerStackTraceOverride*(overrideProc: StackTraceOverrideGetTracebackProc) =
+    ## Override the default stack trace inside rawWriteStackTrace() with your
+    ## own procedure.
+    stackTraceOverrideGetTraceback = overrideProc
+  proc registerStackTraceOverrideGetProgramCounters*(overrideProc: StackTraceOverrideGetProgramCountersProc) =
+    stackTraceOverrideGetProgramCounters = overrideProc
+  proc registerStackTraceOverrideGetDebuggingInfo*(overrideProc: StackTraceOverrideGetDebuggingInfoProc) =
+    stackTraceOverrideGetDebuggingInfo = overrideProc
+
+  # Custom stack trace manipulation.
+  proc auxWriteStackTraceWithOverride*(s: var string) =
+    add(s, stackTraceOverrideGetTraceback())
+
+  proc auxWriteStackTraceWithOverride*(s: var seq[StackTraceEntry]) =
+    let programCounters = stackTraceOverrideGetProgramCounters(maxStackTraceLines)
+    if s.len == 0:
+      s = newSeqOfCap[StackTraceEntry](programCounters.len)
+    for programCounter in programCounters:
+      s.add(StackTraceEntry(programCounter: cast[uint](programCounter)))
+
+  # We may have more stack trace lines in the output, due to inlined procedures.
+  proc addDebuggingInfo*(s: seq[StackTraceEntry]): seq[StackTraceEntry] =
+    var programCounters: seq[cuintptr_t]
+    # We process program counters in groups from complete stack traces, because
+    # we have logic that keeps track of certain functions being inlined or not.
+    for entry in s:
+      if entry.procname.isNil and entry.programCounter != 0:
+        programCounters.add(cast[cuintptr_t](entry.programCounter))
+      elif entry.procname.isNil and (entry.line == reraisedFromBegin or entry.line == reraisedFromEnd):
+        result.add(stackTraceOverrideGetDebuggingInfo(programCounters, maxStackTraceLines))
+        programCounters = @[]
+        result.add(entry)
+      else:
+        result.add(entry)
+    if programCounters.len > 0:
+      result.add(stackTraceOverrideGetDebuggingInfo(programCounters, maxStackTraceLines))

lib/system/strmantle.nim

@@ -248,15 +248,16 @@ proc nimParseBiggestFloat(s: string, number: var BiggestFloat,
   var ti = 0
   let maxlen = t.high - "e+000".len # reserve enough space for exponent
 
-  result = i - start
+  let endPos = i
+  result = endPos - start
   i = start
   # re-parse without error checking, any error should be handled by the code above.
-  if i < s.len and s[i] == '.': i.inc
-  while i < s.len and s[i] in {'0'..'9','+','-'}:
+  if i < endPos and s[i] == '.': i.inc
+  while i < endPos and s[i] in {'0'..'9','+','-'}:
     if ti < maxlen:
       t[ti] = s[i]; inc(ti)
     inc(i)
-    while i < s.len and s[i] in {'.', '_'}: # skip underscore and decimal point
+    while i < endPos and s[i] in {'.', '_'}: # skip underscore and decimal point
       inc(i)
 
   # insert exponent

lib/system/syslocks.nim

@@ -145,13 +145,13 @@ else:
       SysCond = ptr SysCondObj
 
     when not declared(c_malloc):
-      proc c_malloc(size: csize): pointer {.
+      proc c_malloc(size: csize_t): pointer {.
         importc: "malloc", header: "<stdlib.h>".}
       proc c_free(p: pointer) {.
         importc: "free", header: "<stdlib.h>".}
 
     proc initSysLock(L: var SysLock, attr: ptr SysLockAttr = nil) =
-      L = cast[SysLock](c_malloc(sizeof(SysLockObj)))
+      L = cast[SysLock](c_malloc(csize_t(sizeof(SysLockObj))))
       initSysLockAux(L[], attr)
 
     proc deinitSys(L: var SysLock) =
@@ -201,7 +201,7 @@ else:
 
     when defined(ios):
       proc initSysCond(cond: var SysCond, cond_attr: ptr SysCondAttr = nil) =
-        cond = cast[SysCond](c_malloc(sizeof(SysCondObj)))
+        cond = cast[SysCond](c_malloc(csize_t(sizeof(SysCondObj))))
         initSysCondAux(cond[], cond_attr)
 
       proc deinitSysCond(cond: var SysCond) =

lib/system/threads.nim

@@ -311,7 +311,10 @@ else:
     when hasSharedHeap: t.core.stackSize = ThreadStackSize
     var a {.noinit.}: Pthread_attr
     doAssert pthread_attr_init(a) == 0
-    doAssert pthread_attr_setstacksize(a, ThreadStackSize) == 0
+    let setstacksizeResult = pthread_attr_setstacksize(a, ThreadStackSize)
+    when not defined(ios):
+      # This fails on iOS
+      doAssert(setstacksizeResult == 0)
     if pthread_create(t.sys, a, threadProcWrapper[TArg], addr(t)) != 0:
       raise newException(ResourceExhaustedError, "cannot create thread")
     doAssert pthread_attr_destroy(a) == 0

lib/windows/winlean.nim

@@ -717,22 +717,20 @@ proc setHandleInformation*(hObject: Handle, dwMask: DWORD,
 proc getCurrentProcess*(): Handle{.stdcall, dynlib: "kernel32",
                                    importc: "GetCurrentProcess".}
 
-when useWinUnicode:
-  proc createFileW*(lpFileName: WideCString, dwDesiredAccess, dwShareMode: DWORD,
-                    lpSecurityAttributes: pointer,
-                    dwCreationDisposition, dwFlagsAndAttributes: DWORD,
-                    hTemplateFile: Handle): Handle {.
-      stdcall, dynlib: "kernel32", importc: "CreateFileW".}
-  proc deleteFileW*(pathName: WideCString): int32 {.
-    importc: "DeleteFileW", dynlib: "kernel32", stdcall.}
-else:
-  proc createFileA*(lpFileName: cstring, dwDesiredAccess, dwShareMode: DWORD,
-                    lpSecurityAttributes: pointer,
-                    dwCreationDisposition, dwFlagsAndAttributes: DWORD,
-                    hTemplateFile: Handle): Handle {.
-      stdcall, dynlib: "kernel32", importc: "CreateFileA".}
-  proc deleteFileA*(pathName: cstring): int32 {.
-    importc: "DeleteFileA", dynlib: "kernel32", stdcall.}
+proc createFileW*(lpFileName: WideCString, dwDesiredAccess, dwShareMode: DWORD,
+                  lpSecurityAttributes: pointer,
+                  dwCreationDisposition, dwFlagsAndAttributes: DWORD,
+                  hTemplateFile: Handle): Handle {.
+    stdcall, dynlib: "kernel32", importc: "CreateFileW".}
+proc deleteFileW*(pathName: WideCString): int32 {.
+  importc: "DeleteFileW", dynlib: "kernel32", stdcall.}
+proc createFileA*(lpFileName: cstring, dwDesiredAccess, dwShareMode: DWORD,
+                  lpSecurityAttributes: pointer,
+                  dwCreationDisposition, dwFlagsAndAttributes: DWORD,
+                  hTemplateFile: Handle): Handle {.
+    stdcall, dynlib: "kernel32", importc: "CreateFileA".}
+proc deleteFileA*(pathName: cstring): int32 {.
+  importc: "DeleteFileA", dynlib: "kernel32", stdcall.}
 
 proc setEndOfFile*(hFile: Handle): WINBOOL {.stdcall, dynlib: "kernel32",
     importc: "SetEndOfFile".}

lib/wrappers/openssl.nim

@@ -64,9 +64,9 @@ elif useWinVersion:
   from winlean import SocketHandle
 else:
   when defined(osx):
-    const versions = "(.1.1|.38|.39|.41|.43|.44|.45|.46|.47|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)"
+    const versions = "(.1.1|.38|.39|.41|.43|.44|.45|.46|.47|.48|.10|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|)"
   else:
-    const versions = "(.1.1|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.47|.46|.45|.44|.43|.41|.39|.38|.10|)"
+    const versions = "(.1.1|.1.0.2|.1.0.1|.1.0.0|.0.9.9|.0.9.8|.48|.47|.46|.45|.44|.43|.41|.39|.38|.10|)"
 
   when defined(macosx):
     const
@@ -719,6 +719,8 @@ when not defined(nimDisableCertificateValidation) and not defined(windows):
 
   proc X509_check_host*(cert: PX509, name: cstring, namelen: cint, flags:cuint, peername: cstring): cint {.cdecl, dynlib: DLLSSLName, importc.}
 
+  proc X509_free*(cert: PX509) {.cdecl, dynlib: DLLSSLName, importc.}
+
   # Certificates store
 
   type PX509_STORE* = SslPtr

testament/categories.nim

@@ -217,6 +217,7 @@ proc gcTests(r: var TResults, cat: Category, options: string) =
 
   test "stackrefleak"
   test "cyclecollector"
+  test "trace_globals"
 
 proc longGCTests(r: var TResults, cat: Category, options: string) =
   when defined(windows):

testament/important_packages.nim

@@ -24,14 +24,14 @@ pkg "coco", true
 pkg "combparser"
 pkg "compactdict"
 pkg "comprehension", false, "nimble test", "https://github.com/alehander42/comprehension"
-pkg "criterion"
+# pkg "criterion"
 pkg "dashing", false, "nim c tests/functional.nim"
 pkg "docopt"
 pkg "easygl", true, "nim c -o:egl -r src/easygl.nim", "https://github.com/jackmott/easygl"
 pkg "elvis"
 pkg "fragments", false, "nim c -r fragments/dsl.nim"
 pkg "gara"
-pkg "ggplotnim", true, "nimble testCI"
+# pkg "ggplotnim", true, "nimble testCI"
 pkg "glob"
 pkg "gnuplot"
 pkg "hts", false, "nim c -o:htss src/hts.nim"
@@ -52,21 +52,21 @@ pkg "nimcrypto", false, "nim c -r tests/testall.nim"
 pkg "NimData", true, "nim c -o:nimdataa src/nimdata.nim"
 pkg "nimes", true, "nim c src/nimes.nim"
 pkg "nimfp", true, "nim c -o:nfp -r src/fp.nim"
-pkg "nimgame2", true, "nim c nimgame2/nimgame.nim"
-pkg "nimgen", true, "nim c -o:nimgenn -r src/nimgen/runcfg.nim"
+# pkg "nimgame2", true, "nim c nimgame2/nimgame.nim"
+# pkg "nimgen", true, "nim c -o:nimgenn -r src/nimgen/runcfg.nim"
 # pkg "nimlsp", true
-pkg "nimly", true
+# pkg "nimly", true
 # pkg "nimongo", true, "nimble test_ci"
 pkg "nimpy", false, "nim c -r tests/nimfrompy.nim"
-pkg "nimquery"
+# pkg "nimquery"
 pkg "nimsl", true
 pkg "nimsvg"
 # pkg "nimterop", true
 # pkg "nimx", true, "nim c --threads:on test/main.nim"
-pkg "norm", true, "nim c -r tests/tsqliterows.nim"
+# pkg "norm", true, "nim c -r tests/tsqliterows.nim"
 pkg "npeg"
 pkg "ormin", true, "nim c -o:orminn ormin.nim"
-pkg "parsetoml"
+# pkg "parsetoml"
 pkg "patty"
 pkg "plotly", true, "nim c --oldgensym:on examples/all.nim"
 pkg "pnm"
@@ -91,6 +91,6 @@ pkg "unicodeplus", true
 pkg "unpack"
 # pkg "winim", true
 pkg "with"
-pkg "ws"
+# pkg "ws"
 pkg "yaml"
 pkg "zero_functional", false, "nim c -r test.nim"

tests/array/tarray.nim

@@ -590,3 +590,14 @@ block t12466:
     a[0'u16 + i] = i
   for i in 0'u16 ..< 8'u16:
     a[0'u16 + i] = i
+
+block t18643:
+  # https://github.com/nim-lang/Nim/issues/18643
+  let a: array[0, int] = []
+  var caught = false
+  let b = 9999999
+  try:
+    echo a[b]
+  except IndexError:
+    caught = true
+  doAssert caught, "IndexError not caught!"

tests/async/t7758.nim

@@ -1,7 +1,8 @@
 import asyncdispatch
 
 proc task() {.async.} =
-  await sleepAsync(40)
+  const tSleep = 40
+  await sleepAsync(tSleep)
 
 proc main() =
   var counter = 0
@@ -10,6 +11,10 @@ proc main() =
     inc(counter)
     poll(10)
 
-  doAssert counter <= 4
+  const slack = 1
+    # because there is overhead in `async` + `sleepAsync`
+    # as can be seen by increasing `tSleep` from 40 to 49, which increases the number
+    # of failures.
+  doAssert counter <= 4 + slack
 
 for i in 0 .. 4: main()

tests/async/tasync_traceback.nim

@@ -86,7 +86,7 @@ Async traceback:
     asyncfutures\.nim\(\d+?\)\s+?read
   \]#
 Exception message: b failure
-Exception type:
+
 
 bar failure
 Async traceback:
@@ -114,7 +114,7 @@ Async traceback:
     asyncfutures\.nim\(\d+?\)\s+?read
   \]#
 Exception message: bar failure
-Exception type:
+
 """
 
 let resLines = splitLines(result.strip)

tests/async/tbreak_must_exec_finally.nim

@@ -0,0 +1,25 @@
+discard """
+  output: '''
+finally handler 8
+do not duplicate this one
+'''
+"""
+
+# bug #15243
+
+import asyncdispatch
+
+proc f() {.async.} =
+  try:
+    while true:
+      try:
+        await sleepAsync(400)
+        break
+      finally:
+        var localHere = 8
+        echo "finally handler ", localHere
+  finally:
+    echo "do not duplicate this one"
+
+when isMainModule:
+  waitFor f()

tests/async/tfuturevar.nim

@@ -1,8 +1,3 @@
-discard """
-action: compile
-"""
-# XXX: action should be run!
-
 import asyncdispatch
 
 proc completeOnReturn(fut: FutureVar[string], x: bool) {.async.} =