bump NimVersion to 1.2.16

* json: limit recursion depth

* do not run this check for JS backend

(cherry picked from commit c17baaefbc)

.builds/freebsd.yml

@@ -1,27 +1,21 @@
 # see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd
 image: freebsd/latest
-
-# packages:
-# - databases/sqlite3
-# - devel/boehm-gc-threaded
-# - devel/pcre
-# - devel/sdl20
-# - devel/sfml
-# - www/node
-# - devel/gmake
-# - devel/git
+packages:
+- databases/sqlite3
+- devel/boehm-gc-threaded
+- devel/pcre
+- devel/sdl20
+- devel/sfml
+- www/node
+- devel/gmake
 sources:
 - https://github.com/nim-lang/Nim
 environment:
   CC: /usr/bin/clang
 tasks:
 - setup: |
-    # workaround https://github.com/timotheecour/Nim/issues/76
-    sudo pkg update -q -f
-    sudo pkg install -y -q databases/sqlite3 devel/boehm-gc-threaded devel/pcre \
-      devel/sdl20 devel/sfml www/node devel/gmake devel/git
     cd Nim
-    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    git clone --depth 1 -q https://github.com/nim-lang/csources_v1.git csources
     gmake -C csources -j $(sysctl -n hw.ncpu)
     bin/nim c --skipUserCfg --skipParentCfg koch
     echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv

.gitattributes

@@ -0,0 +1,6 @@
+# Avoids changelog conflicts by assuming additions-only, which is by far the common case.
+# In the rare case where branch b1 rebases against branch b2 and both branches
+# modified the same changelog entry, you'll end up with that changelog entry
+# duplicated, which is easily identifiable and fixable.
+/changelog.md merge=union
+

.github/workflows/ci.yml.disabled

@@ -85,11 +85,11 @@ jobs:
           curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
           7z x dist/mingw64.7z -odist
           7z x dist/dlls.zip -obin
-          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
-        run: echo "::add-path::${{ github.workspace }}/bin"
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
       - name: 'Build csources'
         shell: bash

.github/workflows/ci_docs.yml

@@ -49,17 +49,17 @@ jobs:
           curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
           7z x dist/mingw64.7z -odist
           7z x dist/dlls.zip -obin
-          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
-        run: echo "::add-path::${{ github.workspace }}/bin"
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
       - name: 'Get current csources version'
         id: csources-version
         shell: bash
         run: |
-          sha=$(git ls-remote https://github.com/nim-lang/csources master | cut -f 1)
+          sha=$(git ls-remote https://github.com/nim-lang/csources_v1 master | cut -f 1)
           echo "::set-output name=sha::$sha"
 
       - name: 'Get prebuilt csources from cache'
@@ -73,7 +73,7 @@ jobs:
         if: steps.csources-cache.outputs.cache-hit != 'true'
         uses: actions/checkout@v2
         with:
-          repository: nim-lang/csources
+          repository: nim-lang/csources_v1
           path: csources
 
       - name: 'Build 1-stage compiler from csources'

.github/workflows/ci_ssl.yml

@@ -47,11 +47,11 @@ jobs:
           curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
           7z x dist/mingw64.7z -odist
           7z x dist/dlls.zip -obin
-          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
-        run: echo "::add-path::${{ github.workspace }}/bin"
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
       - name: 'Build 1-stage compiler from csources'
         shell: bash

azure-pipelines.yml

@@ -15,10 +15,10 @@ jobs:
   strategy:
     matrix:
       Linux_amd64:
-        vmImage: 'ubuntu-16.04'
+        vmImage: 'ubuntu-18.04'
         CPU: amd64
       Linux_i386:
-        vmImage: 'ubuntu-16.04'
+        vmImage: 'ubuntu-18.04'
         CPU: i386
       OSX_amd64:
         vmImage: 'macOS-10.15'
@@ -30,8 +30,12 @@ jobs:
       Windows_amd64:
         vmImage: 'windows-2019'
         CPU: amd64
-      Windows_amd64_pkg:
-        vmImage: 'windows-2019'
+      Linux_amd64_pkg:
+        vmImage: 'ubuntu-18.04'
+        CPU: amd64
+        NIM_TEST_PACKAGES: true
+      OSX_amd64_pkg:
+        vmImage: 'macOS-10.15'
         CPU: amd64
         NIM_TEST_PACKAGES: true
 
@@ -47,14 +51,15 @@ jobs:
       condition: eq(variables['Agent.OS'], 'Windows_NT')
 
     - checkout: self
+      fetchDepth: 1
 
-    - bash: git clone --depth 1 https://github.com/nim-lang/csources.git
-      displayName: 'Checkout csources'
+    - bash: git clone --depth 1 https://github.com/nim-lang/csources_v1 csources
+      displayName: 'Checkout Nim csources'
 
     - task: NodeTool@0
       inputs:
-        versionSpec: '8.x'
-      displayName: 'Install node.js 8.x'
+        versionSpec: '12.x'
+      displayName: 'Install node.js 12.x'
 
     - bash: |
         sudo apt-fast update -qq
@@ -67,19 +72,12 @@ jobs:
     - bash: |
         sudo dpkg --add-architecture i386
 
-        # Downgrade llvm, libgcc and libstdc++:
+        # Downgrade llvm:
         # - llvm has to be downgraded to have 32bit version installed for sfml.
-        # - libgcc and libstdc++ have to be downgraded as an optimization to
-        #   prevent the use of the toolchain ppa, which has a terrible download
-        #   speed.
         cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
-        Package: libllvm6.0 libgcc1 libstdc++6
+        Package: libllvm6.0
         Pin: origin "azure.archive.ubuntu.com"
         Pin-Priority: 1001
-
-        Package: *
-        Pin: release o=LP-PPA-ubuntu-toolchain-r-test
-        Pin-Priority: 100
         EOF
 
         sudo apt-fast update -qq
@@ -107,7 +105,7 @@ jobs:
       displayName: 'Install dependencies (i386 Linux)'
       condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
 
-    - bash: brew install boehmgc make sfml
+    - bash: brew install boehmgc make sfml gtk+3
       displayName: 'Install dependencies (OSX)'
       condition: eq(variables['Agent.OS'], 'Darwin')
 
@@ -135,8 +133,18 @@ jobs:
         make -v
       displayName: 'System information'
 
+    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
+      displayName: 'Get csources version'
+
+    - task: Cache@2
+      inputs:
+        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+        path: csources/bin
+      displayName: 'Restore built csources'
+
     - bash: |
         ncpu=
+        ext=
         case '$(Agent.OS)' in
         'Linux')
           ncpu=$(nproc)
@@ -146,19 +154,25 @@ jobs:
           ;;
         'Windows_NT')
           ncpu=$NUMBER_OF_PROCESSORS
+          ext=.exe
           ;;
         esac
         [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
 
-        make -C csources -j $ncpu CC=gcc ucpu=$(CPU)
-      displayName: 'Build csources'
+        if [[ -x csources/bin/nim$ext ]]; then
+          echo "Found cached compiler, skipping build"
+        else
+          make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
+        fi
+
+        cp csources/bin/nim$ext bin
+      displayName: 'Build 1-stage compiler from csources'
 
     - bash: nim c koch
       displayName: 'Build koch'
 
       # set result to omit the "bash exited with error code '1'" message
-    - bash: |
-        ./koch runCI || echo '##vso[task.complete result=Failed]'
+    - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
       displayName: 'Run CI'
       env:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)

build_all.bat

@@ -1,7 +1,7 @@
 @echo off
 rem build development version of the compiler; can be rerun safely
 if not exist csources (
-  git clone --depth 1 https://github.com/nim-lang/csources.git
+  git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 )
 if not exist bin\nim.exe (
   cd csources

build_all.sh

@@ -11,7 +11,7 @@ echo_run(){
   "$@"
 }
 
-[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources.git
+[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources_v1.git csources
 
 nim_csources=bin/nim_csources
 

changelog.md

@@ -3,13 +3,356 @@
 
 
 ## Standard library additions and changes
+- Added support for parenthesized expressions in `strformat`
 
+- Fixed buffer overflow bugs in `net`
+
+- Added `sections` iterator in `parsecfg`.
+
+- Make custom op in macros.quote work for all statements.
+
+  For `net` and `nativesockets`, an `inheritable` flag has been added to all
+  `proc`s that create sockets, allowing the user to control whether the
+  resulting socket is inheritable. This flag is provided to ease the writing of
+  multi-process servers, where sockets inheritance is desired.
+
+  For a transistion period, define `nimInheritHandles` to enable file handle
+  inheritance by default. This flag does **not** affect the `selectors` module
+  due to the differing semantics between operating systems.
+
+  `system.setInheritable` and `nativesockets.setInheritable` is also introduced
+  for setting file handle or socket inheritance. Not all platform have these
+  `proc`s defined.
+
+- The file descriptors created for internal bookkeeping by `ioselector_kqueue`
+  and `ioselector_epoll` will no longer be leaked to child processes.
+
+- `strutils.formatFloat` with `precision = 0` has been restored to the version
+  1 behaviour that produces a trailing dot, e.g. `formatFloat(3.14159, precision = 0)`
+  is now `3.`, not `3`.
+- `critbits` adds `commonPrefixLen`.
+
+- `relativePath(rel, abs)` and `relativePath(abs, rel)` used to silently give wrong results
+  (see #13222); instead they now use `getCurrentDir` to resolve those cases,
+  and this can now throw in edge cases where `getCurrentDir` throws.
+  `relativePath` also now works for js with `-d:nodejs`.
+
+- JavaScript and NimScript standard library changes: `streams.StringStream` is
+  now supported in JavaScript, with the limitation that any buffer `pointer`s
+  used must be castable to `ptr string`, any incompatible pointer type will not
+  work. The `lexbase` and `streams` modules used to fail to compile on
+  NimScript due to a bug, but this has been fixed.
+  
+  The following modules now compile on both JS and NimScript: `parsecsv`,
+  `parsecfg`, `parsesql`, `xmlparser`, `htmlparser` and `ropes`. Additionally
+  supported for JS is `cstrutils.startsWith` and `cstrutils.endsWith`, for
+  NimScript: `json`, `parsejson`, `strtabs` and `unidecode`. 
+
+- Added `streams.readStr` and `streams.peekStr` overloads to
+  accept an existing string to modify, which avoids memory
+  allocations, similar to `streams.readLine` (#13857).
+
+- Added high-level `asyncnet.sendTo` and `asyncnet.recvFrom`. UDP functionality.
+
+- `paramCount` & `paramStr` are now defined in os.nim instead of nimscript.nim for nimscript/nimble.
+- `dollars.$` now works for unsigned ints with `nim js`
+
+- Improvements to the `bitops` module, including bitslices, non-mutating versions
+  of the original masking functions, `mask`/`masked`, and varargs support for
+  `bitand`, `bitor`, and `bitxor`.
+
+- `sugar.=>` and `sugar.->` changes: Previously `(x, y: int)` was transformed
+  into `(x: auto, y: int)`, it now becomes `(x: int, y: int)` in consistency
+  with regular proc definitions (although you cannot use semicolons).
+  
+  Pragmas and using a name are now allowed on the lefthand side of `=>`. Here
+  is an aggregate example of these changes:
+  ```nim
+  import sugar
+
+  foo(x, y: int) {.noSideEffect.} => x + y
+
+  # is transformed into
+
+  proc foo(x: int, y: int): auto {.noSideEffect.} = x + y
+  ```
+- The fields of `times.DateTime` are now private, and are accessed with getters and deprecated setters.
+
+- The `times` module now handles the default value for `DateTime` more consistently. Most procs raise an assertion error when given
+  an uninitialized `DateTime`, the exceptions are `==` and `$` (which returns `"Uninitialized DateTime"`). The proc `times.isInitialized`
+  has been added which can be used to check if a `DateTime` has been initialized.
+
+- Fix a bug where calling `close` on io streams in osproc.startProcess was a noop and led to
+  hangs if a process had both reads from stdin and writes (eg to stdout).
+
+- The callback that is passed to `system.onThreadDestruction` must now be `.raises: []`.
+- The callback that is assigned to `system.onUnhandledException` must now be `.gcsafe`.
+
+- `osproc.execCmdEx` now takes an optional `input` for stdin, `workingDir` and `env`
+  parameters.
+
+- Added a `ssl_config` module containing lists of secure ciphers as recommended by
+  [Mozilla OpSec](https://wiki.mozilla.org/Security/Server_Side_TLS)
+
+- `net.newContext` now defaults to the list of ciphers targeting
+  ["Intermediate compatibility"](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29)
+  per Mozilla's recommendation instead of `ALL`. This change should protect
+  users from the use of weak and insecure ciphers while still provides
+  adequate compatibility with the majority of the Internet.
+
+- A new module `std/jsonutils` with hookable `jsonTo,toJson,fromJson` operations for json
+  serialization/deserialization of custom types was added.
+
+- A new proc `heapqueue.find[T](heap: HeapQueue[T], x: T): int` to get index of element ``x``
+  was added.
+- Added `rstgen.rstToLatex` convenience proc for `renderRstToOut` and `initRstGenerator`
+  with `outLatex` output.
+- Added `os.normalizeExe`, e.g.: `koch` => `./koch`.
+- `macros.newLit` now preserves named vs unnamed tuples; use `-d:nimHasWorkaround14720`
+  to keep old behavior.
+- Added `random.gauss`, that uses the ratio of uniforms method of sampling from a Gaussian distribution.
+- Added `typetraits.elementType` to get element type of an iterable.
+- `typetraits.$` changes: `$(int,)` is now `"(int,)"` instead of `"(int)"`;
+  `$tuple[]` is now `"tuple[]"` instead of `"tuple"`;
+  `$((int, float), int)` is now `"((int, float), int)"` instead of `"(tuple of (int, float), int)"`
+- Added `macros.extractDocCommentsAndRunnables` helper
+
+- `strformat.fmt` and `strformat.&` support `= specifier`. `fmt"{expr=}"` now
+  expands to `fmt"expr={expr}"`.
+- deprecations: `os.existsDir` => `dirExists`, `os.existsFile` => `fileExists`
+
+- Added `jsre` module, [Regular Expressions for the JavaScript target.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions)
+- Made `maxLines` argument `Positive` in `logging.newRollingFileLogger`,
+  because negative values will result in a new file being created for each logged
+  line which doesn't make sense.
+- Changed `log` in `logging` to use proper log level on JavaScript target,
+  e.g. `debug` uses `console.debug`, `info` uses `console.info`, `warn` uses `console.warn`, etc.
+- Tables, HashSets, SharedTables and deques don't require anymore that the passed
+  initial size must be a power of two - this is done internally.
+  Proc `rightSize` for Tables and HashSets is deprecated, as it is not needed anymore.
+  `CountTable.inc` takes `val: int` again not `val: Positive`; I.e. it can "count down" again.
+- Removed deprecated symbols from `macros` module, deprecated as far back as `0.15`.
+- On Windows the SSL library now checks for valid certificates.
+  It uses the `cacert.pem` file for this purpose which was extracted
+  from `https://curl.se/ca/cacert.pem`. Besides
+  the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
+  now also need to ship `cacert.pem` with your `.exe` file.
+
+
+- Make `{.requiresInit.}` pragma to work for `distinct` types.
+
+
+- Added `asyncdispatch.activeDescriptors` that returns the number of currently
+  active async event handles/file descriptors
+- Added `asyncdispatch.maxDescriptors` that returns the maximum number of
+  active async event handles/file descriptors.
+
+- Added `getPort` to `asynchttpserver`.
+
+- `--gc:orc` is now 10% faster than previously for common workloads. If
+  you have trouble with its changed behavior, compile with `-d:nimOldOrc`.
+
+
+- `os.FileInfo` (returned by `getFileInfo`) now contains `blockSize`,
+  determining preferred I/O block size for this file object.
+
+- Added a simpler to use `io.readChars` overload.
+
+- Added `**` to jsffi.
+
+- `writeStackTrace` is available in JS backend now.
+
+- Added `decodeQuery` to `std/uri`.
+
+- `strscans.scanf` now supports parsing single characters.
+
+- `strscans.scanTuple` added which uses `strscans.scanf` internally,
+  returning a tuple which can be unpacked for easier usage of `scanf`.
+
+- Added `setutils.toSet` that can take any iterable and convert it to a built-in `set`,
+  if the iterable yields a built-in settable type.
+
+- Added `setutils.fullSet` which returns a full built-in `set` for a valid type.
+
+- Added `setutils.complement` which returns the complement of a built-in `set`.
+
+- Added `setutils.[]=`.
+
+- Added `math.isNaN`.
+
+- Added `jsbigints` module, arbitrary precision integers for JavaScript target.
+
+- Added `math.copySign`.
+
+- Added new operations for singly- and doubly linked lists: `lists.toSinglyLinkedList`
+  and `lists.toDoublyLinkedList` convert from `openArray`s; `lists.copy` implements
+  shallow copying; `lists.add` concatenates two lists - an O(1) variation that consumes
+  its argument, `addMoved`, is also supplied.
+
+- Added `euclDiv` and `euclMod` to `math`.
+
+- Added `httpcore.is1xx` and missing HTTP codes.
+
+- Added `jsconsole.jsAssert` for JavaScript target.
+
+- Added `posix_utils.osReleaseFile` to get system identification from `os-release` file on Linux and the BSDs.
+  https://www.freedesktop.org/software/systemd/man/os-release.html
+
+- Added `socketstream` module that wraps sockets in the stream interface
+
+- Added `sugar.dumpToString` which improves on `sugar.dump`.
+
+- Added `math.signbit`.
+
+- Removed the optional `longestMatch` parameter of the `critbits._WithPrefix` iterators (it never worked reliably)
+
+- In `lists`: renamed `append` to `add` and retained `append` as an alias;
+  added `prepend` and `prependMoved` analogously to `add` and `addMoved`;
+  added `remove` for `SinglyLinkedList`s.
+
+- Deprecated `any`. See https://github.com/nim-lang/RFCs/issues/281
+
+- Added optional `options` argument to `copyFile`, `copyFileToDir`, and
+  `copyFileWithPermissions`. By default, on non-Windows OSes, symlinks are
+  followed (copy files symlinks point to); on Windows, `options` argument is
+  ignored and symlinks are skipped.
+
+- On non-Windows OSes, `copyDir` and `copyDirWithPermissions` copy symlinks as
+  symlinks (instead of skipping them as it was before); on Windows symlinks are
+  skipped.
+
+- On non-Windows OSes, `moveFile` and `moveDir` move symlinks as symlinks
+  (instead of skipping them sometimes as it was before).
+
+- Added optional `followSymlinks` argument to `setFilePermissions`.
+
+- Added `os.isAdmin` to tell whether the caller's process is a member of the
+  Administrators local group (on Windows) or a root (on POSIX).
+
+- Added experimental `linenoise.readLineStatus` to get line and status (e.g. ctrl-D or ctrl-C).
+
+- Added `compilesettings.SingleValueSetting.libPath`.
+
+- `std/wrapnils` doesn't use `experimental:dotOperators` anymore, avoiding
+  issues like https://github.com/nim-lang/Nim/issues/13063 (which affected error messages)
+  for modules importing `std/wrapnils`.
+  Added `??.` macro which returns an `Option`.
+
+- Added `math.frexp` overload procs. Deprecated `c_frexp`, use `frexp` instead.
+
+- `parseopt.initOptParser` has been made available and `parseopt` has been
+  added back to `prelude` for all backends. Previously `initOptParser` was
+  unavailable if the `os` module did not have `paramCount` or `paramStr`,
+  but the use of these in `initOptParser` were conditionally to the runtime
+  arguments passed to it, so `initOptParser` has been changed to raise
+  `ValueError` when the real command line is not available. `parseopt` was
+  previously excluded from `prelude` for JS, as it could not be imported.
+
+- Added `system.prepareStrMutation` for better support of low
+  level `moveMem`, `copyMem` operations for Orc's copy-on-write string
+  implementation.
+
+- Added `std/strbasics` for high performance string operations.
+  Added `strip`, `setSlice`, `add(a: var string, b: openArray[char])`.
+
+
+- Added to `wrapnils` an option-like API via `??.`, `isSome`, `get`.
+
+- `std/options` changed `$some(3)` to `"some(3)"` instead of `"Some(3)"`
+  and `$none(int)` to `"none(int)"` instead of `"None[int]"`.
+
+- Added `algorithm.merge`.
+
+
+- Added `std/jsfetch` module [Fetch](https://developer.mozilla.org/docs/Web/API/Fetch_API) wrapper for JavaScript target.
+
+- Added `std/jsheaders` module [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) wrapper for JavaScript target.
+
+- Added `std/jsformdata` module [FormData](https://developer.mozilla.org/en-US/docs/Web/API/FormData) wrapper for JavaScript target.
+
+- `system.addEscapedChar` now renders `\r` as `\r` instead of `\c`, to be compatible
+  with most other languages.
+
+- Removed support for named procs in `sugar.=>`.
+
+- Added `jscore.debugger` to [call any available debugging functionality, such as breakpoints.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/debugger).
+
+- Added `std/channels`.
+
+- Added `htmlgen.portal` for [making "SPA style" pages using HTML only](https://web.dev/hands-on-portals).
+
+- Added `ZZZ` and `ZZZZ` patterns to `times.nim` `DateTime` parsing, to match time
+  zone offsets without colons, e.g. `UTC+7 -> +0700`.
+
+- Added `jsconsole.dir`, `jsconsole.dirxml`, `jsconsole.timeStamp`.
+
+- Added dollar `$` and `len` for `jsre.RegExp`.
+
+- Added `std/tasks`.
+
+- Added `hasDataBuffered` to `asyncnet`.
+
+- Added `hasClosure` to `std/typetraits`.
+
+- Added `std/tempfiles`.
+
+- Added `genasts.genAst` that avoids the problems inherent with `quote do` and can
+  be used as a replacement.
+
+- Added `copyWithin` [for `seq` and `array` for JavaScript targets](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/copyWithin).
+
+- Fixed premature garbage collection in asyncdispatch, when a stack trace override is in place.
 
 ## Language changes
+- In newruntime it is now allowed to assign discriminator field without restrictions as long as case object doesn't have custom destructor. Discriminator value doesn't have to be a constant either. If you have custom destructor for case object and you do want to freely assign discriminator fields, it is recommended to refactor object into 2 objects like this: 
 
+- The `=destroy` hook no longer has to reset its target, as the compiler now automatically inserts
+  `wasMoved` calls where needed.
+- The `=` hook is now called `=copy` for clarity. The old name `=` is still available so there
+  is no need to update your code. This change was backported to 1.2 too so you can use the
+  more readability `=copy` without loss of compatibility.
+
+- In the newruntime it is now allowed to assign to the discriminator field
+  without restrictions as long as case object doesn't have custom destructor.
+  The discriminator value doesn't have to be a constant either. If you have a
+  custom destructor for a case object and you do want to freely assign discriminator
+  fields, it is recommended to refactor object into 2 objects like this:
+
+  ```nim
+  type
+    MyObj = object
+      case kind: bool
+        of true: y: ptr UncheckedArray[float]
+        of false: z: seq[int]
+
+  proc `=destroy`(x: MyObj) =
+    if x.kind and x.y != nil:
+      deallocShared(x.y)
+      x.y = nil
+  ```
+  Refactor into:
+  ```nim
+  type
+    MySubObj = object
+      val: ptr UncheckedArray[float]
+    MyObj = object
+      case kind: bool
+      of true: y: MySubObj
+      of false: z: seq[int]
+
+  proc `=destroy`(x: MySubObj) =
+    if x.val != nil:
+      deallocShared(x.val)
+      x.val = nil
+  ```
 
 ## Compiler changes
 
 
+- The style checking of the compiler now supports a `--styleCheck:usages` switch. This switch
+  enforces that every symbol is written as it was declared, not enforcing
+  the official Nim style guide. To be enabled, this has to be combined either
+  with `--styleCheck:error` or `--styleCheck:hint`.
+
 ## Tool changes
 

ci/build.bat

@@ -2,7 +2,7 @@ REM Some debug info
 echo "Running on %CI_RUNNER_ID% (%CI_RUNNER_DESCRIPTION%) with tags %CI_RUNNER_TAGS%."
 gcc -v
 
-git clone --depth 1 https://github.com/nim-lang/csources.git
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 cd csources
 call build64.bat
 cd ..

ci/build.sh

@@ -1,7 +1,7 @@
 sh ci/deps.sh
 
 # Build from C sources.
-git clone --depth 1 https://github.com/nim-lang/csources.git
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 cd csources
 sh build.sh
 cd ..

ci/nsis_build.bat

@@ -25,7 +25,7 @@ Rem Build csources
 koch csources -d:release || exit /b
 
 rem Grab C sources and nimsuggest
-git clone --depth 1 https://github.com/nim-lang/csources.git
+git clone --depth 1 https://github.com/nim-lang/csources_v1.git csources
 
 set PATH=%CD%\bin;%PATH%
 

compiler/ast.nim

@@ -292,6 +292,7 @@ type
     sfInjectDestructors # whether the proc needs the 'injectdestructors' transformation
     sfNeverRaises     # proc can never raise an exception, not even OverflowError
                       # or out-of-memory
+    sfUsedInFinallyOrExcept  # symbol is used inside an 'except' or 'finally'
 
   TSymFlags* = set[TSymFlag]
 
@@ -1084,7 +1085,7 @@ proc safeLen*(n: PNode): int {.inline.} =
 
 proc safeArrLen*(n: PNode): int {.inline.} =
   ## works for array-like objects (strings passed as openArray in VM).
-  if n.kind in {nkStrLit..nkTripleStrLit}:result = n.strVal.len
+  if n.kind in {nkStrLit..nkTripleStrLit}: result = n.strVal.len
   elif n.kind in {nkNone..nkFloat128Lit}: result = 0
   else: result = n.len
 
@@ -1326,7 +1327,7 @@ const
   MaxLockLevel* = 1000'i16
   UnknownLockLevel* = TLockLevel(1001'i16)
   AttachedOpToStr*: array[TTypeAttachedOp, string] = [
-    "=destroy", "=", "=sink", "=trace", "=dispose", "=deepcopy"]
+    "=destroy", "=copy", "=sink", "=trace", "=dispose", "=deepcopy"]
 
 proc `$`*(x: TLockLevel): string =
   if x.ord == UnspecifiedLockLevel.ord: result = "<unspecified>"

compiler/astalgo.nim

@@ -1048,3 +1048,7 @@ proc listSymbolNames*(symbols: openArray[PSym]): string =
       result.add ", "
     result.add sym.name.s
 
+proc isDiscriminantField*(n: PNode): bool =
+  if n.kind == nkCheckedFieldExpr: sfDiscriminant in n[0][1].sym.flags
+  elif n.kind == nkDotExpr: sfDiscriminant in n[1].sym.flags
+  else: false

compiler/ccgcalls.nim

@@ -21,11 +21,39 @@ proc canRaiseDisp(p: BProc; n: PNode): bool =
     # we have to be *very* conservative:
     result = canRaiseConservative(n)
 
-proc leftAppearsOnRightSide(le, ri: PNode): bool =
+proc preventNrvo(p: BProc; le, ri: PNode): bool =
+  proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
+    var n = le
+    while true:
+      # do NOT follow nkHiddenDeref here!
+      case n.kind
+      of nkSym:
+        # we don't own the location so it escapes:
+        if n.sym.owner != p.prc:
+          return true
+        elif inTryStmt and sfUsedInFinallyOrExcept in n.sym.flags:
+          # it is also an observable store if the location is used
+          # in 'except' or 'finally'
+          return true
+        return false
+      of nkDotExpr, nkBracketExpr, nkObjUpConv, nkObjDownConv,
+          nkCheckedFieldExpr:
+        n = n[0]
+      of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+        n = n[1]
+      else:
+        # cannot analyse the location; assume the worst
+        return true
+
   if le != nil:
     for i in 1..<ri.len:
       let r = ri[i]
       if isPartOf(le, r) != arNo: return true
+    # we use the weaker 'canRaise' here in order to prevent too many
+    # annoying warnings, see #14514
+    if canRaise(ri[0]) and
+        locationEscapes(p, le, p.nestedTryStmts.len > 0):
+      message(p.config, le.info, warnObservableStores, $le)
 
 proc hasNoInit(call: PNode): bool {.inline.} =
   result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
@@ -51,7 +79,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
     if isInvalidReturnType(p.config, typ[0]):
       if params != nil: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
@@ -113,7 +141,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
 
 proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
 
-proc openArrayLoc(p: BProc, n: PNode): Rope =
+proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
   var a: TLoc
 
   var q = skipConv(n)
@@ -149,8 +177,11 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
     of tyOpenArray, tyVarargs, tyUncheckedArray, tyCString:
       result = "($4*)($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dest]
     of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-          not compileToCpp(p.module):
+      let atyp = skipTypes(a.t, abstractInst)
+      if formalType.skipTypes(abstractInst).kind == tyVar and atyp.kind == tyString and
+          optSeqDestructors in p.config.globalOptions:
+        linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+      if atyp.kind == tyVar and not compileToCpp(p.module):
         result = "($5*)(*$1)$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
       else:
         result = "($5*)$1$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
@@ -158,12 +189,15 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
       internalError(p.config, "openArrayLoc: " & typeToString(a.t))
   else:
     initLocExpr(p, n, a)
-    case skipTypes(a.t, abstractVar).kind
+    case skipTypes(a.t, abstractVar+{tyStatic}).kind
     of tyOpenArray, tyVarargs:
       result = "$1, $1Len_0" % [rdLoc(a)]
     of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-            not compileToCpp(p.module):
+      let ntyp = skipTypes(n.typ, abstractInst)
+      if formalType.skipTypes(abstractInst).kind == tyVar and ntyp.kind == tyString and
+          optSeqDestructors in p.config.globalOptions:
+        linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+      if ntyp.kind == tyVar and not compileToCpp(p.module):
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
         result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
@@ -194,7 +228,7 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
     result = genArgStringToCString(p, n)
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
     var n = if n.kind != nkHiddenAddr: n else: n[0]
-    result = openArrayLoc(p, n)
+    result = openArrayLoc(p, param.typ, n)
   elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
     result = addrLoc(p.config, a)
@@ -286,7 +320,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
     if isInvalidReturnType(p.config, typ[0]):
       if ri.len > 1: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
           getTemp(p, typ[0], d, needsInit=true)

compiler/ccgexprs.nim

@@ -833,6 +833,7 @@ proc genFieldCheck(p: BProc, e: PNode, obj: Rope, field: PSym) =
               [rdLoc(test), strLit, raiseInstr(p)])
 
 proc genCheckedRecordField(p: BProc, e: PNode, d: var TLoc) =
+  assert e[0].kind == nkDotExpr
   if optFieldCheck in p.options:
     var a: TLoc
     genRecordFieldAux(p, e[0], d, a)
@@ -867,7 +868,7 @@ proc genArrayElem(p: BProc, n, x, y: PNode, d: var TLoc) =
   if optBoundsCheck in p.options and ty.kind != tyUncheckedArray:
     if not isConstExpr(y):
       # semantic pass has already checked for const index expressions
-      if firstOrd(p.config, ty) == 0:
+      if firstOrd(p.config, ty) == 0 and lastOrd(p.config, ty) >= 0:
         if (firstOrd(p.config, b.t) < firstOrd(p.config, ty)) or (lastOrd(p.config, b.t) > lastOrd(p.config, ty)):
           linefmt(p, cpsStmts, "if ((NU)($1) > (NU)($2)){ #raiseIndexError2($1, $2); $3}$n",
                   [rdCharLoc(b), intLiteral(lastOrd(p.config, ty)), raiseInstr(p)])
@@ -1194,9 +1195,11 @@ proc genSeqElemAppend(p: BProc, e: PNode, d: var TLoc) =
 proc genReset(p: BProc, n: PNode) =
   var a: TLoc
   initLocExpr(p, n[1], a)
-  linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
-          [addrLoc(p.config, a),
-          genTypeInfo(p.module, skipTypes(a.t, {tyVar}), n.info)])
+  specializeReset(p, a)
+  when false:
+    linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
+            [addrLoc(p.config, a),
+            genTypeInfo(p.module, skipTypes(a.t, {tyVar}), n.info)])
 
 proc genDefault(p: BProc; n: PNode; d: var TLoc) =
   if d.k == locNone: getTemp(p, n.typ, d, needsInit=true)
@@ -1779,9 +1782,9 @@ template binaryExprIn(p: BProc, e: PNode, a, b, d: var TLoc, frmt: string) =
 
 proc genInExprAux(p: BProc, e: PNode, a, b, d: var TLoc) =
   case int(getSize(p.config, skipTypes(e[1].typ, abstractVar)))
-  of 1: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&7U)))!=0)")
-  of 2: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&15U)))!=0)")
-  of 4: binaryExprIn(p, e, a, b, d, "(($1 &(1U<<((NU)($2)&31U)))!=0)")
+  of 1: binaryExprIn(p, e, a, b, d, "(($1 &((NU8)1<<((NU)($2)&7U)))!=0)")
+  of 2: binaryExprIn(p, e, a, b, d, "(($1 &((NU16)1<<((NU)($2)&15U)))!=0)")
+  of 4: binaryExprIn(p, e, a, b, d, "(($1 &((NU32)1<<((NU)($2)&31U)))!=0)")
   of 8: binaryExprIn(p, e, a, b, d, "(($1 &((NU64)1<<((NU)($2)&63U)))!=0)")
   else: binaryExprIn(p, e, a, b, d, "(($1[(NU)($2)>>3] &(1U<<((NU)($2)&7U)))!=0)")
 
@@ -2156,7 +2159,7 @@ proc genMagicExpr(p: BProc, e: PNode, d: var TLoc, op: TMagic) =
     const opr: array[mInc..mDec, string] = ["+=", "-="]
     const fun64: array[mInc..mDec, string] = ["nimAddInt64", "nimSubInt64"]
     const fun: array[mInc..mDec, string] = ["nimAddInt","nimSubInt"]
-    let underlying = skipTypes(e[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent, tyRange})
+    let underlying = skipTypes(e[1].typ, {tyGenericInst, tyAlias, tySink, tyVar, tyLent, tyRange, tyDistinct})
     if optOverflowCheck notin p.options or underlying.kind in {tyUInt..tyUInt64}:
       binaryStmt(p, e, d, opr[op])
     else:
@@ -2738,6 +2741,7 @@ proc expr(p: BProc, n: PNode, d: var TLoc) =
     inc p.splitDecls
     genGotoState(p, n)
   of nkBreakState: genBreakState(p, n, d)
+  of nkMixinStmt, nkBindStmt: discard
   else: internalError(p.config, n.info, "expr(" & $n.kind & "); unknown node kind")
 
 proc genNamedConstExpr(p: BProc, n: PNode; isConst: bool): Rope =
@@ -2787,6 +2791,21 @@ proc getDefaultValue(p: BProc; typ: PType; info: TLineInfo): Rope =
   else:
     globalError(p.config, info, "cannot create null element for: " & $t.kind)
 
+proc caseObjDefaultBranch(obj: PNode; branch: Int128): int =
+  for i in 1 ..< obj.len:
+    for j in 0 .. obj[i].len - 2:
+      if obj[i][j].kind == nkRange:
+        let x = getOrdValue(obj[i][j][0])
+        let y = getOrdValue(obj[i][j][1])
+        if branch >= x and branch <= y:
+          return i
+      elif getOrdValue(obj[i][j]) == branch:
+        return i
+    if obj[i].len == 1:
+      # else branch
+      return i
+  assert(false, "unreachable")
+
 proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode,
                      result: var Rope; count: var int;
                      isConst: bool, info: TLineInfo) =
@@ -2809,31 +2828,14 @@ proc getNullValueAux(p: BProc; t: PType; obj, constOrNil: PNode,
           branch = getOrdValue(constOrNil[i])
           break
 
-    var selectedBranch = -1
-    block branchSelection:
-      for i in 1 ..< obj.len:
-        for j in 0 .. obj[i].len - 2:
-          if obj[i][j].kind == nkRange:
-              let x = getOrdValue(obj[i][j][0])
-              let y = getOrdValue(obj[i][j][1])
-              if branch >= x and branch <= y:
-                selectedBranch = i
-                break branchSelection
-          elif getOrdValue(obj[i][j]) == branch:
-            selectedBranch = i
-            break branchSelection
-        if obj[i].len == 1:
-          # else branch
-          selectedBranch = i
-    assert(selectedBranch >= 1)
-
+    let selectedBranch = caseObjDefaultBranch(obj, branch)
     result.add "{"
     var countB = 0
     let b = lastSon(obj[selectedBranch])
     # designated initilization is the only way to init non first element of unions
     # branches are allowed to have no members (b.len == 0), in this case they don't need initializer
-    if  b.kind == nkRecList and b.len > 0:
-      result.add "._" &  mangleRecFieldName(p.module, obj[0].sym) & "_" & $selectedBranch & " = {"
+    if b.kind == nkRecList and b.len > 0:
+      result.add "._" & mangleRecFieldName(p.module, obj[0].sym) & "_" & $selectedBranch & " = {"
       getNullValueAux(p, t,  b, constOrNil, result, countB, isConst, info)
       result.add "}"
     elif b.kind == nkSym:
@@ -2941,7 +2943,23 @@ proc genConstSeqV2(p: BProc, n: PNode, t: PType; isConst: bool): Rope =
 proc genBracedInit(p: BProc, n: PNode; isConst: bool): Rope =
   case n.kind
   of nkHiddenStdConv, nkHiddenSubConv:
-    result = genBracedInit(p, n[1], isConst)
+    when false:
+      # XXX The frontend doesn't keep conversions to openArray for us. :-(
+      # We need to change 'transformConv' first, but that is hard.
+      if n.typ.kind == tyOpenArray:
+        assert n[1].kind == nkBracket
+        let data = genBracedInit(p, n[1], isConst)
+
+        let payload = getTempName(p.module)
+        let ctype = getTypeDesc(p.module, n.typ.skipTypes(abstractInst)[0])
+        let arrLen = n[1].len
+        appcg(p.module, cfsData,
+          "static $5 $1 $3[$2] = $4;$n", [
+          ctype, arrLen, payload, data,
+          if isConst: "const" else: ""])
+        result = "{($1*)&$2, $3}" % [ctype, payload, rope arrLen]
+    else:
+      result = genBracedInit(p, n[1], isConst)
   else:
     var ty = tyNone
     if n.typ == nil:

compiler/ccgreset.nim

@@ -0,0 +1,94 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# included from cgen.nim
+
+## Code specialization instead of the old, incredibly slow 'genericReset'
+## implementation.
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType)
+
+proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
+                     typ: PType) =
+  if n == nil: return
+  case n.kind
+  of nkRecList:
+    for i in 0..<n.len:
+      specializeResetN(p, accessor, n[i], typ)
+  of nkRecCase:
+    if (n[0].kind != nkSym): internalError(p.config, n.info, "specializeResetN")
+    let disc = n[0].sym
+    if disc.loc.r == nil: fillObjectFields(p.module, typ)
+    if disc.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
+    for i in 1..<n.len:
+      let branch = n[i]
+      assert branch.kind in {nkOfBranch, nkElse}
+      if branch.kind == nkOfBranch:
+        genCaseRange(p, branch)
+      else:
+        lineF(p, cpsStmts, "default:$n", [])
+      specializeResetN(p, accessor, lastSon(branch), typ)
+      lineF(p, cpsStmts, "break;$n", [])
+    lineF(p, cpsStmts, "} $n", [])
+    specializeResetT(p, "$1.$2" % [accessor, disc.loc.r], disc.loc.t)
+  of nkSym:
+    let field = n.sym
+    if field.typ.kind == tyVoid: return
+    if field.loc.r == nil: fillObjectFields(p.module, typ)
+    if field.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    specializeResetT(p, "$1.$2" % [accessor, field.loc.r], field.loc.t)
+  else: internalError(p.config, n.info, "specializeResetN()")
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
+  if typ == nil: return
+
+  case typ.kind
+  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred,
+     tySink, tyOwned:
+    specializeResetT(p, accessor, lastSon(typ))
+  of tyArray:
+    let arraySize = lengthOrd(p.config, typ[0])
+    var i: TLoc
+    getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i)
+    linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
+            [i.r, arraySize])
+    specializeResetT(p, ropecg(p.module, "$1[$2]", [accessor, i.r]), typ[1])
+    lineF(p, cpsStmts, "}$n", [])
+  of tyObject:
+    for i in 0..<typ.len:
+      var x = typ[i]
+      if x != nil: x = x.skipTypes(skipPtrs)
+      specializeResetT(p, accessor.parentObj(p.module), x)
+    if typ.n != nil: specializeResetN(p, accessor, typ.n, typ)
+  of tyTuple:
+    let typ = getUniqueType(typ)
+    for i in 0..<typ.len:
+      specializeResetT(p, ropecg(p.module, "$1.Field$2", [accessor, i]), typ[i])
+
+  of tyString, tyRef, tySequence:
+    lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1, NIM_NIL);$n", [accessor])
+
+  of tyProc:
+    if typ.callConv == ccClosure:
+      lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1.ClE_0, NIM_NIL);$n", [accessor])
+      lineCg(p, cpsStmts, "$1.ClP_0 = NIM_NIL;$n", [accessor])
+    else:
+      lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  of tyChar, tyBool, tyEnum, tyInt..tyUInt64:
+    lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
+  of tyCString, tyPointer, tyPtr, tyVar, tyLent:
+    lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  else:
+    discard
+
+proc specializeReset(p: BProc, a: TLoc) =
+  specializeResetT(p, rdLoc(a), a.t)

compiler/ccgstmts.nim

@@ -25,10 +25,10 @@ proc getTraverseProc(p: BProc, v: PSym): Rope =
 
 proc registerTraverseProc(p: BProc, v: PSym, traverseProc: Rope) =
   if sfThread in v.flags:
-    appcg(p.module, p.module.initProc.procSec(cpsInit),
+    appcg(p.module, p.module.preInitProc.procSec(cpsInit),
       "$n\t#nimRegisterThreadLocalMarker($1);$n$n", [traverseProc])
   else:
-    appcg(p.module, p.module.initProc.procSec(cpsInit),
+    appcg(p.module, p.module.preInitProc.procSec(cpsInit),
       "$n\t#nimRegisterGlobalMarker($1);$n$n", [traverseProc])
 
 proc isAssignedImmediately(conf: ConfigRef; n: PNode): bool {.inline.} =
@@ -688,14 +688,15 @@ proc genBreakStmt(p: BProc, t: PNode) =
 
 proc raiseExit(p: BProc) =
   assert p.config.exc == excGoto
-  p.flags.incl nimErrorFlagAccessed
-  if p.nestedTryStmts.len == 0:
-    p.flags.incl beforeRetNeeded
-    # easy case, simply goto 'ret':
-    lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto BeforeRet_;$n", [])
-  else:
-    lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto LA$1_;$n",
-      [p.nestedTryStmts[^1].label])
+  if nimErrorFlagDisabled notin p.flags:
+    p.flags.incl nimErrorFlagAccessed
+    if p.nestedTryStmts.len == 0:
+      p.flags.incl beforeRetNeeded
+      # easy case, simply goto 'ret':
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto BeforeRet_;$n", [])
+    else:
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto LA$1_;$n",
+        [p.nestedTryStmts[^1].label])
 
 proc finallyActions(p: BProc) =
   if p.config.exc != excGoto and p.nestedTryStmts.len > 0 and p.nestedTryStmts[^1].inExcept:
@@ -981,7 +982,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   inc(p.labels, 2)
   let etmp = p.labels
 
-  lineCg(p, cpsStmts, "std::exception_ptr T$1_ = nullptr;", [etmp])
+  p.procSec(cpsInit).add(ropecg(p.module, "\tstd::exception_ptr T$1_ = nullptr;", [etmp]))
 
   let fin = if t[^1].kind == nkFinally: t[^1] else: nil
   p.nestedTryStmts.add((fin, false, 0.Natural))
@@ -1329,8 +1330,19 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
       linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
     elif isDefined(p.config, "nimSigSetjmp"):
       linefmt(p, cpsStmts, "$1.status = sigsetjmp($1.context, 0);$n", [safePoint])
+    elif isDefined(p.config, "nimBuiltinSetjmp"):
+      linefmt(p, cpsStmts, "$1.status = __builtin_setjmp($1.context);$n", [safePoint])
     elif isDefined(p.config, "nimRawSetjmp"):
-      linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint])
+      if isDefined(p.config, "mswindows"):
+        # The Windows `_setjmp()` takes two arguments, with the second being an
+        # undocumented buffer used by the SEH mechanism for stack unwinding.
+        # Mingw-w64 has been trying to get it right for years, but it's still
+        # prone to stack corruption during unwinding, so we disable that by setting
+        # it to NULL.
+        # More details: https://github.com/status-im/nimbus-eth2/issues/3121
+        linefmt(p, cpsStmts, "$1.status = _setjmp($1.context, 0);$n", [safePoint])
+      else:
+        linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint])
     else:
       linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
     startBlock(p, "if ($1.status == 0) {$n", [safePoint])
@@ -1489,15 +1501,6 @@ proc genPragma(p: BProc, n: PNode) =
       p.module.injectStmt = p.s(cpsStmts)
     else: discard
 
-proc fieldDiscriminantCheckNeeded(p: BProc, asgn: PNode): bool =
-  if optFieldCheck in p.options:
-    var le = asgn[0]
-    if le.kind == nkCheckedFieldExpr:
-      var field = le[0][1].sym
-      result = sfDiscriminant in field.flags
-    elif le.kind == nkDotExpr:
-      var field = le[1].sym
-      result = sfDiscriminant in field.flags
 
 proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
                           field: PSym) =
@@ -1534,24 +1537,20 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   initLocExpr(p, e[0], a)
   getTemp(p, a.t, tmp)
   expr(p, e[1], tmp)
-  let field = dotExpr[1].sym
-  if optTinyRtti in p.config.globalOptions:
-    let t = dotExpr[0].typ.skipTypes(abstractInst)
-    var oldVal, newVal: TLoc
-    genCaseObjDiscMapping(p, e[0], t, field, oldVal)
-    genCaseObjDiscMapping(p, e[1], t, field, newVal)
-    lineCg(p, cpsStmts,
-          "if ($1 != $2) { #raiseObjectCaseTransition(); $3}$n",
-          [rdLoc(oldVal), rdLoc(newVal), raiseInstr(p)])
-  else:
+  if optTinyRtti notin p.config.globalOptions:
+    let field = dotExpr[1].sym
     genDiscriminantCheck(p, a, tmp, dotExpr[0].typ, field)
+    message(p.config, e.info, warnCaseTransition)
   genAssignment(p, a, tmp, {})
 
 proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
   if e[0].kind == nkSym and sfGoto in e[0].sym.flags:
     genLineDir(p, e)
     genGotoVar(p, e[1])
-  elif not fieldDiscriminantCheckNeeded(p, e):
+  elif optFieldCheck in p.options and isDiscriminantField(e[0]):
+    genLineDir(p, e)
+    asgnFieldDiscriminant(p, e)
+  else:
     let le = e[0]
     let ri = e[1]
     var a: TLoc
@@ -1565,10 +1564,6 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
     assert(a.t != nil)
     genLineDir(p, ri)
     loadInto(p, le, ri, a)
-  else:
-    genLineDir(p, e)
-    asgnFieldDiscriminant(p, e)
-    message(p.config, e.info, warnCaseTransition)
 
 proc genStmts(p: BProc, t: PNode) =
   var a: TLoc

compiler/ccgtypes.nim

@@ -220,7 +220,8 @@ proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool =
   # such a poor programming language.
   # We exclude records with refs too. This enhances efficiency and
   # is necessary for proper code generation of assignments.
-  if rettype == nil: result = true
+  if rettype == nil or getSize(conf, rettype) > conf.target.floatSize*3:
+    result = true
   else:
     case mapType(conf, rettype)
     of ctArray:
@@ -1308,6 +1309,7 @@ proc genTypeInfoV2(m: BModule, t, origType: PType, name: Rope; info: TLineInfo)
   else:
     typeName = rope("NIM_NIL")
 
+  discard cgsym(m, "TNimType")
   m.s[cfsData].addf("N_LIB_PRIVATE TNimType $1;$n", [name])
   let destroyImpl = genHook(m, t, info, attachedDestructor)
   let traceImpl = genHook(m, t, info, attachedTrace)

compiler/cgen.nim

@@ -244,7 +244,7 @@ proc safeLineNm(info: TLineInfo): int =
 
 proc genCLineDir(r: var Rope, filename: string, line: int; conf: ConfigRef) =
   assert line >= 0
-  if optLineDir in conf.options:
+  if optLineDir in conf.options and line > 0:
     r.addf("$N#line $2 $1$N",
         [rope(makeSingleLineCString(filename)), rope(line)])
 
@@ -390,6 +390,8 @@ proc isComplexValueType(t: PType): bool {.inline.} =
   result = t.kind in {tyArray, tySet, tyTuple, tyObject} or
     (t.kind == tyProc and t.callConv == ccClosure)
 
+include ccgreset
+
 proc resetLoc(p: BProc, loc: var TLoc) =
   let containsGcRef = optSeqDestructors notin p.config.globalOptions and containsGarbageCollectedRef(loc.t)
   let typ = skipTypes(loc.t, abstractVarRange)
@@ -409,8 +411,10 @@ proc resetLoc(p: BProc, loc: var TLoc) =
     if optNilCheck in p.options:
       linefmt(p, cpsStmts, "#chckNil((void*)$1);$n", [addrLoc(p.config, loc)])
     if loc.storage != OnStack and containsGcRef:
-      linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
-              [addrLoc(p.config, loc), genTypeInfo(p.module, loc.t, loc.lode.info)])
+      specializeReset(p, loc)
+      when false:
+        linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
+                [addrLoc(p.config, loc), genTypeInfo(p.module, loc.t, loc.lode.info)])
       # XXX: generated reset procs should not touch the m_type
       # field, so disabling this should be safe:
       genObjectInit(p, cpsStmts, loc.t, loc, constructObj)
@@ -428,8 +432,14 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) =
   if optSeqDestructors in p.config.globalOptions and skipTypes(typ, abstractInst + {tyStatic}).kind in {tyString, tySequence}:
     linefmt(p, cpsStmts, "$1.len = 0; $1.p = NIM_NIL;$n", [rdLoc(loc)])
   elif not isComplexValueType(typ):
-    linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc),
-      getTypeDesc(p.module, typ)])
+    if containsGarbageCollectedRef(loc.t):
+      var nilLoc: TLoc
+      initLoc(nilLoc, locTemp, loc.lode, OnStack)
+      nilLoc.r = rope("NIM_NIL")
+      genRefAssign(p, loc, nilLoc)
+    else:
+      linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc),
+        getTypeDesc(p.module, typ)])
   else:
     if not isTemp or containsGarbageCollectedRef(loc.t):
       # don't use nimZeroMem for temporary values for performance if we can
@@ -680,6 +690,7 @@ proc loadDynamicLib(m: BModule, lib: PLib) =
     else:
       var p = newProc(nil, m)
       p.options = p.options - {optStackTrace}
+      p.flags.incl nimErrorFlagDisabled
       var dest: TLoc
       initLoc(dest, locTemp, lib.path, OnStack)
       dest.r = getTempName(m)
@@ -833,7 +844,8 @@ proc containsResult(n: PNode): bool =
     for i in 0..<n.safeLen:
       if containsResult(n[i]): return true
 
-const harmless = {nkConstSection, nkTypeSection, nkEmpty, nkCommentStmt, nkTemplateDef, nkMacroDef} +
+const harmless = {nkConstSection, nkTypeSection, nkEmpty, nkCommentStmt, nkTemplateDef,
+                  nkMacroDef, nkMixinStmt, nkBindStmt} +
                   declarativeDefs
 
 proc easyResultAsgn(n: PNode): PNode =
@@ -973,6 +985,9 @@ proc genProcBody(p: BProc; procBody: PNode) =
     p.blocks[0].sections[cpsLocals].add(ropecg(p.module, "NIM_BOOL* nimErr_;$n", []))
     p.blocks[0].sections[cpsInit].add(ropecg(p.module, "nimErr_ = #nimErrorFlag();$n", []))
 
+proc isNoReturn(m: BModule; s: PSym): bool {.inline.} =
+  sfNoReturn in s.flags and m.config.exc != excGoto
+
 proc genProcAux(m: BModule, prc: PSym) =
   var p = newProc(prc, m)
   var header = genProcHeader(m, prc)
@@ -1027,7 +1042,7 @@ proc genProcAux(m: BModule, prc: PSym) =
 
   var generatedProc: Rope
   generatedProc.genCLineDir prc.info, m.config
-  if sfNoReturn in prc.flags:
+  if isNoReturn(p.module, prc):
     if hasDeclspec in extccomp.CC[p.config.cCompiler].props:
       header = "__declspec(noreturn) " & header
   if sfPure in prc.flags:
@@ -1090,13 +1105,13 @@ proc genProcPrototype(m: BModule, sym: PSym) =
     let asPtr = isReloadable(m, sym)
     var header = genProcHeader(m, sym, asPtr)
     if not asPtr:
-      if sfNoReturn in sym.flags and hasDeclspec in extccomp.CC[m.config.cCompiler].props:
+      if isNoReturn(m, sym) and hasDeclspec in extccomp.CC[m.config.cCompiler].props:
         header = "__declspec(noreturn) " & header
       if sym.typ.callConv != ccInline and requiresExternC(m, sym):
         header = "extern \"C\" " & header
       if sfPure in sym.flags and hasAttribute in CC[m.config.cCompiler].props:
         header.add(" __attribute__((naked))")
-      if sfNoReturn in sym.flags and hasAttribute in CC[m.config.cCompiler].props:
+      if isNoReturn(m, sym) and hasAttribute in CC[m.config.cCompiler].props:
         header.add(" __attribute__((noreturn))")
     m.s[cfsProcHeaders].add(ropecg(m, "$1;$N", [header]))
 
@@ -1637,7 +1652,7 @@ proc genInitCode(m: BModule) =
     writeSection(preInitProc, cpsLocals)
     writeSection(preInitProc, cpsInit, m.hcrOn)
     writeSection(preInitProc, cpsStmts)
-    prc.addf("}$N", [])
+    prc.addf("}/* preInitProc end */$N", [])
 
   # add new scope for following code, because old vcc compiler need variable
   # be defined at the top of the block
@@ -1999,8 +2014,9 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
     if m.config.exc == excGoto and getCompilerProc(graph, "nimTestErrorFlag") != nil:
       discard cgsym(m, "nimTestErrorFlag")
 
-    for i in countdown(high(graph.globalDestructors), 0):
-      n.add graph.globalDestructors[i]
+    if {optGenStaticLib, optGenDynLib} * m.config.globalOptions == {}:
+      for i in countdown(high(graph.globalDestructors), 0):
+        n.add graph.globalDestructors[i]
   if passes.skipCodegen(m.config, n): return
   if moduleHasChanged(graph, m.module):
     # if the module is cached, we don't regenerate the main proc

compiler/cgendata.nim

@@ -70,7 +70,8 @@ type
     hasCurFramePointer,
     noSafePoints,
     nimErrorFlagAccessed,
-    nimErrorFlagDeclared
+    nimErrorFlagDeclared,
+    nimErrorFlagDisabled
 
   TCProc = object             # represents C proc that is currently generated
     prc*: PSym                # the Nim proc that this C proc belongs to

compiler/closureiters.nim

@@ -130,7 +130,8 @@
 
 import
   ast, msgs, idents,
-  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
+  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos,
+  tables, options
 
 type
   Ctx = object
@@ -155,7 +156,7 @@ type
 
 const
   nkSkip = {nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
-            nkCommentStmt} + procDefs
+            nkCommentStmt, nkMixinStmt, nkBindStmt} + procDefs
 
 proc newStateAccess(ctx: var Ctx): PNode =
   if ctx.stateVarSym.isNil:
@@ -1102,7 +1103,7 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
 
   result = ctx.states[stateIdx][0].intVal.int
 
-proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
+proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
   result = n
   case n.kind
   of nkSkip:
@@ -1282,6 +1283,101 @@ proc deleteEmptyStates(ctx: var Ctx) =
     else:
       inc i
 
+type
+  PreprocessContext = object
+    finallys: seq[PNode]
+    config: ConfigRef
+    blocks: seq[(PNode, int)]
+  FreshVarsContext = object
+    tab: Table[int, PSym]
+    config: ConfigRef
+    info: TLineInfo
+
+proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
+  case n.kind
+  of nkSym:
+    let x = c.tab.getOrDefault(n.sym.id)
+    if x == nil:
+      result = n
+    else:
+      result = newSymNode(x, n.info)
+  of nkSkip - {nkSym}:
+    result = n
+  of nkLetSection, nkVarSection:
+    result = copyNode(n)
+    for it in n:
+      if it.kind in {nkIdentDefs, nkVarTuple}:
+        let idefs = copyNode(it)
+        for v in 0..it.len-3:
+          if it[v].kind == nkSym:
+            let x = copySym(it[v].sym)
+            c.tab[it[v].sym.id] = x
+            idefs.add newSymNode(x)
+          else:
+            idefs.add it[v]
+
+        for rest in it.len-2 ..< it.len: idefs.add it[rest]
+        result.add idefs
+      else:
+        result.add it
+  of nkRaiseStmt:
+    localError(c.config, c.info, "unsupported control flow: 'finally: ... raise' duplicated because of 'break'")
+  else:
+    result = n
+    for i in 0..<n.safeLen:
+      result[i] = freshVars(n[i], c)
+
+proc preprocess(c: var PreprocessContext; n: PNode): PNode =
+  # in order to fix bug #15243 without risking regressions, we preprocess
+  # the AST so that 'break' statements inside a 'try finally' also have the
+  # finally section. We need to duplicate local variables here and also
+  # detect: 'finally: raises X' which is currently not supported. We produce
+  # an error for this case for now. All this will be done properly with Yuriy's
+  # patch.
+  result = n
+  case n.kind
+  of nkTryStmt:
+    let f = n.lastSon
+    if f.kind == nkFinally:
+      c.finallys.add f.lastSon
+
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
+    if f.kind == nkFinally:
+      discard c.finallys.pop()
+
+  of nkWhileStmt, nkBlockStmt:
+    c.blocks.add((n, c.finallys.len))
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+    discard c.blocks.pop()
+
+  of nkBreakStmt:
+    if c.blocks.len == 0:
+      discard
+    else:
+      var fin = -1
+      if n[0].kind == nkEmpty:
+        fin = c.blocks[^1][1]
+      elif n[0].kind == nkSym:
+        for i in countdown(c.blocks.high, 0):
+          if c.blocks[i][0].kind == nkBlockStmt and c.blocks[i][0][0].kind == nkSym and
+              c.blocks[i][0][0].sym == n[0].sym:
+            fin = c.blocks[i][1]
+            break
+
+      if fin >= 0:
+        result = newNodeI(nkStmtList, n.info)
+        for i in countdown(c.finallys.high, fin):
+          var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info)
+          result.add freshVars(preprocess(c, c.finallys[i]), vars)
+        result.add n
+  of nkSkip: discard
+  else:
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
 proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   var ctx: Ctx
   ctx.g = g
@@ -1294,7 +1390,10 @@ proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
     ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), fn, fn.info)
     ctx.stateVarSym.typ = g.createClosureIterStateType(fn)
   ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), fn, fn.info)
-  var n = n.toStmtList
+  var pc = PreprocessContext(finallys: @[], config: g.config)
+  var n = preprocess(pc, n.toStmtList)
+  #echo "transformed into ", n
+  #var n = n.toStmtList
 
   discard ctx.newState(n, nil)
   let gotoOut = newTree(nkGotoState, g.newIntLit(n.info, -1))

compiler/commands.nim

@@ -478,6 +478,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       of "boehm":
         conf.selectedGC = gcBoehm
         defineSymbol(conf.symbols, "boehmgc")
+        incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS
       of "refc":
         conf.selectedGC = gcRefc
       of "v2":
@@ -850,6 +851,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     of "off": conf.globalOptions = conf.globalOptions - {optStyleHint, optStyleError}
     of "hint": conf.globalOptions = conf.globalOptions + {optStyleHint} - {optStyleError}
     of "error": conf.globalOptions = conf.globalOptions + {optStyleError}
+    of "usages": conf.globalOptions.incl optStyleUsages
     else: localError(conf, info, errOffHintsError % arg)
   of "showallmismatches":
     processOnOffSwitchG(conf, {optShowAllMismatches}, arg, pass, info)

compiler/condsyms.nim

@@ -116,3 +116,5 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimNewIntegerOps")
   defineSymbol("nimHasInvariant")
   defineSymbol("nimHasStacktraceMsgs")
+  defineSymbol("nimHasStacktracesModule")
+  defineSymbol("nimHasJsBigIntBackend")

compiler/dfa.nim

@@ -35,26 +35,30 @@ from patterns import sameTrees
 
 type
   InstrKind* = enum
-    goto, fork, join, def, use
+    goto, fork, def, use
   Instr* = object
     n*: PNode # contains the def/use location.
     case kind*: InstrKind
-    of goto, fork, join: dest*: int
+    of goto, fork: dest*: int
     else: discard
 
   ControlFlowGraph* = seq[Instr]
 
   TPosition = distinct int
+
   TBlock = object
-    label: PSym
-    fixups: seq[TPosition]
+    case isTryBlock: bool
+    of false:
+      label: PSym
+      breakFixups: seq[(TPosition, seq[PNode])] #Contains the gotos for the breaks along with their pending finales
+    of true:
+      finale: PNode
+      raiseFixups: seq[TPosition] #Contains the gotos for the raises
 
   Con = object
     code: ControlFlowGraph
     inCall, inTryStmt: int
     blocks: seq[TBlock]
-    tryStmtFixups: seq[TPosition]
-    forks: seq[TPosition]
     owner: PSym
 
 proc debugInfo(info: TLineInfo): string =
@@ -66,7 +70,7 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
   var jumpTargets = initIntSet()
   let last = if last < 0: c.len-1 else: min(last, c.len-1)
   for i in start..last:
-    if c[i].kind in {goto, fork, join}:
+    if c[i].kind in {goto, fork}:
       jumpTargets.incl(i+c[i].dest)
   var i = start
   while i <= last:
@@ -77,7 +81,7 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
     case c[i].kind
     of def, use:
       result.add renderTree(c[i].n)
-    of goto, fork, join:
+    of goto, fork:
       result.add "L"
       result.addInt c[i].dest+i
     result.add("\t#")
@@ -96,7 +100,6 @@ proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
 proc forkI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: fork, dest: 0)
-  c.forks.add result
 
 proc gotoI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
@@ -104,6 +107,24 @@ proc gotoI(c: var Con; n: PNode): TPosition =
 
 #[
 
+Join is no more
+===============
+Instead of generating join instructions we adapt our traversal of the CFG.
+
+When encountering a fork we split into two paths, we follow the path
+starting at "pc + 1" until it encounters the joinpoint: "pc + forkInstr.dest".
+If we encounter gotos that would jump further than the current joinpoint,
+as can happen with gotos generated by unstructured controlflow such as break, raise or return,
+we simply suspend following the current path, and follow the other path until the new joinpoint
+which is simply the instruction pointer returned to us by the now suspended path.
+If the path we are following now, also encounters a goto that exceeds the joinpoint
+we repeat the process; suspending the current path and evaluating the other one with a new joinpoint.
+If we eventually reach a common joinpoint we join the two paths.
+This new "ping-pong" approach has the obvious advantage of not requiring join instructions, as such
+cutting down on the CFG size but is also mandatory for correctly handling complicated cases
+of unstructured controlflow.
+
+
 Design of join
 ==============
 
@@ -252,10 +273,6 @@ duplicate the 'join' instructions on breaks and return exits!
 
 ]#
 
-proc joinI(c: var Con; fromFork: TPosition; n: PNode) =
-  let dist = fromFork.int - c.code.len
-  c.code.add Instr(n: n, kind: join, dest: dist)
-
 proc genLabel(c: Con): TPosition =
   result = TPosition(c.code.len)
 
@@ -271,14 +288,23 @@ proc patch(c: var Con, p: TPosition) =
   doAssert(low(int) div 2 + 1 < diff and diff < high(int) div 2)
   c.code[p].dest = diff
 
+proc gen(c: var Con; n: PNode) # {.noSideEffect.}
+
 proc popBlock(c: var Con; oldLen: int) =
-  for f in c.blocks[oldLen].fixups:
-    c.patch(f)
+  var exits: seq[TPosition]
+  exits.add c.gotoI(newNode(nkEmpty))
+  for f in c.blocks[oldLen].breakFixups:
+    c.patch(f[0])
+    for finale in f[1]:
+      c.gen(finale)
+    exits.add c.gotoI(newNode(nkEmpty))
+  for e in exits:
+    c.patch e
   c.blocks.setLen(oldLen)
 
 template withBlock(labl: PSym; body: untyped) {.dirty.} =
   var oldLen {.gensym.} = c.blocks.len
-  c.blocks.add TBlock(label: labl, fixups: @[])
+  c.blocks.add TBlock(isTryBlock: false, label: labl)
   body
   popBlock(c, oldLen)
 
@@ -286,8 +312,6 @@ proc isTrue(n: PNode): bool =
   n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or
     n.kind == nkIntLit and n.intVal != 0
 
-proc gen(c: var Con; n: PNode) # {.noSideEffect.}
-
 when true:
   proc genWhile(c: var Con; n: PNode) =
     # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
@@ -299,12 +323,13 @@ when true:
 
     Becomes:
 
-    if cond:
-      body
+    block:
       if cond:
         body
         if cond:
           body
+          if cond:
+            body
 
     We still need to ensure 'break' resolves properly, so an AST to AST
     translation is impossible.
@@ -330,22 +355,19 @@ when true:
     if isTrue(n[0]):
       # 'while true' is an idiom in Nim and so we produce
       # better code for it:
-      for i in 0..2:
-        withBlock(nil):
+      withBlock(nil):
+        for i in 0..2:
           c.gen(n[1])
     else:
-      let oldForksLen = c.forks.len
-      var endings: array[3, TPosition]
-      for i in 0..2:
-        withBlock(nil):
+      withBlock(nil):
+        var endings: array[3, TPosition]
+        for i in 0..2:
           c.gen(n[0])
           endings[i] = c.forkI(n)
           c.gen(n[1])
-      for i in countdown(endings.high, 0):
-        let endPos = endings[i]
-        c.patch(endPos)
-        c.joinI(c.forks.pop(), n)
-      doAssert(c.forks.len == oldForksLen)
+        for i in countdown(endings.high, 0):
+          let endPos = endings[i]
+          c.patch(endPos)
 
 else:
 
@@ -356,7 +378,6 @@ else:
     #   body
     #   jmp lab1
     # lab2:
-    let oldForksLen = c.forks.len
     let lab1 = c.genLabel
     withBlock(nil):
       if isTrue(n[0]):
@@ -368,35 +389,11 @@ else:
         c.gen(n[1])
         c.jmpBack(n, lab1)
         c.patch(lab2)
-    setLen(c.forks, oldForksLen)
-
-proc genBlock(c: var Con; n: PNode) =
-  withBlock(n[0].sym):
-    c.gen(n[1])
-
-proc genJoins(c: var Con; n: PNode) =
-  for i in countdown(c.forks.high, 0): joinI(c, c.forks[i], n)
-
-proc genBreak(c: var Con; n: PNode) =
-  genJoins(c, n)
-  let lab1 = c.gotoI(n)
-  if n[0].kind == nkSym:
-    #echo cast[int](n[0].sym)
-    for i in countdown(c.blocks.len-1, 0):
-      if c.blocks[i].label == n[0].sym:
-        c.blocks[i].fixups.add lab1
-        return
-    #globalError(n.info, "VM problem: cannot find 'break' target")
-  else:
-    c.blocks[c.blocks.high].fixups.add lab1
 
 template forkT(n, body) =
-  let oldLen = c.forks.len
   let lab1 = c.forkI(n)
   body
   c.patch(lab1)
-  c.joinI(lab1, n)
-  setLen(c.forks, oldLen)
 
 proc genIf(c: var Con, n: PNode) =
   #[
@@ -435,7 +432,6 @@ proc genIf(c: var Con, n: PNode) =
     join F1
 
   ]#
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
   for i in 0..<n.len:
     var it = n[i]
@@ -448,8 +444,6 @@ proc genIf(c: var Con, n: PNode) =
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
 
 proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, a
@@ -476,7 +470,6 @@ proc genCase(c: var Con; n: PNode) =
     abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
 
   var endings: seq[TPosition] = @[]
-  let oldLen = c.forks.len
   c.gen(n[0])
   for i in 1..<n.len:
     let it = n[i]
@@ -493,27 +486,53 @@ proc genCase(c: var Con; n: PNode) =
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
+
+proc genBlock(c: var Con; n: PNode) =
+  withBlock(n[0].sym):
+    c.gen(n[1])
+
+proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) =
+  let lab1 = c.gotoI(n)
+  if c.blocks[i].isTryBlock:
+    c.blocks[i].raiseFixups.add lab1
+  else:
+    var trailingFinales: seq[PNode]
+    if c.inTryStmt > 0: #Ok, we are in a try, lets see which (if any) try's we break out from:
+      for b in countdown(c.blocks.high, i):
+        if c.blocks[b].isTryBlock:
+          trailingFinales.add c.blocks[b].finale
+
+    c.blocks[i].breakFixups.add (lab1, trailingFinales)
+
+proc genBreak(c: var Con; n: PNode) =
+  if n[0].kind == nkSym:
+    #echo cast[int](n[0].sym)
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock and c.blocks[i].label == n[0].sym:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    #globalError(n.info, "VM problem: cannot find 'break' target")
+  else:
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
 
 proc genTry(c: var Con; n: PNode) =
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
-  inc c.inTryStmt
-  let oldFixups = c.tryStmtFixups.len
 
+  let oldLen = c.blocks.len
+  c.blocks.add TBlock(isTryBlock: true, finale: if n[^1].kind == nkFinally: n[^1] else: newNode(nkEmpty))
+
+  inc c.inTryStmt
   #let elsePos = c.forkI(n)
   c.gen(n[0])
   dec c.inTryStmt
-  for i in oldFixups..c.tryStmtFixups.high:
-    let f = c.tryStmtFixups[i]
-    c.patch(f)
-    # we also need to produce join instructions
-    # for the 'fork' that might precede the goto instruction
-    if f.int-1 >= 0 and c.code[f.int-1].kind == fork:
-      c.joinI(TPosition(f.int-1), n)
 
-  setLen(c.tryStmtFixups, oldFixups)
+  for f in c.blocks[oldLen].raiseFixups:
+    c.patch(f)
+
+  c.blocks.setLen oldLen
 
   #c.patch(elsePos)
   for i in 1..<n.len:
@@ -526,25 +545,26 @@ proc genTry(c: var Con; n: PNode) =
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
 
   # join the 'elsePos' forkI instruction:
-  #c.joinI(c.forks.pop(), n)
+  #c.joinI(c.blocks[^1].forks.pop(), n)
 
   let fin = lastSon(n)
   if fin.kind == nkFinally:
     c.gen(fin[0])
-  doAssert(c.forks.len == oldLen)
 
 template genNoReturn(c: var Con; n: PNode) =
   # leave the graph
   c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 proc genRaise(c: var Con; n: PNode) =
-  genJoins(c, n)
   gen(c, n[0])
   if c.inTryStmt > 0:
-    c.tryStmtFixups.add c.gotoI(n)
+    for i in countdown(c.blocks.high, 0):
+      if c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    assert false #Unreachable
   else:
     genNoReturn(c, n)
 
@@ -553,12 +573,11 @@ proc genImplicitReturn(c: var Con) =
     gen(c, c.owner.ast[resultPos])
 
 proc genReturn(c: var Con; n: PNode) =
-  genJoins(c, n)
   if n[0].kind != nkEmpty:
     gen(c, n[0])
   else:
     genImplicitReturn(c)
-  genNoReturn(c, n)
+  genBreakOrRaiseAux(c, 0, n)
 
 const
   InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp}
@@ -598,7 +617,8 @@ proc genUse(c: var Con; orig: PNode) =
 proc aliases*(obj, field: PNode): bool =
   var n = field
   var obj = obj
-  while obj.kind in {nkHiddenSubConv, nkHiddenStdConv, nkObjDownConv, nkObjUpConv}:
+  while obj.kind in {nkHiddenSubConv, nkHiddenStdConv, nkObjDownConv, nkObjUpConv,
+                     nkAddr, nkHiddenAddr, nkDerefExpr, nkHiddenDeref}:
     obj = obj[0]
   while true:
     if sameTrees(obj, n): return true
@@ -639,8 +659,10 @@ proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
     of nkHiddenDeref, nkDerefExpr:
       # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
       # pointer indirection.
+      # bug #14159, we cannot reason about sinkParam[].location as it can
+      # still be shared for tyRef.
       n = n[0]
-      return n.kind == nkSym and n.sym.owner == owner and (isSinkParam(n.sym) or
+      return n.kind == nkSym and n.sym.owner == owner and (
           n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
     else:
       break
@@ -706,9 +728,11 @@ proc genCall(c: var Con; n: PNode) =
     # lab1:
     # join F1
     let endGoto = c.forkI(n)
-    c.tryStmtFixups.add c.gotoI(n)
+    for i in countdown(c.blocks.high, 0):
+      if c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        break
     c.patch(endGoto)
-    c.joinI(c.forks.pop(), n)
   dec c.inCall
 
 proc genMagic(c: var Con; n: PNode; m: TMagic) =
@@ -782,6 +806,7 @@ proc gen(c: var Con; n: PNode) =
 proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
   ## constructs a control flow graph for ``body``.
   var c = Con(code: @[], blocks: @[], owner: s)
-  gen(c, body)
-  genImplicitReturn(c)
+  withBlock(s):
+    gen(c, body)
+    genImplicitReturn(c)
   shallowCopy(result, c.code)

compiler/docgen.nim

@@ -86,11 +86,14 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
     result = relativeTo(file, conf.docRoot.AbsoluteDir)
   else:
     bail()
+  if isAbsolute(result.string):
+    result = file.string.splitPath()[1].RelativeFile
   if isTitle:
     result = result.string.nativeToUnix.RelativeFile
   else:
     result = result.string.replace("..", "@@").RelativeFile ## refs #13223
   doAssert not result.isEmpty
+  doAssert not isAbsolute(result.string)
 
 proc whichType(d: PDoc; n: PNode): PSym =
   if n.kind == nkSym:
@@ -1140,7 +1143,10 @@ proc generateIndex*(d: PDoc) =
 proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
   if d.module == nil or sfMainModule in d.module.flags: # nil for eg for commandRst2Html
     if d.conf.outDir.isEmpty: d.conf.outDir = d.conf.docOutDir
-    if d.conf.outFile.isEmpty: d.conf.outFile = outfile.relativeTo(d.conf.outDir)
+    if d.conf.outFile.isEmpty:
+      d.conf.outFile = outfile.relativeTo(d.conf.outDir)
+      if isAbsolute(d.conf.outFile.string):
+        d.conf.outFile = splitPath(d.conf.outFile.string)[1].RelativeFile
 
 proc writeOutput*(d: PDoc, useWarning = false) =
   runAllExamples(d)

compiler/extccomp.nim

@@ -601,6 +601,34 @@ proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile):
       "Compiler '$1' doesn't support the requested target" %
       CC[compiler].name)
 
+proc ccHasSaneOverflow*(conf: ConfigRef): bool =
+  if conf.cCompiler == ccGcc:
+    result = false # assume an old or crappy GCC
+    var exe = getConfigVar(conf, conf.cCompiler, ".exe")
+    if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
+    let (s, exitCode) = try: execCmdEx(exe & " --version") except: ("", 1)
+    if exitCode == 0:
+      var i = 0
+      var j = 0
+      # the version is the last part of the first line:
+      while i < s.len and s[i] != '\n':
+        if s[i] in {' ', '\t'}: j = i+1
+        inc i
+      if j > 0:
+        var major = 0
+        while j < s.len and s[j] in {'0'..'9'}:
+          major = major * 10 + (ord(s[j]) - ord('0'))
+          inc j
+        if i < s.len and s[j] == '.': inc j
+        while j < s.len and s[j] in {'0'..'9'}:
+          inc j
+        if j+1 < s.len and s[j] == '.' and s[j+1] in {'0'..'9'}:
+          # we found a third version number, chances are high
+          # we really parsed the version:
+          result = major >= 5
+  else:
+    result = conf.cCompiler == ccCLang
+
 proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
            elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler
@@ -623,6 +651,10 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
   if useCpp(conf, cfile.cname):
     options.add(' ' & CC[c].cppXsupport)
 
+  if cfile.customArgs != "":
+    options.add ' '
+    options.add cfile.customArgs
+
   var compilePattern: string
   # compute include paths:
   var includeCmd = CC[c].includeCmd & quoteShell(conf.libpath)
@@ -836,12 +868,12 @@ proc execLinkCmd(conf: ConfigRef; linkCmd: string) =
 
 proc maybeRunDsymutil(conf: ConfigRef; exe: AbsoluteFile) =
   when defined(osx):
-    if optCDebug notin conf.globalOptions: return
-    # if needed, add an option to skip or override location
-    let cmd = "dsymutil " & $(exe).quoteShell
-    conf.extraCmds.add cmd
-    tryExceptOSErrorMessage(conf, "invocation of dsymutil failed."):
-      execExternalProgram(conf, cmd, hintExecuting)
+    if optCDebug in conf.globalOptions and optGenStaticLib notin conf.globalOptions:
+      # if needed, add an option to skip or override location
+      let cmd = "dsymutil " & $(exe).quoteShell
+      conf.extraCmds.add cmd
+      tryExceptOSErrorMessage(conf, "invocation of dsymutil failed."):
+        execExternalProgram(conf, cmd, hintExecuting)
 
 proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx: int)) =
   let runCb = proc (idx: int, p: Process) =

compiler/injectdestructors.nim

@@ -22,9 +22,9 @@
 # - eliminate 'wasMoved(x); destroy(x)' pairs as a post processing step.
 
 import
-  intsets, ast, msgs, renderer, magicsys, types, idents,
+  intsets, ast, astalgo, msgs, renderer, magicsys, types, idents,
   strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
-  lineinfos, parampatterns, sighashes
+  lineinfos, parampatterns, sighashes, liftdestructors
 
 from trees import exprStructuralEquivalent
 from algorithm import reverse
@@ -49,6 +49,12 @@ type
     uninit: IntSet # set of uninit'ed vars
     uninitComputed: bool
 
+  ProcessMode = enum
+    normal
+    consumed
+    sinkArg
+
+
 const toDebug {.strdefine.} = ""
 
 template dbg(body) =
@@ -56,9 +62,12 @@ template dbg(body) =
     if c.owner.name.s == toDebug or toDebug == "always":
       body
 
-proc isLastRead(location: PNode; c: var Con; pc, comesFrom: int): int =
+proc p(n: PNode; c: var Con; mode: ProcessMode): PNode
+proc moveOrCopy(dest, ri: PNode; c: var Con): PNode
+
+proc isLastRead(location: PNode; c: var Con; pc, until: int): int =
   var pc = pc
-  while pc < c.g.len:
+  while pc < c.g.len and pc < until:
     case c.g[pc].kind
     of def:
       if defInstrTargets(c.g[pc], location):
@@ -74,17 +83,17 @@ proc isLastRead(location: PNode; c: var Con; pc, comesFrom: int): int =
       pc = pc + c.g[pc].dest
     of fork:
       # every branch must lead to the last read of the location:
-      let variantA = isLastRead(location, c, pc+1, pc)
-      if variantA < 0: return -1
-      var variantB = isLastRead(location, c, pc + c.g[pc].dest, pc)
-      if variantB < 0: return -1
-      elif variantB == high(int):
-        variantB = variantA
-      pc = variantB
-    of InstrKind.join:
-      let dest = pc + c.g[pc].dest
-      if dest == comesFrom: return pc + 1
-      inc pc
+      var variantA = pc + 1
+      var variantB = pc + c.g[pc].dest
+      while variantA != variantB:
+        if min(variantA, variantB) < 0: return -1
+        if max(variantA, variantB) >= c.g.len or min(variantA, variantB) >= until:
+          break
+        if variantA < variantB:
+          variantA = isLastRead(location, c, variantA, min(variantB, until))
+        else:
+          variantB = isLastRead(location, c, variantB, min(variantA, until))
+      pc = min(variantA, variantB)
   return pc
 
 proc isLastRead(n: PNode; c: var Con): bool =
@@ -107,12 +116,12 @@ proc isLastRead(n: PNode; c: var Con): bool =
   # ensure that we don't find another 'use X' instruction.
   if instr+1 >= c.g.len: return true
 
-  result = isLastRead(n, c, instr+1, -1) >= 0
+  result = isLastRead(n, c, instr+1, int.high) >= 0
   dbg: echo "ugh ", c.otherRead.isNil, " ", result
 
-proc isFirstWrite(location: PNode; c: var Con; pc, comesFrom: int; instr: int): int =
+proc isFirstWrite(location: PNode; c: var Con; pc, until: int): int =
   var pc = pc
-  while pc < instr:
+  while pc < until:
     case c.g[pc].kind
     of def:
       if defInstrTargets(c.g[pc], location):
@@ -127,17 +136,17 @@ proc isFirstWrite(location: PNode; c: var Con; pc, comesFrom: int; instr: int):
       pc = pc + c.g[pc].dest
     of fork:
       # every branch must not contain a def/use of our location:
-      let variantA = isFirstWrite(location, c, pc+1, pc, instr)
-      if variantA < 0: return -1
-      var variantB = isFirstWrite(location, c, pc + c.g[pc].dest, pc, instr + c.g[pc].dest)
-      if variantB < 0: return -1
-      elif variantB == high(int):
-        variantB = variantA
-      pc = variantB
-    of InstrKind.join:
-      let dest = pc + c.g[pc].dest
-      if dest == comesFrom: return pc + 1
-      inc pc
+      var variantA = pc + 1
+      var variantB = pc + c.g[pc].dest
+      while variantA != variantB:
+        if min(variantA, variantB) < 0: return -1
+        if max(variantA, variantB) > until:
+          break
+        if variantA < variantB:
+          variantA = isFirstWrite(location, c, variantA, min(variantB, until))
+        else:
+          variantB = isFirstWrite(location, c, variantB, min(variantA, until))
+      pc = min(variantA, variantB)
   return pc
 
 proc isFirstWrite(n: PNode; c: var Con): bool =
@@ -156,10 +165,10 @@ proc isFirstWrite(n: PNode; c: var Con): bool =
   # ensure that we don't find another 'def/use X' instruction.
   if instr == 0: return true
 
-  result = isFirstWrite(n, c, 0, -1, instr) >= 0
+  result = isFirstWrite(n, c, 0, instr) >= 0
 
 proc initialized(code: ControlFlowGraph; pc: int,
-                 init, uninit: var IntSet; comesFrom: int): int =
+                 init, uninit: var IntSet; until: int): int =
   ## Computes the set of definitely initialized variables across all code paths
   ## as an IntSet of IDs.
   var pc = pc
@@ -168,20 +177,22 @@ proc initialized(code: ControlFlowGraph; pc: int,
     of goto:
       pc = pc + code[pc].dest
     of fork:
-      let target = pc + code[pc].dest
       var initA = initIntSet()
       var initB = initIntSet()
-      let pcA = initialized(code, pc+1, initA, uninit, pc)
-      discard initialized(code, target, initB, uninit, pc)
+      var variantA = pc + 1
+      var variantB = pc + code[pc].dest
+      while variantA != variantB:
+        if max(variantA, variantB) > until:
+          break
+        if variantA < variantB:
+          variantA = initialized(code, variantA, initA, uninit, min(variantB, until))
+        else:
+          variantB = initialized(code, variantB, initB, uninit, min(variantA, until))
+      pc = min(variantA, variantB)
       # we add vars if they are in both branches:
       for v in initA:
         if v in initB:
           init.incl v
-      pc = pcA+1
-    of InstrKind.join:
-      let target = pc + code[pc].dest
-      if comesFrom == target: return pc
-      inc pc
     of use:
       let v = code[pc].n.sym
       if v.kind != skParam and v.id notin init:
@@ -201,7 +212,7 @@ template isUnpackedTuple(n: PNode): bool =
 
 proc checkForErrorPragma(c: Con; t: PType; ri: PNode; opname: string) =
   var m = "'" & opname & "' is not available for type <" & typeToString(t) & ">"
-  if opname == "=" and ri != nil:
+  if (opname == "=" or opname == "=copy") and ri != nil:
     m.add "; requires a copy because it's not the last read of '"
     m.add renderTree(ri)
     m.add '\''
@@ -220,16 +231,19 @@ proc makePtrType(c: Con, baseType: PType): PType =
   result = newType(tyPtr, c.owner)
   addSonSkipIntLit(result, baseType)
 
+proc genOp(c: Con; op: PSym; dest: PNode): PNode =
+  let addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
+  addrExp.add(dest)
+  result = newTree(nkCall, newSymNode(op), addrExp)
+
 proc genOp(c: Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode =
   var op = t.attachedOps[kind]
-
   if op == nil or op.ast[genericParamsPos].kind != nkEmpty:
     # give up and find the canonical type instead:
     let h = sighashes.hashType(t, {CoType, CoConsiderOwned, CoDistinct})
     let canon = c.graph.canonTypes.getOrDefault(h)
     if canon != nil:
       op = canon.attachedOps[kind]
-
   if op == nil:
     #echo dest.typ.id
     globalError(c.graph.config, dest.info, "internal error: '" & AttachedOpToStr[kind] &
@@ -241,9 +255,7 @@ proc genOp(c: Con; t: PType; kind: TTypeAttachedOp; dest, ri: PNode): PNode =
     if kind == attachedDestructor:
       echo "destructor is ", op.id, " ", op.ast
   if sfError in op.flags: checkForErrorPragma(c, t, ri, AttachedOpToStr[kind])
-  let addrExp = newNodeIT(nkHiddenAddr, dest.info, makePtrType(c, dest.typ))
-  addrExp.add(dest)
-  result = newTree(nkCall, newSymNode(op), addrExp)
+  genOp(c, op, dest)
 
 proc genDestroy(c: Con; dest: PNode): PNode =
   let t = dest.typ.skipTypes({tyGenericInst, tyAlias, tySink})
@@ -289,7 +301,7 @@ proc genCopy(c: var Con; dest, ri: PNode): PNode =
   if tfHasOwned in t.flags and ri.kind != nkNilLit:
     # try to improve the error message here:
     if c.otherRead == nil: discard isLastRead(ri, c)
-    checkForErrorPragma(c, t, ri, "=")
+    checkForErrorPragma(c, t, ri, "=copy")
   result = genCopyNoCheck(c, dest, ri)
 
 proc addTopVar(c: var Con; v: PNode) =
@@ -300,6 +312,44 @@ proc getTemp(c: var Con; typ: PType; info: TLineInfo): PNode =
   sym.typ = typ
   result = newSymNode(sym)
 
+proc genDiscriminantAsgn(c: var Con; n: PNode): PNode =
+  # discriminator is ordinal value that doesn't need sink destroy
+  # but fields within active case branch might need destruction
+
+  # tmp to support self assignments 
+  let tmp = getTemp(c, n[1].typ, n.info)
+  c.addTopVar(tmp)
+
+  result = newTree(nkStmtList)
+  result.add newTree(nkFastAsgn, tmp, p(n[1], c, consumed))
+  result.add p(n[0], c, normal)
+
+  let le = p(n[0], c, normal)
+  let leDotExpr = if le.kind == nkCheckedFieldExpr: le[0] else: le
+  let objType = leDotExpr[0].typ
+
+  if hasDestructor(objType):
+    if objType.attachedOps[attachedDestructor] != nil and
+        sfOverriden in objType.attachedOps[attachedDestructor].flags:
+      localError(c.graph.config, n.info, errGenerated, """Assignment to discriminant for object's with user defined destructor is not supported, object must have default destructor. 
+It is best to factor out piece of object that needs custom destructor into separate object or not use discriminator assignment""")
+      result.add newTree(nkFastAsgn, le, tmp)
+      return
+
+    # generate: if le != tmp: `=destroy`(le)
+    let branchDestructor = produceDestructorForDiscriminator(c.graph, objType, leDotExpr[1].sym, n.info)
+    let cond = newNodeIT(nkInfix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
+    cond.add newSymNode(getMagicEqSymForType(c.graph, le.typ, n.info))
+    cond.add le
+    cond.add tmp
+    let notExpr = newNodeIT(nkPrefix, n.info, getSysType(c.graph, unknownLineInfo, tyBool))
+    notExpr.add newSymNode(createMagic(c.graph, "not", mNot))
+    notExpr.add cond
+    result.add newTree(nkIfStmt, newTree(nkElifBranch, notExpr, genOp(c, branchDestructor, le)))
+    result.add newTree(nkFastAsgn, le, tmp)
+  else:
+    result.add newTree(nkFastAsgn, le, tmp)
+
 proc genWasMoved(n: PNode; c: var Con): PNode =
   result = newNodeI(nkCall, n.info)
   result.add(newSymNode(createMagic(c.graph, "wasMoved", mWasMoved)))
@@ -313,6 +363,7 @@ proc genDefaultCall(t: PType; c: Con; info: TLineInfo): PNode =
 proc destructiveMoveVar(n: PNode; c: var Con): PNode =
   # generate: (let tmp = v; reset(v); tmp)
   if not hasDestructor(n.typ):
+    assert n.kind != nkSym or not hasDestructor(n.sym.typ)
     result = copyTree(n)
   else:
     result = newNodeIT(nkStmtListExpr, n.info, n.typ)
@@ -338,16 +389,8 @@ proc sinkParamIsLastReadCheck(c: var Con, s: PNode) =
     localError(c.graph.config, c.otherRead.info, "sink parameter `" & $s.sym.name.s &
         "` is already consumed at " & toFileLineCol(c. graph.config, s.info))
 
-type
-  ProcessMode = enum
-    normal
-    consumed
-    sinkArg
-
-proc p(n: PNode; c: var Con; mode: ProcessMode): PNode
-proc moveOrCopy(dest, ri: PNode; c: var Con): PNode
-
-proc isClosureEnv(n: PNode): bool = n.kind == nkSym and n.sym.name.s[0] == ':'
+proc isClosureEnv(n: PNode): bool =
+  n.kind == nkDotExpr and n[0].kind == nkHiddenDeref and n[0][0].typ.kind == tyRef
 
 proc passCopyToSink(n: PNode; c: var Con): PNode =
   result = newNodeIT(nkStmtListExpr, n.info, n.typ)
@@ -362,7 +405,7 @@ proc passCopyToSink(n: PNode; c: var Con): PNode =
     if isLValue(n) and not isClosureEnv(n) and n.typ.skipTypes(abstractInst).kind != tyRef:
       message(c.graph.config, n.info, hintPerformance,
         ("passing '$1' to a sink parameter introduces an implicit copy; " &
-        "use 'move($1)' to prevent it") % $n)
+        "if possible, rearrange your program's control flow to prevent it") % $n)
   else:
     if c.graph.config.selectedGC in {gcArc, gcOrc}:
       assert(not containsGarbageCollectedRef(n.typ))
@@ -867,6 +910,8 @@ proc p(n: PNode; c: var Con; mode: ProcessMode): PNode =
             cycleCheck(n, c)
           assert n[1].kind notin {nkAsgn, nkFastAsgn}
           result = moveOrCopy(p(n[0], c, mode), n[1], c)
+      elif isDiscriminantField(n[0]):
+        result = genDiscriminantAsgn(c, n)
       else:
         result = copyNode(n)
         result.add p(n[0], c, mode)
@@ -900,7 +945,8 @@ proc p(n: PNode; c: var Con; mode: ProcessMode): PNode =
     of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef,
        nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo,
        nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt,
-       nkExportStmt, nkPragma, nkCommentStmt, nkBreakState:
+       nkExportStmt, nkPragma, nkCommentStmt, nkBreakState,
+       nkMixinStmt, nkBindStmt:
       result = n
     of nkBreakStmt:
       inc c.hasUnstructuredCf
@@ -910,6 +956,10 @@ proc p(n: PNode; c: var Con; mode: ProcessMode): PNode =
       for i in 0..<n.len:
         result[i] = p(n[i], c, mode)
       inc c.hasUnstructuredCf
+    of nkCast:
+      result = shallowCopy(n)
+      result[0] = n[0]
+      result[1] = p(n[1], c, mode)
     else:
       result = shallowCopy(n)
       for i in 0..<n.len:
@@ -991,7 +1041,7 @@ proc computeUninit(c: var Con) =
     c.uninitComputed = true
     c.uninit = initIntSet()
     var init = initIntSet()
-    discard initialized(c.g, pc = 0, init, c.uninit, comesFrom = -1)
+    discard initialized(c.g, pc = 0, init, c.uninit, int.high)
 
 proc injectDefaultCalls(n: PNode, c: var Con) =
   case n.kind
@@ -1015,7 +1065,7 @@ proc injectDefaultCalls(n: PNode, c: var Con) =
 proc extractDestroysForTemporaries(c: Con, destroys: PNode): PNode =
   result = newNodeI(nkStmtList, destroys.info)
   for i in 0..<destroys.len:
-    if destroys[i][1][0].sym.kind == skTemp:
+    if destroys[i][1][0].sym.kind in {skTemp, skForVar}:
       result.add destroys[i]
       destroys[i] = c.emptyNode
 

compiler/installer.ini

@@ -6,12 +6,12 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv64
-  macosx: i386;amd64;powerpc64
+  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv32;riscv64
+  macosx: i386;amd64;powerpc64;arm64
   solaris: i386;amd64;sparc;sparc64
-  freebsd: i386;amd64;powerpc64;arm;arm64
+  freebsd: i386;amd64;powerpc64;arm;arm64;riscv64;sparc64;mips;mipsel;mips64;mips64el;powerpc;powerpc64el
   netbsd: i386;amd64
-  openbsd: i386;amd64
+  openbsd: i386;amd64;arm;arm64
   dragonfly: i386;amd64
   haiku: i386;amd64
   android: i386;arm;arm64
@@ -77,6 +77,7 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/fusion"
 
 Files: "tests"
 
@@ -89,6 +90,7 @@ Files: "bin/vccexe.exe"
 Files: "bin/nimgrab.exe"
 Files: "bin/nimpretty.exe"
 Files: "bin/testament.exe"
+Files: "bin/nim-gdb.bat"
 
 Files: "koch.exe"
 Files: "finish.exe"
@@ -109,6 +111,7 @@ Download: r"Aporia Text Editor|dist|aporia.zip|97997|https://nim-lang.org/downlo
 Files: "bin/makelink.exe"
 Files: "bin/7zG.exe"
 Files: "bin/*.dll"
+Files: "bin/cacert.pem"
 
 [UnixBin]
 Files: "bin/nim"
@@ -117,6 +120,8 @@ Files: "bin/nim"
 [Unix]
 InstallScript: "yes"
 UninstallScript: "yes"
+Files: "bin/nim-gdb"
+Files: "bin/nim-gdb.bash"
 
 
 [InnoSetup]

compiler/jsgen.nim

@@ -1591,7 +1591,10 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   var t = skipTypes(typ, abstractInst)
   case t.kind
   of tyInt..tyInt64, tyUInt..tyUInt64, tyEnum, tyChar:
-    result = putToSeq("0", indirect)
+    if $t.sym.loc.r == "bigint":
+      result = putToSeq("0n", indirect)
+    else:
+      result = putToSeq("0", indirect)
   of tyFloat..tyFloat128:
     result = putToSeq("0.0", indirect)
   of tyRange, tyGenericInst, tyAlias, tySink, tyOwned:
@@ -1634,7 +1637,7 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   of tyObject:
     var initList: Rope
     createObjInitList(p, t, initIntSet(), initList)
-    result = ("{$1}") % [initList]
+    result = ("({$1})") % [initList]
     if indirect: result = "[$1]" % [result]
   of tyVar, tyPtr, tyLent, tyRef, tyPointer:
     if mapType(p, t) == etyBaseIndex:
@@ -2124,11 +2127,17 @@ proc genConv(p: PProc, n: PNode, r: var TCompRes) =
   if dest.kind == src.kind:
     # no-op conversion
     return
-  case dest.kind:
-  of tyBool:
+  let toInt = (dest.kind in tyInt..tyInt32)
+  let fromInt = (src.kind in tyInt..tyInt32)
+  let toUint = (dest.kind in tyUInt..tyUInt32)
+  let fromUint = (src.kind in tyUInt..tyUInt32)
+  if toUint and (fromInt or fromUint):
+    let trimmer = unsignedTrimmer(dest.size)
+    r.res = "($1 $2)" % [r.res, trimmer]
+  elif dest.kind == tyBool:
     r.res = "(!!($1))" % [r.res]
     r.kind = resExpr
-  of tyInt:
+  elif toInt:
     r.res = "(($1)|0)" % [r.res]
   else:
     # TODO: What types must we handle here?
@@ -2470,7 +2479,8 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   of nkRaiseStmt: genRaiseStmt(p, n)
   of nkTypeSection, nkCommentStmt, nkIteratorDef, nkIncludeStmt,
      nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
-     nkFromStmt, nkTemplateDef, nkMacroDef, nkStaticStmt: discard
+     nkFromStmt, nkTemplateDef, nkMacroDef, nkStaticStmt,
+     nkMixinStmt, nkBindStmt: discard
   of nkPragma: genPragma(p, n)
   of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef:
     var s = n[namePos].sym

compiler/lambdalifting.nim

@@ -493,7 +493,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
           w = up
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit,
      nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef,
-     nkConverterDef, nkMacroDef, nkFuncDef, nkCommentStmt, nkTypeOfExpr:
+     nkConverterDef, nkMacroDef, nkFuncDef, nkCommentStmt,
+     nkTypeOfExpr, nkMixinStmt, nkBindStmt:
     discard
   of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil:
@@ -747,7 +748,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         result = accessViaEnvVar(n, owner, d, c)
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom,
      nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef, nkConverterDef,
-     nkMacroDef, nkFuncDef:
+     nkMacroDef, nkFuncDef, nkMixinStmt, nkBindStmt:
     discard
   of nkClosure:
     if n[1].kind == nkNilLit:

compiler/lexer.nim

@@ -62,6 +62,7 @@ type
     tkComma, tkSemiColon,
     tkColon, tkColonColon, tkEquals, tkDot, tkDotDot, tkBracketLeColon,
     tkOpr, tkComment, tkAccent,
+    # these are fake tokens used by renderer.nim
     tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
 
   TTokTypes* = set[TTokType]

compiler/liftdestructors.nim

@@ -26,6 +26,8 @@ type
     fn: PSym
     asgnForType: PType
     recurse: bool
+    filterDiscriminator: PSym  # we generating destructor for case branch
+    addMemReset: bool    # add wasMoved() call after destructor call
     c: PContext # c can be nil, then we are called from lambdalifting!
 
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode)
@@ -72,14 +74,62 @@ proc genAddr(g: ModuleGraph; x: PNode): PNode =
     result = newNodeIT(nkHiddenAddr, x.info, makeVarType(x.typ.owner, x.typ))
     result.add x
 
-proc destructorCall(g: ModuleGraph; op: PSym; x: PNode): PNode =
-  result = newNodeIT(nkCall, x.info, op.typ[0])
-  result.add(newSymNode(op))
-  result.add genAddr(g, x)
+
+proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode =
+  result = newNodeI(nkCall, i.info)
+  result.add createMagic(g, name, magic).newSymNode
+  result.add i
+
+proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
+  result = newNodeI(nkWhileStmt, c.info, 2)
+  let cmp = genBuiltin(c.g, mLtI, "<", i)
+  cmp.add genLen(c.g, dest)
+  cmp.typ = getSysType(c.g, c.info, tyBool)
+  result[0] = cmp
+  result[1] = newNodeI(nkStmtList, c.info)
+
+proc genIf(c: var TLiftCtx; cond, action: PNode): PNode =
+  result = newTree(nkIfStmt, newTree(nkElifBranch, cond, action))
+
+proc genContainerOf(c: TLiftCtx; objType: PType, field, x: PSym): PNode = 
+  # generate: cast[ptr ObjType](cast[int](addr(x)) - offsetOf(objType.field))
+  let intType = getSysType(c.g, unknownLineInfo, tyInt)
+
+  let addrOf = newNodeIT(nkAddr, c.info, makePtrType(x.owner, x.typ))
+  addrOf.add newDeref(newSymNode(x))
+  let castExpr1 = newNodeIT(nkCast, c.info, intType)
+  castExpr1.add newNodeIT(nkType, c.info, intType)
+  castExpr1.add addrOf
+
+  let dotExpr = newNodeIT(nkDotExpr, c.info, x.typ)
+  dotExpr.add newNodeIT(nkType, c.info, objType)
+  dotExpr.add newSymNode(field)
+  
+  let offsetOf = genBuiltin(c.g, mOffsetOf, "offsetof", dotExpr)
+  offsetOf.typ = intType
+
+  let minusExpr = genBuiltin(c.g, mSubI, "-", castExpr1)
+  minusExpr.typ = intType
+  minusExpr.add offsetOf
+
+  let objPtr = makePtrType(objType.owner, objType)
+  result = newNodeIT(nkCast, c.info, objPtr)
+  result.add newNodeIT(nkType, c.info, objPtr)
+  result.add minusExpr
+
+proc destructorCall(c: TLiftCtx; op: PSym; x: PNode): PNode =
+  var destroy = newNodeIT(nkCall, x.info, op.typ[0])
+  destroy.add(newSymNode(op))
+  destroy.add genAddr(c.g, x)
+  if c.addMemReset:
+    result = newTree(nkStmtList, destroy, genBuiltin(c.g, mWasMoved,  "wasMoved", x))
+  else:
+    result = destroy
 
 proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool) =
   case n.kind
   of nkSym:
+    if c.filterDiscriminator != nil: return
     let f = n.sym
     let b = if c.kind == attachedTrace: y else: y.dotField(f)
     if (sfCursor in f.flags and f.typ.skipTypes(abstractInst).kind in {tyRef, tyProc} and
@@ -90,6 +140,9 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       fillBody(c, f.typ, body, x.dotField(f), b)
   of nkNilLit: discard
   of nkRecCase:
+    let oldfilterDiscriminator = c.filterDiscriminator
+    if c.filterDiscriminator == n[0].sym:
+      c.filterDiscriminator = nil # we have found the case part, proceed as normal
     # XXX This is only correct for 'attachedSink'!
     var localEnforceDefaultOp = enforceDefaultOp
     if c.kind == attachedSink:
@@ -121,6 +174,7 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       caseStmt.add(branch)
     if emptyBranches != n.len-1:
       body.add(caseStmt)
+    c.filterDiscriminator = oldfilterDiscriminator 
   of nkRecList:
     for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp)
   else:
@@ -275,7 +329,7 @@ proc addDestructorCall(c: var TLiftCtx; orig: PType; body, x: PNode) =
   if op != nil:
     #markUsed(c.g.config, c.info, op, c.g.usageSym)
     onUse(c.info, op)
-    body.add destructorCall(c.g, op, x)
+    body.add destructorCall(c, op, x)
   elif useNoGc(c, t):
     internalError(c.g.config, c.info,
       "type-bound operator could not be resolved")
@@ -293,7 +347,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
 
       #markUsed(c.g.config, c.info, op, c.g.usageSym)
       onUse(c.info, op)
-      body.add destructorCall(c.g, op, x)
+      body.add destructorCall(c, op, x)
       result = true
     #result = addDestructorCall(c, t, body, x)
   of attachedAsgn, attachedSink, attachedTrace:
@@ -318,22 +372,6 @@ proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
   v.addVar(result, lowerings.newIntLit(c.g, body.info, first))
   body.add v
 
-proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode =
-  result = newNodeI(nkCall, i.info)
-  result.add createMagic(g, name, magic).newSymNode
-  result.add i
-
-proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
-  result = newNodeI(nkWhileStmt, c.info, 2)
-  let cmp = genBuiltin(c.g, mLtI, "<", i)
-  cmp.add genLen(c.g, dest)
-  cmp.typ = getSysType(c.g, c.info, tyBool)
-  result[0] = cmp
-  result[1] = newNodeI(nkStmtList, c.info)
-
-proc genIf(c: var TLiftCtx; cond, action: PNode): PNode =
-  result = newTree(nkIfStmt, newTree(nkElifBranch, cond, action))
-
 proc addIncStmt(c: var TLiftCtx; body, i: PNode) =
   let incCall = genBuiltin(c.g, mInc, "inc", i)
   incCall.add lowerings.newIntLit(c.g, c.info, 1)
@@ -382,7 +420,7 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
+    moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
   of attachedDestructor:
     # destroy all elements:
@@ -413,7 +451,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
+    moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
     # alternatively we could do this:
     when false:
@@ -421,7 +459,7 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.add newHookCall(c.g, t.asink, x, y)
   of attachedDestructor:
     doAssert t.destructor != nil
-    body.add destructorCall(c.g, t.destructor, x)
+    body.add destructorCall(c, t.destructor, x)
   of attachedTrace:
     body.add newHookCall(c.g, t.attachedOps[c.kind], x, y)
   of attachedDispose:
@@ -435,7 +473,7 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
+    moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
   of attachedDestructor, attachedDispose:
     body.add genBuiltin(c.g, mDestroy, "destroy", x)
@@ -477,8 +515,8 @@ proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, cond, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
     body.add genIf(c, cond, actions)
+    body.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace:
     if isFinal(elemType):
@@ -526,8 +564,8 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, cond, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(xenv, newNodeIT(nkNilLit, body.info, xenv.typ))
     body.add genIf(c, cond, actions)
+    body.add newAsgnStmt(xenv, newNodeIT(nkNilLit, body.info, xenv.typ))
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace:
     body.add callCodegenProc(c.g, "nimTraceRefDyn", c.info, genAddrOf(xenv), y)
@@ -584,8 +622,8 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, x, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
     body.add genIf(c, x, actions)
+    body.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace, attachedDispose: discard
 
@@ -637,8 +675,8 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, xx, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(xx, newNodeIT(nkNilLit, body.info, xx.typ))
     body.add genIf(c, xx, actions)
+    body.add newAsgnStmt(xx, newNodeIT(nkNilLit, body.info, xx.typ))
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace, attachedDispose: discard
 
@@ -738,33 +776,14 @@ proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType;
   typ.attachedOps[kind] = baseType.attachedOps[kind]
   result = typ.attachedOps[kind]
 
-proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
+proc symPrototype(g: ModuleGraph; typ: PType; kind: TTypeAttachedOp;
               info: TLineInfo): PSym =
-  if typ.kind == tyDistinct:
-    return produceSymDistinctType(g, c, typ, kind, info)
 
-  var a: TLiftCtx
-  a.info = info
-  a.g = g
-  a.kind = kind
-  a.c = c
-  let body = newNodeI(nkStmtList, info)
   let procname = getIdent(g.cache, AttachedOpToStr[kind])
-
   result = newSym(skProc, procname, typ.owner, info)
-  a.fn = result
-  a.asgnForType = typ
-
   let dest = newSym(skParam, getIdent(g.cache, "dest"), result, info)
   let src = newSym(skParam, getIdent(g.cache, if kind == attachedTrace: "env" else: "src"), result, info)
-  var d: PNode
-  #if kind notin {attachedTrace, attachedDispose}:
   dest.typ = makeVarType(typ.owner, typ)
-  d = newDeref(newSymNode(dest))
-  #else:
-  #  dest.typ = typ
-  #  d = newSymNode(dest)
-
   if kind == attachedTrace:
     src.typ = getSysType(g, info, tyPointer)
   else:
@@ -775,6 +794,30 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
   if kind notin {attachedDestructor, attachedDispose}:
     result.typ.addParam src
 
+  var n = newNodeI(nkProcDef, info, bodyPos+1)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = newNodeI(nkStmtList, info)
+  result.ast = n
+  incl result.flags, sfFromGeneric
+  incl result.flags, sfGeneratedOp
+
+
+proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
+              info: TLineInfo): PSym =
+  if typ.kind == tyDistinct:
+    return produceSymDistinctType(g, c, typ, kind, info)
+
+  result = symPrototype(g, typ, kind, info)
+  var a = TLiftCtx(info: info, g: g, kind: kind, c: c, asgnForType:typ)
+  a.fn = result
+
+  let dest = result.typ.n[1].sym
+  let d = newDeref(newSymNode(dest))
+  let src = if kind in {attachedDestructor, attachedDispose}: newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
+            else: newSymNode(result.typ.n[2].sym)
+
   # register this operation already:
   typ.attachedOps[kind] = result
 
@@ -782,8 +825,8 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
        sfOverriden in typ.attachedOps[attachedDestructor].flags:
     ## compiler can use a combination of `=destroy` and memCopy for sink op
     dest.flags.incl sfCursor
-    body.add newOpCall(typ.attachedOps[attachedDestructor], d[0])
-    body.add newAsgnStmt(d, newSymNode(src))
+    result.ast[bodyPos].add newOpCall(typ.attachedOps[attachedDestructor], d[0])
+    result.ast[bodyPos].add newAsgnStmt(d, src)
   else:
     var tk: TTypeKind
     if g.config.selectedGC in {gcArc, gcOrc, gcHooks}:
@@ -792,20 +835,33 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
       tk = tyNone # no special casing for strings and seqs
     case tk
     of tySequence:
-      fillSeqOp(a, typ, body, d, newSymNode(src))
+      fillSeqOp(a, typ, result.ast[bodyPos], d, src)
     of tyString:
-      fillStrOp(a, typ, body, d, newSymNode(src))
+      fillStrOp(a, typ, result.ast[bodyPos], d, src)
     else:
-      fillBody(a, typ, body, d, newSymNode(src))
+      fillBody(a, typ, result.ast[bodyPos], d, src)
+
+
+proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym, info: TLineInfo): PSym =
+  assert(typ.kind == tyObject)
+  result = symPrototype(g, field.typ, attachedDestructor, info)
+  var a = TLiftCtx(info: info, g: g, kind: attachedDestructor, asgnForType: typ)
+  a.fn = result
+  a.asgnForType = typ
+  a.filterDiscriminator = field
+  a.addMemReset = true
+  let discrimantDest = result.typ.n[1].sym
+
+  let dst = newSym(skVar, getIdent(g.cache, "dest"), result, info)
+  dst.typ = makePtrType(typ.owner, typ)
+  let dstSym = newSymNode(dst)
+  let d = newDeref(dstSym)
+  let v = newNodeI(nkVarSection, info)
+  v.addVar(dstSym, genContainerOf(a, typ, field, discrimantDest))
+  result.ast[bodyPos].add v
+  let placeHolder = newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
+  fillBody(a, typ, result.ast[bodyPos], d, placeHolder)
 
-  var n = newNodeI(nkProcDef, info, bodyPos+1)
-  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
-  n[namePos] = newSymNode(result)
-  n[paramsPos] = result.typ.n
-  n[bodyPos] = body
-  result.ast = n
-  incl result.flags, sfFromGeneric
-  incl result.flags, sfGeneratedOp
 
 template liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
   discard "now a nop"

compiler/liftlocals.nim

@@ -42,7 +42,7 @@ proc liftLocals(n: PNode; i: int; c: var Ctx) =
   of nkSym:
     if interestingVar(it.sym):
       n[i] = lookupOrAdd(c, it.sym, it.info)
-  of procDefs, nkTypeSection: discard
+  of procDefs, nkTypeSection, nkMixinStmt, nkBindStmt: discard
   else:
     for i in 0..<it.safeLen:
       liftLocals(it, i, c)

compiler/lineinfos.nim

@@ -42,7 +42,10 @@ type
     warnProveInit, warnProveField, warnProveIndex,
     warnStaticIndexCheck, warnGcUnsafe, warnGcUnsafe2,
     warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed,
-    warnInconsistentSpacing, warnCaseTransition, warnCycleCreated, warnUser,
+    warnInconsistentSpacing, warnCaseTransition, warnCycleCreated,
+    warnObservableStores,
+    warnResultUsed,
+    warnUser,
     hintSuccess, hintSuccessX, hintCC,
     hintLineTooLong, hintXDeclaredButNotUsed,
     hintConvToBaseNotNeeded,
@@ -87,7 +90,8 @@ const
     warnWriteToForeignHeap: "write to foreign heap",
     warnUnsafeCode: "unsafe code: '$1'",
     warnUnusedImportX: "imported and not used: '$1'",
-    warnInheritFromException: "inherit from a more precise exception type like ValueError, IOError or OSError",
+    warnInheritFromException: "inherit from a more precise exception type like ValueError, " &
+      "IOError or OSError. If these don't suit, inherit from CatchableError or Defect.",
     warnEachIdentIsTuple: "each identifier is a tuple",
     warnUnsafeSetLen: "setLen can potentially expand the sequence, " &
                       "but the element type '$1' doesn't have a valid default value",
@@ -106,6 +110,8 @@ const
     warnInconsistentSpacing: "Number of spaces around '$#' is not consistent",
     warnCaseTransition: "Potential object case transition, instantiate new object instead",
     warnCycleCreated: "$1",
+    warnObservableStores: "observable stores to '$1'",
+    warnResultUsed: "used 'result' variable",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
@@ -155,7 +161,8 @@ const
     "ProveInit", "ProveField", "ProveIndex",
     "IndexCheck", "GcUnsafe", "GcUnsafe2", "Uninit",
     "GcMem", "Destructor", "LockLevel", "ResultShadowed",
-    "Spacing", "CaseTransition", "CycleCreated", "User"]
+    "Spacing", "CaseTransition", "CycleCreated",
+    "ObservableStores", "ResultUsed", "User"]
 
   HintsToStr* = [
     "Success", "SuccessX", "CC", "LineTooLong",
@@ -186,7 +193,7 @@ type
   TNoteKinds* = set[TNoteKind]
 
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
-  result[3] = {low(TNoteKind)..high(TNoteKind)} - {}
+  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnResultUsed}
   result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext}
   result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,

compiler/linter.nim

@@ -101,7 +101,7 @@ proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
     lintReport(conf, info, beau, s.name.s)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
-  if {optStyleHint, optStyleError} * conf.globalOptions != {}:
+  if {optStyleHint, optStyleError} * conf.globalOptions != {} and optStyleUsages notin conf.globalOptions:
     nep1CheckDefImpl(conf, info, s, k)
 
 template styleCheckDef*(conf: ConfigRef; info: TLineInfo; s: PSym) =

compiler/magicsys.nim

@@ -177,3 +177,28 @@ proc getNimScriptSymbol*(g: ModuleGraph; name: string): PSym =
   strTableGet(g.exposed, getIdent(g.cache, name))
 
 proc resetNimScriptSymbols*(g: ModuleGraph) = initStrTable(g.exposed)
+
+proc getMagicEqSymForType*(g: ModuleGraph; t: PType; info: TLineInfo): PSym =
+  case t.kind
+  of tyInt,  tyInt8, tyInt16, tyInt32, tyInt64,
+     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64: 
+    result = getSysMagic(g, info, "==", mEqI)
+  of tyEnum: 
+    result = getSysMagic(g, info, "==", mEqEnum)
+  of tyBool: 
+    result = getSysMagic(g, info, "==", mEqB)
+  of tyRef, tyPtr, tyPointer: 
+    result = getSysMagic(g, info, "==", mEqRef)
+  of tyString:
+    result = getSysMagic(g, info, "==", mEqStr)
+  of tyChar:
+    result = getSysMagic(g, info, "==", mEqCh)
+  of tySet:
+    result = getSysMagic(g, info, "==", mEqSet)
+  of tyProc:
+    result = getSysMagic(g, info, "==", mEqProc)
+  else:
+    globalError(g.config, info,
+      "can't find magic equals operator for type kind " & $t.kind)
+
+

compiler/main.nim

@@ -13,7 +13,7 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  llstream, strutils, ast, lexer, syntaxes, options, msgs,
+  llstream, strutils, os, ast, lexer, syntaxes, options, msgs,
   condsyms, times,
   sem, idents, passes, extccomp,
   cgen, json, nversion,
@@ -89,6 +89,9 @@ proc commandCompileToC(graph: ModuleGraph) =
       graph.config.notes = graph.config.mainPackageNotes
       return
 
+  if not extccomp.ccHasSaneOverflow(conf):
+    conf.symbols.defineSymbol("nimEmulateOverflowChecks")
+
   compileProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933
@@ -315,6 +318,7 @@ proc mainCommand*(graph: ModuleGraph) =
 
       var dumpdata = %[
         (key: "version", val: %VersionAsString),
+        (key: "nimExe", val: %(getAppFilename())),
         (key: "prefixdir", val: %conf.getPrefixDir().string),
         (key: "libpath", val: %conf.libpath.string),
         (key: "project_path", val: %conf.projectFull.string),

compiler/msgs.nim

@@ -379,7 +379,7 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
     if context.info != lastinfo and context.info != info:
       if conf.structuredErrorHook != nil:
         conf.structuredErrorHook(conf, context.info, instantiationFrom,
-                                 Severity.Error)
+                                 Severity.Hint)
       else:
         let message = if context.detail == "":
           instantiationFrom

compiler/nim.cfg

@@ -19,5 +19,6 @@ define:useStdoutAsStdmsg
   styleCheck:error
 @end
 
+
 #define:useNodeIds
 #gc:markAndSweep

compiler/options.nim

@@ -59,6 +59,7 @@ type                          # please make sure we have under 32 options
     optRun,                   # run the compiled project
     optStyleHint,             # check that the names adhere to NEP-1
     optStyleError,            # enforce that the names adhere to NEP-1
+    optStyleUsages,           # only enforce consistent **usages** of the symbol
     optSkipSystemConfigFile,  # skip the system's cfg/nims config file
     optSkipProjConfigFile,    # skip the project's cfg/nims config file
     optSkipUserConfigFile,    # skip the users's cfg/nims config file
@@ -180,6 +181,7 @@ type
     nimname*: string
     cname*, obj*: AbsoluteFile
     flags*: set[CfileFlag]
+    customArgs*: string
   CfileList* = seq[Cfile]
 
   Suggest* = ref object
@@ -309,7 +311,7 @@ proc setNote*(conf: ConfigRef, note: TNoteKind, enabled = true) =
 proc hasHint*(conf: ConfigRef, note: TNoteKind): bool =
   optHints in conf.options and note in conf.notes
 
-proc hasWarn*(conf: ConfigRef, note: TNoteKind): bool =
+proc hasWarn*(conf: ConfigRef, note: TNoteKind): bool {.inline.} =
   optWarns in conf.options and note in conf.notes
 
 proc hcrOn*(conf: ConfigRef): bool = return optHotCodeReloading in conf.globalOptions

compiler/parampatterns.nim

@@ -176,6 +176,7 @@ type
     arLocalLValue,            # is an l-value, but local var; must not escape
                               # its stack frame!
     arDiscriminant,           # is a discriminant
+    arLentValue,              # lent value
     arStrange                 # it is a strange beast like 'typedesc[var T]'
 
 proc exprRoot*(n: PNode): PSym =
@@ -260,9 +261,14 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
       # types that are equal modulo distinction preserve l-value:
       result = isAssignable(owner, n[1], isUnsafeAddr)
   of nkHiddenDeref:
-    if isUnsafeAddr and n[0].typ.kind == tyLent: result = arLValue
-    elif n[0].typ.kind == tyLent: result = arDiscriminant
-    else: result = arLValue
+    let n0 = n[0]
+    if n0.typ.kind == tyLent:
+      if isUnsafeAddr or (n0.kind == nkSym and n0.sym.kind == skResult):
+        result = arLValue
+      else:
+        result = arLentValue
+    else:
+      result = arLValue
   of nkDerefExpr, nkHiddenAddr:
     result = arLValue
   of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:

compiler/platform.nim

@@ -22,7 +22,7 @@ type
     osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris,
     osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osAix, osPalmos, osQnx,
     osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks
-    osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osAny
+    osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osAny
 
 type
   TInfoOSProp* = enum
@@ -177,6 +177,10 @@ const
       objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
       scriptExt: ".sh", curDir: ".", exeExt: ".elf", extSep: ".",
       props: {ospNeedsPIC, ospPosix}),
+     (name: "FreeRTOS", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
+      objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
+      scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
+      props: {ospPosix}),
      (name: "Any", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
       objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
       scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
@@ -189,7 +193,7 @@ type
     cpuNone, cpuI386, cpuM68k, cpuAlpha, cpuPowerpc, cpuPowerpc64,
     cpuPowerpc64el, cpuSparc, cpuVm, cpuHppa, cpuIa64, cpuAmd64, cpuMips,
     cpuMipsel, cpuArm, cpuArm64, cpuJS, cpuNimVM, cpuAVR, cpuMSP430,
-    cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV64, cpuWasm32
+    cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV32, cpuRiscV64, cpuEsp, cpuWasm32
 
 type
   TEndian* = enum
@@ -222,7 +226,9 @@ const
     (name: "sparc64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64),
     (name: "mips64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64),
     (name: "mips64el", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
+    (name: "riscv32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
     (name: "riscv64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
+    (name: "esp", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
     (name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32)]
 
 type

compiler/pragmas.nim

@@ -20,7 +20,7 @@ const
 
 const
   declPragmas = {wImportc, wImportObjC, wImportCpp, wImportJs, wExportc, wExportCpp,
-    wExportNims, wExtern, wDeprecated, wNodecl, wError, wUsed, wAlign}
+    wExportNims, wExtern, wDeprecated, wNodecl, wError, wUsed}
     ## common pragmas for declarations, to a good approximation
   procPragmas* = declPragmas + {FirstCallConv..LastCallConv,
     wMagic, wNoSideEffect, wSideEffect, wNoreturn, wNosinks, wDynlib, wHeader,
@@ -54,22 +54,22 @@ const
     wFloatChecks, wInfChecks, wNanChecks, wPragma, wEmit, wUnroll,
     wLinearScanEnd, wPatterns, wTrMacros, wEffects, wNoForward, wReorder, wComputedGoto,
     wInjectStmt, wExperimental, wThis, wUsed, wInvariant, wAssume}
-  lambdaPragmas* = declPragmas + {FirstCallConv..LastCallConv,
+  lambdaPragmas* = {FirstCallConv..LastCallConv,
     wNoSideEffect, wSideEffect, wNoreturn, wNosinks, wDynlib, wHeader,
     wThread, wAsmNoStackFrame,
     wRaises, wLocks, wTags, wRequires, wEnsures,
-    wGcSafe, wCodegenDecl} - {wExportNims, wError, wUsed}  # why exclude these?
+    wGcSafe, wCodegenDecl, wNoInit}
   typePragmas* = declPragmas + {wMagic, wAcyclic,
     wPure, wHeader, wCompilerProc, wCore, wFinal, wSize, wShallow,
     wIncompleteStruct, wByCopy, wByRef,
     wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked,
     wBorrow, wGcSafe, wPartial, wExplain, wPackage}
-  fieldPragmas* = declPragmas + {
-    wGuard, wBitsize, wCursor, wRequiresInit} - {wExportNims, wNodecl} # why exclude these?
+  fieldPragmas* = declPragmas + {wGuard, wBitsize, wCursor,
+    wRequiresInit, wAlign} - {wExportNims, wNodecl} # why exclude these?
   varPragmas* = declPragmas + {wVolatile, wRegister, wThreadVar,
     wMagic, wHeader, wCompilerProc, wCore, wDynlib,
     wNoInit, wCompileTime, wGlobal,
-    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wCursor}
+    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wCursor, wAlign}
   constPragmas* = declPragmas + {wHeader, wMagic,
     wGensym, wInject,
     wIntDefine, wStrDefine, wBoolDefine, wCompilerProc, wCore}
@@ -92,11 +92,10 @@ proc getPragmaVal*(procAst: PNode; name: TSpecialWord): PNode =
 proc pragma*(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords;
             isStatement: bool = false)
 
-proc recordPragma(c: PContext; n: PNode; key, val: string; val2 = "") =
+proc recordPragma(c: PContext; n: PNode; args: varargs[string]) =
   var recorded = newNodeI(nkCommentStmt, n.info)
-  recorded.add newStrNode(key, n.info)
-  recorded.add newStrNode(val, n.info)
-  if val2.len > 0: recorded.add newStrNode(val2, n.info)
+  for i in 0..args.high:
+    recorded.add newStrNode(args[i], n.info)
   c.graph.recordStmt(c.graph, c.module, recorded)
 
 const
@@ -496,11 +495,12 @@ proc relativeFile(c: PContext; n: PNode; ext=""): AbsoluteFile =
       if result.isEmpty: result = AbsoluteFile s
 
 proc processCompile(c: PContext, n: PNode) =
-  proc docompile(c: PContext; it: PNode; src, dest: AbsoluteFile) =
+  proc docompile(c: PContext; it: PNode; src, dest: AbsoluteFile; customArgs: string) =
     var cf = Cfile(nimname: splitFile(src).name,
-                   cname: src, obj: dest, flags: {CfileFlag.External})
+                   cname: src, obj: dest, flags: {CfileFlag.External},
+                   customArgs: customArgs)
     extccomp.addExternalFileToCompile(c.config, cf)
-    recordPragma(c, it, "compile", src.string, dest.string)
+    recordPragma(c, it, "compile", src.string, dest.string, customArgs)
 
   proc getStrLit(c: PContext, n: PNode; i: int): string =
     n[i] = c.semConstExpr(c, n[i])
@@ -518,9 +518,19 @@ proc processCompile(c: PContext, n: PNode) =
     var found = parentDir(toFullPath(c.config, n.info)) / s
     for f in os.walkFiles(found):
       let obj = completeCfilePath(c.config, AbsoluteFile(dest % extractFilename(f)))
-      docompile(c, it, AbsoluteFile f, obj)
+      docompile(c, it, AbsoluteFile f, obj, "")
   else:
-    let s = expectStrLit(c, n)
+    var s = ""
+    var customArgs = ""
+    if n.kind in nkCallKinds:
+      s = getStrLit(c, n, 1)
+      if n.len <= 3:
+        customArgs = getStrLit(c, n, 2)
+      else:
+        localError(c.config, n.info, "'.compile' pragma takes up 2 arguments")
+    else:
+      s = expectStrLit(c, n)
+
     var found = AbsoluteFile(parentDir(toFullPath(c.config, n.info)) / s)
     if not fileExists(found):
       if isAbsolute(s): found = AbsoluteFile s
@@ -528,7 +538,7 @@ proc processCompile(c: PContext, n: PNode) =
         found = findFile(c.config, s)
         if found.isEmpty: found = AbsoluteFile s
     let obj = toObjFile(c.config, completeCfilePath(c.config, found, false))
-    docompile(c, it, found, obj)
+    docompile(c, it, found, obj, customArgs)
 
 proc processLink(c: PContext, n: PNode) =
   let found = relativeFile(c, n, CC[c.config.cCompiler].objExt)
@@ -910,7 +920,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wNoreturn:
         noVal(c, it)
         # Disable the 'noreturn' annotation when in the "Quirky Exceptions" mode!
-        if c.config.exc notin {excQuirky, excGoto}:
+        if c.config.exc != excQuirky:
           incl(sym.flags, sfNoReturn)
         if sym.typ[0] != nil:
           localError(c.config, sym.ast[paramsPos][0].info,

compiler/renderer.nim

@@ -31,6 +31,7 @@ type
   TSrcGen* = object
     indent*: int
     lineLen*: int
+    col: int
     pos*: int              # current position for iteration over the buffer
     idx*: int              # current token index for iteration over the buffer
     tokens*: TRenderTokSeq
@@ -109,11 +110,10 @@ proc initSrcGen(g: var TSrcGen, renderFlags: TRenderFlags; config: ConfigRef) =
   g.config = config
 
 proc addTok(g: var TSrcGen, kind: TTokType, s: string; sym: PSym = nil) =
-  setLen(g.tokens, g.tokens.len + 1)
-  g.tokens[^1].kind = kind
-  g.tokens[^1].length = int16(s.len)
-  g.tokens[^1].sym = sym
+  g.tokens.add TRenderTok(kind: kind, length: int16(s.len), sym: sym)
   g.buf.add(s)
+  if kind != tkSpaces:
+    inc g.col, s.len
 
 proc addPendingNL(g: var TSrcGen) =
   if g.pendingNL >= 0:
@@ -123,6 +123,7 @@ proc addPendingNL(g: var TSrcGen) =
       const newlines = "\n"
     addTok(g, tkSpaces, newlines & spaces(g.pendingNL))
     g.lineLen = g.pendingNL
+    g.col = g.pendingNL
     g.pendingNL = - 1
     g.pendingWhitespace = -1
   elif g.pendingWhitespace >= 0:
@@ -131,7 +132,10 @@ proc addPendingNL(g: var TSrcGen) =
 
 proc putNL(g: var TSrcGen, indent: int) =
   if g.pendingNL >= 0: addPendingNL(g)
-  else: addTok(g, tkSpaces, "\n")
+  else:
+    addTok(g, tkSpaces, "\n")
+    g.col = 0
+
   g.pendingNL = indent
   g.lineLen = indent
   g.pendingWhitespace = -1
@@ -173,16 +177,17 @@ proc put(g: var TSrcGen, kind: TTokType, s: string; sym: PSym = nil) =
     addPendingNL(g)
     if s.len > 0:
       addTok(g, kind, s, sym)
-      inc(g.lineLen, s.len)
   else:
     g.pendingWhitespace = s.len
+    inc g.col, s.len
+  inc(g.lineLen, s.len)
 
 proc putComment(g: var TSrcGen, s: string) =
   if s.len == 0: return
   var i = 0
   let hi = s.len - 1
-  var isCode = (s.len >= 2) and (s[1] != ' ')
-  var ind = g.lineLen
+  let isCode = (s.len >= 2) and (s[1] != ' ')
+  let ind = g.col
   var com = "## "
   while i <= hi:
     case s[i]
@@ -208,7 +213,7 @@ proc putComment(g: var TSrcGen, s: string) =
       # compute length of the following word:
       var j = i
       while j <= hi and s[j] > ' ': inc(j)
-      if not isCode and (g.lineLen + (j - i) > MaxLineLen):
+      if not isCode and (g.col + (j - i) > MaxLineLen):
         put(g, tkComment, com)
         optNL(g, ind)
         com = "## "
@@ -290,15 +295,18 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool =
 proc gcom(g: var TSrcGen, n: PNode) =
   assert(n != nil)
   if shouldRenderComment(g, n):
+    var oneSpaceAdded = 0
     if (g.pendingNL < 0) and (g.buf.len > 0) and (g.buf[^1] != ' '):
       put(g, tkSpaces, Space)
+      oneSpaceAdded = 1
       # Before long comments we cannot make sure that a newline is generated,
       # because this might be wrong. But it is no problem in practice.
     if (g.pendingNL < 0) and (g.buf.len > 0) and
-        (g.lineLen < LineCommentColumn):
+        (g.col < LineCommentColumn):
       var ml = maxLineLength(n.comment)
       if ml + LineCommentColumn <= MaxLineLen:
-        put(g, tkSpaces, spaces(LineCommentColumn - g.lineLen))
+        put(g, tkSpaces, spaces(LineCommentColumn - g.col))
+        dec g.col, oneSpaceAdded
     putComment(g, n.comment)  #assert(g.comStack[high(g.comStack)] = n);
 
 proc gcoms(g: var TSrcGen) =

compiler/reorder.nim

@@ -106,6 +106,7 @@ proc computeDeps(cache: IdentCache; n: PNode, declares, uses: var IntSet; topLev
         decl(a[1])
     else:
       for i in 0..<n.safeLen: deps(n[i])
+  of nkMixinStmt, nkBindStmt: discard
   else:
     for i in 0..<n.safeLen: deps(n[i])
 

compiler/rodimpl.nim

@@ -838,11 +838,12 @@ proc replay(g: ModuleGraph; module: PSym; n: PNode) =
       of "warning": message(g.config, n.info, warnUser, n[1].strVal)
       of "error": localError(g.config, n.info, errUser, n[1].strVal)
       of "compile":
-        internalAssert g.config, n.len == 3 and n[2].kind == nkStrLit
+        internalAssert g.config, n.len == 4 and n[2].kind == nkStrLit
         let cname = AbsoluteFile n[1].strVal
         var cf = Cfile(nimname: splitFile(cname).name, cname: cname,
                        obj: AbsoluteFile n[2].strVal,
-                       flags: {CfileFlag.External})
+                       flags: {CfileFlag.External},
+                       customArgs: n[3].strVal)
         extccomp.addExternalFileToCompile(g.config, cf)
       of "link":
         extccomp.addExternalFileToLink(g.config, AbsoluteFile n[1].strVal)

compiler/semcall.nim

@@ -219,7 +219,11 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
       candidates.add("  first type mismatch at position: " & $err.firstMismatch.arg)
       # candidates.add "\n  reason: " & $err.firstMismatch.kind # for debugging
       case err.firstMismatch.kind
-      of kUnknownNamedParam: candidates.add("\n  unknown named parameter: " & $nArg[0])
+      of kUnknownNamedParam:
+        if nArg == nil:
+          candidates.add("\n  unknown named parameter")
+        else:
+          candidates.add("\n  unknown named parameter: " & $nArg[0])
       of kAlreadyGiven: candidates.add("\n  named param already provided: " & $nArg[0])
       of kPositionalAlreadyGiven: candidates.add("\n  positional param was already given as named param")
       of kExtraArg: candidates.add("\n  extra argument given")
@@ -318,7 +322,7 @@ proc getMsgDiagnostic(c: PContext, flags: TExprFlags, n, f: PNode): string =
       sym = nextOverloadIter(o, c, f)
 
   let ident = considerQuotedIdent(c, f, n).s
-  if nfDotField in n.flags and nfExplicitCall notin n.flags:
+  if {nfDotField, nfExplicitCall} * n.flags == {nfDotField}:
     let sym = n[1].typ.sym
     var typeHint = ""
     if sym == nil:

compiler/semdata.nim

@@ -38,6 +38,7 @@ type
     mappingExists*: bool
     mapping*: TIdTable
     caseContext*: seq[tuple[n: PNode, idx: int]]
+    localBindStmts*: seq[PNode]
 
   TMatchedConcept* = object
     candidateType*: PType
@@ -274,10 +275,13 @@ proc addToLib*(lib: PLib, sym: PSym) =
 proc newTypeS*(kind: TTypeKind, c: PContext): PType =
   result = newType(kind, getCurrOwner(c))
 
-proc makePtrType*(c: PContext, baseType: PType): PType =
-  result = newTypeS(tyPtr, c)
+proc makePtrType*(owner: PSym, baseType: PType): PType =
+  result = newType(tyPtr, owner)
   addSonSkipIntLit(result, baseType)
 
+proc makePtrType*(c: PContext, baseType: PType): PType =
+  makePtrType(getCurrOwner(c), baseType)
+
 proc makeTypeWithModifier*(c: PContext,
                            modifier: TTypeKind,
                            baseType: PType): PType =

compiler/semexprs.nim

@@ -1207,6 +1207,9 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     # not sure the symbol really ends up being used:
     # var len = 0 # but won't be called
     # genericThatUsesLen(x) # marked as taking a closure?
+    if hasWarn(c.config, warnResultUsed):
+      message(c.config, n.info, warnResultUsed)
+
   of skGenericParam:
     onUse(n.info, s)
     if s.typ.kind == tyStatic:
@@ -1703,7 +1706,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
   if le == nil:
     localError(c.config, a.info, "expression has no type")
   elif (skipTypes(le, {tyGenericInst, tyAlias, tySink}).kind != tyVar and
-        isAssignable(c, a) == arNone) or
+        isAssignable(c, a) in {arNone, arLentValue}) or
       skipTypes(le, abstractVar).kind in {tyOpenArray, tyVarargs}:
     # Direct assignment to a discriminant is allowed!
     localError(c.config, a.info, errXCannotBeAssignedTo %
@@ -2415,7 +2418,7 @@ proc checkPar(c: PContext; n: PNode): TParKind =
     for i in 0..<n.len:
       if result == paTupleFields:
         if (n[i].kind != nkExprColonExpr) or
-            n[i][0].kind notin {nkSym, nkIdent}:
+            n[i][0].kind notin {nkSym, nkIdent, nkAccQuoted}:
           localError(c.config, n[i].info, errNamedExprExpected)
           return paNone
       else:
@@ -2830,6 +2833,13 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     for i in 0..<n.len:
       n[i] = semExpr(c, n[i])
   of nkComesFrom: discard "ignore the comes from information for now"
+  of nkMixinStmt: discard
+  of nkBindStmt:
+    if c.p != nil:
+      c.p.localBindStmts.add n
+    else:
+      localError(c.config, n.info, "invalid context for 'bind' statement: " &
+                renderTree(n, {renderNoComments}))
   else:
     localError(c.config, n.info, "invalid expression: " &
                renderTree(n, {renderNoComments}))

compiler/semfields.nim

@@ -23,7 +23,7 @@ proc instFieldLoopBody(c: TFieldInstCtx, n: PNode, forLoop: PNode): PNode =
     result = newNode(nkEmpty)
     return
   case n.kind
-  of nkEmpty..pred(nkIdent), succ(nkSym)..nkNilLit: result = n
+  of nkEmpty..pred(nkIdent), succ(nkSym)..nkNilLit: result = copyNode(n)
   of nkIdent, nkSym:
     result = n
     let ident = considerQuotedIdent(c.c, n)
@@ -52,8 +52,7 @@ proc instFieldLoopBody(c: TFieldInstCtx, n: PNode, forLoop: PNode): PNode =
     if n.kind == nkContinueStmt:
       localError(c.c.config, n.info,
                  "'continue' not supported in a 'fields' loop")
-    result = copyNode(n)
-    newSons(result, n.len)
+    result = shallowCopy(n)
     for i in 0..<n.len:
       result[i] = instFieldLoopBody(c, n[i], forLoop)
 

compiler/semfold.nim

@@ -164,7 +164,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode =
   of mCard: result = newIntNodeT(toInt128(nimsets.cardSet(g.config, a)), n, g)
   of mBitnotI:
     if n.typ.isUnsigned:
-      result = newIntNodeT(bitnot(getInt(a)).maskBytes(int(n.typ.size)), n, g)
+      result = newIntNodeT(bitnot(getInt(a)).maskBytes(int(getSize(g.config, n.typ))), n, g)
     else:
       result = newIntNodeT(bitnot(getInt(a)), n, g)
   of mLengthArray: result = newIntNodeT(lengthOrd(g.config, a.typ), n, g)
@@ -276,23 +276,23 @@ proc evalOp(m: TMagic, n, a, b, c: PNode; g: ModuleGraph): PNode =
   of mBitorI, mOr: result = newIntNodeT(bitor(getInt(a), getInt(b)), n, g)
   of mBitxorI, mXor: result = newIntNodeT(bitxor(getInt(a), getInt(b)), n, g)
   of mAddU:
-    let val = maskBytes(getInt(a) + getInt(b), int(n.typ.size))
+    let val = maskBytes(getInt(a) + getInt(b), int(getSize(g.config, n.typ)))
     result = newIntNodeT(val, n, g)
   of mSubU:
-    let val = maskBytes(getInt(a) - getInt(b), int(n.typ.size))
+    let val = maskBytes(getInt(a) - getInt(b), int(getSize(g.config, n.typ)))
     result = newIntNodeT(val, n, g)
     # echo "subU: ", val, " n: ", n, " result: ", val
   of mMulU:
-    let val = maskBytes(getInt(a) * getInt(b), int(n.typ.size))
+    let val = maskBytes(getInt(a) * getInt(b), int(getSize(g.config, n.typ)))
     result = newIntNodeT(val, n, g)
   of mModU:
-    let argA = maskBytes(getInt(a), int(a.typ.size))
-    let argB = maskBytes(getInt(b), int(a.typ.size))
+    let argA = maskBytes(getInt(a), int(getSize(g.config, a.typ)))
+    let argB = maskBytes(getInt(b), int(getSize(g.config, a.typ)))
     if argB != Zero:
       result = newIntNodeT(argA mod argB, n, g)
   of mDivU:
-    let argA = maskBytes(getInt(a), int(a.typ.size))
-    let argB = maskBytes(getInt(b), int(a.typ.size))
+    let argA = maskBytes(getInt(a), int(getSize(g.config, a.typ)))
+    let argB = maskBytes(getInt(b), int(getSize(g.config, a.typ)))
     if argB != Zero:
       result = newIntNodeT(argA div argB, n, g)
   of mLeSet: result = newIntNodeT(toInt128(ord(containsSets(g.config, a, b))), n, g)
@@ -715,8 +715,15 @@ proc getConstExpr(m: PSym, n: PNode; g: ModuleGraph): PNode =
       result.typ = n.typ
   of nkBracketExpr: result = foldArrayAccess(m, n, g)
   of nkDotExpr: result = foldFieldAccess(m, n, g)
+  of nkCheckedFieldExpr:
+    assert n[0].kind == nkDotExpr
+    result = foldFieldAccess(m, n[0], g)
   of nkStmtListExpr:
-    if n.len == 2 and n[0].kind == nkComesFrom:
-      result = getConstExpr(m, n[1], g)
+    var i = 0
+    while i <= n.len - 2:
+      if n[i].kind in {nkComesFrom, nkCommentStmt, nkEmpty}: i.inc
+      else: break
+    if i == n.len - 1:
+      result = getConstExpr(m, n[i], g)
   else:
     discard

compiler/seminst.nim

@@ -319,6 +319,14 @@ proc instantiateProcType(c: PContext, pt: TIdTable,
   prc.typ = result
   popInfoContext(c.config)
 
+proc fillMixinScope(c: PContext) =
+  var p = c.p
+  while p != nil:
+    for bnd in p.localBindStmts:
+      for n in bnd:
+        addSym(c.currentScope, n.sym)
+    p = p.next
+
 proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
                       info: TLineInfo): PSym =
   ## Generates a new instance of a generic procedure.
@@ -345,6 +353,10 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   result.ast = n
   pushOwner(c, result)
 
+  # mixin scope:
+  openScope(c)
+  fillMixinScope(c)
+
   openScope(c)
   let gp = n[genericParamsPos]
   internalAssert c.config, gp.kind != nkEmpty
@@ -395,6 +407,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable,
   popProcCon(c)
   popInfoContext(c.config)
   closeScope(c)           # close scope for parameters
+  closeScope(c)           # close scope for 'mixin' declarations
   popOwner(c)
   c.currentScope = oldScope
   discard c.friendModules.pop()

compiler/semmagic.nim

@@ -512,12 +512,14 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     if n[^1].kind == nkSym and n[^1].sym.kind notin {skProc, skFunc}:
       localError(c.config, n.info, "finalizer must be a direct reference to a proc")
     elif optTinyRtti in c.config.globalOptions:
+      let fin = if n[^1].kind == nkLambda: n[^1][namePos].sym
+                else: n[^1].sym
       # check if we converted this finalizer into a destructor already:
-      let t = whereToBindTypeHook(c, n[^1].sym.typ[1].skipTypes(abstractInst+{tyRef}))
-      if t != nil and t.attachedOps[attachedDestructor] != nil and t.attachedOps[attachedDestructor].owner == n[^1].sym:
+      let t = whereToBindTypeHook(c, fin.typ[1].skipTypes(abstractInst+{tyRef}))
+      if t != nil and t.attachedOps[attachedDestructor] != nil and t.attachedOps[attachedDestructor].owner == fin:
         discard "already turned this one into a finalizer"
       else:
-        bindTypeHook(c, turnFinalizerIntoDestructor(c, n[^1].sym, n.info), n, attachedDestructor)
+        bindTypeHook(c, turnFinalizerIntoDestructor(c, fin, n.info), n, attachedDestructor)
     result = n
   of mDestroy:
     result = n
@@ -543,6 +545,10 @@ proc magicsAfterOverloadResolution(c: PContext, n: PNode,
     let constructed = result[1].typ.base
     if constructed.requiresInit:
       message(c.config, n.info, warnUnsafeDefault, typeToString(constructed))
+  of mPred:
+    if n[1].typ.skipTypes(abstractInst).kind in {tyUInt..tyUInt64}:
+      n[0].sym.magic = mSubU
+    result = n
   else:
     result = n
 

compiler/semobjconstr.nim

@@ -359,12 +359,17 @@ proc defaultConstructionError(c: PContext, t: PType, info: TLineInfo) =
   while objType.kind != tyObject:
     objType = objType.lastSon
     assert objType != nil
-  var constrCtx = initConstrContext(objType, newNodeI(nkObjConstr, info))
-  let initResult = semConstructTypeAux(c, constrCtx, {})
-  assert constrCtx.missingFields.len > 0
-  localError(c.config, info,
-    "The $1 type doesn't have a default value. The following fields must be initialized: $2.",
-    [typeToString(t), listSymbolNames(constrCtx.missingFields)])
+  if objType.kind == tyObject:
+    var constrCtx = initConstrContext(objType, newNodeI(nkObjConstr, info))
+    let initResult = semConstructTypeAux(c, constrCtx, {})
+    if constrCtx.missingFields.len > 0:
+      localError(c.config, info,
+        "The $1 type doesn't have a default value. The following fields must be initialized: $2." % [typeToString(t), listSymbolNames(constrCtx.missingFields)])
+  elif objType.kind == tyDistinct:
+    localError(c.config, info,
+      "The $1 distinct type doesn't have a default value." % typeToString(t))
+  else:
+    assert false, "Must not enter here."
 
 proc semObjConstr(c: PContext, n: PNode, flags: TExprFlags): PNode =
   var t = semTypeNode(c, n[0], nil)

compiler/semparallel.nim

@@ -391,7 +391,8 @@ proc analyse(c: var AnalysisCtx; n: PNode) =
         addFactNeg(c.guards, canon(n[0], c.guards.o))
     dec c.inLoop
   of nkTypeSection, nkProcDef, nkConverterDef, nkMethodDef, nkIteratorDef,
-      nkMacroDef, nkTemplateDef, nkConstSection, nkPragma, nkFuncDef:
+      nkMacroDef, nkTemplateDef, nkConstSection, nkPragma, nkFuncDef,
+      nkMixinStmt, nkBindStmt, nkExportStmt:
     discard
   else:
     analyseSons(c, n)

compiler/sempass2.nim

@@ -66,7 +66,7 @@ type
   TEffects = object
     exc: PNode  # stack of exceptions
     tags: PNode # list of tags
-    bottom, inTryStmt: int
+    bottom, inTryStmt, inExceptOrFinallyStmt: int
     owner: PSym
     ownerModule: PSym
     init: seq[int] # list of initialized variables
@@ -248,6 +248,8 @@ proc listGcUnsafety(s: PSym; onlyWarning: bool; conf: ConfigRef) =
 
 proc useVar(a: PEffects, n: PNode) =
   let s = n.sym
+  if a.inExceptOrFinallyStmt > 0:
+    incl s.flags, sfUsedInFinallyOrExcept
   if isLocalVar(a, s):
     if sfNoInit in s.flags:
       # If the variable is explicitly marked as .noinit. do not emit any error
@@ -379,6 +381,7 @@ proc trackTryStmt(tracked: PEffects, n: PNode) =
 
   var branches = 1
   var hasFinally = false
+  inc tracked.inExceptOrFinallyStmt
 
   # Collect the exceptions caught by the except branches
   for i in 1..<n.len:
@@ -411,6 +414,7 @@ proc trackTryStmt(tracked: PEffects, n: PNode) =
       hasFinally = true
 
   tracked.bottom = oldBottom
+  dec tracked.inExceptOrFinallyStmt
   if not hasFinally:
     setLen(tracked.init, oldState)
   for id, count in items(inter):
@@ -780,6 +784,8 @@ proc track(tracked: PEffects, n: PNode) =
     useVar(tracked, n)
     if n.sym.typ != nil and tfHasAsgn in n.sym.typ.flags:
       tracked.owner.flags.incl sfInjectDestructors
+      # bug #15038: ensure consistency
+      if not hasDestructor(n.typ) and sameType(n.typ, n.sym.typ): n.typ = n.sym.typ
   of nkRaiseStmt:
     if n[0].kind != nkEmpty:
       n[0].info = n.info
@@ -893,7 +899,8 @@ proc track(tracked: PEffects, n: PNode) =
     when false: cstringCheck(tracked, n)
     if tracked.owner.kind != skMacro:
       createTypeBoundOps(tracked, n[0].typ, n.info)
-    checkForSink(tracked.config, tracked.owner, n[1])
+    if n[0].kind != nkSym or not isLocalVar(tracked, n[0].sym):
+      checkForSink(tracked.config, tracked.owner, n[1])
   of nkVarSection, nkLetSection:
     for child in n:
       let last = lastSon(child)

compiler/semstmts.nim

@@ -161,7 +161,7 @@ proc semIf(c: PContext, n: PNode; flags: TExprFlags): PNode =
     if it.len == 2:
       openScope(c)
       it[0] = forceBool(c, semExprWithType(c, it[0]))
-      it[1] = semExprBranch(c, it[1])
+      it[1] = semExprBranch(c, it[1], flags)
       typ = commonType(typ, it[1])
       closeScope(c)
     elif it.len == 1:
@@ -722,7 +722,7 @@ proc semForVars(c: PContext, n: PNode; flags: TExprFlags): PNode =
   result = n
   let iterBase = n[^2].typ
   var iter = skipTypes(iterBase, {tyGenericInst, tyAlias, tySink, tyOwned})
-  var iterAfterVarLent = iter.skipTypes({tyLent, tyVar})
+  var iterAfterVarLent = iter.skipTypes({tyGenericInst, tyAlias, tyLent, tyVar})
   # n.len == 3 means that there is one for loop variable
   # and thus no tuple unpacking:
   if iterAfterVarLent.kind != tyTuple or n.len == 3:
@@ -1472,9 +1472,14 @@ proc semProcAnnotation(c: PContext, prc: PNode;
     if whichPragma(it) != wInvalid:
       # Not a custom pragma
       continue
-    elif strTableGet(c.userPragmas, considerQuotedIdent(c, key)) != nil:
-      # User-defined pragma
-      continue
+    else:
+      let ident = considerQuotedIdent(c, key)
+      if strTableGet(c.userPragmas, ident) != nil:
+        continue # User defined pragma
+      else: 
+        let sym = searchInScopes(c, ident)
+        if sym != nil and sfCustomPragma in sym.flags: 
+          continue # User custom pragma
 
     # we transform ``proc p {.m, rest.}`` into ``m(do: proc p {.rest.})`` and
     # let the semantic checker deal with it:
@@ -1743,7 +1748,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
                  "signature for 'deepCopy' must be proc[T: ptr|ref](x: T): T")
     incl(s.flags, sfUsed)
     incl(s.flags, sfOverriden)
-  of "=", "=sink":
+  of "=", "=copy", "=sink":
     if s.magic == mAsgn: return
     incl(s.flags, sfUsed)
     incl(s.flags, sfOverriden)
@@ -1765,7 +1770,7 @@ proc semOverride(c: PContext, s: PSym, n: PNode) =
         # attach these ops to the canonical tySequence
         obj = canonType(c, obj)
         #echo "ATTACHING TO ", obj.id, " ", s.name.s, " ", cast[int](obj)
-        let k = if name == "=": attachedAsgn else: attachedSink
+        let k = if name == "=" or name == "=copy": attachedAsgn else: attachedSink
         if obj.attachedOps[k] == s:
           discard "forward declared op"
         elif obj.attachedOps[k].isNil and tfCheckedForDestructor notin obj.flags:
@@ -1823,8 +1828,8 @@ proc semMethodPrototype(c: PContext; s: PSym; n: PNode) =
       let t = tt[col]
       if t != nil and t.kind == tyGenericInvocation:
         var x = skipTypes(t[0], {tyVar, tyLent, tyPtr, tyRef, tyGenericInst,
-                                      tyGenericInvocation, tyGenericBody,
-                                      tyAlias, tySink, tyOwned})
+                                 tyGenericInvocation, tyGenericBody,
+                                 tyAlias, tySink, tyOwned})
         if x.kind == tyObject and t.len-1 == n[genericParamsPos].len:
           foundObj = true
           x.methods.add((col,s))

compiler/semtempl.nim

@@ -86,6 +86,7 @@ proc symChoice(c: PContext, n: PNode, s: PSym, r: TSymChoiceRule;
       a = nextOverloadIter(o, c, n)
 
 proc semBindStmt(c: PContext, n: PNode, toBind: var IntSet): PNode =
+  result = copyNode(n)
   for i in 0..<n.len:
     var a = n[i]
     # If 'a' is an overloaded symbol, we used to use the first symbol
@@ -99,16 +100,24 @@ proc semBindStmt(c: PContext, n: PNode, toBind: var IntSet): PNode =
       let sc = symChoice(c, n, s, scClosed)
       if sc.kind == nkSym:
         toBind.incl(sc.sym.id)
+        result.add sc
       else:
-        for x in items(sc): toBind.incl(x.sym.id)
+        for x in items(sc):
+          toBind.incl(x.sym.id)
+          result.add x
     else:
       illFormedAst(a, c.config)
-  result = newNodeI(nkEmpty, n.info)
 
 proc semMixinStmt(c: PContext, n: PNode, toMixin: var IntSet): PNode =
+  result = copyNode(n)
+  var count = 0
   for i in 0..<n.len:
     toMixin.incl(considerQuotedIdent(c, n[i]).id)
-  result = newNodeI(nkEmpty, n.info)
+    let x = symChoice(c, n[i], nil, scForceOpen)
+    inc count, x.len
+    result.add x
+  if count == 0:
+    result = newNodeI(nkEmpty, n.info)
 
 proc replaceIdentBySym(c: PContext; n: var PNode, s: PNode) =
   case n.kind

compiler/semtypes.nim

@@ -660,7 +660,7 @@ proc semRecordCase(c: PContext, n: PNode, check: var IntSet, pos: var int,
   case typ.kind
   of shouldChckCovered:
     chckCovered = true
-  of tyFloat..tyFloat128, tyString, tyError:
+  of tyFloat..tyFloat128, tyError:
     discard
   of tyRange:
     if skipTypes(typ[0], abstractInst).kind in shouldChckCovered:
@@ -668,7 +668,7 @@ proc semRecordCase(c: PContext, n: PNode, check: var IntSet, pos: var int,
   of tyForward:
     errorUndeclaredIdentifier(c, n[0].info, typ.sym.name.s)
   elif not isOrdinalType(typ):
-    localError(c.config, n[0].info, "selector must be of an ordinal type, float or string")
+    localError(c.config, n[0].info, "selector must be of an ordinal type, float")
   if firstOrd(c.config, typ) != 0:
     localError(c.config, n.info, "low(" & $a[0].sym.name.s &
                                      ") must be 0 for discriminant")
@@ -1576,28 +1576,34 @@ proc applyTypeSectionPragmas(c: PContext; pragmas, operand: PNode): PNode =
 
     if p.kind == nkEmpty or whichPragma(p) != wInvalid:
       discard "builtin pragma"
-    elif strTableGet(c.userPragmas, considerQuotedIdent(c, key)) != nil:
-      discard "User-defined pragma"
     else:
-      # we transform ``(arg1, arg2: T) {.m, rest.}`` into ``m((arg1, arg2: T) {.rest.})`` and
-      # let the semantic checker deal with it:
-      var x = newNodeI(nkCall, key.info)
-      x.add(key)
-      if p.kind in nkPragmaCallKinds and p.len > 1:
-        # pass pragma arguments to the macro too:
-        for i in 1 ..< p.len:
-          x.add(p[i])
-      # Also pass the node the pragma has been applied to
-      x.add(operand.copyTreeWithoutNode(p))
-      # recursion assures that this works for multiple macro annotations too:
-      var r = semOverloadedCall(c, x, x, {skMacro, skTemplate}, {efNoUndeclared})
-      if r != nil:
-        doAssert r[0].kind == nkSym
-        let m = r[0].sym
-        case m.kind
-        of skMacro: return semMacroExpr(c, r, r, m, {efNoSemCheck})
-        of skTemplate: return semTemplateExpr(c, r, m, {efNoSemCheck})
-        else: doAssert(false, "cannot happen")
+      let ident = considerQuotedIdent(c, key)
+      if strTableGet(c.userPragmas, ident) != nil:
+        discard "User-defined pragma"
+      else:
+        let sym = searchInScopes(c, ident)
+        if sym != nil and sfCustomPragma in sym.flags: 
+          discard "Custom user pragma"
+        else:
+          # we transform ``(arg1, arg2: T) {.m, rest.}`` into ``m((arg1, arg2: T) {.rest.})`` and
+          # let the semantic checker deal with it:
+          var x = newNodeI(nkCall, key.info)
+          x.add(key)
+          if p.kind in nkPragmaCallKinds and p.len > 1:
+            # pass pragma arguments to the macro too:
+            for i in 1 ..< p.len:
+              x.add(p[i])
+          # Also pass the node the pragma has been applied to
+          x.add(operand.copyTreeWithoutNode(p))
+          # recursion assures that this works for multiple macro annotations too:
+          var r = semOverloadedCall(c, x, x, {skMacro, skTemplate}, {efNoUndeclared})
+          if r != nil:
+            doAssert r[0].kind == nkSym
+            let m = r[0].sym
+            case m.kind
+            of skMacro: return semMacroExpr(c, r, r, m, {efNoSemCheck})
+            of skTemplate: return semTemplateExpr(c, r, m, {efNoSemCheck})
+            else: doAssert(false, "cannot happen")
 
 proc semProcTypeWithScope(c: PContext, n: PNode,
                           prev: PType, kind: TSymKind): PType =

compiler/sigmatch.nim

@@ -548,7 +548,8 @@ proc allowsNilDeprecated(c: TCandidate, f: PType): TTypeRelation =
     result = isNone
 
 proc inconsistentVarTypes(f, a: PType): bool {.inline.} =
-  result = f.kind != a.kind and (f.kind in {tyVar, tyLent} or a.kind in {tyVar, tyLent})
+  result = f.kind != a.kind and
+    (f.kind in {tyVar, tyLent, tySink} or a.kind in {tyVar, tyLent, tySink})
 
 proc procParamTypeRel(c: var TCandidate, f, a: PType): TTypeRelation =
   ## For example we have:
@@ -1085,7 +1086,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
       let x = typeRel(c, f, branch, flags + {trDontBind})
       if x == isNone: return isNone
       if x < result: result = x
-    return
+    return result
 
   of tyAnd:
     # XXX: deal with the current dual meaning of tyGenericParam
@@ -1420,7 +1421,7 @@ proc typeRel(c: var TCandidate, f, aOrig: PType,
           let ff = rootf[i]
           let aa = roota[i]
           let res = typeRel(c, ff, aa, nextFlags)
-          if res != isEqual: result = isGeneric
+          if res != isNone and res != isEqual: result = isGeneric
           if res notin {isEqual, isGeneric}:
             if trNoCovariance notin flags and ff.kind == aa.kind:
               let paramFlags = rootf.base[i-1].flags

compiler/sinkparameter_inference.nim

@@ -48,8 +48,7 @@ proc checkForSink*(config: ConfigRef; owner: PSym; arg: PNode) =
         # we only report every potential 'sink' parameter only once:
         incl arg.sym.flags, sfWasForwarded
         message(config, arg.info, hintPerformance,
-          ("could not turn '$1' to a sink parameter " &
-          "because '$2' was forward declared") % [arg.sym.name.s, owner.name.s])
+          "could not turn '$1' to a sink parameter" % [arg.sym.name.s])
       #echo config $ arg.info, " candidate for a sink parameter here"
   of nkStmtList, nkStmtListExpr, nkBlockStmt, nkBlockExpr:
     if not isEmptyType(arg.typ):

compiler/sizealignoffsetimpl.nim

@@ -138,12 +138,13 @@ proc computeObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode, packed: bool, a
       accum.offset  = szUnknownSize
       accum.maxAlign = szUnknownSize
     else:
-      # the union neds to be aligned first, before the offsets can be assigned
+      # the union needs to be aligned first, before the offsets can be assigned
       accum.align(maxChildAlign)
       let accumRoot = accum # copy, because each branch should start af the same offset
       for i in 1..<n.len:
-        var branchAccum = accumRoot
+        var branchAccum = OffsetAccum(offset: accumRoot.offset, maxAlign: 1)
         computeObjectOffsetsFoldFunction(conf, n[i].lastSon, packed, branchAccum)
+        discard finish(branchAccum)
         accum.mergeBranch(branchAccum)
   of nkRecList:
     for i, child in n.sons:
@@ -173,9 +174,10 @@ proc computeUnionObjectOffsetsFoldFunction(conf: ConfigRef; n: PNode; accum: var
     localError(conf, n.info, "Illegal use of ``case`` in union type.")
   of nkRecList:
     let accumRoot = accum # copy, because each branch should start af the same offset
-    for i, child in n.sons:
-      var branchAccum = accumRoot
+    for child in n.sons:
+      var branchAccum = OffsetAccum(offset: accumRoot.offset, maxAlign: 1)
       computeUnionObjectOffsetsFoldFunction(conf, child, branchAccum)
+      discard finish(branchAccum)
       accum.mergeBranch(branchAccum)
   of nkSym:
     var size = szUnknownSize

compiler/transf.nim

@@ -830,13 +830,6 @@ proc transformExceptBranch(c: PTransf, n: PNode): PNode =
   else:
     result = transformSons(c, n)
 
-proc dontInlineConstant(orig, cnst: PNode): bool {.inline.} =
-  # symbols that expand to a complex constant (array, etc.) should not be
-  # inlined, unless it's the empty array:
-  result = orig.kind == nkSym and
-           cnst.kind in {nkCurly, nkPar, nkTupleConstr, nkBracket} and
-           cnst.len != 0
-
 proc commonOptimizations*(g: ModuleGraph; c: PSym, n: PNode): PNode =
   result = n
   for i in 0..<n.safeLen:
@@ -1000,7 +993,7 @@ proc transform(c: PTransf, n: PNode): PNode =
   of nkConstSection:
     # do not replace ``const c = 3`` with ``const 3 = 3``
     return transformConstSection(c, n)
-  of nkTypeSection, nkTypeOfExpr:
+  of nkTypeSection, nkTypeOfExpr, nkMixinStmt, nkBindStmt:
     # no need to transform type sections:
     return n
   of nkVarSection, nkLetSection:
@@ -1035,6 +1028,11 @@ proc transform(c: PTransf, n: PNode): PNode =
     return n
   of nkExceptBranch:
     result = transformExceptBranch(c, n)
+  of nkCheckedFieldExpr:
+    result = transformSons(c, n)
+    if result[0].kind != nkDotExpr:
+      # simplfied beyond a dot expression --> simplify further.
+      result = result[0]
   else:
     result = transformSons(c, n)
   when false:
@@ -1090,15 +1088,15 @@ proc liftDeferAux(n: PNode) =
         if n[i].kind == nkDefer:
           let deferPart = newNodeI(nkFinally, n[i].info)
           deferPart.add n[i][0]
-          var tryStmt = newNodeI(nkTryStmt, n[i].info)
-          var body = newNodeI(n.kind, n[i].info)
+          var tryStmt = newNodeIT(nkTryStmt, n[i].info, n.typ)
+          var body = newNodeIT(n.kind, n[i].info, n.typ)
           if i < last:
             body.sons = n.sons[(i+1)..last]
           tryStmt.add body
           tryStmt.add deferPart
           n[i] = tryStmt
           n.sons.setLen(i+1)
-          n.typ = n[i].typ
+          n.typ = tryStmt.typ
           goOn = true
           break
   for i in 0..n.safeLen-1:

compiler/trees.nim

@@ -169,3 +169,10 @@ proc flattenStmts*(n: PNode): PNode =
 proc extractRange*(k: TNodeKind, n: PNode, a, b: int): PNode =
   result = newNodeI(k, n.info, b-a+1)
   for i in 0..b-a: result[i] = n[i+a]
+
+proc dontInlineConstant*(orig, cnst: PNode): bool {.inline.} =
+  # symbols that expand to a complex constant (array, etc.) should not be
+  # inlined, unless it's the empty array:
+  result = orig.kind != cnst.kind and
+           cnst.kind in {nkCurly, nkPar, nkTupleConstr, nkBracket, nkObjConstr} and
+           cnst.len > ord(cnst.kind == nkObjConstr)

compiler/types.nim

@@ -71,7 +71,7 @@ const
                tyInferred, tySink, tyLent, tyOwned}
   # typedescX is used if we're sure tyTypeDesc should be included (or skipped)
   typedescPtrs* = abstractPtrs + {tyTypeDesc}
-  typedescInst* = abstractInst + {tyTypeDesc, tyOwned}
+  typedescInst* = abstractInst + {tyTypeDesc, tyOwned, tyUserTypeClass}
 
 proc invalidGenericInst*(f: PType): bool =
   result = f.kind == tyGenericInst and lastSon(f) == nil

compiler/vm.nim

@@ -510,6 +510,23 @@ const
     "if you are sure this is not a bug in your code, compile with `--maxLoopIterationsVM:number` (current value: $1)"
   errFieldXNotFound = "node lacks field: "
 
+
+template maybeHandlePtr(node2: PNode, reg: TFullReg, isAssign2: bool): bool =
+  let node = node2 # prevent double evaluation
+  if node.kind == nkNilLit:
+    stackTrace(c, tos, pc, errNilAccess)
+  let typ = node.typ
+  if nfIsPtr in node.flags or (typ != nil and typ.kind == tyPtr):
+    assert node.kind == nkIntLit, $(node.kind)
+    assert typ != nil
+    let typ2 = if typ.kind == tyPtr: typ[0] else: typ
+    if not derefPtrToReg(node.intVal, typ2, reg, isAssign = isAssign2):
+      # tyObject not supported in this context
+      stackTrace(c, tos, pc, "deref unsupported ptr type: " & $(typeToString(typ), typ.kind))
+    true
+  else:
+    false
+
 proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
   var pc = start
   var tos = tos
@@ -755,26 +772,12 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
         ensureKind(regs[rb].regAddr.kind)
         regs[ra] = regs[rb].regAddr[]
       of rkNode:
-        if regs[rb].node.kind == nkNilLit:
-          stackTrace(c, tos, pc, errNilAccess)
         if regs[rb].node.kind == nkRefTy:
           regs[ra].node = regs[rb].node[0]
-        else:
-          let node = regs[rb].node
-          let typ = node.typ
-          # see also `nfIsPtr`
-          if node.kind == nkIntLit:
-            var typ2 = typ
-            doAssert typ != nil
-            if typ.kind == tyPtr:
-              typ2 = typ2[0]
-            if not derefPtrToReg(node.intVal, typ2, regs[ra], isAssign = false):
-              # tyObject not supported in this context
-              stackTrace(c, tos, pc, "opcLdDeref unsupported ptr type: " & $(typeToString(typ), typ.kind))
-          else:
-            ## eg: typ.kind = tyObject
-            ensureKind(rkNode)
-            regs[ra].node = regs[rb].node
+        elif not maybeHandlePtr(regs[rb].node, regs[ra], false):
+          ## eg: typ.kind = tyObject
+          ensureKind(rkNode)
+          regs[ra].node = regs[rb].node
       else:
         stackTrace(c, tos, pc, errNilAccess & " kind: " & $regs[rb].kind)
     of opcWrDeref:
@@ -795,18 +798,8 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): TFullReg =
           regs[ra].nodeAddr[] = n
       of rkRegisterAddr: regs[ra].regAddr[] = regs[rc]
       of rkNode:
-        if regs[ra].node.kind == nkNilLit:
-          stackTrace(c, tos, pc, errNilAccess)
-        let node = regs[ra].node
-        let typ = node.typ
-        if nfIsPtr in node.flags or (typ != nil and typ.kind == tyPtr):
-          assert node.kind == nkIntLit, $(node.kind)
-          var typ2 = typ
-          if typ.kind == tyPtr:
-            typ2 = typ2[0]
-          if not derefPtrToReg(node.intVal, typ2, regs[rc], isAssign = true):
-            stackTrace(c, tos, pc, "opcWrDeref unsupported ptr type: " & $(typeToString(typ), typ.kind))
-        else:
+         # xxx: also check for nkRefTy as in opcLdDeref?
+        if not maybeHandlePtr(regs[ra].node, regs[rc], true):
           regs[ra].node[] = regs[rc].regToNode[]
           regs[ra].node.flags.incl nfIsRef
       else: stackTrace(c, tos, pc, errNilAccess)
@@ -2093,6 +2086,7 @@ proc execute(c: PCtx, start: int): PNode =
   result = rawExecute(c, start, tos).regToNode
 
 proc execProc*(c: PCtx; sym: PSym; args: openArray[PNode]): PNode =
+  c.loopIterations = c.config.maxLoopIterationsVM
   if sym.kind in routineKinds:
     if sym.typ.len-1 != args.len:
       localError(c.config, sym.info,

compiler/vmgen.nim

@@ -16,7 +16,7 @@
 #   types that use the 'node' field; the reason is that slots are
 #   re-used in a register based VM. Example:
 #
-#..code-block:: nim
+#.. code-block:: nim
 #   let s = a & b  # no matter what, create fresh node
 #   s = a & b  # no matter what, keep the node
 #
@@ -247,6 +247,8 @@ proc freeTemp(c: PCtx; r: TRegister) =
 proc getTempRange(cc: PCtx; n: int; kind: TSlotKind): TRegister =
   # if register pressure is high, we re-use more aggressively:
   let c = cc.prc
+  # we could also customize via the following (with proper caching in ConfigRef):
+  # let highRegisterPressure = cc.config.getConfigVar("vm.highRegisterPressure", "40").parseInt
   if c.maxSlots >= HighRegisterPressure or c.maxSlots+n >= high(TRegister):
     for i in 0..c.maxSlots-n:
       if not c.slots[i].inUse:
@@ -1521,14 +1523,14 @@ proc genAsgn(c: PCtx; le, ri: PNode; requiresCopy: bool) =
     let tmp = c.genx(ri)
     c.preventFalseAlias(le[0], opcWrObj, objR, idx, tmp)
     c.freeTemp(tmp)
-    c.freeTemp(idx)
+    # c.freeTemp(idx) # BUGFIX, see nkDotExpr
     c.freeTemp(objR)
   of nkDotExpr:
     let dest = c.genx(le[0], {gfNode})
     let idx = genField(c, le[1])
     let tmp = c.genx(ri)
     c.preventFalseAlias(le, opcWrObj, dest, idx, tmp)
-    c.freeTemp(idx)
+    # c.freeTemp(idx) # BUGFIX: idx is an immediate (field position), not a register
     c.freeTemp(tmp)
     c.freeTemp(dest)
   of nkDerefExpr, nkHiddenDeref:
@@ -1957,14 +1959,6 @@ proc genProc*(c: PCtx; s: PSym): int
 
 proc matches(s: PSym; x: string): bool =
   let y = x.split('.')
-  var s = s
-  for i in 1..y.len:
-    if s == nil or (y[^i].cmpIgnoreStyle(s.name.s) != 0 and y[^i] != "*"):
-      return false
-    s = s.owner
-  result = true
-
-proc matches(s: PSym; y: varargs[string]): bool =
   var s = s
   for i in 1..y.len:
     if s == nil or (y[^i].cmpIgnoreStyle(s.name.s) != 0 and y[^i] != "*"):
@@ -2025,11 +2019,11 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
       elif s.kind == skMethod:
         localError(c.config, n.info, "cannot call method " & s.name.s &
           " at compile time")
-      elif matches(s, "stdlib", "marshal", "to"):
+      elif matches(s, "stdlib.marshal.to"):
         # XXX marshal load&store should not be opcodes, but use the
         # general callback mechanisms.
         genMarshalLoad(c, n, dest)
-      elif matches(s, "stdlib", "marshal", "$$"):
+      elif matches(s, "stdlib.marshal.$$"):
         genMarshalStore(c, n, dest)
       else:
         genCall(c, n, dest)
@@ -2112,7 +2106,8 @@ proc gen(c: PCtx; n: PNode; dest: var TDest; flags: TGenFlags = {}) =
     else:
       dest = tmp0
   of nkEmpty, nkCommentStmt, nkTypeSection, nkConstSection, nkPragma,
-     nkTemplateDef, nkIncludeStmt, nkImportStmt, nkFromStmt, nkExportStmt:
+     nkTemplateDef, nkIncludeStmt, nkImportStmt, nkFromStmt, nkExportStmt,
+     nkMixinStmt, nkBindStmt:
     unused(c, n, dest)
   of nkStringToCString, nkCStringToString:
     gen(c, n[0], dest)

compiler/vmops.nim

@@ -110,7 +110,7 @@ when defined(nimHasInvariant):
     of arguments: result = conf.arguments
     of outFile: result = conf.outFile.string
     of outDir: result = conf.outDir.string
-    of nimcacheDir: result = conf.nimcacheDir.string
+    of nimcacheDir: result = conf.getNimcacheDir().string
     of projectName: result = conf.projectName
     of projectPath: result = conf.projectPath.string
     of projectFull: result = conf.projectFull.string

config/nimdoc.cfg

@@ -136,15 +136,10 @@ doc.body_toc_group = """
   </div>
   <div id="global-links">
     <ul class="simple-boot">
-      <li>
-        <a href="manual.html">Manual</a>
-      </li>
-      <li>
-        <a href="lib.html">Standard library</a>
-      </li>
-      <li>
-        <a href="theindex.html">Index</a>
-      </li>
+      <li><a href="manual.html">Manual</a></li>
+      <li><a href="lib.html">Standard library</a></li>
+      <li><a href="theindex.html">Index</a></li>
+      <li><a href="https://nim-lang.github.io/Nim/">devel</a>, <a href="https://nim-lang.org/documentation.html">stable</a></li>
     </ul>
   </div>
   <div id="searchInputDiv">

doc/advopt.txt

@@ -42,6 +42,8 @@ Advanced options:
                             produce hints or errors for Nim identifiers that
                             do not adhere to Nim's official style guide
                             https://nim-lang.org/docs/nep1.html
+  --styleCheck:usages       only enforce consistent spellings of identifiers,
+                            do not enforce the style on declarations
   --showAllMismatches:on|off
                             show all mismatching candidates in overloading
                             resolution

doc/backends.rst

@@ -47,9 +47,11 @@ The commands to compile to either C, C++ or Objective-C are:
 The most significant difference between these commands is that if you look
 into the ``nimcache`` directory you will find ``.c``, ``.cpp`` or ``.m``
 files, other than that all of them will produce a native binary for your
-project.  This allows you to take the generated code and place it directly
-into a project using any of these languages. Here are some typical command
-line invocations::
+project. This allows you to take the generated code and place it directly
+into a project using any of these languages. Here are some typical command-
+line invocations:
+
+.. code:: cmd
 
     $ nim c hallo.nim
     $ nim cpp hallo.nim
@@ -108,8 +110,8 @@ Nim code calling the backend
 Nim code can interface with the backend through the `Foreign function
 interface <manual.html#foreign-function-interface>`_ mainly through the
 `importc pragma <manual.html#foreign-function-interface-importc-pragma>`_.
-The ``importc`` pragma is the *generic* way of making backend symbols available
-in Nim and is available in all the target backends (JavaScript too).  The C++
+The `importc` pragma is the *generic* way of making backend symbols available
+in Nim and is available in all the target backends (JavaScript too). The C++
 or Objective-C backends have their respective `ImportCpp
 <manual.html#implementation-specific-pragmas-importcpp-pragma>`_ and
 `ImportObjC <manual.html#implementation-specific-pragmas-importobjc-pragma>`_
@@ -230,11 +232,6 @@ Also, C code requires you to specify a forward declaration for functions or
 the compiler will assume certain types for the return value and parameters
 which will likely make your program crash at runtime.
 
-The Nim compiler can generate a C interface header through the ``--header``
-command line switch. The generated header will contain all the exported
-symbols and the ``NimMain`` proc which you need to call before any other
-Nim code.
-
 
 Nim invocation example from C
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -253,9 +250,10 @@ Create a ``maths.c`` file with the following content:
 
 .. code-block:: c
 
-  #include "fib.h"
   #include <stdio.h>
 
+  extern int fib(int a);
+
   int main(void)
   {
     NimMain();
@@ -268,13 +266,14 @@ Now you can run the following Unix like commands to first generate C sources
 form the Nim code, then link them into a static binary along your main C
 program::
 
-  $ nim c --noMain --noLinking --header:fib.h fib.nim
-  $ gcc -o m -I$HOME/.cache/nim/fib_d -Ipath/to/nim/lib $HOME/.cache/nim/fib_d/*.c maths.c
+.. code:: cmd
+
+  nim c --noMain --noLinking fib.nim
+  gcc -o m -I$HOME/.cache/nim/fib_d -Ipath/to/nim/lib $HOME/.cache/nim/fib_d/*.c maths.c
 
 The first command runs the Nim compiler with three special options to avoid
-generating a ``main()`` function in the generated files, avoid linking the
-object files into a final binary, and explicitly generate a header file for C
-integration. All the generated files are placed into the ``nimcache``
+generating a `main()`:c: function in the generated files and to avoid linking the
+object files into a final binary. All the generated files are placed into the ``nimcache``
 directory. That's why the next command compiles the ``maths.c`` source plus
 all the ``.c`` files form ``nimcache``. In addition to this path, you also
 have to tell the C compiler where to find Nim's ``nimbase.h`` header file.
@@ -282,12 +281,12 @@ have to tell the C compiler where to find Nim's ``nimbase.h`` header file.
 Instead of depending on the generation of the individual ``.c`` files you can
 also ask the Nim compiler to generate a statically linked library::
 
-  $ nim c --app:staticLib --noMain --header fib.nim
-  $ gcc -o m -Inimcache -Ipath/to/nim/lib libfib.nim.a maths.c
+  nim c --app:staticLib --noMain fib.nim
+  gcc -o m -Inimcache -Ipath/to/nim/lib libfib.nim.a maths.c
 
 The Nim compiler will handle linking the source files generated in the
 ``nimcache`` directory into the ``libfib.nim.a`` static library, which you can
-then link into your C program.  Note that these commands are generic and will
+then link into your C program. Note that these commands are generic and will
 vary for each system. For instance, on Linux systems you will likely need to
 use ``-ldl`` too to link in required dlopen functionality.
 

doc/destructors.rst

@@ -41,7 +41,7 @@ written as:
       dealloc(x.data)
       x.data = nil
 
-  proc `=`*[T](a: var myseq[T]; b: myseq[T]) =
+  proc `=copy`*[T](a: var myseq[T]; b: myseq[T]) =
     # do nothing for self-assignments:
     if a.data == b.data: return
     `=destroy`(a)
@@ -134,7 +134,7 @@ not free the resources afterwards by setting the object to its default value
 default value is written as ``wasMoved(x)``. When not provided the compiler
 is using a combination of `=destroy` and `copyMem` instead. This is efficient
 hence users rarely need to implement their own `=sink` operator, it is enough to
-provide `=destroy` and `=`, compiler will take care about the rest.
+provide `=destroy` and `=copy`, compiler will take care about the rest.
 
 The prototype of this hook for a type ``T`` needs to be:
 
@@ -156,10 +156,10 @@ The general pattern in ``=sink`` looks like:
 How self-assignments are handled is explained later in this document.
 
 
-`=` (copy) hook
+`=copy` hook
 ---------------
 
-The ordinary assignment in Nim conceptually copies the values. The ``=`` hook
+The ordinary assignment in Nim conceptually copies the values. The ``=copy`` hook
 is called for assignments that couldn't be transformed into ``=sink``
 operations.
 
@@ -167,21 +167,21 @@ The prototype of this hook for a type ``T`` needs to be:
 
 .. code-block:: nim
 
-  proc `=`(dest: var T; source: T)
+  proc `=copy`(dest: var T; source: T)
 
 
-The general pattern in ``=`` looks like:
+The general pattern in ``=copy`` looks like:
 
 .. code-block:: nim
 
-  proc `=`(dest: var T; source: T) =
+  proc `=copy`(dest: var T; source: T) =
     # protect against self-assignments:
     if dest.field != source.field:
       `=destroy`(dest)
       dest.field = duplicateResource(source.field)
 
 
-The ``=`` proc can be marked with the ``{.error.}`` pragma. Then any assignment
+The ``=copy`` proc can be marked with the ``{.error.}`` pragma. Then any assignment
 that otherwise would lead to a copy is prevented at compile-time.
 
 
@@ -199,7 +199,7 @@ Swap
 ====
 
 The need to check for self-assignments and also the need to destroy previous
-objects inside ``=`` and ``=sink`` is a strong indicator to treat
+objects inside ``=copy`` and ``=sink`` is a strong indicator to treat
 ``system.swap`` as a builtin primitive of its own that simply swaps every
 field in the involved objects via ``copyMem`` or a comparable mechanism.
 In other words, ``swap(a, b)`` is **not** implemented
@@ -320,7 +320,7 @@ not destroyed at the scope exit, but at the proc exit.
 
   x = y
   ------------------          (copy)
-  `=`(x, y)
+  `=copy`(x, y)
 
 
   f_sink(g())
@@ -330,7 +330,7 @@ not destroyed at the scope exit, but at the proc exit.
 
   f_sink(notLastReadOf y)
   --------------------------     (copy-to-sink)
-  (let tmp; `=`(tmp, y);
+  (let tmp; `=copy`(tmp, y);
   f_sink(tmp))
 
 

doc/manual.rst

@@ -148,7 +148,7 @@ It is mostly accurate to picture semantic analysis proceeding top to bottom and
 left to right in the source code, with compile-time code execution interleaved
 when necessary to compute values that are required for subsequent semantic
 analysis. We will see much later in this document that macro invocation not only
-requires this interleaving, but also creates a situation where semantic analyis
+requires this interleaving, but also creates a situation where semantic analysis
 does not entirely proceed top to bottom and left to right.
 
 
@@ -792,7 +792,7 @@ features supported at compile time (as detailed in the next section below).
 Within such a code block, it is possible to declare variables and then later
 read and update them, or declare variables and pass them to procedures that
 modify them. However, the code in such a block must still adhere to the
-retrictions listed above for referencing values and operations outside the
+restrictions listed above for referencing values and operations outside the
 block.
 
 The ability to access and modify compile-time variables adds flexibility to
@@ -1296,7 +1296,7 @@ A parameter ``A`` may be an *open array*, in which case it is indexed by
 integers from 0 to ``len(A)-1``. An array expression may be constructed by the
 array constructor ``[]``. The element type of this array expression is
 inferred from the type of the first element. All other elements need to be
-implicitly convertable to this type.
+implicitly convertible to this type.
 
 Sequences are similar to arrays but of dynamic length which may change
 during runtime (like strings). Sequences are implemented as growable arrays,
@@ -1802,7 +1802,7 @@ Mixing GC'ed memory with ``ptr``
 
 Special care has to be taken if an untraced object contains traced objects like
 traced references, strings or sequences: in order to free everything properly,
-the built-in procedure ``GCunref`` has to be called before freeing the untraced
+the built-in procedure ``reset`` has to be called before freeing the untraced
 memory manually:
 
 .. code-block:: nim
@@ -1816,12 +1816,12 @@ memory manually:
   d.s = "abc"
 
   # tell the GC that the string is not needed anymore:
-  GCunref(d.s)
+  reset(d.s)
 
   # free the memory:
   dealloc(d)
 
-Without the ``GCunref`` call the memory allocated for the ``d.s`` string would
+Without the ``reset`` call the memory allocated for the ``d.s`` string would
 never be freed. The example also demonstrates two important features for low
 level programming: the ``sizeof`` proc returns the size of a type or value
 in bytes. The ``cast`` operator can circumvent the type system: the compiler
@@ -2362,7 +2362,7 @@ The convertible relation can be relaxed by a user-defined type
   # implicit conversion magic happens here
   x = chr
   echo x # => 97
-  # you can use the explicit form too
+  # one can use the explicit form too
   x = chr.toInt
   echo x # => 97
 
@@ -2483,9 +2483,10 @@ matches) is preferred:
 Overloading based on 'var T'
 ----------------------------
 
-If the formal parameter ``f`` is of type ``var T`` in addition to the ordinary
-type checking, the argument is checked to be an `l-value`:idx:. ``var T``
-matches better than just ``T`` then.
+If the formal parameter ``f`` is of type ``var T``
+in addition to the ordinary type checking,
+the argument is checked to be an `l-value`:idx:.
+``var T`` matches better than just ``T`` then.
 
 .. code-block:: nim
   proc sayHi(x: int): string =
@@ -3856,7 +3857,7 @@ as there are components in the tuple. The i'th iteration variable's type is
 the type of the i'th component. In other words, implicit tuple unpacking in a
 for loop context is supported.
 
-Implict items/pairs invocations
+Implicit items/pairs invocations
 -------------------------------
 
 If the for loop expression ``e`` does not denote an iterator and the for loop
@@ -4250,7 +4251,7 @@ Exception hierarchy
 The exception tree is defined in the `system <system.html>`_ module.
 Every exception inherits from ``system.Exception``. Exceptions that indicate
 programming bugs inherit from ``system.Defect`` (which is a subtype of ``Exception``)
-and are stricly speaking not catchable as they can also be mapped to an operation
+and are strictly speaking not catchable as they can also be mapped to an operation
 that terminates the whole process. If panics are turned into exceptions, these
 exceptions inherit from `Defect`.
 
@@ -4539,8 +4540,8 @@ more complex type classes:
 
 Whilst the syntax of type classes appears to resemble that of ADTs/algebraic data
 types in ML-like languages, it should be understood that type classes are static
-constraints to be enforced at type instantations. Type classes are not really
-types in themsleves, but are instead a system of providing generic "checks" that
+constraints to be enforced at type instantiations. Type classes are not really
+types in themselves, but are instead a system of providing generic "checks" that
 ultimately *resolve* to some singular type. Type classes do not allow for
 runtime type dynamism, unlike object variants or methods.
 
@@ -4702,7 +4703,7 @@ instantiation. The following is not allowed:
   # not valid: 'T' is not inferred to be of type 'var int'
   g(v, i)
 
-  # also not allowed: explict instantiation via 'var int'
+  # also not allowed: explicit instantiation via 'var int'
   g[var int](v, i)
 
 
@@ -4788,6 +4789,50 @@ scope is the default.
 ``bind`` statements only make sense in templates and generics.
 
 
+Delegating bind statements
+--------------------------
+
+The following example outlines a problem that can arise when generic
+instantiations cross multiple different modules:
+
+.. code-block:: nim
+
+  # module A
+  proc genericA*[T](x: T) =
+    mixin init
+    init(x)
+
+
+.. code-block:: nim
+
+  import C
+
+  # module B
+  proc genericB*[T](x: T) =
+    # Without the `bind init` statement C's init proc is
+    # not available when `genericB` is instantiated:
+    bind init
+    genericA(x)
+
+.. code-block:: nim
+
+  # module C
+  type O = object
+  proc init*(x: var O) = discard
+
+.. code-block:: nim
+
+  # module main
+  import B, C
+
+  genericB O()
+
+In module B has an `init` proc from module C in its scope that is not
+taken into account when `genericB` is instantiated which leads to the
+instantiation of `genericA`. The solution is to `forward`:idx these
+symbols by a `bind` statement inside `genericB`.
+
+
 Templates
 =========
 
@@ -4858,7 +4903,7 @@ compiler. Explicit immediate templates are now deprecated.
 Passing a code block to a template
 ----------------------------------
 
-You can pass a block of statements as the last argument to a template
+One can pass a block of statements as the last argument to a template
 following the special ``:`` syntax:
 
 .. code-block:: nim
@@ -5192,7 +5237,7 @@ tree (AST) of the code that is passed to it. The macro can then do
 transformations on it and return the transformed AST. This can be used to
 add custom language features and implement `domain specific languages`:idx:.
 
-Macro invocation is a case where semantic analyis does **not** entirely proceed
+Macro invocation is a case where semantic analysis does **not** entirely proceed
 top to bottom and left to right. Instead, semantic analysis happens at least
 twice:
 
@@ -5444,7 +5489,7 @@ generic type ``static[T]``. The type param can be omitted to obtain the type
 class of all constant expressions. A more specific type class can be created by
 instantiating ``static`` with another type class.
 
-You can force an expression to be evaluated at compile time as a constant
+One can force an expression to be evaluated at compile time as a constant
 expression by coercing it to a corresponding ``static`` type:
 
 .. code-block:: nim
@@ -5458,14 +5503,14 @@ possible type mismatch error.
 typedesc[T]
 -----------
 
-In many contexts, Nim allows you to treat the names of types as regular
+In many contexts, Nim allows to treat the names of types as regular
 values. These values exists only during the compilation phase, but since
 all values must have a type, ``typedesc`` is considered their special type.
 
 ``typedesc`` acts like a generic type. For instance, the type of the symbol
 ``int`` is ``typedesc[int]``. Just like with regular generic types, when the
 generic param is omitted, ``typedesc`` denotes the type class of all types.
-As a syntactic convenience, you can also use ``typedesc`` as a modifier.
+As a syntactic convenience, one can also use ``typedesc`` as a modifier.
 
 Procs featuring ``typedesc`` params are considered implicitly generic.
 They will be instantiated for each unique combination of supplied types
@@ -5543,7 +5588,7 @@ typeof operator
 **Note**: ``typeof(x)`` can for historical reasons also be written as
 ``type(x)`` but ``type(x)`` is discouraged.
 
-You can obtain the type of a given expression by constructing a ``typeof``
+One can obtain the type of a given expression by constructing a ``typeof``
 value from it (in many other languages this is known as the `typeof`:idx:
 operator):
 
@@ -5714,12 +5759,12 @@ avoid ambiguity when there are multiple modules with the same path.
 There are two pseudo directories:
 
 1. ``std``: The ``std`` pseudo directory is the abstract location of Nim's standard
-library. For example, the syntax ``import std / strutils`` is used to unambiguously
-refer to the standard library's ``strutils`` module.
+   library. For example, the syntax ``import std / strutils`` is used to unambiguously
+   refer to the standard library's ``strutils`` module.
 2. ``pkg``: The ``pkg`` pseudo directory is used to unambiguously refer to a Nimble
-package. However, for technical details that lie outside of the scope of this document
-its semantics are: *Use the search path to look for module name but ignore the standard
-library locations*. In other words, it is the opposite of ``std``.
+   package. However, for technical details that lie outside of the scope of this document
+   its semantics are: *Use the search path to look for module name but ignore the standard
+   library locations*. In other words, it is the opposite of ``std``.
 
 
 From import statement
@@ -5770,9 +5815,9 @@ modules don't need to import a module's dependencies:
   echo $x
 
 When the exported symbol is another module, all of its definitions will
-be forwarded. You can use an ``except`` list to exclude some of the symbols.
+be forwarded. One can use an ``except`` list to exclude some of the symbols.
 
-Notice that when exporting, you need to specify only the module name:
+Notice that when exporting, one needs to specify only the module name:
 
 .. code-block:: nim
   import foo/bar/baz
@@ -5981,10 +6026,10 @@ noSideEffect pragma
 The ``noSideEffect`` pragma is used to mark a proc/iterator to have no side
 effects. This means that the proc/iterator only changes locations that are
 reachable from its parameters and the return value only depends on the
-arguments. If none of its parameters have the type ``var T``
-or ``ref T`` or ``ptr T`` this means no locations are modified. It is a static
-error to mark a proc/iterator to have no side effect if the compiler cannot
-verify this.
+arguments. If none of its parameters have the type ``var T`` or ``ref T``
+or ``ptr T`` this means no locations are modified. It is a static error to
+mark a proc/iterator to have no side effect if the compiler cannot verify
+this.
 
 As a special semantic rule, the built-in `debugEcho
 <system.html#debugEcho,varargs[typed,]>`_ pretends to be free of side effects,
@@ -6576,7 +6621,7 @@ and instead the generated code should contain an ``#include``:
     PFile {.importc: "FILE*", header: "<stdio.h>".} = distinct pointer
       # import C's FILE* type; Nim will treat it as a new pointer type
 
-The ``header`` pragma always expects a string constant. The string contant
+The ``header`` pragma always expects a string constant. The string constant
 contains the header file: As usual for C, a system header file is enclosed
 in angle brackets: ``<>``. If no angle brackets are given, Nim
 encloses the header file in ``""`` in the generated C code.
@@ -6604,9 +6649,17 @@ with the project:
   {.compile: "myfile.cpp".}
 
 **Note**: Nim computes a SHA1 checksum and only recompiles the file if it
-has changed. You can use the ``-f`` command line option to force recompilation
+has changed. One can use the ``-f`` command line option to force recompilation
 of the file.
 
+Since 1.4 the `compile` pragma is also available with this syntax:
+
+.. code-block:: Nim
+  {.compile("myfile.cpp", "--custom flags here").}
+
+As can be seen in the example, this new variant allows for custom flags
+that are passed to the C compiler when the file is recompiled.
+
 
 Link pragma
 -----------
@@ -6619,12 +6672,12 @@ The ``link`` pragma can be used to link an additional file with the project:
 PassC pragma
 ------------
 The ``passc`` pragma can be used to pass additional parameters to the C
-compiler like you would using the commandline switch ``--passc``:
+compiler like one would using the commandline switch ``--passc``:
 
 .. code-block:: Nim
   {.passc: "-Wall -Werror".}
 
-Note that you can use ``gorge`` from the `system module <system.html>`_ to
+Note that one can use ``gorge`` from the `system module <system.html>`_ to
 embed parameters from an external command that will be executed
 during semantic analysis:
 
@@ -6647,12 +6700,12 @@ the pragma resides in:
 PassL pragma
 ------------
 The ``passL`` pragma can be used to pass additional parameters to the linker
-like you would using the commandline switch ``--passL``:
+like one would using the commandline switch ``--passL``:
 
 .. code-block:: Nim
   {.passL: "-lSDLmain -lSDL".}
 
-Note that you can use ``gorge`` from the `system module <system.html>`_ to
+Note that one can use ``gorge`` from the `system module <system.html>`_ to
 embed parameters from an external command that will be executed
 during semantic analysis:
 
@@ -6907,7 +6960,7 @@ Importcpp for objects
 ~~~~~~~~~~~~~~~~~~~~~
 
 Generic ``importcpp``'ed objects are mapped to C++ templates. This means that
-you can import C++'s templates rather easily without the need for a pattern
+one can import C++'s templates rather easily without the need for a pattern
 language for object types:
 
 .. code-block:: nim
@@ -7156,9 +7209,9 @@ specified. It is possible to annotate procs, templates, type and variable
 definitions, statements, etc.
 
 Macros module includes helpers which can be used to simplify custom pragma
-access `hasCustomPragma`, `getCustomPragmaVal`. Please consult macros module
-documentation for details. These macros are no magic, they don't do anything
-you cannot do yourself by walking AST object representation.
+access `hasCustomPragma`, `getCustomPragmaVal`. Please consult the macros module
+documentation for details. These macros are not magic, everything they do can
+also be achieved by walking the AST of the object representation.
 
 More examples with custom pragmas:
 

doc/nimc.rst

@@ -149,6 +149,22 @@ ignored too. ``--define:FOO`` and ``--define:foo`` are identical.
 Compile time symbols starting with the ``nim`` prefix are reserved for the
 implementation and should not be used elsewhere.
 
+==========================       ============================================
+Name                             Description
+==========================       ============================================
+nimStdSetjmp                     Use the standard `setjmp()/longjmp()` library
+                                 functions for setjmp-based exceptions. This is
+                                 the default on most platforms.
+nimSigSetjmp                     Use `sigsetjmp()/siglongjmp()` for setjmp-based exceptions.
+nimRawSetjmp                     Use `_setjmp()/_longjmp()` on POSIX and `_setjmp()/longjmp()`
+                                 on Windows, for setjmp-based exceptions. It's the default on
+                                 BSDs and BSD-like platforms, where it's significantly faster
+                                 than the standard functions.
+nimBuiltinSetjmp                 Use `__builtin_setjmp()/__builtin_longjmp()` for setjmp-based
+                                 exceptions. This will not work if an exception is being thrown
+                                 and caught inside the same procedure. Useful for benchmarking.
+==========================       ============================================
+
 
 Configuration files
 -------------------

koch.nim

@@ -10,7 +10,7 @@
 #
 
 const
-  NimbleStableCommit = "4007b2a778429a978e12307bf13a038029b4c4d9" # master
+  NimbleStableCommit = "d13f3b8ce288b4dc8c34c219a4e050aaeaf43fc9" # master
 
 when not defined(windows):
   const
@@ -136,35 +136,11 @@ proc bundleC2nim(args: string) =
 
 proc bundleNimbleExe(latest: bool, args: string) =
   let commit = if latest: "HEAD" else: NimbleStableCommit
-  cloneDependency(distDir, "https://github.com/nim-lang/nimble.git", commit = commit)
+  cloneDependency(distDir, "https://github.com/nim-lang/nimble.git",
+                  commit = commit, allowBundled = true)
   # installer.ini expects it under $nim/bin
   nimCompile("dist/nimble/src/nimble.nim",
-             options = "-d:release --nilseqs:on " & args)
-
-proc buildNimble(latest: bool, args: string) =
-  # if koch is used for a tar.xz, build the dist/nimble we shipped
-  # with the tarball:
-  var installDir = "dist/nimble"
-  if not latest and dirExists(installDir) and not dirExists("dist/nimble/.git"):
-    discard "don't do the git dance"
-  else:
-    if not dirExists("dist/nimble/.git"):
-      if dirExists(installDir):
-        var id = 0
-        while dirExists("dist/nimble" & $id):
-          inc id
-        installDir = "dist/nimble" & $id
-      # consider using/adapting cloneDependency
-      exec("git clone -q https://github.com/nim-lang/nimble.git " & installDir)
-    withDir(installDir):
-      if latest:
-        exec("git checkout -f master")
-        exec("git pull")
-      else:
-        exec("git fetch")
-        exec("git checkout " & NimbleStableCommit)
-  nimCompile(installDir / "src/nimble.nim",
-             options = "--noNimblePath --nilseqs:on -d:release " & args)
+             options = "-d:release --noNimblePath " & args)
 
 proc bundleNimsuggest(args: string) =
   nimCompileFold("Compile nimsuggest", "nimsuggest/nimsuggest.nim",
@@ -664,13 +640,13 @@ when isMainModule:
       of "temp": temp(op.cmdLineRest)
       of "xtemp": xtemp(op.cmdLineRest)
       of "wintools": bundleWinTools(op.cmdLineRest)
-      of "nimble": buildNimble(latest, op.cmdLineRest)
+      of "nimble": bundleNimbleExe(latest, op.cmdLineRest)
       of "nimsuggest": bundleNimsuggest(op.cmdLineRest)
       of "toolsnonimble":
         buildTools(op.cmdLineRest)
       of "tools":
         buildTools(op.cmdLineRest)
-        buildNimble(latest, op.cmdLineRest)
+        bundleNimbleExe(latest, op.cmdLineRest)
       of "pushcsource", "pushcsources": pushCsources()
       of "valgrind": valgrind(op.cmdLineRest)
       of "c2nim": bundleC2nim(op.cmdLineRest)

lib/core/macros.nim

@@ -1262,7 +1262,15 @@ proc `body=`*(someProc: NimNode, val: NimNode) {.compileTime.} =
   else:
     badNodeKind someProc, "body="
 
-proc basename*(a: NimNode): NimNode {.compileTime, benign.}
+proc basename*(a: NimNode): NimNode {.raises: [].} =
+  ## Pull an identifier from prefix/postfix expressions.
+  case a.kind
+  of nnkIdent: result = a
+  of nnkPostfix, nnkPrefix: result = a[1]
+  of nnkPragmaExpr: result = basename(a[0])
+  else:
+    error("Do not know how to get basename of (" & treeRepr(a) & ")\n" &
+      repr(a), a)
 
 proc `$`*(node: NimNode): string {.compileTime.} =
   ## Get the string of an identifier node.
@@ -1325,16 +1333,6 @@ proc insert*(a: NimNode; pos: int; b: NimNode) {.compileTime.} =
       a[i + 1] = a[i]
     a[pos] = b
 
-proc basename*(a: NimNode): NimNode =
-  ## Pull an identifier from prefix/postfix expressions.
-  case a.kind
-  of nnkIdent: result = a
-  of nnkPostfix, nnkPrefix: result = a[1]
-  of nnkPragmaExpr: result = basename(a[0])
-  else:
-    error("Do not know how to get basename of (" & treeRepr(a) & ")\n" &
-      repr(a), a)
-
 proc `basename=`*(a: NimNode; val: string) {.compileTime.}=
   case a.kind
   of nnkIdent:

lib/core/typeinfo.nim

@@ -271,7 +271,7 @@ proc setPointer*(x: Any, y: pointer) =
   ## ``akPointer``, ``akSequence``.
   assert x.rawType.kind in {tyString, tyCString, tyRef, tyPtr, tyPointer,
                             tySequence, tyProc}
-  cast[ppointer](x.value)[] = y
+  genericAssign(x.value, y, x.rawType)
 
 proc fieldsAux(p: pointer, n: ptr TNimNode,
                ret: var seq[tuple[name: cstring, any: Any]]) =

lib/impure/db_odbc.nim

@@ -242,7 +242,7 @@ proc tryExec*(db: var DbConn, query: SqlQuery, args: varargs[string, `$`]): bool
   try:
     db.prepareFetchDirect(query, args)
     var
-      rCnt = -1
+      rCnt:TSqlLen = -1
     res = SQLRowCount(db.stmt, rCnt)
     properFreeResult(SQL_HANDLE_STMT, db.stmt)
     if res != SQL_SUCCESS: dbError(db)
@@ -461,10 +461,10 @@ proc execAffectedRows*(db: var DbConn, query: SqlQuery,
   var q = dbFormat(query, args)
   db.sqlCheck(SQLPrepare(db.stmt, q.PSQLCHAR, q.len.TSqlSmallInt))
   rawExec(db, query, args)
-  var rCnt = -1
+  var rCnt:TSqlLen = -1
   db.sqlCheck(SQLRowCount(db.hDb, rCnt))
   properFreeResult(SQL_HANDLE_STMT, db.stmt)
-  result = rCnt
+  result = rCnt.int64
 
 proc close*(db: var DbConn) {.
       tags: [WriteDbEffect], raises: [].} =
@@ -489,7 +489,7 @@ proc open*(connection, user, password, database: string): DbConn {.
   ## Currently the database parameter is ignored,
   ## but included to match ``open()`` in the other db_xxxxx library modules.
   var
-    val: TSqlInteger = SQL_OV_ODBC3
+    val = SQL_OV_ODBC3
     resLen = 0
   result = (hDb: nil, env: nil, stmt: nil)
   # allocate environment handle
@@ -497,7 +497,7 @@ proc open*(connection, user, password, database: string): DbConn {.
   if res != SQL_SUCCESS: dbError("Error: unable to initialise ODBC environment.")
   res = SQLSetEnvAttr(result.env,
                       SQL_ATTR_ODBC_VERSION.TSqlInteger,
-                      val, resLen.TSqlInteger)
+                      cast[SqlPointer](val.addr), resLen.TSqlInteger)
   if res != SQL_SUCCESS: dbError("Error: unable to set ODBC driver version.")
   # allocate hDb handle
   res = SQLAllocHandle(SQL_HANDLE_DBC, result.env, result.hDb)

lib/impure/nre.nim

@@ -6,6 +6,9 @@
 #    distribution, for details about the copyright.
 #
 
+when defined(js):
+  {.error: "This library needs to be compiled with a c-like backend, and depends on PCRE.".}
+
 ## What is NRE?
 ## ============
 ##
@@ -59,7 +62,6 @@ from pcre import nil
 import nre/private/util
 import tables
 from strutils import `%`
-from math import ceil
 import options
 from unicode import runeLenAt
 
@@ -528,7 +530,7 @@ proc matchImpl(str: string, pattern: Regex, start, endpos: int, flags: int): Opt
       raise RegexInternalError(msg : "Unknown internal error: " & $execRet)
 
 proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[RegexMatch] =
-  ## Like ` ``find(...)`` <#proc-find>`_, but anchored to the start of the
+  ## Like `find(...)<#find,string,Regex,int>`_, but anchored to the start of the
   ## string.
   ##
   runnableExamples:
@@ -538,11 +540,11 @@ proc match*(str: string, pattern: Regex, start = 0, endpos = int.high): Option[R
   return str.matchImpl(pattern, start, endpos, pcre.ANCHORED)
 
 iterator findIter*(str: string, pattern: Regex, start = 0, endpos = int.high): RegexMatch =
-  ## Works the same as ` ``find(...)`` <#proc-find>`_, but finds every
+  ## Works the same as `find(...)<#find,string,Regex,int>`_, but finds every
   ## non-overlapping match. ``"2222".find(re"22")`` is ``"22", "22"``, not
   ## ``"22", "22", "22"``.
   ##
-  ## Arguments are the same as ` ``find(...)`` <#proc-find>`_
+  ## Arguments are the same as `find(...)<#find,string,Regex,int>`_
   ##
   ## Variants:
   ##
@@ -621,7 +623,7 @@ proc split*(str: string, pattern: Regex, maxSplit = -1, start = 0): seq[string]
   ## Splits the string with the given regex. This works according to the
   ## rules that Perl and Javascript use.
   ##
-  ## ``start`` behaves the same as in ` ``find(...)`` <#proc-find>`_.
+  ## ``start`` behaves the same as in `find(...)<#find,string,Regex,int>`_.
   ##
   runnableExamples:
     # -  If the match is zero-width, then the string is still split:
@@ -731,8 +733,25 @@ proc replace*(str: string, pattern: Regex, sub: string): string =
   replaceImpl(str, pattern,
     formatStr(sub, match.captures[name], match.captures[id - 1]))
 
-let SpecialCharMatcher = re"([\\+*?[^\]$(){}=!<>|:-])"
-proc escapeRe*(str: string): string =
-  ## Escapes the string so it doesn’t match any special characters.
+proc escapeRe*(str: string): string {.gcsafe.} =
+  ## Escapes the string so it doesn't match any special characters.
   ## Incompatible with the Extra flag (``X``).
-  str.replace(SpecialCharMatcher, "\\$1")
+  ##
+  ## Escaped char: `\ + * ? [ ^ ] $ ( ) { } = ! < > | : -`
+  runnableExamples:
+    doAssert escapeRe("fly+wind") == "fly\\+wind"
+    doAssert escapeRe("!") == "\\!"
+    doAssert escapeRe("nim*") == "nim\\*"
+
+  #([\\+*?[^\]$(){}=!<>|:-])
+  const SpecialCharMatcher = {'\\', '+', '*', '?', '[', '^', ']', '$', '(',
+                              ')', '{', '}', '=', '!', '<', '>', '|', ':',
+                              '-'}
+
+  for c in items(str):
+    case c
+    of SpecialCharMatcher:
+      result.add("\\")
+      result.add(c)
+    else:
+      result.add(c)

lib/impure/re.nim

@@ -7,6 +7,9 @@
 #    distribution, for details about the copyright.
 #
 
+when defined(js):
+  {.error: "This library needs to be compiled with a c-like backend, and depends on PCRE.".}
+
 ## Regular expression support for Nim.
 ##
 ## This module is implemented by providing a wrapper around the
@@ -678,4 +681,3 @@ when isMainModule:
     doAssert replace("bar", re"^", "foo") == "foobar"
     doAssert replace("foo", re"", "-") == "-foo"
     doAssert replace("foo", re"$", "bar") == "foobar"
-

lib/js/jsffi.nim

@@ -22,7 +22,7 @@
 ##  var document {.importc, nodecl.}: JsObject
 ##  var console {.importc, nodecl.}: JsObject
 ##  # import the "$" function
-##  proc jq(selector: JsObject): JsObject {.importcpp: "$(#)".}
+##  proc jq(selector: JsObject): JsObject {.importcpp: "$$(#)".}
 ##
 ##  # Use jQuery to make the following code run, after the document is ready.
 ##  # This uses an experimental ``.()`` operator for ``JsObject``, to emit

lib/posix/posix_other.nim

@@ -32,7 +32,12 @@ type
   SocketHandle* = distinct cint # The type used to represent socket descriptors
 
 type
-  Time* {.importc: "time_t", header: "<time.h>".} = distinct clong
+  Time* {.importc: "time_t", header: "<time.h>".} = distinct (
+    when defined(nimUse64BitCTime):
+      int64
+    else:
+      clong
+  )
 
   Timespec* {.importc: "struct timespec",
                header: "<time.h>", final, pure.} = object ## struct timespec

lib/pure/algorithm.nim

@@ -504,7 +504,7 @@ template sortedByIt*(seq1, op: untyped): untyped =
     # Nested sort
     assert people.sortedByIt((it.age, it.name)) == @[(name: "p2", age: 20),
        (name: "p3", age: 30), (name: "p4", age: 30), (name: "p1", age: 60)]
-  var result = sorted(seq1, proc(x, y: type(seq1[0])): int =
+  var result = sorted(seq1, proc(x, y: typeof(seq1[0])): int =
     var it {.inject.} = x
     let a = op
     it = y

lib/pure/asyncdispatch.nim

@@ -267,7 +267,7 @@ when defined(windows) or defined(nimdoc):
       pcd: PostCallbackDataPtr
     AsyncEvent* = ptr AsyncEventImpl
 
-    Callback = proc (fd: AsyncFD): bool {.closure, gcsafe.}
+    Callback* = proc (fd: AsyncFD): bool {.closure, gcsafe.}
 
   proc hash(x: AsyncFD): Hash {.borrow.}
   proc `==`*(x: AsyncFD, y: AsyncFD): bool {.borrow.}
@@ -1002,10 +1002,11 @@ when defined(windows) or defined(nimdoc):
   proc addProcess*(pid: int, cb: Callback) =
     ## Registers callback ``cb`` to be called when process with process ID
     ## ``pid`` exited.
+    const NULL = Handle(0)
     let p = getGlobalDispatcher()
     let procFlags = SYNCHRONIZE
     var hProcess = openProcess(procFlags, 0, pid.DWORD)
-    if hProcess == INVALID_HANDLE_VALUE:
+    if hProcess == NULL:
       raiseOSError(osLastError())
 
     var pcd = cast[PostCallbackDataPtr](allocShared0(sizeof(PostCallbackData)))
@@ -1098,7 +1099,7 @@ else:
                                      # queue.
   type
     AsyncFD* = distinct cint
-    Callback = proc (fd: AsyncFD): bool {.closure, gcsafe.}
+    Callback* = proc (fd: AsyncFD): bool {.closure, gcsafe.}
 
     AsyncData = object
       readList: seq[Callback]
@@ -1210,14 +1211,19 @@ else:
     let newLength = max(len(curList), InitCallbackListSize)
     var newList = newSeqOfCap[Callback](newLength)
 
+    var eventsExtinguished = false
     for cb in curList:
+      if eventsExtinguished:
+        newList.add(cb)
+        continue
       if not cb(fd):
         # Callback wants to be called again.
         newList.add(cb)
         # This callback has returned with EAGAIN, so we don't need to
         # call any other callbacks as they are all waiting for the same event
         # on the same fd.
-        break
+        # We do need to ensure they are called again though.
+        eventsExtinguished = true
 
     withData(selector, fd.int, fdData) do:
       # Descriptor is still present in the queue.
@@ -1886,3 +1892,25 @@ proc waitFor*[T](fut: Future[T]): T =
     poll()
 
   fut.read
+
+proc activeDescriptors*(): int {.inline.} =
+  ## Returns the current number of active file descriptors for the current
+  ## event loop. This is a cheap operation that does not involve a system call.
+  when defined(windows):
+    result = getGlobalDispatcher().handles.len
+  elif not defined(nimdoc):
+    result = getGlobalDispatcher().selector.count
+
+when defined(posix):
+  import posix
+
+proc maxDescriptors*(): int {.raises: OSError.} =
+  ## Returns the maximum number of active file descriptors for the current
+  ## process. This involves a system call.
+  when defined(windows):
+    result = 16_700_000
+  else:
+    var fdLim: RLimit
+    if getrlimit(RLIMIT_NOFILE, fdLim) < 0:
+      raiseOSError(osLastError())
+    result = int(fdLim.rlim_cur) - 1

lib/pure/asyncftpclient.nim

@@ -134,7 +134,7 @@ proc expectReply(ftp: AsyncFtpClient): Future[TaintedString] {.async.} =
   var line = await ftp.csock.recvLine()
   result = TaintedString(line)
   var count = 0
-  while line[3] == '-':
+  while line.len > 3 and line[3] == '-':
     ## Multi-line reply.
     line = await ftp.csock.recvLine()
     string(result).add("\n" & line)
@@ -146,7 +146,11 @@ proc send*(ftp: AsyncFtpClient, m: string): Future[TaintedString] {.async.} =
   ## Send a message to the server, and wait for a primary reply.
   ## ``\c\L`` is added for you.
   ##
+  ## You need to make sure that the message ``m`` doesn't contain any newline
+  ## characters. Failing to do so will raise ``AssertionDefect``.
+  ##
   ## **Note:** The server may return multiple lines of coded replies.
+  doAssert(not m.contains({'\c', '\L'}), "message shouldn't contain any newline characters")
   await ftp.csock.send(m & "\c\L")
   return await ftp.expectReply()
 

lib/pure/asyncfutures.nim

@@ -9,6 +9,8 @@
 
 import os, tables, strutils, times, heapqueue, options, deques, cstrutils
 
+import "system/stacktraces"
+
 # TODO: This shouldn't need to be included, but should ideally be exported.
 type
   CallbackFunc = proc () {.closure, gcsafe.}
@@ -296,49 +298,72 @@ proc `callback=`*[T](future: Future[T],
   ## If future has already completed then ``cb`` will be called immediately.
   future.callback = proc () = cb(future)
 
+template getFilenameProcname(entry: StackTraceEntry): (string, string) =
+  when compiles(entry.filenameStr) and compiles(entry.procnameStr):
+    # We can't rely on "entry.filename" and "entry.procname" still being valid
+    # cstring pointers, because the "string.data" buffers they pointed to might
+    # be already garbage collected (this entry being a non-shallow copy,
+    # "entry.filename" no longer points to "entry.filenameStr.data", but to the
+    # buffer of the original object).
+    (entry.filenameStr, entry.procnameStr)
+  else:
+    ($entry.filename, $entry.procname)
+
 proc getHint(entry: StackTraceEntry): string =
   ## We try to provide some hints about stack trace entries that the user
   ## may not be familiar with, in particular calls inside the stdlib.
+
+  let (filename, procname) = getFilenameProcname(entry)
+
   result = ""
-  if entry.procname == cstring"processPendingCallbacks":
-    if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0:
+  if procname == "processPendingCallbacks":
+    if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0:
       return "Executes pending callbacks"
-  elif entry.procname == cstring"poll":
-    if cmpIgnoreStyle(entry.filename, "asyncdispatch.nim") == 0:
+  elif procname == "poll":
+    if cmpIgnoreStyle(filename, "asyncdispatch.nim") == 0:
       return "Processes asynchronous completion events"
 
-  if entry.procname.endsWith(NimAsyncContinueSuffix):
-    if cmpIgnoreStyle(entry.filename, "asyncmacro.nim") == 0:
+  if procname.endsWith(NimAsyncContinueSuffix):
+    if cmpIgnoreStyle(filename, "asyncmacro.nim") == 0:
       return "Resumes an async procedure"
 
-proc `$`*(entries: seq[StackTraceEntry]): string =
+proc `$`*(stackTraceEntries: seq[StackTraceEntry]): string =
+  when defined(nimStackTraceOverride):
+    let entries = addDebuggingInfo(stackTraceEntries)
+  else:
+    let entries = stackTraceEntries
+
   result = ""
   # Find longest filename & line number combo for alignment purposes.
   var longestLeft = 0
   for entry in entries:
-    if entry.procname.isNil: continue
+    let (filename, procname) = getFilenameProcname(entry)
 
-    let left = $entry.filename & $entry.line
-    if left.len > longestLeft:
-      longestLeft = left.len
+    if procname == "": continue
+
+    let leftLen = filename.len + len($entry.line)
+    if leftLen > longestLeft:
+      longestLeft = leftLen
 
   var indent = 2
   # Format the entries.
   for entry in entries:
-    if entry.procname.isNil:
-      if entry.line == -10:
+    let (filename, procname) = getFilenameProcname(entry)
+
+    if procname == "":
+      if entry.line == reraisedFromBegin:
         result.add(spaces(indent) & "#[\n")
         indent.inc(2)
-      else:
+      elif entry.line == reraisedFromEnd:
         indent.dec(2)
         result.add(spaces(indent) & "]#\n")
       continue
 
-    let left = "$#($#)" % [$entry.filename, $entry.line]
+    let left = "$#($#)" % [filename, $entry.line]
     result.add((spaces(indent) & "$#$# $#\n") % [
       left,
       spaces(longestLeft - left.len + 2),
-      $entry.procname
+      procname
     ])
     let hint = getHint(entry)
     if hint.len > 0:
@@ -362,9 +387,9 @@ proc injectStacktrace[T](future: Future[T]) =
     newMsg.add($entries)
 
     newMsg.add("Exception message: " & exceptionMsg & "\n")
-    newMsg.add("Exception type:")
 
     # # For debugging purposes
+    # newMsg.add("Exception type:")
     # for entry in getStackTraceEntries(future.error):
     #   newMsg.add "\n" & $entry
     future.error.msg = newMsg

lib/pure/asynchttpserver.nim

@@ -11,24 +11,36 @@
 ##
 ## This HTTP server has not been designed to be used in production, but
 ## for testing applications locally. Because of this, when deploying your
-## application you should use a reverse proxy (for example nginx) instead of
-## allowing users to connect directly to this server.
+## application in production you should use a reverse proxy (for example nginx)
+## instead of allowing users to connect directly to this server.
 ##
-## Basic usage
-## ===========
+## Example
+## =======
 ##
 ## This example will create an HTTP server on port 8080. The server will
 ## respond to all requests with a ``200 OK`` response code and "Hello World"
 ## as the response body.
 ##
-## .. code-block::nim
-##    import asynchttpserver, asyncdispatch
+## .. code-block:: Nim
 ##
-##    var server = newAsyncHttpServer()
-##    proc cb(req: Request) {.async.} =
-##      await req.respond(Http200, "Hello World")
+##   import asynchttpserver, asyncdispatch
 ##
-##    waitFor server.serve(Port(8080), cb)
+##   proc main {.async.} =
+##     var server = newAsyncHttpServer()
+##     proc cb(req: Request) {.async.} =
+##       let headers = {"Date": "Tue, 29 Apr 2014 23:40:08 GMT",
+##           "Content-type": "text/plain; charset=utf-8"}
+##       await req.respond(Http200, "Hello World", headers.newHttpHeaders())
+##
+##     server.listen Port(8080)
+##     while true:
+##       if server.shouldAcceptRequest():
+##         asyncCheck server.acceptRequest(cb)
+##       else:
+##         poll()
+##
+##   asyncCheck main()
+##   runForever()
 
 import asyncnet, asyncdispatch, parseutils, uri, strutils
 import httpcore
@@ -58,14 +70,12 @@ type
     reuseAddr: bool
     reusePort: bool
     maxBody: int ## The maximum content-length that will be read for the body.
+    maxFDs: int
 
 proc newAsyncHttpServer*(reuseAddr = true, reusePort = false,
                          maxBody = 8388608): AsyncHttpServer =
   ## Creates a new ``AsyncHttpServer`` instance.
-  new result
-  result.reuseAddr = reuseAddr
-  result.reusePort = reusePort
-  result.maxBody = maxBody
+  result = AsyncHttpServer(reuseAddr: reuseAddr, reusePort: reusePort, maxBody: maxBody)
 
 proc addHeaders(msg: var string, headers: HttpHeaders) =
   for k, v in headers:
@@ -99,13 +109,15 @@ proc respond*(req: Request, code: HttpCode, content: string,
 
   if headers != nil:
     msg.addHeaders(headers)
-    
-  if not headers.hasKey("Content-Length"):
+
+  # If the headers did not contain a Content-Length use our own
+  if headers.isNil() or not headers.hasKey("Content-Length"):
     msg.add("Content-Length: ")
     # this particular way saves allocations:
     msg.addInt content.len
-  
-  msg.add "\c\L\c\L"
+    msg.add "\c\L"
+
+  msg.add "\c\L"
   msg.add(content)
   result = req.client.send(msg)
 
@@ -292,13 +304,9 @@ proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string
     )
     if not retry: break
 
-proc serve*(server: AsyncHttpServer, port: Port,
-            callback: proc (request: Request): Future[void] {.closure, gcsafe.},
-            address = "") {.async.} =
-  ## Starts the process of listening for incoming HTTP connections on the
-  ## specified address and port.
-  ##
-  ## When a request is made by a client the specified callback will be called.
+proc listen*(server: AsyncHttpServer; port: Port; address = "") =
+  ## Listen to the given port and address.
+  server.maxFDs = maxDescriptors()
   server.socket = newAsyncSocket()
   if server.reuseAddr:
     server.socket.setSockOpt(OptReuseAddr, true)
@@ -307,9 +315,44 @@ proc serve*(server: AsyncHttpServer, port: Port,
   server.socket.bindAddr(port, address)
   server.socket.listen()
 
+proc shouldAcceptRequest*(server: AsyncHttpServer;
+                          assumedDescriptorsPerRequest = 5): bool {.inline.} =
+  ## Returns true if the process's current number of opened file
+  ## descriptors is still within the maximum limit and so it's reasonable to
+  ## accept yet another request.
+  result = assumedDescriptorsPerRequest < 0 or
+    (activeDescriptors() + assumedDescriptorsPerRequest < server.maxFDs)
+
+proc acceptRequest*(server: AsyncHttpServer,
+            callback: proc (request: Request): Future[void] {.closure, gcsafe.}) {.async.} =
+  ## Accepts a single request. Write an explicit loop around this proc so that
+  ## errors can be handled properly.
+  var (address, client) = await server.socket.acceptAddr()
+  asyncCheck processClient(server, client, address, callback)
+
+proc serve*(server: AsyncHttpServer, port: Port,
+            callback: proc (request: Request): Future[void] {.closure, gcsafe.},
+            address = "";
+            assumedDescriptorsPerRequest = -1) {.async.} =
+  ## Starts the process of listening for incoming HTTP connections on the
+  ## specified address and port.
+  ##
+  ## When a request is made by a client the specified callback will be called.
+  ##
+  ## If `assumedDescriptorsPerRequest` is 0 or greater the server cares about
+  ## the process's maximum file descriptor limit. It then ensures that the
+  ## process still has the resources for `assumedDescriptorsPerRequest`
+  ## file descriptors before accepting a connection.
+  ##
+  ## You should prefer to call `acceptRequest` instead with a custom server
+  ## loop so that you're in control over the error handling and logging.
+  listen server, port, address
   while true:
-    var (address, client) = await server.socket.acceptAddr()
-    asyncCheck processClient(server, client, address, callback)
+    if shouldAcceptRequest(server, assumedDescriptorsPerRequest):
+      var (address, client) = await server.socket.acceptAddr()
+      asyncCheck processClient(server, client, address, callback)
+    else:
+      poll()
     #echo(f.isNil)
     #echo(f.repr)
 
@@ -318,7 +361,7 @@ proc close*(server: AsyncHttpServer) =
   server.socket.close()
 
 when not defined(testing) and isMainModule:
-  proc main =
+  proc main {.async.} =
     var server = newAsyncHttpServer()
     proc cb(req: Request) {.async.} =
       #echo(req.reqMethod, " ", req.url)
@@ -327,6 +370,12 @@ when not defined(testing) and isMainModule:
           "Content-type": "text/plain; charset=utf-8"}
       await req.respond(Http200, "Hello World", headers.newHttpHeaders())
 
-    asyncCheck server.serve(Port(5555), cb)
-    runForever()
-  main()
+    server.listen Port(5555)
+    while true:
+      if server.shouldAcceptRequest():
+        asyncCheck server.acceptRequest(cb)
+      else:
+        poll()
+
+  asyncCheck main()
+  runForever()

lib/pure/asyncmacro.nim

@@ -46,7 +46,7 @@ template createCb(retFutureSym, iteratorNameSym,
         else:
           {.gcsafe.}:
             {.push hint[ConvFromXtoItselfNotNeeded]: off.}
-            next.callback = cast[proc() {.closure, gcsafe.}](identName)
+            next.addCallback cast[proc() {.closure, gcsafe.}](identName)
             {.pop.}
     except:
       futureVarCompletions
@@ -198,7 +198,8 @@ proc getFutureVarIdents(params: NimNode): seq[NimNode] {.compileTime.} =
   for i in 1 ..< len(params):
     expectKind(params[i], nnkIdentDefs)
     if params[i][1].kind == nnkBracketExpr and
-       params[i][1][0].eqIdent("futurevar"):
+       params[i][1][0].eqIdent(FutureVar.astToStr):
+      ## eqIdent: first char is case sensitive!!!
       result.add(params[i][0])
 
 proc isInvalidReturnType(typeName: string): bool =

lib/pure/asyncnet.nim

@@ -691,8 +691,12 @@ elif defined(nimdoc):
 
 proc close*(socket: AsyncSocket) =
   ## Closes the socket.
+  if socket.closed: return
+
   defer:
     socket.fd.AsyncFD.closeSocket()
+    socket.closed = true # TODO: Add extra debugging checks for this.
+
   when defineSsl:
     if socket.isSsl:
       let res = SSL_shutdown(socket.sslHandle)
@@ -701,7 +705,6 @@ proc close*(socket: AsyncSocket) =
         discard
       elif res != 1:
         raiseSSLError()
-  socket.closed = true # TODO: Add extra debugging checks for this.
 
 when defineSsl:
   proc wrapSocket*(ctx: SslContext, socket: AsyncSocket) =

lib/pure/asyncstreams.nim

@@ -110,7 +110,11 @@ proc read*[T](future: FutureStream[T]): owned(Future[(bool, T)]) =
       resFut.complete(res)
 
       # If the saved callback isn't nil then let's call it.
-      if not savedCb.isNil: savedCb()
+      if not savedCb.isNil:
+        if fs.queue.len > 0:
+          savedCb()
+        else:
+          future.cb = savedCb
 
   if future.queue.len > 0 or future.finished:
     newCb(future)

lib/pure/base64.nim

@@ -230,11 +230,11 @@ proc decode*(s: string): string =
 
   template inputChar(x: untyped) =
     let x = int decodeTable[ord(s[inputIndex])]
-    inc inputIndex
     if x == invalidChar:
       raise newException(ValueError,
         "Invalid base64 format character `" & s[inputIndex] &
         "` (ord " & $s[inputIndex].ord & ") at location " & $inputIndex & ".")
+    inc inputIndex
 
   template outputChar(x: untyped) =
     result[outputIndex] = char(x and 255)

lib/pure/browsers.nim

@@ -17,24 +17,30 @@ include "system/inclrtl"
 
 when defined(windows):
   import winlean
+  from os import absolutePath
 else:
   import os, osproc
 
 const osOpenCmd* =
-  when defined(macos) or defined(windows): "open" else: "xdg-open" ## \
+  when defined(macos) or defined(macosx) or defined(windows): "open" else: "xdg-open" ## \
   ## Alias for the operating system specific *"open"* command,
-  ## ``"open"`` on MacOS and Windows, ``"xdg-open"`` on Linux, BSD, etc.
+  ## ``"open"`` on OSX, MacOS and Windows, ``"xdg-open"`` on Linux, BSD, etc.
 
+proc prepare(s: string): string =
+  if s.contains("://"):
+    result = s
+  else:
+    result = "file://" & absolutePath(s)
 
-template openDefaultBrowserImpl(url: string) =
+proc openDefaultBrowserImpl(url: string) =
   when defined(windows):
     var o = newWideCString(osOpenCmd)
-    var u = newWideCString(url)
+    var u = newWideCString(prepare url)
     discard shellExecuteW(0'i32, o, u, nil, nil, SW_SHOWNORMAL)
   elif defined(macosx):
-    discard execShellCmd(osOpenCmd & " " & quoteShell(url))
+    discard execShellCmd(osOpenCmd & " " & quoteShell(prepare url))
   else:
-    var u = quoteShell(url)
+    var u = quoteShell(prepare url)
     if execShellCmd(osOpenCmd & " " & u) == 0: return
     for b in getEnv("BROWSER").string.split(PathSep):
       try:

lib/pure/cgi.nim

@@ -32,12 +32,7 @@
 import strutils, os, strtabs, cookies, uri
 export uri.encodeUrl, uri.decodeUrl
 
-proc handleHexChar(c: char, x: var int) {.inline.} =
-  case c
-  of '0'..'9': x = (x shl 4) or (ord(c) - ord('0'))
-  of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10)
-  of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
-  else: assert(false)
+include includes/decode_helpers
 
 proc addXmlChar(dest: var string, c: char) {.inline.} =
   case c
@@ -93,40 +88,27 @@ proc getEncodedData(allowedMethods: set[RequestMethod]): string =
 iterator decodeData*(data: string): tuple[key, value: TaintedString] =
   ## Reads and decodes CGI data and yields the (name, value) pairs the
   ## data consists of.
+  proc parseData(data: string, i: int, field: var string): int =
+    result = i
+    while result < data.len:
+      case data[result]
+      of '%': add(field, decodePercent(data, result))
+      of '+': add(field, ' ')
+      of '=', '&': break
+      else: add(field, data[result])
+      inc(result)
+
   var i = 0
   var name = ""
   var value = ""
   # decode everything in one pass:
   while i < data.len:
     setLen(name, 0) # reuse memory
-    while i < data.len:
-      case data[i]
-      of '%':
-        var x = 0
-        handleHexChar(data[i+1], x)
-        handleHexChar(data[i+2], x)
-        inc(i, 2)
-        add(name, chr(x))
-      of '+': add(name, ' ')
-      of '=', '&': break
-      else: add(name, data[i])
-      inc(i)
+    i = parseData(data, i, name)
     if i >= data.len or data[i] != '=': cgiError("'=' expected")
     inc(i) # skip '='
     setLen(value, 0) # reuse memory
-    while i < data.len:
-      case data[i]
-      of '%':
-        var x = 0
-        if i+2 < data.len:
-          handleHexChar(data[i+1], x)
-          handleHexChar(data[i+2], x)
-        inc(i, 2)
-        add(value, chr(x))
-      of '+': add(value, ' ')
-      of '&', '\0': break
-      else: add(value, data[i])
-      inc(i)
+    i = parseData(data, i, value)
     yield (name.TaintedString, value.TaintedString)
     if i < data.len:
       if data[i] == '&': inc(i)

lib/pure/collections/sharedtables.nim

@@ -140,6 +140,10 @@ proc hasKeyOrPut*[A, B](t: var SharedTable[A, B], key: A, val: B): bool =
   withLock t:
     hasKeyOrPutImpl(enlarge)
 
+template tabMakeEmpty(i) = t.data[i].hcode = 0
+template tabCellEmpty(i) = isEmpty(t.data[i].hcode)
+template tabCellHash(i)  = t.data[i].hcode
+
 proc withKey*[A, B](t: var SharedTable[A, B], key: A,
                     mapper: proc(key: A, val: var B, pairExists: var bool)) =
   ## Computes a new mapping for the ``key`` with the specified ``mapper``
@@ -181,7 +185,7 @@ proc withKey*[A, B](t: var SharedTable[A, B], key: A,
     if pairExists:
       mapper(t.data[index].key, t.data[index].val, pairExists)
       if not pairExists:
-        delImplIdx(t, index)
+        delImplIdx(t, index, tabMakeEmpty, tabCellEmpty, tabCellHash)
     else:
       var val: B
       mapper(key, val, pairExists)
@@ -202,7 +206,7 @@ proc add*[A, B](t: var SharedTable[A, B], key: A, val: B) =
 proc del*[A, B](t: var SharedTable[A, B], key: A) =
   ## deletes `key` from hash table `t`.
   withLock t:
-    delImpl()
+    delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash)
 
 proc len*[A, B](t: var SharedTable[A, B]): int =
   ## number of elements in `t`

lib/pure/collections/tableimpl.nim

@@ -78,22 +78,52 @@ template hasKeyOrPutImpl(enlarge) {.dirty.} =
     maybeRehashPutImpl(enlarge)
   else: result = true
 
-template delImplIdx(t, i) =
+# delImplIdx is KnuthV3 Algo6.4R adapted to i=i+1 (from i=i-1) which has come to
+# be called "back shift delete".  It shifts elements in the collision cluster of
+# a victim backward to make things as-if the victim were never inserted in the
+# first place.  This is desirable to keep things "ageless" after many deletes.
+# It is trickier than you might guess since initial probe (aka "home") locations
+# of keys in a cluster may collide and since table addresses wrap around.
+#
+# A before-after diagram might look like ('.' means empty):
+#   slot:   0   1   2   3   4   5   6   7
+# before(1)
+#   hash1:  6   7   .   3   .   5   5   6  ; Really hash() and msk
+#   data1:  E   F   .   A   .   B   C   D  ; About to delete C @index 6
+# after(2)
+#   hash2:  7   .   .   3   .   5   6   6  ; Really hash() and msk
+#   data2:  F   .   .   A   .   B   D   E  ; After deletion of C
+#
+# This lowers total search depth over the whole table from 1+1+2+2+2+2=10 to 7.
+# Had the victim been B@5, C would need back shifting to slot 5.  Total depth is
+# always lowered by at least 1, e.g. victim A@3.  This is all quite fast when
+# empty slots are frequent (also needed to keep insert/miss searches fast) and
+# hash() is either fast or avoided (via `.hcode`).  It need not compare keys.
+#
+# delImplIdx realizes the above transformation, but only works for dense Linear
+# Probing, nextTry(h)=h+1.  This is not an important limitation since that's the
+# fastest sequence on any CPU made since the 1980s. { Performance analysis often
+# overweights "key cmp" neglecting cache behavior, giving bad ideas how big/slow
+# tables behave (when perf matters most!).  Comparing hcode first means usually
+# only 1 key cmp is needed for *any* seq.  Timing only predictable activity,
+# small tables, and/or integer keys often perpetuates such bad ideas. }
+
+template delImplIdx(t, i, makeEmpty, cellEmpty, cellHash) =
   let msk = maxHash(t)
   if i >= 0:
     dec(t.counter)
     block outer:
       while true:         # KnuthV3 Algo6.4R adapted for i=i+1 instead of i=i-1
-        var j = i         # The correctness of this depends on (h+1) in nextTry,
+        var j = i         # The correctness of this depends on (h+1) in nextTry
         var r = j         # though may be adaptable to other simple sequences.
-        t.data[i].hcode = 0              # mark current EMPTY
+        makeEmpty(i)                     # mark current EMPTY
         t.data[i].key = default(type(t.data[i].key))
         t.data[i].val = default(type(t.data[i].val))
         while true:
           i = (i + 1) and msk            # increment mod table size
-          if isEmpty(t.data[i].hcode):   # end of collision cluster; So all done
+          if cellEmpty(i):               # end of collision cluster; So all done
             break outer
-          r = t.data[i].hcode and msk    # "home" location of key@i
+          r = cellHash(i) and msk        # initial probe index for key@slot i
           if not ((i >= r and r > j) or (r > j and j > i) or (j > i and i >= r)):
             break
         when defined(js):
@@ -101,10 +131,19 @@ template delImplIdx(t, i) =
         else:
           t.data[j] = move(t.data[i]) # data[j] will be marked EMPTY next loop
 
-template delImpl() {.dirty.} =
+template delImpl(makeEmpty, cellEmpty, cellHash) {.dirty.} =
   var hc: Hash
   var i = rawGet(t, key, hc)
-  delImplIdx(t, i)
+  delImplIdx(t, i, makeEmpty, cellEmpty, cellHash)
+
+template delImplNoHCode(makeEmpty, cellEmpty, cellHash) {.dirty.} =
+  if t.dataLen > 0:
+    var i: Hash = hash(key) and maxHash(t)
+    while not cellEmpty(i):
+      if t.data[i].key == key:
+        delImplIdx(t, i, makeEmpty, cellEmpty, cellHash)
+        break
+      i = nextTry(i, maxHash(t))
 
 template clearImpl() {.dirty.} =
   for i in 0 ..< t.dataLen:

lib/pure/collections/tables.nim

@@ -499,6 +499,10 @@ proc add*[A, B](t: var Table[A, B], key: A, val: B) =
   ## (key, value) pair in the table without introducing duplicates.
   addImpl(enlarge)
 
+template tabMakeEmpty(i) = t.data[i].hcode = 0
+template tabCellEmpty(i) = isEmpty(t.data[i].hcode)
+template tabCellHash(i)  = t.data[i].hcode
+
 proc del*[A, B](t: var Table[A, B], key: A) =
   ## Deletes ``key`` from hash table ``t``. Does nothing if the key does not exist.
   ##
@@ -512,7 +516,7 @@ proc del*[A, B](t: var Table[A, B], key: A) =
     a.del('z')
     doAssert a == {'b': 9, 'c': 13}.toTable
 
-  delImpl()
+  delImpl(tabMakeEmpty, tabCellEmpty, tabCellHash)
 
 proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool =
   ## Deletes the ``key`` from the table.
@@ -540,7 +544,7 @@ proc pop*[A, B](t: var Table[A, B], key: A, val: var B): bool =
   result = index >= 0
   if result:
     val = move(t.data[index].val)
-    delImplIdx(t, index)
+    delImplIdx(t, index, tabMakeEmpty, tabCellEmpty, tabCellHash)
 
 proc take*[A, B](t: var Table[A, B], key: A, val: var B): bool {.inline.} =
   ## Alias for:
@@ -1804,7 +1808,7 @@ proc newOrderedTable*[A, B](pairs: openArray[(A, B)]): <//>OrderedTableRef[A, B]
     assert b == {'a': 5, 'b': 9}.newOrderedTable
 
   result = newOrderedTable[A, B](rightSize(pairs.len))
-  for key, val in items(pairs): result.add(key, val)
+  for key, val in items(pairs): result[key] = val
 
 
 proc `[]`*[A, B](t: OrderedTableRef[A, B], key: A): var B =
@@ -2218,19 +2222,6 @@ proc enlarge[A](t: var CountTable[A]) =
     if t.data[i].val != 0: ctRawInsert(t, n, move t.data[i].key, move t.data[i].val)
   swap(t.data, n)
 
-proc remove[A](t: var CountTable[A], key: A) =
-  var n: seq[tuple[key: A, val: int]]
-  newSeq(n, len(t.data))
-  var removed: bool
-  for i in countup(0, high(t.data)):
-    if t.data[i].val != 0:
-      if t.data[i].key != key:
-        ctRawInsert(t, n, move t.data[i].key, move t.data[i].val)
-      else:
-        removed = true
-  swap(t.data, n)
-  if removed: dec(t.counter)
-
 proc rawGet[A](t: CountTable[A], key: A): int =
   if t.data.len == 0:
     return -1
@@ -2244,7 +2235,7 @@ template ctget(t, key, default: untyped): untyped =
   var index = rawGet(t, key)
   result = if index >= 0: t.data[index].val else: default
 
-proc inc*[A](t: var CountTable[A], key: A, val: Positive = 1)
+proc inc*[A](t: var CountTable[A], key: A, val = 1)
 
 # ----------------------------------------------------------------------
 
@@ -2287,6 +2278,10 @@ proc `[]`*[A](t: CountTable[A], key: A): int =
   assert(not t.isSorted, "CountTable must not be used after sorting")
   ctget(t, key, 0)
 
+template cntMakeEmpty(i) = t.data[i].val = 0
+template cntCellEmpty(i) = t.data[i].val == 0
+template cntCellHash(i)  = hash(t.data[i].key)
+
 proc `[]=`*[A](t: var CountTable[A], key: A, val: int) =
   ## Inserts a ``(key, value)`` pair into ``t``.
   ##
@@ -2297,7 +2292,7 @@ proc `[]=`*[A](t: var CountTable[A], key: A, val: int) =
   assert(not t.isSorted, "CountTable must not be used after sorting")
   assert val >= 0
   if val == 0:
-    t.remove(key)
+    delImplNoHCode(cntMakeEmpty, cntCellEmpty, cntCellHash)
   else:
     let h = rawGet(t, key)
     if h >= 0:
@@ -2305,11 +2300,8 @@ proc `[]=`*[A](t: var CountTable[A], key: A, val: int) =
     else:
       insertImpl()
 
-proc inc*[A](t: var CountTable[A], key: A, val: Positive = 1) =
+proc inc*[A](t: var CountTable[A], key: A, val = 1) =
   ## Increments ``t[key]`` by ``val`` (default: 1).
-  ##
-  ## ``val`` must be a positive number. If you need to decrement a value,
-  ## use a regular ``Table`` instead.
   runnableExamples:
     var a = toCountTable("aab")
     a.inc('a')
@@ -2320,9 +2312,11 @@ proc inc*[A](t: var CountTable[A], key: A, val: Positive = 1) =
   var index = rawGet(t, key)
   if index >= 0:
     inc(t.data[index].val, val)
-    if t.data[index].val == 0: dec(t.counter)
+    if t.data[index].val == 0:
+      delImplIdx(t, index, cntMakeEmpty, cntCellEmpty, cntCellHash)
   else:
-    insertImpl()
+    if val != 0:
+      insertImpl()
 
 proc smallest*[A](t: CountTable[A]): tuple[key: A, val: int] =
   ## Returns the ``(key, value)`` pair with the smallest ``val``. Efficiency: O(n)
@@ -2383,8 +2377,6 @@ proc len*[A](t: CountTable[A]): int =
 proc del*[A](t: var CountTable[A], key: A) {.since: (1, 1).} =
   ## Deletes ``key`` from table ``t``. Does nothing if the key does not exist.
   ##
-  ## O(n) complexity.
-  ##
   ## See also:
   ## * `pop proc<#pop,CountTable[A],A,int>`_
   ## * `clear proc<#clear,CountTable[A]>`_ to empty the whole table
@@ -2397,7 +2389,7 @@ proc del*[A](t: var CountTable[A], key: A) {.since: (1, 1).} =
     a.del('c')
     assert a == toCountTable("aa")
 
-  remove(t, key)
+  delImplNoHCode(cntMakeEmpty, cntCellEmpty, cntCellHash)
 
 proc pop*[A](t: var CountTable[A], key: A, val: var int): bool {.since: (1, 1).} =
   ## Deletes the ``key`` from the table.
@@ -2405,8 +2397,6 @@ proc pop*[A](t: var CountTable[A], key: A, val: var int): bool {.since: (1, 1).}
   ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is
   ## unchanged.
   ##
-  ## O(n) complexity.
-  ##
   ## See also:
   ## * `del proc<#del,CountTable[A],A>`_
   ## * `clear proc<#clear,CountTable[A]>`_ to empty the whole table
@@ -2423,7 +2413,7 @@ proc pop*[A](t: var CountTable[A], key: A, val: var int): bool {.since: (1, 1).}
   result = index >= 0
   if result:
     val = move(t.data[index].val)
-    remove(t, key)
+    delImplIdx(t, index, cntMakeEmpty, cntCellEmpty, cntCellHash)
 
 proc clear*[A](t: var CountTable[A]) =
   ## Resets the table so that it is empty.
@@ -2644,7 +2634,7 @@ proc `[]`*[A](t: CountTableRef[A], key: A): int =
   ## See also:
   ## * `getOrDefault<#getOrDefault,CountTableRef[A],A,int>`_ to return
   ##   a custom value if the key doesn't exist
-  ## * `mget proc<#mget,CountTableRef[A],A>`_
+  ## * `inc proc<#inc,CountTableRef[A],A>`_ to inc even if missing
   ## * `[]= proc<#[]%3D,CountTableRef[A],A,int>`_ for inserting a new
   ##   (key, value) pair in the table
   ## * `hasKey proc<#hasKey,CountTableRef[A],A>`_ for checking if a key
@@ -2717,8 +2707,6 @@ proc len*[A](t: CountTableRef[A]): int =
 proc del*[A](t: CountTableRef[A], key: A) {.since: (1, 1).} =
   ## Deletes ``key`` from table ``t``. Does nothing if the key does not exist.
   ##
-  ## O(n) complexity.
-  ##
   ## See also:
   ## * `pop proc<#pop,CountTableRef[A],A,int>`_
   ## * `clear proc<#clear,CountTableRef[A]>`_ to empty the whole table
@@ -2730,8 +2718,6 @@ proc pop*[A](t: CountTableRef[A], key: A, val: var int): bool {.since: (1, 1).}
   ## mapping of the key. Otherwise, returns ``false``, and the ``val`` is
   ## unchanged.
   ##
-  ## O(n) complexity.
-  ##
   ## See also:
   ## * `del proc<#del,CountTableRef[A],A>`_
   ## * `clear proc<#clear,CountTableRef[A]>`_ to empty the whole table
@@ -3016,37 +3002,37 @@ when isMainModule:
   block: #5482
     var a = [("wrong?", "foo"), ("wrong?", "foo2")].newOrderedTable()
     var b = newOrderedTable[string, string](initialSize = 2)
-    b.add("wrong?", "foo")
-    b.add("wrong?", "foo2")
+    b["wrong?"] = "foo"
+    b["wrong?"] = "foo2"
     assert a == b
 
   block: #5482
     var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable()
     var b = newOrderedTable[string, string](initialSize = 2)
-    b.add("wrong?", "foo")
-    b.add("wrong?", "foo2")
+    b["wrong?"] = "foo"
+    b["wrong?"] = "foo2"
     assert a == b
 
   block: #5487
     var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable()
     var b = newOrderedTable[string, string]()         # notice, default size!
-    b.add("wrong?", "foo")
-    b.add("wrong?", "foo2")
+    b["wrong?"] = "foo"
+    b["wrong?"] = "foo2"
     assert a == b
 
   block: #5487
     var a = [("wrong?", "foo"), ("wrong?", "foo2")].newOrderedTable()
     var b = newOrderedTable[string, string]()         # notice, default size!
-    b.add("wrong?", "foo")
-    b.add("wrong?", "foo2")
+    b["wrong?"] = "foo"
+    b["wrong?"] = "foo2"
     assert a == b
 
   block:
     var a = {"wrong?": "foo", "wrong?": "foo2"}.newOrderedTable()
     var b = [("wrong?", "foo"), ("wrong?", "foo2")].newOrderedTable()
     var c = newOrderedTable[string, string]()         # notice, default size!
-    c.add("wrong?", "foo")
-    c.add("wrong?", "foo2")
+    c["wrong?"] = "foo"
+    c["wrong?"] = "foo2"
     assert a == b
     assert a == c
 

lib/pure/encodings.nim

@@ -290,11 +290,8 @@ else:
     importc: "iconv_open", importIconv.}
   proc iconvClose(c: EncodingConverter) {.
     importc: "iconv_close", importIconv.}
-  proc iconv(c: EncodingConverter, inbuf: var cstring, inbytesLeft: var int,
-             outbuf: var cstring, outbytesLeft: var int): int {.
-    importc: "iconv", importIconv.}
-  proc iconv(c: EncodingConverter, inbuf: pointer, inbytesLeft: pointer,
-             outbuf: var cstring, outbytesLeft: var int): int {.
+  proc iconv(c: EncodingConverter, inbuf: ptr cstring, inbytesLeft: ptr csize_t,
+             outbuf: ptr cstring, outbytesLeft: ptr csize_t): csize_t {.
     importc: "iconv", importIconv.}
 
 proc getCurrentEncoding*(uiApp = false): string =
@@ -428,14 +425,14 @@ when defined(windows):
 else:
   proc convert*(c: EncodingConverter, s: string): string =
     result = newString(s.len)
-    var inLen = len(s)
-    var outLen = len(result)
+    var inLen = csize_t len(s)
+    var outLen = csize_t len(result)
     var src = cstring(s)
     var dst = cstring(result)
-    var iconvres: int
+    var iconvres: csize_t
     while inLen > 0:
-      iconvres = iconv(c, src, inLen, dst, outLen)
-      if iconvres == -1:
+      iconvres = iconv(c, addr src, addr inLen, addr dst, addr outLen)
+      if iconvres == high(csize_t):
         var lerr = errno
         if lerr == EILSEQ or lerr == EINVAL:
           # unknown char, skip
@@ -446,24 +443,24 @@ else:
           dec(outLen)
         elif lerr == E2BIG:
           var offset = cast[int](dst) - cast[int](cstring(result))
-          setLen(result, len(result)+inLen*2+5)
+          setLen(result, len(result) + inLen.int * 2 + 5)
           # 5 is minimally one utf-8 char
           dst = cast[cstring](cast[int](cstring(result)) + offset)
-          outLen = len(result) - offset
+          outLen = csize_t(len(result) - offset)
         else:
           raiseOSError(lerr.OSErrorCode)
     # iconv has a buffer that needs flushing, specially if the last char is
     # not '\0'
-    discard iconv(c, nil, nil, dst, outLen)
-    if iconvres == cint(-1) and errno == E2BIG:
+    discard iconv(c, nil, nil, addr dst, addr outLen)
+    if iconvres == high(csize_t) and errno == E2BIG:
       var offset = cast[int](dst) - cast[int](cstring(result))
-      setLen(result, len(result)+inLen*2+5)
+      setLen(result, len(result) + inLen.int * 2 + 5)
       # 5 is minimally one utf-8 char
       dst = cast[cstring](cast[int](cstring(result)) + offset)
-      outLen = len(result) - offset
-      discard iconv(c, nil, nil, dst, outLen)
+      outLen = csize_t(len(result) - offset)
+      discard iconv(c, nil, nil, addr dst, addr outLen)
     # trim output buffer
-    setLen(result, len(result) - outLen)
+    setLen(result, len(result) - outLen.int)
 
 proc convert*(s: string, destEncoding = "UTF-8",
                          srcEncoding = "CP1252"): string =

lib/pure/httpclient.nim

@@ -241,7 +241,8 @@ proc contentLength*(response: Response | AsyncResponse): int =
   ##
   ## A ``ValueError`` exception will be raised if the value is not an integer.
   var contentLengthHeader = response.headers.getOrDefault("Content-Length")
-  return contentLengthHeader.parseInt()
+  result = contentLengthHeader.parseInt()
+  doAssert(result >= 0 and result <= high(int32))
 
 proc lastModified*(response: Response | AsyncResponse): DateTime =
   ## Retrieves the specified response's last modified time.
@@ -316,7 +317,7 @@ proc getDefaultSSL(): SslContext =
   result = defaultSslContext
   when defined(ssl):
     if result == nil:
-      defaultSslContext = newContext(verifyMode = CVerifyNone)
+      defaultSslContext = newContext(verifyMode = CVerifyPeer)
       result = defaultSslContext
       doAssert result != nil, "failure to initialize the SSL context"
 
@@ -1018,6 +1019,11 @@ proc request*(client: HttpClient | AsyncHttpClient, url: string,
   ##
   ## This procedure will follow redirects up to a maximum number of redirects
   ## specified in ``client.maxRedirects``.
+  ##
+  ## You need to make sure that the ``url`` doesn't contain any newline
+  ## characters. Failing to do so will raise ``AssertionDefect``.
+  doAssert(not url.contains({'\c', '\L'}), "url shouldn't contain any newline characters")
+
   result = await client.requestAux(url, httpMethod, body, headers, multipart)
 
   var lastURL = url
@@ -1134,15 +1140,15 @@ proc downloadFile*(client: HttpClient, url: string, filename: string) =
     client.getBody = true
   let resp = client.get(url)
 
+  if resp.code.is4xx or resp.code.is5xx:
+    raise newException(HttpRequestError, resp.status)
+
   client.bodyStream = newFileStream(filename, fmWrite)
   if client.bodyStream.isNil:
     fileError("Unable to open file")
   parseBody(client, resp.headers, resp.version)
   client.bodyStream.close()
 
-  if resp.code.is4xx or resp.code.is5xx:
-    raise newException(HttpRequestError, resp.status)
-
 proc downloadFile*(client: AsyncHttpClient, url: string,
                    filename: string): Future[void] =
   proc downloadFileEx(client: AsyncHttpClient,