bump NimVersion to 1.4.2

(cherry picked from commit c9a10bb9e4)

.builds/freebsd.yml

@@ -1,25 +1,19 @@
 # see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd
 image: freebsd/latest
-
-# packages:
-# - databases/sqlite3
-# - devel/boehm-gc-threaded
-# - devel/pcre
-# - devel/sdl20
-# - devel/sfml
-# - www/node
-# - devel/gmake
-# - devel/git
+packages:
+- databases/sqlite3
+- devel/boehm-gc-threaded
+- devel/pcre
+- devel/sdl20
+- devel/sfml
+- www/node
+- devel/gmake
 sources:
 - https://github.com/nim-lang/Nim
 environment:
   CC: /usr/bin/clang
 tasks:
 - setup: |
-    # workaround https://github.com/timotheecour/Nim/issues/76
-    sudo pkg update -q -f
-    sudo pkg install -y -q databases/sqlite3 devel/boehm-gc-threaded devel/pcre \
-      devel/sdl20 devel/sfml www/node devel/gmake devel/git
     cd Nim
     git clone --depth 1 -q https://github.com/nim-lang/csources.git
     gmake -C csources -j $(sysctl -n hw.ncpu)

.builds/openbsd_0.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "0_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_1.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "1_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_2.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "2_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.gitattributes

@@ -0,0 +1,6 @@
+# Avoids changelog conflicts by assuming additions-only, which is by far the common case.
+# In the rare case where branch b1 rebases against branch b2 and both branches
+# modified the same changelog entry, you'll end up with that changelog entry
+# duplicated, which is easily identifiable and fixable.
+/changelog.md merge=union
+

.github/ISSUE_TEMPLATE/bug_report.md

@@ -12,9 +12,14 @@ Function `echo` outputs the wrong string.
 ### Example
 ```nim
 echo "Hello World!"
+# This code should be a minimum reproducible example:
+# try to simplify and minimize as much as possible. If it's a compiler
+# issue, try to minimize further by removing any imports if possible.
 ```
 
 ### Current Output
+please check whether the problem still exists in git head before posting,
+see [rebuilding the compiler](https://nim-lang.github.io/Nim/intern.html#rebuilding-the-compiler).
 ```
 Hola mundo!
 ```
@@ -29,12 +34,18 @@ Hello World!
 * In file xyz there is a call that might be the cause of it.
 
 ### Additional Information
+If it's a regression, you can help us by identifying which version introduced
+the bug, see [Bisecting for regressions](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions),
+or at least try known past releases (eg `choosenim 1.2.0`).
+
+If it's a pre-existing compiler bug, see [Debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler)
+which should give more context on a compiler crash.
 
-* It was working in version a.b.c
 * Issue #abc is related, but different because of ...
 * This issue is blocking my project xyz
 
 ```
 $ nim -v
 Nim Compiler Version 0.1.2
+# make sure to include the git hash if not using a tagged release
 ```

.github/stale.yml

@@ -0,0 +1,69 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 365
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 30
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - ARC
+  - bounty
+  - Codegen
+  - Crash
+  - Generics
+  - High Priority
+  - Macros
+  - Next release
+  - Showstopper
+  - Static[T]
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: stale
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This pull request has been automatically marked as stale because it has not had
+  recent activity.
+  If you think it is still a valid PR, please rebase it on the latest devel;
+  otherwise it will be closed. Thank you for your contributions.
+
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 20
+
+# Limit to only `issues` or `pulls`
+only: pulls
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.github/workflows/ci.yml.disabled

@@ -85,11 +85,11 @@ jobs:
           curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
           7z x dist/mingw64.7z -odist
           7z x dist/dlls.zip -obin
-          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
-        run: echo "::add-path::${{ github.workspace }}/bin"
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
       - name: 'Build csources'
         shell: bash

.github/workflows/ci_docs.yml

@@ -1,22 +1,37 @@
 name: Nim Docs CI
 on:
   push:
-    # Run only on changes on these files
     paths:
-      - 'lib/**.nim'
+      - 'compiler/docgen.nim'
+      - 'compiler/renderverbatim.nim'
+      - 'config/nimdoc.cfg'
       - 'doc/**.rst'
       - 'doc/nimdoc.css'
+      - 'lib/**.nim'
+      - 'nimdoc/testproject/expected/testproject.html'
+      - 'tools/dochack/dochack.nim'
+      - 'tools/kochdocs.nim'
+      - '.github/workflows/ci_docs.yml'
+      - 'koch.nim'
+  pull_request:
+    # Run only on changes on these files.
+    paths:
+      - 'compiler/docgen.nim'
+      - 'compiler/renderverbatim.nim'
+      - 'config/nimdoc.cfg'
+      - 'doc/**.rst'
+      - 'doc/nimdoc.css'
+      - 'lib/**.nim'
+      - 'nimdoc/testproject/expected/testproject.html'
+      - 'tools/dochack/dochack.nim'
+      - 'tools/kochdocs.nim'
       - '.github/workflows/ci_docs.yml'
 
-  pull_request:
-    # Run only on changes on these files
-    paths:
-      - 'lib/**.nim'
-      - 'doc/**.rst'
-      - '.github/workflows/ci_docs.yml'
 
 jobs:
   build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
     strategy:
       fail-fast: false
       matrix:
@@ -49,11 +64,11 @@ jobs:
           curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
           7z x dist/mingw64.7z -odist
           7z x dist/dlls.zip -obin
-          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
-        run: echo "::add-path::${{ github.workspace }}/bin"
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
       - name: 'Get current csources version'
         id: csources-version
@@ -111,17 +126,14 @@ jobs:
         shell: bash
         run: ./koch boot -d:release
 
+      - name: 'Clone fusion'
+        shell: bash
+        run: ./koch fusion
+
       - name: 'Build documentation'
         shell: bash
         run: ./koch doc --git.commit:devel
 
-      - name: 'Prepare documentation for deployment'
-        if: |
-          github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
-          matrix.target == 'linux'
-        shell: bash
-        run: cp -f doc/html/{overview,index}.html
-
       - name: 'Publish documentation to Github Pages'
         if: |
           github.event_name == 'push' && github.ref == 'refs/heads/devel' &&

.github/workflows/ci_packages.yml

@@ -0,0 +1,85 @@
+name: Packages CI
+on: [push, pull_request]
+
+jobs:
+  build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-10.15]
+        cpu: [amd64]
+        pkg: [1, 2]
+    name: '${{ matrix.os }} (pkg: ${{ matrix.pkg }})'
+    runs-on: ${{ matrix.os }}
+    env:
+      NIM_TEST_PACKAGES: ${{ matrix.pkg }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install node.js 12.x'
+        uses: actions/setup-node@v1
+        with:
+          node-version: '12.x'
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -yq \
+              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
+              valgrind libc6-dbg libblas-dev xorg-dev
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install boehmgc make sfml gtk+3
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+      - name: 'Run CI'
+        shell: bash
+        run: ./koch runCI
+
+      - name: 'Show failed tests'
+        if: failure()
+        shell: bash
+        run: nim c -r tools/ci_testresults.nim

.github/workflows/ci_ssl.yml

@@ -12,6 +12,8 @@ on:
 
 jobs:
   build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
     strategy:
       fail-fast: false
       matrix:
@@ -47,11 +49,11 @@ jobs:
           curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
           7z x dist/mingw64.7z -odist
           7z x dist/dlls.zip -obin
-          echo "::add-path::${{ github.workspace }}/dist/mingw64/bin"
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
 
       - name: 'Add build binaries to PATH'
         shell: bash
-        run: echo "::add-path::${{ github.workspace }}/bin"
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
 
       - name: 'Build 1-stage compiler from csources'
         shell: bash

.gitignore

@@ -66,6 +66,7 @@ testament.db
 /tests/**/*.js
 /csources
 /dist/
+/lib/fusion
 
 # Private directories and files (IDEs)
 .*/
@@ -82,6 +83,8 @@ megatest.nim
 
 /outputExpected.txt
 /outputGotten.txt
+/t15148.txt
+/tests/vm/tfile_rw.txt
 
 /lib/pure/*.js
 
@@ -92,7 +95,8 @@ megatest.nim
 # ignore debug dirs generated by dsymutil on OSX
 *.dSYM
 
-nimdoc.out.css
-
 # for `nim c -r nimdoc/tester` etc; this can be in multiple places
 htmldocs
+
+## these are not needed anymore unless checkout old older versions
+nimdoc.out.css

azure-pipelines.yml

@@ -27,13 +27,19 @@ jobs:
         vmImage: 'macOS-10.15'
         CPU: amd64
         NIM_COMPILE_TO_CPP: true
-      Windows_amd64:
+      Windows_amd64_batch0_3:
         vmImage: 'windows-2019'
         CPU: amd64
-      Windows_amd64_pkg:
+        # see also: `NIM_TEST_PACKAGES`
+        NIM_TESTAMENT_BATCH: "0_3"
+      Windows_amd64_batch1_3:
         vmImage: 'windows-2019'
         CPU: amd64
-        NIM_TEST_PACKAGES: true
+        NIM_TESTAMENT_BATCH: "1_3"
+      Windows_amd64_batch2_3:
+        vmImage: 'windows-2019'
+        CPU: amd64
+        NIM_TESTAMENT_BATCH: "2_3"
 
   pool:
     vmImage: $(vmImage)
@@ -47,14 +53,15 @@ jobs:
       condition: eq(variables['Agent.OS'], 'Windows_NT')
 
     - checkout: self
+      fetchDepth: 1
 
-    - bash: git clone --depth 1 https://github.com/nim-lang/csources.git
-      displayName: 'Checkout csources'
+    - bash: git clone --depth 1 https://github.com/nim-lang/csources
+      displayName: 'Checkout Nim csources'
 
     - task: NodeTool@0
       inputs:
-        versionSpec: '8.x'
-      displayName: 'Install node.js 8.x'
+        versionSpec: '12.x'
+      displayName: 'Install node.js 12.x'
 
     - bash: |
         sudo apt-fast update -qq
@@ -67,19 +74,12 @@ jobs:
     - bash: |
         sudo dpkg --add-architecture i386
 
-        # Downgrade llvm, libgcc and libstdc++:
+        # Downgrade llvm:
         # - llvm has to be downgraded to have 32bit version installed for sfml.
-        # - libgcc and libstdc++ have to be downgraded as an optimization to
-        #   prevent the use of the toolchain ppa, which has a terrible download
-        #   speed.
         cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
-        Package: libllvm6.0 libgcc1 libstdc++6
+        Package: libllvm6.0
         Pin: origin "azure.archive.ubuntu.com"
         Pin-Priority: 1001
-
-        Package: *
-        Pin: release o=LP-PPA-ubuntu-toolchain-r-test
-        Pin-Priority: 100
         EOF
 
         sudo apt-fast update -qq
@@ -135,8 +135,18 @@ jobs:
         make -v
       displayName: 'System information'
 
+    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
+      displayName: 'Get csources version'
+
+    - task: Cache@2
+      inputs:
+        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+        path: csources/bin
+      displayName: 'Restore built csources'
+
     - bash: |
         ncpu=
+        ext=
         case '$(Agent.OS)' in
         'Linux')
           ncpu=$(nproc)
@@ -146,19 +156,25 @@ jobs:
           ;;
         'Windows_NT')
           ncpu=$NUMBER_OF_PROCESSORS
+          ext=.exe
           ;;
         esac
         [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
 
-        make -C csources -j $ncpu CC=gcc ucpu=$(CPU)
-      displayName: 'Build csources'
+        if [[ -x csources/bin/nim$ext ]]; then
+          echo "Found cached compiler, skipping build"
+        else
+          make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
+        fi
+
+        cp csources/bin/nim$ext bin
+      displayName: 'Build 1-stage compiler from csources'
 
     - bash: nim c koch
       displayName: 'Build koch'
 
       # set result to omit the "bash exited with error code '1'" message
-    - bash: |
-        ./koch runCI || echo '##vso[task.complete result=Failed]'
+    - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
       displayName: 'Run CI'
       env:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken)

bin/nim-gdb

@@ -5,17 +5,15 @@ set -e
 
 which nim > /dev/null || (echo "nim not in PATH"; exit 1)
 which gdb > /dev/null || (echo "gdb not in PATH"; exit 1)
+which readlink > /dev/null || (echo "readlink not in PATH."; exit 1)
 
-if [[ "$(uname -s)" == "Darwin" ]]; then
-  which greadlink > /dev/null || (echo "readlink not in PATH. Please install coreutils from homebrew."; exit 1)
-  READLINK=greadlink
+if [[ "$(uname -s)" == "Darwin" || "(uname -s)" == *"BSD" ]]; then
+  NIM_SYSROOT=$(dirname $(dirname $(readlink -f $(which nim))))
 else
-  which readlink > /dev/null || (echo "readlink not in PATH."; exit 1)
-  READLINK=readlink
+  NIM_SYSROOT=$(dirname $(dirname $(readlink -e $(which nim))))
 fi
 
 # Find out where the pretty printer Python module is
-NIM_SYSROOT=$(dirname $(dirname $($READLINK -e $(which nim))))
 GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"
 
 # Run GDB with the additional arguments that load the pretty printers

build_all.sh

@@ -1,7 +1,7 @@
 #! /bin/sh
 
 # build development version of the compiler; can be rerun safely.
-# arguments can be passed, eg `--os freebsd`
+# arguments can be passed, e.g. `--os freebsd`
 
 set -u # error on undefined variables
 set -e # exit on first error
@@ -24,7 +24,7 @@ build_nim_csources(){
   # avoid changing dir in case of failure
   (
     if [ $# -ne 0 ]; then
-      # some args were passed (eg: `--cpu i386`), need to call build.sh
+      # some args were passed (e.g.: `--cpu i386`), need to call build.sh
       build_nim_csources_via_script "$@"
     else
       # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s)

changelog.md

@@ -1,15 +1,36 @@
-# v1.4.0 - yyyy-mm-dd
+# v1.6.x - yyyy-mm-dd
 
 
 
 ## Standard library additions and changes
 
+- `prelude` now works with the JavaScript target.
+
+- Added `ioutils` module containing `duplicate` and `duplicateTo` to duplicate `FileHandle` using C function `dup` and `dup2`.
+
+- The JSON module can now handle integer literals and floating point literals of arbitrary length and precision.
+  Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are kept as string literals and
+  one can use external BigNum libraries to handle these. The `parseFloat` family of functions also has now optional
+  `rawIntegers` and `rawFloats` parameters that can be used to enforce that all integer or float literals remain
+  in the "raw" string form so that client code can easily treat small and large numbers uniformly.
+
+- Added `randState` template that exposes the default random number generator. Useful for library authors.
+
+- Added `asyncdispatch.activeDescriptors` that returns the number of currently
+  active async event handles/file descriptors
+
 
 ## Language changes
 
+- `nimscript` now handles `except Exception as e`
+
+- The `cstring` doesn't support `[]=` operator in JS backend.
+
+
 
 ## Compiler changes
 
 
+
 ## Tool changes
 

changelogs/changelog_1_2_0.md

@@ -2,7 +2,6 @@
 
 
 ## Standard library additions and changes
-
 - Added overloaded `strformat.fmt` macro that use specified characters as
   delimiter instead of '{' and '}'.
 - Added new procs in `tables.nim`: `OrderedTable.pop`, `CountTable.del`,
@@ -33,7 +32,7 @@
 - Added `expectIdent` to the `macros` module.
 - Added `os.isValidFilename` that returns `true` if `filename` argument is valid
   for cross-platform use.
-- Added `times.isLeapDay`
+- Added `times.isLeapDay`.
 - `base64` adds URL-Safe Base64, implements RFC-4648 Section-7.
 - Added a new module, `std / compilesettings` for querying the compiler about
   diverse configuration settings.
@@ -53,6 +52,8 @@
   was designed for it. For example:
 
 ```nim
+import std/with
+
 type
   Foo = object
     col, pos: string
@@ -113,8 +114,6 @@ echo f
   usage. The `$` operator can be used to obtain the string form of `HttpCode`
   for comparison if desired.
 - `std/oswalkdir` was buggy, it's now deprecated and reuses `std/os` procs.
-- `net.newContext` now performs SSL Certificate checking on Linux and OSX.
-  Define `nimDisableCertificateValidation` to disable it globally.
 - `os.walkDir` and `os.walkDirRec` now have new flag, `checkDir` (default: false).
   If it is set to true, it will throw if input dir is invalid instead of a noop
   (which is the default behaviour, as it was before this change),
@@ -128,8 +127,8 @@ echo f
   of children nodes.
 - `os.splitPath()` behavior synchronized with `os.splitFile()` to return "/"
   as the dir component of `/root_sub_dir` instead of the empty string.
-- The deprecated `lc` macro has been removed from `sugar`. It is now replaced with the
-  more powerful `collect` macro.
+- The deprecated `lc` macro has been removed from `sugar`. It is now replaced
+  with the more powerful `collect` macro.
 - `os.relativePath("foo", "foo")` is now `"."`, not `""`, as `""` means invalid
   path and shouldn't be conflated with `"."`; use `-d:nimOldRelativePathBehavior`
   to restore the old behavior.
@@ -138,25 +137,33 @@ echo f
   Use `base64.encodeMime` instead.
 
 
+### Breaking changes
+
+- `net.newContext` now performs SSL Certificate checking on Linux and OSX.
+  Define `nimDisableCertificateValidation` to disable it globally.
+
+
+
 ## Language changes
 
 - An `align` pragma can now be used for variables and object fields, similar
   to the `alignas` declaration modifier in C/C++.
-- `=sink` type bound operator is now optional. Compiler can now use combination
-  of `=destroy` and `copyMem` to move objects efficiently.
+- The `=sink` type bound operator is now optional. The compiler can now use a
+  combination of `=destroy` and `copyMem` to move objects efficiently.
 - Unsigned integer operators have been fixed to allow promotion of the first operand.
 - Conversions to unsigned integers are unchecked at runtime, imitating earlier Nim
   versions. The documentation was improved to acknowledge this special case.
   See https://github.com/nim-lang/RFCs/issues/175 for more details.
-- New syntax for lvalue references: `var b {.byaddr.} = expr` enabled by
+- There is a new syntax for lvalue references: `var b {.byaddr.} = expr` enabled by
   `import std/decls`.
 - `var a {.foo.}: MyType = expr` now lowers to `foo(a, MyType, expr)` for
   non-builtin pragmas, enabling things like lvalue references (see `decls.byaddr`).
 
 
+
 ## Compiler changes
 
-- Generated JS code uses spaces, instead of mixed spaces and tabs.
+- The generated JS code uses spaces, instead of mixing spaces and tabs.
 - The Nim compiler now supports the ``--asm`` command option for easier
   inspection of the produced assembler code.
 - The Nim compiler now supports a new pragma called ``.localPassc`` to
@@ -192,10 +199,6 @@ echo f
   with `-d:nimEmulateOverflowChecks`.
 
 
-## Tool changes
-
-- Nimpretty doesn't accept negative indentation argument anymore, because it was
-  breaking files.
 
 
 ## Bugfixes

changelogs/changelog_1_4_0.md

@@ -0,0 +1,874 @@
+# v1.4.0 - 2020-10-16
+
+
+
+## Standard library additions and changes
+
+- Added some enhancements to `std/jsonutils` module.
+  * Added a possibility to deserialize JSON arrays directly to `HashSet` and
+    `OrderedSet` types and respectively to serialize those types to JSON arrays
+    via `jsonutils.fromJson` and `jsonutils.toJson` procedures.
+  * Added a possibility to deserialize JSON `null` objects to Nim option objects
+    and respectively to serialize Nim option object to JSON object if `isSome`
+    or to JSON null object if `isNone` via `jsonutils.fromJson` and
+    `jsonutils.toJson` procedures.
+  * Added a `Joptions` parameter to `jsonutils.fromJson` currently
+    containing two boolean options `allowExtraKeys` and `allowMissingKeys`.
+    - If `allowExtraKeys` is `true` Nim's object to which the JSON is parsed is
+      not required to have a field for every JSON key.
+    - If `allowMissingKeys` is `true` Nim's object to which JSON is parsed is
+      allowed to have fields without corresponding JSON keys.
+- Added `bindParams`, `bindParam` to `db_sqlite` for binding parameters into a `SqlPrepared` statement.
+- Added `tryInsert`,`insert` procs to `db_*` libs which accept primary key column name.
+- Added `xmltree.newVerbatimText` support create `style`'s,`script`'s text.
+- `uri` module now implements RFC-2397.
+- Added [DOM Parser](https://developer.mozilla.org/en-US/docs/Web/API/DOMParser)
+  to the `dom` module for the JavaScript target.
+- The default hash for `Ordinal` has changed to something more bit-scrambling.
+  `import hashes; proc hash(x: myInt): Hash = hashIdentity(x)` recovers the old
+  one in an instantiation context while `-d:nimIntHash1` recovers it globally.
+- `deques.peekFirst` and `deques.peekLast` now have `var Deque[T] -> var T` overloads.
+- File handles created from high-level abstractions in the stdlib will no longer
+  be inherited by child processes. In particular, these modules are affected:
+  `asyncdispatch`, `asyncnet`, `system`, `nativesockets`, `net` and `selectors`.
+
+  For `asyncdispatch`, `asyncnet`, `net` and `nativesockets`, an `inheritable`
+  flag has been added to all `proc`s that create sockets, allowing the user to
+  control whether the resulting socket is inheritable. This flag is provided to
+  ease the writing of multi-process servers, where sockets inheritance is
+  desired.
+
+  For a transition period, define `nimInheritHandles` to enable file handle
+  inheritance by default. This flag does **not** affect the `selectors` module
+  due to the differing semantics between operating systems.
+
+  `asyncdispatch.setInheritable`, `system.setInheritable` and
+  `nativesockets.setInheritable` are also introduced for setting file handle or
+  socket inheritance. Not all platforms have these `proc`s defined.
+
+- The file descriptors created for internal bookkeeping by `ioselector_kqueue`
+  and `ioselector_epoll` will no longer be leaked to child processes.
+
+- `strutils.formatFloat` with `precision = 0` has been restored to the version
+  1 behaviour that produces a trailing dot, e.g. `formatFloat(3.14159, precision = 0)`
+  is now `3.`, not `3`.
+- Added `commonPrefixLen` to `critbits`.
+
+- `relativePath(rel, abs)` and `relativePath(abs, rel)` used to silently give wrong results
+  (see #13222); instead they now use `getCurrentDir` to resolve those cases,
+  and this can now throw in edge cases where `getCurrentDir` throws.
+  `relativePath` also now works for js with `-d:nodejs`.
+
+- JavaScript and NimScript standard library changes: `streams.StringStream` is
+  now supported in JavaScript, with the limitation that any buffer `pointer`s
+  used must be castable to `ptr string`, any incompatible pointer type will not
+  work. The `lexbase` and `streams` modules used to fail to compile on
+  NimScript due to a bug, but this has been fixed.
+
+  The following modules now compile on both JS and NimScript: `parsecsv`,
+  `parsecfg`, `parsesql`, `xmlparser`, `htmlparser` and `ropes`. Additionally
+  supported for JS is `cstrutils.startsWith` and `cstrutils.endsWith`, for
+  NimScript: `json`, `parsejson`, `strtabs` and `unidecode`.
+
+- Added `streams.readStr` and `streams.peekStr` overloads to
+  accept an existing string to modify, which avoids memory
+  allocations, similar to `streams.readLine` (#13857).
+
+- Added high-level `asyncnet.sendTo` and `asyncnet.recvFrom` UDP functionality.
+
+- `dollars.$` now works for unsigned ints with `nim js`.
+
+- Improvements to the `bitops` module, including bitslices, non-mutating versions
+  of the original masking functions, `mask`/`masked`, and varargs support for
+  `bitand`, `bitor`, and `bitxor`.
+
+- `sugar.=>` and `sugar.->` changes: Previously `(x, y: int)` was transformed
+  into `(x: auto, y: int)`, it now becomes `(x: int, y: int)` for consistency
+  with regular proc definitions (although you cannot use semicolons).
+
+  Pragmas and using a name are now allowed on the lefthand side of `=>`. Here
+  is an example of these changes:
+  ```nim
+  import sugar
+
+  foo(x, y: int) {.noSideEffect.} => x + y
+
+  # is transformed into
+
+  proc foo(x: int, y: int): auto {.noSideEffect.} = x + y
+  ```
+
+- The fields of `times.DateTime` are now private, and are accessed with getters and deprecated setters.
+
+- The `times` module now handles the default value for `DateTime` more consistently.
+  Most procs raise an assertion error when given
+  an uninitialized `DateTime`, the exceptions are `==` and `$` (which returns `"Uninitialized DateTime"`).
+  The proc `times.isInitialized` has been added which can be used to check if
+  a `DateTime` has been initialized.
+
+- Fix a bug where calling `close` on io streams in `osproc.startProcess` was a noop and led to
+  hangs if a process had both reads from stdin and writes (e.g. to stdout).
+
+- The callback that is passed to `system.onThreadDestruction` must now be `.raises: []`.
+- The callback that is assigned to `system.onUnhandledException` must now be `.gcsafe`.
+
+- `osproc.execCmdEx` now takes an optional `input` for stdin, `workingDir` and `env`
+  parameters.
+
+- Added a `ssl_config` module containing lists of secure ciphers as recommended by
+  [Mozilla OpSec](https://wiki.mozilla.org/Security/Server_Side_TLS)
+
+- `net.newContext` now defaults to the list of ciphers targeting
+  ["Intermediate compatibility"](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29)
+  per Mozilla's recommendation instead of `ALL`. This change should protect
+  users from the use of weak and insecure ciphers while still provides
+  adequate compatibility with the majority of the Internet.
+
+- A new module `std/jsonutils` with hookable `jsonTo,toJson,fromJson` operations for json
+  serialization/deserialization of custom types was added.
+
+- A new proc `heapqueue.find[T](heap: HeapQueue[T], x: T): int` to get index of element ``x``
+  was added.
+- Added `rstgen.rstToLatex` a convenience proc for `renderRstToOut` and `initRstGenerator`.
+- Added `os.normalizeExe`.
+- `macros.newLit` now preserves named vs unnamed tuples.
+- Added `random.gauss`, that uses the ratio of uniforms method of sampling from a Gaussian distribution.
+- Added `typetraits.elementType` to get the element type of an iterable.
+- `typetraits.$` changes: `$(int,)` is now `"(int,)"` instead of `"(int)"`;
+  `$tuple[]` is now `"tuple[]"` instead of `"tuple"`;
+  `$((int, float), int)` is now `"((int, float), int)"` instead of `"(tuple of (int, float), int)"`
+- Added `macros.extractDocCommentsAndRunnables` helper.
+
+- `strformat.fmt` and `strformat.&` support `specifier =`. `fmt"{expr=}"` now
+  expands to `fmt"expr={expr}"`.
+- Deprecations: instead of `os.existsDir` use `dirExists`, instead of `os.existsFile` use `fileExists`.
+
+- Added the `jsre` module, [Regular Expressions for the JavaScript target.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions).
+- Made `maxLines` argument `Positive` in `logging.newRollingFileLogger`,
+  because negative values will result in a new file being created for each logged
+  line which doesn't make sense.
+- Changed `log` in `logging` to use proper log level for JavaScript,
+  e.g. `debug` uses `console.debug`, `info` uses `console.info`, `warn` uses `console.warn`, etc.
+- Tables, HashSets, SharedTables and deques don't require anymore that the passed
+  initial size must be a power of two - this is done internally.
+  Proc `rightSize` for Tables and HashSets is deprecated, as it is not needed anymore.
+  `CountTable.inc` takes `val: int` again not `val: Positive`; i.e. it can "count down" again.
+- Removed deprecated symbols from `macros` module, some of which were deprecated already in `0.15`.
+- Removed `sugar.distinctBase`, deprecated since `0.19`. Use `typetraits.distinctBase`.
+- `asyncdispatch.PDispatcher.handles` is exported so that an external low-level libraries can access it.
+
+- `std/with`, `sugar.dup` now support object field assignment expressions:
+  ```nim
+  import std/with
+
+  type Foo = object
+    x, y: int
+
+  var foo = Foo()
+  with foo:
+    x = 10
+    y = 20
+
+  echo foo
+  ```
+
+- Proc `math.round` is no longer deprecated. The advice to use `strformat` instead
+  cannot be applied to every use case. The limitations and the (lack of) reliability
+  of `round` are well documented.
+
+- Added `getprotobyname` to `winlean`. Added `getProtoByname` to `nativesockets` which returns a protocol code
+  from the database that matches the protocol `name`.
+
+- Added missing attributes and methods to `dom.Navigator` like `deviceMemory`, `onLine`, `vibrate()`, etc.
+
+- Added `strutils.indentation` and `strutils.dedent` which enable indented string literals:
+  ```nim
+  import strutils
+  echo dedent """
+    This
+      is
+        cool!
+    """
+  ```
+
+- Added `initUri(isIpv6: bool)` to `uri` module, now `uri` supports parsing ipv6 hostname.
+
+- Added `readLines(p: Process)` to `osproc`.
+
+- Added the below `toX` procs for collections. The usage is similar to procs such as
+  `sets.toHashSet` and `tables.toTable`. Previously, it was necessary to create the
+  respective empty collection and add items manually.
+    * `critbits.toCritBitTree`, which creates a `CritBitTree` from an `openArray` of
+       items or an `openArray` of pairs.
+    * `deques.toDeque`, which creates a `Deque` from an `openArray`.
+    * `heapqueue.toHeapQueue`, which creates a `HeapQueue` from an `openArray`.
+    * `intsets.toIntSet`, which creates an `IntSet` from an `openArray`.
+
+- Added `progressInterval` argument to `asyncftpclient.newAsyncFtpClient` to control the interval
+  at which progress callbacks are called.
+
+- Added `os.copyFileToDir`.
+
+## Language changes
+
+- The `=destroy` hook no longer has to reset its target, as the compiler now automatically inserts
+  `wasMoved` calls where needed.
+- The `=` hook is now called `=copy` for clarity. The old name `=` is still available so there
+  is no need to update your code. This change was backported to 1.2 too so you can use the
+  more readable `=copy` without loss of compatibility.
+
+- In the newruntime it is now allowed to assign to the discriminator field
+  without restrictions as long as the case object doesn't have a custom destructor.
+  The discriminator value doesn't have to be a constant either. If you have a
+  custom destructor for a case object and you do want to freely assign discriminator
+  fields, it is recommended to refactor the object into 2 objects like this:
+
+  ```nim
+  type
+    MyObj = object
+      case kind: bool
+      of true: y: ptr UncheckedArray[float]
+      of false: z: seq[int]
+
+  proc `=destroy`(x: MyObj) =
+    if x.kind and x.y != nil:
+      deallocShared(x.y)
+  ```
+  Refactor into:
+  ```nim
+  type
+    MySubObj = object
+      val: ptr UncheckedArray[float]
+    MyObj = object
+      case kind: bool
+      of true: y: MySubObj
+      of false: z: seq[int]
+
+  proc `=destroy`(x: MySubObj) =
+    if x.val != nil:
+      deallocShared(x.val)
+  ```
+- `getImpl` on enum type symbols now returns field syms instead of idents. This helps
+  with writing typed macros. The old behavior for backwards compatibility can be restored
+  with `--useVersion:1.0`.
+- The typed AST for proc headers will now have the arguments be syms instead of idents.
+  This helps with writing typed macros. The old behaviour for backwards compatibility can
+  be restored with `--useVersion:1.0`.
+- ``let`` statements can now be used without a value if declared with
+  ``importc``/``importcpp``/``importjs``/``importobjc``.
+- The keyword `from` is now usable as an operator.
+- Exceptions inheriting from `system.Defect` are no longer tracked with
+  the `.raises: []` exception tracking mechanism. This is more consistent with the
+  built-in operations. The following always used to compile (and still does):
+  ```nim
+  proc mydiv(a, b): int {.raises: [].} =
+    a div b # can raise an DivByZeroDefect
+  ```
+
+  Now also this compiles:
+  ```nim
+  proc mydiv(a, b): int {.raises: [].} =
+    if b == 0: raise newException(DivByZeroDefect, "division by zero")
+    else: result = a div b
+  ```
+
+  The reason for this is that `DivByZeroDefect` inherits from `Defect` and
+  with `--panics:on` `Defects` become unrecoverable errors.
+
+- Added the `thiscall` calling convention as specified by Microsoft, mostly for hooking purposes.
+- Deprecated the `{.unroll.}` pragma, because it was always ignored by the compiler anyway.
+- Removed the deprecated `strutils.isNilOrWhitespace`.
+- Removed the deprecated `sharedtables.initSharedTable`.
+- Removed the deprecated `asyncdispatch.newAsyncNativeSocket`.
+- Removed the deprecated `dom.releaseEvents` and `dom.captureEvents`.
+
+- Removed `sharedlists.initSharedList`, was deprecated and produces undefined behaviour.
+
+- There is a new experimental feature called "strictFuncs" which makes the definition of
+  `.noSideEffect` stricter. [See here](manual_experimental.html#stricts-funcs)
+  for more information.
+
+- "for-loop macros" (see [the manual](manual.html#macros-for-loop-macros)) are no longer
+  an experimental feature. In other words, you don't have to write pragma
+  `{.experimental: "forLoopMacros".}` if you want to use them.
+
+- Added the ``.noalias`` pragma. It is mapped to C's ``restrict`` keyword for the increased
+  performance this keyword can enable.
+
+- `items` no longer compiles with enums with holes as its behavior was error prone, see #14004.
+- `system.deepcopy` has to be enabled explicitly for `--gc:arc` and `--gc:orc` via
+  `--deepcopy:on`.
+
+- Added the `std/effecttraits` module for introspection of the inferred effects.
+  We hope this enables `async` macros that are precise about the possible exceptions that
+  can be raised.
+- The pragma blocks `{.gcsafe.}: ...` and `{.noSideEffect.}: ...` can now also be
+  written as `{.cast(gcsafe).}: ...` and `{.cast(noSideEffect).}: ...`. This is the new
+  preferred way of writing these, emphasizing their unsafe nature.
+
+
+## Compiler changes
+
+- Specific warnings can now be turned into errors via `--warningAsError[X]:on|off`.
+- The `define` and `undef` pragmas have been de-deprecated.
+- New command: `nim r main.nim [args...]` which compiles and runs main.nim, and implies `--usenimcache`
+  so that the output is saved to $nimcache/main$exeExt, using the same logic as `nim c -r` to
+  avoid recompilations when sources don't change.
+  Example:
+  ```bash
+  nim r compiler/nim.nim --help # only compiled the first time
+  echo 'import os; echo getCurrentCompilerExe()' | nim r - # this works too
+  nim r compiler/nim.nim --fullhelp # no recompilation
+  nim r --nimcache:/tmp main # binary saved to /tmp/main
+  ```
+- `--hint:processing` is now supported and means `--hint:processing:on`
+  (likewise with other hints and warnings), which is consistent with all other bool flags.
+  (since 1.3.3).
+- `nim doc -r main` and `nim rst2html -r main` now call `openDefaultBrowser`.
+- Added the new hint `--hint:msgOrigin` will show where a compiler msg (hint|warning|error)
+  was generated; this helps in particular when it's non obvious where it came from
+  either because multiple locations generate the same message, or because the
+  message involves runtime formatting.
+- Added the new flag `--backend:js|c|cpp|objc` (or -b:js etc), to change the backend; can be
+  used with any command (e.g. nim r, doc, check etc); safe to re-assign.
+- Added the new flag `--doccmd:cmd` to pass additional flags for runnableExamples,
+  e.g.: `--doccmd:-d:foo --threads`
+  use `--doccmd:skip` to skip runnableExamples and rst test snippets.
+- Added the new flag `--usenimcache` to output binary files to nimcache.
+- `runnableExamples "-b:cpp -r:off": code` is now supported, allowing to override
+  how an example is compiled and run, for example to change the backend.
+- `nim doc` now outputs under `$projectPath/htmldocs` when `--outdir` is unspecified
+  (with or without `--project`); passing `--project` now automatically generates
+  an index and enables search.
+  See [docgen](docgen.html#introduction-quick-start) for details.
+- Removed the `--oldNewlines` switch.
+- Removed the `--laxStrings` switch for mutating the internal zero terminator on strings.
+- Removed the `--oldast` switch.
+- Removed the `--oldgensym` switch.
+- `$getType(untyped)` is now "untyped" instead of "expr", `$getType(typed)` is
+  now "typed" instead of "stmt".
+- Sink inference is now disabled per default and has to enabled explicitly via
+  `--sinkInference:on`. *Note*: For the standard library sink inference remains
+  enabled. This change is most relevant for the `--gc:arc`, `--gc:orc` memory
+  management modes.
+
+
+## Tool changes
+
+- `nimsuggest` now returns both the forward declaration and the
+  implementation location upon a `def` query. Previously the behavior was
+  to return the forward declaration only.
+
+
+## Bugfixes
+
+- Fixed "repr() not available for uint{,8,16,32,64} under --gc:arc"
+  ([#13872](https://github.com/nim-lang/Nim/issues/13872))
+- Fixed "Critical: 1 completed Future, multiple await: Only 1 await will be awakened (the last one)"
+  ([#13889](https://github.com/nim-lang/Nim/issues/13889))
+- Fixed "crash on openarray interator with argument in stmtListExpr"
+  ([#13739](https://github.com/nim-lang/Nim/issues/13739))
+- Fixed "Some compilers on Windows don't work"
+  ([#13910](https://github.com/nim-lang/Nim/issues/13910))
+- Fixed "httpclient hangs if it recieves an HTTP 204 (No Content)"
+  ([#13894](https://github.com/nim-lang/Nim/issues/13894))
+- Fixed ""distinct uint64" type corruption on 32-bit, when using {.borrow.} operators"
+  ([#13902](https://github.com/nim-lang/Nim/issues/13902))
+- Fixed "Regression: impossible to use typed pragmas with proc types"
+  ([#13909](https://github.com/nim-lang/Nim/issues/13909))
+- Fixed "openssl wrapper corrupts stack on OpenSSL 1.1.1f + Android"
+  ([#13903](https://github.com/nim-lang/Nim/issues/13903))
+- Fixed "C compile error with --gc:arc on version 1.2.0 "unknown type name 'TGenericSeq'"
+  ([#13863](https://github.com/nim-lang/Nim/issues/13863))
+- Fixed "var return type for proc doesn't work at c++ backend"
+  ([#13848](https://github.com/nim-lang/Nim/issues/13848))
+- Fixed "TimeFormat() should raise an error but craches at compilation time"
+  ([#12864](https://github.com/nim-lang/Nim/issues/12864))
+- Fixed "gc:arc cannot fully support threadpool with FlowVar"
+  ([#13781](https://github.com/nim-lang/Nim/issues/13781))
+- Fixed "simple 'var openarray[char]' assignment crash when the openarray source is a local string and using gc:arc"
+  ([#14003](https://github.com/nim-lang/Nim/issues/14003))
+- Fixed "Cant use expressions with `when` in `type` sections."
+  ([#14007](https://github.com/nim-lang/Nim/issues/14007))
+- Fixed "`for a in MyEnum` gives incorrect results with enum with holes"
+  ([#14001](https://github.com/nim-lang/Nim/issues/14001))
+- Fixed "Trivial crash"
+  ([#12741](https://github.com/nim-lang/Nim/issues/12741))
+- Fixed "Enum with holes cannot be used as Table index"
+  ([#12834](https://github.com/nim-lang/Nim/issues/12834))
+- Fixed "spawn proc that uses typedesc crashes the compiler"
+  ([#14014](https://github.com/nim-lang/Nim/issues/14014))
+- Fixed "Docs Search `Results` box styling is not Dark Mode Friendly"
+  ([#13972](https://github.com/nim-lang/Nim/issues/13972))
+- Fixed "--gc:arc -d:useSysAssert undeclared identifier `cstderr` with newSeq"
+  ([#14038](https://github.com/nim-lang/Nim/issues/14038))
+- Fixed "issues in the manual"
+  ([#12486](https://github.com/nim-lang/Nim/issues/12486))
+- Fixed "Annoying warning: inherit from a more precise exception type like ValueError, IOError or OSError [InheritFromException]"
+  ([#14052](https://github.com/nim-lang/Nim/issues/14052))
+- Fixed "relativePath("foo", "/") and relativePath("/", "foo") is wrong"
+  ([#13222](https://github.com/nim-lang/Nim/issues/13222))
+- Fixed "[regression] `parseEnum` does not work anymore for enums with holes"
+  ([#14030](https://github.com/nim-lang/Nim/issues/14030))
+- Fixed "Exception types in the stdlib should inherit from `CatchableError` or `Defect`, not `Exception`"
+  ([#10288](https://github.com/nim-lang/Nim/issues/10288))
+- Fixed "Make debugSend and debugRecv procs public in smtp.nim"
+  ([#12189](https://github.com/nim-lang/Nim/issues/12189))
+- Fixed "xmltree need add raw text, when add style element"
+  ([#14064](https://github.com/nim-lang/Nim/issues/14064))
+- Fixed "raises requirement does not propagate to derived methods"
+  ([#8481](https://github.com/nim-lang/Nim/issues/8481))
+- Fixed "tests/stdlib/tgetaddrinfo.nim fails on NetBSD"
+  ([#14091](https://github.com/nim-lang/Nim/issues/14091))
+- Fixed "tests/niminaction/Chapter8/sdl/sdl_test.nim fails on NetBSD"
+  ([#14088](https://github.com/nim-lang/Nim/issues/14088))
+- Fixed "Incorrect escape sequence for example in jsffi library documentation"
+  ([#14110](https://github.com/nim-lang/Nim/issues/14110))
+- Fixed "HCR: Can not link exported const, in external library"
+  ([#13915](https://github.com/nim-lang/Nim/issues/13915))
+- Fixed "Cannot import std/unidecode"
+  ([#14112](https://github.com/nim-lang/Nim/issues/14112))
+- Fixed "macOS: dsymutil should not be called on static libraries"
+  ([#14132](https://github.com/nim-lang/Nim/issues/14132))
+- Fixed "nim jsondoc -o:doc.json filename.nim fails when sequences without a type are used"
+  ([#14066](https://github.com/nim-lang/Nim/issues/14066))
+- Fixed "algorithm.sortedByIt template corrupts tuple input under --gc:arc"
+  ([#14079](https://github.com/nim-lang/Nim/issues/14079))
+- Fixed "Invalid C code with lvalue conversion"
+  ([#14160](https://github.com/nim-lang/Nim/issues/14160))
+- Fixed "strformat: doc example fails"
+  ([#14054](https://github.com/nim-lang/Nim/issues/14054))
+- Fixed "Nim doc fail to run for nim 1.2.0 (nim 1.0.4 is ok)"
+  ([#13986](https://github.com/nim-lang/Nim/issues/13986))
+- Fixed "Exception when converting csize to clong"
+  ([#13698](https://github.com/nim-lang/Nim/issues/13698))
+- Fixed "[Documentation] overloading using named arguments works but is not documented"
+  ([#11932](https://github.com/nim-lang/Nim/issues/11932))
+- Fixed "import os + use of existsDir/dirExists/existsFile/fileExists/findExe in config.nims causes "ambiguous call' error"
+  ([#14142](https://github.com/nim-lang/Nim/issues/14142))
+- Fixed "import os + use of existsDir/dirExists/existsFile/fileExists/findExe in config.nims causes "ambiguous call' error"
+  ([#14142](https://github.com/nim-lang/Nim/issues/14142))
+- Fixed "runnableExamples doc gen crashes compiler with `except Exception as e` syntax"
+  ([#14177](https://github.com/nim-lang/Nim/issues/14177))
+- Fixed "[ARC] Segfault with cyclic references (?)"
+  ([#14159](https://github.com/nim-lang/Nim/issues/14159))
+- Fixed "Semcheck regression when accessing a static parameter in proc"
+  ([#14136](https://github.com/nim-lang/Nim/issues/14136))
+- Fixed "iterator walkDir doesn't work with -d:useWinAnsi"
+  ([#14201](https://github.com/nim-lang/Nim/issues/14201))
+- Fixed "cas is wrong for tcc"
+  ([#14151](https://github.com/nim-lang/Nim/issues/14151))
+- Fixed "proc execCmdEx doesn't work with -d:useWinAnsi"
+  ([#14203](https://github.com/nim-lang/Nim/issues/14203))
+- Fixed "Use -d:nimEmulateOverflowChecks by default?"
+  ([#14209](https://github.com/nim-lang/Nim/issues/14209))
+- Fixed "Old sequences with destructor objects bug"
+  ([#14217](https://github.com/nim-lang/Nim/issues/14217))
+- Fixed "[ARC] ICE when changing the discriminant of a return value"
+  ([#14244](https://github.com/nim-lang/Nim/issues/14244))
+- Fixed "[ARC] ICE with static objects"
+  ([#14236](https://github.com/nim-lang/Nim/issues/14236))
+- Fixed "[ARC] "internal error: environment misses: a" in a finalizer"
+  ([#14243](https://github.com/nim-lang/Nim/issues/14243))
+- Fixed "[ARC] compile failure using repr with object containing `ref seq[string]`"
+  ([#14270](https://github.com/nim-lang/Nim/issues/14270))
+- Fixed "[ARC] implicit move on last use happening on non-last use"
+  ([#14269](https://github.com/nim-lang/Nim/issues/14269))
+- Fixed "[ARC] Compiler crash with a recursive non-ref object variant"
+  ([#14294](https://github.com/nim-lang/Nim/issues/14294))
+- Fixed "htmlparser.parseHtml behaves differently using --gc:arc or --gc:orc"
+  ([#13946](https://github.com/nim-lang/Nim/issues/13946))
+- Fixed "Invalid return value of openProcess is NULL rather than INVALID_HANDLE_VALUE(-1) in windows"
+  ([#14289](https://github.com/nim-lang/Nim/issues/14289))
+- Fixed "ARC codegen bug with inline iterators"
+  ([#14219](https://github.com/nim-lang/Nim/issues/14219))
+- Fixed "Building koch on OpenBSD fails unless the Nim directory is in `$PATH`"
+  ([#13758](https://github.com/nim-lang/Nim/issues/13758))
+- Fixed "[gc:arc] case object assignment SIGSEGV: destroy not called for primitive type "
+  ([#14312](https://github.com/nim-lang/Nim/issues/14312))
+- Fixed "Crash when using thread and --gc:arc "
+  ([#13881](https://github.com/nim-lang/Nim/issues/13881))
+- Fixed "Getting "Warning: Cannot prove that 'result' is initialized" for an importcpp'd proc with `var T` return type"
+  ([#14314](https://github.com/nim-lang/Nim/issues/14314))
+- Fixed "`nim cpp -r --gc:arc` segfaults on caught AssertionError"
+  ([#13071](https://github.com/nim-lang/Nim/issues/13071))
+- Fixed "tests/async/tasyncawait.nim is recently very flaky"
+  ([#14320](https://github.com/nim-lang/Nim/issues/14320))
+- Fixed "Documentation nonexistent quitprocs module"
+  ([#14331](https://github.com/nim-lang/Nim/issues/14331))
+- Fixed "SIGSEV encountered when creating threads in a loop w/ --gc:arc"
+  ([#13935](https://github.com/nim-lang/Nim/issues/13935))
+- Fixed "nim-gdb is missing from all released packages"
+  ([#13104](https://github.com/nim-lang/Nim/issues/13104))
+- Fixed "sysAssert error with gc:arc on 3 line program"
+  ([#13862](https://github.com/nim-lang/Nim/issues/13862))
+- Fixed "compiler error with inline async proc and pragma"
+  ([#13998](https://github.com/nim-lang/Nim/issues/13998))
+- Fixed "[ARC] Compiler crash when adding to a seq[ref Object]"
+  ([#14333](https://github.com/nim-lang/Nim/issues/14333))
+- Fixed "nimvm: sysFatal: unhandled exception: 'sons' is not accessible using discriminant 'kind' of type 'TNode' [FieldError]"
+  ([#14340](https://github.com/nim-lang/Nim/issues/14340))
+- Fixed "[Regression] karax events are not firing "
+  ([#14350](https://github.com/nim-lang/Nim/issues/14350))
+- Fixed "odbcsql module has some wrong integer types"
+  ([#9771](https://github.com/nim-lang/Nim/issues/9771))
+- Fixed "db_sqlite needs sqlPrepared"
+  ([#13559](https://github.com/nim-lang/Nim/issues/13559))
+- Fixed "[Regression] `createThread` is not GC-safe"
+  ([#14370](https://github.com/nim-lang/Nim/issues/14370))
+- Fixed "Broken example on hot code reloading"
+  ([#14380](https://github.com/nim-lang/Nim/issues/14380))
+- Fixed "runnableExamples block with `except` on specified error fails with `nim doc`"
+  ([#12746](https://github.com/nim-lang/Nim/issues/12746))
+- Fixed "compiler as a library: findNimStdLibCompileTime fails to find system.nim"
+  ([#12293](https://github.com/nim-lang/Nim/issues/12293))
+- Fixed "5 bugs with importcpp exceptions"
+  ([#14369](https://github.com/nim-lang/Nim/issues/14369))
+- Fixed "Docs shouldn't collapse pragmas inside runnableExamples/code blocks"
+  ([#14174](https://github.com/nim-lang/Nim/issues/14174))
+- Fixed "Bad codegen/emit for hashes.hiXorLo in some contexts."
+  ([#14394](https://github.com/nim-lang/Nim/issues/14394))
+- Fixed "Boehm GC does not scan thread-local storage"
+  ([#14364](https://github.com/nim-lang/Nim/issues/14364))
+- Fixed "RVO not exception safe"
+  ([#14126](https://github.com/nim-lang/Nim/issues/14126))
+- Fixed "runnableExamples that are only compiled"
+  ([#10731](https://github.com/nim-lang/Nim/issues/10731))
+- Fixed "`foldr` raises IndexError when called on sequence"
+  ([#14404](https://github.com/nim-lang/Nim/issues/14404))
+- Fixed "moveFile does not overwrite destination file"
+  ([#14057](https://github.com/nim-lang/Nim/issues/14057))
+- Fixed "doc2 outputs in current work dir"
+  ([#6583](https://github.com/nim-lang/Nim/issues/6583))
+- Fixed "[docgen] proc doc comments silently omitted after 1st runnableExamples"
+  ([#9227](https://github.com/nim-lang/Nim/issues/9227))
+- Fixed "`nim doc --project` shows '@@/' instead of '../' for relative paths to submodules"
+  ([#14448](https://github.com/nim-lang/Nim/issues/14448))
+- Fixed "re, nre have wrong `start` semantics"
+  ([#14284](https://github.com/nim-lang/Nim/issues/14284))
+- Fixed "runnableExamples should preserve source code doc comments, strings, and (maybe) formatting"
+  ([#8871](https://github.com/nim-lang/Nim/issues/8871))
+- Fixed "`nim doc ..` fails when runnableExamples uses `$`  [devel] [regression]"
+  ([#14485](https://github.com/nim-lang/Nim/issues/14485))
+- Fixed "`items` is 20%~30% slower than iteration via an index"
+  ([#14421](https://github.com/nim-lang/Nim/issues/14421))
+- Fixed "ARC: unreliable setLen "
+  ([#14495](https://github.com/nim-lang/Nim/issues/14495))
+- Fixed "lent is unsafe: after #14447 you can modify variables with "items" loop for sequences"
+  ([#14498](https://github.com/nim-lang/Nim/issues/14498))
+- Fixed "`var op = fn()` wrongly gives warning `ObservableStores` with `object of RootObj` type"
+  ([#14514](https://github.com/nim-lang/Nim/issues/14514))
+- Fixed "Compiler assertion"
+  ([#14562](https://github.com/nim-lang/Nim/issues/14562))
+- Fixed "Can't get `ord` of a value of a Range type in the JS backend "
+  ([#14570](https://github.com/nim-lang/Nim/issues/14570))
+- Fixed "js: can't take addr of param (including implicitly via `lent`)"
+  ([#14576](https://github.com/nim-lang/Nim/issues/14576))
+- Fixed "{.noinit.} ignored in for loop -> bad codegen for non-movable types"
+  ([#14118](https://github.com/nim-lang/Nim/issues/14118))
+- Fixed "generic destructor gives: `Error: unresolved generic parameter`"
+  ([#14315](https://github.com/nim-lang/Nim/issues/14315))
+- Fixed "Memory leak with arc gc"
+  ([#14568](https://github.com/nim-lang/Nim/issues/14568))
+- Fixed "escape analysis broken with `lent`"
+  ([#14557](https://github.com/nim-lang/Nim/issues/14557))
+- Fixed "`wrapWords` seems to ignore linebreaks when wrapping, leaving breaks in the wrong place"
+  ([#14579](https://github.com/nim-lang/Nim/issues/14579))
+- Fixed "`lent` gives wrong results with -d:release"
+  ([#14578](https://github.com/nim-lang/Nim/issues/14578))
+- Fixed "Nested await expressions regression: `await a(await expandValue())` doesnt compile"
+  ([#14279](https://github.com/nim-lang/Nim/issues/14279))
+- Fixed "windows CI docs fails with strange errors"
+  ([#14545](https://github.com/nim-lang/Nim/issues/14545))
+- Fixed "[CI] tests/async/tioselectors.nim flaky test for freebsd + OSX CI"
+  ([#13166](https://github.com/nim-lang/Nim/issues/13166))
+- Fixed "seq.setLen sometimes doesn't zero memory"
+  ([#14655](https://github.com/nim-lang/Nim/issues/14655))
+- Fixed "`nim dump` is roughly 100x slower in 1.3 versus 1.2"
+  ([#14179](https://github.com/nim-lang/Nim/issues/14179))
+- Fixed "Regression: devel docgen cannot generate document for method"
+  ([#14691](https://github.com/nim-lang/Nim/issues/14691))
+- Fixed "recently flaky tests/async/t7758.nim"
+  ([#14685](https://github.com/nim-lang/Nim/issues/14685))
+- Fixed "Bind no longer working in generic procs."
+  ([#11811](https://github.com/nim-lang/Nim/issues/11811))
+- Fixed "The pegs module doesn't work with generics!"
+  ([#14718](https://github.com/nim-lang/Nim/issues/14718))
+- Fixed "Defer is not properly working for asynchronous procedures."
+  ([#13899](https://github.com/nim-lang/Nim/issues/13899))
+- Fixed "Add an ARC test with threads in a loop"
+  ([#14690](https://github.com/nim-lang/Nim/issues/14690))
+- Fixed "[goto exceptions] {.noReturn.} pragma is not detected in a case expression"
+  ([#14458](https://github.com/nim-lang/Nim/issues/14458))
+- Fixed "[exceptions:goto] C compiler error with dynlib pragma calling a proc"
+  ([#14240](https://github.com/nim-lang/Nim/issues/14240))
+- Fixed "Cannot borrow var float64 in infix assignment"
+  ([#14440](https://github.com/nim-lang/Nim/issues/14440))
+- Fixed "lib/pure/memfiles.nim: compilation error with --taintMode:on"
+  ([#14760](https://github.com/nim-lang/Nim/issues/14760))
+- Fixed "newWideCString allocates a multiple of the memory needed"
+  ([#14750](https://github.com/nim-lang/Nim/issues/14750))
+- Fixed "Nim source archive install: 'install.sh' fails with error: cp: cannot stat 'bin/nim-gdb': No such file or directory"
+  ([#14748](https://github.com/nim-lang/Nim/issues/14748))
+- Fixed "`nim cpp -r tests/exception/t9657` hangs"
+  ([#10343](https://github.com/nim-lang/Nim/issues/10343))
+- Fixed "Detect tool fails on FreeBSD"
+  ([#14715](https://github.com/nim-lang/Nim/issues/14715))
+- Fixed "compiler crash: `findUnresolvedStatic` "
+  ([#14802](https://github.com/nim-lang/Nim/issues/14802))
+- Fixed "seq namespace (?) regression"
+  ([#4796](https://github.com/nim-lang/Nim/issues/4796))
+- Fixed "Possible out of bounds string access in std/colors parseColor and isColor"
+  ([#14839](https://github.com/nim-lang/Nim/issues/14839))
+- Fixed "compile error on latest devel with orc and ssl"
+  ([#14647](https://github.com/nim-lang/Nim/issues/14647))
+- Fixed "[minor] `$` wrong for type tuple"
+  ([#13432](https://github.com/nim-lang/Nim/issues/13432))
+- Fixed "Documentation missing on devel asyncftpclient"
+  ([#14846](https://github.com/nim-lang/Nim/issues/14846))
+- Fixed "nimpretty is confused with a trailing comma in enum definition"
+  ([#14401](https://github.com/nim-lang/Nim/issues/14401))
+- Fixed "Output arguments get ignored when compiling with --app:staticlib"
+  ([#12745](https://github.com/nim-lang/Nim/issues/12745))
+- Fixed "[ARC] destructive move destroys the object too early"
+  ([#14396](https://github.com/nim-lang/Nim/issues/14396))
+- Fixed "highlite.getNextToken() crashes if the buffer string is "echo "\"""
+  ([#14830](https://github.com/nim-lang/Nim/issues/14830))
+- Fixed "Memory corruption with --gc:arc with a seq of objects with an empty body."
+  ([#14472](https://github.com/nim-lang/Nim/issues/14472))
+- Fixed "Stropped identifiers don't work as field names in tuple literals"
+  ([#14911](https://github.com/nim-lang/Nim/issues/14911))
+- Fixed "Please revert my commit"
+  ([#14930](https://github.com/nim-lang/Nim/issues/14930))
+- Fixed "[ARC] C compiler error with inline iterators and imports"
+  ([#14864](https://github.com/nim-lang/Nim/issues/14864))
+- Fixed "AsyncHttpClient segfaults with gc:orc, possibly memory corruption"
+  ([#14402](https://github.com/nim-lang/Nim/issues/14402))
+- Fixed "[ARC] Template with a block evaluating to a GC'd value results in a compiler crash"
+  ([#14899](https://github.com/nim-lang/Nim/issues/14899))
+- Fixed "[ARC] Weird issue with if expressions and templates"
+  ([#14900](https://github.com/nim-lang/Nim/issues/14900))
+- Fixed "xmlparser does not compile on devel"
+  ([#14805](https://github.com/nim-lang/Nim/issues/14805))
+- Fixed "returning lent T from a var T param gives codegen errors or SIGSEGV"
+  ([#14878](https://github.com/nim-lang/Nim/issues/14878))
+- Fixed "[ARC] Weird issue with if expressions and templates"
+  ([#14900](https://github.com/nim-lang/Nim/issues/14900))
+- Fixed "threads:on + gc:orc + unittest = C compiler errors"
+  ([#14865](https://github.com/nim-lang/Nim/issues/14865))
+- Fixed "mitems, mpairs doesn't work at compile time anymore"
+  ([#12129](https://github.com/nim-lang/Nim/issues/12129))
+- Fixed "strange result from executing code in const expression"
+  ([#10465](https://github.com/nim-lang/Nim/issues/10465))
+- Fixed "Same warning printed 3 times"
+  ([#11009](https://github.com/nim-lang/Nim/issues/11009))
+- Fixed "type alias for generic typeclass doesn't work"
+  ([#4668](https://github.com/nim-lang/Nim/issues/4668))
+- Fixed "exceptions:goto Bug devel codegen lvalue NIM_FALSE=NIM_FALSE"
+  ([#14925](https://github.com/nim-lang/Nim/issues/14925))
+- Fixed "the --useVersion:1.0 no longer works in devel"
+  ([#14912](https://github.com/nim-lang/Nim/issues/14912))
+- Fixed "template declaration of iterator doesn't compile"
+  ([#4722](https://github.com/nim-lang/Nim/issues/4722))
+- Fixed "Compiler crash on type inheritance with static generic parameter and equality check"
+  ([#12571](https://github.com/nim-lang/Nim/issues/12571))
+- Fixed "Nim crashes while handling a cast in async circumstances."
+  ([#13815](https://github.com/nim-lang/Nim/issues/13815))
+- Fixed "[ARC] Internal compiler error when calling an iterator from an inline proc "
+  ([#14383](https://github.com/nim-lang/Nim/issues/14383))
+- Fixed ""Cannot instantiate" error when template uses generic type"
+  ([#5926](https://github.com/nim-lang/Nim/issues/5926))
+- Fixed "Different raises behaviour for newTerminal between Linux and Windows"
+  ([#12759](https://github.com/nim-lang/Nim/issues/12759))
+- Fixed "Expand on a type (that defines a proc type) in error message "
+  ([#6608](https://github.com/nim-lang/Nim/issues/6608))
+- Fixed "unittest require quits program with an exit code of 0"
+  ([#14475](https://github.com/nim-lang/Nim/issues/14475))
+- Fixed "Range type: Generics vs concrete type, semcheck difference."
+  ([#8426](https://github.com/nim-lang/Nim/issues/8426))
+- Fixed "[Macro] Type mismatch when parameter name is the same as a field"
+  ([#13253](https://github.com/nim-lang/Nim/issues/13253))
+- Fixed "Generic instantiation failure when converting a sequence of circular generic types to strings"
+  ([#10396](https://github.com/nim-lang/Nim/issues/10396))
+- Fixed "initOptParser ignores argument after value option with empty value."
+  ([#13086](https://github.com/nim-lang/Nim/issues/13086))
+- Fixed "[ARC] proc with both explicit and implicit return results in a C compiler error"
+  ([#14985](https://github.com/nim-lang/Nim/issues/14985))
+- Fixed "Alias type forgets implicit generic params depending on order"
+  ([#14990](https://github.com/nim-lang/Nim/issues/14990))
+- Fixed "[ARC] sequtils.insert has different behaviour between ARC/refc"
+  ([#14994](https://github.com/nim-lang/Nim/issues/14994))
+- Fixed "The documentation for "hot code reloading" references a non-existent npm package"
+  ([#13621](https://github.com/nim-lang/Nim/issues/13621))
+- Fixed "existsDir deprecated but breaking `dir` undeclared"
+  ([#15006](https://github.com/nim-lang/Nim/issues/15006))
+- Fixed "uri.decodeUrl crashes on incorrectly formatted input"
+  ([#14082](https://github.com/nim-lang/Nim/issues/14082))
+- Fixed "testament incorrectly reports time for tests, leading to wrong conclusions"
+  ([#14822](https://github.com/nim-lang/Nim/issues/14822))
+- Fixed "Calling peekChar with Stream returned from osproc.outputStream generate runtime error"
+  ([#14906](https://github.com/nim-lang/Nim/issues/14906))
+- Fixed "localPassC pragma should come *after* other flags"
+  ([#14194](https://github.com/nim-lang/Nim/issues/14194))
+- Fixed ""Could not load" dynamic library at runtime because of hidden dependency"
+  ([#2408](https://github.com/nim-lang/Nim/issues/2408))
+- Fixed "--gc:arc generate invalid code for {.global.} (`«nimErr_» in NIM_UNLIKELY`)"
+  ([#14480](https://github.com/nim-lang/Nim/issues/14480))
+- Fixed "Using `^` from stdlib/math along with converters gives a match for types that aren't SomeNumber"
+  ([#15033](https://github.com/nim-lang/Nim/issues/15033))
+- Fixed "[ARC] Weird exception behaviour from doAssertRaises"
+  ([#15026](https://github.com/nim-lang/Nim/issues/15026))
+- Fixed "[ARC] Compiler crash declaring a finalizer proc directly in 'new'"
+  ([#15044](https://github.com/nim-lang/Nim/issues/15044))
+- Fixed "[ARC] C compiler error when creating a var of a const seq"
+  ([#15036](https://github.com/nim-lang/Nim/issues/15036))
+- Fixed "code with named arguments in proc of winim/com can not been compiled"
+  ([#15056](https://github.com/nim-lang/Nim/issues/15056))
+- Fixed "javascript backend produces javascript code with syntax error in object syntax"
+  ([#14534](https://github.com/nim-lang/Nim/issues/14534))
+- Fixed "--gc:arc should be ignored in JS mode."
+  ([#14684](https://github.com/nim-lang/Nim/issues/14684))
+- Fixed "arc: C compilation error with imported global code using a closure iterator"
+  ([#12990](https://github.com/nim-lang/Nim/issues/12990))
+- Fixed "[ARC] Crash when modifying a string with mitems iterator"
+  ([#15052](https://github.com/nim-lang/Nim/issues/15052))
+- Fixed "[ARC] SIGSEGV when calling a closure as a tuple field in a seq"
+  ([#15038](https://github.com/nim-lang/Nim/issues/15038))
+- Fixed "pass varargs[seq[T]] to iterator give empty seq "
+  ([#12576](https://github.com/nim-lang/Nim/issues/12576))
+- Fixed "Compiler crashes when using string as object variant selector with else branch"
+  ([#14189](https://github.com/nim-lang/Nim/issues/14189))
+- Fixed "JS compiler error related to implicit return and return var type"
+  ([#11354](https://github.com/nim-lang/Nim/issues/11354))
+- Fixed "`nkRecWhen` causes internalAssert in semConstructFields"
+  ([#14698](https://github.com/nim-lang/Nim/issues/14698))
+- Fixed "Memory leaks with async (closure iterators?) under ORC"
+  ([#15076](https://github.com/nim-lang/Nim/issues/15076))
+- Fixed "strutil.insertSep() fails on negative numbers"
+  ([#11352](https://github.com/nim-lang/Nim/issues/11352))
+- Fixed "Constructing a uint64 range on a 32-bit machine leads to incorrect codegen"
+  ([#14616](https://github.com/nim-lang/Nim/issues/14616))
+- Fixed "heapqueue pushpop() proc doesn't compile"
+  ([#14139](https://github.com/nim-lang/Nim/issues/14139))
+- Fixed "[ARC] SIGSEGV when trying to swap in a literal/const string"
+  ([#15112](https://github.com/nim-lang/Nim/issues/15112))
+- Fixed "Defer and --gc:arc"
+  ([#15071](https://github.com/nim-lang/Nim/issues/15071))
+- Fixed "internal error: compiler/semobjconstr.nim(324, 20) example"
+  ([#15111](https://github.com/nim-lang/Nim/issues/15111))
+- Fixed "[ARC] Sequence "disappears" with a table inside of a table with an object variant"
+  ([#15122](https://github.com/nim-lang/Nim/issues/15122))
+- Fixed "[ARC] SIGSEGV with tuple assignment caused by cursor inference"
+  ([#15130](https://github.com/nim-lang/Nim/issues/15130))
+- Fixed "Issue with --gc:arc at compile time"
+  ([#15129](https://github.com/nim-lang/Nim/issues/15129))
+- Fixed "Writing an empty string to an AsyncFile raises an IndexDefect"
+  ([#15148](https://github.com/nim-lang/Nim/issues/15148))
+- Fixed "Compiler is confused about call convention of function with nested closure"
+  ([#5688](https://github.com/nim-lang/Nim/issues/5688))
+- Fixed "Nil check on each field fails in generic function"
+  ([#15101](https://github.com/nim-lang/Nim/issues/15101))
+- Fixed "{.nimcall.} convention won't avoid the creation of closures"
+  ([#8473](https://github.com/nim-lang/Nim/issues/8473))
+- Fixed "smtp.nim(161, 40) Error: type mismatch: got <typeof(nil)> but expected 'SslContext = void'"
+  ([#15177](https://github.com/nim-lang/Nim/issues/15177))
+- Fixed "[strscans] scanf doesn't match a single character with $+ if it's the end of the string"
+  ([#15064](https://github.com/nim-lang/Nim/issues/15064))
+- Fixed "Crash and incorrect return values when using readPasswordFromStdin on Windows."
+  ([#15207](https://github.com/nim-lang/Nim/issues/15207))
+- Fixed "Possible capture error with fieldPairs and genericParams"
+  ([#15221](https://github.com/nim-lang/Nim/issues/15221))
+- Fixed "The StmtList processing of template parameters can lead to unexpected errors"
+  ([#5691](https://github.com/nim-lang/Nim/issues/5691))
+- Fixed "[ARC] C compiler error when passing a var openArray to a sink openArray"
+  ([#15035](https://github.com/nim-lang/Nim/issues/15035))
+- Fixed "Inconsistent unsigned -> signed RangeDefect usage across integer sizes"
+  ([#15210](https://github.com/nim-lang/Nim/issues/15210))
+- Fixed "toHex results in RangeDefect exception when used with large uint64"
+  ([#15257](https://github.com/nim-lang/Nim/issues/15257))
+- Fixed "Arc sink arg crash"
+  ([#15238](https://github.com/nim-lang/Nim/issues/15238))
+- Fixed "SQL escape in db_mysql is not enough"
+  ([#15219](https://github.com/nim-lang/Nim/issues/15219))
+- Fixed "Mixing 'return' with expressions is allowed in 1.2"
+  ([#15280](https://github.com/nim-lang/Nim/issues/15280))
+- Fixed "os.getFileInfo() causes ICE with --gc:arc on Windows"
+  ([#15286](https://github.com/nim-lang/Nim/issues/15286))
+- Fixed "[ARC] Sequence "disappears" with a table inside of a table with an object variant"
+  ([#15122](https://github.com/nim-lang/Nim/issues/15122))
+- Fixed "Documentation regression jsre module missing"
+  ([#15183](https://github.com/nim-lang/Nim/issues/15183))
+- Fixed "CountTable.smallest/largest() on empty table either asserts or gives bogus answer"
+  ([#15021](https://github.com/nim-lang/Nim/issues/15021))
+- Fixed "[Regression] Parser regression"
+  ([#15305](https://github.com/nim-lang/Nim/issues/15305))
+- Fixed "[ARC] SIGSEGV with tuple unpacking caused by cursor inference"
+  ([#15147](https://github.com/nim-lang/Nim/issues/15147))
+- Fixed "LwIP/FreeRTOS compile error - missing SIGPIPE and more "
+  ([#15302](https://github.com/nim-lang/Nim/issues/15302))
+- Fixed "Memory leaks with async (closure iterators?) under ORC"
+  ([#15076](https://github.com/nim-lang/Nim/issues/15076))
+- Fixed "Bug compiling with --gc:arg or --gc:orc"
+  ([#15325](https://github.com/nim-lang/Nim/issues/15325))
+- Fixed "memory corruption in tmarshall.nim"
+  ([#9754](https://github.com/nim-lang/Nim/issues/9754))
+- Fixed "typed macros break generic proc definitions"
+  ([#15326](https://github.com/nim-lang/Nim/issues/15326))
+- Fixed "nim doc2 ignores --docSeeSrcUrl parameter"
+  ([#6071](https://github.com/nim-lang/Nim/issues/6071))
+- Fixed "The decodeData Iterator from cgi module crash"
+  ([#15369](https://github.com/nim-lang/Nim/issues/15369))
+- Fixed "|| iterator generates invalid code when compiling with --debugger:native"
+  ([#9710](https://github.com/nim-lang/Nim/issues/9710))
+- Fixed "Wrong number of variables"
+  ([#15360](https://github.com/nim-lang/Nim/issues/15360))
+- Fixed "Coercions with distinct types should traverse pointer modifiers transparently."
+  ([#7165](https://github.com/nim-lang/Nim/issues/7165))
+- Fixed "Error with distinct generic TableRef"
+  ([#6060](https://github.com/nim-lang/Nim/issues/6060))
+- Fixed "Support images in nim docgen"
+  ([#6430](https://github.com/nim-lang/Nim/issues/6430))
+- Fixed "Regression. Double sem check for procs."
+  ([#15389](https://github.com/nim-lang/Nim/issues/15389))
+- Fixed "uri.nim url with literal ipv6 address is printed wrong, and cannot parsed again"
+  ([#15333](https://github.com/nim-lang/Nim/issues/15333))
+- Fixed "[ARC] Object variant gets corrupted with cursor inference"
+  ([#15361](https://github.com/nim-lang/Nim/issues/15361))
+- Fixed "`nim doc ..` compiler crash (regression 0.19.6 => 1.0)"
+  ([#14474](https://github.com/nim-lang/Nim/issues/14474))
+- Fixed "cannot borrow result; what it borrows from is potentially mutated"
+  ([#15403](https://github.com/nim-lang/Nim/issues/15403))
+- Fixed "memory corruption for seq.add(seq) with gc:arc and d:useMalloc "
+  ([#14983](https://github.com/nim-lang/Nim/issues/14983))
+- Fixed "DocGen HTML output appears improperly when encountering text immediately after/before inline monospace; in some cases won't compile"
+  ([#11537](https://github.com/nim-lang/Nim/issues/11537))
+- Fixed "Deepcopy in arc crashes"
+  ([#15405](https://github.com/nim-lang/Nim/issues/15405))
+- Fixed "pop pragma takes invalid input"
+  ([#15430](https://github.com/nim-lang/Nim/issues/15430))
+- Fixed "tests/stdlib/tgetprotobyname fails on NetBSD"
+  ([#15452](https://github.com/nim-lang/Nim/issues/15452))
+- Fixed "defer doesnt work with block, break and await"
+  ([#15243](https://github.com/nim-lang/Nim/issues/15243))
+- Fixed "tests/stdlib/tssl failing on NetBSD"
+  ([#15493](https://github.com/nim-lang/Nim/issues/15493))
+- Fixed "strictFuncs doesn't seem to catch simple ref mutation"
+  ([#15508](https://github.com/nim-lang/Nim/issues/15508))
+- Fixed "Sizeof of case object is incorrect. Showstopper"
+  ([#15516](https://github.com/nim-lang/Nim/issues/15516))
+- Fixed "[ARC] Internal error when trying to use a parallel for loop"
+  ([#15512](https://github.com/nim-lang/Nim/issues/15512))
+- Fixed "[ARC] Type-bound assign op is not being generated"
+  ([#15510](https://github.com/nim-lang/Nim/issues/15510))
+- Fixed "[ARC] Crash when adding openArray proc argument to a local seq"
+  ([#15511](https://github.com/nim-lang/Nim/issues/15511))
+- Fixed "VM: const case object gets some fields zeroed out at runtime"
+  ([#13081](https://github.com/nim-lang/Nim/issues/13081))
+- Fixed "regression(1.2.6 => devel): VM: const case object field access gives: 'sons' is not accessible"
+  ([#15532](https://github.com/nim-lang/Nim/issues/15532))
+- Fixed "Csources: huge size increase (x2.3) in 0.20"
+  ([#12027](https://github.com/nim-lang/Nim/issues/12027))
+- Fixed "Out of date error message for GC options"
+  ([#15547](https://github.com/nim-lang/Nim/issues/15547))
+- Fixed "dbQuote additional escape regression"
+  ([#15560](https://github.com/nim-lang/Nim/issues/15560))

compiler/ast.nim

@@ -17,21 +17,17 @@ export int128
 
 type
   TCallingConvention* = enum
-    ccDefault,                # proc has no explicit calling convention
-    ccStdCall,                # procedure is stdcall
-    ccCDecl,                  # cdecl
-    ccSafeCall,               # safecall
-    ccSysCall,                # system call
-    ccInline,                 # proc should be inlined
-    ccNoInline,               # proc should not be inlined
-    ccFastCall,               # fastcall (pass parameters in registers)
-    ccClosure,                # proc has a closure
-    ccNoConvention            # needed for generating proper C procs sometimes
-
-const
-  CallingConvToStr*: array[TCallingConvention, string] = ["", "stdcall",
-    "cdecl", "safecall", "syscall", "inline", "noinline", "fastcall",
-    "closure", "noconv"]
+    ccNimCall = "nimcall"           # nimcall, also the default
+    ccStdCall = "stdcall"           # procedure is stdcall
+    ccCDecl = "cdecl"               # cdecl
+    ccSafeCall = "safecall"         # safecall
+    ccSysCall = "syscall"           # system call
+    ccInline = "inline"             # proc should be inlined
+    ccNoInline = "noinline"         # proc should not be inlined
+    ccFastCall = "fastcall"         # fastcall (pass parameters in registers)
+    ccThisCall = "thiscall"         # thiscall (parameters are pushed right-to-left)
+    ccClosure  = "closure"          # proc has a closure
+    ccNoConvention = "noconv"       # needed for generating proper C procs sometimes
 
 type
   TNodeKind* = enum # order is extremely important, because ranges are used
@@ -218,7 +214,7 @@ type
     nkEnumFieldDef,       # `ident = expr` in an enumeration
     nkArgList,            # argument list
     nkPattern,            # a special pattern; used for matching
-    nkHiddenTryStmt,      # token used for interpretation
+    nkHiddenTryStmt,      # a hidden try statement
     nkClosure,            # (prc, env)-pair (internally used for code gen)
     nkGotoState,          # used for the state machine (for iterators)
     nkState,              # give a label to a code section (for iterators)
@@ -229,7 +225,7 @@ type
   TNodeKinds* = set[TNodeKind]
 
 type
-  TSymFlag* = enum    # 42 flags!
+  TSymFlag* = enum    # 46 flags!
     sfUsed,           # read access of sym (for warnings) or simply used
     sfExported,       # symbol is exported from module
     sfFromGeneric,    # symbol is instantiation of a generic; this is needed
@@ -290,8 +286,11 @@ type
     sfTemplateParam   # symbol is a template parameter
     sfCursor          # variable/field is a cursor, see RFC 177 for details
     sfInjectDestructors # whether the proc needs the 'injectdestructors' transformation
-    sfNeverRaises     # proc can never raise an exception, not even OverflowError
+    sfNeverRaises     # proc can never raise an exception, not even OverflowDefect
                       # or out-of-memory
+    sfUsedInFinallyOrExcept  # symbol is used inside an 'except' or 'finally'
+    sfSingleUsedTemp  # For temporaries that we know will only be used once
+    sfNoalias         # 'noalias' annotation, means C's 'restrict'
 
   TSymFlags* = set[TSymFlag]
 
@@ -310,10 +309,6 @@ const
     # the compiler will avoid printing such names
     # in user messages.
 
-  sfHoisted* = sfForward
-    # an expression was hoised to an anonymous variable.
-    # the flag is applied to the var/let symbol
-
   sfNoForward* = sfRegister
     # forward declarations are not required (per module)
   sfReorder* = sfForward
@@ -432,8 +427,9 @@ type
       # instantiation and prior to this it has the potential to
       # be any type.
 
-    tyOpt
-      # Builtin optional type
+    tyOptDeprecated
+      # 'out' parameter. Comparable to a 'var' parameter but every
+      # path must assign a value to it before it can be read from.
 
     tyVoid
       # now different from tyEmpty, hurray!
@@ -553,6 +549,10 @@ type
                            # If it has one, t.destructor is not nil.
     tfAcyclic # object type was annotated as .acyclic
     tfIncompleteStruct # treat this type as if it had sizeof(pointer)
+    tfCompleteStruct
+      # (for importc types); type is fully specified, allowing to compute
+      # sizeof, alignof, offsetof at CT
+    tfExplicitCallConv
 
   TTypeFlags* = set[TTypeFlag]
 
@@ -609,7 +609,7 @@ var
 type
   TMagic* = enum # symbols that require compiler magic:
     mNone,
-    mDefined, mDefinedInScope, mCompiles, mArrGet, mArrPut, mAsgn,
+    mDefined, mDeclared, mDeclaredInScope, mCompiles, mArrGet, mArrPut, mAsgn,
     mLow, mHigh, mSizeOf, mAlignOf, mOffsetOf, mTypeTrait,
     mIs, mOf, mAddr, mType, mTypeOf,
     mPlugin, mEcho, mShallowCopy, mSlurp, mStaticExec, mStatic,
@@ -652,8 +652,8 @@ type
     mSwap, mIsNil, mArrToSeq,
     mNewString, mNewStringOfCap, mParseBiggestFloat,
     mMove, mWasMoved, mDestroy,
-    mDefault, mUnown, mAccessEnv, mReset,
-    mArray, mOpenArray, mRange, mSet, mSeq, mOpt, mVarargs,
+    mDefault, mUnown, mIsolate, mAccessEnv, mReset,
+    mArray, mOpenArray, mRange, mSet, mSeq, mVarargs,
     mRef, mPtr, mVar, mDistinct, mVoid, mTuple,
     mOrdinal,
     mInt, mInt8, mInt16, mInt32, mInt64,
@@ -677,7 +677,7 @@ type
     mNBindSym, mNCallSite,
     mEqIdent, mEqNimrodNode, mSameNodeType, mGetImpl, mNGenSym,
     mNHint, mNWarning, mNError,
-    mInstantiationInfo, mGetTypeInfo,
+    mInstantiationInfo, mGetTypeInfo, mGetTypeInfoV2,
     mNimvm, mIntDefine, mStrDefine, mBoolDefine, mRunnableExamples,
     mException, mBuiltinType, mSymOwner, mUncheckedArray, mGetImplTransf,
     mSymIsInstantiationOf, mNodeId
@@ -871,7 +871,7 @@ type
     annex*: PLib              # additional fields (seldom used, so we use a
                               # reference to another object to save space)
     when hasFFI:
-      cname*: string          # resolved C declaration name in importc decl, eg:
+      cname*: string          # resolved C declaration name in importc decl, e.g.:
                               # proc fun() {.importc: "$1aux".} => cname = funaux
     constraint*: PNode        # additional constraints like 'lit|result'; also
                               # misused for the codegenDecl pragma in the hope
@@ -1026,6 +1026,7 @@ const
   nkFloatLiterals* = {nkFloatLit..nkFloat128Lit}
   nkLambdaKinds* = {nkLambda, nkDo}
   declarativeDefs* = {nkProcDef, nkFuncDef, nkMethodDef, nkIteratorDef, nkConverterDef}
+  routineDefs* = declarativeDefs + {nkMacroDef, nkTemplateDef}
   procDefs* = nkLambdaKinds + declarativeDefs
 
   nkSymChoices* = {nkClosedSymChoice, nkOpenSymChoice}
@@ -1084,7 +1085,7 @@ proc safeLen*(n: PNode): int {.inline.} =
 
 proc safeArrLen*(n: PNode): int {.inline.} =
   ## works for array-like objects (strings passed as openArray in VM).
-  if n.kind in {nkStrLit..nkTripleStrLit}:result = n.strVal.len
+  if n.kind in {nkStrLit..nkTripleStrLit}: result = n.strVal.len
   elif n.kind in {nkNone..nkFloat128Lit}: result = 0
   else: result = n.len
 
@@ -1113,12 +1114,49 @@ proc newNode*(kind: TNodeKind): PNode =
       writeStackTrace()
     inc gNodeId
 
+proc newNodeI*(kind: TNodeKind, info: TLineInfo): PNode =
+  result = PNode(kind: kind, info: info)
+  when defined(useNodeIds):
+    result.id = gNodeId
+    if result.id == nodeIdToDebug:
+      echo "KIND ", result.kind
+      writeStackTrace()
+    inc gNodeId
+
+proc newNodeI*(kind: TNodeKind, info: TLineInfo, children: int): PNode =
+  result = PNode(kind: kind, info: info)
+  if children > 0:
+    newSeq(result.sons, children)
+  when defined(useNodeIds):
+    result.id = gNodeId
+    if result.id == nodeIdToDebug:
+      echo "KIND ", result.kind
+      writeStackTrace()
+    inc gNodeId
+
+proc newNodeIT*(kind: TNodeKind, info: TLineInfo, typ: PType): PNode =
+  result = newNode(kind)
+  result.info = info
+  result.typ = typ
+
 proc newTree*(kind: TNodeKind; children: varargs[PNode]): PNode =
   result = newNode(kind)
   if children.len > 0:
     result.info = children[0].info
   result.sons = @children
 
+proc newTreeI*(kind: TNodeKind; info: TLineInfo; children: varargs[PNode]): PNode =
+  result = newNodeI(kind, info)
+  if children.len > 0:
+    result.info = children[0].info
+  result.sons = @children
+
+proc newTreeIT*(kind: TNodeKind; info: TLineInfo; typ: PType; children: varargs[PNode]): PNode =
+  result = newNodeIT(kind, info, typ)
+  if children.len > 0:
+    result.info = children[0].info
+  result.sons = @children
+
 template previouslyInferred*(t: PType): PType =
   if t.sons.len > 1: t.lastSon else: nil
 
@@ -1219,43 +1257,6 @@ proc newSymNode*(sym: PSym, info: TLineInfo): PNode =
   result.typ = sym.typ
   result.info = info
 
-proc newNodeI*(kind: TNodeKind, info: TLineInfo): PNode =
-  result = PNode(kind: kind, info: info)
-  when defined(useNodeIds):
-    result.id = gNodeId
-    if result.id == nodeIdToDebug:
-      echo "KIND ", result.kind
-      writeStackTrace()
-    inc gNodeId
-
-proc newNodeI*(kind: TNodeKind, info: TLineInfo, children: int): PNode =
-  result = PNode(kind: kind, info: info)
-  if children > 0:
-    newSeq(result.sons, children)
-  when defined(useNodeIds):
-    result.id = gNodeId
-    if result.id == nodeIdToDebug:
-      echo "KIND ", result.kind
-      writeStackTrace()
-    inc gNodeId
-
-proc newNode*(kind: TNodeKind, info: TLineInfo, sons: TNodeSeq = @[],
-              typ: PType = nil): PNode =
-  # XXX use shallowCopy here for ownership transfer:
-  result = PNode(kind: kind, info: info, typ: typ)
-  result.sons = sons
-  when defined(useNodeIds):
-    result.id = gNodeId
-    if result.id == nodeIdToDebug:
-      echo "KIND ", result.kind
-      writeStackTrace()
-    inc gNodeId
-
-proc newNodeIT*(kind: TNodeKind, info: TLineInfo, typ: PType): PNode =
-  result = newNode(kind)
-  result.info = info
-  result.typ = typ
-
 proc newIntNode*(kind: TNodeKind, intVal: BiggestInt): PNode =
   result = newNode(kind)
   result.intVal = intVal
@@ -1288,7 +1289,7 @@ proc newIntTypeNode*(intVal: BiggestInt, typ: PType): PNode =
   of tyUInt16:  result = newNode(nkUInt16Lit)
   of tyUInt32:  result = newNode(nkUInt32Lit)
   of tyUInt64:  result = newNode(nkUInt64Lit)
-  of tyBool,tyEnum:
+  of tyBool, tyEnum:
     # XXX: does this really need to be the kind nkIntLit?
     result = newNode(nkIntLit)
   of tyStatic: # that's a pre-existing bug, will fix in another PR
@@ -1326,7 +1327,7 @@ const
   MaxLockLevel* = 1000'i16
   UnknownLockLevel* = TLockLevel(1001'i16)
   AttachedOpToStr*: array[TTypeAttachedOp, string] = [
-    "=destroy", "=", "=sink", "=trace", "=dispose", "=deepcopy"]
+    "=destroy", "=copy", "=sink", "=trace", "=dispose", "=deepcopy"]
 
 proc `$`*(x: TLockLevel): string =
   if x.ord == UnspecifiedLockLevel.ord: result = "<unspecified>"
@@ -1352,9 +1353,9 @@ proc newType*(kind: TTypeKind, owner: PSym): PType =
       writeStackTrace()
 
 proc mergeLoc(a: var TLoc, b: TLoc) =
-  if a.k == low(a.k): a.k = b.k
-  if a.storage == low(a.storage): a.storage = b.storage
-  a.flags = a.flags + b.flags
+  if a.k == low(typeof(a.k)): a.k = b.k
+  if a.storage == low(typeof(a.storage)): a.storage = b.storage
+  a.flags.incl b.flags
   if a.lode == nil: a.lode = b.lode
   if a.r == nil: a.r = b.r
 
@@ -1379,7 +1380,7 @@ proc assignType*(dest, src: PType) =
   # this fixes 'type TLock = TSysLock':
   if src.sym != nil:
     if dest.sym != nil:
-      dest.sym.flags = dest.sym.flags + (src.sym.flags-{sfExported})
+      dest.sym.flags.incl src.sym.flags-{sfExported}
       if dest.sym.annex == nil: dest.sym.annex = src.sym.annex
       mergeLoc(dest.sym.loc, src.sym.loc)
     else:
@@ -1398,9 +1399,6 @@ proc copyType*(t: PType, owner: PSym, keepId: bool): PType =
 
 proc exactReplica*(t: PType): PType = copyType(t, t.owner, true)
 
-template requiresInit*(t: PType): bool =
-  t.flags * {tfRequiresInit, tfNotNil} != {}
-
 proc copySym*(s: PSym): PSym =
   result = newSym(s.kind, s.name, s.owner, s.info, s.options)
   #result.ast = nil            # BUGFIX; was: s.ast which made problems
@@ -1414,6 +1412,7 @@ proc copySym*(s: PSym): PSym =
   result.position = s.position
   result.loc = s.loc
   result.annex = s.annex      # BUGFIX
+  result.constraint = s.constraint
   if result.kind in {skVar, skLet, skField}:
     result.guard = s.guard
     result.bitsize = s.bitsize
@@ -1486,8 +1485,7 @@ proc isGCedMem*(t: PType): bool {.inline.} =
            t.kind == tyProc and t.callConv == ccClosure
 
 proc propagateToOwner*(owner, elem: PType; propagateHasAsgn = true) =
-  const HaveTheirOwnEmpty = {tySequence, tyOpt, tySet, tyPtr, tyRef, tyProc}
-  owner.flags = owner.flags + (elem.flags * {tfHasMeta, tfTriggersCompileTime})
+  owner.flags.incl elem.flags * {tfHasMeta, tfTriggersCompileTime}
   if tfNotNil in elem.flags:
     if owner.kind in {tyGenericInst, tyGenericBody, tyGenericInvocation}:
       owner.flags.incl tfNotNil
@@ -1499,7 +1497,7 @@ proc propagateToOwner*(owner, elem: PType; propagateHasAsgn = true) =
   if mask != {} and propagateHasAsgn:
     let o2 = owner.skipTypes({tyGenericInst, tyAlias, tySink})
     if o2.kind in {tyTuple, tyObject, tyArray,
-                   tySequence, tyOpt, tySet, tyDistinct, tyOpenArray, tyVarargs}:
+                   tySequence, tySet, tyDistinct, tyOpenArray, tyVarargs}:
       o2.flags.incl mask
       owner.flags.incl mask
 
@@ -1749,7 +1747,7 @@ proc isRoutine*(s: PSym): bool {.inline.} =
 
 proc isCompileTimeProc*(s: PSym): bool {.inline.} =
   result = s.kind == skMacro or
-           s.kind == skProc and sfCompileTime in s.flags
+           s.kind in {skProc, skFunc} and sfCompileTime in s.flags
 
 proc isRunnableExamples*(n: PNode): bool =
   # Templates and generics don't perform symbol lookups.
@@ -1778,7 +1776,7 @@ proc isAtom*(n: PNode): bool {.inline.} =
   result = n.kind >= nkNone and n.kind <= nkNilLit
 
 proc isEmptyType*(t: PType): bool {.inline.} =
-  ## 'void' and 'stmt' types are often equivalent to 'nil' these days:
+  ## 'void' and 'typed' types are often equivalent to 'nil' these days:
   result = t == nil or t.kind in {tyVoid, tyTyped}
 
 proc makeStmtList*(n: PNode): PNode =
@@ -1796,17 +1794,18 @@ proc skipStmtList*(n: PNode): PNode =
   else:
     result = n
 
-proc toVar*(typ: PType): PType =
+proc toVar*(typ: PType; kind: TTypeKind): PType =
   ## If ``typ`` is not a tyVar then it is converted into a `var <typ>` and
   ## returned. Otherwise ``typ`` is simply returned as-is.
   result = typ
-  if typ.kind != tyVar:
-    result = newType(tyVar, typ.owner)
+  if typ.kind != kind:
+    result = newType(kind, typ.owner)
     rawAddSon(result, typ)
 
 proc toRef*(typ: PType): PType =
   ## If ``typ`` is a tyObject then it is converted into a `ref <typ>` and
   ## returned. Otherwise ``typ`` is simply returned as-is.
+  result = typ
   if typ.skipTypes({tyAlias, tyGenericInst}).kind == tyObject:
     result = newType(tyRef, typ.owner)
     rawAddSon(result, typ)
@@ -1833,8 +1832,14 @@ proc isImportedException*(t: PType; conf: ConfigRef): bool =
 proc isInfixAs*(n: PNode): bool =
   return n.kind == nkInfix and n[0].kind == nkIdent and n[0].ident.s == "as"
 
+proc skipColon*(n: PNode): PNode =
+  result = n
+  if n.kind == nkExprColonExpr:
+    result = n[1]
+
 proc findUnresolvedStatic*(n: PNode): PNode =
-  if n.kind == nkSym and n.typ.kind == tyStatic and n.typ.n == nil:
+  # n.typ == nil: see issue #14802
+  if n.kind == nkSym and n.typ != nil and n.typ.kind == tyStatic and n.typ.n == nil:
     return n
 
   for son in n:
@@ -1851,6 +1856,10 @@ when false:
       if n[i].containsNil: return true
 
 template hasDestructor*(t: PType): bool = {tfHasAsgn, tfHasOwned} * t.flags != {}
+
+proc hasDisabledAsgn*(t: PType): bool =
+  t.attachedOps[attachedAsgn] != nil and sfError in t.attachedOps[attachedAsgn].flags
+
 template incompleteType*(t: PType): bool =
   t.sym != nil and {sfForward, sfNoForward} * t.sym.flags == {sfForward}
 
@@ -1928,3 +1937,6 @@ proc toHumanStr*(kind: TSymKind): string =
 proc toHumanStr*(kind: TTypeKind): string =
   ## strips leading `tk`
   result = toHumanStrImpl(kind, 2)
+
+proc skipAddr*(n: PNode): PNode {.inline.} =
+  (if n.kind == nkHiddenAddr: n[0] else: n)

compiler/astalgo.nim

@@ -110,14 +110,14 @@ proc iiTablePut*(t: var TIITable, key, val: int)
 
 # implementation
 
-proc skipConvAndClosure*(n: PNode): PNode =
+proc skipConvCastAndClosure*(n: PNode): PNode =
   result = n
   while true:
     case result.kind
     of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64,
        nkClosure:
       result = result[0]
-    of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+    of nkHiddenStdConv, nkHiddenSubConv, nkConv, nkCast:
       result = result[1]
     else: break
 
@@ -345,7 +345,7 @@ proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int,
     result.addf("$N$1\"n\": $2",     [istr, treeToYamlAux(conf, n.n, marker, indent + 2, maxRecDepth - 1)])
     if card(n.flags) > 0:
       result.addf("$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
-    result.addf("$N$1\"callconv\": $2", [istr, makeYamlString(CallingConvToStr[n.callConv])])
+    result.addf("$N$1\"callconv\": $2", [istr, makeYamlString($n.callConv)])
     result.addf("$N$1\"size\": $2", [istr, rope(n.size)])
     result.addf("$N$1\"align\": $2", [istr, rope(n.align)])
     result.addf("$N$1\"sons\": $2", [istr, sonsRope])
@@ -591,10 +591,17 @@ proc value(this: var DebugPrinter; value: PNode) =
   if this.conf != nil:
     this.key "info"
     this.value $lineInfoToStr(this.conf, value.info)
-  if card(value.flags) > 0:
+  if value.flags != {}:
     this.key "flags"
     this.value value.flags
 
+  if value.typ != nil:
+    this.key "typ"
+    this.value value.typ.kind
+  else:
+    this.key "typ"
+    this.value "nil"
+
   case value.kind
   of nkCharLit..nkUInt64Lit:
     this.key "intVal"
@@ -1048,3 +1055,7 @@ proc listSymbolNames*(symbols: openArray[PSym]): string =
       result.add ", "
     result.add sym.name.s
 
+proc isDiscriminantField*(n: PNode): bool =
+  if n.kind == nkCheckedFieldExpr: sfDiscriminant in n[0][1].sym.flags
+  elif n.kind == nkDotExpr: sfDiscriminant in n[1].sym.flags
+  else: false

compiler/btrees.nim

@@ -79,6 +79,9 @@ proc insert[Key, Val](h: Node[Key, Val], key: Key, val: Val): Node[Key, Val] =
   var j = 0
   if not h.isInternal:
     while j < h.entries:
+      if eq(key, h.keys[j]):
+        h.vals[j] = val
+        return
       if less(key, h.keys[j]): break
       inc j
     for i in countdown(h.entries, j+1):
@@ -132,8 +135,7 @@ proc `$`[Key, Val](b: BTree[Key, Val]): string =
   result = ""
   toString(b.root, "", result)
 
-proc hasNext*[Key, Val](b: BTree[Key, Val]; index: int): bool =
-  result = index < b.entries
+proc hasNext*[Key, Val](b: BTree[Key, Val]; index: int): bool = index < b.entries
 
 proc countSubTree[Key, Val](it: Node[Key, Val]): int =
   if it.isInternal:
@@ -166,68 +168,3 @@ iterator pairs*[Key, Val](b: BTree[Key, Val]): (Key, Val) =
     yield (k, v)
 
 proc len*[Key, Val](b: BTree[Key, Val]): int {.inline.} = b.entries
-
-when isMainModule:
-
-  import random, tables
-
-  proc main =
-    var st = initBTree[string, string]()
-    st.add("www.cs.princeton.edu", "abc")
-    st.add("www.princeton.edu",    "128.112.128.15")
-    st.add("www.yale.edu",         "130.132.143.21")
-    st.add("www.simpsons.com",     "209.052.165.60")
-    st.add("www.apple.com",        "17.112.152.32")
-    st.add("www.amazon.com",       "207.171.182.16")
-    st.add("www.ebay.com",         "66.135.192.87")
-    st.add("www.cnn.com",          "64.236.16.20")
-    st.add("www.google.com",       "216.239.41.99")
-    st.add("www.nytimes.com",      "199.239.136.200")
-    st.add("www.microsoft.com",    "207.126.99.140")
-    st.add("www.dell.com",         "143.166.224.230")
-    st.add("www.slashdot.org",     "66.35.250.151")
-    st.add("www.espn.com",         "199.181.135.201")
-    st.add("www.weather.com",      "63.111.66.11")
-    st.add("www.yahoo.com",        "216.109.118.65")
-
-    assert st.getOrDefault("www.cs.princeton.edu") == "abc"
-    assert st.getOrDefault("www.harvardsucks.com") == ""
-
-    assert st.getOrDefault("www.simpsons.com") == "209.052.165.60"
-    assert st.getOrDefault("www.apple.com") == "17.112.152.32"
-    assert st.getOrDefault("www.ebay.com") == "66.135.192.87"
-    assert st.getOrDefault("www.dell.com") == "143.166.224.230"
-    assert(st.entries == 16)
-
-    for k, v in st:
-      echo k, ": ", v
-
-    when false:
-      var b2 = initBTree[string, string]()
-      const iters = 10_000
-      for i in 1..iters:
-        b2.add($i, $(iters - i))
-      for i in 1..iters:
-        let x = b2.getOrDefault($i)
-        if x != $(iters - i):
-          echo "got ", x, ", but expected ", iters - i
-      echo b2.entries
-
-    when true:
-      var b2 = initBTree[int, string]()
-      var t2 = initTable[int, string]()
-      const iters = 100_000
-      for i in 1..iters:
-        let x = rand(high(int))
-        if not t2.hasKey(x):
-          doAssert b2.getOrDefault(x).len == 0, " what, tree has this element " & $x
-          t2[x] = $x
-          b2.add(x, $x)
-
-      doAssert b2.entries == t2.len
-      echo "unique entries ", b2.entries
-      for k, v in t2:
-        doAssert $k == v
-        doAssert b2.getOrDefault(k) == $k
-
-  main()

compiler/ccgcalls.nim

@@ -21,11 +21,39 @@ proc canRaiseDisp(p: BProc; n: PNode): bool =
     # we have to be *very* conservative:
     result = canRaiseConservative(n)
 
-proc leftAppearsOnRightSide(le, ri: PNode): bool =
+proc preventNrvo(p: BProc; le, ri: PNode): bool =
+  proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
+    var n = le
+    while true:
+      # do NOT follow nkHiddenDeref here!
+      case n.kind
+      of nkSym:
+        # we don't own the location so it escapes:
+        if n.sym.owner != p.prc:
+          return true
+        elif inTryStmt and sfUsedInFinallyOrExcept in n.sym.flags:
+          # it is also an observable store if the location is used
+          # in 'except' or 'finally'
+          return true
+        return false
+      of nkDotExpr, nkBracketExpr, nkObjUpConv, nkObjDownConv,
+          nkCheckedFieldExpr:
+        n = n[0]
+      of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+        n = n[1]
+      else:
+        # cannot analyse the location; assume the worst
+        return true
+
   if le != nil:
     for i in 1..<ri.len:
       let r = ri[i]
       if isPartOf(le, r) != arNo: return true
+    # we use the weaker 'canRaise' here in order to prevent too many
+    # annoying warnings, see #14514
+    if canRaise(ri[0]) and
+        locationEscapes(p, le, p.nestedTryStmts.len > 0):
+      message(p.config, le.info, warnObservableStores, $le)
 
 proc hasNoInit(call: PNode): bool {.inline.} =
   result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
@@ -51,7 +79,7 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
     if isInvalidReturnType(p.config, typ[0]):
       if params != nil: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
@@ -113,9 +141,61 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
 
 proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
 
-proc openArrayLoc(p: BProc, n: PNode): Rope =
-  var a: TLoc
+proc reifiedOpenArray(n: PNode): bool {.inline.} =
+  var x = n
+  while x.kind in {nkAddr, nkHiddenAddr, nkHiddenStdConv, nkHiddenDeref}:
+    x = x[0]
+  if x.kind == nkSym and x.sym.kind == skParam:
+    result = false
+  else:
+    result = true
 
+proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope, Rope) =
+  var a, b, c: TLoc
+  initLocExpr(p, q[1], a)
+  initLocExpr(p, q[2], b)
+  initLocExpr(p, q[3], c)
+  # but first produce the required index checks:
+  if optBoundsCheck in p.options:
+    genBoundsCheck(p, a, b, c)
+  let ty = skipTypes(a.t, abstractVar+{tyPtr})
+  let dest = getTypeDesc(p.module, destType)
+  let lengthExpr = "($1)-($2)+1" % [rdLoc(c), rdLoc(b)]
+  case ty.kind
+  of tyArray:
+    let first = toInt64(firstOrd(p.config, ty))
+    if first == 0:
+      result = ("($3*)(($1)+($2))" % [rdLoc(a), rdLoc(b), dest],
+                lengthExpr)
+    else:
+      result = ("($4*)($1)+(($2)-($3))" %
+        [rdLoc(a), rdLoc(b), intLiteral(first), dest],
+        lengthExpr)
+  of tyOpenArray, tyVarargs:
+    if reifiedOpenArray(q[1]):
+      result = ("($3*)($1.Field0)+($2)" % [rdLoc(a), rdLoc(b), dest],
+                lengthExpr)
+    else:
+      result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest],
+                lengthExpr)
+  of tyUncheckedArray, tyCString:
+    result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest],
+              lengthExpr)
+  of tyString, tySequence:
+    let atyp = skipTypes(a.t, abstractInst)
+    if formalType.skipTypes(abstractInst).kind in {tyVar} and atyp.kind == tyString and
+        optSeqDestructors in p.config.globalOptions:
+      linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+    if atyp.kind in {tyVar} and not compileToCpp(p.module):
+      result = ("($4*)(*$1)$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
+                lengthExpr)
+    else:
+      result = ("($4*)$1$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
+                lengthExpr)
+  else:
+    internalError(p.config, "openArrayLoc: " & typeToString(a.t))
+
+proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
   var q = skipConv(n)
   var skipped = false
   while q.kind == nkStmtListExpr and q.len > 0:
@@ -129,41 +209,26 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
         for i in 0..<q.len-1:
           genStmts(p, q[i])
         q = q.lastSon
-    var b, c: TLoc
-    initLocExpr(p, q[1], a)
-    initLocExpr(p, q[2], b)
-    initLocExpr(p, q[3], c)
-    # but first produce the required index checks:
-    if optBoundsCheck in p.options:
-      genBoundsCheck(p, a, b, c)
-    let ty = skipTypes(a.t, abstractVar+{tyPtr})
-    let dest = getTypeDesc(p.module, n.typ[0])
-    case ty.kind
-    of tyArray:
-      let first = toInt64(firstOrd(p.config, ty))
-      if first == 0:
-        result = "($4*)(($1)+($2)), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dest]
-      else:
-        result = "($5*)($1)+(($2)-($4)), ($3)-($2)+1" %
-          [rdLoc(a), rdLoc(b), rdLoc(c), intLiteral(first), dest]
-    of tyOpenArray, tyVarargs, tyUncheckedArray, tyCString:
-      result = "($4*)($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dest]
-    of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-          not compileToCpp(p.module):
-        result = "($5*)(*$1)$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
-      else:
-        result = "($5*)$1$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p), dest]
-    else:
-      internalError(p.config, "openArrayLoc: " & typeToString(a.t))
+    let (x, y) = genOpenArraySlice(p, q, formalType, n.typ[0])
+    result = x & ", " & y
   else:
-    initLocExpr(p, n, a)
+    var a: TLoc
+    initLocExpr(p, if n.kind == nkHiddenStdConv: n[1] else: n, a)
     case skipTypes(a.t, abstractVar).kind
     of tyOpenArray, tyVarargs:
-      result = "$1, $1Len_0" % [rdLoc(a)]
+      if reifiedOpenArray(n):
+        if a.t.kind in {tyVar, tyLent}:
+          result = "$1->Field0, $1->Field1" % [rdLoc(a)]
+        else:
+          result = "$1.Field0, $1.Field1" % [rdLoc(a)]
+      else:
+        result = "$1, $1Len_0" % [rdLoc(a)]
     of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-            not compileToCpp(p.module):
+      let ntyp = skipTypes(n.typ, abstractInst)
+      if formalType.skipTypes(abstractInst).kind in {tyVar} and ntyp.kind == tyString and
+          optSeqDestructors in p.config.globalOptions:
+        linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+      if ntyp.kind in {tyVar} and not compileToCpp(p.module):
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
         result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
@@ -183,22 +248,33 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
         internalError(p.config, "openArrayLoc: " & typeToString(a.t))
     else: internalError(p.config, "openArrayLoc: " & typeToString(a.t))
 
-proc genArgStringToCString(p: BProc, n: PNode): Rope {.inline.} =
+proc withTmpIfNeeded(p: BProc, a: TLoc, needsTmp: bool): TLoc =
+  # Bug https://github.com/status-im/nimbus-eth2/issues/1549
+  # Aliasing is preferred over stack overflows.
+  # Also don't regress for non ARC-builds, too risky.
+  if needsTmp and a.lode.typ != nil and p.config.selectedGC in {gcArc, gcOrc} and
+      getSize(p.config, a.lode.typ) < 1024:
+    getTemp(p, a.lode.typ, result, needsInit=false)
+    genAssignment(p, result, a, {})
+  else:
+    result = a
+
+proc genArgStringToCString(p: BProc, n: PNode, needsTmp: bool): Rope {.inline.} =
   var a: TLoc
   initLocExpr(p, n[0], a)
-  result = ropecg(p.module, "#nimToCStringConv($1)", [a.rdLoc])
+  ropecg(p.module, "#nimToCStringConv($1)", [withTmpIfNeeded(p, a, needsTmp).rdLoc])
 
-proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
+proc genArg(p: BProc, n: PNode, param: PSym; call: PNode, needsTmp = false): Rope =
   var a: TLoc
   if n.kind == nkStringToCString:
-    result = genArgStringToCString(p, n)
+    result = genArgStringToCString(p, n, needsTmp)
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
     var n = if n.kind != nkHiddenAddr: n else: n[0]
-    result = openArrayLoc(p, n)
+    result = openArrayLoc(p, param.typ, n)
   elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
-    result = addrLoc(p.config, a)
-  elif p.module.compileToCpp and param.typ.kind == tyVar and
+    result = addrLoc(p.config, withTmpIfNeeded(p, a, needsTmp))
+  elif p.module.compileToCpp and param.typ.kind in {tyVar} and
       n.kind == nkHiddenAddr:
     initLocExprSingleUse(p, n[0], a)
     # if the proc is 'importc'ed but not 'importcpp'ed then 'var T' still
@@ -212,26 +288,85 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
       result = rdLoc(a)
   else:
     initLocExprSingleUse(p, n, a)
-    result = rdLoc(a)
+    result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
 
-proc genArgNoParam(p: BProc, n: PNode): Rope =
+proc genArgNoParam(p: BProc, n: PNode, needsTmp = false): Rope =
   var a: TLoc
   if n.kind == nkStringToCString:
-    result = genArgStringToCString(p, n)
+    result = genArgStringToCString(p, n, needsTmp)
   else:
     initLocExprSingleUse(p, n, a)
-    result = rdLoc(a)
+    result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
 
-template genParamLoop(params) {.dirty.} =
-  if i < typ.len:
-    assert(typ.n[i].kind == nkSym)
-    let paramType = typ.n[i]
-    if not paramType.typ.isCompileTimeOnly:
-      if params != nil: params.add(~", ")
-      params.add(genArg(p, ri[i], paramType.sym, ri))
+from dfa import instrTargets, InstrTargetKind
+
+proc potentialAlias(n: PNode, potentialWrites: seq[PNode]): bool =
+  for p in potentialWrites:
+    if instrTargets(p, n) != None:
+      return true
+
+proc skipTrivialIndirections(n: PNode): PNode =
+  result = n
+  while true:
+    case result.kind
+    of {nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv}:
+      result = result[0]
+    of {nkHiddenStdConv, nkHiddenSubConv}:
+      result = result[1]
+    else: break
+
+proc getPotentialWrites(n: PNode, mutate = false): seq[PNode] =
+  case n.kind:
+  of nkLiterals, nkIdent: discard
+  of nkSym:
+    if mutate: result.add n
+  of nkAsgn, nkFastAsgn:
+    result.add getPotentialWrites(n[0], true)
+    result.add getPotentialWrites(n[1], mutate)
+  of nkAddr, nkHiddenAddr:
+    result.add getPotentialWrites(n[0], true)
+  of nkCallKinds: #TODO: Find out why in f += 1, f is a nkSym and not a nkHiddenAddr
+    for s in n.sons:
+      result.add getPotentialWrites(s, true)
   else:
-    if params != nil: params.add(~", ")
-    params.add(genArgNoParam(p, ri[i]))
+    for s in n.sons:
+      result.add getPotentialWrites(s, mutate)
+
+proc getPotentialReads(n: PNode): seq[PNode] =
+  case n.kind:
+  of nkLiterals, nkIdent: discard
+  of nkSym: result.add n
+  else:
+    for s in n.sons:
+      result.add getPotentialReads(s)
+
+proc genParams(p: BProc, ri: PNode, typ: PType): Rope =
+  # We must generate temporaries in cases like #14396
+  # to keep the strict Left-To-Right evaluation
+  var needTmp = newSeq[bool](ri.len - 1)
+  var potentialWrites: seq[PNode]
+  for i in countdown(ri.len - 1, 1):
+    if ri[i].skipTrivialIndirections.kind == nkSym:
+      needTmp[i - 1] = potentialAlias(ri[i], potentialWrites)
+    else:
+      for n in getPotentialReads(ri[i]):
+        if not needTmp[i - 1]:
+          needTmp[i - 1] = potentialAlias(n, potentialWrites)
+      potentialWrites.add getPotentialWrites(ri[i])
+    if ri[i].kind == nkHiddenAddr:
+      # Optimization: don't use a temp, if we would only take the adress anyway
+      needTmp[i - 1] = false
+
+  for i in 1..<ri.len:
+    if i < typ.len:
+      assert(typ.n[i].kind == nkSym)
+      let paramType = typ.n[i]
+      if not paramType.typ.isCompileTimeOnly:
+        if result != nil: result.add(~", ")
+        result.add(genArg(p, ri[i], paramType.sym, ri, needTmp[i-1]))
+    else:
+      if result != nil: result.add(~", ")
+      result.add(genArgNoParam(p, ri[i], needTmp[i-1]))
 
 proc addActualSuffixForHCR(res: var Rope, module: PSym, sym: PSym) =
   if sym.flags * {sfImportc, sfNonReloadable} == {} and sym.loc.k == locProc and
@@ -242,13 +377,13 @@ proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var op: TLoc
   # this is a hotspot in the compiler
   initLocExpr(p, ri[0], op)
-  var params: Rope
   # getUniqueType() is too expensive here:
   var typ = skipTypes(ri[0].typ, abstractInstOwned)
   assert(typ.kind == tyProc)
   assert(typ.len == typ.n.len)
-  for i in 1..<ri.len:
-    genParamLoop(params)
+
+  var params = genParams(p, ri, typ)
+
   var callee = rdLoc(op)
   if p.hcrOn and ri[0].kind == nkSym:
     callee.addActualSuffixForHCR(p.module.module, ri[0].sym)
@@ -256,23 +391,21 @@ proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
 
 proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
 
-  proc getRawProcType(p: BProc, t: PType): Rope =
-    result = getClosureType(p.module, t, clHalf)
-
   proc addComma(r: Rope): Rope =
-    result = if r == nil: r else: r & ~", "
+    if r == nil: r else: r & ~", "
 
   const PatProc = "$1.ClE_0? $1.ClP_0($3$1.ClE_0):(($4)($1.ClP_0))($2)"
   const PatIter = "$1.ClP_0($3$1.ClE_0)" # we know the env exists
+
   var op: TLoc
   initLocExpr(p, ri[0], op)
-  var pl: Rope
 
-  var typ = skipTypes(ri[0].typ, abstractInst)
+  # getUniqueType() is too expensive here:
+  var typ = skipTypes(ri[0].typ, abstractInstOwned)
   assert(typ.kind == tyProc)
-  for i in 1..<ri.len:
-    assert(typ.len == typ.n.len)
-    genParamLoop(pl)
+  assert(typ.len == typ.n.len)
+
+  var pl = genParams(p, ri, typ)
 
   template genCallPattern {.dirty.} =
     if tfIterator in typ.flags:
@@ -280,13 +413,13 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
     else:
       lineF(p, cpsStmts, PatProc & ";$n", [rdLoc(op), pl, pl.addComma, rawProc])
 
-  let rawProc = getRawProcType(p, typ)
+  let rawProc = getClosureType(p.module, typ, clHalf)
   let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
   if typ[0] != nil:
     if isInvalidReturnType(p.config, typ[0]):
       if ri.len > 1: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
           getTemp(p, typ[0], d, needsInit=true)
@@ -338,7 +471,7 @@ proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
     assert(paramType.kind == nkSym)
     if paramType.typ.isCompileTimeOnly:
       result = nil
-    elif typ[i].kind == tyVar and ri[i].kind == nkHiddenAddr:
+    elif typ[i].kind in {tyVar} and ri[i].kind == nkHiddenAddr:
       result = genArgNoParam(p, ri[i][0])
     else:
       result = genArgNoParam(p, ri[i]) #, typ.n[i].sym)
@@ -415,7 +548,7 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   var ri = ri[i]
   while ri.kind == nkObjDownConv: ri = ri[0]
   let t = typ[i].skipTypes({tyGenericInst, tyAlias, tySink})
-  if t.kind == tyVar:
+  if t.kind in {tyVar}:
     let x = if ri.kind == nkHiddenAddr: ri[0] else: ri
     if x.typ.kind == tyPtr:
       result = genArgNoParam(p, x)
@@ -610,18 +743,33 @@ proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
     pl.add(~"];$n")
     line(p, cpsStmts, pl)
 
-proc genCall(p: BProc, e: PNode, d: var TLoc) =
-  if e[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
-    genClosureCall(p, nil, e, d)
-  elif e[0].kind == nkSym and sfInfixCall in e[0].sym.flags:
-    genInfixCall(p, nil, e, d)
-  elif e[0].kind == nkSym and sfNamedParamCall in e[0].sym.flags:
-    genNamedParamCall(p, e, d)
-  else:
-    genPrefixCall(p, nil, e, d)
-  postStmtActions(p)
+proc notYetAlive(n: PNode): bool {.inline.} =
+  let r = getRoot(n)
+  result = r != nil and r.loc.lode == nil
+
+proc isInactiveDestructorCall(p: BProc, e: PNode): bool =
+  #[ Consider this example.
+
+    var :tmpD_3281815
+    try:
+      if true:
+        return
+      let args_3280013 =
+        wasMoved_3281816(:tmpD_3281815)
+        `=_3280036`(:tmpD_3281815, [1])
+        :tmpD_3281815
+    finally:
+      `=destroy_3280027`(args_3280013)
+
+  We want to return early but the 'finally' section is traversed before
+  the 'let args = ...' statement. We exploit this to generate better
+  code for 'return'. ]#
+  result = e.len == 2 and e[0].kind == nkSym and
+    e[0].sym.name.s == "=destroy" and notYetAlive(e[1].skipAddr)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
+  if p.withinBlockLeaveActions > 0 and isInactiveDestructorCall(p, ri):
+    return
   if ri[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
   elif ri[0].kind == nkSym and sfInfixCall in ri[0].sym.flags:
@@ -631,3 +779,5 @@ proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
   else:
     genPrefixCall(p, le, ri, d)
   postStmtActions(p)
+
+proc genCall(p: BProc, e: PNode, d: var TLoc) = genAsgnCall(p, nil, e, d)

compiler/ccgmerge.nim

@@ -48,6 +48,10 @@ const
   NimMergeEndMark = "/*\tNIM_merge_END:*/"
 
 proc genSectionStart*(fs: TCFileSection; conf: ConfigRef): Rope =
+  # useful for debugging and only adds at most a few lines in each file
+  result.add("\n/* section: ")
+  result.add(CFileSectionNames[fs])
+  result.add(" */\n")
   if compilationCachePresent(conf):
     result = nil
     result.add("\n/*\t")
@@ -278,7 +282,7 @@ proc mergeRequired*(m: BModule): bool =
     if m.s[i] != nil:
       #echo "not empty: ", i, " ", m.s[i]
       return true
-  for i in low(TCProcSection)..high(TCProcSection):
+  for i in TCProcSection:
     if m.initProc.s(i) != nil:
       #echo "not empty: ", i, " ", m.initProc.s[i]
       return true
@@ -288,7 +292,7 @@ proc mergeFiles*(cfilename: AbsoluteFile, m: BModule) =
   var old: TMergeSections
   readMergeSections(cfilename, old)
   # do the merge; old section before new section:
-  for i in low(TCFileSection)..high(TCFileSection):
+  for i in TCFileSection:
     m.s[i] = old.f[i] & m.s[i]
-  for i in low(TCProcSection)..high(TCProcSection):
+  for i in TCProcSection:
     m.initProc.s(i) = old.p[i] & m.initProc.s(i)

compiler/ccgreset.nim

@@ -0,0 +1,94 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# included from cgen.nim
+
+## Code specialization instead of the old, incredibly slow 'genericReset'
+## implementation.
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType)
+
+proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
+                     typ: PType) =
+  if n == nil: return
+  case n.kind
+  of nkRecList:
+    for i in 0..<n.len:
+      specializeResetN(p, accessor, n[i], typ)
+  of nkRecCase:
+    if (n[0].kind != nkSym): internalError(p.config, n.info, "specializeResetN")
+    let disc = n[0].sym
+    if disc.loc.r == nil: fillObjectFields(p.module, typ)
+    if disc.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
+    for i in 1..<n.len:
+      let branch = n[i]
+      assert branch.kind in {nkOfBranch, nkElse}
+      if branch.kind == nkOfBranch:
+        genCaseRange(p, branch)
+      else:
+        lineF(p, cpsStmts, "default:$n", [])
+      specializeResetN(p, accessor, lastSon(branch), typ)
+      lineF(p, cpsStmts, "break;$n", [])
+    lineF(p, cpsStmts, "} $n", [])
+    specializeResetT(p, "$1.$2" % [accessor, disc.loc.r], disc.loc.t)
+  of nkSym:
+    let field = n.sym
+    if field.typ.kind == tyVoid: return
+    if field.loc.r == nil: fillObjectFields(p.module, typ)
+    if field.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    specializeResetT(p, "$1.$2" % [accessor, field.loc.r], field.loc.t)
+  else: internalError(p.config, n.info, "specializeResetN()")
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
+  if typ == nil: return
+
+  case typ.kind
+  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred,
+     tySink, tyOwned:
+    specializeResetT(p, accessor, lastSon(typ))
+  of tyArray:
+    let arraySize = lengthOrd(p.config, typ[0])
+    var i: TLoc
+    getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i)
+    linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
+            [i.r, arraySize])
+    specializeResetT(p, ropecg(p.module, "$1[$2]", [accessor, i.r]), typ[1])
+    lineF(p, cpsStmts, "}$n", [])
+  of tyObject:
+    for i in 0..<typ.len:
+      var x = typ[i]
+      if x != nil: x = x.skipTypes(skipPtrs)
+      specializeResetT(p, accessor.parentObj(p.module), x)
+    if typ.n != nil: specializeResetN(p, accessor, typ.n, typ)
+  of tyTuple:
+    let typ = getUniqueType(typ)
+    for i in 0..<typ.len:
+      specializeResetT(p, ropecg(p.module, "$1.Field$2", [accessor, i]), typ[i])
+
+  of tyString, tyRef, tySequence:
+    lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1, NIM_NIL);$n", [accessor])
+
+  of tyProc:
+    if typ.callConv == ccClosure:
+      lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1.ClE_0, NIM_NIL);$n", [accessor])
+      lineCg(p, cpsStmts, "$1.ClP_0 = NIM_NIL;$n", [accessor])
+    else:
+      lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  of tyChar, tyBool, tyEnum, tyInt..tyUInt64:
+    lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
+  of tyCString, tyPointer, tyPtr, tyVar, tyLent:
+    lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  else:
+    discard
+
+proc specializeReset(p: BProc, a: TLoc) =
+  specializeResetT(p, rdLoc(a), a.t)

compiler/ccgstmts.nim

@@ -200,6 +200,7 @@ proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
 
   var stack = newSeq[tuple[fin: PNode, inExcept: bool, label: Natural]](0)
 
+  inc p.withinBlockLeaveActions
   for i in 1..howManyTrys:
     let tryStmt = p.nestedTryStmts.pop
     if p.config.exc == excSetjmp:
@@ -217,6 +218,8 @@ proc blockLeaveActions(p: BProc, howManyTrys, howManyExcepts: int) =
     if finallyStmt != nil:
       genStmts(p, finallyStmt[0])
 
+  dec p.withinBlockLeaveActions
+
   # push old elements again:
   for i in countdown(howManyTrys-1, 0):
     p.nestedTryStmts.add(stack[i])
@@ -269,7 +272,7 @@ proc genGotoVar(p: BProc; value: PNode) =
   else:
     lineF(p, cpsStmts, "goto NIMSTATE_$#;$n", [value.intVal.rope])
 
-proc genBracedInit(p: BProc, n: PNode; isConst: bool): Rope
+proc genBracedInit(p: BProc, n: PNode; isConst: bool; optionalType: PType): Rope
 
 proc potentialValueInit(p: BProc; v: PSym; value: PNode): Rope =
   if lfDynamicLib in v.loc.flags or sfThread in v.flags or p.hcrOn:
@@ -277,7 +280,7 @@ proc potentialValueInit(p: BProc; v: PSym; value: PNode): Rope =
   elif sfGlobal in v.flags and value != nil and isDeepConstExpr(value, p.module.compileToCpp) and
       p.withinLoop == 0 and not containsGarbageCollectedRef(v.typ):
     #echo "New code produced for ", v.name.s, " ", p.config $ value.info
-    result = genBracedInit(p, value, isConst = false)
+    result = genBracedInit(p, value, isConst = false, v.typ)
   else:
     result = nil
 
@@ -379,7 +382,13 @@ proc genSingleVar(p: BProc, v: PSym; vn, value: PNode) =
 
 proc genSingleVar(p: BProc, a: PNode) =
   let v = a[0].sym
-  if sfCompileTime in v.flags: return
+  if sfCompileTime in v.flags:
+    # fix issue #12640
+    # {.global, compileTime.} pragma in proc
+    if sfGlobal in v.flags and p.prc != nil and p.prc.kind == skProc:
+      discard
+    else:
+      return
   genSingleVar(p, v, a[0], a[2])
 
 proc genClosureVar(p: BProc, a: PNode) =
@@ -688,14 +697,15 @@ proc genBreakStmt(p: BProc, t: PNode) =
 
 proc raiseExit(p: BProc) =
   assert p.config.exc == excGoto
-  p.flags.incl nimErrorFlagAccessed
-  if p.nestedTryStmts.len == 0:
-    p.flags.incl beforeRetNeeded
-    # easy case, simply goto 'ret':
-    lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto BeforeRet_;$n", [])
-  else:
-    lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto LA$1_;$n",
-      [p.nestedTryStmts[^1].label])
+  if nimErrorFlagDisabled notin p.flags:
+    p.flags.incl nimErrorFlagAccessed
+    if p.nestedTryStmts.len == 0:
+      p.flags.incl beforeRetNeeded
+      # easy case, simply goto 'ret':
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto BeforeRet_;$n", [])
+    else:
+      lineCg(p, cpsStmts, "if (NIM_UNLIKELY(*nimErr_)) goto LA$1_;$n",
+        [p.nestedTryStmts[^1].label])
 
 proc finallyActions(p: BProc) =
   if p.config.exc != excGoto and p.nestedTryStmts.len > 0 and p.nestedTryStmts[^1].inExcept:
@@ -727,6 +737,7 @@ proc genRaiseStmt(p: BProc, t: PNode) =
     initLocExprSingleUse(p, t[0], a)
     finallyActions(p)
     var e = rdLoc(a)
+    discard getTypeDesc(p.module, t[0].typ)
     var typ = skipTypes(t[0].typ, abstractPtrs)
     # XXX For reasons that currently escape me, this is only required by the new
     # C++ based exception handling:
@@ -859,10 +870,10 @@ proc genStringCase(p: BProc, t: PNode, d: var TLoc) =
     genCaseGeneric(p, t, d, "", "if (#eqStrings($1, $2)) goto $3;$n")
 
 proc branchHasTooBigRange(b: PNode): bool =
-  for i in 0..<b.len-1:
+  for it in b:
     # last son is block
-    if (b[i].kind == nkRange) and
-        b[i][1].intVal - b[i][0].intVal > RangeExpandLimit:
+    if (it.kind == nkRange) and
+        it[1].intVal - it[0].intVal > RangeExpandLimit:
       return true
 
 proc ifSwitchSplitPoint(p: BProc, n: PNode): int =
@@ -981,14 +992,19 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
   inc(p.labels, 2)
   let etmp = p.labels
 
-  lineCg(p, cpsStmts, "std::exception_ptr T$1_ = nullptr;", [etmp])
+  p.procSec(cpsInit).add(ropecg(p.module, "\tstd::exception_ptr T$1_ = nullptr;", [etmp]))
 
   let fin = if t[^1].kind == nkFinally: t[^1] else: nil
   p.nestedTryStmts.add((fin, false, 0.Natural))
 
-  startBlock(p, "try {$n")
-  expr(p, t[0], d)
-  endBlock(p)
+  if t.kind == nkHiddenTryStmt:
+    lineCg(p, cpsStmts, "try {$n", [])
+    expr(p, t[0], d)
+    lineCg(p, cpsStmts, "}$n", [])
+  else:
+    startBlock(p, "try {$n")
+    expr(p, t[0], d)
+    endBlock(p)
 
   # First pass: handle Nim based exceptions:
   lineCg(p, cpsStmts, "catch (#Exception* T$1_) {$n", [etmp+1])
@@ -1030,7 +1046,7 @@ proc genTryCpp(p: BProc, t: PNode, d: var TLoc) =
           let checkFor = if optTinyRtti in p.config.globalOptions:
             genTypeInfo2Name(p.module, typeNode.typ)
           else:
-            genTypeInfo(p.module, typeNode.typ, typeNode.info)
+            genTypeInfoV1(p.module, typeNode.typ, typeNode.info)
           let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
           appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
 
@@ -1208,6 +1224,10 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
   p.nestedTryStmts.add((fin, false, Natural lab))
 
   p.flags.incl nimErrorFlagAccessed
+
+  if not isEmptyType(t.typ) and d.k == locNone:
+    getTemp(p, t.typ, d)
+
   expr(p, t[0], d)
 
   if 1 < t.len and t[1].kind == nkExceptBranch:
@@ -1241,7 +1261,7 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
         let checkFor = if optTinyRtti in p.config.globalOptions:
           genTypeInfo2Name(p.module, t[i][j].typ)
         else:
-          genTypeInfo(p.module, t[i][j].typ, t[i][j].info)
+          genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
         let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
         appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
 
@@ -1278,7 +1298,7 @@ proc genTryGoto(p: BProc; t: PNode; d: var TLoc) =
       #    handler present or only handlers that did not match.
       linefmt(p, cpsStmts, "*nimErr_ = oldNimErrFin$1_;$n", [lab])
     endBlock(p)
-  if p.prc != nil: raiseExit(p)
+  raiseExit(p)
   if hasExcept: inc p.withinTryWithExcept
 
 proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
@@ -1333,13 +1353,13 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
       linefmt(p, cpsStmts, "$1.status = _setjmp($1.context);$n", [safePoint])
     else:
       linefmt(p, cpsStmts, "$1.status = setjmp($1.context);$n", [safePoint])
-    startBlock(p, "if ($1.status == 0) {$n", [safePoint])
+    lineCg(p, cpsStmts, "if ($1.status == 0) {$n", [safePoint])
   let fin = if t[^1].kind == nkFinally: t[^1] else: nil
   p.nestedTryStmts.add((fin, quirkyExceptions, 0.Natural))
   expr(p, t[0], d)
   if not quirkyExceptions:
     linefmt(p, cpsStmts, "#popSafePoint();$n", [])
-    endBlock(p)
+    lineCg(p, cpsStmts, "}$n", [])
     startBlock(p, "else {$n")
     linefmt(p, cpsStmts, "#popSafePoint();$n", [])
     genRestoreFrameAfterException(p)
@@ -1369,7 +1389,7 @@ proc genTrySetjmp(p: BProc, t: PNode, d: var TLoc) =
         let checkFor = if optTinyRtti in p.config.globalOptions:
           genTypeInfo2Name(p.module, t[i][j].typ)
         else:
-          genTypeInfo(p.module, t[i][j].typ, t[i][j].info)
+          genTypeInfoV1(p.module, t[i][j].typ, t[i][j].info)
         let memberName = if p.module.compileToCpp: "m_type" else: "Sup.m_type"
         appcg(p.module, orExpr, "#isObj(#nimBorrowCurrentException()->$1, $2)", [memberName, checkFor])
 
@@ -1440,7 +1460,7 @@ proc genAsmOrEmitStmt(p: BProc, t: PNode, isAsmStmt=false): Rope =
         else:
           # ignore empty lines
           result.add("\"")
-          result.add(x)
+          result.add(x.replace("\"", "\\\""))
           result.add("\\n\"\n")
   else:
     res.add("\L")
@@ -1484,26 +1504,17 @@ proc genPragma(p: BProc, n: PNode) =
     of wEmit: genEmit(p, it)
     of wInjectStmt:
       var p = newProc(nil, p.module)
-      p.options = p.options - {optLineTrace, optStackTrace}
+      p.options.excl {optLineTrace, optStackTrace}
       genStmts(p, it[1])
       p.module.injectStmt = p.s(cpsStmts)
     else: discard
 
-proc fieldDiscriminantCheckNeeded(p: BProc, asgn: PNode): bool =
-  if optFieldCheck in p.options:
-    var le = asgn[0]
-    if le.kind == nkCheckedFieldExpr:
-      var field = le[0][1].sym
-      result = sfDiscriminant in field.flags
-    elif le.kind == nkDotExpr:
-      var field = le[1].sym
-      result = sfDiscriminant in field.flags
 
 proc genDiscriminantCheck(p: BProc, a, tmp: TLoc, objtype: PType,
                           field: PSym) =
   var t = skipTypes(objtype, abstractVar)
   assert t.kind == tyObject
-  discard genTypeInfo(p.module, t, a.lode.info)
+  discard genTypeInfoV1(p.module, t, a.lode.info)
   if not containsOrIncl(p.module.declaredThings, field.id):
     appcg(p.module, cfsVars, "extern $1",
           [discriminatorTableDecl(p.module, t, field)])
@@ -1534,28 +1545,24 @@ proc asgnFieldDiscriminant(p: BProc, e: PNode) =
   initLocExpr(p, e[0], a)
   getTemp(p, a.t, tmp)
   expr(p, e[1], tmp)
-  let field = dotExpr[1].sym
-  if optTinyRtti in p.config.globalOptions:
-    let t = dotExpr[0].typ.skipTypes(abstractInst)
-    var oldVal, newVal: TLoc
-    genCaseObjDiscMapping(p, e[0], t, field, oldVal)
-    genCaseObjDiscMapping(p, e[1], t, field, newVal)
-    lineCg(p, cpsStmts,
-          "if ($1 != $2) { #raiseObjectCaseTransition(); $3}$n",
-          [rdLoc(oldVal), rdLoc(newVal), raiseInstr(p)])
-  else:
+  if optTinyRtti notin p.config.globalOptions:
+    let field = dotExpr[1].sym
     genDiscriminantCheck(p, a, tmp, dotExpr[0].typ, field)
+    message(p.config, e.info, warnCaseTransition)
   genAssignment(p, a, tmp, {})
 
 proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
   if e[0].kind == nkSym and sfGoto in e[0].sym.flags:
     genLineDir(p, e)
     genGotoVar(p, e[1])
-  elif not fieldDiscriminantCheckNeeded(p, e):
+  elif optFieldCheck in p.options and isDiscriminantField(e[0]):
+    genLineDir(p, e)
+    asgnFieldDiscriminant(p, e)
+  else:
     let le = e[0]
     let ri = e[1]
     var a: TLoc
-    discard getTypeDesc(p.module, le.typ.skipTypes(skipPtrs))
+    discard getTypeDesc(p.module, le.typ.skipTypes(skipPtrs), skVar)
     initLoc(a, locNone, le, OnUnknown)
     a.flags.incl(lfEnforceDeref)
     a.flags.incl(lfPrepareForMutation)
@@ -1565,10 +1572,6 @@ proc genAsgn(p: BProc, e: PNode, fastAsgn: bool) =
     assert(a.t != nil)
     genLineDir(p, ri)
     loadInto(p, le, ri, a)
-  else:
-    genLineDir(p, e)
-    asgnFieldDiscriminant(p, e)
-    message(p.config, e.info, warnCaseTransition)
 
 proc genStmts(p: BProc, t: PNode) =
   var a: TLoc

compiler/ccgthreadvars.nim

@@ -42,11 +42,12 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
 proc generateThreadLocalStorage(m: BModule) =
   if m.g.nimtv != nil and (usesThreadVars in m.flags or sfMainModule in m.module.flags):
     for t in items(m.g.nimtvDeps): discard getTypeDesc(m, t)
+    finishTypeDescriptions(m)
     m.s[cfsSeqTypes].addf("typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
 
 proc generateThreadVarsSize(m: BModule) =
   if m.g.nimtv != nil:
-    let externc = if m.config.cmd == cmdCompileToCpp or
+    let externc = if m.config.backend == backendCpp or
                        sfCompileToCpp in m.module.flags: "extern \"C\" "
                   else: ""
     m.s[cfsProcs].addf(

compiler/ccgtrav.nim

@@ -173,7 +173,7 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope =
          [result, markerName, getModuleDllPath(m)])
 
 proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope =
-  discard genTypeInfo(m, s.loc.t, info)
+  discard genTypeInfoV1(m, s.loc.t, info)
 
   var c: TTraversalClosure
   var p = newProc(nil, m)

compiler/ccgtypes.nim

@@ -149,7 +149,7 @@ proc mapSetType(conf: ConfigRef; typ: PType): TCTypeKind =
   of 8: result = ctInt64
   else: result = ctArray
 
-proc mapType(conf: ConfigRef; typ: PType): TCTypeKind =
+proc mapType(conf: ConfigRef; typ: PType; kind: TSymKind): TCTypeKind =
   ## Maps a Nim type to a C type
   case typ.kind
   of tyNone, tyTyped: result = ctVoid
@@ -157,14 +157,17 @@ proc mapType(conf: ConfigRef; typ: PType): TCTypeKind =
   of tyChar: result = ctChar
   of tyNil: result = ctPtr
   of tySet: result = mapSetType(conf, typ)
-  of tyOpenArray, tyArray, tyVarargs, tyUncheckedArray: result = ctArray
+  of tyOpenArray, tyVarargs:
+    if kind == skParam: result = ctArray
+    else: result = ctStruct
+  of tyArray, tyUncheckedArray: result = ctArray
   of tyObject, tyTuple: result = ctStruct
   of tyUserTypeClasses:
     doAssert typ.isResolvedUserTypeClass
-    return mapType(conf, typ.lastSon)
+    return mapType(conf, typ.lastSon, kind)
   of tyGenericBody, tyGenericInst, tyGenericParam, tyDistinct, tyOrdinal,
      tyTypeDesc, tyAlias, tySink, tyInferred, tyOwned:
-    result = mapType(conf, lastSon(typ))
+    result = mapType(conf, lastSon(typ), kind)
   of tyEnum:
     if firstOrd(conf, typ) < 0:
       result = ctInt32
@@ -175,7 +178,7 @@ proc mapType(conf: ConfigRef; typ: PType): TCTypeKind =
       of 4: result = ctInt32
       of 8: result = ctInt64
       else: result = ctInt32
-  of tyRange: result = mapType(conf, typ[0])
+  of tyRange: result = mapType(conf, typ[0], kind)
   of tyPtr, tyVar, tyLent, tyRef:
     var base = skipTypes(typ.lastSon, typedescInst)
     case base.kind
@@ -192,14 +195,14 @@ proc mapType(conf: ConfigRef; typ: PType): TCTypeKind =
   of tyInt..tyUInt64:
     result = TCTypeKind(ord(typ.kind) - ord(tyInt) + ord(ctInt))
   of tyStatic:
-    if typ.n != nil: result = mapType(conf, lastSon typ)
+    if typ.n != nil: result = mapType(conf, lastSon typ, kind)
     else: doAssert(false, "mapType")
   else: doAssert(false, "mapType")
 
 proc mapReturnType(conf: ConfigRef; typ: PType): TCTypeKind =
   #if skipTypes(typ, typedescInst).kind == tyArray: result = ctPtr
   #else:
-  result = mapType(conf, typ)
+  result = mapType(conf, typ, skResult)
 
 proc isImportedType(t: PType): bool =
   result = t.sym != nil and sfImportc in t.sym.flags
@@ -209,7 +212,7 @@ proc isImportedCppType(t: PType): bool =
   result = (t.sym != nil and sfInfixCall in t.sym.flags) or
            (x.sym != nil and sfInfixCall in x.sym.flags)
 
-proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope
+proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKind): Rope
 
 proc isObjLackingTypeField(typ: PType): bool {.inline.} =
   result = (typ.kind == tyObject) and ((tfFinal in typ.flags) and
@@ -222,7 +225,7 @@ proc isInvalidReturnType(conf: ConfigRef; rettype: PType): bool =
   # is necessary for proper code generation of assignments.
   if rettype == nil: result = true
   else:
-    case mapType(conf, rettype)
+    case mapType(conf, rettype, skResult)
     of ctArray:
       result = not (skipTypes(rettype, typedescInst).kind in
           {tyVar, tyLent, tyRef, tyPtr})
@@ -238,7 +241,7 @@ const
     "N_STDCALL", "N_CDECL", "N_SAFECALL",
     "N_SYSCALL", # this is probably not correct for all platforms,
                  # but one can #define it to what one wants
-    "N_INLINE", "N_NOINLINE", "N_FASTCALL", "N_CLOSURE", "N_NOCONV"]
+    "N_INLINE", "N_NOINLINE", "N_FASTCALL", "N_THISCALL", "N_CLOSURE", "N_NOCONV"]
 
 proc cacheGetType(tab: TypeCache; sig: SigHash): Rope =
   # returns nil if we need to declare this type
@@ -247,8 +250,13 @@ proc cacheGetType(tab: TypeCache; sig: SigHash): Rope =
   result = tab.getOrDefault(sig)
 
 proc addAbiCheck(m: BModule, t: PType, name: Rope) =
-  if isDefined(m.config, "checkabi") and (let size = getSize(m.config, t); size != szUnknownSize):
-    m.s[cfsTypeInfo].addf("NIM_CHECK_SIZE($1, $2);$n", [name, rope(size)])
+  if isDefined(m.config, "checkAbi") and (let size = getSize(m.config, t); size != szUnknownSize):
+    var msg = "backend & Nim disagree on size for: "
+    msg.addTypeHeader(m.config, t)
+    var msg2 = ""
+    msg2.addQuoted msg # not a hostspot so extra allocation doesn't matter
+    m.s[cfsTypeInfo].addf("NIM_STATIC_ASSERT(sizeof($1) == $2, $3);$n", [name, rope(size), msg2.rope])
+    # see `testCodegenABICheck` for example error message it generates
 
 proc ccgIntroducedPtr(conf: ConfigRef; s: PSym, retType: PType): bool =
   var pt = skipTypes(s.typ, typedescInst)
@@ -263,19 +271,19 @@ proc ccgIntroducedPtr(conf: ConfigRef; s: PSym, retType: PType): bool =
       result = true
     elif (optByRef in s.options) or (getSize(conf, pt) > conf.target.floatSize * 3):
       result = true           # requested anyway
-    elif retType != nil and retType.kind == tyLent:
-      result = true
     elif (tfFinal in pt.flags) and (pt[0] == nil):
       result = false          # no need, because no subtyping possible
     else:
       result = true           # ordinary objects are always passed by reference,
                               # otherwise casting doesn't work
   of tyTuple:
-    if retType != nil and retType.kind == tyLent:
-      result = true
-    else:
-      result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options)
-  else: result = false
+    result = (getSize(conf, pt) > conf.target.floatSize*3) or (optByRef in s.options)
+  else:
+    result = false
+  # first parameter and return type is 'lent T'? --> use pass by pointer
+  if s.position == 0 and retType != nil and retType.kind == tyLent:
+    result = not (pt.kind in {tyVar, tyArray, tyOpenArray, tyVarargs, tyRef, tyPtr, tyPointer} or
+      pt.kind == tySet and mapSetType(conf, pt) == ctArray)
 
 proc fillResult(conf: ConfigRef; param: PNode) =
   fillLoc(param.sym.loc, locParam, param, ~"Result",
@@ -328,7 +336,6 @@ proc getSimpleTypeDesc(m: BModule, typ: PType): Rope =
     let sig = hashType typ
     if cacheGetType(m.typeCache, sig) == nil:
       m.typeCache[sig] = result
-      addAbiCheck(m, typ, result)
 
 proc pushType(m: BModule, typ: PType) =
   for i in 0..high(m.typeStack):
@@ -343,8 +350,8 @@ proc getTypePre(m: BModule, typ: PType; sig: SigHash): Rope =
     if result == nil: result = cacheGetType(m.typeCache, sig)
 
 proc structOrUnion(t: PType): Rope =
-  let cachedUnion {.global.} = rope("union")
-  let cachedStruct {.global.} = rope("struct")
+  let cachedUnion = rope("union")
+  let cachedStruct = rope("struct")
   let t = t.skipTypes({tyAlias, tySink})
   if tfUnion in t.flags: cachedUnion
   else: cachedStruct
@@ -364,7 +371,7 @@ proc getTypeForward(m: BModule, typ: PType; sig: SigHash): Rope =
   if result != nil: return
   result = getTypePre(m, typ, sig)
   if result != nil: return
-  let concrete = typ.skipTypes(abstractInst + {tyOpt})
+  let concrete = typ.skipTypes(abstractInst)
   case concrete.kind
   of tySequence, tyTuple, tyObject:
     result = getTypeName(m, typ, sig)
@@ -376,7 +383,7 @@ proc getTypeForward(m: BModule, typ: PType; sig: SigHash): Rope =
     doAssert m.forwTypeCache[sig] == result
   else: internalError(m.config, "getTypeForward(" & $typ.kind & ')')
 
-proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet): Rope =
+proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet; kind: TSymKind): Rope =
   ## like getTypeDescAux but creates only a *weak* dependency. In other words
   ## we know we only need a pointer to it so we only generate a struct forward
   ## declaration:
@@ -384,7 +391,7 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet): Rope =
   case etB.kind
   of tyObject, tyTuple:
     if isImportedCppType(etB) and t.kind == tyGenericInst:
-      result = getTypeDescAux(m, t, check)
+      result = getTypeDescAux(m, t, check, kind)
     else:
       result = getTypeForward(m, t, hashType(t))
       pushType(m, t)
@@ -414,18 +421,18 @@ proc getTypeDescWeak(m: BModule; t: PType; check: var IntSet): Rope =
       result = getTypeForward(m, t, sig) & seqStar(m)
     pushType(m, t)
   else:
-    result = getTypeDescAux(m, t, check)
+    result = getTypeDescAux(m, t, check, kind)
 
 proc getSeqPayloadType(m: BModule; t: PType): Rope =
   var check = initIntSet()
-  result = getTypeDescWeak(m, t, check) & "_Content"
+  result = getTypeDescWeak(m, t, check, skParam) & "_Content"
   #result = getTypeForward(m, t, hashType(t)) & "_Content"
 
 proc seqV2ContentType(m: BModule; t: PType; check: var IntSet) =
   let sig = hashType(t)
   let result = cacheGetType(m.typeCache, sig)
   if result == nil:
-    discard getTypeDescAux(m, t, check)
+    discard getTypeDescAux(m, t, check, skVar)
   else:
     # little hack for now to prevent multiple definitions of the same
     # Seq_Content:
@@ -434,7 +441,7 @@ $3ifndef $2_Content_PP
 $3define $2_Content_PP
 struct $2_Content { NI cap; $1 data[SEQ_DECL_SIZE];};
 $3endif$N
-      """, [getTypeDescAux(m, t.skipTypes(abstractInst)[0], check), result, rope"#"])
+      """, [getTypeDescAux(m, t.skipTypes(abstractInst)[0], check, skVar), result, rope"#"])
 
 proc paramStorageLoc(param: PSym): TStorageLoc =
   if param.typ.skipTypes({tyVar, tyLent, tyTypeDesc}).kind notin {
@@ -450,7 +457,7 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
   if t[0] == nil or isInvalidReturnType(m.config, t[0]):
     rettype = ~"void"
   else:
-    rettype = getTypeDescAux(m, t[0], check)
+    rettype = getTypeDescAux(m, t[0], check, skResult)
   for i in 1..<t.n.len:
     if t.n[i].kind != nkSym: internalError(m.config, t.n.info, "genProcParams")
     var param = t.n[i].sym
@@ -459,15 +466,17 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
     fillLoc(param.loc, locParam, t.n[i], mangleParamName(m, param),
             param.paramStorageLoc)
     if ccgIntroducedPtr(m.config, param, t[0]):
-      params.add(getTypeDescWeak(m, param.typ, check))
+      params.add(getTypeDescWeak(m, param.typ, check, skParam))
       params.add(~"*")
       incl(param.loc.flags, lfIndirect)
       param.loc.storage = OnUnknown
     elif weakDep:
-      params.add(getTypeDescWeak(m, param.typ, check))
+      params.add(getTypeDescWeak(m, param.typ, check, skParam))
     else:
-      params.add(getTypeDescAux(m, param.typ, check))
+      params.add(getTypeDescAux(m, param.typ, check, skParam))
     params.add(~" ")
+    if sfNoalias in param.flags:
+      params.add(~"NIM_NOALIAS ")
     params.add(param.loc.r)
     # declare the len field for open arrays:
     var arr = param.typ
@@ -484,10 +493,10 @@ proc genProcParams(m: BModule, t: PType, rettype, params: var Rope,
     var arr = t[0]
     if params != nil: params.add(", ")
     if mapReturnType(m.config, t[0]) != ctArray:
-      params.add(getTypeDescWeak(m, arr, check))
+      params.add(getTypeDescWeak(m, arr, check, skResult))
       params.add("*")
     else:
-      params.add(getTypeDescAux(m, arr, check))
+      params.add(getTypeDescAux(m, arr, check, skResult))
     params.addf(" Result", [])
   if t.callConv == ccClosure and declareEnvironment:
     if params != nil: params.add(", ")
@@ -556,19 +565,21 @@ proc genRecordFieldsAux(m: BModule, n: PNode,
     # have to recurse via 'getTypeDescAux'. And not doing so prevents problems
     # with heavily templatized C++ code:
     if not isImportedCppType(rectype):
+      let noAlias = if sfNoalias in field.flags: ~" NIM_NOALIAS" else: nil
+
       let fieldType = field.loc.lode.typ.skipTypes(abstractInst)
       if fieldType.kind == tyUncheckedArray:
         result.addf("$1 $2[SEQ_DECL_SIZE];$n",
-            [getTypeDescAux(m, fieldType.elemType, check), sname])
+            [getTypeDescAux(m, fieldType.elemType, check, skField), sname])
       elif fieldType.kind == tySequence:
         # we need to use a weak dependency here for trecursive_table.
-        result.addf("$1 $2;$n", [getTypeDescWeak(m, field.loc.t, check), sname])
+        result.addf("$1$3 $2;$n", [getTypeDescWeak(m, field.loc.t, check, skField), sname, noAlias])
       elif field.bitsize != 0:
-        result.addf("$1 $2:$3;$n", [getTypeDescAux(m, field.loc.t, check), sname, rope($field.bitsize)])
+        result.addf("$1$4 $2:$3;$n", [getTypeDescAux(m, field.loc.t, check, skField), sname, rope($field.bitsize), noAlias])
       else:
         # don't use fieldType here because we need the
         # tyGenericInst for C++ template support
-        result.addf("$1 $2;$n", [getTypeDescAux(m, field.loc.t, check), sname])
+        result.addf("$1$3 $2;$n", [getTypeDescAux(m, field.loc.t, check, skField), sname, noAlias])
   else: internalError(m.config, n.info, "genRecordFieldsAux()")
 
 proc getRecordFields(m: BModule, typ: PType, check: var IntSet): Rope =
@@ -603,11 +614,14 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
       if (typ.sym != nil and sfPure in typ.sym.flags) or tfFinal in typ.flags:
         appcg(m, result, " {$n", [])
       else:
-        appcg(m, result, " {$n#TNimType* m_type;$n", [])
+        if optTinyRtti in m.config.globalOptions:
+          appcg(m, result, " {$n#TNimTypeV2* m_type;$n", [])
+        else:
+          appcg(m, result, " {$n#TNimType* m_type;$n", [])
         hasField = true
     elif m.compileToCpp:
       appcg(m, result, " : public $1 {$n",
-                      [getTypeDescAux(m, typ[0].skipTypes(skipPtrs), check)])
+                      [getTypeDescAux(m, typ[0].skipTypes(skipPtrs), check, skField)])
       if typ.isException and m.config.exc == excCpp:
         when false:
           appcg(m, result, "virtual void raise() { throw *this; }$n", []) # required for polymorphic exceptions
@@ -620,7 +634,7 @@ proc getRecordDesc(m: BModule, typ: PType, name: Rope,
       hasField = true
     else:
       appcg(m, result, " {$n  $1 Sup;$n",
-                      [getTypeDescAux(m, typ[0].skipTypes(skipPtrs), check)])
+                      [getTypeDescAux(m, typ[0].skipTypes(skipPtrs), check, skField)])
       hasField = true
   else:
     result.addf(" {$n", [name])
@@ -640,7 +654,7 @@ proc getTupleDesc(m: BModule, typ: PType, name: Rope,
   var desc: Rope = nil
   for i in 0..<typ.len:
     desc.addf("$1 Field$2;$n",
-         [getTypeDescAux(m, typ[i], check), rope(i)])
+         [getTypeDescAux(m, typ[i], check, skField), rope(i)])
   if desc == nil: result.add("char dummy;\L")
   else: result.add(desc)
   result.add("};\L")
@@ -674,8 +688,22 @@ proc resolveStarsInCppType(typ: PType, idx, stars: int): PType =
       result = if result.kind == tyGenericInst: result[1]
                else: result.elemType
 
-proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
+proc getOpenArrayDesc(m: BModule, t: PType, check: var IntSet; kind: TSymKind): Rope =
+  let sig = hashType(t)
+  if kind == skParam:
+    result = getTypeDescWeak(m, t[0], check, kind) & "*"
+  else:
+    result = cacheGetType(m.typeCache, sig)
+    if result == nil:
+      result = getTypeName(m, t, sig)
+      m.typeCache[sig] = result
+      let elemType = getTypeDescWeak(m, t[0], check, kind)
+      m.s[cfsTypes].addf("typedef struct {$n$2* Field0;$nNI Field1;$n} $1;$n",
+                         [result, elemType])
+
+proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet; kind: TSymKind): Rope =
   # returns only the type's name
+
   var t = origTyp.skipTypes(irrelevantForBackend-{tyOwned})
   if containsOrIncl(check, t.id):
     if not (isImportedCppType(origTyp) or isImportedCppType(t)):
@@ -686,17 +714,22 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
   if t.sym != nil: useHeader(m, t.sym)
   if t != origTyp and origTyp.sym != nil: useHeader(m, origTyp.sym)
   let sig = hashType(origTyp)
+
+  defer: # defer is the simplest in this case
+    if isImportedType(t) and not m.typeABICache.containsOrIncl(sig):
+      addAbiCheck(m, t, result)
+
   result = getTypePre(m, t, sig)
-  if result != nil:
+  if result != nil and t.kind != tyOpenArray:
     excl(check, t.id)
     return
   case t.kind
   of tyRef, tyPtr, tyVar, tyLent:
-    var star = if t.kind == tyVar and tfVarIsPtr notin origTyp.flags and
+    var star = if t.kind in {tyVar} and tfVarIsPtr notin origTyp.flags and
                     compileToCpp(m): "&" else: "*"
     var et = origTyp.skipTypes(abstractInst).lastSon
     var etB = et.skipTypes(abstractInst)
-    if mapType(m.config, t) == ctPtrToArray:
+    if mapType(m.config, t, kind) == ctPtrToArray and (etB.kind != tyOpenArray or kind == skParam):
       if etB.kind == tySet:
         et = getSysType(m.g.graph, unknownLineInfo, tyUInt8)
       else:
@@ -706,7 +739,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
     case etB.kind
     of tyObject, tyTuple:
       if isImportedCppType(etB) and et.kind == tyGenericInst:
-        result = getTypeDescAux(m, et, check) & star
+        result = getTypeDescAux(m, et, check, kind) & star
       else:
         # no restriction! We have a forward declaration for structs
         let name = getTypeForward(m, et, hashType et)
@@ -714,7 +747,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
         m.typeCache[sig] = result
     of tySequence:
       if optSeqDestructors in m.config.globalOptions:
-        result = getTypeDescWeak(m, et, check) & star
+        result = getTypeDescWeak(m, et, check, kind) & star
         m.typeCache[sig] = result
       else:
         # no restriction! We have a forward declaration for structs
@@ -724,11 +757,10 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
         pushType(m, et)
     else:
       # else we have a strong dependency  :-(
-      result = getTypeDescAux(m, et, check) & star
+      result = getTypeDescAux(m, et, check, kind) & star
       m.typeCache[sig] = result
   of tyOpenArray, tyVarargs:
-    result = getTypeDescWeak(m, t[0], check) & "*"
-    m.typeCache[sig] = result
+    result = getOpenArrayDesc(m, t, check, kind)
   of tyEnum:
     result = cacheGetType(m.typeCache, sig)
     if result == nil:
@@ -774,7 +806,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
              [result, rettype, desc])
   of tySequence:
     if optSeqDestructors in m.config.globalOptions:
-      result = getTypeDescWeak(m, t, check)
+      result = getTypeDescWeak(m, t, check, kind)
     else:
       # we cannot use getTypeForward here because then t would be associated
       # with the name of the struct, not with the pointer to the struct:
@@ -795,11 +827,11 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
           if m.compileToCpp:
             appcg(m, m.s[cfsSeqTypes],
                 cppSeq & "  $1 data[SEQ_DECL_SIZE];$n" &
-                "};$n", [getTypeDescAux(m, t[0], check), result])
+                "};$n", [getTypeDescAux(m, t[0], check, kind), result])
           else:
             appcg(m, m.s[cfsSeqTypes],
                 cSeq & "  $1 data[SEQ_DECL_SIZE];$n" &
-                "};$n", [getTypeDescAux(m, t[0], check), result])
+                "};$n", [getTypeDescAux(m, t[0], check, kind), result])
         else:
           result = rope("TGenericSeq")
       result.add(seqStar(m))
@@ -807,7 +839,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
     result = getTypeName(m, origTyp, sig)
     m.typeCache[sig] = result
     if not isImportedType(t):
-      let foo = getTypeDescAux(m, t[0], check)
+      let foo = getTypeDescAux(m, t[0], check, kind)
       m.s[cfsTypes].addf("typedef $1 $2[1];$n", [foo, result])
   of tyArray:
     var n: BiggestInt = toInt64(lengthOrd(m.config, t))
@@ -815,10 +847,9 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
     result = getTypeName(m, origTyp, sig)
     m.typeCache[sig] = result
     if not isImportedType(t):
-      let foo = getTypeDescAux(m, t[1], check)
+      let foo = getTypeDescAux(m, t[1], check, kind)
       m.s[cfsTypes].addf("typedef $1 $2[$3];$n",
            [foo, result, rope(n)])
-    else: addAbiCheck(m, t, result)
   of tyObject, tyTuple:
     if isImportedCppType(t) and origTyp.kind == tyGenericInst:
       let cppName = getTypeName(m, t, sig)
@@ -832,7 +863,7 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
           internalAssert m.config, ty.n != nil
           result.add ty.n.renderTree
         else:
-          result.add getTypeDescAux(m, ty, check)
+          result.add getTypeDescAux(m, ty, check, kind)
 
       while i < cppName.data.len:
         if cppName.data[i] == '\'':
@@ -879,7 +910,8 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
                       else: getTupleDesc(m, t, result, check)
         if not isImportedType(t):
           m.s[cfsTypes].add(recdesc)
-        elif tfIncompleteStruct notin t.flags: addAbiCheck(m, t, result)
+        elif tfIncompleteStruct notin t.flags:
+          discard # addAbiCheck(m, t, result) # already handled elsewhere
   of tySet:
     # Don't use the imported name as it may be scoped: 'Foo::SomeKind'
     result = $t.kind & '_' & t.lastSon.typeName & $t.lastSon.hashType
@@ -892,16 +924,16 @@ proc getTypeDescAux(m: BModule, origTyp: PType, check: var IntSet): Rope =
              [result, rope(getSize(m.config, t))])
   of tyGenericInst, tyDistinct, tyOrdinal, tyTypeDesc, tyAlias, tySink, tyOwned,
      tyUserTypeClass, tyUserTypeClassInst, tyInferred:
-    result = getTypeDescAux(m, lastSon(t), check)
+    result = getTypeDescAux(m, lastSon(t), check, kind)
   else:
     internalError(m.config, "getTypeDescAux(" & $t.kind & ')')
     result = nil
   # fixes bug #145:
   excl(check, t.id)
 
-proc getTypeDesc(m: BModule, typ: PType): Rope =
+proc getTypeDesc(m: BModule, typ: PType; kind = skParam): Rope =
   var check = initIntSet()
-  result = getTypeDescAux(m, typ, check)
+  result = getTypeDescAux(m, typ, check, kind)
 
 type
   TClosureTypeKind = enum ## In C closures are mapped to 3 different things.
@@ -933,8 +965,9 @@ proc finishTypeDescriptions(m: BModule) =
     if optSeqDestructors in m.config.globalOptions and t.skipTypes(abstractInst).kind == tySequence:
       seqV2ContentType(m, t, check)
     else:
-      discard getTypeDescAux(m, t, check)
+      discard getTypeDescAux(m, t, check, skParam)
     inc(i)
+  m.typeStack.setLen 0
 
 template cgDeclFrmt*(s: PSym): string =
   s.constraint.strVal
@@ -979,7 +1012,7 @@ proc genProcHeader(m: BModule, prc: PSym, asPtr: bool = false): Rope =
 
 # ------------------ type info generation -------------------------------------
 
-proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope
+proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope
 proc getNimNode(m: BModule): Rope =
   result = "$1[$2]" % [m.typeNodesName, rope(m.typeNodes)]
   inc(m.typeNodes)
@@ -999,11 +1032,14 @@ proc genTypeInfoAuxBase(m: BModule; typ, origType: PType;
   let nameHcr = tiNameForHcr(m, name)
 
   var size: Rope
-  if tfIncompleteStruct in typ.flags: size = rope"void*"
-  else: size = getTypeDesc(m, origType)
+  if tfIncompleteStruct in typ.flags:
+    size = rope"void*"
+  else:
+    size = getTypeDesc(m, origType, skVar)
   m.s[cfsTypeInit3].addf(
-       "$1.size = sizeof($2);$n" & "$1.kind = $3;$n" & "$1.base = $4;$n",
-       [nameHcr, size, rope(nimtypeKind), base])
+    "$1.size = sizeof($2);$n$1.align = NIM_ALIGNOF($2);$n$1.kind = $3;$n$1.base = $4;$n",
+    [nameHcr, size, rope(nimtypeKind), base]
+  )
   # compute type flags for GC optimization
   var flags = 0
   if not containsGarbageCollectedRef(typ): flags = flags or 1
@@ -1039,7 +1075,7 @@ proc genTypeInfoAux(m: BModule, typ, origType: PType, name: Rope;
     if typ.kind == tyPtr and x.kind == tyObject and incompleteType(x):
       base = rope("0")
     else:
-      base = genTypeInfo(m, x, info)
+      base = genTypeInfoV1(m, x, info)
   else:
     base = rope("0")
   genTypeInfoAuxBase(m, typ, origType, name, base, info)
@@ -1097,8 +1133,8 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
     m.s[cfsTypeInit3].addf("$1.kind = 3;$n" &
         "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" &
         "$1.name = $5;$n" & "$1.sons = &$6[0];$n" &
-        "$1.len = $7;$n", [expr, getTypeDesc(m, origType), field.loc.r,
-                           genTypeInfo(m, field.typ, info),
+        "$1.len = $7;$n", [expr, getTypeDesc(m, origType, skVar), field.loc.r,
+                           genTypeInfoV1(m, field.typ, info),
                            makeCString(field.name.s),
                            tmp, rope(L)])
     m.s[cfsData].addf("TNimNode* $1[$2];$n", [tmp, rope(L+1)])
@@ -1134,8 +1170,8 @@ proc genObjectFields(m: BModule, typ, origType: PType, n: PNode, expr: Rope;
         internalError(m.config, n.info, "genObjectFields")
       m.s[cfsTypeInit3].addf("$1.kind = 1;$n" &
           "$1.offset = offsetof($2, $3);$n" & "$1.typ = $4;$n" &
-          "$1.name = $5;$n", [expr, getTypeDesc(m, origType),
-          field.loc.r, genTypeInfo(m, field.typ, info), makeCString(field.name.s)])
+          "$1.name = $5;$n", [expr, getTypeDesc(m, origType, skVar),
+          field.loc.r, genTypeInfoV1(m, field.typ, info), makeCString(field.name.s)])
   else: internalError(m.config, n.info, "genObjectFields")
 
 proc genObjectInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo) =
@@ -1170,7 +1206,7 @@ proc genTupleInfo(m: BModule, typ, origType: PType, name: Rope; info: TLineInfo)
           "$1.offset = offsetof($2, Field$3);$n" &
           "$1.typ = $4;$n" &
           "$1.name = \"Field$3\";$n",
-           [tmp2, getTypeDesc(m, origType), rope(i), genTypeInfo(m, a, info)])
+           [tmp2, getTypeDesc(m, origType, skVar), rope(i), genTypeInfoV1(m, a, info)])
     m.s[cfsTypeInit3].addf("$1.len = $2; $1.kind = 2; $1.sons = &$3[0];$n",
          [expr, rope(typ.len), tmp])
   else:
@@ -1227,7 +1263,7 @@ proc genSetInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
        [tmp, rope(firstOrd(m.config, typ)), tiNameForHcr(m, name)])
 
 proc genArrayInfo(m: BModule, typ: PType, name: Rope; info: TLineInfo) =
-  genTypeInfoAuxBase(m, typ, typ, name, genTypeInfo(m, typ[1], info), info)
+  genTypeInfoAuxBase(m, typ, typ, name, genTypeInfoV1(m, typ[1], info), info)
 
 proc fakeClosureType(m: BModule; owner: PSym): PType =
   # we generate the same RTTI as for a tuple[pointer, ref tuple[]]
@@ -1245,13 +1281,14 @@ proc genDeepCopyProc(m: BModule; s: PSym; result: Rope) =
   m.s[cfsTypeInit3].addf("$1.deepcopy =(void* (N_RAW_NIMCALL*)(void*))$2;$n",
      [result, s.loc.r])
 
-proc declareNimType(m: BModule, str: Rope, ownerModule: PSym) =
+proc declareNimType(m: BModule, name: string; str: Rope, ownerModule: PSym) =
+  let nr = rope(name)
   if m.hcrOn:
-    m.s[cfsData].addf("static TNimType* $1;$n", [str])
-    m.s[cfsTypeInit1].addf("\t$1 = (TNimType*)hcrGetGlobal($2, \"$1\");$n",
-          [str, getModuleDllPath(m, ownerModule)])
+    m.s[cfsData].addf("static $2* $1;$n", [str, nr])
+    m.s[cfsTypeInit1].addf("\t$1 = ($3*)hcrGetGlobal($2, \"$1\");$n",
+          [str, getModuleDllPath(m, ownerModule), nr])
   else:
-    m.s[cfsData].addf("extern TNimType $1;$n", [str])
+    m.s[cfsData].addf("extern $2 $1;$n", [str, nr])
 
 proc genTypeInfo2Name(m: BModule; t: PType): Rope =
   var res = "|"
@@ -1284,21 +1321,22 @@ proc genHook(m: BModule; t: PType; info: TLineInfo; op: TTypeAttachedOp): Rope =
     # the prototype of a destructor is ``=destroy(x: var T)`` and that of a
     # finalizer is: ``proc (x: ref T) {.nimcall.}``. We need to check the calling
     # convention at least:
-    if theProc.typ == nil or theProc.typ.callConv != ccDefault:
+    if theProc.typ == nil or theProc.typ.callConv != ccNimCall:
       localError(m.config, info,
         theProc.name.s & " needs to have the 'nimcall' calling convention")
 
     genProc(m, theProc)
     result = theProc.loc.r
   else:
-    if op == attachedTrace and m.config.selectedGC == gcOrc and
-        containsGarbageCollectedRef(t):
-      when false:
-        # re-enable this check
+    when false:
+      if op == attachedTrace and m.config.selectedGC == gcOrc and
+          containsGarbageCollectedRef(t):
+        # unfortunately this check is wrong for an object type that only contains
+        # .cursor fields like 'Node' inside 'cycleleak'.
         internalError(m.config, info, "no attached trace proc found")
     result = rope("NIM_NIL")
 
-proc genTypeInfoV2(m: BModule, t, origType: PType, name: Rope; info: TLineInfo) =
+proc genTypeInfoV2Impl(m: BModule, t, origType: PType, name: Rope; info: TLineInfo) =
   var typeName: Rope
   if t.kind == tyObject:
     if incompleteType(t):
@@ -1308,19 +1346,65 @@ proc genTypeInfoV2(m: BModule, t, origType: PType, name: Rope; info: TLineInfo)
   else:
     typeName = rope("NIM_NIL")
 
-  m.s[cfsData].addf("N_LIB_PRIVATE TNimType $1;$n", [name])
+  discard cgsym(m, "TNimTypeV2")
+  m.s[cfsData].addf("N_LIB_PRIVATE TNimTypeV2 $1;$n", [name])
   let destroyImpl = genHook(m, t, info, attachedDestructor)
   let traceImpl = genHook(m, t, info, attachedTrace)
   let disposeImpl = genHook(m, t, info, attachedDispose)
 
-  m.s[cfsTypeInit3].addf("$1.destructor = (void*)$2; $1.size = sizeof($3);$n" &
-    "$1.name = $4;$n" &
-    "$1.traceImpl = (void*)$5;$n" &
-    "$1.disposeImpl = (void*)$6;$n", [
+  addf(m.s[cfsTypeInit3], "$1.destructor = (void*)$2; $1.size = sizeof($3); $1.align = NIM_ALIGNOF($3); $1.name = $4;$n; $1.traceImpl = (void*)$5; $1.disposeImpl = (void*)$6;", [
     name, destroyImpl, getTypeDesc(m, t), typeName,
     traceImpl, disposeImpl])
 
-proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope =
+  if t.kind == tyObject and t.len > 0 and t[0] != nil and optEnableDeepCopy in m.config.globalOptions:
+    discard genTypeInfoV1(m, t, info)
+
+proc genTypeInfoV2(m: BModule, t: PType; info: TLineInfo): Rope =
+  let origType = t
+  var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses)
+
+  let prefixTI = if m.hcrOn: "(" else: "(&"
+
+  let sig = hashType(origType)
+  result = m.typeInfoMarkerV2.getOrDefault(sig)
+  if result != nil:
+    return prefixTI.rope & result & ")".rope
+
+  let marker = m.g.typeInfoMarkerV2.getOrDefault(sig)
+  if marker.str != nil:
+    discard cgsym(m, "TNimTypeV2")
+    declareNimType(m, "TNimTypeV2", marker.str, marker.owner)
+    # also store in local type section:
+    m.typeInfoMarkerV2[sig] = marker.str
+    return prefixTI.rope & marker.str & ")".rope
+
+  result = "NTIv2$1_" % [rope($sig)]
+  m.typeInfoMarkerV2[sig] = result
+
+  let owner = t.skipTypes(typedescPtrs).owner.getModule
+  if owner != m.module:
+    # make sure the type info is created in the owner module
+    assert m.g.modules[owner.position] != nil
+    discard genTypeInfoV2(m.g.modules[owner.position], origType, info)
+    # reference the type info as extern here
+    discard cgsym(m, "TNimTypeV2")
+    declareNimType(m, "TNimTypeV2", result, owner)
+    return prefixTI.rope & result & ")".rope
+
+  m.g.typeInfoMarkerV2[sig] = (str: result, owner: owner)
+  genTypeInfoV2Impl(m, t, origType, result, info)
+  result = prefixTI.rope & result & ")".rope
+
+proc openArrayToTuple(m: BModule; t: PType): PType =
+  result = newType(tyTuple, t.owner)
+  let p = newType(tyPtr, t.owner)
+  let a = newType(tyUncheckedArray, t.owner)
+  a.add t.lastSon
+  p.add a
+  result.add p
+  result.add getSysType(m.g.graph, t.owner.info, tyInt)
+
+proc genTypeInfoV1(m: BModule, t: PType; info: TLineInfo): Rope =
   let origType = t
   var t = skipTypes(origType, irrelevantForBackend + tyUserTypeClasses)
 
@@ -1335,7 +1419,7 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope =
   if marker.str != nil:
     discard cgsym(m, "TNimType")
     discard cgsym(m, "TNimNode")
-    declareNimType(m, marker.str, marker.owner)
+    declareNimType(m, "TNimType", marker.str, marker.owner)
     # also store in local type section:
     m.typeInfoMarker[sig] = marker.str
     return prefixTI.rope & marker.str & ")".rope
@@ -1347,63 +1431,68 @@ proc genTypeInfo(m: BModule, t: PType; info: TLineInfo): Rope =
   if owner != m.module:
     # make sure the type info is created in the owner module
     assert m.g.modules[owner.position] != nil
-    discard genTypeInfo(m.g.modules[owner.position], origType, info)
+    discard genTypeInfoV1(m.g.modules[owner.position], origType, info)
     # reference the type info as extern here
     discard cgsym(m, "TNimType")
     discard cgsym(m, "TNimNode")
-    declareNimType(m, result, owner)
+    declareNimType(m, "TNimType", result, owner)
     return prefixTI.rope & result & ")".rope
 
   m.g.typeInfoMarker[sig] = (str: result, owner: owner)
 
-  if optTinyRtti in m.config.globalOptions:
-    genTypeInfoV2(m, t, origType, result, info)
-  else:
-    case t.kind
-    of tyEmpty, tyVoid: result = rope"0"
-    of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar, tyLent:
+  case t.kind
+  of tyEmpty, tyVoid: result = rope"0"
+  of tyPointer, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64, tyVar, tyLent:
+    genTypeInfoAuxBase(m, t, t, result, rope"0", info)
+  of tyStatic:
+    if t.n != nil: result = genTypeInfoV1(m, lastSon t, info)
+    else: internalError(m.config, "genTypeInfoV1(" & $t.kind & ')')
+  of tyUserTypeClasses:
+    internalAssert m.config, t.isResolvedUserTypeClass
+    return genTypeInfoV1(m, t.lastSon, info)
+  of tyProc:
+    if t.callConv != ccClosure:
       genTypeInfoAuxBase(m, t, t, result, rope"0", info)
-    of tyStatic:
-      if t.n != nil: result = genTypeInfo(m, lastSon t, info)
-      else: internalError(m.config, "genTypeInfo(" & $t.kind & ')')
-    of tyUserTypeClasses:
-      internalAssert m.config, t.isResolvedUserTypeClass
-      return genTypeInfo(m, t.lastSon, info)
-    of tyProc:
-      if t.callConv != ccClosure:
-        genTypeInfoAuxBase(m, t, t, result, rope"0", info)
-      else:
-        let x = fakeClosureType(m, t.owner)
-        genTupleInfo(m, x, x, result, info)
-    of tySequence:
-      genTypeInfoAux(m, t, t, result, info)
-      if optSeqDestructors notin m.config.globalOptions:
-        if m.config.selectedGC >= gcMarkAndSweep:
-          let markerProc = genTraverseProc(m, origType, sig)
-          m.s[cfsTypeInit3].addf("$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc])
-    of tyRef:
-      genTypeInfoAux(m, t, t, result, info)
-      if m.config.selectedGC >= gcMarkAndSweep:
-        let markerProc = genTraverseProc(m, origType, sig)
-        m.s[cfsTypeInit3].addf("$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc])
-    of tyPtr, tyRange, tyUncheckedArray: genTypeInfoAux(m, t, t, result, info)
-    of tyArray: genArrayInfo(m, t, result, info)
-    of tySet: genSetInfo(m, t, result, info)
-    of tyEnum: genEnumInfo(m, t, result, info)
-    of tyObject:
-      genObjectInfo(m, t, origType, result, info)
-    of tyTuple:
-      # if t.n != nil: genObjectInfo(m, t, result)
-      # else:
-      # BUGFIX: use consistently RTTI without proper field names; otherwise
-      # results are not deterministic!
-      genTupleInfo(m, t, origType, result, info)
-    else: internalError(m.config, "genTypeInfo(" & $t.kind & ')')
+    else:
+      let x = fakeClosureType(m, t.owner)
+      genTupleInfo(m, x, x, result, info)
+  of tySequence:
+    genTypeInfoAux(m, t, t, result, info)
+    if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcV2, gcGo}:
+      let markerProc = genTraverseProc(m, origType, sig)
+      m.s[cfsTypeInit3].addf("$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc])
+  of tyRef:
+    genTypeInfoAux(m, t, t, result, info)
+    if m.config.selectedGC in {gcMarkAndSweep, gcRefc, gcV2, gcGo}:
+      let markerProc = genTraverseProc(m, origType, sig)
+      m.s[cfsTypeInit3].addf("$1.marker = $2;$n", [tiNameForHcr(m, result), markerProc])
+  of tyPtr, tyRange, tyUncheckedArray: genTypeInfoAux(m, t, t, result, info)
+  of tyArray: genArrayInfo(m, t, result, info)
+  of tySet: genSetInfo(m, t, result, info)
+  of tyEnum: genEnumInfo(m, t, result, info)
+  of tyObject:
+    genObjectInfo(m, t, origType, result, info)
+  of tyTuple:
+    # if t.n != nil: genObjectInfo(m, t, result)
+    # else:
+    # BUGFIX: use consistently RTTI without proper field names; otherwise
+    # results are not deterministic!
+    genTupleInfo(m, t, origType, result, info)
+  of tyOpenArray:
+    let x = openArrayToTuple(m, t)
+    genTupleInfo(m, x, origType, result, info)
+  else: internalError(m.config, "genTypeInfoV1(" & $t.kind & ')')
 
   if t.attachedOps[attachedDeepCopy] != nil:
     genDeepCopyProc(m, t.attachedOps[attachedDeepCopy], result)
   elif origType.attachedOps[attachedDeepCopy] != nil:
     genDeepCopyProc(m, origType.attachedOps[attachedDeepCopy], result)
+
+  if optTinyRtti in m.config.globalOptions and t.kind == tyObject and sfImportc notin t.sym.flags:
+    let v2info = genTypeInfoV2(m, origType, info)
+    addf(m.s[cfsTypeInit3], "$1->typeInfoV1 = (void*)&$2; $2.typeInfoV2 = (void*)$1;$n", [
+      v2info, result])
+
   result = prefixTI.rope & result & ")".rope
 
 proc genTypeSection(m: BModule, n: PNode) =

compiler/cgen.nim

@@ -244,7 +244,7 @@ proc safeLineNm(info: TLineInfo): int =
 
 proc genCLineDir(r: var Rope, filename: string, line: int; conf: ConfigRef) =
   assert line >= 0
-  if optLineDir in conf.options:
+  if optLineDir in conf.options and line > 0:
     r.addf("$N#line $2 $1$N",
         [rope(makeSingleLineCString(filename)), rope(line)])
 
@@ -279,7 +279,7 @@ proc genProc(m: BModule, prc: PSym)
 proc raiseInstr(p: BProc): Rope
 
 template compileToCpp(m: BModule): untyped =
-  m.config.cmd == cmdCompileToCpp or sfCompileToCpp in m.module.flags
+  m.config.backend == backendCpp or sfCompileToCpp in m.module.flags
 
 proc getTempName(m: BModule): Rope =
   result = m.tmpBase & rope(m.labels)
@@ -310,14 +310,19 @@ include ccgtypes
 
 # ------------------------------ Manager of temporaries ------------------
 
+template mapTypeChooser(n: PNode): TSymKind =
+  (if n.kind == nkSym: n.sym.kind else: skVar)
+
+template mapTypeChooser(a: TLoc): TSymKind = mapTypeChooser(a.lode)
+
 proc addrLoc(conf: ConfigRef; a: TLoc): Rope =
   result = a.r
-  if lfIndirect notin a.flags and mapType(conf, a.t) != ctArray:
+  if lfIndirect notin a.flags and mapType(conf, a.t, mapTypeChooser(a)) != ctArray:
     result = "(&" & result & ")"
 
 proc byRefLoc(p: BProc; a: TLoc): Rope =
   result = a.r
-  if lfIndirect notin a.flags and mapType(p.config, a.t) != ctArray and not
+  if lfIndirect notin a.flags and mapType(p.config, a.t, mapTypeChooser(a)) != ctArray and not
       p.module.compileToCpp:
     result = "(&" & result & ")"
 
@@ -355,7 +360,10 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: var TLoc,
       while s.kind == tyObject and s[0] != nil:
         r.add(".Sup")
         s = skipTypes(s[0], skipPtrs)
-    linefmt(p, section, "$1.m_type = $2;$n", [r, genTypeInfo(p.module, t, a.lode.info)])
+    if optTinyRtti in p.config.globalOptions:
+      linefmt(p, section, "$1.m_type = $2;$n", [r, genTypeInfoV2(p.module, t, a.lode.info)])
+    else:
+      linefmt(p, section, "$1.m_type = $2;$n", [r, genTypeInfoV1(p.module, t, a.lode.info)])
   of frEmbedded:
     if optTinyRtti in p.config.globalOptions:
       var tmp: TLoc
@@ -364,14 +372,14 @@ proc genObjectInit(p: BProc, section: TCProcSection, t: PType, a: var TLoc,
         rawConstExpr(p, newNodeIT(nkType, a.lode.info, objType), tmp)
         linefmt(p, cpsStmts,
             "#nimCopyMem((void*)$1, (NIM_CONST void*)&$2, sizeof($3));$n",
-            [rdLoc(a), rdLoc(tmp), getTypeDesc(p.module, objType)])
+            [rdLoc(a), rdLoc(tmp), getTypeDesc(p.module, objType, mapTypeChooser(a))])
       else:
         rawConstExpr(p, newNodeIT(nkType, a.lode.info, t), tmp)
         genAssignment(p, a, tmp, {})
     else:
       # worst case for performance:
       var r = if mode == constructObj: addrLoc(p.config, a) else: rdLoc(a)
-      linefmt(p, section, "#objectInit($1, $2);$n", [r, genTypeInfo(p.module, t, a.lode.info)])
+      linefmt(p, section, "#objectInit($1, $2);$n", [r, genTypeInfoV1(p.module, t, a.lode.info)])
 
   if isException(t):
     var r = rdLoc(a)
@@ -387,9 +395,11 @@ proc genRefAssign(p: BProc, dest, src: TLoc)
 
 proc isComplexValueType(t: PType): bool {.inline.} =
   let t = t.skipTypes(abstractInst + tyUserTypeClasses)
-  result = t.kind in {tyArray, tySet, tyTuple, tyObject} or
+  result = t.kind in {tyArray, tySet, tyTuple, tyObject, tyOpenArray} or
     (t.kind == tyProc and t.callConv == ccClosure)
 
+include ccgreset
+
 proc resetLoc(p: BProc, loc: var TLoc) =
   let containsGcRef = optSeqDestructors notin p.config.globalOptions and containsGarbageCollectedRef(loc.t)
   let typ = skipTypes(loc.t, abstractVarRange)
@@ -406,11 +416,11 @@ proc resetLoc(p: BProc, loc: var TLoc) =
     else:
       linefmt(p, cpsStmts, "$1 = 0;$n", [rdLoc(loc)])
   else:
-    if optNilCheck in p.options:
-      linefmt(p, cpsStmts, "#chckNil((void*)$1);$n", [addrLoc(p.config, loc)])
     if loc.storage != OnStack and containsGcRef:
-      linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
-              [addrLoc(p.config, loc), genTypeInfo(p.module, loc.t, loc.lode.info)])
+      specializeReset(p, loc)
+      when false:
+        linefmt(p, cpsStmts, "#genericReset((void*)$1, $2);$n",
+                [addrLoc(p.config, loc), genTypeInfoV1(p.module, loc.t, loc.lode.info)])
       # XXX: generated reset procs should not touch the m_type
       # field, so disabling this should be safe:
       genObjectInit(p, cpsStmts, loc.t, loc, constructObj)
@@ -418,7 +428,8 @@ proc resetLoc(p: BProc, loc: var TLoc) =
       # array passed as argument decayed into pointer, bug #7332
       # so we use getTypeDesc here rather than rdLoc(loc)
       linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n",
-              [addrLoc(p.config, loc), getTypeDesc(p.module, loc.t)])
+              [addrLoc(p.config, loc),
+              getTypeDesc(p.module, loc.t, mapTypeChooser(loc))])
       # XXX: We can be extra clever here and call memset only
       # on the bytes following the m_type field?
       genObjectInit(p, cpsStmts, loc.t, loc, constructObj)
@@ -429,14 +440,14 @@ proc constructLoc(p: BProc, loc: var TLoc, isTemp = false) =
     linefmt(p, cpsStmts, "$1.len = 0; $1.p = NIM_NIL;$n", [rdLoc(loc)])
   elif not isComplexValueType(typ):
     linefmt(p, cpsStmts, "$1 = ($2)0;$n", [rdLoc(loc),
-      getTypeDesc(p.module, typ)])
+      getTypeDesc(p.module, typ, mapTypeChooser(loc))])
   else:
     if not isTemp or containsGarbageCollectedRef(loc.t):
       # don't use nimZeroMem for temporary values for performance if we can
       # avoid it:
       if not isImportedCppType(typ):
         linefmt(p, cpsStmts, "#nimZeroMem((void*)$1, sizeof($2));$n",
-                [addrLoc(p.config, loc), getTypeDesc(p.module, typ)])
+                [addrLoc(p.config, loc), getTypeDesc(p.module, typ, mapTypeChooser(loc))])
     genObjectInit(p, cpsStmts, loc.t, loc, constructObj)
 
 proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) =
@@ -454,17 +465,25 @@ proc initLocalVar(p: BProc, v: PSym, immediateAsgn: bool) =
 proc getTemp(p: BProc, t: PType, result: var TLoc; needsInit=false) =
   inc(p.labels)
   result.r = "T" & rope(p.labels) & "_"
-  linefmt(p, cpsLocals, "$1 $2;$n", [getTypeDesc(p.module, t), result.r])
+  linefmt(p, cpsLocals, "$1 $2;$n", [getTypeDesc(p.module, t, skVar), result.r])
   result.k = locTemp
   result.lode = lodeTyp t
   result.storage = OnStack
   result.flags = {}
   constructLoc(p, result, not needsInit)
+  when false:
+    # XXX Introduce a compiler switch in order to detect these easily.
+    if getSize(p.config, t) > 1024 * 1024:
+      if p.prc != nil:
+        echo "ENORMOUS TEMPORARY! ", p.config $ p.prc.info
+      else:
+        echo "ENORMOUS TEMPORARY! ", p.config $ p.lastLineInfo
+      writeStackTrace()
 
 proc getTempCpp(p: BProc, t: PType, result: var TLoc; value: Rope) =
   inc(p.labels)
   result.r = "T" & rope(p.labels) & "_"
-  linefmt(p, cpsStmts, "$1 $2 = $3;$n", [getTypeDesc(p.module, t), result.r, value])
+  linefmt(p, cpsStmts, "$1 $2 = $3;$n", [getTypeDesc(p.module, t, skVar), result.r, value])
   result.k = locTemp
   result.lode = lodeTyp t
   result.storage = OnStack
@@ -486,12 +505,13 @@ proc localVarDecl(p: BProc; n: PNode): Rope =
     if s.kind == skLet: incl(s.loc.flags, lfNoDeepCopy)
   if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0:
     result.addf("NIM_ALIGN($1) ", [rope(s.alignment)])
-  result.add getTypeDesc(p.module, s.typ)
+  result.add getTypeDesc(p.module, s.typ, skVar)
   if s.constraint.isNil:
     if sfRegister in s.flags: result.add(" register")
     #elif skipTypes(s.typ, abstractInst).kind in GcTypeKinds:
     #  decl.add(" GC_GUARD")
     if sfVolatile in s.flags: result.add(" volatile")
+    if sfNoalias in s.flags: result.add(" NIM_NOALIAS")
     result.add(" ")
     result.add(s.loc.r)
   else:
@@ -539,7 +559,7 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
         internalError(p.config, n.info, ".threadvar variables cannot have a value")
     else:
       var decl: Rope = nil
-      var td = getTypeDesc(p.module, s.loc.t)
+      var td = getTypeDesc(p.module, s.loc.t, skVar)
       if s.constraint.isNil:
         if s.kind in {skLet, skVar, skField, skForVar} and s.alignment > 0:
           decl.addf "NIM_ALIGN($1) ", [rope(s.alignment)]
@@ -552,6 +572,7 @@ proc assignGlobalVar(p: BProc, n: PNode; value: Rope) =
         if p.hcrOn: decl.add("*")
         if sfRegister in s.flags: decl.add(" register")
         if sfVolatile in s.flags: decl.add(" volatile")
+        if sfNoalias in s.flags: decl.add(" NIM_NOALIAS")
         if value != nil:
           decl.addf(" $1 = $2;$n", [s.loc.r, value])
         else:
@@ -679,12 +700,13 @@ proc loadDynamicLib(m: BModule, lib: PLib) =
             [loadlib, genStringLiteral(m, lib.path)])
     else:
       var p = newProc(nil, m)
-      p.options = p.options - {optStackTrace}
+      p.options.excl optStackTrace
+      p.flags.incl nimErrorFlagDisabled
       var dest: TLoc
       initLoc(dest, locTemp, lib.path, OnStack)
       dest.r = getTempName(m)
       appcg(m, m.s[cfsDynLibInit],"$1 $2;$n",
-           [getTypeDesc(m, lib.path.typ), rdLoc(dest)])
+           [getTypeDesc(m, lib.path.typ, skVar), rdLoc(dest)])
       expr(p, lib.path, dest)
 
       m.s[cfsVars].add(p.s(cpsLocals))
@@ -724,7 +746,7 @@ proc symInDynamicLib(m: BModule, sym: PSym) =
       params.add(rdLoc(a))
       params.add(", ")
     let load = "\t$1 = ($2) ($3$4));$n" %
-        [tmp, getTypeDesc(m, sym.typ), params, makeCString($extname)]
+        [tmp, getTypeDesc(m, sym.typ, skVar), params, makeCString($extname)]
     var last = lastSon(n)
     if last.kind == nkHiddenStdConv: last = last[1]
     internalAssert(m.config, last.kind == nkStrLit)
@@ -738,8 +760,8 @@ proc symInDynamicLib(m: BModule, sym: PSym) =
   else:
     appcg(m, m.s[cfsDynLibInit],
         "\t$1 = ($2) #nimGetProcAddr($3, $4);$n",
-        [tmp, getTypeDesc(m, sym.typ), lib.name, makeCString($extname)])
-  m.s[cfsVars].addf("$2 $1;$n", [sym.loc.r, getTypeDesc(m, sym.loc.t)])
+        [tmp, getTypeDesc(m, sym.typ, skVar), lib.name, makeCString($extname)])
+  m.s[cfsVars].addf("$2 $1;$n", [sym.loc.r, getTypeDesc(m, sym.loc.t, skVar)])
 
 proc varInDynamicLib(m: BModule, sym: PSym) =
   var lib = sym.annex
@@ -751,9 +773,9 @@ proc varInDynamicLib(m: BModule, sym: PSym) =
   inc(m.labels, 2)
   appcg(m, m.s[cfsDynLibInit],
       "$1 = ($2*) #nimGetProcAddr($3, $4);$n",
-      [tmp, getTypeDesc(m, sym.typ), lib.name, makeCString($extname)])
+      [tmp, getTypeDesc(m, sym.typ, skVar), lib.name, makeCString($extname)])
   m.s[cfsVars].addf("$2* $1;$n",
-      [sym.loc.r, getTypeDesc(m, sym.loc.t)])
+      [sym.loc.r, getTypeDesc(m, sym.loc.t, skVar)])
 
 proc symInDynamicLibPartial(m: BModule, sym: PSym) =
   sym.loc.r = mangleDynLibProc(sym)
@@ -973,6 +995,9 @@ proc genProcBody(p: BProc; procBody: PNode) =
     p.blocks[0].sections[cpsLocals].add(ropecg(p.module, "NIM_BOOL* nimErr_;$n", []))
     p.blocks[0].sections[cpsInit].add(ropecg(p.module, "nimErr_ = #nimErrorFlag();$n", []))
 
+proc isNoReturn(m: BModule; s: PSym): bool {.inline.} =
+  sfNoReturn in s.flags and m.config.exc != excGoto
+
 proc genProcAux(m: BModule, prc: PSym) =
   var p = newProc(prc, m)
   var header = genProcHeader(m, prc)
@@ -1027,7 +1052,7 @@ proc genProcAux(m: BModule, prc: PSym) =
 
   var generatedProc: Rope
   generatedProc.genCLineDir prc.info, m.config
-  if sfNoReturn in prc.flags:
+  if isNoReturn(p.module, prc):
     if hasDeclspec in extccomp.CC[p.config.cCompiler].props:
       header = "__declspec(noreturn) " & header
   if sfPure in prc.flags:
@@ -1068,11 +1093,11 @@ proc genProcAux(m: BModule, prc: PSym) =
 proc requiresExternC(m: BModule; sym: PSym): bool {.inline.} =
   result = (sfCompileToCpp in m.module.flags and
            sfCompileToCpp notin sym.getModule().flags and
-           m.config.cmd != cmdCompileToCpp) or (
+           m.config.backend != backendCpp) or (
            sym.flags * {sfInfixCall, sfCompilerProc, sfMangleCpp} == {} and
            sym.flags * {sfImportc, sfExportc} != {} and
            sym.magic == mNone and
-           m.config.cmd == cmdCompileToCpp)
+           m.config.backend == backendCpp)
 
 proc genProcPrototype(m: BModule, sym: PSym) =
   useHeader(m, sym)
@@ -1090,13 +1115,13 @@ proc genProcPrototype(m: BModule, sym: PSym) =
     let asPtr = isReloadable(m, sym)
     var header = genProcHeader(m, sym, asPtr)
     if not asPtr:
-      if sfNoReturn in sym.flags and hasDeclspec in extccomp.CC[m.config.cCompiler].props:
+      if isNoReturn(m, sym) and hasDeclspec in extccomp.CC[m.config.cCompiler].props:
         header = "__declspec(noreturn) " & header
       if sym.typ.callConv != ccInline and requiresExternC(m, sym):
         header = "extern \"C\" " & header
       if sfPure in sym.flags and hasAttribute in CC[m.config.cCompiler].props:
         header.add(" __attribute__((naked))")
-      if sfNoReturn in sym.flags and hasAttribute in CC[m.config.cCompiler].props:
+      if isNoReturn(m, sym) and hasAttribute in CC[m.config.cCompiler].props:
         header.add(" __attribute__((noreturn))")
     m.s[cfsProcHeaders].add(ropecg(m, "$1;$N", [header]))
 
@@ -1168,21 +1193,44 @@ proc requestConstImpl(p: BProc, sym: PSym) =
   useHeader(m, sym)
   if sym.loc.k == locNone:
     fillLoc(sym.loc, locData, sym.ast, mangleName(p.module, sym), OnStatic)
+  if m.hcrOn: incl(sym.loc.flags, lfIndirect)
+
   if lfNoDecl in sym.loc.flags: return
   # declare implementation:
   var q = findPendingModule(m, sym)
   if q != nil and not containsOrIncl(q.declaredThings, sym.id):
     assert q.initProc.module == q
+    # add a suffix for hcr - will later init the global pointer with this data
+    let actualConstName = if m.hcrOn: sym.loc.r & "_const" else: sym.loc.r
     q.s[cfsData].addf("N_LIB_PRIVATE NIM_CONST $1 $2 = $3;$n",
-        [getTypeDesc(q, sym.typ), sym.loc.r, genBracedInit(q.initProc, sym.ast, isConst = true)])
+        [getTypeDesc(q, sym.typ), actualConstName,
+        genBracedInit(q.initProc, sym.ast, isConst = true, sym.typ)])
+    if m.hcrOn:
+      # generate the global pointer with the real name
+      q.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r])
+      # register it (but ignore the boolean result of hcrRegisterGlobal)
+      q.initProc.procSec(cpsLocals).addf(
+        "\thcrRegisterGlobal($1, \"$2\", sizeof($3), NULL, (void**)&$2);$n",
+        [getModuleDllPath(q, sym), sym.loc.r, rdLoc(sym.loc)])
+      # always copy over the contents of the actual constant with the _const
+      # suffix ==> this means that the constant is reloadable & updatable!
+      q.initProc.procSec(cpsLocals).add(ropecg(q,
+        "\t#nimCopyMem((void*)$1, (NIM_CONST void*)&$2, sizeof($3));$n",
+        [sym.loc.r, actualConstName, rdLoc(sym.loc)]))
   # declare header:
   if q != m and not containsOrIncl(m.declaredThings, sym.id):
     assert(sym.loc.r != nil)
-    let headerDecl = "extern NIM_CONST $1 $2;$n" %
-        [getTypeDesc(m, sym.loc.t), sym.loc.r]
-    m.s[cfsData].add(headerDecl)
-    if sfExportc in sym.flags and p.module.g.generatedHeader != nil:
-      p.module.g.generatedHeader.s[cfsData].add(headerDecl)
+    if m.hcrOn:
+      m.s[cfsVars].addf("static $1* $2;$n", [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]);
+      m.initProc.procSec(cpsLocals).addf(
+        "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r,
+        getTypeDesc(m, sym.loc.t, skVar), getModuleDllPath(q, sym)])
+    else:
+      let headerDecl = "extern NIM_CONST $1 $2;$n" %
+          [getTypeDesc(m, sym.loc.t, skVar), sym.loc.r]
+      m.s[cfsData].add(headerDecl)
+      if sfExportc in sym.flags and p.module.g.generatedHeader != nil:
+        p.module.g.generatedHeader.s[cfsData].add(headerDecl)
 
 proc isActivated(prc: PSym): bool = prc.typ != nil
 
@@ -1219,15 +1267,16 @@ proc genVarPrototype(m: BModule, n: PNode) =
       if sym.kind in {skLet, skVar, skField, skForVar} and sym.alignment > 0:
         m.s[cfsVars].addf "NIM_ALIGN($1) ", [rope(sym.alignment)]
       m.s[cfsVars].add(if m.hcrOn: "static " else: "extern ")
-      m.s[cfsVars].add(getTypeDesc(m, sym.loc.t))
+      m.s[cfsVars].add(getTypeDesc(m, sym.loc.t, skVar))
       if m.hcrOn: m.s[cfsVars].add("*")
       if lfDynamicLib in sym.loc.flags: m.s[cfsVars].add("*")
       if sfRegister in sym.flags: m.s[cfsVars].add(" register")
       if sfVolatile in sym.flags: m.s[cfsVars].add(" volatile")
+      if sfNoalias in sym.flags: m.s[cfsVars].add(" NIM_NOALIAS")
       m.s[cfsVars].addf(" $1;$n", [sym.loc.r])
       if m.hcrOn: m.initProc.procSec(cpsLocals).addf(
         "\t$1 = ($2*)hcrGetGlobal($3, \"$1\");$n", [sym.loc.r,
-        getTypeDesc(m, sym.loc.t), getModuleDllPath(m, sym)])
+        getTypeDesc(m, sym.loc.t, skVar), getModuleDllPath(m, sym)])
 
 proc addNimDefines(result: var Rope; conf: ConfigRef) {.inline.} =
   result.addf("#define NIM_INTBITS $1\L", [
@@ -1533,7 +1582,7 @@ proc registerModuleToMain(g: BModuleList; m: BModule) =
   if sfSystemModule in m.module.flags:
     if emulatedThreadVars(m.config) and m.config.target.targetOS != osStandalone:
       g.mainDatInit.add(ropecg(m, "\t#initThreadVarsEmulation();$N", []))
-    if m.config.target.targetOS != osStandalone and m.config.selectedGC != gcNone:
+    if m.config.target.targetOS != osStandalone and m.config.selectedGC notin {gcNone, gcArc, gcOrc}:
       g.mainDatInit.add(ropecg(m, "\t#initStackBottomWith((void *)&inner);$N", []))
 
   if m.s[cfsInitProc].len > 0:
@@ -1585,10 +1634,10 @@ proc hcrGetProcLoadCode(m: BModule, sym, prefix, handle, getProcFunc: string): R
   prc.typ.sym = nil
 
   if not containsOrIncl(m.declaredThings, prc.id):
-    m.s[cfsVars].addf("static $2 $1;$n", [prc.loc.r, getTypeDesc(m, prc.loc.t)])
+    m.s[cfsVars].addf("static $2 $1;$n", [prc.loc.r, getTypeDesc(m, prc.loc.t, skVar)])
 
   result = "\t$1 = ($2) $3($4, $5);$n" %
-      [tmp, getTypeDesc(m, prc.typ), getProcFunc.rope, handle.rope, makeCString(prefix & sym)]
+      [tmp, getTypeDesc(m, prc.typ, skVar), getProcFunc.rope, handle.rope, makeCString(prefix & sym)]
 
 proc genInitCode(m: BModule) =
   ## this function is called in cgenWriteModules after all modules are closed,
@@ -1638,6 +1687,10 @@ proc genInitCode(m: BModule) =
     writeSection(preInitProc, cpsInit, m.hcrOn)
     writeSection(preInitProc, cpsStmts)
     prc.addf("}$N", [])
+    when false:
+      m.initProc.blocks[0].sections[cpsLocals].add m.preInitProc.s(cpsLocals)
+      m.initProc.blocks[0].sections[cpsInit].prepend m.preInitProc.s(cpsInit)
+      m.initProc.blocks[0].sections[cpsStmts].prepend m.preInitProc.s(cpsStmts)
 
   # add new scope for following code, because old vcc compiler need variable
   # be defined at the top of the block
@@ -1749,11 +1802,6 @@ proc genModule(m: BModule, cfile: Cfile): Rope =
   if moduleIsEmpty:
     result = nil
 
-proc newPreInitProc(m: BModule): BProc =
-  result = newProc(nil, m)
-  # little hack so that unique temporaries are generated:
-  result.labels = 100_000
-
 proc initProcOptions(m: BModule): TOptions =
   let opts = m.config.options
   if sfSystemModule in m.module.flags: opts-{optStackTrace} else: opts
@@ -1774,7 +1822,9 @@ proc rawNewModule(g: BModuleList; module: PSym, filename: AbsoluteFile): BModule
   result.sigConflicts = initCountTable[SigHash]()
   result.initProc = newProc(nil, result)
   result.initProc.options = initProcOptions(result)
-  result.preInitProc = newPreInitProc(result)
+  result.preInitProc = newProc(nil, result)
+  result.preInitProc.flags.incl nimErrorFlagDisabled
+  result.preInitProc.labels = 100_000 # little hack so that unique temporaries are generated
   initNodeTable(result.dataCache)
   result.typeStack = @[]
   result.typeNodesName = getTempName(result)
@@ -1804,7 +1854,10 @@ template injectG() {.dirty.} =
     graph.backend = newModuleList(graph)
   let g = BModuleList(graph.backend)
 
-proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
+when not defined(nimHasSinkInference):
+  {.pragma: nosinks.}
+
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext {.nosinks.} =
   injectG()
   result = newModule(g, module, graph.config)
   if optGenIndex in graph.config.globalOptions and g.generatedHeader == nil:
@@ -1844,7 +1897,7 @@ proc writeHeader(m: BModule) =
 proc getCFile(m: BModule): AbsoluteFile =
   let ext =
       if m.compileToCpp: ".nim.cpp"
-      elif m.config.cmd == cmdCompileToOC or sfCompileToObjc in m.module.flags: ".nim.m"
+      elif m.config.backend == backendObjc or sfCompileToObjc in m.module.flags: ".nim.m"
       else: ".nim.c"
   result = changeFileExt(completeCfilePath(m.config, withPackageName(m.config, m.cfilename)), ext)
 
@@ -1999,8 +2052,9 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
     if m.config.exc == excGoto and getCompilerProc(graph, "nimTestErrorFlag") != nil:
       discard cgsym(m, "nimTestErrorFlag")
 
-    for i in countdown(high(graph.globalDestructors), 0):
-      n.add graph.globalDestructors[i]
+    if {optGenStaticLib, optGenDynLib} * m.config.globalOptions == {}:
+      for i in countdown(high(graph.globalDestructors), 0):
+        n.add graph.globalDestructors[i]
   if passes.skipCodegen(m.config, n): return
   if moduleHasChanged(graph, m.module):
     # if the module is cached, we don't regenerate the main proc
@@ -2039,7 +2093,8 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
       let disp = generateMethodDispatchers(graph)
       for x in disp: genProcAux(m, x.sym)
 
-  m.g.modulesClosed.add m
+  let mm = m
+  m.g.modulesClosed.add mm
 
 proc genForwardedProcs(g: BModuleList) =
   # Forward declared proc:s lack bodies when first encountered, so they're given

compiler/cgendata.nim

@@ -11,7 +11,7 @@
 
 import
   ast, ropes, options, intsets,
-  tables, ndi, lineinfos, pathutils, modulegraphs
+  tables, ndi, lineinfos, pathutils, modulegraphs, sets
 
 type
   TLabel* = Rope              # for the C generator a label is just a rope
@@ -25,7 +25,7 @@ type
                               # this is needed for strange type generation
                               # reasons
     cfsFieldInfo,             # section for field information
-    cfsTypeInfo,              # section for type information
+    cfsTypeInfo,              # section for type information (ag ABI checks)
     cfsProcHeaders,           # section for C procs prototypes
     cfsData,                  # section for C constant data
     cfsVars,                  # section for C variable declarations
@@ -70,7 +70,8 @@ type
     hasCurFramePointer,
     noSafePoints,
     nimErrorFlagAccessed,
-    nimErrorFlagDeclared
+    nimErrorFlagDeclared,
+    nimErrorFlagDisabled
 
   TCProc = object             # represents C proc that is currently generated
     prc*: PSym                # the Nim proc that this C proc belongs to
@@ -96,6 +97,7 @@ type
                               # requires 'T x = T()' to become 'T x; x = T()'
                               # (yes, C++ is weird like that)
     withinTryWithExcept*: int # required for goto based exception handling
+    withinBlockLeaveActions*: int # complex to explain
     sigConflicts*: CountTable[string]
 
   TTypeSeq* = seq[PType]
@@ -120,6 +122,7 @@ type
     forwardedProcs*: seq[PSym] # proc:s that did not yet have a body
     generatedHeader*: BModule
     typeInfoMarker*: TypeCacheWithOwner
+    typeInfoMarkerV2*: TypeCacheWithOwner
     config*: ConfigRef
     graph*: ModuleGraph
     strVersion*, seqVersion*: int # version of the string/seq implementation to use
@@ -144,11 +147,16 @@ type
                               # without extension)
     tmpBase*: Rope            # base for temp identifier generation
     typeCache*: TypeCache     # cache the generated types
+    typeABICache*: HashSet[SigHash] # cache for ABI checks; reusing typeCache
+                              # would be ideal but for some reason enums
+                              # don't seem to get cached so it'd generate
+                              # 1 ABI check per occurence in code
     forwTypeCache*: TypeCache # cache for forward declarations of types
     declaredThings*: IntSet   # things we have declared in this .c file
     declaredProtos*: IntSet   # prototypes we have declared in this .c file
     headerFiles*: seq[string] # needed headers to include
     typeInfoMarker*: TypeCache # needed for generating type information
+    typeInfoMarkerV2*: TypeCache
     initProc*: BProc          # code for init procedure
     preInitProc*: BProc       # code executed before the init proc
     hcrCreateTypeInfosProc*: Rope # type info globals are in here when HCR=on
@@ -184,8 +192,8 @@ proc newProc*(prc: PSym, module: BModule): BProc =
   new(result)
   result.prc = prc
   result.module = module
-  if prc != nil: result.options = prc.options
-  else: result.options = module.config.options
+  result.options = if prc != nil: prc.options
+                   else: module.config.options
   newSeq(result.blocks, 1)
   result.nestedTryStmts = @[]
   result.finallySafePoints = @[]

compiler/cgmeth.nim

@@ -113,7 +113,7 @@ proc createDispatcher(s: PSym): PSym =
   excl(disp.flags, sfExported)
   disp.typ = copyType(disp.typ, disp.typ.owner, false)
   # we can't inline the dispatcher itself (for now):
-  if disp.typ.callConv == ccInline: disp.typ.callConv = ccDefault
+  if disp.typ.callConv == ccInline: disp.typ.callConv = ccNimCall
   disp.ast = copyTree(s.ast)
   disp.ast[bodyPos] = newNodeI(nkEmpty, s.info)
   disp.loc.r = nil

compiler/closureiters.nim

@@ -130,7 +130,8 @@
 
 import
   ast, msgs, idents,
-  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
+  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos,
+  tables, options
 
 type
   Ctx = object
@@ -388,6 +389,10 @@ proc hasYieldsInExpressions(n: PNode): bool =
           return true
     else:
       result = n.hasYields
+  of nkCast:
+    for i in 1..<n.len:
+      if n[i].hasYieldsInExpressions:
+        return true
   else:
     for c in n:
       if c.hasYieldsInExpressions:
@@ -481,7 +486,6 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
     if ns:
       needsSplit = true
       var tmp: PSym
-      var s: PNode
       let isExpr = not isEmptyType(n.typ)
       if isExpr:
         tmp = ctx.newTempVar(n.typ)
@@ -686,7 +690,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
   of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv, nkObjDownConv,
       nkDerefExpr, nkHiddenDeref:
     var ns = false
-    for i in 0..<n.len:
+    for i in ord(n.kind == nkCast)..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -738,8 +742,6 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       result.add(n)
 
   of nkWhileStmt:
-    var ns = false
-
     var condNeedsSplit = false
     n[0] = ctx.lowerStmtListExprs(n[0], condNeedsSplit)
     var bodyNeedsSplit = false
@@ -1102,7 +1104,7 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
 
   result = ctx.states[stateIdx][0].intVal.int
 
-proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
+proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
   result = n
   case n.kind
   of nkSkip:
@@ -1282,6 +1284,101 @@ proc deleteEmptyStates(ctx: var Ctx) =
     else:
       inc i
 
+type
+  PreprocessContext = object
+    finallys: seq[PNode]
+    config: ConfigRef
+    blocks: seq[(PNode, int)]
+  FreshVarsContext = object
+    tab: Table[int, PSym]
+    config: ConfigRef
+    info: TLineInfo
+
+proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
+  case n.kind
+  of nkSym:
+    let x = c.tab.getOrDefault(n.sym.id)
+    if x == nil:
+      result = n
+    else:
+      result = newSymNode(x, n.info)
+  of nkSkip - {nkSym}:
+    result = n
+  of nkLetSection, nkVarSection:
+    result = copyNode(n)
+    for it in n:
+      if it.kind in {nkIdentDefs, nkVarTuple}:
+        let idefs = copyNode(it)
+        for v in 0..it.len-3:
+          if it[v].kind == nkSym:
+            let x = copySym(it[v].sym)
+            c.tab[it[v].sym.id] = x
+            idefs.add newSymNode(x)
+          else:
+            idefs.add it[v]
+
+        for rest in it.len-2 ..< it.len: idefs.add it[rest]
+        result.add idefs
+      else:
+        result.add it
+  of nkRaiseStmt:
+    localError(c.config, c.info, "unsupported control flow: 'finally: ... raise' duplicated because of 'break'")
+  else:
+    result = n
+    for i in 0..<n.safeLen:
+      result[i] = freshVars(n[i], c)
+
+proc preprocess(c: var PreprocessContext; n: PNode): PNode =
+  # in order to fix bug #15243 without risking regressions, we preprocess
+  # the AST so that 'break' statements inside a 'try finally' also have the
+  # finally section. We need to duplicate local variables here and also
+  # detect: 'finally: raises X' which is currently not supported. We produce
+  # an error for this case for now. All this will be done properly with Yuriy's
+  # patch.
+  result = n
+  case n.kind
+  of nkTryStmt:
+    let f = n.lastSon
+    if f.kind == nkFinally:
+      c.finallys.add f.lastSon
+
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
+    if f.kind == nkFinally:
+      discard c.finallys.pop()
+
+  of nkWhileStmt, nkBlockStmt:
+    c.blocks.add((n, c.finallys.len))
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+    discard c.blocks.pop()
+
+  of nkBreakStmt:
+    if c.blocks.len == 0:
+      discard
+    else:
+      var fin = -1
+      if n[0].kind == nkEmpty:
+        fin = c.blocks[^1][1]
+      elif n[0].kind == nkSym:
+        for i in countdown(c.blocks.high, 0):
+          if c.blocks[i][0].kind == nkBlockStmt and c.blocks[i][0][0].kind == nkSym and
+              c.blocks[i][0][0].sym == n[0].sym:
+            fin = c.blocks[i][1]
+            break
+
+      if fin >= 0:
+        result = newNodeI(nkStmtList, n.info)
+        for i in countdown(c.finallys.high, fin):
+          var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info)
+          result.add freshVars(preprocess(c, c.finallys[i]), vars)
+        result.add n
+  of nkSkip: discard
+  else:
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
 proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   var ctx: Ctx
   ctx.g = g
@@ -1294,7 +1391,10 @@ proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
     ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), fn, fn.info)
     ctx.stateVarSym.typ = g.createClosureIterStateType(fn)
   ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), fn, fn.info)
-  var n = n.toStmtList
+  var pc = PreprocessContext(finallys: @[], config: g.config)
+  var n = preprocess(pc, n.toStmtList)
+  #echo "transformed into ", n
+  #var n = n.toStmtList
 
   discard ctx.newState(n, nil)
   let gotoOut = newTree(nkGotoState, g.newIntLit(n.info, -1))

compiler/cmdlinehelper.nim

@@ -7,20 +7,33 @@
 #    distribution, for details about the copyright.
 #
 
-## Helpers for binaries that use compiler passes, eg: nim, nimsuggest, nimfix
+## Helpers for binaries that use compiler passes, e.g.: nim, nimsuggest, nimfix
 
 import
   options, idents, nimconf, extccomp, commands, msgs,
-  lineinfos, modulegraphs, condsyms, os, pathutils
+  lineinfos, modulegraphs, condsyms, os, pathutils, parseopt
 
 from strutils import normalize
 
+proc prependCurDir*(f: AbsoluteFile): AbsoluteFile =
+  when defined(unix):
+    if os.isAbsolute(f.string): result = f
+    else: result = AbsoluteFile("./" & f.string)
+  else:
+    result = f
+
+proc addCmdPrefix*(result: var string, kind: CmdLineKind) =
+  # consider moving this to std/parseopt
+  case kind
+  of cmdLongOption: result.add "--"
+  of cmdShortOption: result.add "-"
+  of cmdArgument, cmdEnd: discard
+
 type
   NimProg* = ref object
     suggestMode*: bool
     supportsStdinFile*: bool
     processCmdLine*: proc(pass: TCmdLinePass, cmd: string; config: ConfigRef)
-    mainCommand*: proc(graph: ModuleGraph)
 
 proc initDefinesProg*(self: NimProg, conf: ConfigRef, name: string) =
   condsyms.initDefines(conf.symbols)
@@ -42,37 +55,28 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   else:
     conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir())
 
-proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef): bool =
+proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef;
+                                   graph: ModuleGraph): bool =
   if self.suggestMode:
     conf.command = "nimsuggest"
   loadConfigs(DefaultConfig, cache, conf) # load all config files
 
-  block:
+  if not self.suggestMode:
     let scriptFile = conf.projectFull.changeFileExt("nims")
-    if not self.suggestMode:
-      # 'nim foo.nims' means to just run the NimScript file and do nothing more:
-      if fileExists(scriptFile) and scriptFile == conf.projectFull:
-        if conf.command == "":
-          conf.command = "e"
-          return false
-        elif conf.command.normalize == "e":
-          return false
+    # 'nim foo.nims' means to just run the NimScript file and do nothing more:
+    if fileExists(scriptFile) and scriptFile == conf.projectFull:
+      if conf.command == "":
+        conf.command = "e"
+        return false
+      elif conf.command.normalize == "e":
+        return false
 
   # now process command line arguments again, because some options in the
   # command line can overwrite the config file's settings
   extccomp.initVars(conf)
-  # XXX This is hacky. We need to find a better way.
-  case conf.command
-  of "cpp", "compiletocpp":
-    conf.cmd = cmdCompileToCpp
-  else:
-    discard
-
   self.processCmdLine(passCmd2, "", conf)
   if conf.command == "":
     rawMessage(conf, errGenerated, "command missing")
 
-  let graph = newModuleGraph(cache, conf)
   graph.suggestMode = self.suggestMode
-  self.mainCommand(graph)
   return true

compiler/commands.nim

@@ -53,15 +53,14 @@ const
       "Copyright (c) 2006-" & copyrightYear & " by Andreas Rumpf\n"
 
 proc genFeatureDesc[T: enum](t: typedesc[T]): string {.compileTime.} =
-  var x = ""
-  for f in low(T)..high(T):
-    if x.len > 0: x.add "|"
-    x.add $f
-  x
+  result = ""
+  for f in T:
+    if result.len > 0: result.add "|"
+    result.add $f
 
 const
-  Usage = slurp"../doc/basicopt.txt".replace(" //", " ")
-  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", " ") % [genFeatureDesc(Feature), genFeatureDesc(LegacyFeature)]
+  Usage = slurp"../doc/basicopt.txt".replace(" //", "   ")
+  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", "   ") % [genFeatureDesc(Feature), genFeatureDesc(LegacyFeature)]
 
 proc getCommandLineDesc(conf: ConfigRef): string =
   result = (HelpMessage % [VersionAsString, platform.OS[conf.target.hostOS].name,
@@ -98,7 +97,7 @@ proc writeVersionInfo(conf: ConfigRef; pass: TCmdLinePass) =
                                  CPU[conf.target.hostCPU].name, CompileDate]),
                {msgStdout})
 
-    const gitHash = gorge("git log -n 1 --format=%H").strip
+    const gitHash {.strdefine.} = gorge("git log -n 1 --format=%H").strip
     when gitHash.len == 40:
       msgWriteln(conf, "git hash: " & gitHash, {msgStdout})
 
@@ -146,24 +145,24 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
 proc processOnOffSwitch(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass,
                         info: TLineInfo) =
   case arg.normalize
-  of "","on": conf.options = conf.options + op
-  of "off": conf.options = conf.options - op
+  of "","on": conf.options.incl op
+  of "off": conf.options.excl op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
 proc processOnOffSwitchOrList(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass,
                               info: TLineInfo): bool =
   result = false
   case arg.normalize
-  of "on": conf.options = conf.options + op
-  of "off": conf.options = conf.options - op
+  of "on": conf.options.incl op
+  of "off": conf.options.excl op
   of "list": result = true
   else: localError(conf, info, errOnOffOrListExpectedButXFound % arg)
 
 proc processOnOffSwitchG(conf: ConfigRef; op: TGlobalOptions, arg: string, pass: TCmdLinePass,
                          info: TLineInfo) =
   case arg.normalize
-  of "", "on": conf.globalOptions = conf.globalOptions + op
-  of "off": conf.globalOptions = conf.globalOptions - op
+  of "", "on": conf.globalOptions.incl op
+  of "off": conf.globalOptions.excl op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
 proc expectArg(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
@@ -176,7 +175,7 @@ proc expectNoArg(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, info:
 
 proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
                          info: TLineInfo; orig: string; conf: ConfigRef) =
-  var id = ""  # arg = key:val or [key]:val;  with val=on|off
+  var id = ""  # arg = key or [key] or key:val or [key]:val;  with val=on|off
   var i = 0
   var n = hintMin
   var isBracket = false
@@ -190,18 +189,20 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
     if i < arg.len and arg[i] == ']': inc(i)
     else: invalidCmdLineOption(conf, pass, orig, info)
 
-  if i < arg.len and (arg[i] in {':', '='}): inc(i)
+  if i == arg.len: discard
+  elif i < arg.len and (arg[i] in {':', '='}): inc(i)
   else: invalidCmdLineOption(conf, pass, orig, info)
   if state == wHint:
-    let x = findStr(lineinfos.HintsToStr, id)
-    if x >= 0: n = TNoteKind(x + ord(hintMin))
+    let x = findStr(hintMin..hintMax, id, errUnknown)
+    if x != errUnknown: n = TNoteKind(x)
     else: localError(conf, info, "unknown hint: " & id)
   else:
-    let x = findStr(lineinfos.WarningsToStr, id)
-    if x >= 0: n = TNoteKind(x + ord(warnMin))
+    let x = findStr(warnMin..warnMax, id, errUnknown)
+    if x != errUnknown: n = TNoteKind(x)
     else: localError(conf, info, "unknown warning: " & id)
 
-  let val = substr(arg, i).normalize
+  var val = substr(arg, i).normalize
+  if val == "": val = "on"
   if val notin ["on", "off"]:
     localError(conf, info, errOnOrOffExpectedButXFound % arg)
   elif n notin conf.cmdlineNotes or pass == passCmd1:
@@ -209,12 +210,18 @@ proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
     incl(conf.modifiedyNotes, n)
     case val
     of "on":
-      incl(conf.notes, n)
-      incl(conf.mainPackageNotes, n)
+      if state == wWarningAsError:
+        incl(conf.warningAsErrors, n)
+      else:
+        incl(conf.notes, n)
+        incl(conf.mainPackageNotes, n)
     of "off":
-      excl(conf.notes, n)
-      excl(conf.mainPackageNotes, n)
-      excl(conf.foreignPackageNotes, n)
+      if state == wWarningAsError:
+        excl(conf.warningAsErrors, n)
+      else:
+        excl(conf.notes, n)
+        excl(conf.mainPackageNotes, n)
+        excl(conf.foreignPackageNotes, n)
 
 proc processCompile(conf: ConfigRef; filename: string) =
   var found = findFile(conf, filename)
@@ -222,7 +229,7 @@ proc processCompile(conf: ConfigRef; filename: string) =
   extccomp.addExternalFileToCompile(conf, found)
 
 const
-  errNoneBoehmRefcExpectedButXFound = "'none', 'boehm' or 'refc' expected, but '$1' found"
+  errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found"
   errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found"
@@ -291,7 +298,7 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
     result = conf.options * {optNaNCheck, optInfCheck} == {optNaNCheck, optInfCheck}
   of "infchecks": result = contains(conf.options, optInfCheck)
   of "nanchecks": result = contains(conf.options, optNaNCheck)
-  of "nilchecks": result = contains(conf.options, optNilCheck)
+  of "nilchecks": result = false # not a thing
   of "objchecks": result = contains(conf.options, optObjCheck)
   of "fieldchecks": result = contains(conf.options, optFieldCheck)
   of "rangechecks": result = contains(conf.options, optRangeCheck)
@@ -312,7 +319,6 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "patterns", "trmacros": result = contains(conf.options, optTrMacros)
   of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace)
   of "nilseqs": result = contains(conf.options, optNilSeqs)
-  of "oldast": result = contains(conf.options, optOldAst)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc processPath(conf: ConfigRef; path: string, info: TLineInfo,
@@ -426,11 +432,18 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "outdir":
     expectArg(conf, switch, arg, pass, info)
     conf.outDir = processPath(conf, arg, info, notRelativeToProj=true)
+  of "usenimcache":
+    processOnOffSwitchG(conf, {optUseNimcache}, arg, pass, info)
   of "docseesrcurl":
     expectArg(conf, switch, arg, pass, info)
     conf.docSeeSrcUrl = arg
   of "docroot":
-    conf.docRoot = if arg.len == 0: "@default" else: arg
+    conf.docRoot = if arg.len == 0: docRootDefault else: arg
+  of "backend", "b":
+    let backend = parseEnum(arg.normalize, TBackend.default)
+    if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
+    conf.backend = backend
+  of "doccmd": conf.docCmd = arg
   of "mainmodule", "m":
     discard "allow for backwards compatibility, but don't do anything"
   of "define", "d":
@@ -470,14 +483,20 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "forcebuild", "f":
     processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
   of "project":
-    processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
+    processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info)
   of "gc":
+    if conf.backend == backendJs or conf.command == "js":
+      # for: bug #16033
+      # This might still be imperfect, in rarse corner cases
+      # (where command is reset in nimscript, maybe).
+      return
     expectArg(conf, switch, arg, pass, info)
     if pass in {passCmd2, passPP}:
       case arg.normalize
       of "boehm":
         conf.selectedGC = gcBoehm
         defineSymbol(conf.symbols, "boehmgc")
+        incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS
       of "refc":
         conf.selectedGC = gcRefc
       of "v2":
@@ -487,8 +506,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
         defineSymbol(conf.symbols, "gcmarkandsweep")
       of "destructors", "arc":
         conf.selectedGC = gcArc
-        if conf.cmd != cmdCompileToCpp:
-          conf.exc = excGoto
         defineSymbol(conf.symbols, "gcdestructors")
         defineSymbol(conf.symbols, "gcarc")
         incl conf.globalOptions, optSeqDestructors
@@ -498,8 +515,6 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
           defineSymbol(conf.symbols, "nimV2")
       of "orc":
         conf.selectedGC = gcOrc
-        if conf.cmd != cmdCompileToCpp:
-          conf.exc = excGoto
         defineSymbol(conf.symbols, "gcdestructors")
         defineSymbol(conf.symbols, "gcorc")
         incl conf.globalOptions, optSeqDestructors
@@ -528,6 +543,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if processOnOffSwitchOrList(conf, {optWarns}, arg, pass, info): listWarnings(conf)
   of "warning": processSpecificNote(arg, wWarning, pass, info, switch, conf)
   of "hint": processSpecificNote(arg, wHint, pass, info, switch, conf)
+  of "warningaserror": processSpecificNote(arg, wWarningAsError, pass, info, switch, conf)
   of "hints":
     if processOnOffSwitchOrList(conf, {optHints}, arg, pass, info): listHints(conf)
   of "threadanalysis": processOnOffSwitchG(conf, {optThreadAnalysis}, arg, pass, info)
@@ -569,25 +585,13 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     else:
       undefSymbol(conf.symbols, "hotcodereloading")
       undefSymbol(conf.symbols, "useNimRtl")
-  of "oldnewlines":
-    case arg.normalize
-    of "", "on":
-      conf.oldNewlines = true
-      defineSymbol(conf.symbols, "nimOldNewlines")
-    of "off":
-      conf.oldNewlines = false
-      undefSymbol(conf.symbols, "nimOldNewlines")
-    else:
-      localError(conf, info, errOnOrOffExpectedButXFound % arg)
-  of "laxstrings": processOnOffSwitch(conf, {optLaxStrings}, arg, pass, info)
   of "nilseqs": processOnOffSwitch(conf, {optNilSeqs}, arg, pass, info)
-  of "oldast": processOnOffSwitch(conf, {optOldAst}, arg, pass, info)
   of "checks", "x": processOnOffSwitch(conf, ChecksOptions, arg, pass, info)
   of "floatchecks":
     processOnOffSwitch(conf, {optNaNCheck, optInfCheck}, arg, pass, info)
   of "infchecks": processOnOffSwitch(conf, {optInfCheck}, arg, pass, info)
   of "nanchecks": processOnOffSwitch(conf, {optNaNCheck}, arg, pass, info)
-  of "nilchecks": processOnOffSwitch(conf, {optNilCheck}, arg, pass, info)
+  of "nilchecks": discard "no such thing anymore"
   of "objchecks": processOnOffSwitch(conf, {optObjCheck}, arg, pass, info)
   of "fieldchecks": processOnOffSwitch(conf, {optFieldCheck}, arg, pass, info)
   of "rangechecks": processOnOffSwitch(conf, {optRangeCheck}, arg, pass, info)
@@ -866,8 +870,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "expandmacro":
     expectArg(conf, switch, arg, pass, info)
     conf.macrosToExpand[arg] = "T"
-  of "oldgensym":
-    processOnOffSwitchG(conf, {optNimV019}, arg, pass, info)
+  of "expandarc":
+    expectArg(conf, switch, arg, pass, info)
+    conf.arcToExpand[arg] = "T"
   of "useversion":
     expectArg(conf, switch, arg, pass, info)
     case arg
@@ -878,14 +883,26 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       defineSymbol(conf.symbols, "NimPatch", "100")
       # old behaviors go here:
       defineSymbol(conf.symbols, "nimOldRelativePathBehavior")
+      undefSymbol(conf.symbols, "nimDoesntTrackDefects")
       ast.eqTypeFlags.excl {tfGcSafe, tfNoSideEffect}
       conf.globalOptions.incl optNimV1Emulation
+    of "1.2":
+      defineSymbol(conf.symbols, "NimMajor", "1")
+      defineSymbol(conf.symbols, "NimMinor", "2")
+      # always be compatible with 1.2.100:
+      defineSymbol(conf.symbols, "NimPatch", "100")
+      conf.globalOptions.incl optNimV12Emulation
     else:
       localError(conf, info, "unknown Nim version; currently supported values are: {1.0}")
   of "benchmarkvm":
     processOnOffSwitchG(conf, {optBenchmarkVM}, arg, pass, info)
+  of "profilevm":
+    processOnOffSwitchG(conf, {optProfileVM}, arg, pass, info)
   of "sinkinference":
     processOnOffSwitch(conf, {optSinkInference}, arg, pass, info)
+  of "cursorinference":
+    # undocumented, for debugging purposes only:
+    processOnOffSwitch(conf, {optCursorInference}, arg, pass, info)
   of "panics":
     processOnOffSwitchG(conf, {optPanics}, arg, pass, info)
     if optPanics in conf.globalOptions:
@@ -893,15 +910,14 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "sourcemap":
     conf.globalOptions.incl optSourcemap
     conf.options.incl optLineDir
-    # processOnOffSwitchG(conf, {optSourcemap, opt}, arg, pass, info)
+  of "deepcopy":
+    processOnOffSwitchG(conf, {optEnableDeepCopy}, arg, pass, info)
   of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -)
     handleStdinInput(conf)
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)
 
-template gCmdLineInfo*(): untyped = newLineInfo(commandLineIdx, 1, 1)
-
 proc processCommand*(switch: string, pass: TCmdLinePass; config: ConfigRef) =
   var cmd, arg: string
   splitSwitch(config, switch, cmd, arg, pass, gCmdLineInfo)

compiler/condsyms.nim

@@ -13,7 +13,7 @@ import
   strtabs
 
 from options import Feature
-from lineinfos import HintsToStr, WarningsToStr
+from lineinfos import hintMin, hintMax, warnMin, warnMax
 
 proc defineSymbol*(symbols: StringTableRef; symbol: string, value: string = "true") =
   symbols[symbol] = value
@@ -86,13 +86,13 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimMacrosSizealignof")
   defineSymbol("nimNoZeroExtendMagic")
   defineSymbol("nimMacrosGetNodeId")
-  for f in low(Feature)..high(Feature):
+  for f in Feature:
     defineSymbol("nimHas" & $f)
 
-  for s in WarningsToStr:
-    defineSymbol("nimHasWarning" & s)
-  for s in HintsToStr:
-    defineSymbol("nimHasHint" & s)
+  for s in warnMin..warnMax:
+    defineSymbol("nimHasWarning" & $s)
+  for s in hintMin..hintMax:
+    defineSymbol("nimHasHint" & $s)
 
   defineSymbol("nimFixedOwned")
   defineSymbol("nimHasStyleChecks")
@@ -102,6 +102,7 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimnomagic64")
   defineSymbol("nimNewShiftOps")
   defineSymbol("nimHasCursor")
+  defineSymbol("nimAlignPragma")
   defineSymbol("nimHasExceptionsQuery")
   defineSymbol("nimHasIsNamedTuple")
   defineSymbol("nimHashOrdinalFixed")
@@ -116,3 +117,9 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimNewIntegerOps")
   defineSymbol("nimHasInvariant")
   defineSymbol("nimHasStacktraceMsgs")
+  defineSymbol("nimDoesntTrackDefects")
+  defineSymbol("nimHasLentIterators")
+  defineSymbol("nimHasDeclaredMagic")
+  defineSymbol("nimHasStacktracesModule")
+  defineSymbol("nimHasEffectTraitsModule")
+  defineSymbol("nimHasCastPragmaBlocks")

compiler/depends.nim

@@ -51,7 +51,10 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) =
       rope(project.splitFile.name), b.dotGraph],
             changeFileExt(project, "dot"))
 
-proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
+when not defined(nimHasSinkInference):
+  {.pragma: nosinks.}
+
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext {.nosinks.} =
   var g: PGen
   new(g)
   g.module = module

compiler/dfa.nim

@@ -29,44 +29,45 @@
 ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
 ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
 
-import ast, types, intsets, lineinfos, renderer
+import ast, types, intsets, lineinfos, renderer, asciitables
 
 from patterns import sameTrees
 
 type
   InstrKind* = enum
-    goto, fork, join, def, use
+    goto, fork, def, use
   Instr* = object
     n*: PNode # contains the def/use location.
     case kind*: InstrKind
-    of goto, fork, join: dest*: int
+    of goto, fork: dest*: int
     else: discard
 
   ControlFlowGraph* = seq[Instr]
 
   TPosition = distinct int
+
   TBlock = object
-    label: PSym
-    fixups: seq[TPosition]
+    case isTryBlock: bool
+    of false:
+      label: PSym
+      breakFixups: seq[(TPosition, seq[PNode])] #Contains the gotos for the breaks along with their pending finales
+    of true:
+      finale: PNode
+      raiseFixups: seq[TPosition] #Contains the gotos for the raises
 
   Con = object
     code: ControlFlowGraph
-    inCall, inTryStmt: int
+    inTryStmt: int
     blocks: seq[TBlock]
-    tryStmtFixups: seq[TPosition]
-    forks: seq[TPosition]
     owner: PSym
 
-proc debugInfo(info: TLineInfo): string =
-  result = $info.line #info.toFilename & ":" & $info.line
-
-proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
+proc codeListing(c: ControlFlowGraph, start = 0; last = -1): string =
   # for debugging purposes
   # first iteration: compute all necessary labels:
   var jumpTargets = initIntSet()
   let last = if last < 0: c.len-1 else: min(last, c.len-1)
   for i in start..last:
-    if c[i].kind in {goto, fork, join}:
+    if c[i].kind in {goto, fork}:
       jumpTargets.incl(i+c[i].dest)
   var i = start
   while i <= last:
@@ -77,26 +78,22 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
     case c[i].kind
     of def, use:
       result.add renderTree(c[i].n)
-    of goto, fork, join:
+    of goto, fork:
       result.add "L"
       result.addInt c[i].dest+i
     result.add("\t#")
-    result.add(debugInfo(c[i].n.info))
+    result.add($c[i].n.info.line)
     result.add("\n")
     inc i
   if i in jumpTargets: result.add("L" & $i & ": End\n")
-  # consider calling `asciitables.alignTable`
 
-proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
+proc echoCfg*(c: ControlFlowGraph; start = 0; last = -1) {.deprecated.} =
   ## echos the ControlFlowGraph for debugging purposes.
-  var buf = ""
-  codeListing(c, buf, start, last)
-  echo buf
+  echo codeListing(c, start, last).alignTable
 
 proc forkI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: fork, dest: 0)
-  c.forks.add result
 
 proc gotoI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
@@ -104,6 +101,24 @@ proc gotoI(c: var Con; n: PNode): TPosition =
 
 #[
 
+Join is no more
+===============
+Instead of generating join instructions we adapt our traversal of the CFG.
+
+When encountering a fork we split into two paths, we follow the path
+starting at "pc + 1" until it encounters the joinpoint: "pc + forkInstr.dest".
+If we encounter gotos that would jump further than the current joinpoint,
+as can happen with gotos generated by unstructured controlflow such as break, raise or return,
+we simply suspend following the current path, and follow the other path until the new joinpoint
+which is simply the instruction pointer returned to us by the now suspended path.
+If the path we are following now, also encounters a goto that exceeds the joinpoint
+we repeat the process; suspending the current path and evaluating the other one with a new joinpoint.
+If we eventually reach a common joinpoint we join the two paths.
+This new "ping-pong" approach has the obvious advantage of not requiring join instructions, as such
+cutting down on the CFG size but is also mandatory for correctly handling complicated cases
+of unstructured controlflow.
+
+
 Design of join
 ==============
 
@@ -252,33 +267,36 @@ duplicate the 'join' instructions on breaks and return exits!
 
 ]#
 
-proc joinI(c: var Con; fromFork: TPosition; n: PNode) =
-  let dist = fromFork.int - c.code.len
-  c.code.add Instr(n: n, kind: join, dest: dist)
+proc genLabel(c: Con): TPosition = TPosition(c.code.len)
 
-proc genLabel(c: Con): TPosition =
-  result = TPosition(c.code.len)
+template checkedDistance(dist): int =
+  doAssert low(int) div 2 + 1 < dist and dist < high(int) div 2
+  dist
 
 proc jmpBack(c: var Con, n: PNode, p = TPosition(0)) =
-  let dist = p.int - c.code.len
-  doAssert(low(int) div 2 + 1 < dist and dist < high(int) div 2)
-  c.code.add Instr(n: n, kind: goto, dest: dist)
+  c.code.add Instr(n: n, kind: goto, dest: checkedDistance(p.int - c.code.len))
 
 proc patch(c: var Con, p: TPosition) =
   # patch with current index
-  let p = p.int
-  let diff = c.code.len - p
-  doAssert(low(int) div 2 + 1 < diff and diff < high(int) div 2)
-  c.code[p].dest = diff
+  c.code[p.int].dest = checkedDistance(c.code.len - p.int)
+
+proc gen(c: var Con; n: PNode)
 
 proc popBlock(c: var Con; oldLen: int) =
-  for f in c.blocks[oldLen].fixups:
-    c.patch(f)
+  var exits: seq[TPosition]
+  exits.add c.gotoI(newNode(nkEmpty))
+  for f in c.blocks[oldLen].breakFixups:
+    c.patch(f[0])
+    for finale in f[1]:
+      c.gen(finale)
+    exits.add c.gotoI(newNode(nkEmpty))
+  for e in exits:
+    c.patch e
   c.blocks.setLen(oldLen)
 
-template withBlock(labl: PSym; body: untyped) {.dirty.} =
-  var oldLen {.gensym.} = c.blocks.len
-  c.blocks.add TBlock(label: labl, fixups: @[])
+template withBlock(labl: PSym; body: untyped) =
+  let oldLen = c.blocks.len
+  c.blocks.add TBlock(isTryBlock: false, label: labl)
   body
   popBlock(c, oldLen)
 
@@ -286,8 +304,6 @@ proc isTrue(n: PNode): bool =
   n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or
     n.kind == nkIntLit and n.intVal != 0
 
-proc gen(c: var Con; n: PNode) # {.noSideEffect.}
-
 when true:
   proc genWhile(c: var Con; n: PNode) =
     # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
@@ -299,12 +315,13 @@ when true:
 
     Becomes:
 
-    if cond:
-      body
+    block:
       if cond:
         body
         if cond:
           body
+          if cond:
+            body
 
     We still need to ensure 'break' resolves properly, so an AST to AST
     translation is impossible.
@@ -330,25 +347,20 @@ when true:
     if isTrue(n[0]):
       # 'while true' is an idiom in Nim and so we produce
       # better code for it:
-      for i in 0..2:
-        withBlock(nil):
+      withBlock(nil):
+        for i in 0..2:
           c.gen(n[1])
     else:
-      let oldForksLen = c.forks.len
-      var endings: array[3, TPosition]
-      for i in 0..2:
-        withBlock(nil):
+      withBlock(nil):
+        var endings: array[3, TPosition]
+        for i in 0..2:
           c.gen(n[0])
           endings[i] = c.forkI(n)
           c.gen(n[1])
-      for i in countdown(endings.high, 0):
-        let endPos = endings[i]
-        c.patch(endPos)
-        c.joinI(c.forks.pop(), n)
-      doAssert(c.forks.len == oldForksLen)
+        for i in countdown(endings.high, 0):
+          c.patch(endings[i])
 
 else:
-
   proc genWhile(c: var Con; n: PNode) =
     # lab1:
     #   cond, tmp
@@ -356,7 +368,6 @@ else:
     #   body
     #   jmp lab1
     # lab2:
-    let oldForksLen = c.forks.len
     let lab1 = c.genLabel
     withBlock(nil):
       if isTrue(n[0]):
@@ -364,39 +375,14 @@ else:
         c.jmpBack(n, lab1)
       else:
         c.gen(n[0])
-        let lab2 = c.forkI(n)
-        c.gen(n[1])
-        c.jmpBack(n, lab1)
-        c.patch(lab2)
-    setLen(c.forks, oldForksLen)
-
-proc genBlock(c: var Con; n: PNode) =
-  withBlock(n[0].sym):
-    c.gen(n[1])
-
-proc genJoins(c: var Con; n: PNode) =
-  for i in countdown(c.forks.high, 0): joinI(c, c.forks[i], n)
-
-proc genBreak(c: var Con; n: PNode) =
-  genJoins(c, n)
-  let lab1 = c.gotoI(n)
-  if n[0].kind == nkSym:
-    #echo cast[int](n[0].sym)
-    for i in countdown(c.blocks.len-1, 0):
-      if c.blocks[i].label == n[0].sym:
-        c.blocks[i].fixups.add lab1
-        return
-    #globalError(n.info, "VM problem: cannot find 'break' target")
-  else:
-    c.blocks[c.blocks.high].fixups.add lab1
+        forkT(n):
+          c.gen(n[1])
+          c.jmpBack(n, lab1)
 
 template forkT(n, body) =
-  let oldLen = c.forks.len
   let lab1 = c.forkI(n)
   body
   c.patch(lab1)
-  c.joinI(lab1, n)
-  setLen(c.forks, oldLen)
 
 proc genIf(c: var Con, n: PNode) =
   #[
@@ -435,21 +421,16 @@ proc genIf(c: var Con, n: PNode) =
     join F1
 
   ]#
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
   for i in 0..<n.len:
-    var it = n[i]
+    let it = n[i]
     c.gen(it[0])
     if it.len == 2:
-      let elsePos = forkI(c, it[1])
-      c.gen(it[1])
-      endings.add(c.gotoI(it[1]))
-      c.patch(elsePos)
+      forkT(it[1]):
+        c.gen(it[1])
+        endings.add c.gotoI(it[1])
   for i in countdown(endings.high, 0):
-    let endPos = endings[i]
-    c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
+    c.patch(endings[i])
 
 proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, a
@@ -476,75 +457,90 @@ proc genCase(c: var Con; n: PNode) =
     abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
 
   var endings: seq[TPosition] = @[]
-  let oldLen = c.forks.len
   c.gen(n[0])
   for i in 1..<n.len:
     let it = n[i]
-    if it.len == 1:
-      c.gen(it[0])
-    elif i == n.len-1 and isExhaustive:
+    if it.len == 1 or (i == n.len-1 and isExhaustive):
       # treat the last branch as 'else' if this is an exhaustive case statement.
       c.gen(it.lastSon)
     else:
-      let elsePos = c.forkI(it.lastSon)
-      c.gen(it.lastSon)
-      endings.add(c.gotoI(it.lastSon))
-      c.patch(elsePos)
+      forkT(it.lastSon):
+        c.gen(it.lastSon)
+        endings.add c.gotoI(it.lastSon)
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
+
+proc genBlock(c: var Con; n: PNode) =
+  withBlock(n[0].sym):
+    c.gen(n[1])
+
+proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) =
+  let lab1 = c.gotoI(n)
+  if c.blocks[i].isTryBlock:
+    c.blocks[i].raiseFixups.add lab1
+  else:
+    var trailingFinales: seq[PNode]
+    if c.inTryStmt > 0: #Ok, we are in a try, lets see which (if any) try's we break out from:
+      for b in countdown(c.blocks.high, i):
+        if c.blocks[b].isTryBlock:
+          trailingFinales.add c.blocks[b].finale
+
+    c.blocks[i].breakFixups.add (lab1, trailingFinales)
+
+proc genBreak(c: var Con; n: PNode) =
+  if n[0].kind == nkSym:
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock and c.blocks[i].label == n[0].sym:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    #globalError(n.info, "VM problem: cannot find 'break' target")
+  else:
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
 
 proc genTry(c: var Con; n: PNode) =
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
-  inc c.inTryStmt
-  let oldFixups = c.tryStmtFixups.len
 
-  #let elsePos = c.forkI(n)
+  let oldLen = c.blocks.len
+  c.blocks.add TBlock(isTryBlock: true, finale: if n[^1].kind == nkFinally: n[^1] else: newNode(nkEmpty))
+
+  inc c.inTryStmt
   c.gen(n[0])
   dec c.inTryStmt
-  for i in oldFixups..c.tryStmtFixups.high:
-    let f = c.tryStmtFixups[i]
+
+  for f in c.blocks[oldLen].raiseFixups:
     c.patch(f)
-    # we also need to produce join instructions
-    # for the 'fork' that might precede the goto instruction
-    if f.int-1 >= 0 and c.code[f.int-1].kind == fork:
-      c.joinI(TPosition(f.int-1), n)
 
-  setLen(c.tryStmtFixups, oldFixups)
+  c.blocks.setLen oldLen
 
-  #c.patch(elsePos)
   for i in 1..<n.len:
     let it = n[i]
     if it.kind != nkFinally:
-      let endExcept = c.forkI(it)
-      c.gen(it.lastSon)
-      endings.add(c.gotoI(it))
-      c.patch(endExcept)
+      forkT(it):
+        c.gen(it.lastSon)
+        endings.add c.gotoI(it)
   for i in countdown(endings.high, 0):
-    let endPos = endings[i]
-    c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-
-  # join the 'elsePos' forkI instruction:
-  #c.joinI(c.forks.pop(), n)
+    c.patch(endings[i])
 
   let fin = lastSon(n)
   if fin.kind == nkFinally:
     c.gen(fin[0])
-  doAssert(c.forks.len == oldLen)
 
 template genNoReturn(c: var Con; n: PNode) =
   # leave the graph
   c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 proc genRaise(c: var Con; n: PNode) =
-  genJoins(c, n)
   gen(c, n[0])
   if c.inTryStmt > 0:
-    c.tryStmtFixups.add c.gotoI(n)
+    for i in countdown(c.blocks.high, 0):
+      if c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    assert false #Unreachable
   else:
     genNoReturn(c, n)
 
@@ -553,12 +549,11 @@ proc genImplicitReturn(c: var Con) =
     gen(c, c.owner.ast[resultPos])
 
 proc genReturn(c: var Con; n: PNode) =
-  genJoins(c, n)
   if n[0].kind != nkEmpty:
     gen(c, n[0])
   else:
     genImplicitReturn(c)
-  genNoReturn(c, n)
+  genBreakOrRaiseAux(c, 0, n)
 
 const
   InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp}
@@ -578,58 +573,43 @@ proc skipConvDfa*(n: PNode): PNode =
       result = result[1]
     else: break
 
-proc genUse(c: var Con; orig: PNode) =
-  var n = orig
+proc aliases*(obj, field: PNode): bool =
+  var n = field
+  var obj = obj
   while true:
+    case obj.kind
+    of {nkObjDownConv, nkObjUpConv, nkAddr, nkHiddenAddr, nkDerefExpr, nkHiddenDeref}:
+      obj = obj[0]
+    of PathKinds1:
+      obj = obj[1]
+    else: break
+  while true:
+    if sameTrees(obj, n): return true
     case n.kind
-    of PathKinds0 - {nkBracketExpr}:
-      n = n[0]
-    of nkBracketExpr:
-      gen(c, n[1])
+    of PathKinds0:
       n = n[0]
     of PathKinds1:
       n = n[1]
     else: break
-    if n.kind in nkCallKinds:
-      gen(c, n)
-  if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: orig, kind: use)
 
-proc aliases*(obj, field: PNode): bool =
-  var n = field
-  var obj = obj
-  while obj.kind in {nkHiddenSubConv, nkHiddenStdConv, nkObjDownConv, nkObjUpConv}:
-    obj = obj[0]
-  while true:
-    if sameTrees(obj, n): return true
-    case n.kind
-    of PathKinds0, PathKinds1:
-      n = n[0]
-    else:
-      break
+type InstrTargetKind* = enum
+  None, Full, Partial
 
-proc useInstrTargets*(ins: Instr; loc: PNode): bool =
-  assert ins.kind == use
-  result = sameTrees(ins.n, loc) or
-    ins.n.aliases(loc) or loc.aliases(ins.n)
-  # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
-  # use x.f;  question: does it affect the full 'x'? No.
-  # use x; question does it affect 'x.f'? Yes.
-
-proc defInstrTargets*(ins: Instr; loc: PNode): bool =
-  assert ins.kind == def
-  result = sameTrees(ins.n, loc) or ins.n.aliases(loc)
-  # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
-  # def x.f; question: does it affect the full 'x'? No.
-  # def x; question: does it affect the 'x.f'? Yes.
+proc instrTargets*(insloc, loc: PNode): InstrTargetKind =
+  if sameTrees(insloc, loc) or insloc.aliases(loc):
+    Full    # x -> x; x -> x.f
+  elif loc.aliases(insloc):
+    Partial # x.f -> x
+  else: None
 
 proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
   var n = orig
   while true:
     case n.kind
-    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-       nkObjDownConv, nkObjUpConv, nkHiddenAddr, nkAddr:
+    of PathKinds0 - {nkBracketExpr, nkHiddenDeref, nkDerefExpr}:
       n = n[0]
+    of PathKinds1:
+      n = n[1]
     of nkBracketExpr:
       # in a[i] the 'i' must be known
       if n.len > 1 and n[1].kind in {nkCharLit..nkUInt64Lit}:
@@ -639,15 +619,16 @@ proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
     of nkHiddenDeref, nkDerefExpr:
       # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
       # pointer indirection.
+      # bug #14159, we cannot reason about sinkParam[].location as it can
+      # still be shared for tyRef.
       n = n[0]
-      return n.kind == nkSym and n.sym.owner == owner and (isSinkParam(n.sym) or
-          n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
-    else:
-      break
+      return n.kind == nkSym and n.sym.owner == owner and
+         (n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
+    else: break
   # XXX Allow closure deref operations here if we know
   # the owner controlled the closure allocation?
   result = n.kind == nkSym and n.sym.owner == owner and
-    owner.kind != skModule and
+    {sfGlobal, sfThread, sfCursor} * n.sym.flags == {} and
     (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar)
   # Note: There is a different move analyzer possible that checks for
   # consume(param.key); param.key = newValue  for all paths. Then code like
@@ -660,43 +641,42 @@ proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
   # free the old data and all is lost! Lesson: Don't be too smart, trust the
   # lower level C++ optimizer to specialize this code.
 
-proc genDef(c: var Con; n: PNode) =
-  var m = n
-  # XXX do something about this duplicated logic here.
+proc skipTrivials(c: var Con, n: PNode): PNode =
+  result = n
   while true:
-    case m.kind
-    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-        nkObjDownConv, nkObjUpConv, nkHiddenAddr, nkAddr:
-      m = m[0]
+    case result.kind
+    of PathKinds0 - {nkBracketExpr}:
+      result = result[0]
     of nkBracketExpr:
-      gen(c, m[1])
-      m = m[0]
-    of nkHiddenDeref, nkDerefExpr:
-      m = m[0]
-    else:
-      break
+      gen(c, result[1])
+      result = result[0]
+    of PathKinds1:
+      result = result[1]
+    else: break
+
+proc genUse(c: var Con; orig: PNode) =
+  let n = c.skipTrivials(orig)
 
   if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: n, kind: def)
-  elif isAnalysableFieldAccess(n, c.owner):
-    c.code.add Instr(n: n, kind: def)
-  else:
-    # bug #13314: An assignment to t5.w = -5 is a usage of 't5'
-    # we still need to gather the use information:
+    c.code.add Instr(n: orig, kind: use)
+  elif n.kind in nkCallKinds:
     gen(c, n)
 
+proc genDef(c: var Con; orig: PNode) =
+  let n = c.skipTrivials(orig)
+
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    c.code.add Instr(n: orig, kind: def)
+
 proc genCall(c: var Con; n: PNode) =
   gen(c, n[0])
   var t = n[0].typ
   if t != nil: t = t.skipTypes(abstractInst)
-  inc c.inCall
   for i in 1..<n.len:
     gen(c, n[i])
     when false:
-      if t != nil and i < t.len and t[i].kind == tyVar:
-        # This is wrong! Pass by var is a 'might def', not a 'must def'
-        # like the other defs we emit. This is not good enough for a move
-        # optimizer.
+      if t != nil and i < t.len and t[i].kind == tyOut:
+        # Pass by 'out' is a 'must def'. Good enough for a move optimizer.
         genDef(c, n[i])
   # every call can potentially raise:
   if c.inTryStmt > 0 and canRaiseConservative(n[0]):
@@ -705,11 +685,11 @@ proc genCall(c: var Con; n: PNode) =
     # goto exceptionHandler (except or finally)
     # lab1:
     # join F1
-    let endGoto = c.forkI(n)
-    c.tryStmtFixups.add c.gotoI(n)
-    c.patch(endGoto)
-    c.joinI(c.forks.pop(), n)
-  dec c.inCall
+    forkT(n):
+      for i in countdown(c.blocks.high, 0):
+        if c.blocks[i].isTryBlock:
+          genBreakOrRaiseAux(c, i, n)
+          break
 
 proc genMagic(c: var Con; n: PNode; m: TMagic) =
   case m
@@ -753,7 +733,7 @@ proc gen(c: var Con; n: PNode) =
     # "uses" 'i'. But we are only talking about builtin array indexing so
     # it doesn't matter and 'x = 34' is NOT a usage of 'x'.
     genDef(c, n[0])
-  of PathKinds0 - {nkHiddenStdConv, nkHiddenSubConv, nkObjDownConv, nkObjUpConv}:
+  of PathKinds0 - {nkObjDownConv, nkObjUpConv}:
     genUse(c, n)
   of nkIfStmt, nkIfExpr: genIf(c, n)
   of nkWhenStmt:
@@ -770,18 +750,21 @@ proc gen(c: var Con; n: PNode) =
      nkBracket, nkCurly, nkPar, nkTupleConstr, nkClosure, nkObjConstr, nkYieldStmt:
     for x in n: gen(c, x)
   of nkPragmaBlock: gen(c, n.lastSon)
-  of nkDiscardStmt, nkObjDownConv, nkObjUpConv: gen(c, n[0])
-  of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast, nkHiddenSubConv, nkHiddenStdConv:
+  of nkDiscardStmt, nkObjDownConv, nkObjUpConv, nkStringToCString, nkCStringToString:
+    gen(c, n[0])
+  of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast, PathKinds1:
     gen(c, n[1])
-  of nkStringToCString, nkCStringToString: gen(c, n[0])
   of nkVarSection, nkLetSection: genVarSection(c, n)
-  of nkDefer:
-    doAssert false, "dfa construction pass requires the elimination of 'defer'"
+  of nkDefer: doAssert false, "dfa construction pass requires the elimination of 'defer'"
   else: discard
 
 proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
   ## constructs a control flow graph for ``body``.
   var c = Con(code: @[], blocks: @[], owner: s)
-  gen(c, body)
-  genImplicitReturn(c)
-  shallowCopy(result, c.code)
+  withBlock(s):
+    gen(c, body)
+    genImplicitReturn(c)
+  when defined(gcArc) or defined(gcOrc):
+    result = c.code # will move
+  else:
+    shallowCopy(result, c.code)

compiler/docgen.nim

@@ -17,19 +17,28 @@ import
   packages/docutils/rst, packages/docutils/rstgen,
   json, xmltree, cgi, trees, types,
   typesrenderer, astalgo, lineinfos, intsets,
-  pathutils, trees
+  pathutils, trees, tables, nimpaths, renderverbatim, osproc
+
+from std/private/globs import nativeToUnixPath
 
 const
   exportSection = skField
-  htmldocsDir = RelativeDir"htmldocs"
+  docCmdSkip = "skip"
 
 type
   TSections = array[TSymKind, Rope]
+  ExampleGroup = ref object
+    ## a group of runnableExamples with same rdoccmd
+    rdoccmd: string ## from 1st arg in `runnableExamples(rdoccmd): body`
+    docCmd: string ## from user config, e.g. --doccmd:-d:foo
+    code: string ## contains imports; each import contains `body`
+    index: int ## group index
   TDocumentor = object of rstgen.RstGenerator
     modDesc: Rope       # module description
     module: PSym
     modDeprecationMsg: Rope
-    toc, section: TSections
+    toc, toc2, section: TSections
+    tocTable: array[TSymKind, Table[string, Rope]]
     indexValFilename: string
     analytics: string  # Google Analytics javascript, "" if doesn't exist
     seenSymbols: StringTableRef # avoids duplicate symbol generation for HTML.
@@ -41,22 +50,12 @@ type
     exampleCounter: int
     emitted: IntSet # we need to track which symbols have been emitted
                     # already. See bug #3655
-    destFile*: AbsoluteFile
     thisDir*: AbsoluteDir
-    examples: string
-    wroteCss*: bool
+    exampleGroups: OrderedTable[string, ExampleGroup]
+    wroteSupportFiles*: bool
 
   PDoc* = ref TDocumentor ## Alias to type less.
 
-proc nativeToUnix(path: string): string =
-  doAssert not path.isAbsolute # absolute files need more care for the drive
-  when DirSep == '\\':
-    result = replace(path, '\\', '/')
-  else: result = path
-
-proc docOutDir(conf: ConfigRef, subdir: RelativeDir = RelativeDir""): AbsoluteDir =
-  if not conf.outDir.isEmpty: conf.outDir else: conf.projectPath / subdir
-
 proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): RelativeFile =
   ## returns a relative file that will be appended to outDir
   let file2 = $file
@@ -65,7 +64,7 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
   proc nimbleDir(): AbsoluteDir =
     getNimbleFile(conf, file2).parentDir.AbsoluteDir
   case conf.docRoot:
-  of "@default": # using `@` instead of `$` to avoid shell quoting complications
+  of docRootDefault:
     result = getRelativePathFromConfigPath(conf, file)
     let dir = nimbleDir()
     if not dir.isEmpty:
@@ -81,16 +80,22 @@ proc presentationPath*(conf: ConfigRef, file: AbsoluteFile, isTitle = false): Re
     result = getRelativePathFromConfigPath(conf, file)
     if result.isEmpty: bail()
   elif conf.docRoot.len > 0:
-    doAssert conf.docRoot.isAbsolute, conf.docRoot # or globalError
-    doAssert conf.docRoot.existsDir, conf.docRoot
-    result = relativeTo(file, conf.docRoot.AbsoluteDir)
+    # we're (currently) requiring `isAbsolute` to avoid confusion when passing
+    # a relative path (would it be relative wrt $PWD or to projectfile)
+    conf.globalAssert conf.docRoot.isAbsolute, arg=conf.docRoot
+    conf.globalAssert conf.docRoot.dirExists, arg=conf.docRoot
+    # needed because `canonicalizePath` called on `file`
+    result = file.relativeTo conf.docRoot.expandFilename.AbsoluteDir
   else:
     bail()
+  if isAbsolute(result.string):
+    result = file.string.splitPath()[1].RelativeFile
   if isTitle:
-    result = result.string.nativeToUnix.RelativeFile
+    result = result.string.nativeToUnixPath.RelativeFile
   else:
-    result = result.string.replace("..", "@@").RelativeFile ## refs #13223
+    result = result.string.replace("..", dotdotMangle).RelativeFile
   doAssert not result.isEmpty
+  doAssert not isAbsolute(result.string)
 
 proc whichType(d: PDoc; n: PNode): PSym =
   if n.kind == nkSym:
@@ -136,7 +141,7 @@ template declareClosures =
     result = options.findFile(conf, s).string
     if result.len == 0:
       result = getCurrentDir() / s
-      if not existsFile(result): result = ""
+      if not fileExists(result): result = ""
 
 proc parseRst(text, filename: string,
               line, column: int, hasToc: var bool,
@@ -147,13 +152,9 @@ proc parseRst(text, filename: string,
                     docgenFindFile, compilerMsgHandler)
 
 proc getOutFile2(conf: ConfigRef; filename: RelativeFile,
-                 ext: string, dir: RelativeDir; guessTarget: bool): AbsoluteFile =
-  if optWholeProject in conf.globalOptions:
-    let d = conf.docOutDir(dir)
-    createDir(d)
-    result = d / changeFileExt(filename, ext)
-  elif guessTarget:
-    let d = conf.docOutDir
+                 ext: string, guessTarget: bool): AbsoluteFile =
+  if optWholeProject in conf.globalOptions or guessTarget:
+    let d = conf.outDir
     createDir(d)
     result = d / changeFileExt(filename, ext)
   elif not conf.outFile.isEmpty:
@@ -167,6 +168,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   result.module = module
   result.conf = conf
   result.cache = cache
+  result.outDir = conf.outDir.string
   initRstGenerator(result[], (if conf.cmd != cmdRst2tex: outHtml else: outLatex),
                    conf.configVars, filename.string, {roSupportRawDirective, roSupportMarkdown},
                    docgenFindFile, compilerMsgHandler)
@@ -193,6 +195,7 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
   initStrTable result.types
   result.onTestSnippet =
     proc (gen: var RstGenerator; filename, cmd: string; status: int; content: string) =
+      if conf.docCmd == docCmdSkip: return
       inc(gen.id)
       var d = TDocumentor(gen)
       var outp: AbsoluteFile
@@ -211,16 +214,24 @@ proc newDocumentor*(filename: AbsoluteFile; cache: IdentCache; conf: ConfigRef,
       # Include the current file if we're parsing a nim file
       let importStmt = if d.isPureRst: "" else: "import \"$1\"\n" % [d.filename.replace("\\", "/")]
       writeFile(outp, importStmt & content)
-      let c = if cmd.startsWith("nim "): os.getAppFilename() & cmd.substr(3)
-              else: cmd
-      let c2 = c % quoteShell(outp)
-      rawMessage(conf, hintExecuting, c2)
-      if execShellCmd(c2) != status:
-        rawMessage(conf, errGenerated, "executing of external program failed: " & c2)
+
+      proc interpSnippetCmd(cmd: string): string =
+        # backward compatibility hacks; interpolation commands should explicitly use `$`
+        if cmd.startsWith "nim ": result = "$nim " & cmd[4..^1]
+        else: result = cmd
+        result = result.replace("$1", "$options") % [
+          "nim", os.getAppFilename().quoteShell,
+          "backend", $d.conf.backend,
+          "options", outp.quoteShell,
+        ]
+      let cmd = cmd.interpSnippetCmd
+      rawMessage(conf, hintExecuting, cmd)
+      let (output, gotten) = execCmdEx(cmd)
+      if gotten != status:
+        rawMessage(conf, errGenerated, "snippet failed: cmd: '$1' status: $2 expected: $3 output: $4" % [cmd, $gotten, $status, output])
   result.emitted = initIntSet()
-  result.destFile = getOutFile2(conf, presentationPath(conf, filename),
-                                outExt, htmldocsDir, false)
-  result.thisDir = result.destFile.splitFile.dir
+  result.destFile = getOutFile2(conf, presentationPath(conf, filename), outExt, false).string
+  result.thisDir = result.destFile.AbsoluteFile.splitFile.dir
 
 template dispA(conf: ConfigRef; dest: var Rope, xml, tex: string, args: openArray[Rope]) =
   if conf.cmd != cmdRst2tex: dest.addf(xml, args)
@@ -292,11 +303,13 @@ proc ropeFormatNamedVars(conf: ConfigRef; frmt: FormatStr,
 
 proc genComment(d: PDoc, n: PNode): string =
   result = ""
-  var dummyHasToc: bool
   if n.comment.len > 0:
-    renderRstToOut(d[], parseRst(n.comment, toFullPath(d.conf, n.info),
-                               toLinenumber(n.info), toColumn(n.info),
-                               dummyHasToc, d.options, d.conf), result)
+    let comment = n.comment
+    when false:
+      # RFC: to preseve newlines in comments, this would work:
+      comment = comment.replace("\n", "\n\n")
+    renderRstToOut(d[], parseRst(comment, toFullPath(d.conf, n.info), toLinenumber(n.info),
+                   toColumn(n.info), (var dummy: bool; dummy), d.options, d.conf), result)
 
 proc genRecCommentAux(d: PDoc, n: PNode): Rope =
   if n == nil: return nil
@@ -328,11 +341,10 @@ proc getPlainDocstring(n: PNode): string =
   ## You need to call this before genRecComment, whose side effects are removal
   ## of comments from the tree. The proc will recursively scan and return all
   ## the concatenated ``##`` comments of the node.
-  result = ""
-  if n == nil: return
-  if startsWith(n.comment, "##"):
+  if n == nil: result = ""
+  elif startsWith(n.comment, "##"):
     result = n.comment
-  if result.len < 1:
+  else:
     for i in 0..<n.safeLen:
       result = getPlainDocstring(n[i])
       if result.len > 0: return
@@ -343,8 +355,7 @@ proc belongsToPackage(conf: ConfigRef; module: PSym): bool =
 proc externalDep(d: PDoc; module: PSym): string =
   if optWholeProject in d.conf.globalOptions or d.conf.docRoot.len > 0:
     let full = AbsoluteFile toFullPath(d.conf, FileIndex module.position)
-    let tmp = getOutFile2(d.conf, presentationPath(d.conf, full), HtmlExt,
-                          htmldocsDir, sfMainModule notin module.flags)
+    let tmp = getOutFile2(d.conf, presentationPath(d.conf, full), HtmlExt, sfMainModule notin module.flags)
     result = relativeTo(tmp, d.thisDir, '/').string
   else:
     result = extractFilename toFullPath(d.conf, FileIndex module.position)
@@ -357,6 +368,7 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
   var kind = tkEof
   var tokenPos = 0
   var procTokenPos = 0
+  template escLit(): untyped = rope(esc(d.target, literal))
   while true:
     getNextTok(r, kind, literal)
     inc tokenPos
@@ -365,7 +377,7 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
       break
     of tkComment:
       dispA(d.conf, result, "<span class=\"Comment\">$1</span>", "\\spanComment{$1}",
-            [rope(esc(d.target, literal))])
+            [escLit])
     of tokKeywordLow..tokKeywordHigh:
       if kind in {tkProc, tkMethod, tkIterator, tkMacro, tkTemplate, tkFunc, tkConverter}:
         procTokenPos = tokenPos
@@ -373,53 +385,53 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
             [rope(literal)])
     of tkOpr:
       dispA(d.conf, result, "<span class=\"Operator\">$1</span>", "\\spanOperator{$1}",
-            [rope(esc(d.target, literal))])
+            [escLit])
     of tkStrLit..tkTripleStrLit:
       dispA(d.conf, result, "<span class=\"StringLit\">$1</span>",
-            "\\spanStringLit{$1}", [rope(esc(d.target, literal))])
+            "\\spanStringLit{$1}", [escLit])
     of tkCharLit:
       dispA(d.conf, result, "<span class=\"CharLit\">$1</span>", "\\spanCharLit{$1}",
-            [rope(esc(d.target, literal))])
+            [escLit])
     of tkIntLit..tkUInt64Lit:
       dispA(d.conf, result, "<span class=\"DecNumber\">$1</span>",
-            "\\spanDecNumber{$1}", [rope(esc(d.target, literal))])
+            "\\spanDecNumber{$1}", [escLit])
     of tkFloatLit..tkFloat128Lit:
       dispA(d.conf, result, "<span class=\"FloatNumber\">$1</span>",
-            "\\spanFloatNumber{$1}", [rope(esc(d.target, literal))])
+            "\\spanFloatNumber{$1}", [escLit])
     of tkSymbol:
       let s = getTokSym(r)
       # -2 because of the whitespace in between:
       if procTokenPos == tokenPos-2 and procLink != nil:
         dispA(d.conf, result, "<a href=\"#$2\"><span class=\"Identifier\">$1</span></a>",
-              "\\spanIdentifier{$1}", [rope(esc(d.target, literal)), procLink])
+              "\\spanIdentifier{$1}", [escLit, procLink])
       elif s != nil and s.kind in {skType, skVar, skLet, skConst} and
            sfExported in s.flags and s.owner != nil and
            belongsToPackage(d.conf, s.owner) and d.target == outHtml:
         let external = externalDep(d, s.owner)
         result.addf "<a href=\"$1#$2\"><span class=\"Identifier\">$3</span></a>",
           [rope changeFileExt(external, "html"), rope literal,
-           rope(esc(d.target, literal))]
+           escLit]
       else:
         dispA(d.conf, result, "<span class=\"Identifier\">$1</span>",
-              "\\spanIdentifier{$1}", [rope(esc(d.target, literal))])
+              "\\spanIdentifier{$1}", [escLit])
     of tkSpaces, tkInvalid:
       result.add(literal)
     of tkCurlyDotLe:
-      dispA(d.conf, result, "<span>" & # This span is required for the JS to work properly
+      template fun(s) = dispA(d.conf, result, s, "\\spanOther{$1}", [escLit])
+      if renderRunnableExamples in renderFlags: fun "$1"
+      else: fun: "<span>" & # This span is required for the JS to work properly
         """<span class="Other">{</span><span class="Other pragmadots">...</span><span class="Other">}</span>
 </span>
 <span class="pragmawrap">
 <span class="Other">$1</span>
-<span class="pragma">""".replace("\n", ""),  # Must remove newlines because wrapped in a <pre>
-                    "\\spanOther{$1}",
-                  [rope(esc(d.target, literal))])
+<span class="pragma">""".replace("\n", "")  # Must remove newlines because wrapped in a <pre>
     of tkCurlyDotRi:
-      dispA(d.conf, result, """
+      template fun(s) = dispA(d.conf, result, s, "\\spanOther{$1}", [escLit])
+      if renderRunnableExamples in renderFlags: fun "$1"
+      else: fun """
 </span>
 <span class="Other">$1</span>
-</span>""".replace("\n", ""),
-                    "\\spanOther{$1}",
-                  [rope(esc(d.target, literal))])
+</span>""".replace("\n", "")
     of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi,
        tkBracketDotLe, tkBracketDotRi, tkParDotLe,
        tkParDotRi, tkComma, tkSemiColon, tkColon, tkEquals, tkDot, tkDotDot,
@@ -427,51 +439,73 @@ proc nodeToHighlightedHtml(d: PDoc; n: PNode; result: var Rope; renderFlags: TRe
        tkGStrLit, tkGTripleStrLit, tkInfixOpr, tkPrefixOpr, tkPostfixOpr,
        tkBracketLeColon:
       dispA(d.conf, result, "<span class=\"Other\">$1</span>", "\\spanOther{$1}",
-            [rope(esc(d.target, literal))])
+            [escLit])
 
-proc testExample(d: PDoc; ex: PNode) =
+proc exampleOutputDir(d: PDoc): AbsoluteDir = d.conf.getNimcacheDir / RelativeDir"runnableExamples"
+
+proc writeExample(d: PDoc; ex: PNode, rdoccmd: string) =
   if d.conf.errorCounter > 0: return
-  let outputDir = d.conf.getNimcacheDir / RelativeDir"runnableExamples"
+  let outputDir = d.exampleOutputDir
   createDir(outputDir)
   inc d.exampleCounter
   let outp = outputDir / RelativeFile(extractFilename(d.filename.changeFileExt"" &
       "_examples" & $d.exampleCounter & ".nim"))
   #let nimcache = outp.changeFileExt"" & "_nimcache"
   renderModule(ex, d.filename, outp.string, conf = d.conf)
-  d.examples.add "import r\"" & outp.string & "\"\n"
+  if rdoccmd notin d.exampleGroups: d.exampleGroups[rdoccmd] = ExampleGroup(rdoccmd: rdoccmd, docCmd: d.conf.docCmd, index: d.exampleGroups.len)
+  d.exampleGroups[rdoccmd].code.add "import r\"$1\"\n" % outp.string
 
 proc runAllExamples(d: PDoc) =
-  if d.examples.len == 0: return
-  let outputDir = d.conf.getNimcacheDir / RelativeDir"runnableExamples"
-  let outp = outputDir / RelativeFile(extractFilename(d.filename.changeFileExt"" &
-      "_examples.nim"))
-  writeFile(outp, d.examples)
-  let backend = if isDefined(d.conf, "js"): "js"
-                elif isDefined(d.conf, "cpp"): "cpp"
-                elif isDefined(d.conf, "objc"): "objc"
-                else: "c"
-  if os.execShellCmd(os.getAppFilename() & " " & backend &
-                    " --warning[UnusedImport]:off" &
-                    " --path:" & quoteShell(d.conf.projectPath) &
-                    " --nimcache:" & quoteShell(outputDir) &
-                    " -r " & quoteShell(outp)) != 0:
-    quit "[Examples] failed: see " & outp.string
-  else:
-    # keep generated source file `outp` to allow inspection.
-    rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
-    removeFile(outp.changeFileExt(ExeExt))
+  # This used to be: `let backend = if isDefined(d.conf, "js"): "js"` (etc), however
+  # using `-d:js` (etc) cannot work properly, e.g. would fail with `importjs`
+  # since semantics are affected by `config.backend`, not by isDefined(d.conf, "js")
+  let outputDir = d.exampleOutputDir
+  for _, group in d.exampleGroups:
+    if group.docCmd == docCmdSkip: continue
+    let outp = outputDir / RelativeFile("$1_group$2_examples.nim" % [d.filename.splitFile.name, $group.index])
+    group.code = "# autogenerated by docgen\n# source: $1\n# rdoccmd: $2\n$3" % [d.filename, group.rdoccmd, group.code]
+    writeFile(outp, group.code)
+    # most useful semantics is that `docCmd` comes after `rdoccmd`, so that we can (temporarily) override
+    # via command line
+    let cmd = "$nim $backend -r --warning:UnusedImport:off --path:$path --nimcache:$nimcache $rdoccmd $docCmd $file" % [
+      "nim", os.getAppFilename(),
+      "backend", $d.conf.backend,
+      "path", quoteShell(d.conf.projectPath),
+      "nimcache", quoteShell(outputDir),
+      "file", quoteShell(outp),
+      "rdoccmd", group.rdoccmd,
+      "docCmd", group.docCmd,
+    ]
+    if os.execShellCmd(cmd) != 0:
+      quit "[runnableExamples] failed: generated file: '$1' group: '$2' cmd: $3" % [outp.string, $group[], cmd]
+    else:
+      # keep generated source file `outp` to allow inspection.
+      rawMessage(d.conf, hintSuccess, ["runnableExamples: " & outp.string])
+      # removeFile(outp.changeFileExt(ExeExt)) # it's in nimcache, no need to remove
 
+proc prepareExample(d: PDoc; n: PNode): tuple[rdoccmd: string, code: string] =
+  ## returns `rdoccmd` and source code for this runnableExamples
+  var rdoccmd = ""
+  if n.len < 2 or n.len > 3: globalError(d.conf, n.info, "runnableExamples invalid")
+  if n.len == 3:
+    let n1 = n[1]
+    # xxx this should be evaluated during sempass
+    if n1.kind notin nkStrKinds: globalError(d.conf, n1.info, "string litteral expected")
+    rdoccmd = n1.strVal
 
-proc prepareExamples(d: PDoc; n: PNode) =
   var docComment = newTree(nkCommentStmt)
   let loc = d.conf.toFileLineCol(n.info)
-  docComment.comment = "autogenerated by docgen from " & loc
+
+  docComment.comment = "autogenerated by docgen\nloc: $1\nrdoccmd: $2" % [loc, rdoccmd]
   var runnableExamples = newTree(nkStmtList,
       docComment,
       newTree(nkImportStmt, newStrNode(nkStrLit, d.filename)))
   runnableExamples.info = n.info
+  let ret = extractRunnableExamplesSource(d.conf, n)
   for a in n.lastSon: runnableExamples.add a
-  testExample(d, runnableExamples)
+  # we could also use `ret` instead here, to keep sources verbatim
+  writeExample(d, runnableExamples, rdoccmd)
+  result = (rdoccmd, ret)
   when false:
     proc extractImports(n: PNode; result: PNode) =
       if n.kind in {nkImportStmt, nkImportExceptStmt, nkFromStmt}:
@@ -485,36 +519,101 @@ proc prepareExamples(d: PDoc; n: PNode) =
     for imp in imports: runnableExamples.add imp
     runnableExamples.add newTree(nkBlockStmt, newNode(nkEmpty), copyTree savedLastSon)
 
-proc getAllRunnableExamplesRec(d: PDoc; n, orig: PNode; dest: var Rope) =
-  if n.info.fileIndex != orig.info.fileIndex: return
+type RunnableState = enum
+  rsStart
+  rsComment
+  rsRunnable
+  rsDone
+
+proc getAllRunnableExamplesImpl(d: PDoc; n: PNode, dest: var Rope, state: RunnableState): RunnableState =
+  ##[
+  Simple state machine to tell whether we render runnableExamples and doc comments.
+  This is to ensure that we can interleave runnableExamples and doc comments freely;
+  the logic is easy to change but currently a doc comment following another doc comment
+  will not render, to avoid rendering in following case:
+
+  proc fn* =
+    runnableExamples: discard
+    ## d1
+    runnableExamples: discard
+    ## d2
+
+    ## internal explanation  # <- this one should be out; it's part of rest of function body and would likey not make sense in doc comment
+    discard # some code
+  ]##
+
   case n.kind
+  of nkCommentStmt:
+    if state in {rsStart, rsRunnable}:
+      dest.add genRecComment(d, n)
+      return rsComment
   of nkCallKinds:
     if isRunnableExamples(n[0]) and
-        n.len >= 2 and n.lastSon.kind == nkStmtList:
-      prepareExamples(d, n)
+        n.len >= 2 and n.lastSon.kind == nkStmtList and state in {rsStart, rsComment, rsRunnable}:
+      let (rdoccmd, code) = prepareExample(d, n)
+      var msg = "Example:"
+      if rdoccmd.len > 0: msg.add " cmd: " & rdoccmd
       dispA(d.conf, dest, "\n<p><strong class=\"examples_text\">$1</strong></p>\n",
-          "\n\\textbf{$1}\n", [rope"Examples:"])
+          "\n\\textbf{$1}\n", [msg.rope])
       inc d.listingCounter
       let id = $d.listingCounter
       dest.add(d.config.getOrDefault"doc.listing_start" % [id, "langNim"])
-      # this is a rather hacky way to get rid of the initial indentation
-      # that the renderer currently produces:
-      var i = 0
-      var body = n.lastSon
-      if body.len == 1 and body.kind == nkStmtList and
-          body.lastSon.kind == nkStmtList:
-        body = body.lastSon
-      for b in body:
-        if i > 0: dest.add "\n"
-        inc i
-        nodeToHighlightedHtml(d, b, dest, {}, nil)
+      var dest2 = ""
+      renderNimCode(dest2, code, isLatex = d.conf.cmd == cmdRst2tex)
+      dest.add dest2
       dest.add(d.config.getOrDefault"doc.listing_end" % id)
+      return rsRunnable
   else: discard
-  for i in 0..<n.safeLen:
-    getAllRunnableExamplesRec(d, n[i], orig, dest)
+  return rsDone
+    # change this to `rsStart` if you want to keep generating doc comments
+    # and runnableExamples that occur after some code in routine
 
-proc getAllRunnableExamples(d: PDoc; n: PNode; dest: var Rope) =
-  getAllRunnableExamplesRec(d, n, n, dest)
+proc getRoutineBody(n: PNode): PNode =
+  ##[
+  nim transforms these quite differently:
+
+  proc someType*(): int =
+    ## foo
+    result = 3
+=>
+  result =
+    ## foo
+    3;
+
+  proc someType*(): int =
+    ## foo
+    3
+=>
+  ## foo
+  result = 3;
+
+  so we normalize the results to get to the statement list containing the
+  (0 or more) doc comments and runnableExamples.
+  ]##
+  result = n[bodyPos]
+
+  # This won't be transformed: result.id = 10. Namely result[0].kind != nkSym.
+  if result.kind == nkAsgn and result[0].kind == nkSym and
+                               n.len > bodyPos+1 and n[bodyPos+1].kind == nkSym:
+    doAssert result.len == 2
+    result = result[1]
+
+proc getAllRunnableExamples(d: PDoc, n: PNode, dest: var Rope) =
+  var n = n
+  var state = rsStart
+  template fn(n2) =
+    state = getAllRunnableExamplesImpl(d, n2, dest, state)
+  dest.add genComment(d, n).rope
+  case n.kind
+  of routineDefs:
+    n = n.getRoutineBody
+    case n.kind
+    of nkCommentStmt, nkCallKinds: fn(n)
+    else:
+      for i in 0..<n.safeLen:
+        fn(n[i])
+        if state == rsDone: return
+  else: fn(n)
 
 proc isVisible(d: PDoc; n: PNode): bool =
   result = false
@@ -618,7 +717,7 @@ proc complexName(k: TSymKind, n: PNode, baseName: string): string =
   of skTemplate: result.add(".t")
   of skConverter: result.add(".c")
   else: discard
-  if n.len > paramsPos and n[paramsPos].kind == nkFormalParams:
+  if n.safeLen > paramsPos and n[paramsPos].kind == nkFormalParams:
     let params = renderParamTypes(n[paramsPos])
     if params.len > 0:
       result.add(defaultParamSeparator)
@@ -689,8 +788,12 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
   var result: Rope = nil
   var literal, plainName = ""
   var kind = tkEof
-  var comm = genRecComment(d, n)  # call this here for the side-effect!
-  getAllRunnableExamples(d, n, comm)
+  var comm: Rope = nil
+  if n.kind in routineDefs:
+    getAllRunnableExamples(d, n, comm)
+  else:
+    comm.add genRecComment(d, n)
+
   var r: TSrcGen
   # Obtain the plain rendered string for hyperlink titles.
   initTokRender(r, n, {renderNoBody, renderNoComments, renderDocComments,
@@ -748,7 +851,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
       symbolOrIdRope, plainSymbolEncRope, symbolOrIdEncRope, seeSrcRope,
       deprecationMsgRope]))
 
-  let external = d.destFile.relativeTo(d.conf.outDir, '/').changeFileExt(HtmlExt).string
+  let external = d.destFile.AbsoluteFile.relativeTo(d.conf.outDir, '/').changeFileExt(HtmlExt).string
 
   var attype: Rope
   if k in routineKinds and nameNode.kind == nkSym:
@@ -765,11 +868,13 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind, docFlags: DocFlags) =
         xmltree.escape(getPlainDocstring(e).docstringSummary))
 
   d.toc[k].add(ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.item.toc"),
-    ["name", "header", "desc", "itemID", "header_plain", "itemSym",
-      "itemSymOrID", "itemSymEnc", "itemSymOrIDEnc", "attype"],
-    [rope(getName(d, nameNode, d.splitAfter)), result, comm,
-      itemIDRope, plainNameRope, plainSymbolRope, symbolOrIdRope,
-      plainSymbolEncRope, symbolOrIdEncRope, attype]))
+    ["name", "header_plain", "itemSymOrIDEnc"],
+    [nameRope, plainNameRope, symbolOrIdEncRope]))
+
+  d.tocTable[k].mgetOrPut(cleanPlainSymbol, nil).add(ropeFormatNamedVars(
+    d.conf, getConfigVar(d.conf, "doc.item.tocTable"),
+    ["name", "header_plain", "itemSymOrID", "itemSymOrIDEnc"],
+    [nameRope, plainNameRope, rope(symbolOrId.replace(",", ",<wbr>")), symbolOrIdEncRope]))
 
   # Ironically for types the complexSymbol is *cleaner* than the plainName
   # because it doesn't include object fields or documentation comments. So we
@@ -846,7 +951,7 @@ proc traceDeps(d: PDoc, it: PNode) =
     if d.section[k] != nil: d.section[k].add(", ")
     dispA(d.conf, d.section[k],
           "<a class=\"reference external\" href=\"$2\">$1</a>",
-          "$1", [rope esc(d.target, changeFileExt(external, "")),
+          "$1", [rope esc(d.target, external.prettyLink),
           rope changeFileExt(external, "html")])
 
 proc exportSym(d: PDoc; s: PSym) =
@@ -856,18 +961,22 @@ proc exportSym(d: PDoc; s: PSym) =
     if d.section[k] != nil: d.section[k].add(", ")
     dispA(d.conf, d.section[k],
           "<a class=\"reference external\" href=\"$2\">$1</a>",
-          "$1", [rope esc(d.target, changeFileExt(external, "")),
+          "$1", [rope esc(d.target, external.prettyLink),
           rope changeFileExt(external, "html")])
   elif s.kind != skModule and s.owner != nil:
     let module = originatingModule(s)
     if belongsToPackage(d.conf, module):
-      let external = externalDep(d, module)
+      let
+        complexSymbol = complexName(s.kind, s.ast, s.name.s)
+        symbolOrIdRope = rope(d.newUniquePlainSymbol(complexSymbol))
+        external = externalDep(d, module)
       if d.section[k] != nil: d.section[k].add(", ")
       # XXX proper anchor generation here
       dispA(d.conf, d.section[k],
-            "<a href=\"$2#$1\"><span class=\"Identifier\">$1</span></a>",
+            "<a href=\"$2#$3\"><span class=\"Identifier\">$1</span></a>",
             "$1", [rope esc(d.target, s.name.s),
-            rope changeFileExt(external, "html")])
+            rope changeFileExt(external, "html"),
+            symbolOrIdRope])
 
 proc documentNewEffect(cache: IdentCache; n: PNode): PNode =
   let s = n[namePos].sym
@@ -892,8 +1001,8 @@ proc documentEffect(cache: IdentCache; n, x: PNode, effectType: TSpecialWord, id
       # set the type so that the following analysis doesn't screw up:
       effects[i].typ = real[i].typ
 
-    result = newNode(nkExprColonExpr, n.info, @[
-      newIdentNode(getIdent(cache, specialWords[effectType]), n.info), effects])
+    result = newTreeI(nkExprColonExpr, n.info,
+      newIdentNode(getIdent(cache, $effectType), n.info), effects)
 
 proc documentWriteEffect(cache: IdentCache; n: PNode; flag: TSymFlag; pragmaName: string): PNode =
   let s = n[namePos].sym
@@ -905,8 +1014,8 @@ proc documentWriteEffect(cache: IdentCache; n: PNode; flag: TSymFlag; pragmaName
       effects.add params[i]
 
   if effects.len > 0:
-    result = newNode(nkExprColonExpr, n.info, @[
-      newIdentNode(getIdent(cache, pragmaName), n.info), effects])
+    result = newTreeI(nkExprColonExpr, n.info,
+      newIdentNode(getIdent(cache, pragmaName), n.info), effects)
 
 proc documentRaises*(cache: IdentCache; n: PNode) =
   if n[namePos].kind != nkSym: return
@@ -970,7 +1079,7 @@ proc generateDoc*(d: PDoc, n, orig: PNode, docFlags: DocFlags = kDefault) =
       if it.kind == nkSym:
         if d.module != nil and d.module == it.sym.owner:
           generateDoc(d, it.sym.ast, orig, kForceExport)
-        else:
+        elif it.sym.ast != nil:
           exportSym(d, it.sym)
   of nkExportExceptStmt: discard "transformed into nkExportStmt by semExportExcept"
   of nkFromStmt, nkImportExceptStmt: traceDeps(d, n[0])
@@ -1067,7 +1176,7 @@ proc generateTags*(d: PDoc, n: PNode, r: var Rope) =
       generateTags(d, lastSon(n[0]), r)
   else: discard
 
-proc genSection(d: PDoc, kind: TSymKind) =
+proc genSection(d: PDoc, kind: TSymKind, groupedToc = false) =
   const sectionNames: array[skModule..skField, string] = [
     "Imports", "Types", "Vars", "Lets", "Consts", "Vars", "Procs", "Funcs",
     "Methods", "Iterators", "Converters", "Macros", "Templates", "Exports"
@@ -1077,17 +1186,23 @@ proc genSection(d: PDoc, kind: TSymKind) =
   d.section[kind] = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.section"), [
       "sectionid", "sectionTitle", "sectionTitleID", "content"], [
       ord(kind).rope, title, rope(ord(kind) + 50), d.section[kind]])
+
+  var tocSource = d.toc
+  if groupedToc:
+    for p in d.tocTable[kind].keys:
+      d.toc2[kind].add ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.section.toc2"), [
+          "sectionid", "sectionTitle", "sectionTitleID", "content", "plainName"], [
+          ord(kind).rope, title, rope(ord(kind) + 50), d.tocTable[kind][p], p.rope])
+    tocSource = d.toc2
+
   d.toc[kind] = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.section.toc"), [
       "sectionid", "sectionTitle", "sectionTitleID", "content"], [
-      ord(kind).rope, title, rope(ord(kind) + 50), d.toc[kind]])
+      ord(kind).rope, title, rope(ord(kind) + 50), tocSource[kind]])
 
-const nimdocOutCss = "nimdoc.out.css"
-  # `out` to make it easier to use with gitignore in user's repos
+proc relLink(outDir: AbsoluteDir, destFile: AbsoluteFile, linkto: RelativeFile): Rope =
+  rope($relativeTo(outDir / linkto, destFile.splitFile().dir, '/'))
 
-proc cssHref(outDir: AbsoluteDir, destFile: AbsoluteFile): Rope =
-  rope($relativeTo(outDir / nimdocOutCss.RelativeFile, destFile.splitFile().dir, '/'))
-
-proc genOutFile(d: PDoc): Rope =
+proc genOutFile(d: PDoc, groupedToc = false): Rope =
   var
     code, content: Rope
     title = ""
@@ -1095,36 +1210,42 @@ proc genOutFile(d: PDoc): Rope =
   var tmp = ""
   renderTocEntries(d[], j, 1, tmp)
   var toc = tmp.rope
-  for i in low(TSymKind)..high(TSymKind):
-    genSection(d, i)
+  for i in TSymKind:
+    var shouldSort = i in {skProc, skFunc} and groupedToc
+    genSection(d, i, shouldSort)
     toc.add(d.toc[i])
   if toc != nil:
     toc = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.toc"), ["content"], [toc])
-  for i in low(TSymKind)..high(TSymKind): code.add(d.section[i])
+  for i in TSymKind: code.add(d.section[i])
 
   # Extract the title. Non API modules generate an entry in the index table.
   if d.meta[metaTitle].len != 0:
     title = d.meta[metaTitle]
-    let external = presentationPath(d.conf, AbsoluteFile d.filename).changeFileExt(HtmlExt).string.nativeToUnix
+    let external = presentationPath(d.conf, AbsoluteFile d.filename).changeFileExt(HtmlExt).string.nativeToUnixPath
     setIndexTerm(d[], external, "", title)
   else:
     # Modules get an automatic title for the HTML, but no entry in the index.
     # better than `extractFilename(changeFileExt(d.filename, ""))` as it disambiguates dups
     title = $presentationPath(d.conf, AbsoluteFile d.filename, isTitle = true).changeFileExt("")
 
-  let bodyname = if d.hasToc and not d.isPureRst: "doc.body_toc_group"
+  var groupsection = getConfigVar(d.conf, "doc.body_toc_groupsection")
+  let bodyname = if d.hasToc and not d.isPureRst:
+                   groupsection.setLen 0
+                   "doc.body_toc_group"
                  elif d.hasToc: "doc.body_toc"
                  else: "doc.body_no_toc"
   content = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, bodyname), ["title",
-      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg"],
+      "tableofcontents", "moduledesc", "date", "time", "content", "deprecationMsg", "theindexhref", "body_toc_groupsection"],
       [title.rope, toc, d.modDesc, rope(getDateStr()),
-      rope(getClockStr()), code, d.modDeprecationMsg])
+      rope(getClockStr()), code, d.modDeprecationMsg, relLink(d.conf.outDir, d.destFile.AbsoluteFile, theindexFname.RelativeFile), groupsection.rope])
   if optCompileOnly notin d.conf.globalOptions:
     # XXX what is this hack doing here? 'optCompileOnly' means raw output!?
     code = ropeFormatNamedVars(d.conf, getConfigVar(d.conf, "doc.file"), [
-        "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
+        "nimdoccss", "dochackjs",  "title", "tableofcontents", "moduledesc", "date", "time",
         "content", "author", "version", "analytics", "deprecationMsg"],
-        [cssHref(d.conf.outDir, d.destFile), title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
+        [relLink(d.conf.outDir, d.destFile.AbsoluteFile, nimdocOutCss.RelativeFile),
+        relLink(d.conf.outDir, d.destFile.AbsoluteFile, docHackJsFname.RelativeFile),
+        title.rope, toc, d.modDesc, rope(getDateStr()), rope(getClockStr()),
         content, d.meta[metaAuthor].rope, d.meta[metaVersion].rope, d.analytics.rope, d.modDeprecationMsg])
   else:
     code = content
@@ -1132,35 +1253,39 @@ proc genOutFile(d: PDoc): Rope =
 
 proc generateIndex*(d: PDoc) =
   if optGenIndex in d.conf.globalOptions:
-    let dir = d.conf.docOutDir(htmldocsDir)
+    let dir = d.conf.outDir
     createDir(dir)
     let dest = dir / changeFileExt(presentationPath(d.conf, AbsoluteFile d.filename), IndexExt)
     writeIndexFile(d[], dest.string)
 
 proc updateOutfile(d: PDoc, outfile: AbsoluteFile) =
-  if d.module == nil or sfMainModule in d.module.flags: # nil for eg for commandRst2Html
-    if d.conf.outDir.isEmpty: d.conf.outDir = d.conf.docOutDir
-    if d.conf.outFile.isEmpty: d.conf.outFile = outfile.relativeTo(d.conf.outDir)
+  if d.module == nil or sfMainModule in d.module.flags: # nil for e.g. for commandRst2Html
+    if d.conf.outFile.isEmpty:
+      d.conf.outFile = outfile.relativeTo(d.conf.outDir)
+      if isAbsolute(d.conf.outFile.string):
+        d.conf.outFile = splitPath(d.conf.outFile.string)[1].RelativeFile
 
-proc writeOutput*(d: PDoc, useWarning = false) =
+proc writeOutput*(d: PDoc, useWarning = false, groupedToc = false) =
   runAllExamples(d)
-  var content = genOutFile(d)
+  var content = genOutFile(d, groupedToc)
   if optStdout in d.conf.globalOptions:
     writeRope(stdout, content)
   else:
-    template outfile: untyped = d.destFile
-    #let outfile = getOutFile2(d.conf, shortenDir(d.conf, filename), outExt, htmldocsDir)
+    template outfile: untyped = d.destFile.AbsoluteFile
+    #let outfile = getOutFile2(d.conf, shortenDir(d.conf, filename), outExt)
     let dir = outfile.splitFile.dir
     createDir(dir)
     updateOutfile(d, outfile)
     if not writeRope(content, outfile):
       rawMessage(d.conf, if useWarning: warnCannotOpenFile else: errCannotOpenFile,
         outfile.string)
-    elif not d.wroteCss:
-      let cssSource = $d.conf.getPrefixDir() / "doc" / "nimdoc.css"
-      let cssDest = $dir / nimdocOutCss
-      copyFile(cssSource, cssDest)
-      d.wroteCss = true
+    elif not d.wroteSupportFiles: # nimdoc.css + dochack.js
+      let nimr = $d.conf.getPrefixDir()
+      copyFile(docCss.interp(nimr = nimr), $d.conf.outDir / nimdocOutCss)
+      if optGenIndex in d.conf.globalOptions:
+        let docHackJs2 = getDocHacksJs(nimr, nim = getAppFilename())
+        copyFile(docHackJs2, $d.conf.outDir / docHackJs2.lastPathPart)
+      d.wroteSupportFiles = true
 
 proc writeOutputJson*(d: PDoc, useWarning = false) =
   runAllExamples(d)
@@ -1175,18 +1300,20 @@ proc writeOutputJson*(d: PDoc, useWarning = false) =
     write(stdout, $content)
   else:
     var f: File
-    if open(f, d.destFile.string, fmWrite):
+    if open(f, d.destFile, fmWrite):
       write(f, $content)
       close(f)
-      updateOutfile(d, d.destFile)
+      updateOutfile(d, d.destFile.AbsoluteFile)
     else:
       localError(d.conf, newLineInfo(d.conf, AbsoluteFile d.filename, -1, -1),
-                 warnUser, "unable to open file \"" & d.destFile.string &
+                 warnUser, "unable to open file \"" & d.destFile &
                  "\" for writing")
 
 proc handleDocOutputOptions*(conf: ConfigRef) =
   if optWholeProject in conf.globalOptions:
     # Backward compatibility with previous versions
+    # xxx this is buggy when user provides `nim doc --project -o:sub/bar.html main`,
+    # it'd write to `sub/bar.html/main.html`
     conf.outDir = AbsoluteDir(conf.outDir / conf.outFile)
 
 proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
@@ -1261,18 +1388,20 @@ proc commandTags*(cache: IdentCache, conf: ConfigRef) =
     if not writeRope(content, filename):
       rawMessage(conf, errCannotOpenFile, filename.string)
 
-proc commandBuildIndex*(cache: IdentCache, conf: ConfigRef) =
-  var content = mergeIndexes(conf.projectFull.string).rope
+proc commandBuildIndex*(conf: ConfigRef, dir: string, outFile = RelativeFile"") =
+  var content = mergeIndexes(dir).rope
 
-  var outFile = RelativeFile"theindex"
-  if conf.outFile != RelativeFile"":
-    outFile = conf.outFile
+  var outFile = outFile
+  if outFile.isEmpty: outFile = theindexFname.RelativeFile.changeFileExt("")
   let filename = getOutFile(conf, outFile, HtmlExt)
 
   let code = ropeFormatNamedVars(conf, getConfigVar(conf, "doc.file"), [
-      "nimdoccss", "title", "tableofcontents", "moduledesc", "date", "time",
+      "nimdoccss", "dochackjs",
+      "title", "tableofcontents", "moduledesc", "date", "time",
       "content", "author", "version", "analytics"],
-      [cssHref(conf.outDir, filename), rope"Index", nil, nil, rope(getDateStr()),
+      [relLink(conf.outDir, filename, nimdocOutCss.RelativeFile),
+      relLink(conf.outDir, filename, docHackJsFname.RelativeFile),
+      rope"Index", nil, nil, rope(getDateStr()),
       rope(getClockStr()), content, nil, nil, nil])
   # no analytics because context is not available
 

compiler/docgen2.nim

@@ -29,6 +29,7 @@ proc shouldProcess(g: PGen): bool =
 template closeImpl(body: untyped) {.dirty.} =
   var g = PGen(p)
   let useWarning = sfMainModule notin g.module.flags
+  let groupedToc = true
   if shouldProcess(g):
     body
     try:
@@ -38,7 +39,7 @@ template closeImpl(body: untyped) {.dirty.} =
 
 proc close(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
-    writeOutput(g.doc, useWarning)
+    writeOutput(g.doc, useWarning, groupedToc)
 
 proc closeJson(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:

compiler/evalffi.nim

@@ -17,6 +17,8 @@ when defined(windows):
   const libcDll = "msvcrt.dll"
 elif defined(linux):
   const libcDll = "libc.so(.6|.5|)"
+elif defined(openbsd):
+  const libcDll = "/usr/lib/libc.so(.95.1|)"
 elif defined(bsd):
   const libcDll = "/lib/libc.so.7"
 elif defined(osx):
@@ -102,7 +104,7 @@ proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type =
 
 proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI =
   case cc
-  of ccDefault: result = DEFAULT_ABI
+  of ccNimCall: result = DEFAULT_ABI
   of ccStdCall: result = when defined(windows) and defined(x86): STDCALL else: DEFAULT_ABI
   of ccCDecl: result = DEFAULT_ABI
   else:

compiler/evaltempl.nim

@@ -21,6 +21,7 @@ type
                       # new symbol
     config: ConfigRef
     ic: IdentCache
+    instID: int
 
 proc copyNode(ctx: TemplCtx, a, b: PNode): PNode =
   result = copyNode(a)
@@ -53,8 +54,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
           #if x.kind == skParam and x.owner.kind == skModule:
           #  internalAssert c.config, false
           idTablePut(c.mapping, s, x)
-        if sfGenSym in s.flags and optNimV019 notin c.config.globalOptions:
-          result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $x.id),
+        if sfGenSym in s.flags:
+          result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $c.instID),
             if c.instLines: actual.info else: templ.info)
         else:
           result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
@@ -166,7 +167,7 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
 
 proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
                    conf: ConfigRef;
-                   ic: IdentCache;
+                   ic: IdentCache; instID: ref int;
                    fromHlo=false): PNode =
   inc(conf.evalTemplateCounter)
   if conf.evalTemplateCounter > evalTemplateLimit:
@@ -181,6 +182,7 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   ctx.config = conf
   ctx.ic = ic
   initIdTable(ctx.mapping)
+  ctx.instID = instID[]
 
   let body = tmpl.getBody
   #echo "instantion of ", renderTree(body, {renderIds})
@@ -203,3 +205,5 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   #if ctx.debugActive:
   #  echo "instantion of ", renderTree(result, {renderIds})
   dec(conf.evalTemplateCounter)
+  # The instID must be unique for every template instantiation, so we increment it here
+  inc instID[]

compiler/extccomp.nim

@@ -143,16 +143,16 @@ compiler vcc:
   result = (
     name: "vcc",
     objExt: "obj",
-    optSpeed: " /Ogityb2 /G7 ",
-    optSize: " /O1 /G7 ",
+    optSpeed: " /Ogityb2 ",
+    optSize: " /O1 ",
     compilerExe: "cl",
     cppCompiler: "cl",
-    compileTmpl: "/c$vccplatform $options $include /Fo$objfile $file",
-    buildGui: " /link /SUBSYSTEM:WINDOWS ",
+    compileTmpl: "/c$vccplatform $options $include /nologo /Fo$objfile $file",
+    buildGui: " /SUBSYSTEM:WINDOWS user32.lib ",
     buildDll: " /LD",
     buildLib: "lib /OUT:$libfile $objfiles",
     linkerExe: "cl",
-    linkTmpl: "$builddll$vccplatform /Fe$exefile $objfiles $buildgui $options",
+    linkTmpl: "$builddll$vccplatform /Fe$exefile $objfiles $buildgui /nologo $options",
     includeCmd: " /I",
     linkDirCmd: " /LIBPATH:",
     linkLibCmd: " $1.lib",
@@ -170,6 +170,7 @@ compiler clangcl:
   result.compilerExe = "clang-cl"
   result.cppCompiler = "clang-cl"
   result.linkerExe = "clang-cl"
+  result.linkTmpl = "-fuse-ld=lld " & result.linkTmpl
 
 # Intel C/C++ Compiler
 compiler icl:
@@ -185,32 +186,6 @@ compiler icc:
   result.compilerExe = "icc"
   result.linkerExe = "icc"
 
-# Local C Compiler
-compiler lcc:
-  result = (
-    name: "lcc",
-    objExt: "obj",
-    optSpeed: " -O -p6 ",
-    optSize: " -O -p6 ",
-    compilerExe: "lcc",
-    cppCompiler: "",
-    compileTmpl: "$options $include -Fo$objfile $file",
-    buildGui: " -subsystem windows",
-    buildDll: " -dll",
-    buildLib: "", # XXX: not supported yet
-    linkerExe: "lcclnk",
-    linkTmpl: "$options $buildgui $builddll -O $exefile $objfiles",
-    includeCmd: " -I",
-    linkDirCmd: "", # XXX: not supported yet
-    linkLibCmd: "", # XXX: not supported yet
-    debug: " -g5 ",
-    pic: "",
-    asmStmtFrmt: "_asm{$n$1$n}$n",
-    structStmtFmt: "$1 $2",
-    produceAsm: "",
-    cppXsupport: "",
-    props: {})
-
 # Borland C Compiler
 compiler bcc:
   result = (
@@ -238,58 +213,6 @@ compiler bcc:
     props: {hasSwitchRange, hasComputedGoto, hasCpp, hasGcGuard,
             hasAttribute})
 
-# Digital Mars C Compiler
-compiler dmc:
-  result = (
-    name: "dmc",
-    objExt: "obj",
-    optSpeed: " -ff -o -6 ",
-    optSize: " -ff -o -6 ",
-    compilerExe: "dmc",
-    cppCompiler: "",
-    compileTmpl: "-c $options $include -o$objfile $file",
-    buildGui: " -L/exet:nt/su:windows",
-    buildDll: " -WD",
-    buildLib: "", # XXX: not supported yet
-    linkerExe: "dmc",
-    linkTmpl: "$options $buildgui $builddll -o$exefile $objfiles",
-    includeCmd: " -I",
-    linkDirCmd: "", # XXX: not supported yet
-    linkLibCmd: "", # XXX: not supported yet
-    debug: " -g ",
-    pic: "",
-    asmStmtFrmt: "__asm{$n$1$n}$n",
-    structStmtFmt: "$3$n$1 $2",
-    produceAsm: "",
-    cppXsupport: "",
-    props: {hasCpp})
-
-# Watcom C Compiler
-compiler wcc:
-  result = (
-    name: "wcc",
-    objExt: "obj",
-    optSpeed: " -ox -on -6 -d0 -fp6 -zW ",
-    optSize: "",
-    compilerExe: "wcl386",
-    cppCompiler: "",
-    compileTmpl: "-c $options $include -fo=$objfile $file",
-    buildGui: " -bw",
-    buildDll: " -bd",
-    buildLib: "", # XXX: not supported yet
-    linkerExe: "wcl386",
-    linkTmpl: "$options $buildgui $builddll -fe=$exefile $objfiles ",
-    includeCmd: " -i=",
-    linkDirCmd: "", # XXX: not supported yet
-    linkLibCmd: "", # XXX: not supported yet
-    debug: " -d2 ",
-    pic: "",
-    asmStmtFrmt: "__asm{$n$1$n}$n",
-    structStmtFmt: "$1 $2",
-    produceAsm: "",
-    cppXsupport: "",
-    props: {hasCpp})
-
 # Tiny C Compiler
 compiler tcc:
   result = (
@@ -316,43 +239,16 @@ compiler tcc:
     cppXsupport: "",
     props: {hasSwitchRange, hasComputedGoto, hasGnuAsm})
 
-# Pelles C Compiler
-compiler pcc:
-  # Pelles C
-  result = (
-    name: "pcc",
-    objExt: "obj",
-    optSpeed: " -Ox ",
-    optSize: " -Os ",
-    compilerExe: "cc",
-    cppCompiler: "",
-    compileTmpl: "-c $options $include -Fo$objfile $file",
-    buildGui: " -SUBSYSTEM:WINDOWS",
-    buildDll: " -DLL",
-    buildLib: "", # XXX: not supported yet
-    linkerExe: "cc",
-    linkTmpl: "$options $buildgui $builddll -OUT:$exefile $objfiles",
-    includeCmd: " -I",
-    linkDirCmd: "", # XXX: not supported yet
-    linkLibCmd: "", # XXX: not supported yet
-    debug: " -Zi ",
-    pic: "",
-    asmStmtFrmt: "__asm{$n$1$n}$n",
-    structStmtFmt: "$1 $2",
-    produceAsm: "",
-    cppXsupport: "",
-    props: {})
-
 # Your C Compiler
-compiler ucc:
+compiler envcc:
   result = (
-    name: "ucc",
+    name: "env",
     objExt: "o",
     optSpeed: " -O3 ",
     optSize: " -O1 ",
-    compilerExe: "cc",
+    compilerExe: "",
     cppCompiler: "",
-    compileTmpl: "-c $options $include -o $objfile $file",
+    compileTmpl: "-c $ccenvflags $options $include -o $objfile $file",
     buildGui: "",
     buildDll: " -shared ",
     buildLib: "", # XXX: not supported yet
@@ -367,7 +263,7 @@ compiler ucc:
     structStmtFmt: "$1 $2",
     produceAsm: "",
     cppXsupport: "",
-    props: {})
+    props: {hasGnuAsm})
 
 const
   CC*: array[succ(low(TSystemCC))..high(TSystemCC), TInfoCC] = [
@@ -375,23 +271,16 @@ const
     nintendoSwitchGCC(),
     llvmGcc(),
     clang(),
-    lcc(),
     bcc(),
-    dmc(),
-    wcc(),
     vcc(),
     tcc(),
-    pcc(),
-    ucc(),
+    envcc(),
     icl(),
     icc(),
     clangcl()]
 
   hExt* = ".h"
 
-proc libNameTmpl(conf: ConfigRef): string {.inline.} =
-  result = if conf.target.targetOS == osWindows: "$1.lib" else: "lib$1.a"
-
 proc nameToCC*(name: string): TSystemCC =
   ## Returns the kind of compiler referred to by `name`, or ccNone
   ## if the name doesn't refer to any known compiler.
@@ -414,15 +303,14 @@ proc isVSCompatible*(conf: ConfigRef): bool =
 proc getConfigVar(conf: ConfigRef; c: TSystemCC, suffix: string): string =
   # use ``cpu.os.cc`` for cross compilation, unless ``--compileOnly`` is given
   # for niminst support
-  let fullSuffix =
-    if conf.cmd == cmdCompileToCpp:
-      ".cpp" & suffix
-    elif conf.cmd == cmdCompileToOC:
-      ".objc" & suffix
-    elif conf.cmd == cmdCompileToJS:
-      ".js" & suffix
-    else:
-      suffix
+  var fullSuffix = suffix
+  case conf.backend
+  of backendCpp, backendJs, backendObjc: fullSuffix = "." & $conf.backend & suffix
+  of backendC: discard
+  of backendInvalid:
+    # during parsing of cfg files; we don't know the backend yet, no point in
+    # guessing wrong thing
+    return ""
 
   if (conf.target.hostOS != conf.target.targetOS or conf.target.hostCPU != conf.target.targetCPU) and
       optCompileOnly notin conf.globalOptions:
@@ -443,7 +331,7 @@ proc setCC*(conf: ConfigRef; ccname: string; info: TLineInfo) =
   conf.compileOptions = getConfigVar(conf, conf.cCompiler, ".options.always")
   conf.linkOptions = ""
   conf.cCompilerPath = getConfigVar(conf, conf.cCompiler, ".path")
-  for i in low(CC)..high(CC): undefSymbol(conf.symbols, CC[i].name)
+  for c in CC: undefSymbol(conf.symbols, c.name)
   defineSymbol(conf.symbols, CC[conf.cCompiler].name)
 
 proc addOpt(dest: var string, src: string) =
@@ -465,7 +353,7 @@ proc addCompileOptionCmd*(conf: ConfigRef; option: string) =
 
 proc initVars*(conf: ConfigRef) =
   # we need to define the symbol here, because ``CC`` may have never been set!
-  for i in low(CC)..high(CC): undefSymbol(conf.symbols, CC[i].name)
+  for c in CC: undefSymbol(conf.symbols, c.name)
   defineSymbol(conf.symbols, CC[conf.cCompiler].name)
   addCompileOption(conf, getConfigVar(conf, conf.cCompiler, ".options.always"))
   #addLinkOption(getConfigVar(cCompiler, ".options.linker"))
@@ -541,7 +429,6 @@ proc noAbsolutePaths(conf: ConfigRef): bool {.inline.} =
 
 proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string =
   result = conf.compileOptions
-  addOpt(result, conf.cfileSpecificOptions.getOrDefault(fullNimFile))
 
   for option in conf.compileOptionsCmd:
     if strutils.find(result, option, 0) < 0:
@@ -562,6 +449,8 @@ proc cFileSpecificOptions(conf: ConfigRef; nimname, fullNimFile: string): string
   let key = nimname & ".always"
   if existsConfigVar(conf, key): addOpt(result, getConfigVar(conf, key))
 
+  addOpt(result, conf.cfileSpecificOptions.getOrDefault(fullNimFile))
+
 proc getCompileOptions(conf: ConfigRef): string =
   result = cFileSpecificOptions(conf, "__dummy__", "__dummy__")
 
@@ -589,21 +478,61 @@ proc needsExeExt(conf: ConfigRef): bool {.inline.} =
            (conf.target.hostOS == osWindows)
 
 proc useCpp(conf: ConfigRef; cfile: AbsoluteFile): bool =
-  conf.cmd == cmdCompileToCpp and not cfile.string.endsWith(".c")
+  conf.backend == backendCpp and not cfile.string.endsWith(".c")
+
+proc envFlags(conf: ConfigRef): string =
+  result = if conf.backend == backendCpp:
+            getEnv("CXXFLAGS")
+          else:
+            getEnv("CFLAGS")
 
 proc getCompilerExe(conf: ConfigRef; compiler: TSystemCC; cfile: AbsoluteFile): string =
-  result = if useCpp(conf, cfile):
-             CC[compiler].cppCompiler
-           else:
-             CC[compiler].compilerExe
+  if compiler == ccEnv:
+    result = if useCpp(conf, cfile):
+              getEnv("CXX")
+            else:
+              getEnv("CC")
+  else:
+    result = if useCpp(conf, cfile):
+              CC[compiler].cppCompiler
+            else:
+              CC[compiler].compilerExe
   if result.len == 0:
     rawMessage(conf, errGenerated,
       "Compiler '$1' doesn't support the requested target" %
       CC[compiler].name)
 
+proc ccHasSaneOverflow*(conf: ConfigRef): bool =
+  if conf.cCompiler == ccGcc:
+    result = false # assume an old or crappy GCC
+    var exe = getConfigVar(conf, conf.cCompiler, ".exe")
+    if exe.len == 0: exe = CC[conf.cCompiler].compilerExe
+    let (s, exitCode) = try: execCmdEx(exe & " --version") except: ("", 1)
+    if exitCode == 0:
+      var i = 0
+      var j = 0
+      # the version is the last part of the first line:
+      while i < s.len and s[i] != '\n':
+        if s[i] in {' ', '\t'}: j = i+1
+        inc i
+      if j > 0:
+        var major = 0
+        while j < s.len and s[j] in {'0'..'9'}:
+          major = major * 10 + (ord(s[j]) - ord('0'))
+          inc j
+        if i < s.len and s[j] == '.': inc j
+        while j < s.len and s[j] in {'0'..'9'}:
+          inc j
+        if j+1 < s.len and s[j] == '.' and s[j+1] in {'0'..'9'}:
+          # we found a third version number, chances are high
+          # we really parsed the version:
+          result = major >= 5
+  else:
+    result = conf.cCompiler == ccCLang
+
 proc getLinkerExe(conf: ConfigRef; compiler: TSystemCC): string =
   result = if CC[compiler].linkerExe.len > 0: CC[compiler].linkerExe
-           elif optMixedMode in conf.globalOptions and conf.cmd != cmdCompileToCpp: CC[compiler].cppCompiler
+           elif optMixedMode in conf.globalOptions and conf.backend != backendCpp: CC[compiler].cppCompiler
            else: getCompilerExe(conf, compiler, AbsoluteFile"")
 
 proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
@@ -612,6 +541,11 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
   # We produce files like module.nim.cpp, so the absolute Nim filename is not
   # cfile.name but `cfile.cname.changeFileExt("")`:
   var options = cFileSpecificOptions(conf, cfile.nimname, cfile.cname.changeFileExt("").string)
+  if useCpp(conf, cfile.cname):
+    # needs to be prepended so that --passc:-std=c++17 can override default.
+    # we could avoid allocation by making cFileSpecificOptions inplace
+    options = CC[c].cppXsupport & ' ' & options
+
   var exe = getConfigVar(conf, c, ".exe")
   if exe.len == 0: exe = getCompilerExe(conf, c, cfile.cname)
 
@@ -620,8 +554,9 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
       ospNeedsPIC in platform.OS[conf.target.targetOS].props:
     options.add(' ' & CC[c].pic)
 
-  if useCpp(conf, cfile.cname):
-    options.add(' ' & CC[c].cppXsupport)
+  if cfile.customArgs != "":
+    options.add ' '
+    options.add cfile.customArgs
 
   var compilePattern: string
   # compute include paths:
@@ -659,7 +594,8 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
     "dfile", dfile,
     "file", cfsh, "objfile", quoteShell(objfile), "options", options,
     "include", includeCmd, "nim", getPrefixDir(conf).string,
-    "lib", conf.libpath.string])
+    "lib", conf.libpath.string,
+    "ccenvflags", envFlags(conf)])
 
   if optProduceAsm in conf.globalOptions:
     if CC[conf.cCompiler].produceAsm.len > 0:
@@ -679,7 +615,8 @@ proc getCompileCFileCmd*(conf: ConfigRef; cfile: Cfile,
     "options", options, "include", includeCmd,
     "nim", quoteShell(getPrefixDir(conf)),
     "lib", quoteShell(conf.libpath),
-    "vccplatform", vccplatform(conf)])
+    "vccplatform", vccplatform(conf),
+    "ccenvflags", envFlags(conf)])
 
 proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
   result = secureHash(
@@ -690,8 +627,7 @@ proc footprint(conf: ConfigRef; cfile: Cfile): SecureHash =
     getCompileCFileCmd(conf, cfile))
 
 proc externalFileChanged(conf: ConfigRef; cfile: Cfile): bool =
-  if conf.cmd notin {cmdCompileToC, cmdCompileToCpp, cmdCompileToOC, cmdCompileToLLVM, cmdNone}:
-    return false
+  if conf.backend == backendJs: return false # pre-existing behavior, but not sure it's good
 
   var hashFile = toGeneratedFile(conf, conf.withPackageName(cfile.cname), "sha1")
   var currentHash = footprint(conf, cfile)
@@ -726,14 +662,7 @@ proc addExternalFileToCompile*(conf: ConfigRef; filename: AbsoluteFile) =
 proc getLinkCmd(conf: ConfigRef; output: AbsoluteFile,
                 objfiles: string, isDllBuild: bool): string =
   if optGenStaticLib in conf.globalOptions:
-    var libname: string
-    if not conf.outFile.isEmpty:
-      libname = conf.outFile.string.expandTilde
-      if not libname.isAbsolute():
-        libname = getCurrentDir() / libname
-    else:
-      libname = (libNameTmpl(conf) % splitFile(conf.projectName).name)
-    result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(libname),
+    result = CC[conf.cCompiler].buildLib % ["libfile", quoteShell(output),
                                             "objfiles", objfiles]
   else:
     var linkerExe = getConfigVar(conf, conf.cCompiler, ".linkerexe")
@@ -830,19 +759,16 @@ template tryExceptOSErrorMessage(conf: ConfigRef; errorPrefix: string = "", body
         (ose.msg & " " & $ose.errorCode))
     raise
 
+proc getExtraCmds(conf: ConfigRef; output: AbsoluteFile): seq[string] =
+  when defined(macosx):
+    if optCDebug in conf.globalOptions and optGenStaticLib notin conf.globalOptions:
+      # if needed, add an option to skip or override location
+      result.add "dsymutil " & $(output).quoteShell
+
 proc execLinkCmd(conf: ConfigRef; linkCmd: string) =
   tryExceptOSErrorMessage(conf, "invocation of external linker program failed."):
     execExternalProgram(conf, linkCmd, hintLinking)
 
-proc maybeRunDsymutil(conf: ConfigRef; exe: AbsoluteFile) =
-  when defined(osx):
-    if optCDebug notin conf.globalOptions: return
-    # if needed, add an option to skip or override location
-    let cmd = "dsymutil " & $(exe).quoteShell
-    conf.extraCmds.add cmd
-    tryExceptOSErrorMessage(conf, "invocation of dsymutil failed."):
-      execExternalProgram(conf, cmd, hintExecuting)
-
 proc execCmdsInParallel(conf: ConfigRef; cmds: seq[string]; prettyCb: proc (idx: int)) =
   let runCb = proc (idx: int, p: Process) =
     let exitCode = p.peekExitCode
@@ -913,6 +839,7 @@ proc displayProgressCC(conf: ConfigRef, path, compileCmd: string): string =
 proc callCCompiler*(conf: ConfigRef) =
   var
     linkCmd: string
+    extraCmds: seq[string]
   if conf.globalOptions * {optCompileOnly, optGenScript} == {optCompileOnly}:
     return # speed up that call if only compiling and no script shall be
            # generated
@@ -983,6 +910,7 @@ proc callCCompiler*(conf: ConfigRef) =
       let mainOutput = if optGenScript notin conf.globalOptions: conf.prepareToWriteOutput
                        else: AbsoluteFile(conf.projectName)
       linkCmd = getLinkCmd(conf, mainOutput, objfiles)
+      extraCmds = getExtraCmds(conf, mainOutput)
       if optCompileOnly notin conf.globalOptions:
         const MaxCmdLen = when defined(windows): 8_000 else: 32_000
         if linkCmd.len > MaxCmdLen:
@@ -992,7 +920,8 @@ proc callCCompiler*(conf: ConfigRef) =
           linkViaResponseFile(conf, linkCmd)
         else:
           execLinkCmd(conf, linkCmd)
-        maybeRunDsymutil(conf, mainOutput)
+        for cmd in extraCmds:
+          execExternalProgram(conf, cmd, hintExecuting)
   else:
     linkCmd = ""
   if optGenScript in conf.globalOptions:
@@ -1066,10 +995,16 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
   var buf = newStringOfCap(50)
 
   let jsonFile = conf.getNimcacheDir / RelativeFile(conf.projectName & ".json")
+  conf.jsonBuildFile = jsonFile
+  let output = conf.absOutFile
 
   var f: File
   if open(f, jsonFile.string, fmWrite):
-    lit "{\"compile\":[\L"
+    lit "{\L"
+    lit "\"outputFile\": "
+    str $output
+
+    lit ",\L\"compile\":[\L"
     cfiles(conf, f, buf, conf.toCompile, false)
     lit "],\L\"link\":[\L"
     var objfiles = ""
@@ -1077,10 +1012,10 @@ proc writeJsonBuildInstructions*(conf: ConfigRef) =
     linkfiles(conf, f, buf, objfiles, conf.toCompile, conf.externalToLink)
 
     lit "],\L\"linkcmd\": "
-    str getLinkCmd(conf, conf.absOutFile, objfiles)
+    str getLinkCmd(conf, output, objfiles)
 
     lit ",\L\"extraCmds\": "
-    lit $(%* conf.extraCmds)
+    lit $(%* getExtraCmds(conf, conf.absOutFile))
 
     lit ",\L\"stdinInput\": "
     lit $(%* conf.projectIsStdin)
@@ -1137,6 +1072,17 @@ proc runJsonBuildInstructions*(conf: ConfigRef; projectfile: AbsoluteFile) =
   let jsonFile = toGeneratedFile(conf, projectfile, "json")
   try:
     let data = json.parseFile(jsonFile.string)
+
+    let output = data["outputFile"].getStr
+    createDir output.parentDir
+    let outputCurrent = $conf.absOutFile
+    if output != outputCurrent:
+      # previously, any specified output file would be silently ignored;
+      # simply copying won't work in some cases, for example with `extraCmds`,
+      # so we just make it an error, user should use same command for jsonscript
+      # as was used with --compileOnly.
+      globalError(conf, gCmdLineInfo, "jsonscript command outputFile '$1' must match '$2' which was specified during --compileOnly, see \"outputFile\" entry in '$3' " % [outputCurrent, output, jsonFile.string])
+
     let toCompile = data["compile"]
     doAssert toCompile.kind == JArray
     var cmds: TStringSeq

compiler/filter_tmpl.nim

@@ -199,8 +199,8 @@ proc parseLine(p: var TTmplParser) =
           inc(j)
     llStreamWrite(p.outp, "\\n\"")
 
-proc filterTmpl*(stdin: PLLStream, filename: AbsoluteFile,
-                 call: PNode; conf: ConfigRef): PLLStream =
+proc filterTmpl*(conf: ConfigRef, stdin: PLLStream, filename: AbsoluteFile,
+                 call: PNode): PLLStream =
   var p: TTmplParser
   p.config = conf
   p.info = newLineInfo(conf, filename, 0, 0)
@@ -214,9 +214,9 @@ proc filterTmpl*(stdin: PLLStream, filename: AbsoluteFile,
   p.x = newStringOfCap(120)
   # do not process the first line which contains the directive:
   if llStreamReadLine(p.inp, p.x):
-    p.info.line = p.info.line + 1'u16
+    inc p.info.line
   while llStreamReadLine(p.inp, p.x):
-    p.info.line = p.info.line + 1'u16
+    inc p.info.line
     parseLine(p)
   newLine(p)
   result = p.outp

compiler/forloops.nim

@@ -1,89 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements for loop detection for better C code generation.
-
-import ast, astalgo
-
-const
-  someCmp = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
-    mLeI, mLeF64, mLeU, mLeEnum,
-    mLeCh, mLeB, mLePtr, mLtI, mLtF64, mLtU, mLtEnum,
-    mLtCh, mLtB, mLtPtr}
-
-proc isCounter(s: PSym): bool {.inline.} =
-  s.kind in {skResult, skVar, skLet, skTemp} and
-  {sfGlobal, sfAddrTaken} * s.flags == {}
-
-proc isCall(n: PNode): bool {.inline.} =
-  n.kind in nkCallKinds and n[0].kind == nkSym
-
-proc fromSystem(op: PSym): bool = sfSystemModule in getModule(op).flags
-
-proc getCounter(lastStmt: PNode): PSym =
-  if lastStmt.isCall:
-    let op = lastStmt.sym
-    if op.magic in {mDec, mInc} or
-        ((op.name.s == "+=" or op.name.s == "-=") and op.fromSystem):
-      if op[1].kind == nkSym and isCounter(op[1].sym):
-        result = op[1].sym
-
-proc counterInTree(n, loop: PNode; counter: PSym): bool =
-  # prune the search tree: within the loop the counter may be used:
-  if n == loop: return
-  case n.kind
-  of nkSym:
-    if n.sym == counter: return true
-  of nkVarSection, nkLetSection:
-    # definitions are fine!
-    for it in n:
-      if counterInTree(it.lastSon): return true
-  else:
-    for i in 0..<n.safeLen:
-      if counterInTree(n[i], loop, counter): return true
-
-proc copyExcept(n: PNode, x, dest: PNode) =
-  if x == n: return
-  if n.kind in {nkStmtList, nkStmtListExpr}:
-    for i in 0..<n.len: copyExcept(n[i], x, dest)
-  else:
-    dest.add n
-
-type
-  ForLoop* = object
-    counter*: PSym
-    init*, cond*, increment*, body*: PNode
-
-proc extractForLoop*(loop, fullTree: PNode): ForLoop =
-  ## returns 'counter == nil' if the while loop 'n' is not a for loop:
-  assert loop.kind == nkWhileStmt
-  let cond == loop[0]
-
-  if not cond.isCall: return
-  if cond[0].sym.magic notin someCmp: return
-
-  var lastStmt = loop[1]
-  while lastStmt.kind in {nkStmtList, nkStmtListExpr}:
-    lastStmt = lastStmt.lastSon
-
-  let counter = getCounter(lastStmt)
-  if counter.isNil or counter.ast.isNil: return
-
-  template `=~`(a, b): expr = a.kind == nkSym and a.sym == b
-
-  if cond[1] =~ counter or cond[2] =~ counter:
-    # ok, now check 'counter' is not used *after* the loop
-    if counterInTree(fullTree, loop, counter): return
-    # ok, success, fill in the fields:
-    result.counter = counter
-    result.init = counter.ast
-    result.cond = cond
-    result.increment = lastStmt
-    result.body = newNodeI(nkStmtList, loop[1].info)
-    copyExcept(loop[1], lastStmt, result.body)

compiler/guards.nim

@@ -46,7 +46,7 @@ proc isLet(n: PNode): bool =
     if n.sym.kind in {skLet, skTemp, skForVar}:
       result = true
     elif n.sym.kind == skParam and skipTypes(n.sym.typ,
-                                             abstractInst).kind != tyVar:
+                                             abstractInst).kind notin {tyVar}:
       result = true
 
 proc isVar(n: PNode): bool =
@@ -84,8 +84,8 @@ proc interestingCaseExpr*(m: PNode): bool = isLetLocation(m, true)
 
 type
   Operators* = object
-    opNot, opContains, opLe, opLt, opAnd, opOr, opIsNil, opEq: PSym
-    opAdd, opSub, opMul, opDiv, opLen: PSym
+    opNot*, opContains*, opLe*, opLt*, opAnd*, opOr*, opIsNil*, opEq*: PSym
+    opAdd*, opSub*, opMul*, opDiv*, opLen*: PSym
 
 proc initOperators*(g: ModuleGraph): Operators =
   result.opLe = createMagic(g, "<=", mLeI)
@@ -156,12 +156,12 @@ proc neg(n: PNode; o: Operators): PNode =
     result[0] = newSymNode(o.opNot)
     result[1] = n
 
-proc buildCall(op: PSym; a: PNode): PNode =
+proc buildCall*(op: PSym; a: PNode): PNode =
   result = newNodeI(nkCall, a.info, 2)
   result[0] = newSymNode(op)
   result[1] = a
 
-proc buildCall(op: PSym; a, b: PNode): PNode =
+proc buildCall*(op: PSym; a, b: PNode): PNode =
   result = newNodeI(nkInfix, a.info, 3)
   result[0] = newSymNode(op)
   result[1] = a
@@ -464,7 +464,7 @@ proc hasSubTree(n, x: PNode): bool =
     for i in 0..n.safeLen-1:
       if hasSubTree(n[i], x): return true
 
-proc invalidateFacts*(m: var TModel, n: PNode) =
+proc invalidateFacts*(s: var seq[PNode], n: PNode) =
   # We are able to guard local vars (as opposed to 'let' variables)!
   # 'while p != nil: f(p); p = p.next'
   # This is actually quite easy to do:
@@ -482,8 +482,11 @@ proc invalidateFacts*(m: var TModel, n: PNode) =
   # The same mechanism could be used for more complex data stored on the heap;
   # procs that 'write: []' cannot invalidate 'n.kind' for instance. In fact, we
   # could CSE these expressions then and help C's optimizer.
-  for i in 0..high(m.s):
-    if m.s[i] != nil and m.s[i].hasSubTree(n): m.s[i] = nil
+  for i in 0..high(s):
+    if s[i] != nil and s[i].hasSubTree(n): s[i] = nil
+
+proc invalidateFacts*(m: var TModel, n: PNode) =
+  invalidateFacts(m.s, n)
 
 proc valuesUnequal(a, b: PNode): bool =
   if a.isValue and b.isValue:

compiler/idents.nim

@@ -110,8 +110,8 @@ proc newIdentCache*(): IdentCache =
   result.idDelegator = result.getIdent":delegator"
   result.emptyIdent = result.getIdent("")
   # initialize the keywords:
-  for s in succ(low(specialWords))..high(specialWords):
-    result.getIdent(specialWords[s], hashIgnoreStyle(specialWords[s])).id = ord(s)
+  for s in succ(low(TSpecialWord))..high(TSpecialWord):
+    result.getIdent($s, hashIgnoreStyle($s)).id = ord(s)
 
 proc whichKeyword*(id: PIdent): TSpecialWord =
   if id.id < 0: result = wInvalid

compiler/importer.nim

@@ -187,10 +187,9 @@ proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
   let it = transformImportAs(c, it)
   let m = myImportModule(c, it, importStmtResult)
   if m != nil:
-    var emptySet: IntSet
     # ``addDecl`` needs to be done before ``importAllSymbols``!
     addDecl(c, m, it.info) # add symbol to symbol table of module
-    importAllSymbolsExcept(c, m, emptySet)
+    importAllSymbols(c, m)
     #importForwarded(c, m.ast, emptySet, m)
 
 proc evalImport*(c: PContext, n: PNode): PNode =

compiler/index.nim

@@ -0,0 +1,18 @@
+##[
+This module only exists to generate docs for the compiler.
+
+## links
+* [main docs](../lib.html)
+* [compiler user guide](../nimc.html)
+* [Internals of the Nim Compiler](../intern.html)
+]##
+
+#[
+note: this is named `index` so that navigating to https://nim-lang.github.io/Nim/compiler/
+will work.
+
+xxx this should also import other modules, not transitively imported by `compiler/nim.nim`,
+eg `evalffi`, otherwise these aren't shown. A glob could be used at CT.
+]#
+
+import nim

compiler/installer.ini

@@ -9,9 +9,9 @@ Platforms: """
   linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv64
   macosx: i386;amd64;powerpc64
   solaris: i386;amd64;sparc;sparc64
-  freebsd: i386;amd64;powerpc64;arm;arm64
+  freebsd: i386;amd64;powerpc64;arm;arm64;riscv64;sparc64;mips;mipsel;mips64;mips64el;powerpc;powerpc64el
   netbsd: i386;amd64
-  openbsd: i386;amd64
+  openbsd: i386;amd64;arm;arm64
   dragonfly: i386;amd64
   haiku: i386;amd64
   android: i386;arm;arm64
@@ -35,9 +35,8 @@ App: Console
 License: "copying.txt"
 
 [Config]
-Files: "config/nim.cfg"
-Files: "config/nimdoc.cfg"
-Files: "config/nimdoc.tex.cfg"
+Files: "config/*.cfg"
+Files: "config/config.nims"
 
 [Documentation]
 ; Files: "doc/*.html"
@@ -77,6 +76,7 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/fusion"
 
 Files: "tests"
 
@@ -89,6 +89,7 @@ Files: "bin/vccexe.exe"
 Files: "bin/nimgrab.exe"
 Files: "bin/nimpretty.exe"
 Files: "bin/testament.exe"
+Files: "bin/nim-gdb.bat"
 
 Files: "koch.exe"
 Files: "finish.exe"
@@ -117,6 +118,8 @@ Files: "bin/nim"
 [Unix]
 InstallScript: "yes"
 UninstallScript: "yes"
+Files: "bin/nim-gdb"
+Files: "bin/nim-gdb.bash"
 
 
 [InnoSetup]

compiler/int128.nim

@@ -373,8 +373,8 @@ proc `*`*(lhs,rhs: Int128): Int128 =
   let b00 = cast[uint64](rhs.udata[0])
 
   result = makeInt128(high64(lhs) * low64(rhs) + low64(lhs) * high64(rhs) + a32 * b32, a00 * b00)
-  result = result + toInt128(a32 * b00) shl 32
-  result = result + toInt128(a00 * b32) shl 32
+  result += toInt128(a32 * b00) shl 32
+  result += toInt128(a00 * b32) shl 32
 
 proc `*=`*(a: var Int128, b: Int128) =
   a = a * b
@@ -427,7 +427,7 @@ proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
   for i in 0..shift:
     quotient = quotient shl 1
     if dividend >= denominator:
-      dividend = dividend - denominator
+      dividend -= denominator
       quotient = bitor(quotient, One)
 
     denominator = denominator shr 1
@@ -592,190 +592,3 @@ proc maskBytes*(arg: Int128, numbytes: int): Int128 {.noinit.} =
     return maskUInt64(arg)
   else:
     assert(false, "masking only implemented for 1, 2, 4 and 8 bytes")
-
-
-
-
-when isMainModule:
-  let (a,b) = divMod(Ten,Ten)
-
-  doAssert $One == "1"
-  doAssert $Ten == "10"
-  doAssert $Zero == "0"
-  let c = parseDecimalInt128("12345678989876543210123456789")
-  doAssert $c == "12345678989876543210123456789"
-
-  var d : array[39, Int128]
-  d[0] =  parseDecimalInt128("1")
-  d[1] =  parseDecimalInt128("10")
-  d[2] =  parseDecimalInt128("100")
-  d[3] =  parseDecimalInt128("1000")
-  d[4] =  parseDecimalInt128("10000")
-  d[5] =  parseDecimalInt128("100000")
-  d[6] =  parseDecimalInt128("1000000")
-  d[7] =  parseDecimalInt128("10000000")
-  d[8] =  parseDecimalInt128("100000000")
-  d[9] =  parseDecimalInt128("1000000000")
-  d[10] = parseDecimalInt128("10000000000")
-  d[11] = parseDecimalInt128("100000000000")
-  d[12] = parseDecimalInt128("1000000000000")
-  d[13] = parseDecimalInt128("10000000000000")
-  d[14] = parseDecimalInt128("100000000000000")
-  d[15] = parseDecimalInt128("1000000000000000")
-  d[16] = parseDecimalInt128("10000000000000000")
-  d[17] = parseDecimalInt128("100000000000000000")
-  d[18] = parseDecimalInt128("1000000000000000000")
-  d[19] = parseDecimalInt128("10000000000000000000")
-  d[20] = parseDecimalInt128("100000000000000000000")
-  d[21] = parseDecimalInt128("1000000000000000000000")
-  d[22] = parseDecimalInt128("10000000000000000000000")
-  d[23] = parseDecimalInt128("100000000000000000000000")
-  d[24] = parseDecimalInt128("1000000000000000000000000")
-  d[25] = parseDecimalInt128("10000000000000000000000000")
-  d[26] = parseDecimalInt128("100000000000000000000000000")
-  d[27] = parseDecimalInt128("1000000000000000000000000000")
-  d[28] = parseDecimalInt128("10000000000000000000000000000")
-  d[29] = parseDecimalInt128("100000000000000000000000000000")
-  d[30] = parseDecimalInt128("1000000000000000000000000000000")
-  d[31] = parseDecimalInt128("10000000000000000000000000000000")
-  d[32] = parseDecimalInt128("100000000000000000000000000000000")
-  d[33] = parseDecimalInt128("1000000000000000000000000000000000")
-  d[34] = parseDecimalInt128("10000000000000000000000000000000000")
-  d[35] = parseDecimalInt128("100000000000000000000000000000000000")
-  d[36] = parseDecimalInt128("1000000000000000000000000000000000000")
-  d[37] = parseDecimalInt128("10000000000000000000000000000000000000")
-  d[38] = parseDecimalInt128("100000000000000000000000000000000000000")
-
-  for i in 0..<d.len:
-    for j in 0..<d.len:
-      doAssert(cmp(d[i], d[j]) == cmp(i,j))
-      if i + j < d.len:
-        doAssert d[i] * d[j] == d[i+j]
-      if i - j >= 0:
-        doAssert d[i] div d[j] == d[i-j]
-
-  var sum: Int128
-
-  for it in d:
-    sum += it
-
-  doAssert $sum == "111111111111111111111111111111111111111"
-
-  for it in d.mitems:
-    it = -it
-
-  for i in 0..<d.len:
-    for j in 0..<d.len:
-      doAssert(cmp(d[i], d[j]) == -cmp(i,j))
-      if i + j < d.len:
-        doAssert d[i] * d[j] == -d[i+j]
-      if i - j >= 0:
-        doAssert d[i] div d[j] == -d[i-j]
-
-  doAssert $high(Int128) == "170141183460469231731687303715884105727"
-  doAssert $low(Int128) == "-170141183460469231731687303715884105728"
-
-  var ma = 100'i64
-  var mb = 13
-
-  doAssert toInt128(ma) * toInt128(0) == toInt128(0)
-  doAssert toInt128(-ma) * toInt128(0) == toInt128(0)
-
-  # sign correctness
-  doAssert divMod(toInt128( ma),toInt128( mb)) == (toInt128( ma div  mb), toInt128( ma mod  mb))
-  doAssert divMod(toInt128(-ma),toInt128( mb)) == (toInt128(-ma div  mb), toInt128(-ma mod  mb))
-  doAssert divMod(toInt128( ma),toInt128(-mb)) == (toInt128( ma div -mb), toInt128( ma mod -mb))
-  doAssert divMod(toInt128(-ma),toInt128(-mb)) == (toInt128(-ma div -mb), toInt128(-ma mod -mb))
-
-  doAssert divMod(toInt128( mb),toInt128( mb)) == (toInt128( mb div  mb), toInt128( mb mod  mb))
-  doAssert divMod(toInt128(-mb),toInt128( mb)) == (toInt128(-mb div  mb), toInt128(-mb mod  mb))
-  doAssert divMod(toInt128( mb),toInt128(-mb)) == (toInt128( mb div -mb), toInt128( mb mod -mb))
-  doAssert divMod(toInt128(-mb),toInt128(-mb)) == (toInt128(-mb div -mb), toInt128(-mb mod -mb))
-
-  doAssert divMod(toInt128( mb),toInt128( ma)) == (toInt128( mb div  ma), toInt128( mb mod  ma))
-  doAssert divMod(toInt128(-mb),toInt128( ma)) == (toInt128(-mb div  ma), toInt128(-mb mod  ma))
-  doAssert divMod(toInt128( mb),toInt128(-ma)) == (toInt128( mb div -ma), toInt128( mb mod -ma))
-  doAssert divMod(toInt128(-mb),toInt128(-ma)) == (toInt128(-mb div -ma), toInt128(-mb mod -ma))
-
-  let e = parseDecimalInt128("70997106675279150998592376708984375")
-
-  let strArray = [
-    # toHex(e shr 0), toHex(e shr 1), toHex(e shr 2), toHex(e shr 3)
-    "000dac6d782d266a37300c32591eee37", "0006d636bc1693351b9806192c8f771b", "00036b1b5e0b499a8dcc030c9647bb8d", "0001b58daf05a4cd46e601864b23ddc6",
-    "0000dac6d782d266a37300c32591eee3", "00006d636bc1693351b9806192c8f771", "000036b1b5e0b499a8dcc030c9647bb8", "00001b58daf05a4cd46e601864b23ddc",
-    "00000dac6d782d266a37300c32591eee", "000006d636bc1693351b9806192c8f77", "0000036b1b5e0b499a8dcc030c9647bb", "000001b58daf05a4cd46e601864b23dd",
-    "000000dac6d782d266a37300c32591ee", "0000006d636bc1693351b9806192c8f7", "00000036b1b5e0b499a8dcc030c9647b", "0000001b58daf05a4cd46e601864b23d",
-    "0000000dac6d782d266a37300c32591e", "00000006d636bc1693351b9806192c8f", "000000036b1b5e0b499a8dcc030c9647", "00000001b58daf05a4cd46e601864b23",
-    "00000000dac6d782d266a37300c32591", "000000006d636bc1693351b9806192c8", "0000000036b1b5e0b499a8dcc030c964", "000000001b58daf05a4cd46e601864b2",
-    "000000000dac6d782d266a37300c3259", "0000000006d636bc1693351b9806192c", "00000000036b1b5e0b499a8dcc030c96", "0000000001b58daf05a4cd46e601864b",
-    "0000000000dac6d782d266a37300c325", "00000000006d636bc1693351b9806192", "000000000036b1b5e0b499a8dcc030c9", "00000000001b58daf05a4cd46e601864",
-    "00000000000dac6d782d266a37300c32", "000000000006d636bc1693351b980619", "0000000000036b1b5e0b499a8dcc030c", "000000000001b58daf05a4cd46e60186",
-    "000000000000dac6d782d266a37300c3", "0000000000006d636bc1693351b98061", "00000000000036b1b5e0b499a8dcc030", "0000000000001b58daf05a4cd46e6018",
-    "0000000000000dac6d782d266a37300c", "00000000000006d636bc1693351b9806", "000000000000036b1b5e0b499a8dcc03", "00000000000001b58daf05a4cd46e601",
-    "00000000000000dac6d782d266a37300", "000000000000006d636bc1693351b980", "0000000000000036b1b5e0b499a8dcc0", "000000000000001b58daf05a4cd46e60",
-    "000000000000000dac6d782d266a3730", "0000000000000006d636bc1693351b98", "00000000000000036b1b5e0b499a8dcc", "0000000000000001b58daf05a4cd46e6",
-    "0000000000000000dac6d782d266a373", "00000000000000006d636bc1693351b9", "000000000000000036b1b5e0b499a8dc", "00000000000000001b58daf05a4cd46e",
-    "00000000000000000dac6d782d266a37", "000000000000000006d636bc1693351b", "0000000000000000036b1b5e0b499a8d", "000000000000000001b58daf05a4cd46",
-    "000000000000000000dac6d782d266a3", "0000000000000000006d636bc1693351", "00000000000000000036b1b5e0b499a8", "0000000000000000001b58daf05a4cd4",
-    "0000000000000000000dac6d782d266a", "00000000000000000006d636bc169335", "000000000000000000036b1b5e0b499a", "00000000000000000001b58daf05a4cd",
-    "00000000000000000000dac6d782d266", "000000000000000000006d636bc16933", "0000000000000000000036b1b5e0b499", "000000000000000000001b58daf05a4c",
-    "000000000000000000000dac6d782d26", "0000000000000000000006d636bc1693", "00000000000000000000036b1b5e0b49", "0000000000000000000001b58daf05a4",
-    "0000000000000000000000dac6d782d2", "00000000000000000000006d636bc169", "000000000000000000000036b1b5e0b4", "00000000000000000000001b58daf05a",
-    "00000000000000000000000dac6d782d", "000000000000000000000006d636bc16", "0000000000000000000000036b1b5e0b", "000000000000000000000001b58daf05",
-    "000000000000000000000000dac6d782", "0000000000000000000000006d636bc1", "00000000000000000000000036b1b5e0", "0000000000000000000000001b58daf0",
-    "0000000000000000000000000dac6d78", "00000000000000000000000006d636bc", "000000000000000000000000036b1b5e", "00000000000000000000000001b58daf",
-    "00000000000000000000000000dac6d7", "000000000000000000000000006d636b", "0000000000000000000000000036b1b5", "000000000000000000000000001b58da",
-    "000000000000000000000000000dac6d", "0000000000000000000000000006d636", "00000000000000000000000000036b1b", "0000000000000000000000000001b58d",
-    "0000000000000000000000000000dac6", "00000000000000000000000000006d63", "000000000000000000000000000036b1", "00000000000000000000000000001b58",
-    "00000000000000000000000000000dac", "000000000000000000000000000006d6", "0000000000000000000000000000036b", "000000000000000000000000000001b5",
-    "000000000000000000000000000000da", "0000000000000000000000000000006d", "00000000000000000000000000000036", "0000000000000000000000000000001b",
-    "0000000000000000000000000000000d", "00000000000000000000000000000006", "00000000000000000000000000000003", "00000000000000000000000000000001",
-    "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000",
-    "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000",
-    "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000", "00000000000000000000000000000000",
-  ]
-
-  for i in 0 ..< 128:
-    let str1 = toHex(e shr i)
-    let str2 = strArray[i]
-    doAssert str1 == str2
-
-  let strArray2 = [
-    "000dac6d782d266a37300c32591eee37", "001b58daf05a4cd46e601864b23ddc6e", "0036b1b5e0b499a8dcc030c9647bb8dc", "006d636bc1693351b9806192c8f771b8",
-    "00dac6d782d266a37300c32591eee370", "01b58daf05a4cd46e601864b23ddc6e0", "036b1b5e0b499a8dcc030c9647bb8dc0", "06d636bc1693351b9806192c8f771b80",
-    "0dac6d782d266a37300c32591eee3700", "1b58daf05a4cd46e601864b23ddc6e00", "36b1b5e0b499a8dcc030c9647bb8dc00", "6d636bc1693351b9806192c8f771b800",
-    "dac6d782d266a37300c32591eee37000", "b58daf05a4cd46e601864b23ddc6e000", "6b1b5e0b499a8dcc030c9647bb8dc000", "d636bc1693351b9806192c8f771b8000",
-    "ac6d782d266a37300c32591eee370000", "58daf05a4cd46e601864b23ddc6e0000", "b1b5e0b499a8dcc030c9647bb8dc0000", "636bc1693351b9806192c8f771b80000",
-    "c6d782d266a37300c32591eee3700000", "8daf05a4cd46e601864b23ddc6e00000", "1b5e0b499a8dcc030c9647bb8dc00000", "36bc1693351b9806192c8f771b800000",
-    "6d782d266a37300c32591eee37000000", "daf05a4cd46e601864b23ddc6e000000", "b5e0b499a8dcc030c9647bb8dc000000", "6bc1693351b9806192c8f771b8000000",
-    "d782d266a37300c32591eee370000000", "af05a4cd46e601864b23ddc6e0000000", "5e0b499a8dcc030c9647bb8dc0000000", "bc1693351b9806192c8f771b80000000",
-    "782d266a37300c32591eee3700000000", "f05a4cd46e601864b23ddc6e00000000", "e0b499a8dcc030c9647bb8dc00000000", "c1693351b9806192c8f771b800000000",
-    "82d266a37300c32591eee37000000000", "05a4cd46e601864b23ddc6e000000000", "0b499a8dcc030c9647bb8dc000000000", "1693351b9806192c8f771b8000000000",
-    "2d266a37300c32591eee370000000000", "5a4cd46e601864b23ddc6e0000000000", "b499a8dcc030c9647bb8dc0000000000", "693351b9806192c8f771b80000000000",
-    "d266a37300c32591eee3700000000000", "a4cd46e601864b23ddc6e00000000000", "499a8dcc030c9647bb8dc00000000000", "93351b9806192c8f771b800000000000",
-    "266a37300c32591eee37000000000000", "4cd46e601864b23ddc6e000000000000", "99a8dcc030c9647bb8dc000000000000", "3351b9806192c8f771b8000000000000",
-    "66a37300c32591eee370000000000000", "cd46e601864b23ddc6e0000000000000", "9a8dcc030c9647bb8dc0000000000000", "351b9806192c8f771b80000000000000",
-    "6a37300c32591eee3700000000000000", "d46e601864b23ddc6e00000000000000", "a8dcc030c9647bb8dc00000000000000", "51b9806192c8f771b800000000000000",
-    "a37300c32591eee37000000000000000", "46e601864b23ddc6e000000000000000", "8dcc030c9647bb8dc000000000000000", "1b9806192c8f771b8000000000000000",
-    "37300c32591eee370000000000000000", "6e601864b23ddc6e0000000000000000", "dcc030c9647bb8dc0000000000000000", "b9806192c8f771b80000000000000000",
-    "7300c32591eee3700000000000000000", "e601864b23ddc6e00000000000000000", "cc030c9647bb8dc00000000000000000", "9806192c8f771b800000000000000000",
-    "300c32591eee37000000000000000000", "601864b23ddc6e000000000000000000", "c030c9647bb8dc000000000000000000", "806192c8f771b8000000000000000000",
-    "00c32591eee370000000000000000000", "01864b23ddc6e0000000000000000000", "030c9647bb8dc0000000000000000000", "06192c8f771b80000000000000000000",
-    "0c32591eee3700000000000000000000", "1864b23ddc6e00000000000000000000", "30c9647bb8dc00000000000000000000", "6192c8f771b800000000000000000000",
-    "c32591eee37000000000000000000000", "864b23ddc6e000000000000000000000", "0c9647bb8dc000000000000000000000", "192c8f771b8000000000000000000000",
-    "32591eee370000000000000000000000", "64b23ddc6e0000000000000000000000", "c9647bb8dc0000000000000000000000", "92c8f771b80000000000000000000000",
-    "2591eee3700000000000000000000000", "4b23ddc6e00000000000000000000000", "9647bb8dc00000000000000000000000", "2c8f771b800000000000000000000000",
-    "591eee37000000000000000000000000", "b23ddc6e000000000000000000000000", "647bb8dc000000000000000000000000", "c8f771b8000000000000000000000000",
-    "91eee370000000000000000000000000", "23ddc6e0000000000000000000000000", "47bb8dc0000000000000000000000000", "8f771b80000000000000000000000000",
-    "1eee3700000000000000000000000000", "3ddc6e00000000000000000000000000", "7bb8dc00000000000000000000000000", "f771b800000000000000000000000000",
-    "eee37000000000000000000000000000", "ddc6e000000000000000000000000000", "bb8dc000000000000000000000000000", "771b8000000000000000000000000000",
-    "ee370000000000000000000000000000", "dc6e0000000000000000000000000000", "b8dc0000000000000000000000000000", "71b80000000000000000000000000000",
-    "e3700000000000000000000000000000", "c6e00000000000000000000000000000", "8dc00000000000000000000000000000", "1b800000000000000000000000000000",
-    "37000000000000000000000000000000", "6e000000000000000000000000000000", "dc000000000000000000000000000000", "b8000000000000000000000000000000",
-    "70000000000000000000000000000000", "e0000000000000000000000000000000", "c0000000000000000000000000000000", "80000000000000000000000000000000",
-  ]
-
-  for i in 0 ..< 128:
-    let str1 = toHex(e shl i)
-    let str2 = strArray2[i]
-    doAssert str1 == str2

compiler/isolation_check.nim

@@ -0,0 +1,126 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Implementation of the check that `recover` needs, see
+## https://github.com/nim-lang/RFCs/issues/244 for more details.
+
+import
+  ast, types, renderer, idents, intsets
+
+proc canAlias(arg, ret: PType; marker: var IntSet): bool
+
+proc canAliasN(arg: PType; n: PNode; marker: var IntSet): bool =
+  case n.kind
+  of nkRecList:
+    for i in 0..<n.len:
+      result = canAliasN(arg, n[i], marker)
+      if result: return
+  of nkRecCase:
+    assert(n[0].kind == nkSym)
+    result = canAliasN(arg, n[0], marker)
+    if result: return
+    for i in 1..<n.len:
+      case n[i].kind
+      of nkOfBranch, nkElse:
+        result = canAliasN(arg, lastSon(n[i]), marker)
+        if result: return
+      else: discard
+  of nkSym:
+    result = canAlias(arg, n.sym.typ, marker)
+  else: discard
+
+proc canAlias(arg, ret: PType; marker: var IntSet): bool =
+  if containsOrIncl(marker, ret.id):
+    return false
+
+  if ret.kind in {tyPtr, tyPointer}:
+    # unsafe so we don't care:
+    return false
+  if compareTypes(arg, ret, dcEqIgnoreDistinct):
+    return true
+  case ret.kind
+  of tyObject:
+    if isFinal(ret):
+      result = canAliasN(arg, ret.n, marker)
+      if not result and ret.len > 0 and ret[0] != nil:
+        result = canAlias(arg, ret[0], marker)
+    else:
+      result = true
+  of tyTuple:
+    for i in 0..<ret.len:
+      result = canAlias(arg, ret[i], marker)
+      if result: break
+  of tyArray, tySequence, tyDistinct, tyGenericInst,
+     tyAlias, tyInferred, tySink, tyLent, tyOwned, tyRef:
+    result = canAlias(arg, ret.lastSon, marker)
+  of tyProc:
+    result = ret.callConv == ccClosure
+  else:
+    result = false
+
+proc isValueOnlyType(t: PType): bool = 
+  # t doesn't contain pointers and references
+  proc wrap(t: PType): bool {.nimcall.} = t.kind in {tyRef, tyPtr, tyVar, tyLent}
+  result = not types.searchTypeFor(t, wrap)
+
+proc canAlias*(arg, ret: PType): bool =
+  if isValueOnlyType(arg):
+    # can alias only with unsafeAddr(arg.x) and we don't care if it is not safe
+    result = false
+  else:
+    var marker = initIntSet()
+    result = canAlias(arg, ret, marker)
+
+proc checkIsolate*(n: PNode): bool =
+  if types.containsTyRef(n.typ):
+    # XXX Maybe require that 'n.typ' is acyclic. This is not much
+    # worse than the already exisiting inheritance and closure restrictions.
+    case n.kind
+    of nkCharLit..nkNilLit:
+      result = true
+    of nkCallKinds:
+      if n[0].typ.flags * {tfGcSafe, tfNoSideEffect} == {}:
+        return false
+      for i in 1..<n.len:
+        if checkIsolate(n[i]):
+          discard "fine, it is isolated already"
+        else:
+          let argType = n[i].typ
+          if argType != nil and not isCompileTimeOnly(argType) and containsTyRef(argType):
+            if argType.canAlias(n.typ):
+              return false
+      result = true
+    of nkIfStmt, nkIfExpr:
+      for it in n:
+        result = checkIsolate(it.lastSon)
+        if not result: break
+    of nkCaseStmt, nkObjConstr:
+      for i in 1..<n.len:
+        result = checkIsolate(n[i].lastSon)
+        if not result: break
+    of nkBracket, nkTupleConstr, nkPar:
+      for it in n:
+        result = checkIsolate(it)
+        if not result: break
+    of nkHiddenStdConv, nkHiddenSubConv, nkCast, nkConv:
+      result = checkIsolate(n[1])
+    of nkObjUpConv, nkObjDownConv, nkDotExpr:
+      result = checkIsolate(n[0])
+    of nkStmtList, nkStmtListExpr:
+      if n.len > 0:
+        result = checkIsolate(n[^1])
+      else:
+        result = false
+    else:
+      # unanalysable expression:
+      result = false
+  else:
+    # no ref, no cry:
+    result = true
+

compiler/jsgen.nim

@@ -86,6 +86,7 @@ type
     generatedSyms: IntSet
     typeInfoGenerated: IntSet
     unique: int    # for temp identifier generation
+    inSystem: bool
 
   PProc = ref TProc
   TProc = object
@@ -159,6 +160,14 @@ proc newProc(globals: PGlobals, module: BModule, procDef: PNode,
     extraIndent: int(procDef != nil))
   if procDef != nil: result.prc = procDef[namePos].sym
 
+proc initProcOptions(module: BModule): TOptions =
+  result = module.config.options
+  if PGlobals(module.graph.backend).inSystem:
+    result.excl(optStackTrace)
+
+proc newInitProc(globals: PGlobals, module: BModule): PProc =
+  result = newProc(globals, module, nil, initProcOptions(module))
+
 proc declareGlobal(p: PProc; id: int; r: Rope) =
   if p.prc != nil and not p.declaredGlobals.containsOrIncl(id):
     p.locals.addf("global $1;$n", [r])
@@ -184,7 +193,7 @@ proc mapType(typ: PType): TJSTypeKind =
   of tyBool: result = etyBool
   of tyFloat..tyFloat128: result = etyFloat
   of tySet: result = etyObject # map a set to a table
-  of tyString, tySequence, tyOpt: result = etySeq
+  of tyString, tySequence: result = etySeq
   of tyObject, tyArray, tyTuple, tyOpenArray, tyVarargs, tyUncheckedArray:
     result = etyObject
   of tyNil: result = etyNull
@@ -201,12 +210,11 @@ proc mapType(typ: PType): TJSTypeKind =
     else: result = etyNone
   of tyProc: result = etyProc
   of tyCString: result = etyString
+  of tyOptDeprecated: doAssert false
 
 proc mapType(p: PProc; typ: PType): TJSTypeKind =
   result = mapType(typ)
 
-var mangled = initSet[string]()
-
 proc mangleName(m: BModule, s: PSym): Rope =
   proc validJsName(name: string): bool =
     result = true
@@ -240,11 +248,7 @@ proc mangleName(m: BModule, s: PSym): Rope =
       while i < s.name.s.len:
         let c = s.name.s[i]
         case c
-        of 'A'..'Z':
-          if i > 0 and s.name.s[i-1] in {'a'..'z'}:
-            x.add '_'
-          x.add(chr(c.ord - 'A'.ord + 'a'.ord))
-        of 'a'..'z', '_', '0'..'9':
+        of 'A'..'Z', 'a'..'z', '_', '0'..'9':
           x.add c
         else:
           x.add("HEX" & toHex(ord(c), 2))
@@ -261,11 +265,6 @@ proc mangleName(m: BModule, s: PSym): Rope =
         result.add(rope(s.id))
     s.loc.r = result
 
-  # TODO: optimize
-  let s = $result
-  if '_' in s:
-    mangled.incl(s)
-
 proc escapeJSString(s: string): string =
   result = newStringOfCap(s.len + s.len shr 2)
   result.add("\"")
@@ -466,7 +465,48 @@ proc maybeMakeTemp(p: PProc, n: PNode; x: TCompRes): tuple[a, tmp: Rope] =
   else:
     (a: a, tmp: b)
 
-template binaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
+proc maybeMakeTempAssignable(p: PProc, n: PNode; x: TCompRes): tuple[a, tmp: Rope] =
+  var
+    a = x.rdLoc
+    b = a
+  if needsTemp(p, n):
+    # if we have tmp just use it
+    if x.tmpLoc != nil and (mapType(n.typ) == etyBaseIndex or n.kind in {nkHiddenDeref, nkDerefExpr}):
+      b = "$1[0][$1[1]]" % [x.tmpLoc]
+      result = (a: a, tmp: b)
+    elif x.tmpLoc != nil and n.kind == nkBracketExpr:
+      # genArrayAddr
+      var
+        address, index: TCompRes
+        first: Int128
+      gen(p, n[0], address)
+      gen(p, n[1], index)
+      let (m1, tmp1) = maybeMakeTemp(p, n[0], address)
+      let typ = skipTypes(n[0].typ, abstractPtrs)
+      if typ.kind == tyArray:
+        first = firstOrd(p.config, typ[0])
+      if optBoundsCheck in p.options:
+        useMagic(p, "chckIndx")
+        if first == 0: # save a couple chars
+          index.res = "chckIndx($1, 0, ($2).length-1)" % [index.res, tmp1]
+        else:
+          index.res = "chckIndx($1, $2, ($3).length+($2)-1)-($2)" % [
+            index.res, rope(first), tmp1]
+      elif first != 0:
+        index.res = "($1)-($2)" % [index.res, rope(first)]
+      else:
+        discard # index.res = index.res
+      let (n1, tmp2) = maybeMakeTemp(p, n[1], index)
+      result = (a: "$1[$2]" % [m1, n1], tmp: "$1[$2]" % [tmp1, tmp2])
+    # could also put here: nkDotExpr -> genFieldAccess, nkCheckedFieldExpr -> genCheckedFieldOp
+    # but the uses of maybeMakeTempAssignable don't need them
+    else:
+      result = (a: a, tmp: b)
+  else:
+    result = (a: a, tmp: b)
+
+template binaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string,
+                    reassign = false) =
   # $1 and $2 in the `frmt` string bind to lhs and rhs of the expr,
   # if $3 or $4 are present they will be substituted with temps for
   # lhs and rhs respectively
@@ -478,8 +518,11 @@ template binaryExpr(p: PProc, n: PNode, r: var TCompRes, magic, frmt: string) =
   var
     a, tmp = x.rdLoc
     b, tmp2 = y.rdLoc
-  when "$3" in frmt: (a, tmp) = maybeMakeTemp(p, n[1], x)
-  when "$4" in frmt: (b, tmp2) = maybeMakeTemp(p, n[2], y)
+  when reassign:
+    (a, tmp) = maybeMakeTempAssignable(p, n[1], x)
+  else:
+    when "$3" in frmt: (a, tmp) = maybeMakeTemp(p, n[1], x)
+    when "$4" in frmt: (b, tmp2) = maybeMakeTemp(p, n[2], y)
 
   r.res = frmt % [a, b, tmp, tmp2]
   r.kind = resExpr
@@ -496,13 +539,13 @@ template unsignedTrimmer(size: BiggestInt): Rope =
   size.unsignedTrimmerJS
 
 proc binaryUintExpr(p: PProc, n: PNode, r: var TCompRes, op: string,
-                    reassign = false) =
+                    reassign: static[bool] = false) =
   var x, y: TCompRes
   gen(p, n[1], x)
   gen(p, n[2], y)
   let trimmer = unsignedTrimmer(n[1].typ.skipTypes(abstractRange).size)
-  if reassign:
-    let (a, tmp) = maybeMakeTemp(p, n[1], x)
+  when reassign:
+    let (a, tmp) = maybeMakeTempAssignable(p, n[1], x)
     r.res = "$1 = (($5 $2 $3) $4)" % [a, rope op, y.rdLoc, trimmer, tmp]
   else:
     r.res = "(($1 $2 $3) $4)" % [x.rdLoc, rope op, y.rdLoc, trimmer]
@@ -541,11 +584,10 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
     gen(p, n[1], r)
     xLoc = r.rdLoc
 
-  template applyFormat(frmtA, frmtB: string) =
-    if i == 0:
-      r.res = frmtA % [xLoc, yLoc]
-    else:
-      r.res = frmtB % [xLoc, yLoc]
+  template applyFormat(frmt) =
+    r.res = frmt % [xLoc, yLoc]
+  template applyFormat(frmtA, frmtB) =
+    if i == 0: applyFormat(frmtA) else: applyFormat(frmtB)
 
   case op:
   of mAddI: applyFormat("addInt($1, $2)", "($1 + $2)")
@@ -560,8 +602,16 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mMulF64: applyFormat("($1 * $2)", "($1 * $2)")
   of mDivF64: applyFormat("($1 / $2)", "($1 / $2)")
   of mShrI: applyFormat("", "")
-  of mShlI: applyFormat("($1 << $2)", "($1 << $2)")
-  of mAshrI: applyFormat("($1 >> $2)", "($1 >> $2)")
+  of mShlI: 
+    if n[1].typ.size <= 4:
+      applyFormat("($1 << $2)", "($1 << $2)")
+    else:
+      applyFormat("($1 * Math.pow(2,$2))", "($1 * Math.pow(2,$2))")
+  of mAshrI: 
+    if n[1].typ.size <= 4:
+      applyFormat("($1 >> $2)", "($1 >> $2)")
+    else:
+      applyFormat("Math.floor($1 / Math.pow(2,$2))", "Math.floor($1 / Math.pow(2,$2))")
   of mBitandI: applyFormat("($1 & $2)", "($1 & $2)")
   of mBitorI: applyFormat("($1 | $2)", "($1 | $2)")
   of mBitxorI: applyFormat("($1 ^ $2)", "($1 ^ $2)")
@@ -607,9 +657,11 @@ proc arithAux(p: PProc, n: PNode, r: var TCompRes, op: TMagic) =
   of mBoolToStr: applyFormat("nimBoolToStr($1)", "nimBoolToStr($1)")
   of mIntToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
   of mInt64ToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
-  of mFloatToStr: applyFormat("cstrToNimstr(($1)+\"\")", "cstrToNimstr(($1)+\"\")")
+  of mFloatToStr:
+    useMagic(p, "nimFloatToString")
+    applyFormat "cstrToNimstr(nimFloatToString($1))"
   of mCStrToStr: applyFormat("cstrToNimstr($1)", "cstrToNimstr($1)")
-  of mStrToStr, mUnown: applyFormat("$1", "$1")
+  of mStrToStr, mUnown, mIsolate: applyFormat("$1", "$1")
   else:
     assert false, $op
 
@@ -813,6 +865,7 @@ proc genRaiseStmt(p: PProc, n: PNode) =
 proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
   var
     cond, stmt: TCompRes
+    totalRange = 0
   genLineDir(p, n)
   gen(p, n[0], cond)
   let stringSwitch = skipTypes(n[0].typ, abstractVar).kind == tyString
@@ -832,6 +885,10 @@ proc genCaseJS(p: PProc, n: PNode, r: var TCompRes) =
         let e = it[j]
         if e.kind == nkRange:
           var v = copyNode(e[0])
+          inc(totalRange, int(e[1].intVal - v.intVal))
+          if totalRange > 65535:
+            localError(p.config, n.info, 
+                       "Your case statement contains too many branches, consider using if/else instead!")
           while v.intVal <= e[1].intVal:
             gen(p, v, cond)
             lineF(p, "case $1:$n", [cond.rdLoc])
@@ -992,13 +1049,17 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
   var a, b: TCompRes
   var xtyp = mapType(p, x.typ)
 
+  # disable `[]=` for cstring
+  if x.kind == nkBracketExpr and x.len >= 2 and x[0].typ.skipTypes(abstractInst).kind == tyCString:
+    localError(p.config, x.info, "cstring doesn't support `[]=` operator")
+
   gen(p, x, a)
   genLineDir(p, y)
   gen(p, y, b)
 
   # we don't care if it's an etyBaseIndex (global) of a string, it's
   # still a string that needs to be copied properly:
-  if x.typ.skipTypes(abstractInst).kind in {tySequence, tyOpt, tyString}:
+  if x.typ.skipTypes(abstractInst).kind in {tySequence, tyString}:
     xtyp = etySeq
   case xtyp
   of etySeq:
@@ -1009,7 +1070,7 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
       lineF(p, "$1 = nimCopy(null, $2, $3);$n",
                [a.rdLoc, b.res, genTypeInfo(p, y.typ)])
   of etyObject:
-    if x.typ.kind == tyVar or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
+    if x.typ.kind in {tyVar} or (needsNoCopy(p, y) and needsNoCopy(p, x)) or noCopyNeeded:
       lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
     else:
       useMagic(p, "nimCopy")
@@ -1022,8 +1083,15 @@ proc genAsgnAux(p: PProc, x, y: PNode, noCopyNeeded: bool) =
         lineF(p, "var $1 = $4; $2 = $1[0]; $3 = $1[1];$n", [tmp, a.address, a.res, b.rdLoc])
       elif b.typ == etyBaseIndex:
         lineF(p, "$# = [$#, $#];$n", [a.res, b.address, b.res])
+      elif b.typ == etyNone:
+        internalAssert p.config, b.address == nil
+        lineF(p, "$# = [$#, 0];$n", [a.address, b.res])
+      elif x.typ.kind == tyVar and y.typ.kind == tyPtr:
+        lineF(p, "$# = [$#, $#];$n", [a.res, b.address, b.res])
+        lineF(p, "$1 = $2;$n", [a.address, b.res])
+        lineF(p, "$1 = $2;$n", [a.rdLoc, b.rdLoc])
       else:
-        internalError(p.config, x.info, "genAsgn")
+        internalError(p.config, x.info, $("genAsgn", b.typ, a.typ))
     else:
       lineF(p, "$1 = $2; $3 = $4;$n", [a.address, b.address, a.res, b.res])
   else:
@@ -1040,7 +1108,7 @@ proc genFastAsgn(p: PProc, n: PNode) =
   # See bug #5933. So we try to be more compatible with the C backend semantics
   # here for 'shallowCopy'. This is an educated guess and might require further
   # changes later:
-  let noCopy = n[0].typ.skipTypes(abstractInst).kind in {tySequence, tyOpt, tyString}
+  let noCopy = n[0].typ.skipTypes(abstractInst).kind in {tySequence, tyString}
   genAsgnAux(p, n[0], n[1], noCopyNeeded=noCopy)
 
 proc genSwap(p: PProc, n: PNode) =
@@ -1141,7 +1209,7 @@ proc genCheckedFieldOp(p: PProc, n: PNode, addrTyp: PType, r: var TCompRes) =
 
   useMagic(p, "raiseFieldError")
   useMagic(p, "makeNimstrLit")
-  let msg = genFieldError(field, disc)
+  let msg = genFieldDefect(field, disc)
   lineF(p, "if ($1[$2.$3]$4undefined) { raiseFieldError(makeNimstrLit($5)); }$n",
     setx.res, tmp, disc.loc.r, if negCheck: ~"!==" else: ~"===",
     makeJSString(msg))
@@ -1171,9 +1239,13 @@ proc genArrayAddr(p: PProc, n: PNode, r: var TCompRes) =
     first = firstOrd(p.config, typ[0])
   if optBoundsCheck in p.options:
     useMagic(p, "chckIndx")
-    r.res = "chckIndx($1, $2, ($3 != null ? $3.length : 0)+$2-1)-$2" % [b.res, rope(first), tmp]
+    if first == 0: # save a couple chars
+      r.res = "chckIndx($1, 0, ($2).length-1)" % [b.res, tmp]
+    else:
+      r.res = "chckIndx($1, $2, ($3).length+($2)-1)-($2)" % [
+        b.res, rope(first), tmp]
   elif first != 0:
-    r.res = "($1)-$2" % [b.res, rope(first)]
+    r.res = "($1)-($2)" % [b.res, rope(first)]
   else:
     r.res = b.res
   r.kind = resExpr
@@ -1219,6 +1291,10 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
     let s = n[0].sym
     if s.loc.r == nil: internalError(p.config, n.info, "genAddr: 3")
     case s.kind
+    of skParam:
+      r.res = s.loc.r
+      r.address = nil
+      r.typ = etyNone
     of skVar, skLet, skResult:
       r.kind = resExpr
       let jsType = mapType(p, n.typ)
@@ -1240,7 +1316,7 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
         # 'var openArray' for instance produces an 'addr' but this is harmless:
         gen(p, n[0], r)
         #internalError(p.config, n.info, "genAddr: 4 " & renderTree(n))
-    else: internalError(p.config, n.info, "genAddr: 2")
+    else: internalError(p.config, n.info, $("genAddr: 2", s.kind))
   of nkCheckedFieldExpr:
     genCheckedFieldOp(p, n[0], n.typ, r)
   of nkDotExpr:
@@ -1263,7 +1339,12 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
   of nkObjDownConv:
     gen(p, n[0], r)
   of nkHiddenDeref:
-    gen(p, n[0][0], r)
+    gen(p, n[0], r)
+  of nkHiddenAddr:
+    gen(p, n[0], r)
+  of nkStmtListExpr:
+    if n.len == 1: gen(p, n[0], r)
+    else: internalError(p.config, n[0].info, "genAddr for complex nkStmtListExpr")
   else: internalError(p.config, n[0].info, "genAddr: " & $n[0].kind)
 
 proc attachProc(p: PProc; content: Rope; s: PSym) =
@@ -1634,14 +1715,16 @@ proc createVar(p: PProc, typ: PType, indirect: bool): Rope =
   of tyObject:
     var initList: Rope
     createObjInitList(p, t, initIntSet(), initList)
-    result = ("{$1}") % [initList]
+    result = ("({$1})") % [initList]
     if indirect: result = "[$1]" % [result]
   of tyVar, tyPtr, tyLent, tyRef, tyPointer:
     if mapType(p, t) == etyBaseIndex:
       result = putToSeq("[null, 0]", indirect)
     else:
       result = putToSeq("null", indirect)
-  of tySequence, tyOpt, tyString, tyCString, tyProc:
+  of tySequence, tyString:
+    result = putToSeq("[]", indirect)
+  of tyCString, tyProc:
     result = putToSeq("null", indirect)
   of tyStatic:
     if t.n != nil:
@@ -1662,6 +1745,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
     varCode: string
     varName = mangleName(p.module, v)
     useReloadingGuard = sfGlobal in v.flags and p.config.hcrOn
+    useGlobalPragmas = sfGlobal in v.flags and ({sfPure, sfThread} * v.flags != {})
 
   if v.constraint.isNil:
     if useReloadingGuard:
@@ -1669,6 +1753,10 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
       lineF(p, "if ($1 === undefined) {$n", varName)
       varCode = $varName
       inc p.extraIndent
+    elif useGlobalPragmas:
+      lineF(p, "if (globalThis.$1 === undefined) {$n", varName)
+      varCode = $varName
+      inc p.extraIndent
     else:
       varCode = "var $2"
   else:
@@ -1721,7 +1809,7 @@ proc genVarInit(p: PProc, v: PSym, n: PNode) =
     else:
       line(p, runtimeFormat(varCode & " = $3;$n", [returnType, v.loc.r, s]))
 
-  if useReloadingGuard:
+  if useReloadingGuard or useGlobalPragmas:
     dec p.extraIndent
     lineF(p, "}$n")
 
@@ -1773,7 +1861,7 @@ proc genNewSeq(p: PProc, n: PNode) =
     x.rdLoc, y.rdLoc, createVar(p, t, false)])
 
 proc genOrd(p: PProc, n: PNode, r: var TCompRes) =
-  case skipTypes(n[1].typ, abstractVar).kind
+  case skipTypes(n[1].typ, abstractVar + abstractRange).kind
   of tyEnum, tyInt..tyUInt64, tyChar: gen(p, n[1], r)
   of tyBool: unaryExpr(p, n, r, "", "($1 ? 1:0)")
   else: internalError(p.config, n.info, "genOrd")
@@ -1848,6 +1936,7 @@ proc genRepr(p: PProc, n: PNode, r: var TCompRes) =
     genReprAux(p, n, r, "reprJSONStringify")
   else:
     genReprAux(p, n, r, "reprAny", genTypeInfo(p, t))
+  r.kind = resExpr
 
 proc genOf(p: PProc, n: PNode, r: var TCompRes) =
   var x: TCompRes
@@ -1872,7 +1961,7 @@ proc genReset(p: PProc, n: PNode) =
   if x.typ == etyBaseIndex:
     lineF(p, "$1 = null, $2 = 0;$n", [x.address, x.res])
   else:
-    let (a, tmp) = maybeMakeTemp(p, n[1], x)
+    let (a, tmp) = maybeMakeTempAssignable(p, n[1], x)
     lineF(p, "$1 = genericReset($3, $2);$n", [a,
                   genTypeInfo(p, n[1].typ), tmp])
 
@@ -1898,37 +1987,33 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
   of mSwap: genSwap(p, n)
   of mAppendStrCh:
     binaryExpr(p, n, r, "addChar",
-        "if ($1 != null) { addChar($3, $2); } else { $3 = [$2]; }")
+        "addChar($1, $2);")
   of mAppendStrStr:
     var lhs, rhs: TCompRes
     gen(p, n[1], lhs)
     gen(p, n[2], rhs)
 
-    let rhsIsLit = n[2].kind in nkStrKinds
-    let (a, tmp) = maybeMakeTemp(p, n[1], lhs)
     if skipTypes(n[1].typ, abstractVarRange).kind == tyCString:
-      r.res = "if ($1 != null) { $4 += $2; } else { $4 = $2$3; }" % [
-        a, rhs.rdLoc, if rhsIsLit: nil else: ~".slice()", tmp]
+      r.res = "$1 += $2;" % [lhs.rdLoc, rhs.rdLoc]
     else:
-      r.res = "if ($1 != null) { $4 = ($4).concat($2); } else { $4 = $2$3; }" % [
-          lhs.rdLoc, rhs.rdLoc, if rhsIsLit: nil else: ~".slice()", tmp]
+      let (a, tmp) = maybeMakeTemp(p, n[1], lhs)
+      r.res = "$1.push.apply($3, $2);" % [a, rhs.rdLoc, tmp]
     r.kind = resExpr
   of mAppendSeqElem:
     var x, y: TCompRes
     gen(p, n[1], x)
     gen(p, n[2], y)
-    let (a, tmp) = maybeMakeTemp(p, n[1], x)
     if mapType(n[2].typ) == etyBaseIndex:
       let c = "[$1, $2]" % [y.address, y.res]
-      r.res = "if ($1 != null) { $3.push($2); } else { $3 = [$2]; }" % [a, c, tmp]
+      r.res = "$1.push($2);" % [x.rdLoc, c]
     elif needsNoCopy(p, n[2]):
-      r.res = "if ($1 != null) { $3.push($2); } else { $3 = [$2]; }" % [a, y.rdLoc, tmp]
+      r.res = "$1.push($2);" % [x.rdLoc, y.rdLoc]
     else:
       useMagic(p, "nimCopy")
       let c = getTemp(p, defineInLocals=false)
       lineF(p, "var $1 = nimCopy(null, $2, $3);$n",
             [c, y.rdLoc, genTypeInfo(p, n[2].typ)])
-      r.res = "if ($1 != null) { $3.push($2); } else { $3 = [$2]; }" % [a, c, tmp]
+      r.res = "$1.push($2);" % [x.rdLoc, c]
     r.kind = resExpr
   of mConStrStr:
     genConStrStr(p, n, r)
@@ -1960,23 +2045,23 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
   of mDestroy: discard "ignore calls to the default destructor"
   of mOrd: genOrd(p, n, r)
   of mLengthStr, mLengthSeq, mLengthOpenArray, mLengthArray:
-    unaryExpr(p, n, r, "", "($1 != null ? $2.length : 0)")
+    unaryExpr(p, n, r, "", "($1).length")
   of mHigh:
-    unaryExpr(p, n, r, "", "($1 != null ? ($2.length-1) : -1)")
+    unaryExpr(p, n, r, "", "(($1).length-1)")
   of mInc:
     if n[1].typ.skipTypes(abstractRange).kind in {tyUInt..tyUInt64}:
       binaryUintExpr(p, n, r, "+", true)
     else:
       if optOverflowCheck notin p.options: binaryExpr(p, n, r, "", "$1 += $2")
-      else: binaryExpr(p, n, r, "addInt", "$1 = addInt($3, $2)")
+      else: binaryExpr(p, n, r, "addInt", "$1 = addInt($3, $2)", true)
   of ast.mDec:
     if n[1].typ.skipTypes(abstractRange).kind in {tyUInt..tyUInt64}:
       binaryUintExpr(p, n, r, "-", true)
     else:
       if optOverflowCheck notin p.options: binaryExpr(p, n, r, "", "$1 -= $2")
-      else: binaryExpr(p, n, r, "subInt", "$1 = subInt($3, $2)")
+      else: binaryExpr(p, n, r, "subInt", "$1 = subInt($3, $2)", true)
   of mSetLengthStr:
-    binaryExpr(p, n, r, "mnewString", "($1 == null ? $3 = mnewString($2) : $3.length = $2)")
+    binaryExpr(p, n, r, "mnewString", "($1.length = $2)")
   of mSetLengthSeq:
     var x, y: TCompRes
     gen(p, n[1], x)
@@ -1984,8 +2069,7 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
     let t = skipTypes(n[1].typ, abstractVar)[0]
     let (a, tmp) = maybeMakeTemp(p, n[1], x)
     let (b, tmp2) = maybeMakeTemp(p, n[2], y)
-    r.res = """if ($1 === null) $4 = [];
-               if ($4.length < $2) { for (var i=$4.length;i<$5;++i) $4.push($3); }
+    r.res = """if ($1.length < $2) { for (var i=$4.length;i<$5;++i) $4.push($3); }
                else { $4.length = $5; }""" % [a, b, createVar(p, t, false), tmp, tmp2]
     r.kind = resExpr
   of mCard: unaryExpr(p, n, r, "SetCard", "SetCard($1)")
@@ -2124,11 +2208,17 @@ proc genConv(p: PProc, n: PNode, r: var TCompRes) =
   if dest.kind == src.kind:
     # no-op conversion
     return
-  case dest.kind:
-  of tyBool:
+  let toInt = (dest.kind in tyInt..tyInt32)
+  let fromInt = (src.kind in tyInt..tyInt32)
+  let toUint = (dest.kind in tyUInt..tyUInt32)
+  let fromUint = (src.kind in tyUInt..tyUInt32)
+  if toUint and (fromInt or fromUint):
+    let trimmer = unsignedTrimmer(dest.size)
+    r.res = "($1 $2)" % [r.res, trimmer]
+  elif dest.kind == tyBool:
     r.res = "(!!($1))" % [r.res]
     r.kind = resExpr
-  of tyInt:
+  elif toInt:
     r.res = "(($1)|0)" % [r.res]
   else:
     # TODO: What types must we handle here?
@@ -2197,7 +2287,7 @@ proc genProcBody(p: PProc, prc: PSym): Rope =
   if hasFrameInfo(p):
     result = frameCreate(p,
               makeJSString(prc.owner.name.s & '.' & prc.name.s),
-              makeJSString(toFilename(p.config, prc.info)))
+              makeJSString(toFilenameOption(p.config, prc.info.fileIndex, foStacktrace)))
   else:
     result = nil
   if p.beforeRetNeeded:
@@ -2326,8 +2416,7 @@ proc genCast(p: PProc, n: PNode, r: var TCompRes) =
     r.res = "($1 $2)" % [r.res, trimmer]
   elif toInt:
     if fromInt:
-      let trimmer = unsignedTrimmer(dest.size)
-      r.res = "($1 $2)" % [r.res, trimmer]
+      return
     elif fromUint:
       if src.size == 4 and dest.size == 4:
         # XXX prevent multi evaluations
@@ -2353,7 +2442,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
   if r.kind != resCallee: r.kind = resNone
   #r.address = nil
   r.res = nil
-  
+
   case n.kind
   of nkSym:
     genSym(p, n, r)
@@ -2408,7 +2497,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
         n.len >= 1:
       genInfixCall(p, n, r)
     else:
-      genCall(p, n, r)    
+      genCall(p, n, r)
   of nkClosure: gen(p, n[0], r)
   of nkCurly: genSetConstr(p, n, r)
   of nkBracket: genArrayConstr(p, n, r)
@@ -2465,12 +2554,16 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
     if n[0].kind != nkEmpty:
       genLineDir(p, n)
       gen(p, n[0], r)
+      r.res = "var _ = " & r.res
   of nkAsmStmt: genAsmOrEmitStmt(p, n)
   of nkTryStmt, nkHiddenTryStmt: genTry(p, n, r)
   of nkRaiseStmt: genRaiseStmt(p, n)
-  of nkTypeSection, nkCommentStmt, nkIteratorDef, nkIncludeStmt,
+  of nkTypeSection, nkCommentStmt, nkIncludeStmt,
      nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt,
      nkFromStmt, nkTemplateDef, nkMacroDef, nkStaticStmt: discard
+  of nkIteratorDef:
+    if n[0].sym.typ.callConv == TCallingConvention.ccClosure:
+      globalError(p.config, n.info, "Closure iterators are not supported by JS backend!")
   of nkPragma: genPragma(p, n)
   of nkProcDef, nkFuncDef, nkMethodDef, nkConverterDef:
     var s = n[namePos].sym
@@ -2478,7 +2571,7 @@ proc gen(p: PProc, n: PNode, r: var TCompRes) =
       genSym(p, n[namePos], r)
       r.res = nil
   of nkGotoState, nkState:
-    internalError(p.config, n.info, "first class iterators not implemented")
+    globalError(p.config, n.info, "First class iterators not implemented")
   of nkPragmaBlock: gen(p, n.lastSon, r)
   of nkComesFrom:
     discard "XXX to implement for better stack traces"
@@ -2492,20 +2585,14 @@ proc newModule(g: ModuleGraph; module: PSym): BModule =
     g.backend = newGlobals()
   result.graph = g
   result.config = g.config
+  if sfSystemModule in module.flags:
+    PGlobals(g.backend).inSystem = true
 
 proc genHeader(): Rope =
   result = rope("""/* Generated by the Nim Compiler v$1 */
     var framePtr = null;
     var excHandler = 0;
     var lastJSError = null;
-    if (typeof Int8Array === 'undefined') Int8Array = Array;
-    if (typeof Int16Array === 'undefined') Int16Array = Array;
-    if (typeof Int32Array === 'undefined') Int32Array = Array;
-    if (typeof Uint8Array === 'undefined') Uint8Array = Array;
-    if (typeof Uint16Array === 'undefined') Uint16Array = Array;
-    if (typeof Uint32Array === 'undefined') Uint32Array = Array;
-    if (typeof Float32Array === 'undefined') Float32Array = Array;
-    if (typeof Float64Array === 'undefined') Float64Array = Array;
   """.unindent.format(VersionAsString))
 
 proc addHcrInitGuards(p: PProc, n: PNode,
@@ -2535,7 +2622,7 @@ proc genModule(p: PProc, n: PNode) =
   if optStackTrace in p.options:
     p.body.add(frameCreate(p,
         makeJSString("module " & p.module.module.name.s),
-        makeJSString(toFilename(p.config, p.module.module.info))))
+        makeJSString(toFilenameOption(p.config, p.module.module.info.fileIndex, foStacktrace))))
   var transformedN = transformStmt(p.module.graph, p.module.module, n)
   if sfInjectDestructors in p.module.module.flags:
     transformedN = injectDestructorCalls(p.module.graph, p.module.module, transformedN)
@@ -2565,7 +2652,7 @@ proc myProcess(b: PPassContext, n: PNode): PNode =
   if passes.skipCodegen(m.config, n): return n
   if m.module == nil: internalError(m.config, n.info, "myProcess")
   let globals = PGlobals(m.graph.backend)
-  var p = newProc(globals, m, nil, m.module.options)
+  var p = newInitProc(globals, m)
   p.unique = globals.unique
   genModule(p, n)
   p.g.code.add(p.locals)
@@ -2575,14 +2662,14 @@ proc wholeCode(graph: ModuleGraph; m: BModule): Rope =
   let globals = PGlobals(graph.backend)
   for prc in globals.forwarded:
     if not globals.generatedSyms.containsOrIncl(prc.id):
-      var p = newProc(globals, m, nil, m.module.options)
+      var p = newInitProc(globals, m)
       attachProc(p, prc)
 
   var disp = generateMethodDispatchers(graph)
   for i in 0..<disp.len:
     let prc = disp[i].sym
     if not globals.generatedSyms.containsOrIncl(prc.id):
-      var p = newProc(globals, m, nil, m.module.options)
+      var p = newInitProc(globals, m)
       attachProc(p, prc)
 
   result = globals.typeInfo & globals.constants & globals.code
@@ -2602,6 +2689,8 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
   if sfMainModule in m.module.flags:
     for destructorCall in graph.globalDestructors:
       n.add destructorCall
+  if sfSystemModule in m.module.flags:
+    PGlobals(graph.backend).inSystem = false
   if passes.skipCodegen(m.config, n): return n
   if sfMainModule in m.module.flags:
     var code = genHeader() & wholeCode(graph, m)
@@ -2609,10 +2698,10 @@ proc myClose(graph: ModuleGraph; b: PPassContext, n: PNode): PNode =
 
     if optSourcemap in m.config.globalOptions:
       var map: SourceMap
-      (code, map) = genSourceMap($(code), mangled, outFile.string)
+      (code, map) = genSourceMap($(code), outFile.string)
       writeFile(outFile.string & ".map", $(%map))
     discard writeRopeIfNotEqual(code, outFile)
-    
+
 proc myOpen(graph: ModuleGraph; s: PSym): PPassContext =
   result = newModule(graph, s)
 

compiler/lambdalifting.nim

@@ -235,7 +235,7 @@ template isIterator*(owner: PSym): bool =
 proc liftingHarmful(conf: ConfigRef; owner: PSym): bool {.inline.} =
   ## lambda lifting can be harmful for JS-like code generators.
   let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
-  result = conf.cmd == cmdCompileToJS and not isCompileTime
+  result = conf.backend == backendJs and not isCompileTime
 
 proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; owner: PSym) =
   createTypeBoundOps(g, nil, refType.lastSon, info)
@@ -291,10 +291,11 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
   if illegalCapture(s):
     localError(g.config, n.info,
       ("'$1' is of type <$2> which cannot be captured as it would violate memory" &
-       " safety, declared here: $3") % [s.name.s, typeToString(s.typ), g.config$s.info])
-  elif owner.typ.callConv notin {ccClosure, ccDefault}:
+       " safety, declared here: $3; using '-d:nimWorkaround14447' helps in some cases") %
+      [s.name.s, typeToString(s.typ), g.config$s.info])
+  elif not (owner.typ.callConv == ccClosure or owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags):
     localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
-      [s.name.s, owner.name.s, CallingConvToStr[owner.typ.callConv]])
+      [s.name.s, owner.name.s, $owner.typ.callConv])
   incl(owner.typ.flags, tfCapturesEnv)
   owner.typ.callConv = ccClosure
 
@@ -821,7 +822,8 @@ proc semCaptureSym*(s, owner: PSym) =
       var o = owner.skipGenericOwner
       while o != nil and o.kind != skModule:
         if s.owner == o:
-          if owner.typ.callConv in {ccClosure, ccDefault} or owner.kind == skIterator:
+          if owner.typ.callConv == ccClosure or owner.kind == skIterator or
+             owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags:
             owner.typ.callConv = ccClosure
             propagateClosure(owner.skipGenericOwner, s.owner)
           else:
@@ -846,14 +848,14 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType): PNo
   fn.typ.callConv = oldCC
 
 proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PNode =
-  # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
+  # XXX backend == backendJs does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
 
   # However we can do lifting for the stuff which is *only* compiletime.
   let isCompileTime = sfCompileTime in fn.flags or fn.kind == skMacro
 
   if body.kind == nkEmpty or (
-      g.config.cmd == cmdCompileToJS and not isCompileTime) or
+      g.config.backend == backendJs and not isCompileTime) or
       fn.skipGenericOwner.kind != skModule:
 
     # ignore forward declaration:

compiler/layouter.nim

@@ -35,7 +35,7 @@ type
   Emitter* = object
     config: ConfigRef
     fid: FileIndex
-    lastTok: TTokType
+    lastTok: TokType
     inquote, lastTokWasTerse: bool
     semicolons: SemicolonKind
     col, lastLineNumber, lineSpan, indentLevel, indWidth*, inSection: int
@@ -327,7 +327,7 @@ const
 
   splitters = openPars + {tkComma, tkSemiColon} # do not add 'tkColon' here!
   oprSet = {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
-            tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor}
+            tkIsnot, tkNot, tkOf, tkAs, tkFrom, tkDotDot, tkAnd, tkOr, tkXor}
 
 template goodCol(col): bool = col >= em.maxLineLen div 2
 
@@ -402,7 +402,7 @@ proc endsInAlpha(em: Emitter): bool =
   while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection}: dec(i)
   result = if i >= 0: em.tokens[i].lastChar in SymChars+{'_'} else: false
 
-proc emitComment(em: var Emitter; tok: TToken; dontIndent: bool) =
+proc emitComment(em: var Emitter; tok: Token; dontIndent: bool) =
   var col = em.col
   let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA, tok.commentOffsetB)
   em.lineSpan = countNewlines(lit)
@@ -417,7 +417,7 @@ proc emitComment(em: var Emitter; tok: TToken; dontIndent: bool) =
       inc col
     emitMultilineComment(em, lit, col, dontIndent)
 
-proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
+proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
   template wasExportMarker(em): bool =
     em.kinds.len > 0 and em.kinds[^1] == ltExportMarker
 
@@ -452,6 +452,9 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
       if tok.tokType in openPars and tok.indent > em.indentStack[^1]:
         while em.indentStack[^1] < tok.indent:
           em.indentStack.add(em.indentStack[^1] + em.indWidth)
+      while em.indentStack[^1] > tok.indent:
+        discard em.indentStack.pop()
+
       # aka: we are in an expression context:
       let alignment = max(tok.indent - em.indentStack[^1], 0)
       em.indentLevel = alignment + em.indentStack.high * em.indWidth
@@ -491,7 +494,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
       wrSpace em
 
     if not em.inquote:
-      wr(em, TokTypeToStr[tok.tokType], ltKeyword)
+      wr(em, $tok.tokType, ltKeyword)
       if tok.tokType in {tkAnd, tkOr, tkIn, tkNotin}:
         rememberSplit(splitIn)
         wrSpace em
@@ -500,28 +503,28 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
       wr(em, tok.ident.s, ltIdent)
 
   of tkColon:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     wrSpace em
   of tkSemiColon, tkComma:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     rememberSplit(splitComma)
     wrSpace em
   of openPars:
     if tok.strongSpaceA > 0 and not em.endsInWhite and
         (not em.wasExportMarker or tok.tokType == tkCurlyDotLe):
       wrSpace em
-    wr(em, TokTypeToStr[tok.tokType], ltSomeParLe)
+    wr(em, $tok.tokType, ltSomeParLe)
     rememberSplit(splitParLe)
   of closedPars:
-    wr(em, TokTypeToStr[tok.tokType], ltSomeParRi)
+    wr(em, $tok.tokType, ltSomeParRi)
   of tkColonColon:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
   of tkDot:
     lastTokWasTerse = true
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
   of tkEquals:
     if not em.inquote and not em.endsInWhite: wrSpace(em)
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     if not em.inquote: wrSpace(em)
   of tkOpr, tkDotDot:
     if em.inquote or ((tok.strongSpaceA == 0 and tok.strongSpaceB == 0) and
@@ -541,7 +544,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
         wrSpace(em)
   of tkAccent:
     if not em.inquote and endsInAlpha(em): wrSpace(em)
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     em.inquote = not em.inquote
   of tkComment:
     if not preventComment:

compiler/lexer.nim

@@ -30,41 +30,49 @@ const
 # don't forget to update the 'highlite' module if these charsets should change
 
 type
-  TTokType* = enum
-    tkInvalid, tkEof,         # order is important here!
-    tkSymbol, # keywords:
-    tkAddr, tkAnd, tkAs, tkAsm,
-    tkBind, tkBlock, tkBreak, tkCase, tkCast,
-    tkConcept, tkConst, tkContinue, tkConverter,
-    tkDefer, tkDiscard, tkDistinct, tkDiv, tkDo,
-    tkElif, tkElse, tkEnd, tkEnum, tkExcept, tkExport,
-    tkFinally, tkFor, tkFrom, tkFunc,
-    tkIf, tkImport, tkIn, tkInclude, tkInterface,
-    tkIs, tkIsnot, tkIterator,
-    tkLet,
-    tkMacro, tkMethod, tkMixin, tkMod, tkNil, tkNot, tkNotin,
-    tkObject, tkOf, tkOr, tkOut,
-    tkProc, tkPtr, tkRaise, tkRef, tkReturn,
-    tkShl, tkShr, tkStatic,
-    tkTemplate,
-    tkTry, tkTuple, tkType, tkUsing,
-    tkVar, tkWhen, tkWhile, tkXor,
-    tkYield, # end of keywords
-    tkIntLit, tkInt8Lit, tkInt16Lit, tkInt32Lit, tkInt64Lit,
-    tkUIntLit, tkUInt8Lit, tkUInt16Lit, tkUInt32Lit, tkUInt64Lit,
-    tkFloatLit, tkFloat32Lit, tkFloat64Lit, tkFloat128Lit,
-    tkStrLit, tkRStrLit, tkTripleStrLit,
-    tkGStrLit, tkGTripleStrLit, tkCharLit, tkParLe, tkParRi, tkBracketLe,
-    tkBracketRi, tkCurlyLe, tkCurlyRi,
-    tkBracketDotLe, tkBracketDotRi, # [. and  .]
-    tkCurlyDotLe, tkCurlyDotRi, # {.  and  .}
-    tkParDotLe, tkParDotRi,   # (. and .)
-    tkComma, tkSemiColon,
-    tkColon, tkColonColon, tkEquals, tkDot, tkDotDot, tkBracketLeColon,
-    tkOpr, tkComment, tkAccent,
+  TokType* = enum
+    tkInvalid = "tkInvalid", tkEof = "[EOF]", # order is important here!
+    tkSymbol = "tkSymbol", # keywords:
+    tkAddr = "addr", tkAnd = "and", tkAs = "as", tkAsm = "asm",
+    tkBind = "bind", tkBlock = "block", tkBreak = "break", tkCase = "case", tkCast = "cast",
+    tkConcept = "concept", tkConst = "const", tkContinue = "continue", tkConverter = "converter",
+    tkDefer = "defer", tkDiscard = "discard", tkDistinct = "distinct", tkDiv = "div", tkDo = "do",
+    tkElif = "elif", tkElse = "else", tkEnd = "end", tkEnum = "enum", tkExcept = "except", tkExport = "export",
+    tkFinally = "finally", tkFor = "for", tkFrom = "from", tkFunc = "func",
+    tkIf = "if", tkImport = "import", tkIn = "in", tkInclude = "include", tkInterface = "interface",
+    tkIs = "is", tkIsnot = "isnot", tkIterator = "iterator",
+    tkLet = "let",
+    tkMacro = "macro", tkMethod = "method", tkMixin = "mixin", tkMod = "mod", tkNil = "nil", tkNot = "not", tkNotin = "notin",
+    tkObject = "object", tkOf = "of", tkOr = "or", tkOut = "out",
+    tkProc = "proc", tkPtr = "ptr", tkRaise = "raise", tkRef = "ref", tkReturn = "return",
+    tkShl = "shl", tkShr = "shr", tkStatic = "static",
+    tkTemplate = "template",
+    tkTry = "try", tkTuple = "tuple", tkType = "type", tkUsing = "using",
+    tkVar = "var", tkWhen = "when", tkWhile = "while", tkXor = "xor",
+    tkYield = "yield", # end of keywords
+
+    tkIntLit = "tkIntLit", tkInt8Lit = "tkInt8Lit", tkInt16Lit = "tkInt16Lit", 
+    tkInt32Lit = "tkInt32Lit", tkInt64Lit = "tkInt64Lit",
+    tkUIntLit = "tkUIntLit", tkUInt8Lit = "tkUInt8Lit", tkUInt16Lit = "tkUInt16Lit", 
+    tkUInt32Lit = "tkUInt32Lit", tkUInt64Lit = "tkUInt64Lit",
+    tkFloatLit = "tkFloatLit", tkFloat32Lit = "tkFloat32Lit",
+    tkFloat64Lit = "tkFloat64Lit", tkFloat128Lit = "tkFloat128Lit",
+    tkStrLit = "tkStrLit", tkRStrLit = "tkRStrLit", tkTripleStrLit = "tkTripleStrLit",
+    tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit", 
+    
+    tkParLe = "(", tkParRi = ")", tkBracketLe = "[",
+    tkBracketRi = "]", tkCurlyLe = "{", tkCurlyRi = "}",
+    tkBracketDotLe = "[.", tkBracketDotRi = ".]",
+    tkCurlyDotLe = "{.", tkCurlyDotRi = ".}",
+    tkParDotLe = "(.", tkParDotRi = ".)",
+    tkComma = ",", tkSemiColon = ";",
+    tkColon = ":", tkColonColon = "::", tkEquals = "=", 
+    tkDot = ".", tkDotDot = "..", tkBracketLeColon = "[:",
+    tkOpr, tkComment, tkAccent = "`",
+    # these are fake tokens used by renderer.nim
     tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
 
-  TTokTypes* = set[TTokType]
+  TokTypes* = set[TokType]
 
 const
   weakTokens = {tkComma, tkSemiColon, tkColon,
@@ -73,53 +81,21 @@ const
     # tokens that should not be considered for previousToken
   tokKeywordLow* = succ(tkSymbol)
   tokKeywordHigh* = pred(tkIntLit)
-  TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]",
-    "tkSymbol",
-    "addr", "and", "as", "asm",
-    "bind", "block", "break", "case", "cast",
-    "concept", "const", "continue", "converter",
-    "defer", "discard", "distinct", "div", "do",
-    "elif", "else", "end", "enum", "except", "export",
-    "finally", "for", "from", "func", "if",
-    "import", "in", "include", "interface", "is", "isnot", "iterator",
-    "let",
-    "macro", "method", "mixin", "mod",
-    "nil", "not", "notin", "object", "of", "or",
-    "out", "proc", "ptr", "raise", "ref", "return",
-    "shl", "shr", "static",
-    "template",
-    "try", "tuple", "type", "using",
-    "var", "when", "while", "xor",
-    "yield",
-    "tkIntLit", "tkInt8Lit", "tkInt16Lit", "tkInt32Lit", "tkInt64Lit",
-    "tkUIntLit", "tkUInt8Lit", "tkUInt16Lit", "tkUInt32Lit", "tkUInt64Lit",
-    "tkFloatLit", "tkFloat32Lit", "tkFloat64Lit", "tkFloat128Lit",
-    "tkStrLit", "tkRStrLit",
-    "tkTripleStrLit", "tkGStrLit", "tkGTripleStrLit", "tkCharLit", "(",
-    ")", "[", "]", "{", "}", "[.", ".]", "{.", ".}", "(.", ".)",
-    ",", ";",
-    ":", "::", "=", ".", "..", "[:",
-    "tkOpr", "tkComment", "`",
-    "tkSpaces", "tkInfixOpr",
-    "tkPrefixOpr", "tkPostfixOpr"]
 
 type
-  TNumericalBase* = enum
+  NumericalBase* = enum
     base10,                   # base10 is listed as the first element,
                               # so that it is the correct default value
     base2, base8, base16
 
-  CursorPosition* {.pure.} = enum ## XXX remove this again
-    None, InToken, BeforeToken, AfterToken
-
-  TToken* = object            # a Nim token
-    tokType*: TTokType        # the type of the token
+  Token* = object             # a Nim token
+    tokType*: TokType         # the type of the token
     indent*: int              # the indentation; != -1 if the token has been
                               # preceded with indentation
     ident*: PIdent            # the parsed identifier
     iNumber*: BiggestInt      # the parsed integer literal
     fNumber*: BiggestFloat    # the parsed floating point literal
-    base*: TNumericalBase     # the numerical base; only valid for int
+    base*: NumericalBase      # the numerical base; only valid for int
                               # or float literals
     strongSpaceA*: int8       # leading spaces of an operator
     strongSpaceB*: int8       # trailing spaces of an operator
@@ -127,26 +103,25 @@ type
                               # documentation comments are here too
     line*, col*: int
     when defined(nimpretty):
-      offsetA*, offsetB*: int   # used for pretty printing so that literals
-                                # like 0b01 or  r"\L" are unaffected
+      offsetA*, offsetB*: int # used for pretty printing so that literals
+                              # like 0b01 or  r"\L" are unaffected
       commentOffsetA*, commentOffsetB*: int
 
-  TErrorHandler* = proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string)
-  TLexer* = object of TBaseLexer
+  ErrorHandler* = proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string)
+  Lexer* = object of TBaseLexer
     fileIdx*: FileIndex
     indentAhead*: int         # if > 0 an indentation has already been read
                               # this is needed because scanning comments
                               # needs so much look-ahead
     currLineIndent*: int
     strongSpaces*, allowTabs*: bool
-    cursor*: CursorPosition
-    errorHandler*: TErrorHandler
+    errorHandler*: ErrorHandler
     cache*: IdentCache
     when defined(nimsuggest):
       previousToken: TLineInfo
     config*: ConfigRef
 
-proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
+proc getLineInfo*(L: Lexer, tok: Token): TLineInfo {.inline.} =
   result = newLineInfo(L.fileIdx, tok.line, tok.col)
   when defined(nimpretty):
     result.offsetA = tok.offsetA
@@ -154,8 +129,8 @@ proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
     result.commentOffsetA = tok.commentOffsetA
     result.commentOffsetB = tok.commentOffsetB
 
-proc isKeyword*(kind: TTokType): bool =
-  result = (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)
+proc isKeyword*(kind: TokType): bool =
+  (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)
 
 template ones(n): untyped = ((1 shl n)-1) # for utf-8 conversion
 
@@ -169,28 +144,26 @@ proc isNimIdentifier*(s: string): bool =
       inc(i)
     result = true
 
-proc `$`*(tok: TToken): string =
+proc `$`*(tok: Token): string =
   case tok.tokType
-  of tkIntLit..tkInt64Lit: result = $tok.iNumber
-  of tkFloatLit..tkFloat64Lit: result = $tok.fNumber
-  of tkInvalid, tkStrLit..tkCharLit, tkComment: result = tok.literal
-  of tkParLe..tkColon, tkEof, tkAccent:
-    result = TokTypeToStr[tok.tokType]
+  of tkIntLit..tkInt64Lit: $tok.iNumber
+  of tkFloatLit..tkFloat64Lit: $tok.fNumber
+  of tkInvalid, tkStrLit..tkCharLit, tkComment: tok.literal
+  of tkParLe..tkColon, tkEof, tkAccent: $tok.tokType
   else:
     if tok.ident != nil:
-      result = tok.ident.s
+      tok.ident.s
     else:
-      result = ""
+      ""
 
-proc prettyTok*(tok: TToken): string =
-  if isKeyword(tok.tokType): result = "keyword " & tok.ident.s
-  else: result = $tok
+proc prettyTok*(tok: Token): string =
+  if isKeyword(tok.tokType): "keyword " & tok.ident.s
+  else: $tok
 
-proc printTok*(conf: ConfigRef; tok: TToken) =
-  msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" &
-      TokTypeToStr[tok.tokType] & " " & $tok)
+proc printTok*(conf: ConfigRef; tok: Token) =
+  msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" & $tok.tokType & " " & $tok)
 
-proc initToken*(L: var TToken) =
+proc initToken*(L: var Token) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
@@ -203,7 +176,7 @@ proc initToken*(L: var TToken) =
     L.commentOffsetA = 0
     L.commentOffsetB = 0
 
-proc fillToken(L: var TToken) =
+proc fillToken(L: var Token) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
@@ -216,7 +189,7 @@ proc fillToken(L: var TToken) =
     L.commentOffsetA = 0
     L.commentOffsetB = 0
 
-proc openLexer*(lex: var TLexer, fileIdx: FileIndex, inputstream: PLLStream;
+proc openLexer*(lex: var Lexer, fileIdx: FileIndex, inputstream: PLLStream;
                  cache: IdentCache; config: ConfigRef) =
   openBaseLexer(lex, inputstream)
   lex.fileIdx = fileIdx
@@ -228,36 +201,36 @@ proc openLexer*(lex: var TLexer, fileIdx: FileIndex, inputstream: PLLStream;
     lex.previousToken.fileIndex = fileIdx
   lex.config = config
 
-proc openLexer*(lex: var TLexer, filename: AbsoluteFile, inputstream: PLLStream;
+proc openLexer*(lex: var Lexer, filename: AbsoluteFile, inputstream: PLLStream;
                 cache: IdentCache; config: ConfigRef) =
   openLexer(lex, fileInfoIdx(config, filename), inputstream, cache, config)
 
-proc closeLexer*(lex: var TLexer) =
+proc closeLexer*(lex: var Lexer) =
   if lex.config != nil:
     inc(lex.config.linesCompiled, lex.lineNumber)
   closeBaseLexer(lex)
 
-proc getLineInfo(L: TLexer): TLineInfo =
+proc getLineInfo(L: Lexer): TLineInfo =
   result = newLineInfo(L.fileIdx, L.lineNumber, getColNumber(L, L.bufpos))
 
-proc dispMessage(L: TLexer; info: TLineInfo; msg: TMsgKind; arg: string) =
+proc dispMessage(L: Lexer; info: TLineInfo; msg: TMsgKind; arg: string) =
   if L.errorHandler.isNil:
     msgs.message(L.config, info, msg, arg)
   else:
     L.errorHandler(L.config, info, msg, arg)
 
-proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "") =
+proc lexMessage*(L: Lexer, msg: TMsgKind, arg = "") =
   L.dispMessage(getLineInfo(L), msg, arg)
 
-proc lexMessageTok*(L: TLexer, msg: TMsgKind, tok: TToken, arg = "") =
+proc lexMessageTok*(L: Lexer, msg: TMsgKind, tok: Token, arg = "") =
   var info = newLineInfo(L.fileIdx, tok.line, tok.col)
   L.dispMessage(info, msg, arg)
 
-proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
+proc lexMessagePos(L: var Lexer, msg: TMsgKind, pos: int, arg = "") =
   var info = newLineInfo(L.fileIdx, L.lineNumber, pos - L.lineStart)
   L.dispMessage(info, msg, arg)
 
-proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
+proc matchTwoChars(L: Lexer, first: char, second: set[char]): bool =
   result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
 
 template tokenBegin(tok, pos) {.dirty.} =
@@ -271,7 +244,6 @@ template tokenEnd(tok, pos) {.dirty.} =
     let colB = getColNumber(L, pos)+1
     if L.fileIdx == L.config.m.trackPos.fileIndex and L.config.m.trackPos.col in colA..colB and
         L.lineNumber == L.config.m.trackPos.line.int and L.config.ideCmd in {ideSug, ideCon}:
-      L.cursor = CursorPosition.InToken
       L.config.m.trackPos.col = colA.int16
     colA = 0
   when defined(nimpretty):
@@ -296,23 +268,22 @@ template tokenEndPrevious(tok, pos) =
     let colB = getColNumber(L, pos)
     if L.fileIdx == L.config.m.trackPos.fileIndex and L.config.m.trackPos.col in colA..colB and
         L.lineNumber == L.config.m.trackPos.line.int and L.config.ideCmd in {ideSug, ideCon}:
-      L.cursor = CursorPosition.BeforeToken
       L.config.m.trackPos = L.previousToken
       L.config.m.trackPosAttached = true
     colA = 0
   when defined(nimpretty):
     tok.offsetB = L.offsetBase + pos
 
-template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
+template eatChar(L: var Lexer, t: var Token, replacementChar: char) =
   t.literal.add(replacementChar)
   inc(L.bufpos)
 
-template eatChar(L: var TLexer, t: var TToken) =
+template eatChar(L: var Lexer, t: var Token) =
   t.literal.add(L.buf[L.bufpos])
   inc(L.bufpos)
 
-proc getNumber(L: var TLexer, result: var TToken) =
-  proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]): Natural =
+proc getNumber(L: var Lexer, result: var Token) =
+  proc matchUnderscoreChars(L: var Lexer, tok: var Token, chars: set[char]): Natural =
     var pos = L.bufpos              # use registers for pos, buf
     result = 0
     while true:
@@ -332,19 +303,19 @@ proc getNumber(L: var TLexer, result: var TToken) =
         inc(pos)
     L.bufpos = pos
 
-  proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
+  proc matchChars(L: var Lexer, tok: var Token, chars: set[char]) =
     var pos = L.bufpos              # use registers for pos, buf
     while L.buf[pos] in chars:
       tok.literal.add(L.buf[pos])
       inc(pos)
     L.bufpos = pos
 
-  proc lexMessageLitNum(L: var TLexer, msg: string, startpos: int, msgKind = errGenerated) =
+  proc lexMessageLitNum(L: var Lexer, msg: string, startpos: int, msgKind = errGenerated) =
     # Used to get slightly human friendlier err messages.
     const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O',
       'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
     var msgPos = L.bufpos
-    var t: TToken
+    var t: Token
     t.literal = ""
     L.bufpos = startpos # Use L.bufpos as pos because of matchChars
     matchChars(L, t, literalishChars)
@@ -571,7 +542,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
         try:
           len = parseBiggestUInt(result.literal, iNumber)
         except ValueError:
-          raise newException(OverflowError, "number out of range: " & $result.literal)
+          raise newException(OverflowDefect, "number out of range: " & $result.literal)
         if len != result.literal.len:
           raise newException(ValueError, "invalid integer: " & $result.literal)
         result.iNumber = cast[int64](iNumber)
@@ -581,7 +552,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
         try:
           len = parseBiggestInt(result.literal, iNumber)
         except ValueError:
-          raise newException(OverflowError, "number out of range: " & $result.literal)
+          raise newException(OverflowDefect, "number out of range: " & $result.literal)
         if len != result.literal.len:
           raise newException(ValueError, "invalid integer: " & $result.literal)
         result.iNumber = iNumber
@@ -607,12 +578,12 @@ proc getNumber(L: var TLexer, result: var TToken) =
 
   except ValueError:
     lexMessageLitNum(L, "invalid number: '$1'", startpos)
-  except OverflowError, RangeError:
+  except OverflowDefect, RangeDefect:
     lexMessageLitNum(L, "number out of range: '$1'", startpos)
   tokenEnd(result, postPos-1)
   L.bufpos = postPos
 
-proc handleHexChar(L: var TLexer, xi: var int; position: range[0..4]) =
+proc handleHexChar(L: var Lexer, xi: var int; position: range[0..4]) =
   template invalid() =
     lexMessage(L, errGenerated,
       "expected a hex digit, but found: " & L.buf[L.bufpos] &
@@ -637,7 +608,7 @@ proc handleHexChar(L: var TLexer, xi: var int; position: range[0..4]) =
     # Need to progress for `nim check`
     inc(L.bufpos)
 
-proc handleDecChars(L: var TLexer, xi: var int) =
+proc handleDecChars(L: var Lexer, xi: var int) =
   while L.buf[L.bufpos] in {'0'..'9'}:
     xi = (xi * 10) + (ord(L.buf[L.bufpos]) - ord('0'))
     inc(L.bufpos)
@@ -680,16 +651,11 @@ proc addUnicodeCodePoint(s: var string, i: int) =
     s[pos+4] = chr(i shr 6 and ones(6) or 0b10_0000_00)
     s[pos+5] = chr(i and ones(6) or 0b10_0000_00)
 
-proc getEscapedChar(L: var TLexer, tok: var TToken) =
+proc getEscapedChar(L: var Lexer, tok: var Token) =
   inc(L.bufpos)               # skip '\'
   case L.buf[L.bufpos]
   of 'n', 'N':
-    if L.config.oldNewlines:
-      if tok.tokType == tkCharLit:
-        lexMessage(L, errGenerated, "\\n not allowed in character literal")
-      tok.literal.add(L.config.target.tnl)
-    else:
-      tok.literal.add('\L')
+    tok.literal.add('\L')
     inc(L.bufpos)
   of 'p', 'P':
     if tok.tokType == tkCharLit:
@@ -765,18 +731,13 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
     else: lexMessage(L, errGenerated, "invalid character constant")
   else: lexMessage(L, errGenerated, "invalid character constant")
 
-proc newString(s: cstring, len: int): string =
-  ## XXX, how come there is no support for this?
-  result = newString(len)
-  for i in 0..<len:
-    result[i] = s[i]
-
-proc handleCRLF(L: var TLexer, pos: int): int =
+proc handleCRLF(L: var Lexer, pos: int): int =
   template registerLine =
     let col = L.getColNumber(pos)
 
-    if col > MaxLineLength:
-      lexMessagePos(L, hintLineTooLong, pos)
+    when not defined(nimpretty):
+      if col > MaxLineLength:
+        lexMessagePos(L, hintLineTooLong, pos)
 
   case L.buf[pos]
   of CR:
@@ -793,7 +754,7 @@ type
     raw,
     generalized
 
-proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
+proc getString(L: var Lexer, tok: var Token, mode: StringMode) =
   var pos = L.bufpos
   var line = L.lineNumber         # save linenumber for better error message
   tokenBegin(tok, pos - ord(mode == raw))
@@ -859,7 +820,7 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
         inc(pos)
     L.bufpos = pos
 
-proc getCharacter(L: var TLexer, tok: var TToken) =
+proc getCharacter(L: var Lexer, tok: var Token) =
   tokenBegin(tok, L.bufpos)
   inc(L.bufpos)               # skip '
   var c = L.buf[L.bufpos]
@@ -876,7 +837,7 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
   tokenEndIgnore(tok, L.bufpos)
   inc(L.bufpos)               # skip '
 
-proc getSymbol(L: var TLexer, tok: var TToken) =
+proc getSymbol(L: var Lexer, tok: var Token) =
   var h: Hash = 0
   var pos = L.bufpos
   tokenBegin(tok, pos)
@@ -906,21 +867,21 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
       (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
     tok.tokType = tkSymbol
   else:
-    tok.tokType = TTokType(tok.ident.id + ord(tkSymbol))
+    tok.tokType = TokType(tok.ident.id + ord(tkSymbol))
     if suspicious and {optStyleHint, optStyleError} * L.config.globalOptions != {}:
       lintReport(L.config, getLineInfo(L), tok.ident.s.normalize, tok.ident.s)
   L.bufpos = pos
 
 
-proc endOperator(L: var TLexer, tok: var TToken, pos: int,
+proc endOperator(L: var Lexer, tok: var Token, pos: int,
                  hash: Hash) {.inline.} =
   var h = !$hash
   tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
   if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr
-  else: tok.tokType = TTokType(tok.ident.id - oprLow + ord(tkColon))
+  else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon))
   L.bufpos = pos
 
-proc getOperator(L: var TLexer, tok: var TToken) =
+proc getOperator(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tokenBegin(tok, pos)
   var h: Hash = 0
@@ -940,18 +901,15 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   if L.buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
-proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
+proc getPrecedence*(tok: Token): int =
   ## Calculates the precedence of the given token.
-  template considerStrongSpaces(x): untyped =
-    x + (if strongSpaces: 100 - tok.strongSpaceA.int*10 else: 0)
-
   case tok.tokType
   of tkOpr:
     let relevantChar = tok.ident.s[0]
 
     # arrow like?
     if tok.ident.s.len > 1 and tok.ident.s[^1] == '>' and
-      tok.ident.s[^2] in {'-', '~', '='}: return considerStrongSpaces(1)
+      tok.ident.s[^2] in {'-', '~', '='}: return 1
 
     template considerAsgn(value: untyped) =
       result = if tok.ident.s[^1] == '=': 1 else: value
@@ -967,15 +925,13 @@ proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
     of '?': result = 2
     else: considerAsgn(2)
   of tkDiv, tkMod, tkShl, tkShr: result = 9
-  of tkIn, tkNotin, tkIs, tkIsnot, tkOf, tkAs: result = 5
   of tkDotDot: result = 6
+  of tkIn, tkNotin, tkIs, tkIsnot, tkOf, tkAs, tkFrom: result = 5
   of tkAnd: result = 4
   of tkOr, tkXor, tkPtr, tkRef: result = 3
   else: return -10
-  result = considerStrongSpaces(result)
 
-
-proc newlineFollows*(L: TLexer): bool =
+proc newlineFollows*(L: Lexer): bool =
   var pos = L.bufpos
   while true:
     case L.buf[pos]
@@ -991,7 +947,7 @@ proc newlineFollows*(L: TLexer): bool =
     else:
       break
 
-proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
+proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
                           isDoc: bool) =
   var pos = start
   var toStrip = 0
@@ -1056,14 +1012,14 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
   when defined(nimpretty):
     tok.commentOffsetB = L.offsetBase + pos - 1
 
-proc scanComment(L: var TLexer, tok: var TToken) =
+proc scanComment(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tok.tokType = tkComment
   # iNumber contains the number of '\n' in the token
   tok.iNumber = 0
   assert L.buf[pos+1] == '#'
   when defined(nimpretty):
-    tok.commentOffsetA = L.offsetBase + pos - 1
+    tok.commentOffsetA = L.offsetBase + pos
 
   if L.buf[pos+2] == '[':
     skipMultiLineComment(L, tok, pos+3, true)
@@ -1106,7 +1062,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
   when defined(nimpretty):
     tok.commentOffsetB = L.offsetBase + pos - 1
 
-proc skip(L: var TLexer, tok: var TToken) =
+proc skip(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tokenBegin(tok, pos)
   tok.strongSpaceA = 0
@@ -1178,7 +1134,7 @@ proc skip(L: var TLexer, tok: var TToken) =
       tok.tokType = tkComment
       tok.indent = commentIndent
 
-proc rawGetTok*(L: var TLexer, tok: var TToken) =
+proc rawGetTok*(L: var Lexer, tok: var Token) =
   template atTokenEnd() {.dirty.} =
     when defined(nimsuggest):
       # we attach the cursor to the last *strong* token
@@ -1186,8 +1142,6 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
         L.previousToken.line = tok.line.uint16
         L.previousToken.col = tok.col.int16
 
-  when defined(nimsuggest):
-    L.cursor = CursorPosition.None
   fillToken(tok)
   if L.indentAhead >= 0:
     tok.indent = L.indentAhead
@@ -1258,7 +1212,6 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
         if L.fileIdx == L.config.m.trackPos.fileIndex and tok.col+1 == L.config.m.trackPos.col and
             tok.line == L.config.m.trackPos.line.int and L.config.ideCmd == ideSug:
           tok.tokType = tkDot
-          L.cursor = CursorPosition.InToken
           L.config.m.trackPos.col = tok.col.int16
           inc(L.bufpos)
           atTokenEnd()
@@ -1331,8 +1284,8 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
 
 proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
                      cache: IdentCache; config: ConfigRef): int =
-  var lex: TLexer
-  var tok: TToken
+  var lex: Lexer
+  var tok: Token
   initToken(tok)
   openLexer(lex, fileIdx, inputstream, cache, config)
   var prevToken = tkEof
@@ -1346,11 +1299,11 @@ proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
 
 proc getPrecedence*(ident: PIdent): int =
   ## assumes ident is binary operator already
-  var tok: TToken
+  var tok: Token
   initToken(tok)
   tok.ident = ident
   tok.tokType =
     if tok.ident.id in ord(tokKeywordLow) - ord(tkSymbol)..ord(tokKeywordHigh) - ord(tkSymbol):
-      TTokType(tok.ident.id + ord(tkSymbol))
+      TokType(tok.ident.id + ord(tkSymbol))
     else: tkOpr
-  getPrecedence(tok, false)
+  getPrecedence(tok)

compiler/liftdestructors.nim

@@ -10,9 +10,6 @@
 ## This module implements lifting for type-bound operations
 ## (``=sink``, ``=``, ``=destroy``, ``=deepCopy``).
 
-# Todo:
-# - use openArray instead of array to avoid over-specializations
-
 import modulegraphs, lineinfos, idents, ast, renderer, semdata,
   sighashes, lowerings, options, types, msgs, magicsys, tables
 
@@ -26,6 +23,9 @@ type
     fn: PSym
     asgnForType: PType
     recurse: bool
+    addMemReset: bool    # add wasMoved() call after destructor call
+    canRaise: bool
+    filterDiscriminator: PSym  # we generating destructor for case branch
     c: PContext # c can be nil, then we are called from lambdalifting!
 
 proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode)
@@ -40,6 +40,9 @@ proc at(a, i: PNode, elemType: PType): PNode =
   result[1] = i
   result.typ = elemType
 
+proc destructorOverriden(t: PType): bool =
+  t.attachedOps[attachedDestructor] != nil and sfOverriden in t.attachedOps[attachedDestructor].flags
+
 proc fillBodyTup(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   for i in 0..<t.len:
     let lit = lowerings.newIntLit(c.g, x.info, i)
@@ -60,9 +63,18 @@ proc newAsgnStmt(le, ri: PNode): PNode =
   result[0] = le
   result[1] = ri
 
+proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode =
+  result = newNodeI(nkCall, i.info)
+  result.add createMagic(g, name, magic).newSymNode
+  result.add i
+
 proc defaultOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   if c.kind in {attachedAsgn, attachedDeepCopy, attachedSink}:
     body.add newAsgnStmt(x, y)
+  elif c.kind == attachedDestructor and c.addMemReset:
+    let call = genBuiltin(c.g, mDefault, "default", x)
+    call.typ = t
+    body.add newAsgnStmt(x, call)
 
 proc genAddr(g: ModuleGraph; x: PNode): PNode =
   if x.kind == nkHiddenDeref:
@@ -72,14 +84,58 @@ proc genAddr(g: ModuleGraph; x: PNode): PNode =
     result = newNodeIT(nkHiddenAddr, x.info, makeVarType(x.typ.owner, x.typ))
     result.add x
 
-proc destructorCall(g: ModuleGraph; op: PSym; x: PNode): PNode =
-  result = newNodeIT(nkCall, x.info, op.typ[0])
-  result.add(newSymNode(op))
-  result.add genAddr(g, x)
+proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
+  result = newNodeI(nkWhileStmt, c.info, 2)
+  let cmp = genBuiltin(c.g, mLtI, "<", i)
+  cmp.add genLen(c.g, dest)
+  cmp.typ = getSysType(c.g, c.info, tyBool)
+  result[0] = cmp
+  result[1] = newNodeI(nkStmtList, c.info)
+
+proc genIf(c: var TLiftCtx; cond, action: PNode): PNode =
+  result = newTree(nkIfStmt, newTree(nkElifBranch, cond, action))
+
+proc genContainerOf(c: TLiftCtx; objType: PType, field, x: PSym): PNode =
+  # generate: cast[ptr ObjType](cast[int](addr(x)) - offsetOf(objType.field))
+  let intType = getSysType(c.g, unknownLineInfo, tyInt)
+
+  let addrOf = newNodeIT(nkAddr, c.info, makePtrType(x.owner, x.typ))
+  addrOf.add newDeref(newSymNode(x))
+  let castExpr1 = newNodeIT(nkCast, c.info, intType)
+  castExpr1.add newNodeIT(nkType, c.info, intType)
+  castExpr1.add addrOf
+
+  let dotExpr = newNodeIT(nkDotExpr, c.info, x.typ)
+  dotExpr.add newNodeIT(nkType, c.info, objType)
+  dotExpr.add newSymNode(field)
+
+  let offsetOf = genBuiltin(c.g, mOffsetOf, "offsetof", dotExpr)
+  offsetOf.typ = intType
+
+  let minusExpr = genBuiltin(c.g, mSubI, "-", castExpr1)
+  minusExpr.typ = intType
+  minusExpr.add offsetOf
+
+  let objPtr = makePtrType(objType.owner, objType)
+  result = newNodeIT(nkCast, c.info, objPtr)
+  result.add newNodeIT(nkType, c.info, objPtr)
+  result.add minusExpr
+
+proc destructorCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
+  var destroy = newNodeIT(nkCall, x.info, op.typ[0])
+  destroy.add(newSymNode(op))
+  destroy.add genAddr(c.g, x)
+  if sfNeverRaises notin op.flags:
+    c.canRaise = true
+  if c.addMemReset:
+    result = newTree(nkStmtList, destroy, genBuiltin(c.g, mWasMoved,  "wasMoved", x))
+  else:
+    result = destroy
 
 proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool) =
   case n.kind
   of nkSym:
+    if c.filterDiscriminator != nil: return
     let f = n.sym
     let b = if c.kind == attachedTrace: y else: y.dotField(f)
     if (sfCursor in f.flags and f.typ.skipTypes(abstractInst).kind in {tyRef, tyProc} and
@@ -101,8 +157,14 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       c.kind = prevKind
       localEnforceDefaultOp = true
 
-    # copy the selector:
-    fillBodyObj(c, n[0], body, x, y, enforceDefaultOp = false)
+    if c.kind != attachedDestructor:
+      # copy the selector before case stmt, but destroy after case stmt
+      fillBodyObj(c, n[0], body, x, y, enforceDefaultOp = false)
+
+    let oldfilterDiscriminator = c.filterDiscriminator
+    if c.filterDiscriminator == n[0].sym:
+      c.filterDiscriminator = nil # we have found the case part, proceed as normal
+
     # we need to generate a case statement:
     var caseStmt = newNodeI(nkCaseStmt, c.info)
     # XXX generate 'if' that checks same branches
@@ -121,6 +183,11 @@ proc fillBodyObj(c: var TLiftCtx; n, body, x, y: PNode; enforceDefaultOp: bool)
       caseStmt.add(branch)
     if emptyBranches != n.len-1:
       body.add(caseStmt)
+
+    if c.kind == attachedDestructor:
+      # destructor for selector is done after case stmt
+      fillBodyObj(c, n[0], body, x, y, enforceDefaultOp = false)
+    c.filterDiscriminator = oldfilterDiscriminator
   of nkRecList:
     for t in items(n): fillBodyObj(c, t, body, x, y, enforceDefaultOp)
   else:
@@ -179,29 +246,37 @@ proc fillBodyObjT(c: var TLiftCtx; t: PType, body, x, y: PNode) =
   else:
     fillBodyObjTImpl(c, t, body, x, y)
 
-proc newHookCall(g: ModuleGraph; op: PSym; x, y: PNode): PNode =
+proc newHookCall(c: var TLiftCtx; op: PSym; x, y: PNode): PNode =
   #if sfError in op.flags:
   #  localError(c.config, x.info, "usage of '$1' is a user-defined error" % op.name.s)
   result = newNodeI(nkCall, x.info)
   result.add newSymNode(op)
+  if sfNeverRaises notin op.flags:
+    c.canRaise = true
   if op.typ.sons[1].kind == tyVar:
-    result.add genAddr(g, x)
+    result.add genAddr(c.g, x)
   else:
     result.add x
   if y != nil:
     result.add y
 
-proc newOpCall(op: PSym; x: PNode): PNode =
+proc newOpCall(c: var TLiftCtx; op: PSym; x: PNode): PNode =
   result = newNodeIT(nkCall, x.info, op.typ[0])
   result.add(newSymNode(op))
   result.add x
+  if sfNeverRaises notin op.flags:
+    c.canRaise = true
 
-proc newDeepCopyCall(op: PSym; x, y: PNode): PNode =
-  result = newAsgnStmt(x, newOpCall(op, y))
+proc newDeepCopyCall(c: var TLiftCtx; op: PSym; x, y: PNode): PNode =
+  result = newAsgnStmt(x, newOpCall(c, op, y))
+
+proc usesBuiltinArc(t: PType): bool =
+  proc wrap(t: PType): bool {.nimcall.} = ast.isGCedMem(t)
+  result = types.searchTypeFor(t, wrap)
 
 proc useNoGc(c: TLiftCtx; t: PType): bool {.inline.} =
   result = optSeqDestructors in c.g.config.globalOptions and
-    ({tfHasGCedMem, tfHasOwned} * t.flags != {} or t.isGCedMem)
+    ({tfHasGCedMem, tfHasOwned} * t.flags != {} or usesBuiltinArc(t))
 
 proc requiresDestructor(c: TLiftCtx; t: PType): bool {.inline.} =
   result = optSeqDestructors in c.g.config.globalOptions and
@@ -218,14 +293,20 @@ proc instantiateGeneric(c: var TLiftCtx; op: PSym; t, typeInst: PType): PSym =
 proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
                         field: var PSym): bool =
   if optSeqDestructors in c.g.config.globalOptions:
-    let op = field
-    if field != nil and sfOverriden in field.flags:
+    var op = field
+    let destructorOverriden = destructorOverriden(t)
+    if op != nil and op != c.fn and 
+        (sfOverriden in op.flags or destructorOverriden):
       if sfError in op.flags:
         incl c.fn.flags, sfError
       #else:
       #  markUsed(c.g.config, c.info, op, c.g.usageSym)
       onUse(c.info, op)
-      body.add newHookCall(c.g, op, x, y)
+      body.add newHookCall(c, op, x, y)
+      result = true
+    elif op == nil and destructorOverriden:
+      op = produceSym(c.g, c.c, t, c.kind, c.info)
+      body.add newHookCall(c, op, x, y)
       result = true
   elif tfHasAsgn in t.flags:
     var op: PSym
@@ -254,7 +335,7 @@ proc considerAsgnOrSink(c: var TLiftCtx; t: PType; body, x, y: PNode;
       #debug(t)
       #return false
     assert op.ast[genericParamsPos].kind == nkEmpty
-    body.add newHookCall(c.g, op, x, y)
+    body.add newHookCall(c, op, x, y)
     result = true
 
 proc addDestructorCall(c: var TLiftCtx; orig: PType; body, x: PNode) =
@@ -275,7 +356,7 @@ proc addDestructorCall(c: var TLiftCtx; orig: PType; body, x: PNode) =
   if op != nil:
     #markUsed(c.g.config, c.info, op, c.g.usageSym)
     onUse(c.info, op)
-    body.add destructorCall(c.g, op, x)
+    body.add destructorCall(c, op, x)
   elif useNoGc(c, t):
     internalError(c.g.config, c.info,
       "type-bound operator could not be resolved")
@@ -293,7 +374,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
 
       #markUsed(c.g.config, c.info, op, c.g.usageSym)
       onUse(c.info, op)
-      body.add destructorCall(c.g, op, x)
+      body.add destructorCall(c, op, x)
       result = true
     #result = addDestructorCall(c, t, body, x)
   of attachedAsgn, attachedSink, attachedTrace:
@@ -305,7 +386,7 @@ proc considerUserDefinedOp(c: var TLiftCtx; t: PType; body, x, y: PNode): bool =
     if op != nil:
       #markUsed(c.g.config, c.info, op, c.g.usageSym)
       onUse(c.info, op)
-      body.add newDeepCopyCall(op, x, y)
+      body.add newDeepCopyCall(c, op, x, y)
       result = true
 
 proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
@@ -318,21 +399,15 @@ proc declareCounter(c: var TLiftCtx; body: PNode; first: BiggestInt): PNode =
   v.addVar(result, lowerings.newIntLit(c.g, body.info, first))
   body.add v
 
-proc genBuiltin(g: ModuleGraph; magic: TMagic; name: string; i: PNode): PNode =
-  result = newNodeI(nkCall, i.info)
-  result.add createMagic(g, name, magic).newSymNode
-  result.add i
+proc declareTempOf(c: var TLiftCtx; body: PNode; value: PNode): PNode =
+  var temp = newSym(skTemp, getIdent(c.g.cache, lowerings.genPrefix), c.fn, c.info)
+  temp.typ = value.typ
+  incl(temp.flags, sfFromGeneric)
 
-proc genWhileLoop(c: var TLiftCtx; i, dest: PNode): PNode =
-  result = newNodeI(nkWhileStmt, c.info, 2)
-  let cmp = genBuiltin(c.g, mLtI, "<", i)
-  cmp.add genLen(c.g, dest)
-  cmp.typ = getSysType(c.g, c.info, tyBool)
-  result[0] = cmp
-  result[1] = newNodeI(nkStmtList, c.info)
-
-proc genIf(c: var TLiftCtx; cond, action: PNode): PNode =
-  result = newTree(nkIfStmt, newTree(nkElifBranch, cond, action))
+  var v = newNodeI(nkVarSection, c.info)
+  result = newSymNode(temp)
+  v.addVar(result, value)
+  body.add v
 
 proc addIncStmt(c: var TLiftCtx; body, i: PNode) =
   let incCall = genBuiltin(c.g, mInc, "inc", i)
@@ -382,7 +457,7 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
+    moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
   of attachedDestructor:
     # destroy all elements:
@@ -392,6 +467,7 @@ proc fillSeqOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     # follow all elements:
     forallElements(c, t, body, x, y)
   of attachedDispose:
+    forallElements(c, t, body, x, y)
     body.add genBuiltin(c.g, mDestroy, "destroy", x)
 
 proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
@@ -406,26 +482,32 @@ proc useSeqOrStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
 
   case c.kind
   of attachedAsgn, attachedDeepCopy:
-    doAssert t.assignment != nil
-    body.add newHookCall(c.g, t.assignment, x, y)
+    # XXX: replace these with assertions.
+    if t.assignment == nil:
+      return # protect from recursion
+    body.add newHookCall(c, t.assignment, x, y)
   of attachedSink:
     # we always inline the move for better performance:
     let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
+    moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
     # alternatively we could do this:
     when false:
       doAssert t.asink != nil
-      body.add newHookCall(c.g, t.asink, x, y)
+      body.add newHookCall(c, t.asink, x, y)
   of attachedDestructor:
     doAssert t.destructor != nil
-    body.add destructorCall(c.g, t.destructor, x)
+    body.add destructorCall(c, t.destructor, x)
   of attachedTrace:
-    body.add newHookCall(c.g, t.attachedOps[c.kind], x, y)
+    if t.attachedOps[c.kind] == nil:
+      return # protect from recursion
+    body.add newHookCall(c, t.attachedOps[c.kind], x, y)
   of attachedDispose:
-    body.add newHookCall(c.g, t.attachedOps[c.kind], x, nil)
+    if t.attachedOps[c.kind] == nil:
+      return # protect from recursion
+    body.add newHookCall(c, t.attachedOps[c.kind], x, nil)
 
 proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case c.kind
@@ -435,7 +517,7 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     let moveCall = genBuiltin(c.g, mMove, "move", x)
     moveCall.add y
     doAssert t.destructor != nil
-    moveCall.add destructorCall(c.g, t.destructor, x)
+    moveCall.add destructorCall(c, t.destructor, x)
     body.add moveCall
   of attachedDestructor, attachedDispose:
     body.add genBuiltin(c.g, mDestroy, "destroy", x)
@@ -443,46 +525,81 @@ proc fillStrOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     discard "strings are atomic and have no inner elements that are to trace"
 
 proc atomicRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
+  #[ bug #15753 is really subtle. Usually the classical write barrier for reference
+  counting looks like this::
+
+    incRef source  # increment first; this takes care of self-assignments1
+    decRef dest
+    dest[] = source
+
+  However, 'decRef dest' might trigger a cycle collection and then the collector
+  traverses the graph. It is crucial that when it follows the pointers the assignment
+  'dest[] = source' already happened so that we don't do trial deletion on a wrong
+  graph -- this causes premature freeing of objects! The correct barrier looks like
+  this::
+
+    let tmp = dest
+    incRef source
+    dest[] = source
+    decRef tmp
+
+  ]#
   var actions = newNodeI(nkStmtList, c.info)
   let elemType = t.lastSon
 
-  if isFinal(elemType):
-    addDestructorCall(c, elemType, actions, genDeref(x, nkDerefExpr))
-    actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x)
-  else:
-    addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(x, nkDerefExpr))
-    actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, x)
+  createTypeBoundOps(c.g, c.c, elemType, c.info)
+  let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(elemType)
 
-  let isCyclic = c.g.config.selectedGC == gcOrc and types.canFormAcycle(t)
+  let tmp =
+    if isCyclic and c.kind in {attachedAsgn, attachedSink}:
+      declareTempOf(c, body, x)
+    else:
+      x
+
+  if isFinal(elemType):
+    addDestructorCall(c, elemType, actions, genDeref(tmp, nkDerefExpr))
+    var alignOf = genBuiltin(c.g, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType))
+    alignOf.typ = getSysType(c.g, c.info, tyInt)
+    actions.add callCodegenProc(c.g, "nimRawDispose", c.info, tmp, alignOf)
+  else:
+    addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(tmp, nkDerefExpr))
+    actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, tmp)
 
   var cond: PNode
   if isCyclic:
     if isFinal(elemType):
-      let typInfo = genBuiltin(c.g, mGetTypeInfo, "getTypeInfo", newNodeIT(nkType, x.info, elemType))
+      let typInfo = genBuiltin(c.g, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
       typInfo.typ = getSysType(c.g, c.info, tyPointer)
-      cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, x, typInfo)
+      cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicStatic", c.info, tmp, typInfo)
     else:
-      cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicDyn", c.info, x)
+      cond = callCodegenProc(c.g, "nimDecRefIsLastCyclicDyn", c.info, tmp)
   else:
     cond = callCodegenProc(c.g, "nimDecRefIsLast", c.info, x)
   cond.typ = getSysType(c.g, x.info, tyBool)
 
   case c.kind
   of attachedSink:
-    body.add genIf(c, cond, actions)
-    body.add newAsgnStmt(x, y)
+    if isCyclic:
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, cond, actions)
+    else:
+      body.add genIf(c, cond, actions)
+      body.add newAsgnStmt(x, y)
   of attachedAsgn:
     body.add genIf(c, y, callCodegenProc(c.g,
         if isCyclic: "nimIncRefCyclic" else: "nimIncRef", c.info, y))
-    body.add genIf(c, cond, actions)
-    body.add newAsgnStmt(x, y)
+    if isCyclic:
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, cond, actions)
+    else:
+      body.add genIf(c, cond, actions)
+      body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
     body.add genIf(c, cond, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace:
     if isFinal(elemType):
-      let typInfo = genBuiltin(c.g, mGetTypeInfo, "getTypeInfo", newNodeIT(nkType, x.info, elemType))
+      let typInfo = genBuiltin(c.g, mGetTypeInfoV2, "getTypeInfoV2", newNodeIT(nkType, x.info, elemType))
       typInfo.typ = getSysType(c.g, c.info, tyPointer)
       body.add callCodegenProc(c.g, "nimTraceRef", c.info, genAddrOf(x), typInfo, y)
     else:
@@ -503,19 +620,30 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   let xenv = genBuiltin(c.g, mAccessEnv, "accessEnv", x)
   xenv.typ = getSysType(c.g, c.info, tyPointer)
 
+  let isCyclic = c.g.config.selectedGC == gcOrc
+  let tmp =
+    if isCyclic and c.kind in {attachedAsgn, attachedSink}:
+      declareTempOf(c, body, xenv)
+    else:
+      xenv
+
   var actions = newNodeI(nkStmtList, c.info)
-  actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, xenv)
+  actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, tmp)
 
   let decRefProc =
-    if c.g.config.selectedGC == gcOrc: "nimDecRefIsLastCyclicDyn"
+    if isCyclic: "nimDecRefIsLastCyclicDyn"
     else: "nimDecRefIsLast"
-  let cond = callCodegenProc(c.g, decRefProc, c.info, xenv)
+  let cond = callCodegenProc(c.g, decRefProc, c.info, tmp)
   cond.typ = getSysType(c.g, x.info, tyBool)
 
   case c.kind
   of attachedSink:
-    body.add genIf(c, cond, actions)
-    body.add newAsgnStmt(x, y)
+    if isCyclic:
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, cond, actions)
+    else:
+      body.add genIf(c, cond, actions)
+      body.add newAsgnStmt(x, y)
   of attachedAsgn:
     let yenv = genBuiltin(c.g, mAccessEnv, "accessEnv", y)
     yenv.typ = getSysType(c.g, c.info, tyPointer)
@@ -523,10 +651,13 @@ proc atomicClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       if c.g.config.selectedGC == gcOrc: "nimIncRefCyclic"
       else: "nimIncRef"
     body.add genIf(c, yenv, callCodegenProc(c.g, incRefProc, c.info, yenv))
-    body.add genIf(c, cond, actions)
-    body.add newAsgnStmt(x, y)
+    if isCyclic:
+      body.add newAsgnStmt(x, y)
+      body.add genIf(c, cond, actions)
+    else:
+      body.add genIf(c, cond, actions)
+      body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(xenv, newNodeIT(nkNilLit, body.info, xenv.typ))
     body.add genIf(c, cond, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace:
@@ -556,7 +687,6 @@ proc weakrefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     # prevent wrong "dangling refs exist" problems:
     var actions = newNodeI(nkStmtList, c.info)
     actions.add callCodegenProc(c.g, "nimDecWeakRef", c.info, x)
-    actions.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
     let des = genIf(c, x, actions)
     if body.len == 0:
       body.add des
@@ -574,7 +704,9 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
 
   if isFinal(elemType):
     addDestructorCall(c, elemType, actions, genDeref(x, nkDerefExpr))
-    actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x)
+    var alignOf = genBuiltin(c.g, mAlignOf, "alignof", newNodeIT(nkType, c.info, elemType))
+    alignOf.typ = getSysType(c.g, c.info, tyInt)
+    actions.add callCodegenProc(c.g, "nimRawDispose", c.info, x, alignOf)
   else:
     addDestructorCall(c, elemType, newNodeI(nkStmtList, c.info), genDeref(x, nkDerefExpr))
     actions.add callCodegenProc(c.g, "nimDestroyAndDispose", c.info, x)
@@ -584,7 +716,6 @@ proc ownedRefOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, x, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(x, newNodeIT(nkNilLit, body.info, t))
     body.add genIf(c, x, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace, attachedDispose: discard
@@ -615,10 +746,7 @@ proc closureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
       body.add genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
       body.add newAsgnStmt(x, y)
     of attachedDestructor:
-      var actions = newNodeI(nkStmtList, c.info)
-      actions.add callCodegenProc(c.g, "nimDecWeakRef", c.info, xx)
-      actions.add newAsgnStmt(xx, newNodeIT(nkNilLit, body.info, xx.typ))
-      let des = genIf(c, xx, actions)
+      let des = genIf(c, xx, callCodegenProc(c.g, "nimDecWeakRef", c.info, xx))
       if body.len == 0:
         body.add des
       else:
@@ -637,7 +765,6 @@ proc ownedClosureOp(c: var TLiftCtx; t: PType; body, x, y: PNode) =
     body.add genIf(c, xx, actions)
     body.add newAsgnStmt(x, y)
   of attachedDestructor:
-    actions.add newAsgnStmt(xx, newNodeIT(nkNilLit, body.info, xx.typ))
     body.add genIf(c, xx, actions)
   of attachedDeepCopy: assert(false, "cannot happen")
   of attachedTrace, attachedDispose: discard
@@ -646,7 +773,7 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   case t.kind
   of tyNone, tyEmpty, tyVoid: discard
   of tyPointer, tySet, tyBool, tyChar, tyEnum, tyInt..tyUInt64, tyCString,
-      tyPtr, tyOpt, tyUncheckedArray, tyVar, tyLent:
+      tyPtr, tyUncheckedArray, tyVar, tyLent:
     defaultOp(c, t, body, x, y)
   of tyRef:
     if c.g.config.selectedGC in {gcArc, gcOrc}:
@@ -722,12 +849,13 @@ proc fillBody(c: var TLiftCtx; t: PType; body, x, y: PNode) =
   of tyFromExpr, tyProxy, tyBuiltInTypeClass, tyUserTypeClass,
      tyUserTypeClassInst, tyCompositeTypeClass, tyAnd, tyOr, tyNot, tyAnything,
      tyGenericParam, tyGenericBody, tyNil, tyUntyped, tyTyped,
-     tyTypeDesc, tyGenericInvocation, tyForward:
+     tyTypeDesc, tyGenericInvocation, tyForward, tyStatic:
     #internalError(c.g.config, c.info, "assignment requested for type: " & typeToString(t))
     discard
   of tyOrdinal, tyRange, tyInferred,
-     tyGenericInst, tyStatic, tyAlias, tySink:
+     tyGenericInst, tyAlias, tySink:
     fillBody(c, lastSon(t), body, x, y)
+  of tyOptDeprecated: doAssert false
 
 proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType;
                             kind: TTypeAttachedOp; info: TLineInfo): PSym =
@@ -738,52 +866,59 @@ proc produceSymDistinctType(g: ModuleGraph; c: PContext; typ: PType;
   typ.attachedOps[kind] = baseType.attachedOps[kind]
   result = typ.attachedOps[kind]
 
-proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
+proc symPrototype(g: ModuleGraph; typ: PType; owner: PSym; kind: TTypeAttachedOp;
               info: TLineInfo): PSym =
-  if typ.kind == tyDistinct:
-    return produceSymDistinctType(g, c, typ, kind, info)
 
-  var a: TLiftCtx
-  a.info = info
-  a.g = g
-  a.kind = kind
-  a.c = c
-  let body = newNodeI(nkStmtList, info)
   let procname = getIdent(g.cache, AttachedOpToStr[kind])
-
-  result = newSym(skProc, procname, typ.owner, info)
-  a.fn = result
-  a.asgnForType = typ
-
+  result = newSym(skProc, procname, owner, info)
   let dest = newSym(skParam, getIdent(g.cache, "dest"), result, info)
   let src = newSym(skParam, getIdent(g.cache, if kind == attachedTrace: "env" else: "src"), result, info)
-  var d: PNode
-  #if kind notin {attachedTrace, attachedDispose}:
   dest.typ = makeVarType(typ.owner, typ)
-  d = newDeref(newSymNode(dest))
-  #else:
-  #  dest.typ = typ
-  #  d = newSymNode(dest)
-
   if kind == attachedTrace:
     src.typ = getSysType(g, info, tyPointer)
   else:
     src.typ = typ
 
-  result.typ = newProcType(info, typ.owner)
+  result.typ = newProcType(info, owner)
   result.typ.addParam dest
   if kind notin {attachedDestructor, attachedDispose}:
     result.typ.addParam src
 
+  var n = newNodeI(nkProcDef, info, bodyPos+1)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = newNodeI(nkStmtList, info)
+  result.ast = n
+  incl result.flags, sfFromGeneric
+  incl result.flags, sfGeneratedOp
+
+
+proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
+              info: TLineInfo): PSym =
+  if typ.kind == tyDistinct:
+    return produceSymDistinctType(g, c, typ, kind, info)
+
+  result = typ.attachedOps[kind]
+  if result == nil:
+    result = symPrototype(g, typ, typ.owner, kind, info)
+
+  var a = TLiftCtx(info: info, g: g, kind: kind, c: c, asgnForType:typ)
+  a.fn = result
+
+  let dest = result.typ.n[1].sym
+  let d = newDeref(newSymNode(dest))
+  let src = if kind in {attachedDestructor, attachedDispose}: newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
+            else: newSymNode(result.typ.n[2].sym)
+
   # register this operation already:
   typ.attachedOps[kind] = result
 
-  if kind == attachedSink and typ.attachedOps[attachedDestructor] != nil and
-       sfOverriden in typ.attachedOps[attachedDestructor].flags:
+  if kind == attachedSink and destructorOverriden(typ):
     ## compiler can use a combination of `=destroy` and memCopy for sink op
     dest.flags.incl sfCursor
-    body.add newOpCall(typ.attachedOps[attachedDestructor], d[0])
-    body.add newAsgnStmt(d, newSymNode(src))
+    result.ast[bodyPos].add newOpCall(a, typ.attachedOps[attachedDestructor], d[0])
+    result.ast[bodyPos].add newAsgnStmt(d, src)
   else:
     var tk: TTypeKind
     if g.config.selectedGC in {gcArc, gcOrc, gcHooks}:
@@ -792,20 +927,35 @@ proc produceSym(g: ModuleGraph; c: PContext; typ: PType; kind: TTypeAttachedOp;
       tk = tyNone # no special casing for strings and seqs
     case tk
     of tySequence:
-      fillSeqOp(a, typ, body, d, newSymNode(src))
+      fillSeqOp(a, typ, result.ast[bodyPos], d, src)
     of tyString:
-      fillStrOp(a, typ, body, d, newSymNode(src))
+      fillStrOp(a, typ, result.ast[bodyPos], d, src)
     else:
-      fillBody(a, typ, body, d, newSymNode(src))
+      fillBody(a, typ, result.ast[bodyPos], d, src)
+  if not a.canRaise: incl result.flags, sfNeverRaises
+
+
+proc produceDestructorForDiscriminator*(g: ModuleGraph; typ: PType; field: PSym, info: TLineInfo): PSym =
+  assert(typ.skipTypes({tyAlias, tyGenericInst}).kind == tyObject)
+  result = symPrototype(g, field.typ, typ.owner, attachedDestructor, info)
+  var a = TLiftCtx(info: info, g: g, kind: attachedDestructor, asgnForType: typ)
+  a.fn = result
+  a.asgnForType = typ
+  a.filterDiscriminator = field
+  a.addMemReset = true
+  let discrimantDest = result.typ.n[1].sym
+
+  let dst = newSym(skVar, getIdent(g.cache, "dest"), result, info)
+  dst.typ = makePtrType(typ.owner, typ)
+  let dstSym = newSymNode(dst)
+  let d = newDeref(dstSym)
+  let v = newNodeI(nkVarSection, info)
+  v.addVar(dstSym, genContainerOf(a, typ, field, discrimantDest))
+  result.ast[bodyPos].add v
+  let placeHolder = newNodeIT(nkSym, info, getSysType(g, info, tyPointer))
+  fillBody(a, typ, result.ast[bodyPos], d, placeHolder)
+  if not a.canRaise: incl result.flags, sfNeverRaises
 
-  var n = newNodeI(nkProcDef, info, bodyPos+1)
-  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
-  n[namePos] = newSymNode(result)
-  n[paramsPos] = result.typ.n
-  n[bodyPos] = body
-  result.ast = n
-  incl result.flags, sfFromGeneric
-  incl result.flags, sfGeneratedOp
 
 template liftTypeBoundOps*(c: PContext; typ: PType; info: TLineInfo) =
   discard "now a nop"
@@ -843,11 +993,6 @@ template inst(field, t) =
 proc isTrival(s: PSym): bool {.inline.} =
   s == nil or (s.ast != nil and s.ast[bodyPos].len == 0)
 
-
-proc isEmptyContainer(g: ModuleGraph, t: PType): bool =
-  (t.kind == tyArray and lengthOrd(g.config, t[0]) == 0) or
-    (t.kind == tySequence and t[0].kind == tyError)
-
 proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInfo) =
   ## In the semantic pass this is called in strategic places
   ## to ensure we lift assignment, destructors and moves properly.
@@ -856,7 +1001,7 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
   incl orig.flags, tfCheckedForDestructor
 
   let skipped = orig.skipTypes({tyGenericInst, tyAlias, tySink})
-  if isEmptyContainer(skipped): return
+  if isEmptyContainer(skipped) or skipped.kind == tyStatic: return
 
   let h = sighashes.hashType(skipped, {CoType, CoConsiderOwned, CoDistinct})
   var canon = g.canonTypes.getOrDefault(h)
@@ -876,9 +1021,18 @@ proc createTypeBoundOps(g: ModuleGraph; c: PContext; orig: PType; info: TLineInf
   let lastAttached = if g.config.selectedGC == gcOrc: attachedDispose
                      else: attachedSink
 
+  # bug #15122: We need to produce all prototypes before entering the
+  # mind boggling recursion. Hacks like these imply we shoule rewrite
+  # this module.
+  var generics: array[attachedDestructor..attachedDispose, bool]
+  for k in attachedDestructor..lastAttached:
+    generics[k] = canon.attachedOps[k] != nil
+    if not generics[k]:
+      canon.attachedOps[k] = symPrototype(g, canon, canon.owner, k, info)
+
   # we generate the destructor first so that other operators can depend on it:
   for k in attachedDestructor..lastAttached:
-    if canon.attachedOps[k] == nil:
+    if not generics[k]:
       discard produceSym(g, c, canon, k, info)
     else:
       inst(canon.attachedOps[k], canon)

compiler/lineinfos.nim

@@ -10,10 +10,20 @@
 ## This module contains the ``TMsgKind`` enum as well as the
 ## ``TLineInfo`` object.
 
-import ropes, tables, pathutils
+import ropes, tables, pathutils, hashes
 
 const
-  explanationsBaseUrl* = "https://nim-lang.org/docs"
+  explanationsBaseUrl* = "https://nim-lang.github.io/Nim"
+    # was: "https://nim-lang.org/docs" but we're now usually showing devel docs
+    # instead of latest release docs.
+
+proc createDocLink*(urlSuffix: string): string =
+  # os.`/` is not appropriate for urls.
+  result = explanationsBaseUrl
+  if urlSuffix.len > 0 and urlSuffix[0] == '/':
+    result.add urlSuffix
+  else:
+    result.add "/" & urlSuffix
 
 type
   TMsgKind* = enum
@@ -23,37 +33,42 @@ type
     errGeneralParseError,
     errNewSectionExpected,
     errInvalidDirectiveX,
-    errProveInit,
+    errProveInit, # deadcode
     errGenerated,
     errUser,
-    warnCannotOpenFile,
-    warnOctalEscape, warnXIsNeverRead, warnXmightNotBeenInit,
-    warnDeprecated, warnConfigDeprecated,
-    warnSmallLshouldNotBeUsed, warnUnknownMagic, warnRedefinitionOfLabel,
-    warnUnknownSubstitutionX, warnLanguageXNotSupported,
-    warnFieldXNotSupported, warnCommentXIgnored,
-    warnTypelessParam,
-    warnUseBase, warnWriteToForeignHeap, warnUnsafeCode,
-    warnUnusedImportX,
-    warnInheritFromException,
-    warnEachIdentIsTuple,
-    warnUnsafeSetLen,
-    warnUnsafeDefault,
-    warnProveInit, warnProveField, warnProveIndex,
-    warnStaticIndexCheck, warnGcUnsafe, warnGcUnsafe2,
-    warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed,
-    warnInconsistentSpacing, warnCaseTransition, warnCycleCreated, warnUser,
-    hintSuccess, hintSuccessX, hintCC,
-    hintLineTooLong, hintXDeclaredButNotUsed,
-    hintConvToBaseNotNeeded,
-    hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled,
-    hintProcessing, hintCodeBegin, hintCodeEnd, hintConf, hintPath,
-    hintConditionAlwaysTrue, hintConditionAlwaysFalse, hintName, hintPattern,
-    hintExecuting, hintLinking, hintDependency,
-    hintSource, hintPerformance, hintStackTrace, hintGCStats,
-    hintGlobalVar, hintExpandMacro,
-    hintUser, hintUserRaw,
-    hintExtendedContext
+
+    warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape",
+    warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit",
+    warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated",
+    warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic",
+    warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX",
+    warnLanguageXNotSupported = "LanguageXNotSupported", warnFieldXNotSupported = "FieldXNotSupported",
+    warnCommentXIgnored = "CommentXIgnored", warnTypelessParam = "TypelessParam",
+    warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap",
+    warnUnsafeCode = "UnsafeCode", warnUnusedImportX = "UnusedImport",
+    warnInheritFromException = "InheritFromException", warnEachIdentIsTuple = "EachIdentIsTuple",
+    warnUnsafeSetLen = "UnsafeSetLen", warnUnsafeDefault = "UnsafeDefault",
+    warnProveInit = "ProveInit", warnProveField = "ProveField", warnProveIndex = "ProveIndex",
+    warnUnreachableElse = "UnreachableElse", warnUnreachableCode = "UnreachableCode",
+    warnStaticIndexCheck = "IndexCheck", warnGcUnsafe = "GcUnsafe", warnGcUnsafe2 = "GcUnsafe2",
+    warnUninit = "Uninit", warnGcMem = "GcMem", warnDestructor = "Destructor",
+    warnLockLevel = "LockLevel", warnResultShadowed = "ResultShadowed",
+    warnInconsistentSpacing = "Spacing",  warnCaseTransition = "CaseTransition",
+    warnCycleCreated = "CycleCreated", warnObservableStores = "ObservableStores",
+    warnUser = "User",
+
+    hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC",
+    hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed",
+    hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded",
+    hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX",
+    hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintCodeBegin = "CodeBegin",
+    hintCodeEnd = "CodeEnd", hintConf = "Conf", hintPath = "Path",
+    hintConditionAlwaysTrue = "CondTrue", hintConditionAlwaysFalse = "CondFalse", hintName = "Name",
+    hintPattern = "Pattern", hintExecuting = "Exec", hintLinking = "Link", hintDependency = "Dependency",
+    hintSource = "Source", hintPerformance = "Performance", hintStackTrace = "StackTrace",
+    hintGCStats = "GCStats", hintGlobalVar = "GlobalVar", hintExpandMacro = "ExpandMacro",
+    hintUser = "User", hintUserRaw = "UserRaw", hintExtendedContext = "ExtendedContext",
+    hintMsgOrigin = "MsgOrigin", # since 1.3.5
 
 const
   MsgKindToStr*: array[TMsgKind, string] = [
@@ -66,7 +81,7 @@ const
     errGeneralParseError: "general parse error",
     errNewSectionExpected: "new section expected",
     errInvalidDirectiveX: "invalid directive: '$1'",
-    errProveInit: "Cannot prove that '$1' is initialized.",
+    errProveInit: "Cannot prove that '$1' is initialized.",  # deadcode
     errGenerated: "$1",
     errUser: "$1",
     warnCannotOpenFile: "cannot open '$1'",
@@ -87,7 +102,8 @@ const
     warnWriteToForeignHeap: "write to foreign heap",
     warnUnsafeCode: "unsafe code: '$1'",
     warnUnusedImportX: "imported and not used: '$1'",
-    warnInheritFromException: "inherit from a more precise exception type like ValueError, IOError or OSError",
+    warnInheritFromException: "inherit from a more precise exception type like ValueError, " &
+      "IOError or OSError. If these don't suit, inherit from CatchableError or Defect.",
     warnEachIdentIsTuple: "each identifier is a tuple",
     warnUnsafeSetLen: "setLen can potentially expand the sequence, " &
                       "but the element type '$1' doesn't have a valid default value",
@@ -95,10 +111,12 @@ const
     warnProveInit: "Cannot prove that '$1' is initialized. This will become a compile time error in the future.",
     warnProveField: "cannot prove that field '$1' is accessible",
     warnProveIndex: "cannot prove index '$1' is valid",
+    warnUnreachableElse: "unreachable else, all cases are already covered",
+    warnUnreachableCode: "unreachable code after 'return' statement or '{.noReturn.}' proc",
     warnStaticIndexCheck: "$1",
     warnGcUnsafe: "not GC-safe: '$1'",
     warnGcUnsafe2: "$1",
-    warnUninit: "'$1' might not have been initialized",
+    warnUninit: "use explicit initialization of '$1' for clarity",
     warnGcMem: "'$1' uses GC'ed memory",
     warnDestructor: "usage of a type with a destructor in a non destructible context. This will become a compile time error in the future.",
     warnLockLevel: "$1",
@@ -106,13 +124,15 @@ const
     warnInconsistentSpacing: "Number of spaces around '$#' is not consistent",
     warnCaseTransition: "Potential object case transition, instantiate new object instead",
     warnCycleCreated: "$1",
+    warnObservableStores: "observable stores to '$1'",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
     # keep in sync with `testament.isSuccess`
-    hintSuccessX: "$loc LOC; $sec sec; $mem; $build build; proj: $project; out: $output",
+    hintSuccessX: "${loc} lines; ${sec}s; $mem; $build build; proj: $project; out: $output",
     hintCC: "CC: $1",
     hintLineTooLong: "line too long",
     hintXDeclaredButNotUsed: "'$1' is declared but not used",
+    hintXCannotRaiseY: "$1",
     hintConvToBaseNotNeeded: "conversion to base object is not needed",
     hintConvFromXtoItselfNotNeeded: "conversion from $1 to itself is pointless",
     hintExprAlwaysX: "expression evaluates always to '$1'",
@@ -138,33 +158,7 @@ const
     hintUser: "$1",
     hintUserRaw: "$1",
     hintExtendedContext: "$1",
-  ]
-
-const
-  WarningsToStr* = ["CannotOpenFile", "OctalEscape",
-    "XIsNeverRead", "XmightNotBeenInit",
-    "Deprecated", "ConfigDeprecated",
-    "SmallLshouldNotBeUsed", "UnknownMagic",
-    "RedefinitionOfLabel", "UnknownSubstitutionX",
-    "LanguageXNotSupported", "FieldXNotSupported",
-    "CommentXIgnored",
-    "TypelessParam", "UseBase", "WriteToForeignHeap",
-    "UnsafeCode", "UnusedImport", "InheritFromException",
-    "EachIdentIsTuple",
-    "UnsafeSetLen", "UnsafeDefault",
-    "ProveInit", "ProveField", "ProveIndex",
-    "IndexCheck", "GcUnsafe", "GcUnsafe2", "Uninit",
-    "GcMem", "Destructor", "LockLevel", "ResultShadowed",
-    "Spacing", "CaseTransition", "CycleCreated", "User"]
-
-  HintsToStr* = [
-    "Success", "SuccessX", "CC", "LineTooLong",
-    "XDeclaredButNotUsed",
-    "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded",
-    "ExprAlwaysX", "QuitCalled", "Processing", "CodeBegin", "CodeEnd", "Conf",
-    "Path", "CondTrue", "CondFalse", "Name", "Pattern", "Exec", "Link", "Dependency",
-    "Source", "Performance", "StackTrace", "GCStats", "GlobalVar", "ExpandMacro",
-    "User", "UserRaw", "ExtendedContext",
+    hintMsgOrigin: "$1",
   ]
 
 const
@@ -177,20 +171,16 @@ const
   hintMin* = hintSuccess
   hintMax* = high(TMsgKind)
 
-static:
-  doAssert HintsToStr.len == ord(hintMax) - ord(hintMin) + 1
-  doAssert WarningsToStr.len == ord(warnMax) - ord(warnMin) + 1
-
 type
   TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints
   TNoteKinds* = set[TNoteKind]
 
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
-  result[3] = {low(TNoteKind)..high(TNoteKind)} - {}
+  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores}
   result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext}
   result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
-    hintSource, hintGlobalVar, hintGCStats}
+    hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin}
   result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf,
     hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC}
 
@@ -245,6 +235,9 @@ type
 
 proc `==`*(a, b: FileIndex): bool {.borrow.}
 
+proc hash*(i: TLineInfo): Hash =
+  hash (i.line.int, i.col.int, i.fileIndex.int)
+
 proc raiseRecoverableError*(msg: string) {.noinline.} =
   raise newException(ERecoverableError, msg)
 

compiler/linter.nim

@@ -71,12 +71,9 @@ proc beautifyName(s: string, k: TSymKind): string =
 
 proc differ*(line: string, a, b: int, x: string): string =
   proc substrEq(s: string, pos, last: int, substr: string): bool =
-    var i = 0
-    while i < substr.len and pos+i <= last and s[pos+i] == substr[i]:
-      inc i
-    return i == substr.len
-
-  let last = min(b, line.len)
+    result = true
+    for i in 0..<substr.len:
+      if pos+i > last or s[pos+i] != substr[i]: return false
 
   result = ""
   if not substrEq(line, a, b, x):
@@ -84,11 +81,6 @@ proc differ*(line: string, a, b: int, x: string): string =
     if cmpIgnoreStyle(y, x) == 0:
       result = y
 
-proc checkStyle(conf: ConfigRef; cache: IdentCache; info: TLineInfo, s: string, k: TSymKind; sym: PSym) =
-  let beau = beautifyName(s, k)
-  if s != beau:
-    lintReport(conf, info, beau, s)
-
 proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
   # operators stay as they are:
   if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
@@ -136,6 +128,6 @@ proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) =
     lintReport(conf, info, newName, oldName)
 
 proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
-  let wanted = canonPragmaSpelling(w)
+  let wanted = $w
   if pragmaName != wanted:
     lintReport(conf, info, wanted, pragmaName)

compiler/lookups.nim

@@ -91,8 +91,15 @@ proc skipAlias*(s: PSym; n: PNode; conf: ConfigRef): PSym =
       message(conf, n.info, warnDeprecated, "use " & result.name.s & " instead; " &
               s.name.s & " is deprecated")
 
+proc isShadowScope*(s: PScope): bool {.inline.} = s.parent != nil and s.parent.depthLevel == s.depthLevel
+
 proc localSearchInScope*(c: PContext, s: PIdent): PSym =
-  result = strTableGet(c.currentScope.symbols, s)
+  var scope = c.currentScope
+  result = strTableGet(scope.symbols, s)
+  while result == nil and scope.isShadowScope:
+    # We are in a shadow scope, check in the parent too
+    scope = scope.parent
+    result = strTableGet(scope.symbols, s)
 
 proc searchInScopes*(c: PContext, s: PIdent): PSym =
   for scope in walkScopes(c.currentScope):
@@ -231,6 +238,23 @@ proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) =
   addOverloadableSymAt(c, scope, sym)
   addInterfaceDeclAux(c, sym)
 
+proc openShadowScope*(c: PContext) =
+  c.currentScope = PScope(parent: c.currentScope,
+                          symbols: newStrTable(),
+                          depthLevel: c.scopeDepth)
+
+proc closeShadowScope*(c: PContext) =
+  c.closeScope
+
+proc mergeShadowScope*(c: PContext) =
+  let shadowScope = c.currentScope
+  c.rawCloseScope
+  for sym in shadowScope.symbols:
+    if sym.kind in OverloadableSyms:
+      c.addInterfaceOverloadableSymAt(c.currentScope, sym)
+    else:
+      c.addInterfaceDecl(sym)
+
 when defined(nimfix):
   # when we cannot find the identifier, retry with a changed identifier:
   proc altSpelling(x: PIdent): PIdent =
@@ -316,8 +340,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
       fixSpelling(n, ident, searchInScopes)
       errorUndeclaredIdentifier(c, n.info, ident.s)
       result = errorSym(c, n)
-    elif checkAmbiguity in flags and result != nil and
-        contains(c.ambiguousSymbols, result.id):
+    elif checkAmbiguity in flags and result != nil and result.id in c.ambiguousSymbols:
       errorUseQualifier(c, n.info, result)
   of nkSym:
     result = n.sym

compiler/lowerings.nim

@@ -59,6 +59,13 @@ proc newFastAsgnStmt*(le, ri: PNode): PNode =
   result[0] = le
   result[1] = ri
 
+proc newFastMoveStmt*(g: ModuleGraph, le, ri: PNode): PNode =
+  result = newNodeI(nkFastAsgn, le.info, 2)
+  result[0] = le
+  result[1] = newNodeIT(nkCall, ri.info, ri.typ)
+  result[1].add newSymNode(getSysMagic(g, ri.info, "move", mMove))
+  result[1].add ri
+
 proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   assert n.kind == nkVarTuple
   let value = n.lastSon
@@ -129,6 +136,7 @@ proc lowerSwap*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   var temp = newSym(skVar, getIdent(g.cache, genPrefix), owner, n.info, owner.options)
   temp.typ = n[1].typ
   incl(temp.flags, sfFromGeneric)
+  incl(temp.flags, sfGenSym)
 
   var v = newNodeI(nkVarSection, n.info)
   let tempAsNode = newSymNode(temp)
@@ -151,7 +159,7 @@ proc createObj*(g: ModuleGraph; owner: PSym, info: TLineInfo; final=true): PType
   else:
     rawAddSon(result, getCompilerProc(g, "RootObj").typ)
   result.n = newNodeI(nkRecList, info)
-  let s = newSym(skType, getIdent(g.cache, "Env_" & toFilename(g.config, info)),
+  let s = newSym(skType, getIdent(g.cache, "Env_" & toFilename(g.config, info) & "_" & $owner.name.s),
                   owner, info, owner.options)
   incl s.flags, sfAnon
   s.typ = result
@@ -362,14 +370,3 @@ proc genLen*(g: ModuleGraph; n: PNode): PNode =
     result[0] = newSymNode(getSysMagic(g, n.info, "len", mLengthSeq))
     result[1] = n
 
-proc hoistExpr*(varSection, expr: PNode, name: PIdent, owner: PSym): PSym =
-  result = newSym(skLet, name, owner, varSection.info, owner.options)
-  result.flags.incl sfHoisted
-  result.typ = expr.typ
-
-  var varDef = newNodeI(nkIdentDefs, varSection.info, 3)
-  varDef[0] = newSymNode(result)
-  varDef[1] = newNodeI(nkEmpty, varSection.info)
-  varDef[2] = expr
-
-  varSection.add varDef

compiler/magicsys.nim

@@ -177,3 +177,28 @@ proc getNimScriptSymbol*(g: ModuleGraph; name: string): PSym =
   strTableGet(g.exposed, getIdent(g.cache, name))
 
 proc resetNimScriptSymbols*(g: ModuleGraph) = initStrTable(g.exposed)
+
+proc getMagicEqSymForType*(g: ModuleGraph; t: PType; info: TLineInfo): PSym =
+  case t.kind
+  of tyInt,  tyInt8, tyInt16, tyInt32, tyInt64,
+     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64: 
+    result = getSysMagic(g, info, "==", mEqI)
+  of tyEnum: 
+    result = getSysMagic(g, info, "==", mEqEnum)
+  of tyBool: 
+    result = getSysMagic(g, info, "==", mEqB)
+  of tyRef, tyPtr, tyPointer: 
+    result = getSysMagic(g, info, "==", mEqRef)
+  of tyString:
+    result = getSysMagic(g, info, "==", mEqStr)
+  of tyChar:
+    result = getSysMagic(g, info, "==", mEqCh)
+  of tySet:
+    result = getSysMagic(g, info, "==", mEqSet)
+  of tyProc:
+    result = getSysMagic(g, info, "==", mEqProc)
+  else:
+    globalError(g.config, info,
+      "can't find magic equals operator for type kind " & $t.kind)
+
+

compiler/main.nim

@@ -13,13 +13,13 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  llstream, strutils, ast, lexer, syntaxes, options, msgs,
+  llstream, strutils, os, ast, lexer, syntaxes, options, msgs,
   condsyms, times,
   sem, idents, passes, extccomp,
   cgen, json, nversion,
   platform, nimconf, passaux, depends, vm, idgen,
   modules,
-  modulegraphs, tables, rod, lineinfos, pathutils
+  modulegraphs, tables, rod, lineinfos, pathutils, vmprofiler
 
 when not defined(leanCompiler):
   import jsgen, docgen, docgen2
@@ -51,7 +51,7 @@ proc commandGenDepend(graph: ModuleGraph) =
       ' ' & changeFileExt(project, "dot").string)
 
 proc commandCheck(graph: ModuleGraph) =
-  graph.config.errorMax = high(int)  # do not stop after first error
+  graph.config.setErrorMaxHighMaybe
   defineSymbol(graph.config.symbols, "nimcheck")
   semanticPasses(graph)  # use an empty backend for semantic checking only
   compileProject(graph)
@@ -59,7 +59,7 @@ proc commandCheck(graph: ModuleGraph) =
 when not defined(leanCompiler):
   proc commandDoc2(graph: ModuleGraph; json: bool) =
     handleDocOutputOptions graph.config
-    graph.config.errorMax = high(int)  # do not stop after first error
+    graph.config.setErrorMaxHighMaybe
     semanticPasses(graph)
     if json: registerPass(graph, docgen2JsonPass)
     else: registerPass(graph, docgen2Pass)
@@ -68,16 +68,7 @@ when not defined(leanCompiler):
 
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
-
-  if conf.outDir.isEmpty:
-    conf.outDir = conf.projectPath
-  if conf.outFile.isEmpty:
-    let targetName = if optGenDynLib in conf.globalOptions:
-      platform.OS[conf.target.targetOS].dllFrmt % conf.projectName
-    else:
-      conf.projectName & platform.OS[conf.target.targetOS].exeExt
-    conf.outFile = RelativeFile targetName
-
+  setOutFile(conf)
   extccomp.initVars(conf)
   semanticPasses(graph)
   registerPass(graph, cgenPass)
@@ -89,6 +80,9 @@ proc commandCompileToC(graph: ModuleGraph) =
       graph.config.notes = graph.config.mainPackageNotes
       return
 
+  if not extccomp.ccHasSaneOverflow(conf):
+    conf.symbols.defineSymbol("nimEmulateOverflowChecks")
+
   compileProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933
@@ -105,13 +99,13 @@ proc commandJsonScript(graph: ModuleGraph) =
   let proj = changeFileExt(graph.config.projectFull, "")
   extccomp.runJsonBuildInstructions(graph.config, proj)
 
-when not defined(leanCompiler):
-  proc commandCompileToJS(graph: ModuleGraph) =
+proc commandCompileToJS(graph: ModuleGraph) =
+  when defined(leanCompiler):
+    globalError(graph.config, unknownLineInfo, "compiler wasn't built with JS code generator")
+  else:
     let conf = graph.config
     conf.exc = excCpp
 
-    if conf.outDir.isEmpty:
-      conf.outDir = conf.projectPath
     if conf.outFile.isEmpty:
       conf.outFile = RelativeFile(conf.projectName & ".js")
 
@@ -119,7 +113,6 @@ when not defined(leanCompiler):
     setTarget(graph.config.target, osJS, cpuJS)
     #initDefines()
     defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility
-    defineSymbol(graph.config.symbols, "js")
     semanticPasses(graph)
     registerPass(graph, JSgenPass)
     compileProject(graph)
@@ -136,7 +129,7 @@ proc interactivePasses(graph: ModuleGraph) =
   registerPass(graph, evalPass)
 
 proc commandInteractive(graph: ModuleGraph) =
-  graph.config.errorMax = high(int)  # do not stop after first error
+  graph.config.setErrorMaxHighMaybe
   interactivePasses(graph)
   compileSystemModule(graph)
   if graph.config.commandArgs.len > 0:
@@ -156,8 +149,8 @@ proc commandScan(cache: IdentCache, config: ConfigRef) =
   var stream = llStreamOpen(f, fmRead)
   if stream != nil:
     var
-      L: TLexer
-      tok: TToken
+      L: Lexer
+      tok: Token
     initToken(tok)
     openLexer(L, f, stream, cache, config)
     while true:
@@ -181,63 +174,103 @@ proc mainCommand*(graph: ModuleGraph) =
   conf.lastCmdTime = epochTime()
   conf.searchPaths.add(conf.libpath)
   setId(100)
-  case conf.command.normalize
-  of "c", "cc", "compile", "compiletoc":
-    # compile means compileToC currently
-    conf.cmd = cmdCompileToC
-    if conf.exc == excNone: conf.exc = excSetjmp
-    defineSymbol(graph.config.symbols, "c")
-    commandCompileToC(graph)
-  of "cpp", "compiletocpp":
-    conf.cmd = cmdCompileToCpp
-    if conf.exc == excNone: conf.exc = excCpp
-    defineSymbol(graph.config.symbols, "cpp")
-    commandCompileToC(graph)
-  of "objc", "compiletooc":
-    conf.cmd = cmdCompileToOC
-    defineSymbol(graph.config.symbols, "objc")
-    commandCompileToC(graph)
-  of "run":
-    conf.cmd = cmdRun
-    when hasTinyCBackend:
-      extccomp.setCC(conf, "tcc", unknownLineInfo)
-      commandCompileToC(graph)
-    else:
-      rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
-  of "js", "compiletojs":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with JS code generator"
-    else:
-      conf.cmd = cmdCompileToJS
+
+  proc customizeForBackend(backend: TBackend) =
+    ## Sets backend specific options but don't compile to backend yet in
+    ## case command doesn't require it. This must be called by all commands.
+    if conf.backend == backendInvalid:
+      # only set if wasn't already set, to allow override via `nim c -b:cpp`
+      conf.backend = backend
+
+    defineSymbol(graph.config.symbols, $conf.backend)
+    case conf.backend
+    of backendC:
+      if conf.exc == excNone: conf.exc = excSetjmp
+    of backendCpp:
+      if conf.exc == excNone: conf.exc = excCpp
+    of backendObjc: discard
+    of backendJs:
       if conf.hcrOn:
         # XXX: At the moment, system.nim cannot be compiled in JS mode
         # with "-d:useNimRtl". The HCR option has been processed earlier
         # and it has added this define implictly, so we must undo that here.
         # A better solution might be to fix system.nim
         undefSymbol(conf.symbols, "useNimRtl")
-      commandCompileToJS(graph)
-  of "doc0":
+    of backendInvalid: doAssert false
+    if conf.selectedGC in {gcArc, gcOrc} and conf.backend != backendCpp:
+      conf.exc = excGoto
+
+  var commandAlreadyProcessed = false
+
+  proc compileToBackend(backend: TBackend, cmd = cmdCompileToBackend) =
+    commandAlreadyProcessed = true
+    conf.cmd = cmd
+    customizeForBackend(backend)
+    case conf.backend
+    of backendC: commandCompileToC(graph)
+    of backendCpp: commandCompileToC(graph)
+    of backendObjc: commandCompileToC(graph)
+    of backendJs: commandCompileToJS(graph)
+    of backendInvalid: doAssert false
+
+  template docLikeCmd(body) =
     when defined(leanCompiler):
       quit "compiler wasn't built with documentation generator"
     else:
       wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandDoc(cache, conf)
-  of "doc2", "doc":
-    conf.setNoteDefaults(warnLockLevel, false) # issue #13218
-    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
-      # because currently generates lots of false positives due to conflation
-      # of labels links in doc comments, eg for random.rand:
-      #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
-      #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
       conf.cmd = cmdDoc
       loadConfigs(DocConfig, cache, conf)
       defineSymbol(conf.symbols, "nimdoc")
+      body
+
+  block: ## command prepass
+    var docLikeCmd2 = false # includes what calls `docLikeCmd` + some more
+    case conf.command.normalize
+    of "r": conf.globalOptions.incl {optRun, optUseNimcache}
+    of "doc0",  "doc2", "doc", "rst2html", "rst2tex", "jsondoc0", "jsondoc2",
+      "jsondoc", "ctags", "buildindex": docLikeCmd2 = true
+    else: discard
+    if conf.outDir.isEmpty:
+      # doc like commands can generate a lot of files (especially with --project)
+      # so by default should not end up in $PWD nor in $projectPath.
+      conf.outDir = block:
+        var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
+                  else: conf.projectPath
+        doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
+        if docLikeCmd2: ret = ret / htmldocsDir
+        ret
+
+  ## process all backend commands
+  case conf.command.normalize
+  of "c", "cc", "compile", "compiletoc": compileToBackend(backendC) # compile means compileToC currently
+  of "cpp", "compiletocpp": compileToBackend(backendCpp)
+  of "objc", "compiletooc": compileToBackend(backendObjc)
+  of "js", "compiletojs": compileToBackend(backendJs)
+  of "r": compileToBackend(backendC) # different from `"run"`!
+  of "run":
+    when hasTinyCBackend:
+      extccomp.setCC(conf, "tcc", unknownLineInfo)
+      if conf.backend notin {backendC, backendInvalid}:
+        rawMessage(conf, errGenerated, "'run' requires c backend, got: '$1'" % $conf.backend)
+      compileToBackend(backendC, cmd = cmdRun)
+    else:
+      rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
+  else: customizeForBackend(backendC) # fallback for other commands
+
+  ## process all other commands
+  case conf.command.normalize # synchronize with `cmdUsingHtmlDocs`
+  of "doc0": docLikeCmd commandDoc(cache, conf)
+  of "doc2", "doc":
+    docLikeCmd():
+      conf.setNoteDefaults(warnLockLevel, false) # issue #13218
+      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+        # because currently generates lots of false positives due to conflation
+        # of labels links in doc comments, e.g. for random.rand:
+        #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
+        #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
       commandDoc2(graph, false)
+      if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
+        commandBuildIndex(conf, $conf.outDir)
   of "rst2html":
     conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
     when defined(leanCompiler):
@@ -253,41 +286,10 @@ proc mainCommand*(graph: ModuleGraph) =
       conf.cmd = cmdRst2tex
       loadConfigs(DocTexConfig, cache, conf)
       commandRst2TeX(cache, conf)
-  of "jsondoc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandJson(cache, conf)
-  of "jsondoc2", "jsondoc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandDoc2(graph, true)
-  of "ctags":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandTags(cache, conf)
-  of "buildindex":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandBuildIndex(cache, conf)
+  of "jsondoc0": docLikeCmd commandJson(cache, conf)
+  of "jsondoc2", "jsondoc": docLikeCmd commandDoc2(graph, true)
+  of "ctags": docLikeCmd commandTags(cache, conf)
+  of "buildindex": docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
   of "gendepend":
     conf.cmd = cmdGenDepend
     commandGenDepend(graph)
@@ -306,15 +308,14 @@ proc mainCommand*(graph: ModuleGraph) =
 
       var hints = newJObject() # consider factoring with `listHints`
       for a in hintMin..hintMax:
-        let key = lineinfos.HintsToStr[ord(a) - ord(hintMin)]
-        hints[key] = %(a in conf.notes)
+        hints[$a] = %(a in conf.notes)
       var warnings = newJObject()
       for a in warnMin..warnMax:
-        let key = lineinfos.WarningsToStr[ord(a) - ord(warnMin)]
-        warnings[key] = %(a in conf.notes)
+        warnings[$a] = %(a in conf.notes)
 
       var dumpdata = %[
         (key: "version", val: %VersionAsString),
+        (key: "nimExe", val: %(getAppFilename())),
         (key: "prefixdir", val: %conf.getPrefixDir().string),
         (key: "libpath", val: %conf.libpath.string),
         (key: "project_path", val: %conf.projectFull.string),
@@ -362,7 +363,9 @@ proc mainCommand*(graph: ModuleGraph) =
     conf.cmd = cmdDump
   of "jsonscript":
     conf.cmd = cmdJsonScript
+    setOutFile(graph.config)
     commandJsonScript(graph)
+  elif commandAlreadyProcessed: discard # already handled
   else:
     rawMessage(conf, errGenerated, "invalid command: " & conf.command)
 
@@ -377,8 +380,18 @@ proc mainCommand*(graph: ModuleGraph) =
                 else: "Debug"
     let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3)
     let project = if optListFullPaths in conf.globalOptions: $conf.projectFull else: $conf.projectName
-    var output = $conf.absOutFile
+
+    var output: string
+    if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonScript:
+      output = $conf.jsonBuildFile
+    elif conf.outFile.isEmpty and conf.cmd notin {cmdJsonScript, cmdCompileToBackend, cmdDoc}:
+      # for some cmd we expect a valid absOutFile
+      output = "unknownOutput"
+    else:
+      output = $conf.absOutFile
     if optListFullPaths notin conf.globalOptions: output = output.AbsoluteFile.extractFilename
+    if optProfileVM in conf.globalOptions:
+      echo conf.dump(conf.vmProfileData)
     rawMessage(conf, hintSuccessX, [
       "loc", loc,
       "sec", sec,
@@ -395,5 +408,3 @@ proc mainCommand*(graph: ModuleGraph) =
     echo "  int tries: ", gCacheIntTries
     echo "  efficiency: ", formatFloat(1-(gCacheMisses.float/gCacheTries.float),
                                        ffDecimal, 3)
-
-  resetAttributes(conf)

compiler/modulegraphs.nim

@@ -72,9 +72,7 @@ type
     onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     globalDestructors*: seq[PNode]
-    proofEngine*: proc (graph: ModuleGraph; assumptions: seq[PNode]; toProve: PNode): (bool, string) {.nimcall.}
-    requirementsCheck*: proc (graph: ModuleGraph; assumptions: seq[PNode];
-                             call, requirement: PNode): (bool, string) {.nimcall.}
+    strongSemCheck*: proc (graph: ModuleGraph; owner: PSym; body: PNode) {.nimcall.}
     compatibleProps*: proc (graph: ModuleGraph; formal, actual: PType): bool {.nimcall.}
 
   TPassContext* = object of RootObj # the pass's context

compiler/modulepaths.nim

@@ -91,8 +91,7 @@ when false:
             if result.len > 0: return result
 
   proc scriptableImport(pkg, sub: string; info: TLineInfo): string =
-    result = resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info)
-    if result.isNil: result = ""
+    resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info)
 
   proc lookupPackage(pkg, subdir: PNode): string =
     let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: ""
@@ -112,8 +111,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
   case n.kind
   of nkStrLit, nkRStrLit, nkTripleStrLit:
     try:
-      result =
-        pathSubs(conf, n.strVal, toFullPath(conf, n.info).splitFile().dir)
+      result = pathSubs(conf, n.strVal, toFullPath(conf, n.info).splitFile().dir)
     except ValueError:
       localError(conf, n.info, "invalid path: " & n.strVal)
       result = n.strVal

compiler/modules.nim

@@ -17,22 +17,29 @@ import
 proc resetSystemArtifacts*(g: ModuleGraph) =
   magicsys.resetSysTypes(g)
 
-proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
+template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent =
+  getIdent(graph.cache, splitFile(filename).name)
+
+proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+  ## returns package symbol (skPackage) for yet to be defined module for fileIdx
+  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+  let name = getModuleIdent(graph, filename)
+  let info = newLineInfo(fileIdx, 1, 1)
   let
     pck = getPackageName(graph.config, filename.string)
     pck2 = if pck.len > 0: pck else: "unknown"
     pack = getIdent(graph.cache, pck2)
   var packSym = graph.packageSyms.strTableGet(pack)
   if packSym == nil:
-    packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, result.info)
+    packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, info)
     initStrTable(packSym.tab)
     graph.packageSyms.strTableAdd(packSym)
   else:
-    let existing = strTableGet(packSym.tab, result.name)
-    if existing != nil and existing.info.fileIndex != result.info.fileIndex:
+    let existing = strTableGet(packSym.tab, name)
+    if existing != nil and existing.info.fileIndex != info.fileIndex:
       when false:
         # we used to produce an error:
-        localError(graph.config, result.info,
+        localError(graph.config, info,
           "module names need to be unique per Nimble package; module clashes with " &
             toFullPath(graph.config, existing.info.fileIndex))
       else:
@@ -40,10 +47,13 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil
         # to resolve the conflicts:
         let pck3 = fakePackageName(graph.config, filename)
         # this makes the new `packSym`'s owner be the original `packSym`
-        packSym = newSym(skPackage, getIdent(graph.cache, pck3), packSym, result.info)
+        packSym = newSym(skPackage, getIdent(graph.cache, pck3), packSym, info)
         initStrTable(packSym.tab)
         graph.packageSyms.strTableAdd(packSym)
+  result = packSym
 
+proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
+  let packSym = getPackage(graph, fileIdx)
   result.owner = packSym
   result.position = int fileIdx
 
@@ -60,13 +70,15 @@ proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID.
   result = PSym(kind: skModule, id: -1, # for better error checking
-                name: getIdent(graph.cache, splitFile(filename).name),
+                name: getModuleIdent(graph, filename),
                 info: newLineInfo(fileIdx, 1, 1))
   if not isNimIdentifier(result.name.s):
     rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
   partialInitModule(result, graph, fileIdx, filename)
 
 proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym =
+  var flags = flags
+  if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
   result = graph.getModule(fileIdx)
   if result == nil:
     let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
@@ -74,7 +86,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
     result = r
     if result == nil:
       result = newModule(graph, fileIdx)
-      result.flags = result.flags + flags
+      result.flags.incl flags
       result.id = id
       registerModule(graph, result)
     else:
@@ -104,7 +116,7 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
   #  localError(result.info, errAttemptToRedefine, result.name.s)
   # restore the notes for outer module:
   graph.config.notes =
-    if s.owner.id == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
+    if s.getnimblePkgId == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
     else: graph.config.foreignPackageNotes
 
 proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode =
@@ -125,7 +137,7 @@ proc compileSystemModule*(graph: ModuleGraph) =
 
 proc wantMainModule*(conf: ConfigRef) =
   if conf.projectFull.isEmpty:
-    fatal(conf, newLineInfo(conf, AbsoluteFile"command line", 1, 1), errGenerated,
+    fatal(conf, newLineInfo(conf, AbsoluteFile(commandLineDesc), 1, 1), errGenerated,
         "command expects a filename")
   conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt))
 
@@ -135,7 +147,12 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
   wantMainModule(conf)
   let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
   let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
+  conf.projectMainIdx2 = projectFile
+
+  let packSym = getPackage(graph, projectFile)
+  graph.config.mainPackageId = packSym.getnimblePkgId
   graph.importStack.add projectFile
+
   if projectFile == systemFileIdx:
     discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})
   else:

compiler/msgs.nim

@@ -10,6 +10,17 @@
 import
   options, strutils, os, tables, ropes, terminal, macros,
   lineinfos, pathutils
+import std/private/miscdollars
+import strutils2
+
+type InstantiationInfo* = typeof(instantiationInfo())
+template instLoc(): InstantiationInfo = instantiationInfo(-2, fullPaths = true)
+
+template flushDot(conf, stdorr) =
+  ## safe to call multiple times
+  if conf.lastMsgWasDot:
+    conf.lastMsgWasDot = false
+    write(stdorr, "\n")
 
 proc toCChar*(c: char; result: var string) =
   case c
@@ -34,7 +45,6 @@ proc makeCString*(s: string): Rope =
   res.add('\"')
   result.add(rope(res))
 
-
 proc newFileInfo(fullPath: AbsoluteFile, projPath: RelativeFile): TFileInfo =
   result.fullPath = fullPath
   #shallow(result.fullPath)
@@ -57,6 +67,12 @@ when defined(nimpretty):
   proc fileSection*(conf: ConfigRef; fid: FileIndex; a, b: int): string =
     substr(conf.m.fileInfos[fid.int].fullContent, a, b)
 
+proc canonicalCase(path: var string) =
+  ## the idea is to only use this for checking whether a path is already in
+  ## the table but otherwise keep the original case
+  when FileSystemCaseSensitive: discard
+  else: toLowerAscii(path)
+
 proc fileInfoKnown*(conf: ConfigRef; filename: AbsoluteFile): bool =
   var
     canon: AbsoluteFile
@@ -64,6 +80,7 @@ proc fileInfoKnown*(conf: ConfigRef; filename: AbsoluteFile): bool =
     canon = canonicalizePath(conf, filename)
   except OSError:
     canon = filename
+  canon.string.canonicalCase
   result = conf.m.filenameToIndexTbl.hasKey(canon.string)
 
 proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile; isKnownFile: var bool): FileIndex =
@@ -80,15 +97,19 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile; isKnownFile: var bool
     # This flag indicates that we are working with such a path here
     pseudoPath = true
 
-  if conf.m.filenameToIndexTbl.hasKey(canon.string):
+  var canon2: string
+  forceCopy(canon2, canon.string) # because `canon` may be shallow
+  canon2.canonicalCase
+
+  if conf.m.filenameToIndexTbl.hasKey(canon2):
     isKnownFile = true
-    result = conf.m.filenameToIndexTbl[canon.string]
+    result = conf.m.filenameToIndexTbl[canon2]
   else:
     isKnownFile = false
     result = conf.m.fileInfos.len.FileIndex
     conf.m.fileInfos.add(newFileInfo(canon, if pseudoPath: RelativeFile filename
                                             else: relativeTo(canon, conf.projectPath)))
-    conf.m.filenameToIndexTbl[canon.string] = result
+    conf.m.filenameToIndexTbl[canon2] = result
 
 proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile): FileIndex =
   var dummy: bool
@@ -108,6 +129,7 @@ proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo =
 proc newLineInfo*(conf: ConfigRef; filename: AbsoluteFile, line, col: int): TLineInfo {.inline.} =
   result = newLineInfo(fileInfoIdx(conf, filename), line, col)
 
+const gCmdLineInfo* = newLineInfo(commandLineIdx, 1, 1)
 
 proc concat(strings: openArray[string]): string =
   var totalLen = 0
@@ -132,7 +154,6 @@ proc suggestQuit*() =
 # this format is understood by many text editors: it is the same that
 # Borland and Freepascal use
 const
-  PosFormat    = "$1($2, $3) "
   KindFormat   = " [$1]"
   KindColor    = fgCyan
   ErrorTitle   = "Error: "
@@ -145,6 +166,7 @@ const
   # but column numbers start with 0, however most editors expect
   # first column to be 1, so we need to +1 here
   ColOffset*   = 1
+  commandLineDesc* = "command line"
 
 proc getInfoContextLen*(conf: ConfigRef): int = return conf.m.msgContext.len
 proc setInfoContextLen*(conf: ConfigRef; L: int) = setLen(conf.m.msgContext, L)
@@ -160,9 +182,6 @@ proc getInfoContext*(conf: ConfigRef; index: int): TLineInfo =
   if i >=% conf.m.msgContext.len: result = unknownLineInfo
   else: result = conf.m.msgContext[i].info
 
-const
-  commandLineDesc = "command line"
-
 template toFilename*(conf: ConfigRef; fileIdx: FileIndex): string =
   if fileIdx.int32 < 0 or conf == nil:
     (if fileIdx == commandLineIdx: commandLineDesc else: "???")
@@ -213,16 +232,36 @@ template toFullPath*(conf: ConfigRef; info: TLineInfo): string =
 template toFullPathConsiderDirty*(conf: ConfigRef; info: TLineInfo): string =
   string toFullPathConsiderDirty(conf, info.fileIndex)
 
+type FilenameOption* = enum
+  foAbs # absolute path, e.g.: /pathto/bar/foo.nim
+  foRelProject # relative to project path, e.g.: ../foo.nim
+  foMagicSauce # magic sauce, shortest of (foAbs, foRelProject)
+  foName # lastPathPart, e.g.: foo.nim
+  foStacktrace # if optExcessiveStackTrace: foAbs else: foName
+
+proc toFilenameOption*(conf: ConfigRef, fileIdx: FileIndex, opt: FilenameOption): string =
+  case opt
+  of foAbs: result = toFullPath(conf, fileIdx)
+  of foRelProject: result = toProjPath(conf, fileIdx)
+  of foMagicSauce:
+    let
+      absPath = toFullPath(conf, fileIdx)
+      relPath = toProjPath(conf, fileIdx)
+    result = if (optListFullPaths in conf.globalOptions) or
+                (relPath.len > absPath.len) or
+                (relPath.count("..") > 2):
+               absPath
+             else:
+               relPath
+  of foName: result = toProjPath(conf, fileIdx).lastPathPart
+  of foStacktrace:
+    if optExcessiveStackTrace in conf.globalOptions:
+      result = toFilenameOption(conf, fileIdx, foAbs)
+    else:
+      result = toFilenameOption(conf, fileIdx, foName)
+
 proc toMsgFilename*(conf: ConfigRef; info: FileIndex): string =
-  let
-    absPath = toFullPath(conf, info)
-    relPath = toProjPath(conf, info)
-  result = if (optListFullPaths in conf.globalOptions) or
-              (relPath.len > absPath.len) or
-              (relPath.count("..") > 2):
-             absPath
-           else:
-             relPath
+  toFilenameOption(conf, info, foMagicSauce)
 
 template toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
   toMsgFilename(conf, info.fileIndex)
@@ -233,10 +272,11 @@ proc toLinenumber*(info: TLineInfo): int {.inline.} =
 proc toColumn*(info: TLineInfo): int {.inline.} =
   result = info.col
 
+proc toFileLineCol(info: InstantiationInfo): string {.inline.} =
+  result.toLocation(info.filename, info.line, info.column + ColOffset)
+
 proc toFileLineCol*(conf: ConfigRef; info: TLineInfo): string {.inline.} =
-  # consider calling `helpers.lineInfoToString` instead
-  result = toMsgFilename(conf, info) & "(" & $info.line & ", " &
-    $(info.col + ColOffset) & ")"
+  result.toLocation(toMsgFilename(conf, info), info.line.int, info.col.int + ColOffset)
 
 proc `$`*(conf: ConfigRef; info: TLineInfo): string = toFileLineCol(conf, info)
 
@@ -261,15 +301,16 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) =
   ## This is used for 'nim dump' etc. where we don't have nimsuggest
   ## support.
   #if conf.cmd == cmdIdeTools and optCDebug notin gGlobalOptions: return
-
   if not isNil(conf.writelnHook) and msgSkipHook notin flags:
     conf.writelnHook(s)
   elif optStdout in conf.globalOptions or msgStdout in flags:
     if eStdOut in conf.m.errorOutputs:
+      flushDot(conf, stdout)
       writeLine(stdout, s)
       flushFile(stdout)
   else:
     if eStdErr in conf.m.errorOutputs:
+      flushDot(conf, stderr)
       writeLine(stderr, s)
       # On Windows stderr is fully-buffered when piped, regardless of C std.
       when defined(windows):
@@ -298,40 +339,48 @@ macro callStyledWriteLineStderr(args: varargs[typed]): untyped =
   result.add(bindSym"stderr")
   for arg in children(args[0][1]):
     result.add(arg)
+  when false:
+    # not needed because styledWriteLine already ends with resetAttributes
+    result = newStmtList(result, newCall(bindSym"resetAttributes", bindSym"stderr"))
 
 template callWritelnHook(args: varargs[string, `$`]) =
   conf.writelnHook concat(args)
 
+proc msgWrite(conf: ConfigRef; s: string) =
+  if conf.m.errorOutputs != {}:
+    let stdOrr =
+      if optStdout in conf.globalOptions:
+        stdout
+      else:
+        stderr
+    write(stdOrr, s)
+    flushFile(stdOrr)
+    conf.lastMsgWasDot = true # subsequent writes need `flushDot`
+
 template styledMsgWriteln*(args: varargs[typed]) =
   if not isNil(conf.writelnHook):
     callIgnoringStyle(callWritelnHook, nil, args)
   elif optStdout in conf.globalOptions:
     if eStdOut in conf.m.errorOutputs:
+      flushDot(conf, stdout)
       callIgnoringStyle(writeLine, stdout, args)
       flushFile(stdout)
-  else:
-    if eStdErr in conf.m.errorOutputs:
-      if optUseColors in conf.globalOptions:
-        callStyledWriteLineStderr(args)
-      else:
-        callIgnoringStyle(writeLine, stderr, args)
-      # On Windows stderr is fully-buffered when piped, regardless of C std.
-      when defined(windows):
-        flushFile(stderr)
+  elif eStdErr in conf.m.errorOutputs:
+    flushDot(conf, stderr)
+    if optUseColors in conf.globalOptions:
+      callStyledWriteLineStderr(args)
+    else:
+      callIgnoringStyle(writeLine, stderr, args)
+    # On Windows stderr is fully-buffered when piped, regardless of C std.
+    when defined(windows):
+      flushFile(stderr)
 
-proc coordToStr(coord: int): string =
-  if coord == -1: result = "???"
-  else: result = $coord
-
-proc msgKindToString*(kind: TMsgKind): string =
+proc msgKindToString*(kind: TMsgKind): string = MsgKindToStr[kind]
   # later versions may provide translated error messages
-  result = MsgKindToStr[kind]
 
-proc getMessageStr(msg: TMsgKind, arg: string): string =
-  result = msgKindToString(msg) % [arg]
+proc getMessageStr(msg: TMsgKind, arg: string): string = msgKindToString(msg) % [arg]
 
-type
-  TErrorHandling = enum doNothing, doAbort, doRaise
+type TErrorHandling* = enum doNothing, doAbort, doRaise
 
 proc log*(s: string) =
   var f: File
@@ -345,16 +394,18 @@ proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} =
       if stackTraceAvailable() and isNil(conf.writelnHook):
         writeStackTrace()
       else:
-        styledMsgWriteln(fgRed, "No stack traceback available\n" &
-            "To create a stacktrace, rerun compilation with ./koch temp " &
-            conf.command & " <file>")
+        styledMsgWriteln(fgRed, """
+No stack traceback available
+To create a stacktrace, rerun compilation with './koch temp $1 <file>', see $2 for details""" %
+          [conf.command, "intern.html#debugging-the-compiler".createDocLink])
   quit 1
 
 proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) =
   if msg >= fatalMin and msg <= fatalMax:
     if conf.cmd == cmdIdeTools: log(s)
     quit(conf, msg)
-  if msg >= errMin and msg <= errMax:
+  if msg >= errMin and msg <= errMax or
+      (msg in warnMin..warnMax and msg in conf.warningAsErrors):
     inc(conf.errorCounter)
     conf.exitcode = 1'i8
     if conf.errorCounter >= conf.errorMax:
@@ -379,86 +430,40 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
     if context.info != lastinfo and context.info != info:
       if conf.structuredErrorHook != nil:
         conf.structuredErrorHook(conf, context.info, instantiationFrom,
-                                 Severity.Error)
+                                 Severity.Hint)
       else:
         let message = if context.detail == "":
           instantiationFrom
         else:
           instantiationOfFrom.format(context.detail)
-        styledMsgWriteln(styleBright,
-                         PosFormat % [toMsgFilename(conf, context.info),
-                                      coordToStr(context.info.line.int),
-                                      coordToStr(context.info.col+ColOffset)],
-                         resetStyle,
-                         message)
+        styledMsgWriteln(styleBright, conf.toFileLineCol(context.info), " ", resetStyle, message)
     info = context.info
 
 proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
   msg >= errGenerated and conf.cmd == cmdIdeTools and optIdeDebug notin conf.globalOptions
 
-proc rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
-  var
-    title: string
-    color: ForegroundColor
-    kind: string
-    sev: Severity
-  case msg
-  of errMin..errMax:
-    sev = Severity.Error
-    writeContext(conf, unknownLineInfo)
-    title = ErrorTitle
-    color = ErrorColor
-  of warnMin..warnMax:
-    sev = Severity.Warning
-    if not conf.hasWarn(msg): return
-    writeContext(conf, unknownLineInfo)
-    title = WarningTitle
-    color = WarningColor
-    kind = WarningsToStr[ord(msg) - ord(warnMin)]
-    inc(conf.warnCounter)
-  of hintMin..hintMax:
-    sev = Severity.Hint
-    if not conf.hasHint(msg): return
-    title = HintTitle
-    color = HintColor
-    if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
-    inc(conf.hintCounter)
-  let s = msgKindToString(msg) % args
-
-  if conf.structuredErrorHook != nil:
-    conf.structuredErrorHook(conf, unknownLineInfo,
-      s & (if kind.len > 0: KindFormat % kind else: ""), sev)
-
-  if not ignoreMsgBecauseOfIdeTools(conf, msg):
-    if kind.len > 0:
-      styledMsgWriteln(color, title, resetStyle, s,
-                       KindColor, `%`(KindFormat, kind))
-    else:
-      styledMsgWriteln(color, title, resetStyle, s)
-  handleError(conf, msg, doAbort, s)
-
-proc rawMessage*(conf: ConfigRef; msg: TMsgKind, arg: string) =
-  rawMessage(conf, msg, [arg])
-
-proc resetAttributes*(conf: ConfigRef) =
-  if {optUseColors, optStdout} * conf.globalOptions == {optUseColors}:
-    terminal.resetAttributes(stderr)
-
 proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
   conf.m.fileInfos[fileIdx.int32].lines.add line
 
-proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
-  if i.fileIndex.int32 < 0: return ""
-
-  if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
+proc numLines*(conf: ConfigRef, fileIdx: FileIndex): int =
+  ## xxx there's an off by 1 error that should be fixed; if a file ends with "foo" or "foo\n"
+  ## it will return same number of lines (ie, a trailing empty line is discounted)
+  result = conf.m.fileInfos[fileIdx.int32].lines.len
+  if result == 0:
     try:
-      for line in lines(toFullPathConsiderDirty(conf, i)):
-        addSourceLine conf, i.fileIndex, line.string
+      for line in lines(toFullPathConsiderDirty(conf, fileIdx).string):
+        addSourceLine conf, fileIdx, line.string
     except IOError:
       discard
-  assert i.fileIndex.int32 < conf.m.fileInfos.len
+    result = conf.m.fileInfos[fileIdx.int32].lines.len
+
+proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
+  ## 1-based index (matches editor line numbers); 1st line is for i.line = 1
+  ## last valid line is `numLines` inclusive
+  if i.fileIndex.int32 < 0: return ""
+  let num = numLines(conf, i.fileIndex)
   # can happen if the error points to EOF:
-  if i.line.int > conf.m.fileInfos[i.fileIndex.int32].lines.len: return ""
+  if i.line.int > num: return ""
 
   result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1]
 
@@ -473,101 +478,127 @@ proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): s
               of warnMin..warnMax: WarningTitle
               of hintMin..hintMax: HintTitle
               else: ErrorTitle
-  result = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                        coordToStr(info.col+ColOffset)] &
-           title &
-           getMessageStr(msg, arg)
+  conf.toFileLineCol(info) & " " & title & getMessageStr(msg, arg)
 
-proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
-               eh: TErrorHandling) =
+proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
+               eh: TErrorHandling, info2: InstantiationInfo, isRaw = false) {.noinline.} =
   var
     title: string
     color: ForegroundColor
-    kind:  string
     ignoreMsg = false
     sev: Severity
+  let kind = if msg in warnMin..hintMax and msg != hintUserRaw: $msg else: "" # xxx not sure why hintUserRaw is special
   case msg
   of errMin..errMax:
     sev = Severity.Error
     writeContext(conf, info)
     title = ErrorTitle
     color = ErrorColor
-    # we try to filter error messages so that not two error message
-    # in the same file and line are produced:
-    #ignoreMsg = lastError == info and eh != doAbort
-    conf.m.lastError = info
+    when false:
+      # we try to filter error messages so that not two error message
+      # in the same file and line are produced:
+      # xxx `lastError` is only used in this disabled code; but could be useful to revive
+      ignoreMsg = conf.m.lastError == info and info != unknownLineInfo and eh != doAbort
+    if info != unknownLineInfo: conf.m.lastError = info
   of warnMin..warnMax:
     sev = Severity.Warning
     ignoreMsg = not conf.hasWarn(msg)
+    if msg in conf.warningAsErrors:
+      ignoreMsg = false
+      title = ErrorTitle
+    else:
+      title = WarningTitle
     if not ignoreMsg: writeContext(conf, info)
-    title = WarningTitle
     color = WarningColor
-    kind = WarningsToStr[ord(msg) - ord(warnMin)]
     inc(conf.warnCounter)
   of hintMin..hintMax:
     sev = Severity.Hint
     ignoreMsg = not conf.hasHint(msg)
     title = HintTitle
     color = HintColor
-    if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
     inc(conf.hintCounter)
-  let x = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                       coordToStr(info.col+ColOffset)]
-  let s = getMessageStr(msg, arg)
 
+  let s = if isRaw: arg else: getMessageStr(msg, arg)
   if not ignoreMsg:
+    let loc = if info != unknownLineInfo: conf.toFileLineCol(info) & " " else: ""
+    var kindmsg = if kind.len > 0: KindFormat % kind else: ""
     if conf.structuredErrorHook != nil:
-      conf.structuredErrorHook(conf, info, s & (if kind.len > 0: KindFormat % kind else: ""), sev)
+      conf.structuredErrorHook(conf, info, s & kindmsg, sev)
     if not ignoreMsgBecauseOfIdeTools(conf, msg):
-      if kind.len > 0:
-        styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s,
-                         KindColor, `%`(KindFormat, kind))
+      if msg == hintProcessing:
+        msgWrite(conf, ".")
       else:
-        styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s)
-      if conf.hasHint(hintSource):
-        conf.writeSurroundingSrc(info)
+        styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg)
+        if conf.hasHint(hintSource) and info != unknownLineInfo:
+          conf.writeSurroundingSrc(info)
+        if hintMsgOrigin in conf.mainPackageNotes:
+          styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle,
+            " compiler msg initiated here", KindColor,
+            KindFormat % $hintMsgOrigin,
+            resetStyle)
   handleError(conf, msg, eh, s)
 
-proc fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  # this fixes bug #7080 so that it is at least obvious 'fatal'
-  # was executed.
+template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
+  let arg = msgKindToString(msg) % args
+  liMessage(conf, unknownLineInfo, msg, arg, eh = doAbort, instLoc(), isRaw = true)
+
+template rawMessage*(conf: ConfigRef; msg: TMsgKind, arg: string) =
+  liMessage(conf, unknownLineInfo, msg, arg, eh = doAbort, instLoc())
+
+template fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  # this fixes bug #7080 so that it is at least obvious 'fatal' was executed.
   conf.m.errorOutputs = {eStdOut, eStdErr}
-  liMessage(conf, info, msg, arg, doAbort)
+  liMessage(conf, info, msg, arg, doAbort, instLoc())
 
-proc globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  liMessage(conf, info, msg, arg, doRaise)
+template globalAssert*(conf: ConfigRef; cond: untyped, info: TLineInfo = unknownLineInfo, arg = "") =
+  ## avoids boilerplate
+  if not cond:
+    var arg2 = "'$1' failed" % [astToStr(cond)]
+    if arg.len > 0: arg2.add "; " & astToStr(arg) & ": " & arg
+    liMessage(conf, info, errGenerated, arg2, doRaise, instLoc())
 
-proc globalError*(conf: ConfigRef; info: TLineInfo, arg: string) =
-  liMessage(conf, info, errGenerated, arg, doRaise)
+template globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  ## `local` means compilation keeps going until errorMax is reached (via `doNothing`),
+  ## `global` means it stops.
+  liMessage(conf, info, msg, arg, doRaise, instLoc())
 
-proc localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  liMessage(conf, info, msg, arg, doNothing)
+template globalError*(conf: ConfigRef; info: TLineInfo, arg: string) =
+  liMessage(conf, info, errGenerated, arg, doRaise, instLoc())
 
-proc localError*(conf: ConfigRef; info: TLineInfo, arg: string) =
-  liMessage(conf, info, errGenerated, arg, doNothing)
+template localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  liMessage(conf, info, msg, arg, doNothing, instLoc())
 
-proc localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openArray[string]) =
-  localError(conf, info, format % params)
+template localError*(conf: ConfigRef; info: TLineInfo, arg: string) =
+  liMessage(conf, info, errGenerated, arg, doNothing, instLoc())
 
-proc message*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  liMessage(conf, info, msg, arg, doNothing)
+template localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openArray[string]) =
+  liMessage(conf, info, errGenerated, format % params, doNothing, instLoc())
 
-proc internalError*(conf: ConfigRef; info: TLineInfo, errMsg: string) =
+template message*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  liMessage(conf, info, msg, arg, doNothing, instLoc())
+
+proc internalErrorImpl(conf: ConfigRef; info: TLineInfo, errMsg: string, info2: InstantiationInfo) =
   if conf.cmd == cmdIdeTools and conf.structuredErrorHook.isNil: return
   writeContext(conf, info)
-  liMessage(conf, info, errInternal, errMsg, doAbort)
+  liMessage(conf, info, errInternal, errMsg, doAbort, info2)
 
-proc internalError*(conf: ConfigRef; errMsg: string) =
-  if conf.cmd == cmdIdeTools and conf.structuredErrorHook.isNil: return
-  writeContext(conf, unknownLineInfo)
-  rawMessage(conf, errInternal, errMsg)
+template internalError*(conf: ConfigRef; info: TLineInfo, errMsg: string) =
+  internalErrorImpl(conf, info, errMsg, instLoc())
 
-template assertNotNil*(conf: ConfigRef; e): untyped =
-  if e == nil: internalError(conf, $instantiationInfo())
-  e
+template internalError*(conf: ConfigRef; errMsg: string) =
+  internalErrorImpl(conf, unknownLineInfo, errMsg, instLoc())
 
 template internalAssert*(conf: ConfigRef, e: bool) =
-  if not e: internalError(conf, $instantiationInfo())
+  # xxx merge with `globalAssert`
+  if not e:
+    const info2 = instLoc()
+    let arg = info2.toFileLineCol
+    internalErrorImpl(conf, unknownLineInfo, arg, info2)
+
+template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string) =
+  let m = "'$2' should be: '$1'" % [beau, got]
+  let msg = if optStyleError in conf.globalOptions: errGenerated else: hintName
+  liMessage(conf, info, msg, m, doNothing, instLoc())
 
 proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
   if i.fileIndex.int32 < 0:
@@ -577,25 +608,9 @@ proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
   else:
     result = conf.m.fileInfos[i.fileIndex.int32].quotedName
 
-proc listWarnings*(conf: ConfigRef) =
-  msgWriteln(conf, "Warnings:")
-  for warn in warnMin..warnMax:
-    msgWriteln(conf, "  [$1] $2" % [
-      if warn in conf.notes: "x" else: " ",
-      lineinfos.WarningsToStr[ord(warn) - ord(warnMin)]
-    ])
+template listMsg(title, r) =
+  msgWriteln(conf, title)
+  for a in r: msgWriteln(conf, "  [$1] $2" % [if a in conf.notes: "x" else: " ", $a])
 
-proc listHints*(conf: ConfigRef) =
-  msgWriteln(conf, "Hints:")
-  for hint in hintMin..hintMax:
-    msgWriteln(conf, "  [$1] $2" % [
-      if hint in conf.notes: "x" else: " ",
-      lineinfos.HintsToStr[ord(hint) - ord(hintMin)]
-    ])
-
-proc lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string) =
-  let m = "'$2' should be: '$1'" % [beau, got]
-  if optStyleError in conf.globalOptions:
-    localError(conf, info, m)
-  else:
-    message(conf, info, hintName, m)
+proc listWarnings*(conf: ConfigRef) = listMsg("Warnings:", warnMin..warnMax)
+proc listHints*(conf: ConfigRef) = listMsg("Hints:", hintMin..hintMax)

compiler/nim.cfg

@@ -19,5 +19,10 @@ define:useStdoutAsStdmsg
   styleCheck:error
 @end
 
+
 #define:useNodeIds
 #gc:markAndSweep
+
+@if nimHasWarningObservableStores:
+  warning[ObservableStores]: off
+@end

compiler/nim.nim

@@ -22,9 +22,10 @@ import
   commands, options, msgs,
   extccomp, strutils, os, main, parseopt,
   idents, lineinfos, cmdlinehelper,
-  pathutils
+  pathutils, modulegraphs
 
-include nodejs
+from std/browsers import openDefaultBrowser
+from nodejs import findNodeJs
 
 when hasTinyCBackend:
   import tccgen
@@ -33,20 +34,6 @@ when defined(profiler) or defined(memProfiler):
   {.hint: "Profiling support is turned on!".}
   import nimprof
 
-proc prependCurDir(f: AbsoluteFile): AbsoluteFile =
-  when defined(unix):
-    if os.isAbsolute(f.string): result = f
-    else: result = AbsoluteFile("./" & f.string)
-  else:
-    result = f
-
-proc addCmdPrefix*(result: var string, kind: CmdLineKind) =
-  # consider moving this to std/parseopt
-  case kind
-  of cmdLongOption: result.add "--"
-  of cmdShortOption: result.add "-"
-  of cmdArgument, cmdEnd: discard
-
 proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
   var p = parseopt.initOptParser(cmd)
   var argsCount = 0
@@ -66,7 +53,7 @@ proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
         config.commandLine.add ':'
         config.commandLine.add p.val.quoteShell
 
-      if p.key == " ":
+      if p.key == "": # `-` was passed to indicate main project is stdin
         p.key = "-"
         if processArgument(pass, p, argsCount, config): break
       else:
@@ -77,14 +64,13 @@ proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
       if processArgument(pass, p, argsCount, config): break
   if pass == passCmd2:
     if {optRun, optWasNimscript} * config.globalOptions == {} and
-        config.arguments.len > 0 and config.command.normalize notin ["run", "e"]:
+        config.arguments.len > 0 and config.command.normalize notin ["run", "e", "r"]:
       rawMessage(config, errGenerated, errArgsNeedRunOption)
 
 proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
   let self = NimProg(
     supportsStdinFile: true,
-    processCmdLine: processCmdLine,
-    mainCommand: mainCommand
+    processCmdLine: processCmdLine
   )
   self.initDefinesProg(conf, "nim_compiler")
   if paramCount() == 0:
@@ -92,7 +78,9 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
     return
 
   self.processCmdLineAndProjectPath(conf)
-  if not self.loadConfigsAndRunMainCommand(cache, conf): return
+  var graph = newModuleGraph(cache, conf)
+  if not self.loadConfigsAndRunMainCommand(cache, conf, graph): return
+  mainCommand(graph)
   if conf.hasHint(hintGCStats): echo(GC_getStatistics())
   #echo(GC_getStatistics())
   if conf.errorCounter != 0: return
@@ -100,11 +88,23 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
     if conf.cmd == cmdRun:
       tccgen.run(conf, conf.arguments)
   if optRun in conf.globalOptions:
-    var ex = quoteShell conf.absOutFile
-    if conf.cmd == cmdCompileToJS:
-      execExternalProgram(conf, findNodeJs() & " " & ex & ' ' & conf.arguments)
+    let output = conf.absOutFile
+    case conf.cmd
+    of cmdCompileToBackend:
+      var cmdPrefix = ""
+      case conf.backend
+      of backendC, backendCpp, backendObjc: discard
+      of backendJs: cmdPrefix = findNodeJs() & " "
+      else: doAssert false, $conf.backend
+      execExternalProgram(conf, cmdPrefix & output.quoteShell & ' ' & conf.arguments)
+    of cmdDoc, cmdRst2html:
+      if conf.arguments.len > 0:
+        # reserved for future use
+        rawMessage(conf, errGenerated, "'$1 cannot handle arguments" % [$conf.cmd])
+      openDefaultBrowser($output)
     else:
-      execExternalProgram(conf, ex & ' ' & conf.arguments)
+      # support as needed
+      rawMessage(conf, errGenerated, "'$1 cannot handle --run" % [$conf.cmd])
 
 when declared(GC_setMaxPause):
   GC_setMaxPause 2_000

compiler/nimblecmd.nim

@@ -134,30 +134,3 @@ proc nimblePath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
   if i != -1:
     conf.nimblePaths.delete(i)
   conf.nimblePaths.insert(path, 0)
-
-when isMainModule:
-  proc v(s: string): Version = s.newVersion
-  # #head is special in the sense that it's assumed to always be newest.
-  doAssert v"1.0" < v"#head"
-  doAssert v"1.0" < v"1.1"
-  doAssert v"1.0.1" < v"1.1"
-  doAssert v"1" < v"1.1"
-  doAssert v"#aaaqwe" < v"1.1" # We cannot assume that a branch is newer.
-  doAssert v"#a111" < v"#head"
-
-  let conf = newConfigRef()
-  var rr = newStringTable()
-  addPackage conf, rr, "irc-#a111", unknownLineInfo
-  addPackage conf, rr, "irc-#head", unknownLineInfo
-  addPackage conf, rr, "irc-0.1.0", unknownLineInfo
-  #addPackage conf, rr, "irc", unknownLineInfo
-  #addPackage conf, rr, "another", unknownLineInfo
-  addPackage conf, rr, "another-0.1", unknownLineInfo
-
-  addPackage conf, rr, "ab-0.1.3", unknownLineInfo
-  addPackage conf, rr, "ab-0.1", unknownLineInfo
-  addPackage conf, rr, "justone-1.0", unknownLineInfo
-
-  doAssert toSeq(rr.chosen) ==
-    @["irc-#head", "another-0.1", "ab-0.1.3", "justone-1.0"]
-

compiler/nimconf.nim

@@ -16,13 +16,13 @@ import
 # ---------------- configuration file parser -----------------------------
 # we use Nim's scanner here to save space and work
 
-proc ppGetTok(L: var TLexer, tok: var TToken) =
+proc ppGetTok(L: var Lexer, tok: var Token) =
   # simple filter
   rawGetTok(L, tok)
   while tok.tokType in {tkComment}: rawGetTok(L, tok)
 
-proc parseExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool
-proc parseAtom(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
+proc parseExpr(L: var Lexer, tok: var Token; config: ConfigRef): bool
+proc parseAtom(L: var Lexer, tok: var Token; config: ConfigRef): bool =
   if tok.tokType == tkParLe:
     ppGetTok(L, tok)
     result = parseExpr(L, tok, config)
@@ -35,21 +35,21 @@ proc parseAtom(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
     result = isDefined(config, tok.ident.s)
     ppGetTok(L, tok)
 
-proc parseAndExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
+proc parseAndExpr(L: var Lexer, tok: var Token; config: ConfigRef): bool =
   result = parseAtom(L, tok, config)
   while tok.tokType == tkAnd:
     ppGetTok(L, tok)          # skip "and"
     var b = parseAtom(L, tok, config)
     result = result and b
 
-proc parseExpr(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
+proc parseExpr(L: var Lexer, tok: var Token; config: ConfigRef): bool =
   result = parseAndExpr(L, tok, config)
   while tok.tokType == tkOr:
     ppGetTok(L, tok)          # skip "or"
     var b = parseAndExpr(L, tok, config)
     result = result or b
 
-proc evalppIf(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
+proc evalppIf(L: var Lexer, tok: var Token; config: ConfigRef): bool =
   ppGetTok(L, tok)            # skip 'if' or 'elif'
   result = parseExpr(L, tok, config)
   if tok.tokType == tkColon: ppGetTok(L, tok)
@@ -57,7 +57,7 @@ proc evalppIf(L: var TLexer, tok: var TToken; config: ConfigRef): bool =
 
 #var condStack: seq[bool] = @[]
 
-proc doEnd(L: var TLexer, tok: var TToken; condStack: var seq[bool]) =
+proc doEnd(L: var Lexer, tok: var Token; condStack: var seq[bool]) =
   if high(condStack) < 0: lexMessage(L, errGenerated, "expected @if")
   ppGetTok(L, tok)            # skip 'end'
   setLen(condStack, high(condStack))
@@ -66,21 +66,21 @@ type
   TJumpDest = enum
     jdEndif, jdElseEndif
 
-proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest; config: ConfigRef;
+proc jumpToDirective(L: var Lexer, tok: var Token, dest: TJumpDest; config: ConfigRef;
                      condStack: var seq[bool])
-proc doElse(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var seq[bool]) =
+proc doElse(L: var Lexer, tok: var Token; config: ConfigRef; condStack: var seq[bool]) =
   if high(condStack) < 0: lexMessage(L, errGenerated, "expected @if")
   ppGetTok(L, tok)
   if tok.tokType == tkColon: ppGetTok(L, tok)
   if condStack[high(condStack)]: jumpToDirective(L, tok, jdEndif, config, condStack)
 
-proc doElif(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var seq[bool]) =
+proc doElif(L: var Lexer, tok: var Token; config: ConfigRef; condStack: var seq[bool]) =
   if high(condStack) < 0: lexMessage(L, errGenerated, "expected @if")
   var res = evalppIf(L, tok, config)
   if condStack[high(condStack)] or not res: jumpToDirective(L, tok, jdElseEndif, config, condStack)
   else: condStack[high(condStack)] = true
 
-proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest; config: ConfigRef;
+proc jumpToDirective(L: var Lexer, tok: var Token, dest: TJumpDest; config: ConfigRef;
                      condStack: var seq[bool]) =
   var nestedIfs = 0
   while true:
@@ -110,7 +110,7 @@ proc jumpToDirective(L: var TLexer, tok: var TToken, dest: TJumpDest; config: Co
     else:
       ppGetTok(L, tok)
 
-proc parseDirective(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var seq[bool]) =
+proc parseDirective(L: var Lexer, tok: var Token; config: ConfigRef; condStack: var seq[bool]) =
   ppGetTok(L, tok)            # skip @
   case whichKeyword(tok.ident)
   of wIf:
@@ -149,16 +149,16 @@ proc parseDirective(L: var TLexer, tok: var TToken; config: ConfigRef; condStack
     else:
       lexMessage(L, errGenerated, "invalid directive: '$1'" % $tok)
 
-proc confTok(L: var TLexer, tok: var TToken; config: ConfigRef; condStack: var seq[bool]) =
+proc confTok(L: var Lexer, tok: var Token; config: ConfigRef; condStack: var seq[bool]) =
   ppGetTok(L, tok)
   while tok.ident != nil and tok.ident.s == "@":
     parseDirective(L, tok, config, condStack)    # else: give the token to the parser
 
-proc checkSymbol(L: TLexer, tok: TToken) =
+proc checkSymbol(L: Lexer, tok: Token) =
   if tok.tokType notin {tkSymbol..tkInt64Lit, tkStrLit..tkTripleStrLit}:
     lexMessage(L, errGenerated, "expected identifier, but got: " & $tok)
 
-proc parseAssignment(L: var TLexer, tok: var TToken;
+proc parseAssignment(L: var Lexer, tok: var Token;
                      config: ConfigRef; condStack: var seq[bool]) =
   if tok.ident != nil:
     if tok.ident.s == "-" or tok.ident.s == "--":
@@ -211,8 +211,8 @@ proc parseAssignment(L: var TLexer, tok: var TToken;
 proc readConfigFile*(filename: AbsoluteFile; cache: IdentCache;
                     config: ConfigRef): bool =
   var
-    L: TLexer
-    tok: TToken
+    L: Lexer
+    tok: Token
     stream: PLLStream
   stream = llStreamOpen(filename, fmRead)
   if stream != nil:

compiler/nimeval.nim

@@ -10,9 +10,9 @@
 ## exposes the Nim VM to clients.
 import
   ast, astalgo, modules, passes, condsyms,
-  options, sem, semdata, llstream, vm, vmdef,
-  modulegraphs, idents, os, pathutils, passaux,
-  scriptconfig
+  options, sem, semdata, llstream, lineinfos, vm,
+  vmdef, modulegraphs, idents, os, pathutils,
+  passaux, scriptconfig
 
 type
   Interpreter* = ref object ## Use Nim as an interpreter with this object
@@ -81,6 +81,9 @@ proc findNimStdLib*(): string =
   ## Returns "" on failure.
   try:
     let nimexe = os.findExe("nim")
+      # this can't work with choosenim shims, refs https://github.com/dom96/choosenim/issues/189
+      # it'd need `nim dump --dump.format:json . | jq -r .libpath`
+      # which we should simplify as `nim dump --key:libpath`
     if nimexe.len == 0: return ""
     result = nimexe.splitPath()[0] /../ "lib"
     if not fileExists(result / "system.nim"):
@@ -93,20 +96,22 @@ proc findNimStdLib*(): string =
 proc findNimStdLibCompileTime*(): string =
   ## Same as ``findNimStdLib`` but uses source files used at compile time,
   ## and asserts on error.
-  const sourcePath = currentSourcePath()
-  result = sourcePath.parentDir.parentDir / "lib"
+  const exe = getCurrentCompilerExe()
+  result = exe.splitFile.dir.parentDir / "lib"
   doAssert fileExists(result / "system.nim"), "result:" & result
 
 proc createInterpreter*(scriptName: string;
                         searchPaths: openArray[string];
-                        flags: TSandboxFlags = {}): Interpreter =
+                        flags: TSandboxFlags = {},
+                        defines = @[("nimscript", "true")],
+                        registerOps = true): Interpreter =
   var conf = newConfigRef()
   var cache = newIdentCache()
   var graph = newModuleGraph(cache, conf)
   connectCallbacks(graph)
   initDefines(conf.symbols)
-  defineSymbol(conf.symbols, "nimscript")
-  defineSymbol(conf.symbols, "nimconfig")
+  for define in defines:
+    defineSymbol(conf.symbols, define[0], define[1])
   registerPass(graph, semPass)
   registerPass(graph, evalPass)
 
@@ -119,6 +124,8 @@ proc createInterpreter*(scriptName: string;
   var vm = newCtx(m, cache, graph)
   vm.mode = emRepl
   vm.features = flags
+  if registerOps:
+    vm.registerAdditionalOps() # Required to register parts of stdlib modules
   graph.vm = vm
   graph.compileSystemModule()
   result = Interpreter(mainModule: m, graph: graph, scriptName: scriptName)
@@ -127,9 +134,15 @@ proc destroyInterpreter*(i: Interpreter) =
   ## destructor.
   discard "currently nothing to do."
 
+proc registerErrorHook*(i: Interpreter, hook:
+                        proc (config: ConfigRef; info: TLineInfo; msg: string;
+                              severity: Severity) {.gcsafe.}) =
+  i.graph.config.structuredErrorHook = hook
+
 proc runRepl*(r: TLLRepl;
               searchPaths: openArray[string];
               supportNimscript: bool) =
+  ## deadcode but please don't remove... might be revived
   var conf = newConfigRef()
   var cache = newIdentCache()
   var graph = newModuleGraph(cache, conf)
@@ -139,7 +152,7 @@ proc runRepl*(r: TLLRepl;
     if conf.libpath.isEmpty: conf.libpath = AbsoluteDir p
 
   conf.cmd = cmdInteractive
-  conf.errorMax = high(int)
+  conf.setErrorMaxHighMaybe
   initDefines(conf.symbols)
   defineSymbol(conf.symbols, "nimscript")
   if supportNimscript: defineSymbol(conf.symbols, "nimconfig")

compiler/nimlexbase.nim

@@ -39,7 +39,9 @@ const
 type
   TBaseLexer* = object of RootObj
     bufpos*: int
-    buf*: string
+    buf*: cstring
+    bufStorage: string
+    bufLen: int
     stream*: PLLStream        # we read from this stream
     lineNumber*: int          # the current line number
                               # private data:
@@ -75,8 +77,8 @@ proc fillBuffer(L: var TBaseLexer) =
     oldBufLen: int
   # we know here that pos == L.sentinel, but not if this proc
   # is called the first time by initBaseLexer()
-  assert(L.sentinel < L.buf.len)
-  toCopy = L.buf.len - L.sentinel - 1
+  assert(L.sentinel < L.bufLen)
+  toCopy = L.bufLen - L.sentinel - 1
   assert(toCopy >= 0)
   if toCopy > 0:
     moveMem(addr L.buf[0], addr L.buf[L.sentinel + 1], toCopy)
@@ -90,7 +92,7 @@ proc fillBuffer(L: var TBaseLexer) =
     # compute sentinel:
     dec(s)                    # BUGFIX (valgrind)
     while true:
-      assert(s < L.buf.len)
+      assert(s < L.bufLen)
       while (s >= 0) and not (L.buf[s] in NewLines): dec(s)
       if s >= 0:
         # we found an appropriate character for a sentinel:
@@ -99,16 +101,18 @@ proc fillBuffer(L: var TBaseLexer) =
       else:
         # rather than to give up here because the line is too long,
         # double the buffer's size and try again:
-        oldBufLen = L.buf.len
-        L.buf.setLen(L.buf.len * 2)
-        assert(L.buf.len - oldBufLen == oldBufLen)
+        oldBufLen = L.bufLen
+        L.bufLen = L.bufLen * 2
+        L.bufStorage.setLen(L.bufLen)
+        L.buf = L.bufStorage
+        assert(L.bufLen - oldBufLen == oldBufLen)
         charsRead = llStreamRead(L.stream, addr(L.buf[oldBufLen]),
                                  oldBufLen)
         if charsRead < oldBufLen:
           L.buf[oldBufLen + charsRead] = EndOfFile
           L.sentinel = oldBufLen + charsRead
           break
-        s = L.buf.len - 1
+        s = L.bufLen - 1
 
 proc fillBaseLexer(L: var TBaseLexer, pos: int): int =
   assert(pos <= L.sentinel)
@@ -142,7 +146,9 @@ proc openBaseLexer(L: var TBaseLexer, inputstream: PLLStream, bufLen = 8192) =
   assert(bufLen > 0)
   L.bufpos = 0
   L.offsetBase = 0
-  L.buf = newString(bufLen)
+  L.bufStorage = newString(bufLen)
+  L.buf = L.bufStorage
+  L.bufLen = bufLen
   L.sentinel = bufLen - 1
   L.lineStart = 0
   L.lineNumber = 1            # lines start at 1
@@ -156,9 +162,9 @@ proc getColNumber(L: TBaseLexer, pos: int): int =
 proc getCurrentLine(L: TBaseLexer, marker: bool = true): string =
   result = ""
   var i = L.lineStart
-  while not (L.buf[i] in {CR, LF, EndOfFile}):
-    result.add(L.buf[i])
-    inc(i)
-  result.add("\n")
+  while L.buf[i] notin {CR, LF, EndOfFile}:
+    result.add L.buf[i]
+    inc i
+  result.add "\n"
   if marker:
-    result.add(spaces(getColNumber(L, L.bufpos)) & '^' & "\n")
+    result.add spaces(getColNumber(L, L.bufpos)) & '^' & "\n"

compiler/nimpaths.nim

@@ -0,0 +1,47 @@
+##[
+Represents absolute paths, but using a symbolic variables (eg $nimr) which can be
+resolved at runtime; this avoids hardcoding at compile time absolute paths so
+that the project root can be relocated.
+
+xxx consider some refactoring with $nim/testament/lib/stdtest/specialpaths.nim;
+specialpaths is simpler because it doesn't need variables to be relocatable at
+runtime (eg for use in testament)
+
+interpolation variables:
+  $nimr: such that `$nimr/lib/system.nim` exists (avoids confusion with $nim binary)
+         in compiler, it's obtainable via getPrefixDir(); for other tools (eg koch),
+        this could be getCurrentDir() or getAppFilename().parentDir.parentDir,
+        depending on use case
+
+Unstable API
+]##
+
+import std/[os,strutils]
+
+const
+  docCss* = "$nimr/doc/nimdoc.css"
+  docHackNim* = "$nimr/tools/dochack/dochack.nim"
+  docHackJs* = docHackNim.changeFileExt("js")
+  docHackJsFname* = docHackJs.lastPathPart
+  theindexFname* = "theindex.html"
+  nimdocOutCss* = "nimdoc.out.css"
+    # `out` to make it easier to use with gitignore in user's repos
+  htmldocsDirname* = "htmldocs"
+  dotdotMangle* = "_._"  ## refs #13223
+    # if this changes, make sure it's consistent with `esc` and `escapeLink`
+    # lots of other obvious options won't work, see #14454; `_` could work too
+
+proc interp*(path: string, nimr: string): string =
+  result = path % ["nimr", nimr]
+  doAssert '$' notin result, $(path, nimr, result) # avoids un-interpolated variables in output
+
+proc getDocHacksJs*(nimr: string, nim = getCurrentCompilerExe(), forceRebuild = false): string =
+  ## return absolute path to dochack.js, rebuilding if it doesn't exist or if
+  ## `forceRebuild`.
+  let docHackJs2 = docHackJs.interp(nimr = nimr)
+  if forceRebuild or not docHackJs2.fileExists:
+    let cmd =  "$nim js -d:release $file" % ["nim", nim.quoteShell, "file", docHackNim.interp(nimr = nimr).quoteShell]
+    echo "getDocHacksJs: cmd: " & cmd
+    doAssert execShellCmd(cmd) == 0, $(cmd)
+  doAssert docHackJs2.fileExists
+  result = docHackJs2

compiler/nimsets.nim

@@ -58,18 +58,18 @@ proc someInSet*(s: PNode, a, b: PNode): bool =
         return true
   result = false
 
-proc toBitSet*(conf: ConfigRef; s: PNode, b: var TBitSet) =
+proc toBitSet*(conf: ConfigRef; s: PNode): TBitSet =
   var first, j: Int128
   first = firstOrd(conf, s.typ[0])
-  bitSetInit(b, int(getSize(conf, s.typ)))
+  bitSetInit(result, int(getSize(conf, s.typ)))
   for i in 0..<s.len:
     if s[i].kind == nkRange:
       j = getOrdValue(s[i][0], first)
       while j <= getOrdValue(s[i][1], first):
-        bitSetIncl(b, toInt64(j - first))
+        bitSetIncl(result, toInt64(j - first))
         inc(j)
     else:
-      bitSetIncl(b, toInt64(getOrdValue(s[i]) - first))
+      bitSetIncl(result, toInt64(getOrdValue(s[i]) - first))
 
 proc toTreeSet*(conf: ConfigRef; s: TBitSet, settype: PType, info: TLineInfo): PNode =
   var
@@ -106,9 +106,8 @@ proc toTreeSet*(conf: ConfigRef; s: TBitSet, settype: PType, info: TLineInfo): P
     inc(e)
 
 template nodeSetOp(a, b: PNode, op: untyped) {.dirty.} =
-  var x, y: TBitSet
-  toBitSet(conf, a, x)
-  toBitSet(conf, b, y)
+  var x = toBitSet(conf, a)
+  let y = toBitSet(conf, b)
   op(x, y)
   result = toTreeSet(conf, x, a.typ, a.info)
 
@@ -118,31 +117,26 @@ proc intersectSets*(conf: ConfigRef; a, b: PNode): PNode = nodeSetOp(a, b, bitSe
 proc symdiffSets*(conf: ConfigRef; a, b: PNode): PNode = nodeSetOp(a, b, bitSetSymDiff)
 
 proc containsSets*(conf: ConfigRef; a, b: PNode): bool =
-  var x, y: TBitSet
-  toBitSet(conf, a, x)
-  toBitSet(conf, b, y)
+  let x = toBitSet(conf, a)
+  let y = toBitSet(conf, b)
   result = bitSetContains(x, y)
 
 proc equalSets*(conf: ConfigRef; a, b: PNode): bool =
-  var x, y: TBitSet
-  toBitSet(conf, a, x)
-  toBitSet(conf, b, y)
+  let x = toBitSet(conf, a)
+  let y = toBitSet(conf, b)
   result = bitSetEquals(x, y)
 
 proc complement*(conf: ConfigRef; a: PNode): PNode =
-  var x: TBitSet
-  toBitSet(conf, a, x)
+  var x = toBitSet(conf, a)
   for i in 0..high(x): x[i] = not x[i]
   result = toTreeSet(conf, x, a.typ, a.info)
 
 proc deduplicate*(conf: ConfigRef; a: PNode): PNode =
-  var x: TBitSet
-  toBitSet(conf, a, x)
+  let x = toBitSet(conf, a)
   result = toTreeSet(conf, x, a.typ, a.info)
 
 proc cardSet*(conf: ConfigRef; a: PNode): BiggestInt =
-  var x: TBitSet
-  toBitSet(conf, a, x)
+  let x = toBitSet(conf, a)
   result = bitSetCard(x)
 
 proc setHasRange*(s: PNode): bool =

compiler/optimizer.nim

@@ -0,0 +1,285 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Optimizer:
+## - elide 'wasMoved(x); destroy(x)' pairs
+## - recognize "all paths lead to 'wasMoved(x)'"
+
+import
+  ast, renderer, idents, intsets
+
+from trees import exprStructuralEquivalent
+
+const
+  nfMarkForDeletion = nfNone # faster than a lookup table
+
+type
+  BasicBlock = object
+    wasMovedLocs: seq[PNode]
+    kind: TNodeKind
+    hasReturn, hasBreak: bool
+    label: PSym # can be nil
+    parent: ptr BasicBlock
+
+  Con = object
+    somethingTodo: bool
+    inFinally: int
+
+proc nestedBlock(parent: var BasicBlock; kind: TNodeKind): BasicBlock =
+  BasicBlock(wasMovedLocs: @[], kind: kind, hasReturn: false, hasBreak: false,
+    label: nil, parent: addr(parent))
+
+proc breakStmt(b: var BasicBlock; n: PNode) =
+  var it = addr(b)
+  while it != nil:
+    it.wasMovedLocs.setLen 0
+    it.hasBreak = true
+
+    if n.kind == nkSym:
+      if it.label == n.sym: break
+    else:
+      # unnamed break leaves the block is nkWhileStmt or the like:
+      if it.kind in {nkWhileStmt, nkBlockStmt, nkBlockExpr}: break
+
+    it = it.parent
+
+proc returnStmt(b: var BasicBlock) =
+  b.hasReturn = true
+  var it = addr(b)
+  while it != nil:
+    it.wasMovedLocs.setLen 0
+    it = it.parent
+
+proc mergeBasicBlockInfo(parent: var BasicBlock; this: BasicBlock) {.inline.} =
+  if this.hasReturn:
+    parent.wasMovedLocs.setLen 0
+    parent.hasReturn = true
+
+proc wasMovedTarget(matches: var IntSet; branch: seq[PNode]; moveTarget: PNode): bool =
+  result = false
+  for i in 0..<branch.len:
+    if exprStructuralEquivalent(branch[i][1].skipAddr, moveTarget,
+                                strictSymEquality = true):
+      result = true
+      matches.incl i
+
+proc intersect(summary: var seq[PNode]; branch: seq[PNode]) =
+  # keep all 'wasMoved(x)' calls in summary that are also in 'branch':
+  var i = 0
+  var matches = initIntSet()
+  while i < summary.len:
+    if wasMovedTarget(matches, branch, summary[i][1].skipAddr):
+      inc i
+    else:
+      summary.del i
+  for m in matches:
+    summary.add branch[m]
+
+
+proc invalidateWasMoved(c: var BasicBlock; x: PNode) =
+  var i = 0
+  while i < c.wasMovedLocs.len:
+    if exprStructuralEquivalent(c.wasMovedLocs[i][1].skipAddr, x,
+                                strictSymEquality = true):
+      c.wasMovedLocs.del i
+    else:
+      inc i
+
+proc wasMovedDestroyPair(c: var Con; b: var BasicBlock; d: PNode) =
+  var i = 0
+  while i < b.wasMovedLocs.len:
+    if exprStructuralEquivalent(b.wasMovedLocs[i][1].skipAddr, d[1].skipAddr,
+                                strictSymEquality = true):
+      b.wasMovedLocs[i].flags.incl nfMarkForDeletion
+      c.somethingTodo = true
+      d.flags.incl nfMarkForDeletion
+      b.wasMovedLocs.del i
+    else:
+      inc i
+
+proc analyse(c: var Con; b: var BasicBlock; n: PNode) =
+  case n.kind
+  of nkCallKinds:
+    var special = false
+    var reverse = false
+    if n[0].kind == nkSym:
+      let s = n[0].sym
+      if s.magic == mWasMoved:
+        b.wasMovedLocs.add n
+        special = true
+      elif s.name.s == "=destroy":
+        if c.inFinally > 0 and (b.hasReturn or b.hasBreak):
+          discard "cannot optimize away the destructor"
+        else:
+          c.wasMovedDestroyPair b, n
+        special = true
+      elif s.name.s == "=sink":
+        reverse = true
+
+    if not special:
+      if not reverse:
+        for i in 0 ..< n.len:
+          analyse(c, b, n[i])
+      else:
+        #[ Test tmatrix.test3:
+        Prevent this from being elided. We should probably
+        find a better solution...
+
+            `=sink`(b, - (
+              let blitTmp = b;
+              wasMoved(b);
+              blitTmp + a)
+            `=destroy`(b)
+
+        ]#
+        for i in countdown(n.len-1, 0):
+          analyse(c, b, n[i])
+      if canRaise(n[0]): returnStmt(b)
+
+  of nkSym:
+    # any usage of the location before destruction implies we
+    # cannot elide the 'wasMoved(x)':
+    b.invalidateWasMoved n
+
+  of nkNone..pred(nkSym), succ(nkSym)..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef,
+      nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo,
+      nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt,
+      nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, nkTypeOfExpr:
+    discard "do not follow the construct"
+
+  of nkAsgn, nkFastAsgn:
+    # reverse order, see remark for `=sink`:
+    analyse(c, b, n[1])
+    analyse(c, b, n[0])
+
+  of nkIfStmt, nkIfExpr:
+    let isExhaustive = n[^1].kind in {nkElse, nkElseExpr}
+    var wasMovedSet: seq[PNode] = @[]
+
+    for i in 0 ..< n.len:
+      var branch = nestedBlock(b, n[i].kind)
+
+      analyse(c, branch, n[i])
+      mergeBasicBlockInfo(b, branch)
+      if isExhaustive:
+        if i == 0:
+          wasMovedSet = move(branch.wasMovedLocs)
+        else:
+          wasMovedSet.intersect(branch.wasMovedLocs)
+    for i in 0..<wasMovedSet.len:
+      b.wasMovedLocs.add wasMovedSet[i]
+
+  of nkCaseStmt:
+    let isExhaustive = skipTypes(n[0].typ,
+      abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString} or
+      n[^1].kind == nkElse
+
+    analyse(c, b, n[0])
+
+    var wasMovedSet: seq[PNode] = @[]
+
+    for i in 1 ..< n.len:
+      var branch = nestedBlock(b, n[i].kind)
+
+      analyse(c, branch, n[i])
+      mergeBasicBlockInfo(b, branch)
+      if isExhaustive:
+        if i == 1:
+          wasMovedSet = move(branch.wasMovedLocs)
+        else:
+          wasMovedSet.intersect(branch.wasMovedLocs)
+    for i in 0..<wasMovedSet.len:
+      b.wasMovedLocs.add wasMovedSet[i]
+
+  of nkTryStmt:
+    for i in 0 ..< n.len:
+      var tryBody = nestedBlock(b, nkTryStmt)
+
+      analyse(c, tryBody, n[i])
+      mergeBasicBlockInfo(b, tryBody)
+
+  of nkWhileStmt:
+    analyse(c, b, n[0])
+    var loopBody = nestedBlock(b, nkWhileStmt)
+    analyse(c, loopBody, n[1])
+    mergeBasicBlockInfo(b, loopBody)
+
+  of nkBlockStmt, nkBlockExpr:
+    var blockBody = nestedBlock(b, n.kind)
+    if n[0].kind == nkSym:
+      blockBody.label = n[0].sym
+    analyse(c, blockBody, n[1])
+    mergeBasicBlockInfo(b, blockBody)
+
+  of nkBreakStmt:
+    breakStmt(b, n[0])
+
+  of nkReturnStmt, nkRaiseStmt:
+    for child in n: analyse(c, b, child)
+    returnStmt(b)
+
+  of nkFinally:
+    inc c.inFinally
+    for child in n: analyse(c, b, child)
+    dec c.inFinally
+
+  else:
+    for child in n: analyse(c, b, child)
+
+proc opt(c: Con; n, parent: PNode; parentPos: int) =
+  template recurse() =
+    let x = shallowCopy(n)
+    for i in 0 ..< n.len:
+      opt(c, n[i], x, i)
+    parent[parentPos] = x
+
+  case n.kind
+  of nkCallKinds:
+    if nfMarkForDeletion in n.flags:
+      parent[parentPos] = newNodeI(nkEmpty, n.info)
+    else:
+      recurse()
+
+  of nkNone..nkNilLit, nkTypeSection, nkProcDef, nkConverterDef,
+      nkMethodDef, nkIteratorDef, nkMacroDef, nkTemplateDef, nkLambda, nkDo,
+      nkFuncDef, nkConstSection, nkConstDef, nkIncludeStmt, nkImportStmt,
+      nkExportStmt, nkPragma, nkCommentStmt, nkBreakState, nkTypeOfExpr:
+    parent[parentPos] = n
+
+  else:
+    recurse()
+
+
+proc optimize*(n: PNode): PNode =
+  # optimize away simple 'wasMoved(x); destroy(x)' pairs.
+  #[ Unfortunately this optimization is only really safe when no exceptions
+     are possible, see for example:
+
+  proc main(inp: string; cond: bool) =
+    if cond:
+      try:
+        var s = ["hi", inp & "more"]
+        for i in 0..4:
+          use s
+        consume(s)
+        wasMoved(s)
+      finally:
+        destroy(s)
+
+    Now assume 'use' raises, then we shouldn't do the 'wasMoved(s)'
+  ]#
+  var c: Con
+  var b: BasicBlock
+  analyse(c, b, n)
+  if c.somethingTodo:
+    result = shallowCopy(n)
+    for i in 0 ..< n.safeLen:
+      opt(c, n[i], result, i)
+  else:
+    result = n

compiler/options.nim

@@ -9,7 +9,7 @@
 
 import
   os, strutils, strtabs, sets, lineinfos, platform,
-  prefixmatches, pathutils
+  prefixmatches, pathutils, nimpaths, tables
 
 from terminal import isatty
 from times import utc, fromUnix, local, getTime, format, DateTime
@@ -25,7 +25,7 @@ type                          # please make sure we have under 32 options
                               # (improves code efficiency a lot!)
   TOption* = enum             # **keep binary compatible**
     optNone, optObjCheck, optFieldCheck, optRangeCheck, optBoundsCheck,
-    optOverflowCheck, optNilCheck, optRefCheck,
+    optOverflowCheck, optRefCheck,
     optNaNCheck, optInfCheck, optStaticBoundsCheck, optStyleCheck,
     optAssert, optLineDir, optWarns, optHints,
     optOptimizeSpeed, optOptimizeSize,
@@ -39,10 +39,9 @@ type                          # please make sure we have under 32 options
                               # evaluation
     optTrMacros,              # en/disable pattern matching
     optMemTracker,
-    optLaxStrings,
     optNilSeqs,
-    optOldAst,
     optSinkInference          # 'sink T' inference
+    optCursorInference
 
 
   TOptions* = set[TOption]
@@ -57,6 +56,7 @@ type                          # please make sure we have under 32 options
     optGenScript,             # generate a script file to compile the *.c files
     optGenMapping,            # generate a mapping file
     optRun,                   # run the compiled project
+    optUseNimcache,           # save artifacts (including binary) in $nimcache
     optStyleHint,             # check that the names adhere to NEP-1
     optStyleError,            # enforce that the names adhere to NEP-1
     optSkipSystemConfigFile,  # skip the system's cfg/nims config file
@@ -77,7 +77,7 @@ type                          # please make sure we have under 32 options
     optIdeTerse               # idetools: use terse descriptions
     optExcessiveStackTrace    # fully qualified module filenames
     optShowAllMismatches      # show all overloading resolution candidates
-    optWholeProject           # for 'doc2': output any dependency
+    optWholeProject           # for 'doc': output any dependency
     optDocInternal            # generate documentation for non-exported symbols
     optMixedMode              # true if some module triggered C++ codegen
     optListFullPaths          # use full paths in toMsgFilename
@@ -90,12 +90,14 @@ type                          # please make sure we have under 32 options
                               # implementation
     optOwnedRefs              # active if the Nim compiler knows about 'owned'.
     optMultiMethods
-    optNimV019
     optBenchmarkVM            # Enables cpuTime() in the VM
     optProduceAsm             # produce assembler code
     optPanics                 # turn panics (sysFatal) into a process termination
     optNimV1Emulation         # emulate Nim v1.0
+    optNimV12Emulation        # emulate Nim v1.2
     optSourcemap
+    optProfileVM              # enable VM profiler
+    optEnableDeepCopy         # ORC specific: enable 'deepcopy' for all types.
 
   TGlobalOptions* = set[TGlobalOption]
 
@@ -103,12 +105,26 @@ const
   harmlessOptions* = {optForceFullMake, optNoLinking, optRun,
                       optUseColors, optStdout}
 
+type
+  TBackend* = enum
+    backendInvalid = "" # for parseEnum
+    backendC = "c"
+    backendCpp = "cpp"  # was cmdCompileToCpp
+    backendJs = "js" # was cmdCompileToJS
+    backendObjc = "objc" # was cmdCompileToOC
+    # backendNimscript = "nimscript" # this could actually work
+    # backendLlvm = "llvm" # probably not well supported; was cmdCompileToLLVM
+
 type
   TCommands* = enum           # Nim's commands
                               # **keep binary compatible**
-    cmdNone, cmdCompileToC, cmdCompileToCpp, cmdCompileToOC,
-    cmdCompileToJS,
-    cmdCompileToLLVM, cmdInterpret, cmdPretty, cmdDoc,
+    cmdNone,
+    cmdCompileToC,            # deadcode
+    cmdCompileToCpp,          # deadcode
+    cmdCompileToOC,           # deadcode
+    cmdCompileToJS,           # deadcode
+    cmdCompileToLLVM,         # deadcode
+    cmdInterpret, cmdPretty, cmdDoc,
     cmdGenDepend, cmdDump,
     cmdCheck,                 # semantic checking for whole project
     cmdParse,                 # parse a single file (for debugging)
@@ -120,11 +136,11 @@ type
     cmdInteractive,           # start interactive session
     cmdRun,                   # run the project via TCC backend
     cmdJsonScript             # compile a .json build file
+    cmdCompileToBackend,      # compile to backend in TBackend
   TStringSeq* = seq[string]
   TGCMode* = enum             # the selected GC
-    gcUnselected, gcNone, gcBoehm, gcRegions, gcMarkAndSweep, gcArc, gcOrc,
-    gcHooks,
-    gcRefc, gcV2, gcGo
+    gcUnselected, gcNone, gcBoehm, gcRegions, gcArc, gcOrc,
+    gcMarkAndSweep, gcHooks, gcRefc, gcV2, gcGo
     # gcRefc and the GCs that follow it use a write barrier,
     # as far as usesWriteBarrier() is concerned
 
@@ -140,13 +156,16 @@ type
     destructor,
     notnil,
     dynamicBindSym,
-    forLoopMacros,
+    forLoopMacros, # not experimental anymore; remains here for backwards compatibility
     caseStmtMacros,
     codeReordering,
     compiletimeFFI,
       ## This requires building nim with `-d:nimHasLibFFI`
       ## which itself requires `nimble install libffi`, see #10150
       ## Note: this feature can't be localized with {.push.}
+    vmopsDanger,
+    strictFuncs,
+    views
 
   LegacyFeature* = enum
     allowSemcheckedAstModification,
@@ -162,8 +181,8 @@ type
     disabledSf, writeOnlySf, readOnlySf, v2Sf
 
   TSystemCC* = enum
-    ccNone, ccGcc, ccNintendoSwitch, ccLLVM_Gcc, ccCLang, ccLcc, ccBcc, ccDmc, ccWcc, ccVcc,
-    ccTcc, ccPcc, ccUcc, ccIcl, ccIcc, ccClangCl
+    ccNone, ccGcc, ccNintendoSwitch, ccLLVM_Gcc, ccCLang, ccBcc, ccVcc,
+    ccTcc, ccEnv, ccIcl, ccIcc, ccClangCl
 
   ExceptionSystem* = enum
     excNone,   # no exception system selected yet
@@ -180,6 +199,7 @@ type
     nimname*: string
     cname*, obj*: AbsoluteFile
     flags*: set[CfileFlag]
+    customArgs*: string
   CfileList* = seq[Cfile]
 
   Suggest* = ref object
@@ -202,15 +222,24 @@ type
     version*: int
   Suggestions* = seq[Suggest]
 
+  ProfileInfo* = object
+    time*: float
+    count*: int
+
+  ProfileData* = ref object
+    data*: TableRef[TLineInfo, ProfileInfo]
+
   ConfigRef* = ref object ## every global configuration
                           ## fields marked with '*' are subject to
                           ## the incremental compilation mechanisms
                           ## (+) means "part of the dependency"
+    backend*: TBackend # set via `nim x` or `nim --backend:x`
     target*: Target       # (+)
     linesCompiled*: int   # all lines that have been compiled
     options*: TOptions    # (+)
     globalOptions*: TGlobalOptions # (+)
     macrosToExpand*: StringTableRef
+    arcToExpand*: StringTableRef
     m*: MsgConfig
     evalTemplateCounter*: int
     evalMacroCounter*: int
@@ -237,6 +266,7 @@ type
     cmdlineNotes*: TNoteKinds # notes that have been set/unset from cmdline
     foreignPackageNotes*: TNoteKinds
     notes*: TNoteKinds # notes after resolving all logic(defaults, verbosity)/cmdline/configs
+    warningAsErrors*: TNoteKinds
     mainPackageNotes*: TNoteKinds
     mainPackageId*: int
     errorCounter*: int
@@ -254,13 +284,16 @@ type
     lazyPaths*: seq[AbsoluteDir]
     outFile*: RelativeFile
     outDir*: AbsoluteDir
+    jsonBuildFile*: AbsoluteFile
     prefixDir*, libpath*, nimcacheDir*: AbsoluteDir
     dllOverrides, moduleOverrides*, cfileSpecificOptions*: StringTableRef
     projectName*: string # holds a name like 'nim'
     projectPath*: AbsoluteDir # holds a path like /home/alice/projects/nim/compiler/
     projectFull*: AbsoluteFile # projectPath/projectName
     projectIsStdin*: bool # whether we're compiling from stdin
+    lastMsgWasDot*: bool # the last compiler message was a single '.'
     projectMainIdx*: FileIndex # the canonical path id of the main module
+    projectMainIdx2*: FileIndex # consider merging with projectMainIdx
     command*: string # the main command (e.g. cc, check, scan, etc)
     commandArgs*: seq[string] # any arguments after the main command
     commandLine*: string
@@ -271,6 +304,7 @@ type
     docSeeSrcUrl*: string # if empty, no seeSrc will be generated. \
     # The string uses the formatting variables `path` and `line`.
     docRoot*: string ## see nim --fullhelp for --docRoot
+    docCmd*: string ## see nim --fullhelp for --docCmd
 
      # the used compiler
     cIncludes*: seq[AbsoluteDir]  # directories to search for included files
@@ -293,6 +327,15 @@ type
     structuredErrorHook*: proc (config: ConfigRef; info: TLineInfo; msg: string;
                                 severity: Severity) {.closure, gcsafe.}
     cppCustomNamespace*: string
+    vmProfileData*: ProfileData
+
+proc assignIfDefault*[T](result: var T, val: T, def = default(T)) =
+  ## if `result` was already assigned to a value (that wasn't `def`), this is a noop.
+  if result == def: result = val
+
+template setErrorMaxHighMaybe*(conf: ConfigRef) =
+  ## do not stop after first error (but honor --errorMax if provided)
+  assignIfDefault(conf.errorMax, high(int))
 
 proc setNoteDefaults*(conf: ConfigRef, note: TNoteKind, enabled = true) =
   template fun(op) =
@@ -307,7 +350,10 @@ proc setNote*(conf: ConfigRef, note: TNoteKind, enabled = true) =
     if enabled: incl(conf.notes, note) else: excl(conf.notes, note)
 
 proc hasHint*(conf: ConfigRef, note: TNoteKind): bool =
-  optHints in conf.options and note in conf.notes
+  if optHints notin conf.options: false
+  elif note in {hintConf}: # could add here other special notes like hintSource
+    note in conf.mainPackageNotes
+  else: note in conf.notes
 
 proc hasWarn*(conf: ConfigRef, note: TNoteKind): bool =
   optWarns in conf.options and note in conf.notes
@@ -323,14 +369,14 @@ template depConfigFields*(fn) {.dirty.} =
 const oldExperimentalFeatures* = {implicitDeref, dotOperators, callOperator, parallel}
 
 const
-  ChecksOptions* = {optObjCheck, optFieldCheck, optRangeCheck, optNilCheck,
+  ChecksOptions* = {optObjCheck, optFieldCheck, optRangeCheck,
     optOverflowCheck, optBoundsCheck, optAssert, optNaNCheck, optInfCheck,
     optStyleCheck}
 
   DefaultOptions* = {optObjCheck, optFieldCheck, optRangeCheck,
     optBoundsCheck, optOverflowCheck, optAssert, optWarns, optRefCheck,
     optHints, optStackTrace, optLineTrace, # consider adding `optStackTraceMsgs`
-    optTrMacros, optNilCheck, optStyleCheck, optSinkInference}
+    optTrMacros, optStyleCheck, optCursorInference}
   DefaultGlobalOptions* = {optThreadAnalysis,
     optExcessiveStackTrace, optListFullPaths}
 
@@ -358,6 +404,9 @@ template newPackageCache*(): untyped =
                  else:
                    modeCaseSensitive)
 
+proc newProfileData(): ProfileData =
+  ProfileData(data: newTable[TLineInfo, ProfileInfo]())
+
 proc newConfigRef*(): ConfigRef =
   result = ConfigRef(
     selectedGC: gcRefc,
@@ -366,6 +415,7 @@ proc newConfigRef*(): ConfigRef =
     options: DefaultOptions,
     globalOptions: DefaultGlobalOptions,
     macrosToExpand: newStringTable(modeStyleInsensitive),
+    arcToExpand: newStringTable(modeStyleInsensitive),
     m: initMsgConfig(),
     evalExpr: "",
     cppDefines: initHashSet[string](),
@@ -399,7 +449,7 @@ proc newConfigRef*(): ConfigRef =
     cIncludes: @[],   # directories to search for included files
     cLibs: @[],       # directories to search for lib files
     cLinkedLibs: @[],  # libraries to link
-
+    backend: backendInvalid,
     externalToLink: @[],
     linkOptionsCmd: "",
     compileOptionsCmd: @[],
@@ -410,6 +460,7 @@ proc newConfigRef*(): ConfigRef =
     arguments: "",
     suggestMaxResults: 10_000,
     maxLoopIterationsVM: 10_000_000,
+    vmProfileData: newProfileData(),
   )
   setTargetFromSystem(result.target)
   # enable colors by default on terminals
@@ -447,7 +498,7 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool =
                             osQnx, osAtari, osAix,
                             osHaiku, osVxWorks, osSolaris, osNetbsd,
                             osFreebsd, osOpenbsd, osDragonfly, osMacosx, osIos,
-                            osAndroid, osNintendoSwitch}
+                            osAndroid, osNintendoSwitch, osFreeRTOS}
     of "linux":
       result = conf.target.targetOS in {osLinux, osAndroid}
     of "bsd":
@@ -463,6 +514,10 @@ proc isDefined*(conf: ConfigRef; symbol: string): bool =
     of "sunos": result = conf.target.targetOS == osSolaris
     of "nintendoswitch":
       result = conf.target.targetOS == osNintendoSwitch
+    of "freertos":
+      result = conf.target.targetOS == osFreeRTOS
+    of "lwip":
+      result = conf.target.targetOS in {osFreeRTOS}
     of "littleendian": result = CPU[conf.target.targetCPU].endian == platform.littleEndian
     of "bigendian": result = CPU[conf.target.targetCPU].endian == platform.bigEndian
     of "cpu8": result = CPU[conf.target.targetCPU].bit == 8
@@ -497,8 +552,9 @@ const
   DefaultConfigNims* = RelativeFile"config.nims"
   DocConfig* = RelativeFile"nimdoc.cfg"
   DocTexConfig* = RelativeFile"nimdoc.tex.cfg"
-
-const oKeepVariableNames* = true
+  htmldocsDir* = htmldocsDirname.RelativeDir
+  docRootDefault* = "@default" # using `@` instead of `$` to avoid shell quoting complications
+  oKeepVariableNames* = true
 
 proc mainCommandArg*(conf: ConfigRef): string =
   ## This is intended for commands like check or parse
@@ -519,21 +575,23 @@ proc setConfigVar*(conf: ConfigRef; key, val: string) =
   conf.configVars[key] = val
 
 proc getOutFile*(conf: ConfigRef; filename: RelativeFile, ext: string): AbsoluteFile =
-  conf.outDir / changeFileExt(filename, ext)
+  # explains regression https://github.com/nim-lang/Nim/issues/6583#issuecomment-625711125
+  # Yet another reason why "" should not mean ".";  `""/something` should raise
+  # instead of implying "" == "." as it's bug prone.
+  doAssert conf.outDir.string.len > 0
+  result = conf.outDir / changeFileExt(filename, ext)
 
 proc absOutFile*(conf: ConfigRef): AbsoluteFile =
+  doAssert not conf.outDir.isEmpty
+  doAssert not conf.outFile.isEmpty
   result = conf.outDir / conf.outFile
   when defined(posix):
-    if dirExists(result.string):
-      result.string.add ".out"
+    if dirExists(result.string): result.string.add ".out"
 
 proc prepareToWriteOutput*(conf: ConfigRef): AbsoluteFile =
   ## Create the output directory and returns a full path to the output file
-  createDir conf.outDir
-  result = conf.outDir / conf.outFile
-  when defined(posix):
-    if dirExists(result.string):
-      result.string.add ".out"
+  result = conf.absOutFile
+  createDir result.string.parentDir
 
 proc getPrefixDir*(conf: ConfigRef): AbsoluteDir =
   ## Gets the prefix dir, usually the parent directory where the binary resides.
@@ -573,17 +631,6 @@ proc setDefaultLibpath*(conf: ConfigRef) =
 proc canonicalizePath*(conf: ConfigRef; path: AbsoluteFile): AbsoluteFile =
   result = AbsoluteFile path.string.expandFilename
 
-proc shortenDir*(conf: ConfigRef; dir: string): string {.
-    deprecated: "use 'relativeTo' instead".} =
-  ## returns the interesting part of a dir
-  var prefix = conf.projectPath.string & DirSep
-  if startsWith(dir, prefix):
-    return substr(dir, prefix.len)
-  prefix = getPrefixDir(conf).string & DirSep
-  if startsWith(dir, prefix):
-    return substr(dir, prefix.len)
-  result = dir
-
 proc removeTrailingDirSep*(path: string): string =
   if (path.len > 0) and (path[^1] == DirSep):
     result = substr(path, 0, path.len - 2)
@@ -611,7 +658,7 @@ proc getNimcacheDir*(conf: ConfigRef): AbsoluteDir =
   # XXX projectName should always be without a file extension!
   result = if not conf.nimcacheDir.isEmpty:
              conf.nimcacheDir
-           elif conf.cmd == cmdCompileToJS:
+           elif conf.backend == backendJs:
              conf.projectPath / genSubDir
            else:
             AbsoluteDir(getOsCacheDir() / splitFile(conf.projectName).name &
@@ -641,12 +688,10 @@ iterator nimbleSubs*(conf: ConfigRef; p: string): string =
 proc toGeneratedFile*(conf: ConfigRef; path: AbsoluteFile,
                       ext: string): AbsoluteFile =
   ## converts "/home/a/mymodule.nim", "rod" to "/home/a/nimcache/mymodule.rod"
-  let (head, tail) = splitPath(path.string)
-  result = getNimcacheDir(conf) / RelativeFile changeFileExt(tail, ext)
+  result = getNimcacheDir(conf) / RelativeFile path.string.splitPath.tail.changeFileExt(ext)
 
 proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile,
                                 createSubDir: bool = true): AbsoluteFile =
-  let (head, tail) = splitPath(f.string)
   let subdir = getNimcacheDir(conf)
   if createSubDir:
     try:
@@ -654,7 +699,7 @@ proc completeGeneratedFilePath*(conf: ConfigRef; f: AbsoluteFile,
     except OSError:
       writeLine(stdout, "cannot create directory: " & subdir.string)
       quit(1)
-  result = subdir / RelativeFile tail
+  result = subdir / RelativeFile f.string.splitPath.tail
   #echo "completeGeneratedFilePath(", f, ") = ", result
 
 proc rawFindFile(conf: ConfigRef; f: RelativeFile; suppressStdlib: bool): AbsoluteFile =
@@ -704,7 +749,7 @@ proc getRelativePathFromConfigPath*(conf: ConfigRef; f: AbsoluteFile): RelativeF
 
 proc findFile*(conf: ConfigRef; f: string; suppressStdlib = false): AbsoluteFile =
   if f.isAbsolute:
-    result = if f.existsFile: AbsoluteFile(f) else: AbsoluteFile""
+    result = if f.fileExists: AbsoluteFile(f) else: AbsoluteFile""
   else:
     result = rawFindFile(conf, RelativeFile f, suppressStdlib)
     if result.isEmpty:
@@ -718,7 +763,8 @@ proc findFile*(conf: ConfigRef; f: string; suppressStdlib = false): AbsoluteFile
 const stdlibDirs = [
   "pure", "core", "arch",
   "pure/collections",
-  "pure/concurrency", "impure",
+  "pure/concurrency",
+  "pure/unidecode", "impure",
   "wrappers", "wrappers/linenoise",
   "windows", "posix", "js"]
 
@@ -839,3 +885,18 @@ proc floatInt64Align*(conf: ConfigRef): int16 =
       # to 4bytes (except with -malign-double)
       return 4
   return 8
+
+proc setOutFile*(conf: ConfigRef) =
+  proc libNameTmpl(conf: ConfigRef): string {.inline.} =
+    result = if conf.target.targetOS == osWindows: "$1.lib" else: "lib$1.a"
+
+  if conf.outFile.isEmpty:
+    let base = conf.projectName
+    let targetName =
+      if optGenDynLib in conf.globalOptions:
+        platform.OS[conf.target.targetOS].dllFrmt % base
+      elif optGenStaticLib in conf.globalOptions:
+        libNameTmpl(conf) % base
+      else:
+        base & platform.OS[conf.target.targetOS].exeExt
+    conf.outFile = RelativeFile targetName

compiler/packagehandling.nim

@@ -16,7 +16,7 @@ iterator myParentDirs(p: string): string =
     yield current
 
 proc getNimbleFile*(conf: ConfigRef; path: string): string =
-  ## returns absolute path to nimble file, eg: /pathto/cligen.nimble
+  ## returns absolute path to nimble file, e.g.: /pathto/cligen.nimble
   var parents = 0
   block packageSearch:
     for d in myParentDirs(path):
@@ -28,8 +28,6 @@ proc getNimbleFile*(conf: ConfigRef; path: string): string =
         result = file
         break packageSearch
   # we also store if we didn't find anything:
-  when not defined(nimNoNilSeqs):
-    if result.isNil: result = ""
   for d in myParentDirs(path):
     #echo "set cache ", d, " |", result, "|", parents
     conf.packageCache[d] = result
@@ -37,7 +35,7 @@ proc getNimbleFile*(conf: ConfigRef; path: string): string =
     if parents <= 0: break
 
 proc getPackageName*(conf: ConfigRef; path: string): string =
-  ## returns nimble package name, eg: `cligen`
+  ## returns nimble package name, e.g.: `cligen`
   let path = getNimbleFile(conf, path)
   result = path.splitFile.name
 

compiler/parampatterns.nim

@@ -99,14 +99,14 @@ proc compileConstraints(p: PNode, result: var TPatternCode; conf: ConfigRef) =
     else:
       # check all symkinds:
       internalAssert conf, int(high(TSymKind)) < 255
-      for i in low(TSymKind)..high(TSymKind):
+      for i in TSymKind:
         if cmpIgnoreStyle(($i).substr(2), spec) == 0:
           result.add(ppSymKind)
           result.add(chr(i.ord))
           return
       # check all nodekinds:
       internalAssert conf, int(high(TNodeKind)) < 255
-      for i in low(TNodeKind)..high(TNodeKind):
+      for i in TNodeKind:
         if cmpIgnoreStyle($i, spec) == 0:
           result.add(ppNodeKind)
           result.add(chr(i.ord))
@@ -176,6 +176,7 @@ type
     arLocalLValue,            # is an l-value, but local var; must not escape
                               # its stack frame!
     arDiscriminant,           # is a discriminant
+    arLentValue,              # lent value
     arStrange                 # it is a strange beast like 'typedesc[var T]'
 
 proc exprRoot*(n: PNode): PSym =
@@ -199,7 +200,7 @@ proc exprRoot*(n: PNode): PSym =
       if it.len > 0 and it.typ != nil: it = it.lastSon
       else: break
     of nkCallKinds:
-      if it.typ != nil and it.typ.kind == tyVar and it.len > 1:
+      if it.typ != nil and it.typ.kind in {tyVar, tyLent} and it.len > 1:
         # See RFC #7373, calls returning 'var T' are assumed to
         # return a view into the first argument (if there is one):
         it = it[1]
@@ -213,7 +214,7 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
   result = arNone
   case n.kind
   of nkEmpty:
-    if n.typ != nil and n.typ.kind == tyVar:
+    if n.typ != nil and n.typ.kind in {tyVar}:
       result = arLValue
   of nkSym:
     let kinds = if isUnsafeAddr: {skVar, skResult, skTemp, skParam, skLet, skForVar}
@@ -222,6 +223,8 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
       result = arLValue
     elif isUnsafeAddr and n.sym.kind == skParam:
       result = arLValue
+    elif isUnsafeAddr and n.sym.kind == skConst and dontInlineConstant(n, n.sym.ast):
+      result = arLValue
     elif n.sym.kind in kinds:
       if owner != nil and owner == n.sym.owner and
           sfGlobal notin n.sym.flags:
@@ -230,7 +233,7 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
         result = arLValue
     elif n.sym.kind == skType:
       let t = n.sym.typ.skipTypes({tyTypeDesc})
-      if t.kind == tyVar: result = arStrange
+      if t.kind in {tyVar}: result = arStrange
   of nkDotExpr:
     let t = skipTypes(n[0].typ, abstractInst-{tyTypeDesc})
     if t.kind in {tyVar, tySink, tyPtr, tyRef}:
@@ -260,9 +263,14 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
       # types that are equal modulo distinction preserve l-value:
       result = isAssignable(owner, n[1], isUnsafeAddr)
   of nkHiddenDeref:
-    if isUnsafeAddr and n[0].typ.kind == tyLent: result = arLValue
-    elif n[0].typ.kind == tyLent: result = arDiscriminant
-    else: result = arLValue
+    let n0 = n[0]
+    if n0.typ.kind == tyLent:
+      if isUnsafeAddr or (n0.kind == nkSym and n0.sym.kind == skResult):
+        result = arLValue
+      else:
+        result = arLentValue
+    else:
+      result = arLValue
   of nkDerefExpr, nkHiddenAddr:
     result = arLValue
   of nkObjUpConv, nkObjDownConv, nkCheckedFieldExpr:
@@ -271,7 +279,7 @@ proc isAssignable*(owner: PSym, n: PNode; isUnsafeAddr=false): TAssignableResult
     # builtin slice keeps lvalue-ness:
     if getMagic(n) in {mArrGet, mSlice}:
       result = isAssignable(owner, n[1], isUnsafeAddr)
-    elif n.typ != nil and n.typ.kind == tyVar:
+    elif n.typ != nil and n.typ.kind in {tyVar}:
       result = arLValue
     elif isUnsafeAddr and n.typ != nil and n.typ.kind == tyLent:
       result = arLValue

compiler/passes.nim

@@ -113,10 +113,8 @@ const
     nkExportStmt, nkExportExceptStmt, nkFromStmt, nkImportStmt, nkImportExceptStmt}
 
 proc prepareConfigNotes(graph: ModuleGraph; module: PSym) =
-  if sfMainModule in module.flags:
-    graph.config.mainPackageId = module.owner.id
   # don't be verbose unless the module belongs to the main package:
-  if module.owner.id == graph.config.mainPackageId:
+  if module.getnimblePkgId == graph.config.mainPackageId:
     graph.config.notes = graph.config.mainPackageNotes
   else:
     if graph.config.mainPackageNotes == {}: graph.config.mainPackageNotes = graph.config.notes
@@ -128,7 +126,7 @@ proc moduleHasChanged*(graph: ModuleGraph; module: PSym): bool {.inline.} =
 proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {.discardable.} =
   if graph.stopCompile(): return true
   var
-    p: TParsers
+    p: Parser
     a: TPassContextArray
     s: PLLStream
     fileIdx = module.fileIdx
@@ -166,7 +164,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
     else:
       s = stream
     while true:
-      openParsers(p, fileIdx, s, graph.cache, graph.config)
+      openParser(p, fileIdx, s, graph.cache, graph.config)
 
       if module.owner == nil or module.owner.name.s != "stdlib" or module.name.s == "distros":
         # XXX what about caching? no processing then? what if I change the
@@ -214,7 +212,7 @@ proc processModule*(graph: ModuleGraph; module: PSym, stream: PLLStream): bool {
         else:
           #echo "----- single\n", n
           if not processTopLevelStmt(graph, n, a): break
-      closeParsers(p)
+      closeParser(p)
       if s.kind != llsStdIn: break
     closePasses(graph, a)
     # id synchronization point for more consistent code generation:

compiler/pathutils.nim

@@ -99,19 +99,3 @@ when true:
   proc addFileExt*(x: RelativeFile; ext: string): RelativeFile {.borrow.}
 
   proc writeFile*(x: AbsoluteFile; content: string) {.borrow.}
-
-when isMainModule:
-  doAssert AbsoluteDir"/Users/me///" / RelativeFile"z.nim" == AbsoluteFile"/Users/me/z.nim"
-  doAssert AbsoluteDir"/Users/me" / RelativeFile"../z.nim" == AbsoluteFile"/Users/z.nim"
-  doAssert AbsoluteDir"/Users/me/" / RelativeFile"../z.nim" == AbsoluteFile"/Users/z.nim"
-  doAssert relativePath("/foo/bar.nim", "/foo/", '/') == "bar.nim"
-  doAssert $RelativeDir"foo/bar" == "foo/bar"
-  doAssert RelativeDir"foo/bar" == RelativeDir"foo/bar"
-  doAssert RelativeFile"foo/bar".changeFileExt(".txt") == RelativeFile"foo/bar.txt"
-  doAssert RelativeFile"foo/bar".addFileExt(".txt") == RelativeFile"foo/bar.txt"
-  doAssert not RelativeDir"foo/bar".isEmpty
-  doAssert RelativeDir"".isEmpty
-
-when isMainModule and defined(windows):
-  let nasty = string(AbsoluteDir(r"C:\Users\rumpf\projects\nim\tests\nimble\nimbleDir\linkedPkgs\pkgB-#head\../../simplePkgs/pkgB-#head/") / RelativeFile"pkgA/module.nim")
-  doAssert nasty.replace('/', '\\') == r"C:\Users\rumpf\projects\nim\tests\nimble\nimbleDir\simplePkgs\pkgB-#head\pkgA\module.nim"

compiler/platform.nim

@@ -22,7 +22,7 @@ type
     osNone, osDos, osWindows, osOs2, osLinux, osMorphos, osSkyos, osSolaris,
     osIrix, osNetbsd, osFreebsd, osOpenbsd, osDragonfly, osAix, osPalmos, osQnx,
     osAmiga, osAtari, osNetware, osMacos, osMacosx, osIos, osHaiku, osAndroid, osVxWorks
-    osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osAny
+    osGenode, osJS, osNimVM, osStandalone, osNintendoSwitch, osFreeRTOS, osAny
 
 type
   TInfoOSProp* = enum
@@ -177,6 +177,10 @@ const
       objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
       scriptExt: ".sh", curDir: ".", exeExt: ".elf", extSep: ".",
       props: {ospNeedsPIC, ospPosix}),
+     (name: "FreeRTOS", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
+      objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
+      scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
+      props: {ospPosix}),
      (name: "Any", parDir: "..", dllFrmt: "lib$1.so", altDirSep: "/",
       objExt: ".o", newLine: "\x0A", pathSep: ":", dirSep: "/",
       scriptExt: ".sh", curDir: ".", exeExt: "", extSep: ".",
@@ -189,7 +193,7 @@ type
     cpuNone, cpuI386, cpuM68k, cpuAlpha, cpuPowerpc, cpuPowerpc64,
     cpuPowerpc64el, cpuSparc, cpuVm, cpuHppa, cpuIa64, cpuAmd64, cpuMips,
     cpuMipsel, cpuArm, cpuArm64, cpuJS, cpuNimVM, cpuAVR, cpuMSP430,
-    cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV64, cpuWasm32
+    cpuSparc64, cpuMips64, cpuMips64el, cpuRiscV64, cpuEsp, cpuWasm32
 
 type
   TEndian* = enum
@@ -223,6 +227,7 @@ const
     (name: "mips64", intSize: 64, endian: bigEndian, floatSize: 64, bit: 64),
     (name: "mips64el", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
     (name: "riscv64", intSize: 64, endian: littleEndian, floatSize: 64, bit: 64),
+    (name: "esp", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32),
     (name: "wasm32", intSize: 32, endian: littleEndian, floatSize: 64, bit: 32)]
 
 type

compiler/pragmas.nim

@@ -53,7 +53,7 @@ const
     wDeprecated,
     wFloatChecks, wInfChecks, wNanChecks, wPragma, wEmit, wUnroll,
     wLinearScanEnd, wPatterns, wTrMacros, wEffects, wNoForward, wReorder, wComputedGoto,
-    wInjectStmt, wExperimental, wThis, wUsed, wInvariant, wAssume}
+    wInjectStmt, wExperimental, wThis, wUsed, wInvariant, wAssume, wAssert}
   lambdaPragmas* = declPragmas + {FirstCallConv..LastCallConv,
     wNoSideEffect, wSideEffect, wNoreturn, wNosinks, wDynlib, wHeader,
     wThread, wAsmNoStackFrame,
@@ -61,18 +61,19 @@ const
     wGcSafe, wCodegenDecl} - {wExportNims, wError, wUsed}  # why exclude these?
   typePragmas* = declPragmas + {wMagic, wAcyclic,
     wPure, wHeader, wCompilerProc, wCore, wFinal, wSize, wShallow,
-    wIncompleteStruct, wByCopy, wByRef,
+    wIncompleteStruct, wCompleteStruct, wByCopy, wByRef,
     wInheritable, wGensym, wInject, wRequiresInit, wUnchecked, wUnion, wPacked,
     wBorrow, wGcSafe, wPartial, wExplain, wPackage}
   fieldPragmas* = declPragmas + {
-    wGuard, wBitsize, wCursor, wRequiresInit} - {wExportNims, wNodecl} # why exclude these?
+    wGuard, wBitsize, wCursor, wRequiresInit, wNoalias} - {wExportNims, wNodecl} # why exclude these?
   varPragmas* = declPragmas + {wVolatile, wRegister, wThreadVar,
     wMagic, wHeader, wCompilerProc, wCore, wDynlib,
     wNoInit, wCompileTime, wGlobal,
-    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wCursor}
+    wGensym, wInject, wCodegenDecl, wGuard, wGoto, wCursor, wNoalias}
   constPragmas* = declPragmas + {wHeader, wMagic,
     wGensym, wInject,
     wIntDefine, wStrDefine, wBoolDefine, wCompilerProc, wCore}
+  paramPragmas* = {wNoalias, wInject, wGensym}
   letPragmas* = varPragmas
   procTypePragmas* = {FirstCallConv..LastCallConv, wVarargs, wNoSideEffect,
                       wThread, wRaises, wLocks, wTags, wGcSafe,
@@ -92,11 +93,10 @@ proc getPragmaVal*(procAst: PNode; name: TSpecialWord): PNode =
 proc pragma*(c: PContext, sym: PSym, n: PNode, validPragmas: TSpecialWords;
             isStatement: bool = false)
 
-proc recordPragma(c: PContext; n: PNode; key, val: string; val2 = "") =
+proc recordPragma(c: PContext; n: PNode; args: varargs[string]) =
   var recorded = newNodeI(nkCommentStmt, n.info)
-  recorded.add newStrNode(key, n.info)
-  recorded.add newStrNode(val, n.info)
-  if val2.len > 0: recorded.add newStrNode(val2, n.info)
+  for i in 0..args.high:
+    recorded.add newStrNode(args[i], n.info)
   c.graph.recordStmt(c.graph, c.module, recorded)
 
 const
@@ -179,7 +179,7 @@ proc processImportCpp(c: PContext; s: PSym, extname: string, info: TLineInfo) =
   incl(s.flags, sfImportc)
   incl(s.flags, sfInfixCall)
   excl(s.flags, sfForward)
-  if c.config.cmd == cmdCompileToC:
+  if c.config.backend == backendC:
     let m = s.getModule()
     incl(m.flags, sfCompileToCpp)
   incl c.config.globalOptions, optMixedMode
@@ -238,7 +238,7 @@ proc processMagic(c: PContext, n: PNode, s: PSym) =
   var v: string
   if n[1].kind == nkIdent: v = n[1].ident.s
   else: v = expectStrLit(c, n)
-  for m in low(TMagic)..high(TMagic):
+  for m in TMagic:
     if substr($m, 1) == v:
       s.magic = m
       break
@@ -247,7 +247,7 @@ proc processMagic(c: PContext, n: PNode, s: PSym) =
 proc wordToCallConv(sw: TSpecialWord): TCallingConvention =
   # this assumes that the order of special words and calling conventions is
   # the same
-  result = TCallingConvention(ord(ccDefault) + ord(sw) - ord(wNimcall))
+  TCallingConvention(ord(ccNimCall) + ord(sw) - ord(wNimcall))
 
 proc isTurnedOn(c: PContext, n: PNode): bool =
   if n.kind in nkPragmaCallKinds and n.len == 2:
@@ -257,8 +257,8 @@ proc isTurnedOn(c: PContext, n: PNode): bool =
   localError(c.config, n.info, "'on' or 'off' expected")
 
 proc onOff(c: PContext, n: PNode, op: TOptions, resOptions: var TOptions) =
-  if isTurnedOn(c, n): resOptions = resOptions + op
-  else: resOptions = resOptions - op
+  if isTurnedOn(c, n): resOptions.incl op
+  else: resOptions.excl op
 
 proc pragmaNoForward(c: PContext, n: PNode; flag=sfNoForward) =
   if isTurnedOn(c, n):
@@ -326,35 +326,32 @@ proc processDynLib(c: PContext, n: PNode, sym: PSym) =
     # a calling convention that doesn't introduce custom name mangling
     # cdecl is the default - the user can override this explicitly
     if sym.kind in routineKinds and sym.typ != nil and
-        sym.typ.callConv == ccDefault:
+       tfExplicitCallConv notin sym.typ.flags:
       sym.typ.callConv = ccCDecl
 
 proc processNote(c: PContext, n: PNode) =
+  template handleNote(enumVals, notes) =
+    let x = findStr(enumVals, n[0][1].ident.s, errUnknown)
+    if x !=  errUnknown:
+      nk = TNoteKind(x)
+      let x = c.semConstBoolExpr(c, n[1])
+      n[1] = x
+      if x.kind == nkIntLit and x.intVal != 0: incl(notes, nk)
+      else: excl(notes, nk)
+    else:
+      invalidPragma(c, n)
+
   if n.kind in nkPragmaCallKinds and n.len == 2 and
       n[0].kind == nkBracketExpr and
       n[0].len == 2 and
       n[0][1].kind == nkIdent and n[0][0].kind == nkIdent:
     var nk: TNoteKind
     case whichKeyword(n[0][0].ident)
-    of wHint:
-      var x = findStr(HintsToStr, n[0][1].ident.s)
-      if x >= 0: nk = TNoteKind(x + ord(hintMin))
-      else: invalidPragma(c, n); return
-    of wWarning:
-      var x = findStr(WarningsToStr, n[0][1].ident.s)
-      if x >= 0: nk = TNoteKind(x + ord(warnMin))
-      else: invalidPragma(c, n); return
-    else:
-      invalidPragma(c, n)
-      return
-
-    let x = c.semConstBoolExpr(c, n[1])
-    n[1] = x
-    if x.kind == nkIntLit and x.intVal != 0: incl(c.config.notes, nk)
-    else: excl(c.config.notes, nk)
-    # checkme: honor cmdlineNotes with: c.setNote(nk, x.kind == nkIntLit and x.intVal != 0)
-  else:
-    invalidPragma(c, n)
+    of wHint: handleNote(hintMin .. hintMax, c.config.notes)
+    of wWarning: handleNote(warnMin .. warnMax, c.config.notes)
+    of wWarningAsError: handleNote(warnMin .. warnMax, c.config.warningAsErrors)
+    else: invalidPragma(c, n)
+  else: invalidPragma(c, n)
 
 proc pragmaToOptions(w: TSpecialWord): TOptions {.inline.} =
   case w
@@ -364,7 +361,6 @@ proc pragmaToOptions(w: TSpecialWord): TOptions {.inline.} =
   of wRangeChecks: {optRangeCheck}
   of wBoundChecks: {optBoundsCheck}
   of wOverflowChecks: {optOverflowCheck}
-  of wNilChecks: {optNilCheck}
   of wFloatChecks: {optNaNCheck, optInfCheck}
   of wNanChecks: {optNaNCheck}
   of wInfChecks: {optInfCheck}
@@ -473,14 +469,12 @@ proc processPop(c: PContext, n: PNode) =
 proc processDefine(c: PContext, n: PNode) =
   if (n.kind in nkPragmaCallKinds and n.len == 2) and (n[1].kind == nkIdent):
     defineSymbol(c.config.symbols, n[1].ident.s)
-    message(c.config, n.info, warnDeprecated, "define is deprecated")
   else:
     invalidPragma(c, n)
 
 proc processUndef(c: PContext, n: PNode) =
   if (n.kind in nkPragmaCallKinds and n.len == 2) and (n[1].kind == nkIdent):
     undefSymbol(c.config.symbols, n[1].ident.s)
-    message(c.config, n.info, warnDeprecated, "undef is deprecated")
   else:
     invalidPragma(c, n)
 
@@ -496,11 +490,12 @@ proc relativeFile(c: PContext; n: PNode; ext=""): AbsoluteFile =
       if result.isEmpty: result = AbsoluteFile s
 
 proc processCompile(c: PContext, n: PNode) =
-  proc docompile(c: PContext; it: PNode; src, dest: AbsoluteFile) =
+  proc docompile(c: PContext; it: PNode; src, dest: AbsoluteFile; customArgs: string) =
     var cf = Cfile(nimname: splitFile(src).name,
-                   cname: src, obj: dest, flags: {CfileFlag.External})
+                   cname: src, obj: dest, flags: {CfileFlag.External},
+                   customArgs: customArgs)
     extccomp.addExternalFileToCompile(c.config, cf)
-    recordPragma(c, it, "compile", src.string, dest.string)
+    recordPragma(c, it, "compile", src.string, dest.string, customArgs)
 
   proc getStrLit(c: PContext, n: PNode; i: int): string =
     n[i] = c.semConstExpr(c, n[i])
@@ -518,9 +513,19 @@ proc processCompile(c: PContext, n: PNode) =
     var found = parentDir(toFullPath(c.config, n.info)) / s
     for f in os.walkFiles(found):
       let obj = completeCfilePath(c.config, AbsoluteFile(dest % extractFilename(f)))
-      docompile(c, it, AbsoluteFile f, obj)
+      docompile(c, it, AbsoluteFile f, obj, "")
   else:
-    let s = expectStrLit(c, n)
+    var s = ""
+    var customArgs = ""
+    if n.kind in nkCallKinds:
+      s = getStrLit(c, n, 1)
+      if n.len <= 3:
+        customArgs = getStrLit(c, n, 2)
+      else:
+        localError(c.config, n.info, "'.compile' pragma takes up 2 arguments")
+    else:
+      s = expectStrLit(c, n)
+
     var found = AbsoluteFile(parentDir(toFullPath(c.config, n.info)) / s)
     if not fileExists(found):
       if isAbsolute(s): found = AbsoluteFile s
@@ -528,7 +533,7 @@ proc processCompile(c: PContext, n: PNode) =
         found = findFile(c.config, s)
         if found.isEmpty: found = AbsoluteFile s
     let obj = toObjFile(c.config, completeCfilePath(c.config, found, false))
-    docompile(c, it, found, obj)
+    docompile(c, it, found, obj, customArgs)
 
 proc processLink(c: PContext, n: PNode) =
   let found = relativeFile(c, n, CC[c.config.cCompiler].objExt)
@@ -538,7 +543,7 @@ proc processLink(c: PContext, n: PNode) =
 proc semAsmOrEmit*(con: PContext, n: PNode, marker: char): PNode =
   case n[1].kind
   of nkStrLit, nkRStrLit, nkTripleStrLit:
-    result = newNode(if n.kind == nkAsmStmt: nkAsmStmt else: nkArgList, n.info)
+    result = newNodeI(if n.kind == nkAsmStmt: nkAsmStmt else: nkArgList, n.info)
     var str = n[1].strVal
     if str == "":
       localError(con.config, n.info, "empty 'asm' statement")
@@ -569,7 +574,7 @@ proc semAsmOrEmit*(con: PContext, n: PNode, marker: char): PNode =
       a = c + 1
   else:
     illFormedAstLocal(n, con.config)
-    result = newNode(nkAsmStmt, n.info)
+    result = newNodeI(nkAsmStmt, n.info)
 
 proc pragmaEmit(c: PContext, n: PNode) =
   if n.kind notin nkPragmaCallKinds or n.len != 2:
@@ -632,7 +637,7 @@ proc processPragma(c: PContext, n: PNode, i: int) =
     invalidPragma(c, n)
 
   var userPragma = newSym(skTemplate, it[1].ident, nil, it.info, c.config.options)
-  userPragma.ast = newNode(nkPragma, n.info, n.sons[i+1..^1])
+  userPragma.ast = newTreeI(nkPragma, n.info, n.sons[i+1..^1])
   strTableAdd(c.userPragmas, userPragma)
 
 proc pragmaRaisesOrTags(c: PContext, n: PNode) =
@@ -776,6 +781,15 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
   if key.kind == nkBracketExpr:
     processNote(c, it)
     return
+  elif key.kind == nkCast:
+    if comesFromPush:
+      localError(c.config, n.info, "a 'cast' pragma cannot be pushed")
+    elif not isStatement:
+      localError(c.config, n.info, "'cast' pragma only allowed in a statement context")
+    case whichPragma(key[1])
+    of wRaises, wTags: pragmaRaisesOrTags(c, key[1])
+    else: discard
+    return
   elif key.kind notin nkIdentKinds:
     n[i] = semCustomPragma(c, it)
     return
@@ -803,8 +817,8 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wExportc, wExportCpp:
         makeExternExport(c, sym, getOptionalStr(c, it, "$1"), it.info)
         if k == wExportCpp:
-          if c.config.cmd != cmdCompileToCpp:
-            localError(c.config, it.info, "exportcpp requires `nim cpp`, got " & $c.config.cmd)
+          if c.config.backend != backendCpp:
+            localError(c.config, it.info, "exportcpp requires `cpp` backend, got " & $c.config.backend)
           else:
             incl(sym.flags, sfMangleCpp)
         incl(sym.flags, sfUsed) # avoid wrong hints
@@ -825,7 +839,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wImportCpp:
         processImportCpp(c, sym, getOptionalStr(c, it, "$1"), it.info)
       of wImportJs:
-        if c.config.cmd != cmdCompileToJS:
+        if c.config.backend != backendJs:
           localError(c.config, it.info, "`importjs` pragma requires the JavaScript target")
         let name = getOptionalStr(c, it, "$1")
         incl(sym.flags, sfImportc)
@@ -870,6 +884,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wRegister:
         noVal(c, it)
         incl(sym.flags, sfRegister)
+      of wNoalias:
+        noVal(c, it)
+        incl(sym.flags, sfNoalias)
       of wThreadVar:
         noVal(c, it)
         incl(sym.flags, {sfThread, sfGlobal})
@@ -879,7 +896,11 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wMagic: processMagic(c, it, sym)
       of wCompileTime:
         noVal(c, it)
-        incl(sym.flags, sfCompileTime)
+        if comesFromPush:
+          if sym.kind in {skProc, skFunc}:
+            incl(sym.flags, sfCompileTime)
+        else:
+          incl(sym.flags, sfCompileTime)
         #incl(sym.loc.flags, lfNoDecl)
       of wGlobal:
         noVal(c, it)
@@ -910,7 +931,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wNoreturn:
         noVal(c, it)
         # Disable the 'noreturn' annotation when in the "Quirky Exceptions" mode!
-        if c.config.exc notin {excQuirky, excGoto}:
+        if c.config.exc != excQuirky:
           incl(sym.flags, sfNoReturn)
         if sym.typ[0] != nil:
           localError(c.config, sym.ast[paramsPos][0].info,
@@ -984,7 +1005,7 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         incl(sym.flags, sfProcvar)
         if sym.typ != nil:
           incl(sym.typ.flags, tfThread)
-          if sym.typ.callConv == ccClosure: sym.typ.callConv = ccDefault
+          if sym.typ.callConv == ccClosure: sym.typ.callConv = ccNimCall
       of wGcSafe:
         noVal(c, it)
         if sym != nil:
@@ -1039,7 +1060,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of wPush:
         processPush(c, n, i + 1)
         result = true
-      of wPop: processPop(c, it)
+      of wPop:
+        processPop(c, it)
+        result = true
       of wPragma:
         if not sym.isNil and sym.kind == skTemplate:
           sym.flags.incl sfCustomPragma
@@ -1067,7 +1090,9 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
       of FirstCallConv..LastCallConv:
         assert(sym != nil)
         if sym.typ == nil: invalidPragma(c, it)
-        else: sym.typ.callConv = wordToCallConv(k)
+        else:
+          sym.typ.callConv = wordToCallConv(k)
+          sym.typ.flags.incl tfExplicitCallConv
       of wEmit: pragmaEmit(c, it)
       of wUnroll: pragmaUnroll(c, it)
       of wLinearScanEnd, wComputedGoto: noVal(c, it)
@@ -1078,6 +1103,10 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         noVal(c, it)
         if sym.typ == nil: invalidPragma(c, it)
         else: incl(sym.typ.flags, tfIncompleteStruct)
+      of wCompleteStruct:
+        noVal(c, it)
+        if sym.typ == nil: invalidPragma(c, it)
+        else: incl(sym.typ.flags, tfCompleteStruct)
       of wUnchecked:
         noVal(c, it)
         if sym.typ == nil or sym.typ.kind notin {tyArray, tyUncheckedArray}:
@@ -1085,9 +1114,12 @@ proc singlePragma(c: PContext, sym: PSym, n: PNode, i: var int,
         else:
           sym.typ.kind = tyUncheckedArray
       of wUnion:
-        noVal(c, it)
-        if sym.typ == nil: invalidPragma(c, it)
-        else: incl(sym.typ.flags, tfUnion)
+        if c.config.backend == backendJs:
+          localError(c.config, it.info, "`{.union.}` is not implemented for js backend.")
+        else:
+          noVal(c, it)
+          if sym.typ == nil: invalidPragma(c, it)
+          else: incl(sym.typ.flags, tfUnion)
       of wRequiresInit:
         noVal(c, it)
         if sym.kind == skField:

compiler/prefixmatches.nim

@@ -54,36 +54,3 @@ proc prefixMatch*(p, s: string): PrefixMatch =
     else:
       return PrefixMatch.None
   return PrefixMatch.None
-
-when isMainModule:
-  import macros
-
-  macro check(val, body: untyped): untyped =
-    result = newStmtList()
-    expectKind body, nnkStmtList
-    for b in body:
-      expectKind b, nnkPar
-      expectLen b, 2
-      let p = b[0]
-      let s = b[1]
-      result.add quote do:
-        echo prefixMatch(`p`, `s`) == `val`
-
-  check PrefixMatch.Prefix:
-    ("abc", "abc")
-    ("a", "abc")
-    ("xyz", "X_yzzzZe")
-
-  check PrefixMatch.Substr:
-    ("b", "abc")
-    ("abc", "fooabcabc")
-    ("abC", "foo_AB_c")
-
-  check PrefixMatch.Abbrev:
-    ("abc", "AxxxBxxxCxxx")
-    ("xyz", "X_yabcZe")
-
-  check PrefixMatch.None:
-    ("foobar", "afkslfjd_as")
-    ("xyz", "X_yuuZuuZe")
-    ("ru", "remotes")

compiler/procfind.nim

@@ -11,7 +11,7 @@
 # This is needed for proper handling of forward declarations.
 
 import
-  ast, astalgo, msgs, semdata, types, trees, strutils
+  ast, astalgo, msgs, semdata, types, trees, strutils, lookups
 
 proc equalGenericParams(procA, procB: PNode): bool =
   if procA.len != procB.len: return false
@@ -28,40 +28,7 @@ proc equalGenericParams(procA, procB: PNode): bool =
       if not exprStructuralEquivalent(a.ast, b.ast): return
   result = true
 
-proc searchForProcOld*(c: PContext, scope: PScope, fn: PSym): PSym =
-  # Searches for a forward declaration or a "twin" symbol of fn
-  # in the symbol table. If the parameter lists are exactly
-  # the same the sym in the symbol table is returned, else nil.
-  var it: TIdentIter
-  result = initIdentIter(it, scope.symbols, fn.name)
-  if isGenericRoutine(fn):
-    # we simply check the AST; this is imprecise but nearly the best what
-    # can be done; this doesn't work either though as type constraints are
-    # not kept in the AST ..
-    while result != nil:
-      if result.kind == fn.kind and isGenericRoutine(result):
-        let genR = result.ast[genericParamsPos]
-        let genF = fn.ast[genericParamsPos]
-        if exprStructuralEquivalent(genR, genF) and
-           exprStructuralEquivalent(result.ast[paramsPos],
-                                    fn.ast[paramsPos]) and
-           equalGenericParams(genR, genF):
-            return
-      result = nextIdentIter(it, scope.symbols)
-  else:
-    while result != nil:
-      if result.kind == fn.kind and not isGenericRoutine(result):
-        case equalParams(result.typ.n, fn.typ.n)
-        of paramsEqual:
-          return
-        of paramsIncompatible:
-          localError(c.config, fn.info, "overloaded '$1' leads to ambiguous calls" % fn.name.s)
-          return
-        of paramsNotEqual:
-          discard
-      result = nextIdentIter(it, scope.symbols)
-
-proc searchForProcNew(c: PContext, scope: PScope, fn: PSym): PSym =
+proc searchForProcAux(c: PContext, scope: PScope, fn: PSym): PSym =
   const flags = {ExactGenericParams, ExactTypeDescValues,
                  ExactConstraints, IgnoreCC}
   var it: TIdentIter
@@ -83,15 +50,13 @@ proc searchForProcNew(c: PContext, scope: PScope, fn: PSym): PSym =
         discard
     result = nextIdentIter(it, scope.symbols)
 
-proc searchForProc*(c: PContext, scope: PScope, fn: PSym): PSym =
-  result = searchForProcNew(c, scope, fn)
-  when false:
-    let old = searchForProcOld(c, scope, fn)
-    if old != result:
-      echo "Mismatch in searchForProc: ", fn.info
-      debug fn.typ
-      debug if result != nil: result.typ else: nil
-      debug if old != nil: old.typ else: nil
+proc searchForProc*(c: PContext, scope: PScope, fn: PSym): tuple[proto: PSym, comesFromShadowScope: bool] =
+  var scope = scope
+  result.proto = searchForProcAux(c, scope, fn)
+  while result.proto == nil and scope.isShadowScope:
+    scope = scope.parent
+    result.proto = searchForProcAux(c, scope, fn)
+    result.comesFromShadowScope = true
 
 when false:
   proc paramsFitBorrow(child, parent: PNode): bool =

compiler/readme.md

@@ -0,0 +1,7 @@
+## Nim Compiler
+
+- This directory contains the Nim compiler written in Nim.
+- Note that this code has been translated from a bootstrapping version written in Pascal.
+- So the code is **not** a poster child of good Nim code.
+
+See [Internals of the Nim Compiler](https://nim-lang.org/docs/intern.html) for more information.

compiler/readme.txt

@@ -1,5 +0,0 @@
-This directory contains the Nim compiler written in Nim. Note that this
-code has been translated from a bootstrapping version written in Pascal, so
-the code is **not** a poster child of good Nim code.
-
-See https://nim-lang.org/docs/intern.html for more information.

compiler/renderer.nim

@@ -20,10 +20,11 @@ import
 type
   TRenderFlag* = enum
     renderNone, renderNoBody, renderNoComments, renderDocComments,
-    renderNoPragmas, renderIds, renderNoProcDefs, renderSyms
+    renderNoPragmas, renderIds, renderNoProcDefs, renderSyms, renderRunnableExamples,
+    renderIr
   TRenderFlags* = set[TRenderFlag]
   TRenderTok* = object
-    kind*: TTokType
+    kind*: TokType
     length*: int16
     sym*: PSym
 
@@ -31,6 +32,7 @@ type
   TSrcGen* = object
     indent*: int
     lineLen*: int
+    col: int
     pos*: int              # current position for iteration over the buffer
     idx*: int              # current token index for iteration over the buffer
     tokens*: TRenderTokSeq
@@ -47,11 +49,20 @@ type
       pendingNewlineCount: int
     fid*: FileIndex
     config*: ConfigRef
+    mangler: seq[PSym]
 
 # We render the source code in a two phases: The first
 # determines how long the subtree will likely be, the second
 # phase appends to a buffer that will be the output.
 
+proc disamb(g: var TSrcGen; s: PSym): int =
+  # we group by 's.name.s' to compute the stable name ID.
+  result = 0
+  for i in 0 ..< g.mangler.len:
+    if s == g.mangler[i]: return result
+    if s.name.s == g.mangler[i].name.s: inc result
+  g.mangler.add s
+
 proc isKeyword*(i: PIdent): bool =
   if (i.id >= ord(tokKeywordLow) - ord(tkSymbol)) and
       (i.id <= ord(tokKeywordHigh) - ord(tkSymbol)):
@@ -108,12 +119,11 @@ proc initSrcGen(g: var TSrcGen, renderFlags: TRenderFlags; config: ConfigRef) =
   g.inGenericParams = false
   g.config = config
 
-proc addTok(g: var TSrcGen, kind: TTokType, s: string; sym: PSym = nil) =
-  setLen(g.tokens, g.tokens.len + 1)
-  g.tokens[^1].kind = kind
-  g.tokens[^1].length = int16(s.len)
-  g.tokens[^1].sym = sym
+proc addTok(g: var TSrcGen, kind: TokType, s: string; sym: PSym = nil) =
+  g.tokens.add TRenderTok(kind: kind, length: int16(s.len), sym: sym)
   g.buf.add(s)
+  if kind != tkSpaces:
+    inc g.col, s.len
 
 proc addPendingNL(g: var TSrcGen) =
   if g.pendingNL >= 0:
@@ -123,6 +133,7 @@ proc addPendingNL(g: var TSrcGen) =
       const newlines = "\n"
     addTok(g, tkSpaces, newlines & spaces(g.pendingNL))
     g.lineLen = g.pendingNL
+    g.col = g.pendingNL
     g.pendingNL = - 1
     g.pendingWhitespace = -1
   elif g.pendingWhitespace >= 0:
@@ -131,7 +142,10 @@ proc addPendingNL(g: var TSrcGen) =
 
 proc putNL(g: var TSrcGen, indent: int) =
   if g.pendingNL >= 0: addPendingNL(g)
-  else: addTok(g, tkSpaces, "\n")
+  else:
+    addTok(g, tkSpaces, "\n")
+    g.col = 0
+
   g.pendingNL = indent
   g.lineLen = indent
   g.pendingWhitespace = -1
@@ -168,21 +182,22 @@ proc dedent(g: var TSrcGen) =
     dec(g.pendingNL, IndentWidth)
     dec(g.lineLen, IndentWidth)
 
-proc put(g: var TSrcGen, kind: TTokType, s: string; sym: PSym = nil) =
+proc put(g: var TSrcGen, kind: TokType, s: string; sym: PSym = nil) =
   if kind != tkSpaces:
     addPendingNL(g)
     if s.len > 0:
       addTok(g, kind, s, sym)
-      inc(g.lineLen, s.len)
   else:
     g.pendingWhitespace = s.len
+    inc g.col, s.len
+  inc(g.lineLen, s.len)
 
 proc putComment(g: var TSrcGen, s: string) =
   if s.len == 0: return
   var i = 0
   let hi = s.len - 1
-  var isCode = (s.len >= 2) and (s[1] != ' ')
-  var ind = g.lineLen
+  let isCode = (s.len >= 2) and (s[1] != ' ')
+  let ind = g.col
   var com = "## "
   while i <= hi:
     case s[i]
@@ -208,7 +223,7 @@ proc putComment(g: var TSrcGen, s: string) =
       # compute length of the following word:
       var j = i
       while j <= hi and s[j] > ' ': inc(j)
-      if not isCode and (g.lineLen + (j - i) > MaxLineLen):
+      if not isCode and (g.col + (j - i) > MaxLineLen):
         put(g, tkComment, com)
         optNL(g, ind)
         com = "## "
@@ -240,7 +255,7 @@ proc maxLineLength(s: string): int =
       inc(lineLen)
       inc(i)
 
-proc putRawStr(g: var TSrcGen, kind: TTokType, s: string) =
+proc putRawStr(g: var TSrcGen, kind: TokType, s: string) =
   var i = 0
   let hi = s.len - 1
   var str = ""
@@ -290,15 +305,18 @@ proc shouldRenderComment(g: var TSrcGen, n: PNode): bool =
 proc gcom(g: var TSrcGen, n: PNode) =
   assert(n != nil)
   if shouldRenderComment(g, n):
+    var oneSpaceAdded = 0
     if (g.pendingNL < 0) and (g.buf.len > 0) and (g.buf[^1] != ' '):
       put(g, tkSpaces, Space)
+      oneSpaceAdded = 1
       # Before long comments we cannot make sure that a newline is generated,
       # because this might be wrong. But it is no problem in practice.
     if (g.pendingNL < 0) and (g.buf.len > 0) and
-        (g.lineLen < LineCommentColumn):
+        (g.col < LineCommentColumn):
       var ml = maxLineLength(n.comment)
       if ml + LineCommentColumn <= MaxLineLen:
-        put(g, tkSpaces, spaces(LineCommentColumn - g.lineLen))
+        put(g, tkSpaces, spaces(LineCommentColumn - g.col))
+        dec g.col, oneSpaceAdded
     putComment(g, n.comment)  #assert(g.comStack[high(g.comStack)] = n);
 
 proc gcoms(g: var TSrcGen) =
@@ -309,8 +327,8 @@ proc lsub(g: TSrcGen; n: PNode): int
 proc litAux(g: TSrcGen; n: PNode, x: BiggestInt, size: int): string =
   proc skip(t: PType): PType =
     result = t
-    while result != nil and result.kind in {tyGenericInst, tyRange, tyVar, tyLent, tyDistinct,
-                          tyOrdinal, tyAlias, tySink}:
+    while result != nil and result.kind in {tyGenericInst, tyRange, tyVar,
+                          tyLent, tyDistinct, tyOrdinal, tyAlias, tySink}:
       result = lastSon(result)
 
   let typ = n.typ.skip
@@ -326,7 +344,7 @@ proc litAux(g: TSrcGen; n: PNode, x: BiggestInt, size: int): string =
 
   if nfBase2 in n.flags: result = "0b" & toBin(x, size * 8)
   elif nfBase8 in n.flags:
-    var y = if size < sizeof(BiggestInt): x and ((1 shl (size*8)) - 1)
+    var y = if size < sizeof(BiggestInt): x and ((1.BiggestInt shl (size*8)) - 1)
             else: x
     result = "0o" & toOct(y, size * 3)
   elif nfBase16 in n.flags: result = "0x" & toHex(x, size * 2)
@@ -449,8 +467,8 @@ proc lsub(g: TSrcGen; n: PNode): int =
   of nkDo: result = lsons(g, n) + len("do__:_")
   of nkConstDef, nkIdentDefs:
     result = lcomma(g, n, 0, - 3)
-    if n[^2].kind != nkEmpty: result = result + lsub(g, n[^2]) + 2
-    if n[^1].kind != nkEmpty: result = result + lsub(g, n[^1]) + 3
+    if n[^2].kind != nkEmpty: result += lsub(g, n[^2]) + 2
+    if n[^1].kind != nkEmpty: result += lsub(g, n[^1]) + 3
   of nkVarTuple:
     if n[^1].kind == nkEmpty:
       result = lcomma(g, n, 0, - 2) + len("()")
@@ -461,8 +479,8 @@ proc lsub(g: TSrcGen; n: PNode): int =
   of nkChckRange: result = len("chckRange") + 2 + lcomma(g, n)
   of nkObjDownConv, nkObjUpConv:
     result = 2
-    if n.len >= 1: result = result + lsub(g, n[0])
-    result = result + lcomma(g, n, 1)
+    if n.len >= 1: result += lsub(g, n[0])
+    result += lcomma(g, n, 1)
   of nkExprColonExpr: result = lsons(g, n) + 2
   of nkInfix: result = lsons(g, n) + 2
   of nkPrefix:
@@ -526,7 +544,7 @@ proc lsub(g: TSrcGen; n: PNode): int =
   of nkGenericParams: result = lcomma(g, n) + 2
   of nkFormalParams:
     result = lcomma(g, n, 1) + 2
-    if n[0].kind != nkEmpty: result = result + lsub(g, n[0]) + 2
+    if n[0].kind != nkEmpty: result += lsub(g, n[0]) + 2
   of nkExceptBranch:
     result = lcomma(g, n, 0, -2) + lsub(g, lastSon(n)) + len("except_:_")
   of nkObjectTy:
@@ -565,7 +583,7 @@ proc hasCom(n: PNode): bool =
     for i in 0..<n.len:
       if hasCom(n[i]): return true
 
-proc putWithSpace(g: var TSrcGen, kind: TTokType, s: string) =
+proc putWithSpace(g: var TSrcGen, kind: TokType, s: string) =
   put(g, kind, s)
   put(g, tkSpaces, Space)
 
@@ -579,7 +597,7 @@ proc gcommaAux(g: var TSrcGen, n: PNode, ind: int, start: int = 0,
     gsub(g, n[i])
     if c:
       if g.tokens.len > oldLen:
-        putWithSpace(g, separator, TokTypeToStr[separator])
+        putWithSpace(g, separator, $separator)
       if hasCom(n[i]):
         gcoms(g)
         optNL(g, ind)
@@ -608,7 +626,7 @@ proc gsons(g: var TSrcGen, n: PNode, c: TContext, start: int = 0,
            theEnd: int = - 1) =
   for i in start..n.len + theEnd: gsub(g, n[i], c)
 
-proc gsection(g: var TSrcGen, n: PNode, c: TContext, kind: TTokType,
+proc gsection(g: var TSrcGen, n: PNode, c: TContext, kind: TokType,
               k: string) =
   if n.len == 0: return # empty var sections are possible
   putWithSpace(g, kind, k)
@@ -837,7 +855,7 @@ proc gident(g: var TSrcGen, n: PNode) =
     if sfAnon in n.sym.flags or
       (n.typ != nil and tfImplicitTypeParam in n.typ.flags): return
 
-  var t: TTokType
+  var t: TokType
   var s = atom(g, n)
   if s.len > 0 and s[0] in lexer.SymChars:
     if n.kind == nkIdent:
@@ -845,12 +863,19 @@ proc gident(g: var TSrcGen, n: PNode) =
           (n.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
         t = tkSymbol
       else:
-        t = TTokType(n.ident.id + ord(tkSymbol))
+        t = TokType(n.ident.id + ord(tkSymbol))
     else:
       t = tkSymbol
   else:
     t = tkOpr
-  if n.kind == nkSym and (renderIds in g.flags or sfGenSym in n.sym.flags or n.sym.kind == skTemp):
+  if renderIr in g.flags and n.kind == nkSym:
+    let localId = disamb(g, n.sym)
+    if localId != 0 and n.sym.magic == mNone:
+      s.add '_'
+      s.addInt localId
+    if sfCursor in n.sym.flags:
+      s.add "_cursor"
+  elif n.kind == nkSym and (renderIds in g.flags or sfGenSym in n.sym.flags or n.sym.kind == skTemp):
     s.add '_'
     s.addInt n.sym.id
     when defined(debugMagics):
@@ -874,12 +899,23 @@ proc gsub(g: var TSrcGen; n: PNode; i: int) =
   else:
     put(g, tkOpr, "<<" & $i & "th child missing for " & $n.kind & " >>")
 
-proc isBracket*(n: PNode): bool =
-  case n.kind
-  of nkClosedSymChoice, nkOpenSymChoice:
-    if n.len > 0: result = isBracket(n[0])
-  of nkSym: result = n.sym.name.s == "[]"
-  else: result = false
+type
+  BracketKind = enum
+    bkNone, bkBracket, bkBracketAsgn, bkCurly, bkCurlyAsgn
+
+proc bracketKind*(g: TSrcGen, n: PNode): BracketKind =
+  if renderIds notin g.flags:
+    case n.kind
+    of nkClosedSymChoice, nkOpenSymChoice:
+      if n.len > 0: result = bracketKind(g, n[0])
+    of nkSym:
+      result = case n.sym.name.s
+        of "[]": bkBracket
+        of "[]=": bkBracketAsgn
+        of "{}": bkCurly
+        of "{}=": bkCurlyAsgn
+        else: bkNone
+    else: result = bkNone
 
 proc skipHiddenNodes(n: PNode): PNode =
   result = n
@@ -892,10 +928,11 @@ proc skipHiddenNodes(n: PNode): PNode =
     else: break
 
 proc accentedName(g: var TSrcGen, n: PNode) =
+  const backticksNeeded = OpChars + {'[', '{'}
   if n == nil: return
   let isOperator =
-    if n.kind == nkIdent and n.ident.s.len > 0 and n.ident.s[0] in OpChars: true
-    elif n.kind == nkSym and n.sym.name.s.len > 0 and n.sym.name.s[0] in OpChars: true
+    if n.kind == nkIdent and n.ident.s.len > 0 and n.ident.s[0] in backticksNeeded: true
+    elif n.kind == nkSym and n.sym.name.s.len > 0 and n.sym.name.s[0] in backticksNeeded: true
     else: false
 
   if isOperator:
@@ -954,12 +991,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
   of nkCharLit: put(g, tkCharLit, atom(g, n))
   of nkNilLit: put(g, tkNil, atom(g, n))    # complex expressions
   of nkCall, nkConv, nkDotCall, nkPattern, nkObjConstr:
-    if renderIds notin g.flags and n.len > 0 and isBracket(n[0]):
-      gsub(g, n, 1)
-      put(g, tkBracketLe, "[")
-      gcomma(g, n, 2)
-      put(g, tkBracketRi, "]")
-    elif n.len > 1 and n.lastSon.kind == nkStmtList:
+    if n.len > 1 and n.lastSon.kind in {nkStmtList, nkStmtListExpr}:
       accentedName(g, n[0])
       if n.len > 2:
         put(g, tkParLe, "(")
@@ -967,10 +999,41 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
         put(g, tkParRi, ")")
       put(g, tkColon, ":")
       gsub(g, n, n.len-1)
+    elif n.len >= 1:
+      case bracketKind(g, n[0])
+      of bkBracket:
+        gsub(g, n, 1)
+        put(g, tkBracketLe, "[")
+        gcomma(g, n, 2)
+        put(g, tkBracketRi, "]")
+      of bkBracketAsgn:
+        gsub(g, n, 1)
+        put(g, tkBracketLe, "[")
+        gcomma(g, n, 2, -2)
+        put(g, tkBracketRi, "]")
+        put(g, tkSpaces, Space)
+        putWithSpace(g, tkEquals, "=")
+        gsub(g, n, n.len - 1)
+      of bkCurly:
+        gsub(g, n, 1)
+        put(g, tkCurlyLe, "{")
+        gcomma(g, n, 2)
+        put(g, tkCurlyRi, "}")
+      of bkCurlyAsgn:
+        gsub(g, n, 1)
+        put(g, tkCurlyLe, "{")
+        gcomma(g, n, 2, -2)
+        put(g, tkCurlyRi, "}")
+        put(g, tkSpaces, Space)
+        putWithSpace(g, tkEquals, "=")
+        gsub(g, n, n.len - 1)
+      of bkNone:
+        accentedName(g, n[0])
+        put(g, tkParLe, "(")
+        gcomma(g, n, 1)
+        put(g, tkParRi, ")")
     else:
-      if n.len >= 1: accentedName(g, n[0])
       put(g, tkParLe, "(")
-      gcomma(g, n, 1)
       put(g, tkParRi, ")")
   of nkCallStrLit:
     if n.len > 0: accentedName(g, n[0])
@@ -978,16 +1041,34 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
       put(g, tkRStrLit, '\"' & replace(n[1].strVal, "\"", "\"\"") & '\"')
     else:
       gsub(g, n, 1)
-  of nkHiddenStdConv, nkHiddenSubConv, nkHiddenCallConv:
+  of nkHiddenStdConv, nkHiddenSubConv:
     if n.len >= 2:
+      when false:
+        # if {renderIds, renderIr} * g.flags != {}:
+        put(g, tkSymbol, "(conv)")
+        put(g, tkParLe, "(")
+        gsub(g, n[1])
+        put(g, tkParRi, ")")
+      else:
+        gsub(g, n[1])
+    else:
+      put(g, tkSymbol, "(wrong conv)")
+  of nkHiddenCallConv:
+    if {renderIds, renderIr} * g.flags != {}:
+      accentedName(g, n[0])
+      put(g, tkParLe, "(")
+      gcomma(g, n, 1)
+      put(g, tkParRi, ")")
+    elif n.len >= 2:
       gsub(g, n[1])
     else:
       put(g, tkSymbol, "(wrong conv)")
   of nkCast:
     put(g, tkCast, "cast")
-    put(g, tkBracketLe, "[")
-    gsub(g, n, 0)
-    put(g, tkBracketRi, "]")
+    if n.len > 0 and n[0].kind != nkEmpty:
+      put(g, tkBracketLe, "[")
+      gsub(g, n, 0)
+      put(g, tkBracketRi, "]")
     put(g, tkParLe, "(")
     gsub(g, n, 1)
     put(g, tkParRi, ")")
@@ -1145,7 +1226,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) =
     infixArgument(g, n, 1)
     put(g, tkSpaces, Space)
     gsub(g, n, 0)        # binary operator
-    # eg: `n1 == n2` decompses as following sum:
+    # e.g.: `n1 == n2` decompses as following sum:
     if n.len == 3 and not fits(g, oldLineLen + lsub(g, n[1]) + lsub(g, n[2]) + lsub(g, n[0]) + len("  ")):
       optNL(g, g.indent + longIndentWid)
     else:
@@ -1579,7 +1660,7 @@ proc initTokRender*(r: var TSrcGen, n: PNode, renderFlags: TRenderFlags = {}) =
   initSrcGen(r, renderFlags, newPartialConfigRef())
   gsub(r, n)
 
-proc getNextTok*(r: var TSrcGen, kind: var TTokType, literal: var string) =
+proc getNextTok*(r: var TSrcGen, kind: var TokType, literal: var string) =
   if r.idx < r.tokens.len:
     kind = r.tokens[r.idx].kind
     let length = r.tokens[r.idx].length.int
@@ -1599,9 +1680,9 @@ proc quoteExpr*(a: string): string {.inline.} =
   ## can be used for quoting expressions in error msgs.
   "'" & a & "'"
 
-proc genFieldError*(field: PSym, disc: PSym): string =
+proc genFieldDefect*(field: PSym, disc: PSym): string =
   ## this needs to be in a module accessible by jsgen, ccgexprs, and vm to
-  ## provide this error msg FieldError; msgs would be better but it does not
+  ## provide this error msg FieldDefect; msgs would be better but it does not
   ## import ast
   result = field.name.s.quoteExpr & " is not accessible using discriminant " &
     disc.name.s.quoteExpr & " of type " &

compiler/renderverbatim.nim

@@ -0,0 +1,152 @@
+import strutils
+from xmltree import addEscaped
+
+import ast, options, msgs
+import packages/docutils/highlite
+
+const isDebug = false
+when isDebug:
+  import renderer
+  import astalgo
+
+proc lastNodeRec(n: PNode): PNode =
+  result = n
+  while result.safeLen > 0: result = result[^1]
+
+proc isInIndentationBlock(src: string, indent: int): bool =
+  #[
+  we stop at the first de-indentation; there's an inherent ambiguity with non
+  doc comments since they can have arbitrary indentation, so we just take the
+  practical route and require a runnableExamples to keep its code (including non
+  doc comments) to its indentation level.
+  ]#
+  for j in 0..<indent:
+    if src.len <= j: return true
+    if src[j] != ' ': return false
+  return true
+
+type LineData = object
+  ## keep track of which lines are starting inside a multiline doc comment.
+  ## We purposefully avoid re-doing parsing which is already done (we get a PNode)
+  ## so we don't worry about whether we're inside (nested) doc comments etc.
+  ## But we sill need some logic to disambiguate different multiline styles.
+  conf: ConfigRef
+  lineFirst: int
+  lines: seq[bool]
+    ## lines[index] is true if line `lineFirst+index` starts inside a multiline string
+    ## Using a HashSet (extra dependency) would simplify but not by much.
+
+proc tripleStrLitStartsAtNextLine(conf: ConfigRef, n: PNode): bool =
+  # enabling TLineInfo.offsetA,offsetB would probably make this easier
+  const tripleQuote = "\"\"\""
+  let src = sourceLine(conf, n.info)
+  let col = n.info.col
+  doAssert src.continuesWith(tripleQuote, col) # sanity check
+  var i = col + 3
+  var onlySpace = true
+  while true:
+    if src.len <= i:
+      doAssert src.len == i
+      return onlySpace
+    elif src.continuesWith(tripleQuote, i) and (src.len == i+3 or src[i+3] != '\"'):
+      return false # triple lit is in 1 line
+    elif src[i] != ' ': onlySpace = false
+    i.inc
+
+proc visitMultilineStrings(ldata: var LineData, n: PNode) =
+  var cline = ldata.lineFirst
+
+  template setLine() =
+    let index = cline - ldata.lineFirst
+    if ldata.lines.len < index+1: ldata.lines.setLen index+1
+    ldata.lines[index] = true
+
+  case n.kind
+  of nkTripleStrLit:
+    # same logic should be applied for any multiline token
+    # we could also consider nkCommentStmt but right now we just assume doc comments,
+    # unlike triple string litterals, don't de-indent from runnableExamples.
+    cline = n.info.line.int
+    if tripleStrLitStartsAtNextLine(ldata.conf, n):
+      cline.inc
+      setLine()
+    for ai in n.strVal:
+      case ai
+      of '\n':
+        cline.inc
+        setLine()
+      else: discard
+  else:
+    for i in 0..<n.safeLen:
+      visitMultilineStrings(ldata, n[i])
+
+proc startOfLineInsideTriple(ldata: LineData, line: int): bool =
+  let index = line - ldata.lineFirst
+  if index >= ldata.lines.len: false
+  else: ldata.lines[index]
+
+proc extractRunnableExamplesSource*(conf: ConfigRef; n: PNode): string =
+  ## TLineInfo.offsetA,offsetB would be cleaner but it's only enabled for nimpretty,
+  ## we'd need to check performance impact to enable it for nimdoc.
+  var first = n.lastSon.info
+  if first.line == n[0].info.line:
+    #[
+    runnableExamples: assert true
+    ]#
+    discard
+  else:
+    #[
+    runnableExamples:
+      # non-doc comment that we want to capture even though `first` points to `assert true`
+      assert true
+    ]#
+    first.line = n[0].info.line + 1
+
+  let last = n.lastNodeRec.info
+  var info = first
+  var indent = info.col
+  let numLines = numLines(conf, info.fileIndex).uint16
+  var lastNonemptyPos = 0
+
+  var ldata = LineData(lineFirst: first.line.int, conf: conf)
+  visitMultilineStrings(ldata, n[^1])
+  when isDebug:
+    debug(n)
+    for i in 0..<ldata.lines.len:
+      echo (i+ldata.lineFirst, ldata.lines[i])
+
+  result = ""
+  for line in first.line..numLines: # bugfix, see `testNimDocTrailingExample`
+    info.line = line
+    let src = sourceLine(conf, info)
+    let special = startOfLineInsideTriple(ldata, line.int)
+    if line > last.line and not special and not isInIndentationBlock(src, indent):
+      break
+    if line > first.line: result.add "\n"
+    if special:
+      result.add src
+      lastNonemptyPos = result.len
+    elif src.len > indent:
+      result.add src[indent..^1]
+      lastNonemptyPos = result.len
+  result.setLen lastNonemptyPos
+
+proc renderNimCode*(result: var string, code: string, isLatex = false) =
+  var toknizr: GeneralTokenizer
+  initGeneralTokenizer(toknizr, code)
+  var buf = ""
+  template append(kind, val) =
+    buf.setLen 0
+    buf.addEscaped(val)
+    let class = tokenClassToStr[kind]
+    if isLatex:
+      result.addf "\\span$1{$2}", [class, buf]
+    else:
+      result.addf  "<span class=\"$1\">$2</span>", [class, buf]
+  while true:
+    getNextToken(toknizr, langNim)
+    case toknizr.kind
+    of gtEof: break  # End Of File (or string)
+    else:
+      # TODO: avoid alloc; maybe toOpenArray
+      append(toknizr.kind, substr(code, toknizr.start, toknizr.length + toknizr.start - 1))

compiler/reorder.nim

@@ -1,6 +1,6 @@
 
 import
-  intsets, ast, idents, algorithm, renderer, os, strutils,
+  intsets, ast, idents, algorithm, renderer, strutils,
   msgs, modulegraphs, syntaxes, options, modulepaths,
   lineinfos
 
@@ -100,37 +100,14 @@ proc computeDeps(cache: IdentCache; n: PNode, declares, uses: var IntSet; topLev
     for i in 0..<n.len: computeDeps(cache, n[i], declares, uses, topLevel)
   of nkPragma:
     let a = n[0]
-    if a.kind == nkExprColonExpr and a[0].kind == nkIdent and
-       a[0].ident.s == "pragma":
-        # user defined pragma
-        decl(a[1])
+    if a.kind == nkExprColonExpr and a[0].kind == nkIdent and a[0].ident.s == "pragma":
+      # user defined pragma
+      decl(a[1])
     else:
       for i in 0..<n.safeLen: deps(n[i])
   else:
     for i in 0..<n.safeLen: deps(n[i])
 
-proc cleanPath(s: string): string =
-  # Here paths may have the form A / B or "A/B"
-  result = ""
-  for c in s:
-    if c != ' ' and c != '\"':
-      result.add c
-
-proc joinPath(parts: seq[string]): string =
-  let nb = parts.len
-  assert nb > 0
-  if nb == 1:
-    return parts[0]
-  result = parts[0] / parts[1]
-  for i in 2..<parts.len:
-    result = result / parts[i]
-
-proc getIncludePath(n: PNode, modulePath: string): string =
-  let istr = n.renderTree.cleanPath
-  let (pdir, _) = modulePath.splitPath
-  let p = istr.split('/').joinPath.addFileExt("nim")
-  result = pdir / p
-
 proc hasIncludes(n:PNode): bool =
   for a in n:
     if a.kind == nkIncludeStmt:
@@ -209,42 +186,42 @@ proc mergeSections(conf: ConfigRef; comps: seq[seq[DepN]], res: PNode) =
           sn.add dn.pnode[0]
         res.add sn
       else:
-          # Problematic circular dependency, we arrange the nodes into
-          # their original relative order and make sure to re-merge
-          # consecutive type and const sections
-          var wmsg = "Circular dependency detected. `codeReordering` pragma may not be able to" &
-            " reorder some nodes properely"
-          when defined(debugReorder):
-            wmsg &= ":\n"
-            for i in 0..<cs.len-1:
-                for j in i..<cs.len:
-                  for ci in 0..<cs[i].kids.len:
-                    if cs[i].kids[ci].id == cs[j].id:
-                      wmsg &= "line " & $cs[i].pnode.info.line &
-                        " depends on line " & $cs[j].pnode.info.line &
-                        ": " & cs[i].expls[ci] & "\n"
-            for j in 0..<cs.len-1:
-                for ci in 0..<cs[^1].kids.len:
-                  if cs[^1].kids[ci].id == cs[j].id:
-                    wmsg &= "line " & $cs[^1].pnode.info.line &
-                      " depends on line " & $cs[j].pnode.info.line &
-                      ": " & cs[^1].expls[ci] & "\n"
-          message(conf, cs[0].pnode.info, warnUser, wmsg)
+        # Problematic circular dependency, we arrange the nodes into
+        # their original relative order and make sure to re-merge
+        # consecutive type and const sections
+        var wmsg = "Circular dependency detected. `codeReordering` pragma may not be able to" &
+          " reorder some nodes properely"
+        when defined(debugReorder):
+          wmsg &= ":\n"
+          for i in 0..<cs.len-1:
+            for j in i..<cs.len:
+              for ci in 0..<cs[i].kids.len:
+                if cs[i].kids[ci].id == cs[j].id:
+                  wmsg &= "line " & $cs[i].pnode.info.line &
+                    " depends on line " & $cs[j].pnode.info.line &
+                    ": " & cs[i].expls[ci] & "\n"
+          for j in 0..<cs.len-1:
+            for ci in 0..<cs[^1].kids.len:
+              if cs[^1].kids[ci].id == cs[j].id:
+                wmsg &= "line " & $cs[^1].pnode.info.line &
+                  " depends on line " & $cs[j].pnode.info.line &
+                  ": " & cs[^1].expls[ci] & "\n"
+        message(conf, cs[0].pnode.info, warnUser, wmsg)
 
-          var i = 0
-          while i < cs.len:
-            if cs[i].pnode.kind in {nkTypeSection, nkConstSection}:
-              let ckind = cs[i].pnode.kind
-              var sn = newNode(ckind)
+        var i = 0
+        while i < cs.len:
+          if cs[i].pnode.kind in {nkTypeSection, nkConstSection}:
+            let ckind = cs[i].pnode.kind
+            var sn = newNode(ckind)
+            sn.add cs[i].pnode[0]
+            inc i
+            while i < cs.len and cs[i].pnode.kind == ckind:
               sn.add cs[i].pnode[0]
               inc i
-              while i < cs.len and cs[i].pnode.kind == ckind:
-                sn.add cs[i].pnode[0]
-                inc i
-              res.add sn
-            else:
-              res.add cs[i].pnode
-              inc i
+            res.add sn
+          else:
+            res.add cs[i].pnode
+            inc i
 
 proc hasImportStmt(n: PNode): bool =
   # Checks if the node is an import statement or
@@ -291,17 +268,17 @@ proc hasAccQuoted(n: PNode): bool =
     if hasAccQuoted(a):
       return true
 
-const extandedProcDefs = procDefs + {nkMacroDef,  nkTemplateDef}
+const extendedProcDefs = procDefs + {nkMacroDef, nkTemplateDef}
 
 proc hasAccQuotedDef(n: PNode): bool =
   # Checks if the node is a function, macro, template ...
   # with a quoted name or if it contains one
   case n.kind
-  of extandedProcDefs:
+  of extendedProcDefs:
     result = n[0].hasAccQuoted
   of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse, nkStaticStmt:
     for a in n:
-      if a.hasAccQuotedDef:
+      if hasAccQuotedDef(a):
         return true
   else:
     result = false
@@ -317,7 +294,7 @@ proc hasBody(n: PNode): bool =
   case n.kind
   of nkCommand, nkCall:
     result = true
-  of extandedProcDefs:
+  of extendedProcDefs:
     result = n[^1].kind == nkStmtList
   of nkStmtList, nkStmtListExpr, nkWhenStmt, nkElifBranch, nkElse, nkStaticStmt:
     for a in n:
@@ -410,8 +387,7 @@ proc strongConnect(v: var DepN, idx: var int, s: var seq[DepN],
 proc getStrongComponents(g: var DepG): seq[seq[DepN]] =
   ## Tarjan's algorithm. Performs a topological sort
   ## and detects strongly connected components.
-  result = newSeq[seq[DepN]]()
-  var s = newSeq[DepN]()
+  var s: seq[DepN]
   var idx = 0
   for v in g.mitems:
     if v.idx < 0:
@@ -423,7 +399,7 @@ proc hasForbiddenPragma(n: PNode): bool =
   for a in n:
     if a.kind == nkPragma and a[0].kind == nkIdent and
         a[0].ident.s == "push":
-          return true
+      return true
 
 proc reorder*(graph: ModuleGraph, n: PNode, module: PSym): PNode =
   if n.hasForbiddenPragma:

compiler/rodimpl.nim

@@ -838,11 +838,12 @@ proc replay(g: ModuleGraph; module: PSym; n: PNode) =
       of "warning": message(g.config, n.info, warnUser, n[1].strVal)
       of "error": localError(g.config, n.info, errUser, n[1].strVal)
       of "compile":
-        internalAssert g.config, n.len == 3 and n[2].kind == nkStrLit
+        internalAssert g.config, n.len == 4 and n[2].kind == nkStrLit
         let cname = AbsoluteFile n[1].strVal
         var cf = Cfile(nimname: splitFile(cname).name, cname: cname,
                        obj: AbsoluteFile n[2].strVal,
-                       flags: {CfileFlag.External})
+                       flags: {CfileFlag.External},
+                       customArgs: n[3].strVal)
         extccomp.addExternalFileToCompile(g.config, cf)
       of "link":
         extccomp.addExternalFileToLink(g.config, AbsoluteFile n[1].strVal)
@@ -916,6 +917,8 @@ proc loadNode*(g: ModuleGraph; module: PSym): PNode =
   replay(g, module, result)
 
 proc setupModuleCache*(g: ModuleGraph) =
+  # historical note: there used to be a `rodfiles` dir with special tests
+  # for incremental compilation via symbol files. This was likely replaced by ic.
   if g.config.symbolFiles == disabledSf: return
   g.recordStmt = recordStmt
   let dbfile = getNimcacheDir(g.config) / RelativeFile"rodfiles.db"

compiler/rodutils.nim

@@ -80,16 +80,14 @@ proc decodeStr*(s: cstring, pos: var int): string =
     else: break
   pos = i
 
-const
-  chars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+const chars = "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 
 {.push overflowChecks: off.}
 
 # since negative numbers require a leading '-' they use up 1 byte. Thus we
 # subtract/add `vintDelta` here to save space for small negative numbers
 # which are common in ROD files:
-const
-  vintDelta = 5
+const vintDelta = 5
 
 template encodeIntImpl(self) =
   var d: char

compiler/saturate.nim

@@ -15,33 +15,33 @@ proc `|+|`*(a, b: BiggestInt): BiggestInt =
   if (result xor a) >= 0'i64 or (result xor b) >= 0'i64:
     return result
   if a < 0 or b < 0:
-    result = low(result)
+    result = low(typeof(result))
   else:
-    result = high(result)
+    result = high(typeof(result))
 
 proc `|-|`*(a, b: BiggestInt): BiggestInt =
   result = a -% b
   if (result xor a) >= 0'i64 or (result xor not b) >= 0'i64:
     return result
   if b > 0:
-    result = low(result)
+    result = low(typeof(result))
   else:
-    result = high(result)
+    result = high(typeof(result))
 
 proc `|abs|`*(a: BiggestInt): BiggestInt =
-  if a != low(a):
+  if a != low(typeof(a)):
     if a >= 0: result = a
     else: result = -a
   else:
-    result = low(a)
+    result = low(typeof(a))
 
 proc `|div|`*(a, b: BiggestInt): BiggestInt =
   # (0..5) div (0..4) == (0..5) div (1..4) == (0 div 4)..(5 div 1)
   if b == 0'i64:
     # make the same as ``div 1``:
     result = a
-  elif a == low(a) and b == -1'i64:
-    result = high(result)
+  elif a == low(typeof(a)) and b == -1'i64:
+    result = high(typeof(result))
   else:
     result = a div b
 
@@ -74,6 +74,6 @@ proc `|*|`*(a, b: BiggestInt): BiggestInt =
     return result
 
   if floatProd >= 0.0:
-    result = high(result)
+    result = high(typeof(result))
   else:
-    result = low(result)
+    result = low(typeof(result))

compiler/scriptconfig.nim

@@ -228,11 +228,14 @@ proc runNimScript*(cache: IdentCache; scriptName: AbsoluteFile;
 
   # watch out, "newruntime" can be set within NimScript itself and then we need
   # to remember this:
+  if conf.selectedGC == gcUnselected:
+    conf.selectedGC = oldSelectedGC
   if optOwnedRefs in oldGlobalOptions:
     conf.globalOptions.incl {optTinyRtti, optOwnedRefs, optSeqDestructors}
     defineSymbol(conf.symbols, "nimv2")
-  if conf.selectedGC == gcUnselected:
-    conf.selectedGC = oldSelectedGC
+  if conf.selectedGC in {gcArc, gcOrc}:
+    conf.globalOptions.incl {optTinyRtti, optSeqDestructors}
+    defineSymbol(conf.symbols, "nimv2")
 
   # ensure we load 'system.nim' again for the real non-config stuff!
   resetSystemArtifacts(graph)

compiler/sem.nim

@@ -16,7 +16,8 @@ import
   procfind, lookups, pragmas, passes, semdata, semtypinst, sigmatch,
   intsets, transf, vmdef, vm, idgen, aliases, cgmeth, lambdalifting,
   evaltempl, patterns, parampatterns, sempass2, linter, semmacrosanity,
-  lowerings, plugins/active, rod, lineinfos, strtabs, int128
+  lowerings, plugins/active, rod, lineinfos, strtabs, int128,
+  isolation_check, typeallowed
 
 from modulegraphs import ModuleGraph, PPassContext, onUse, onDef, onDefResolveForward
 
@@ -197,15 +198,13 @@ proc newSymS(kind: TSymKind, n: PNode, c: PContext): PSym =
     suggestDecl(c, n, result)
 
 proc newSymG*(kind: TSymKind, n: PNode, c: PContext): PSym =
-  proc `$`(kind: TSymKind): string = substr(system.`$`(kind), 2).toLowerAscii
-
   # like newSymS, but considers gensym'ed symbols
   if n.kind == nkSym:
     # and sfGenSym in n.sym.flags:
     result = n.sym
     if result.kind notin {kind, skTemp}:
-      localError(c.config, n.info, "cannot use symbol of kind '" &
-                 $result.kind & "' as a '" & $kind & "'")
+      localError(c.config, n.info, "cannot use symbol of kind '$1' as a '$2'" %
+        [result.kind.toHumanStr, kind.toHumanStr])
     when false:
       if sfGenSym in result.flags and result.kind notin {skTemplate, skMacro, skParam}:
         # declarative context, so produce a fresh gensym:
@@ -229,20 +228,22 @@ proc semIdentVis(c: PContext, kind: TSymKind, n: PNode,
 proc semIdentWithPragma(c: PContext, kind: TSymKind, n: PNode,
                         allowed: TSymFlags): PSym
 
-proc typeAllowedCheck(conf: ConfigRef; info: TLineInfo; typ: PType; kind: TSymKind;
+proc typeAllowedCheck(c: PContext; info: TLineInfo; typ: PType; kind: TSymKind;
                       flags: TTypeAllowedFlags = {}) =
-  let t = typeAllowed(typ, kind, flags)
+  let t = typeAllowed(typ, kind, c, flags)
   if t != nil:
+    var err: string
     if t == typ:
-      localError(conf, info, "invalid type: '" & typeToString(typ) &
-        "' for " & substr($kind, 2).toLowerAscii)
+      err = "invalid type: '$1' for $2" % [typeToString(typ), toHumanStr(kind)]
+      if kind in {skVar, skLet, skConst} and taIsTemplateOrMacro in flags:
+        err &= ". Did you mean to call the $1 with '()'?" % [toHumanStr(typ.owner.kind)]
     else:
-      localError(conf, info, "invalid type: '" & typeToString(t) &
-        "' in this context: '" & typeToString(typ) &
-        "' for " & substr($kind, 2).toLowerAscii)
+      err = "invalid type: '$1' in this context: '$2' for $3" % [typeToString(t),
+              typeToString(typ), toHumanStr(kind)]
+    localError(c.config, info, err)
 
 proc paramsTypeCheck(c: PContext, typ: PType) {.inline.} =
-  typeAllowedCheck(c.config, typ.n.info, typ, skProc)
+  typeAllowedCheck(c, typ.n.info, typ, skProc)
 
 proc expectMacroOrTemplateCall(c: PContext, n: PNode): PSym
 proc semDirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode
@@ -323,7 +324,7 @@ proc tryConstExpr(c: PContext, n: PNode): PNode =
   let oldErrorOutputs = c.config.m.errorOutputs
 
   c.config.m.errorOutputs = {}
-  c.config.errorMax = high(int)
+  c.config.errorMax = high(int) # `setErrorMaxHighMaybe` not appropriate here
 
   try:
     result = evalConstExpr(c.module, c.graph, e)
@@ -377,14 +378,15 @@ proc semExprFlagDispatched(c: PContext, n: PNode, flags: TExprFlags): PNode =
       evaluated = evalAtCompileTime(c, result)
       if evaluated != nil: return evaluated
 
+when not defined(nimHasSinkInference):
+  {.pragma: nosinks.}
+
 include hlo, seminst, semcall
 
-when false:
-  # hopefully not required:
-  proc resetSemFlag(n: PNode) =
-    excl n.flags, nfSem
-    for i in 0..<n.safeLen:
-      resetSemFlag(n[i])
+proc resetSemFlag(n: PNode) =
+  excl n.flags, nfSem
+  for i in 0..<n.safeLen:
+    resetSemFlag(n[i])
 
 proc semAfterMacroCall(c: PContext, call, macroResult: PNode,
                        s: PSym, flags: TExprFlags): PNode =
@@ -399,8 +401,7 @@ proc semAfterMacroCall(c: PContext, call, macroResult: PNode,
   c.friendModules.add(s.owner.getModule)
   idSynchronizationPoint(5000)
   result = macroResult
-  excl(result.flags, nfSem)
-  #resetSemFlag n
+  resetSemFlag result
   if s.typ[0] == nil:
     result = semStmt(c, result, flags)
   else:
@@ -468,7 +469,7 @@ proc semMacroExpr(c: PContext, n, nOrig: PNode, sym: PSym,
 
   #if c.evalContext == nil:
   #  c.evalContext = c.createEvalContext(emStatic)
-  result = evalMacroCall(c.module, c.graph, n, nOrig, sym)
+  result = evalMacroCall(c.module, c.graph, c.templInstCounter, n, nOrig, sym)
   if efNoSemCheck notin flags:
     result = semAfterMacroCall(c, n, result, sym, flags)
   if c.config.macrosToExpand.hasKey(sym.name.s):
@@ -481,15 +482,10 @@ proc forceBool(c: PContext, n: PNode): PNode =
   if result == nil: result = n
 
 proc semConstBoolExpr(c: PContext, n: PNode): PNode =
-  let nn = semExprWithType(c, n)
-  result = fitNode(c, getSysType(c.graph, n.info, tyBool), nn, nn.info)
-  if result == nil:
+  result = forceBool(c, semConstExpr(c, n))
+  if result.kind != nkIntLit:
     localError(c.config, n.info, errConstExprExpected)
-    return nn
-  result = getConstExpr(c.module, result, c.graph)
-  if result == nil:
-    localError(c.config, n.info, errConstExprExpected)
-    result = nn
+
 
 proc semGenericStmt(c: PContext, n: PNode): PNode
 proc semConceptBody(c: PContext, n: PNode): PNode
@@ -506,7 +502,7 @@ proc addCodeForGenerics(c: PContext, n: PNode) =
         n.add prc.ast
   c.lastGenericIdx = c.generics.len
 
-proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext {.nosinks.} =
   var c = newContext(graph, module)
   if c.p != nil: internalError(graph.config, module.info, "sem.myOpen")
   c.semConstExpr = semConstExpr
@@ -522,6 +518,7 @@ proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
   c.semTypeNode = semTypeNode
   c.instTypeBoundOp = sigmatch.instTypeBoundOp
   c.hasUnresolvedArgs = hasUnresolvedArgs
+  c.templInstCounter = new int
 
   pushProcCon(c, module)
   pushOwner(c, c.module)
@@ -597,7 +594,7 @@ proc recoverContext(c: PContext) =
   while getCurrOwner(c).kind != skModule: popOwner(c)
   while c.p != nil and c.p.owner.kind != skModule: c.p = c.p.next
 
-proc myProcess(context: PPassContext, n: PNode): PNode =
+proc myProcess(context: PPassContext, n: PNode): PNode {.nosinks.} =
   var c = PContext(context)
   # no need for an expensive 'try' if we stop after the first error anyway:
   if c.config.errorMax <= 1:

compiler/semcall.nim

@@ -94,10 +94,6 @@ proc pickBestCandidate(c: PContext, headSymbol: PNode,
     if c.currentScope.symbols.counter == counterInitial or syms.len != 0:
       matches(c, n, orig, z)
       if z.state == csMatch:
-        #if sym.name.s == "==" and (n.info ?? "temp3"):
-        #  echo typeToString(sym.typ)
-        #  writeMatches(z)
-
         # little hack so that iterators are preferred over everything else:
         if sym.kind == skIterator: inc(z.exactMatches, 200)
         case best.state
@@ -219,7 +215,11 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
       candidates.add("  first type mismatch at position: " & $err.firstMismatch.arg)
       # candidates.add "\n  reason: " & $err.firstMismatch.kind # for debugging
       case err.firstMismatch.kind
-      of kUnknownNamedParam: candidates.add("\n  unknown named parameter: " & $nArg[0])
+      of kUnknownNamedParam:
+        if nArg == nil:
+          candidates.add("\n  unknown named parameter")
+        else:
+          candidates.add("\n  unknown named parameter: " & $nArg[0])
       of kAlreadyGiven: candidates.add("\n  named param already provided: " & $nArg[0])
       of kPositionalAlreadyGiven: candidates.add("\n  positional param was already given as named param")
       of kExtraArg: candidates.add("\n  extra argument given")
@@ -230,6 +230,9 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
         doAssert err.firstMismatch.formal != nil
         candidates.add("\n  required type for " & nameParam &  ": ")
         candidates.add typeToString(wanted)
+        when false:
+          if wanted.sym != nil:
+            candidates.add "(" & (c.config $ wanted.sym.info) & ")"
         candidates.add "\n  but expression '"
         if err.firstMismatch.kind == kVarNeeded:
           candidates.add renderNotLValue(nArg)
@@ -239,6 +242,10 @@ proc presentFailedCandidates(c: PContext, n: PNode, errors: CandidateErrors):
           candidates.add "' is of type: "
           var got = nArg.typ
           candidates.add typeToString(got)
+          when false:
+            if got.sym != nil:
+              candidates.add "(" & (c.config $ got.sym.info) & ")"
+
           doAssert wanted != nil
           if got != nil: effectProblem(wanted, got, candidates, c)
       of kUnknown: discard "do not break 'nim check'"
@@ -318,12 +325,12 @@ proc getMsgDiagnostic(c: PContext, flags: TExprFlags, n, f: PNode): string =
       sym = nextOverloadIter(o, c, f)
 
   let ident = considerQuotedIdent(c, f, n).s
-  if nfDotField in n.flags and nfExplicitCall notin n.flags:
+  if {nfDotField, nfExplicitCall} * n.flags == {nfDotField}:
     let sym = n[1].typ.sym
     var typeHint = ""
     if sym == nil:
       # Perhaps we're in a `compiles(foo.bar)` expression, or
-      # in a concept, eg:
+      # in a concept, e.g.:
       #   ExplainedConcept {.explain.} = concept x
       #     x.foo is int
       # We could use: `(c.config $ n[1].info)` to get more context.
@@ -495,9 +502,14 @@ proc updateDefaultParams(call: PNode) =
 proc getCallLineInfo(n: PNode): TLineInfo =
   case n.kind
   of nkAccQuoted, nkBracketExpr, nkCall, nkCallStrLit, nkCommand:
-    getCallLineInfo(n[0])
-  of nkDotExpr: getCallLineInfo(n[1])
-  else: n.info
+    if len(n) > 0:
+      return getCallLineInfo(n[0])
+  of nkDotExpr:
+    if len(n) > 1:
+      return getCallLineInfo(n[1])
+  else:
+    discard
+  result = n.info
 
 proc semResolvedCall(c: PContext, x: TCandidate,
                      n: PNode, flags: TExprFlags): PNode =
@@ -527,7 +539,10 @@ proc semResolvedCall(c: PContext, x: TCandidate,
       for s in instantiateGenericParamList(c, gp, x.bindings):
         case s.kind
         of skConst:
-          x.call.add s.ast
+          if not s.ast.isNil:
+            x.call.add s.ast
+          else:
+            x.call.add c.graph.emptyNode
         of skType:
           x.call.add newSymNode(s, n.info)
         else:
@@ -549,7 +564,7 @@ proc tryDeref(n: PNode): PNode =
   result.add n
 
 proc semOverloadedCall(c: PContext, n, nOrig: PNode,
-                       filter: TSymKinds, flags: TExprFlags): PNode =
+                       filter: TSymKinds, flags: TExprFlags): PNode {.nosinks.} =
   var errors: CandidateErrors = @[] # if efExplain in flags: @[] else: nil
   var r = resolveOverloads(c, n, nOrig, filter, flags, errors, efExplain in flags)
   if r.state == csMatch:
@@ -633,7 +648,7 @@ proc explicitGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
     if s.ast[genericParamsPos].safeLen != n.len-1:
       let expected = s.ast[genericParamsPos].safeLen
       localError(c.config, getCallLineInfo(n), errGenerated, "cannot instantiate: '" & renderTree(n) &
-         "'; got " & $(n.len-1) & " type(s) but expected " & $expected)
+         "'; got " & $(n.len-1) & " typeof(s) but expected " & $expected)
       return n
     result = explicitGenericSym(c, n, s)
     if result == nil: result = explicitGenericInstError(c, n)
@@ -673,13 +688,14 @@ proc searchForBorrowProc(c: PContext, startScope: PScope, fn: PSym): PSym =
     if t.kind == tyDistinct or param.typ.kind == tyDistinct: hasDistinct = true
     var x: PType
     if param.typ.kind == tyVar:
-      x = newTypeS(tyVar, c)
+      x = newTypeS(param.typ.kind, c)
       x.addSonSkipIntLit t.baseOfDistinct
     else:
       x = t.baseOfDistinct
     call.add(newNodeIT(nkEmpty, fn.info, x))
   if hasDistinct:
-    var resolved = semOverloadedCall(c, call, call, {fn.kind}, {})
+    let filter = if fn.kind in {skProc, skFunc}: {skProc, skFunc} else: {fn.kind}
+    var resolved = semOverloadedCall(c, call, call, filter, {})
     if resolved != nil:
       result = resolved[0].sym
       if not compareTypes(result.typ[0], fn.typ[0], dcEqIgnoreDistinct):

compiler/semdata.nim

@@ -21,6 +21,7 @@ type
     notes*: TNoteKinds
     features*: set[Feature]
     otherPragmas*: PNode      # every pragma can be pushed
+    warningAsErrors*: TNoteKinds
 
   POptionEntry* = ref TOptionEntry
   PProcCon* = ref TProcCon
@@ -34,7 +35,6 @@ type
     inTryStmt*: int           # whether we are in a try statement; works also
                               # in standalone ``except`` and ``finally``
     next*: PProcCon           # used for stacking procedure contexts
-    wasForwarded*: bool       # whether the current proc has a separate header
     mappingExists*: bool
     mapping*: TIdTable
     caseContext*: seq[tuple[n: PNode, idx: int]]
@@ -84,6 +84,7 @@ type
                                # this is used so that generic instantiations
                                # can access private object fields
     instCounter*: int          # to prevent endless instantiations
+    templInstCounter*: ref int # gives every template instantiation a unique id
 
     ambiguousSymbols*: IntSet  # ids of all ambiguous symbols (cannot
                                # store this info in the syms themselves!)
@@ -142,6 +143,7 @@ type
       # would otherwise fail.
     unusedImports*: seq[(PSym, TLineInfo)]
     exportIndirections*: HashSet[(int, int)]
+    lastTLineInfo*: TLineInfo
 
 template config*(c: PContext): ConfigRef = c.graph.config
 
@@ -210,9 +212,10 @@ proc considerGenSyms*(c: PContext; n: PNode) =
 proc newOptionEntry*(conf: ConfigRef): POptionEntry =
   new(result)
   result.options = conf.options
-  result.defaultCC = ccDefault
+  result.defaultCC = ccNimCall
   result.dynlib = nil
   result.notes = conf.notes
+  result.warningAsErrors = conf.warningAsErrors
 
 proc pushOptionEntry*(c: PContext): POptionEntry =
   new(result)
@@ -221,12 +224,14 @@ proc pushOptionEntry*(c: PContext): POptionEntry =
   result.defaultCC = prev.defaultCC
   result.dynlib = prev.dynlib
   result.notes = c.config.notes
+  result.warningAsErrors = c.config.warningAsErrors
   result.features = c.features
   c.optionStack.add(result)
 
 proc popOptionEntry*(c: PContext) =
   c.config.options = c.optionStack[^1].options
   c.config.notes = c.optionStack[^1].notes
+  c.config.warningAsErrors = c.optionStack[^1].warningAsErrors
   c.features = c.optionStack[^1].features
   c.optionStack.setLen(c.optionStack.len - 1)
 
@@ -274,10 +279,13 @@ proc addToLib*(lib: PLib, sym: PSym) =
 proc newTypeS*(kind: TTypeKind, c: PContext): PType =
   result = newType(kind, getCurrOwner(c))
 
-proc makePtrType*(c: PContext, baseType: PType): PType =
-  result = newTypeS(tyPtr, c)
+proc makePtrType*(owner: PSym, baseType: PType): PType =
+  result = newType(tyPtr, owner)
   addSonSkipIntLit(result, baseType)
 
+proc makePtrType*(c: PContext, baseType: PType): PType =
+  makePtrType(getCurrOwner(c), baseType)
+
 proc makeTypeWithModifier*(c: PContext,
                            modifier: TTypeKind,
                            baseType: PType): PType =
@@ -314,7 +322,8 @@ proc makeTypeDesc*(c: PContext, typ: PType): PType =
 proc makeTypeSymNode*(c: PContext, typ: PType, info: TLineInfo): PNode =
   let typedesc = newTypeS(tyTypeDesc, c)
   incl typedesc.flags, tfCheckedForDestructor
-  typedesc.addSonSkipIntLit(assertNotNil(c.config, typ))
+  internalAssert(c.config, typ != nil)
+  typedesc.addSonSkipIntLit(typ)
   let sym = newSym(skType, c.cache.idAnon, getCurrOwner(c), info,
                    c.config.options).linkTo(typedesc)
   return newSymNode(sym, info)
@@ -372,8 +381,7 @@ proc makeNotType*(c: PContext, t1: PType): PType =
   result.flags.incl tfHasMeta
 
 proc nMinusOne(c: PContext; n: PNode): PNode =
-  result = newNode(nkCall, n.info, @[
-    newSymNode(getSysMagic(c.graph, n.info, "pred", mPred)), n])
+  result = newTreeI(nkCall, n.info, newSymNode(getSysMagic(c.graph, n.info, "pred", mPred)), n)
 
 # Remember to fix the procs below this one when you make changes!
 proc makeRangeWithStaticExpr*(c: PContext, n: PNode): PType =
@@ -382,9 +390,8 @@ proc makeRangeWithStaticExpr*(c: PContext, n: PNode): PType =
   result.sons = @[intType]
   if n.typ != nil and n.typ.n == nil:
     result.flags.incl tfUnresolved
-  result.n = newNode(nkRange, n.info, @[
-    newIntTypeNode(0, intType),
-    makeStaticExpr(c, nMinusOne(c, n))])
+  result.n = newTreeI(nkRange, n.info, newIntTypeNode(0, intType),
+    makeStaticExpr(c, nMinusOne(c, n)))
 
 template rangeHasUnresolvedStatic*(t: PType): bool =
   tfUnresolved in t.flags

compiler/semexprs.nim

@@ -33,7 +33,7 @@ proc semTemplateExpr(c: PContext, n: PNode, s: PSym,
   # Note: This is n.info on purpose. It prevents template from creating an info
   # context when called from an another template
   pushInfoContext(c.config, n.info, s.detailedInfo)
-  result = evalTemplate(n, s, getCurrOwner(c), c.config, c.cache, efFromHlo in flags)
+  result = evalTemplate(n, s, getCurrOwner(c), c.config, c.cache, c.templInstCounter, efFromHlo in flags)
   if efNoSemCheck notin flags: result = semAfterMacroCall(c, n, result, s, flags)
   popInfoContext(c.config)
 
@@ -62,12 +62,18 @@ proc semOperand(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
                renderTree(result, {renderNoComments}))
     result.typ = errorType(c)
 
-proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
+proc semExprCheck(c: PContext, n: PNode, flags: TExprFlags): PNode =
   rejectEmptyNode(n)
   result = semExpr(c, n, flags+{efWantValue})
   if result.kind == nkEmpty:
+    # bug #12741, redundant error messages are the lesser evil here:
+    localError(c.config, n.info, errExprXHasNoType %
+                renderTree(result, {renderNoComments}))
     # do not produce another redundant error message:
     result = errorNode(c, n)
+
+proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
+  result = semExprCheck(c, n, flags)
   if result.typ == nil or result.typ == c.enforceVoidContext:
     localError(c.config, n.info, errExprXHasNoType %
                 renderTree(result, {renderNoComments}))
@@ -76,11 +82,7 @@ proc semExprWithType(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
     if result.typ.kind in {tyVar, tyLent}: result = newDeref(result)
 
 proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
-  rejectEmptyNode(n)
-  result = semExpr(c, n, flags+{efWantValue})
-  if result.kind == nkEmpty:
-    # do not produce another redundant error message:
-    result = errorNode(c, n)
+  result = semExprCheck(c, n, flags)
   if result.typ == nil:
     localError(c.config, n.info, errExprXHasNoType %
                renderTree(result, {renderNoComments}))
@@ -176,7 +178,7 @@ proc checkConvertible(c: PContext, targetTyp: PType, src: PNode): TConvStatus =
     else:
       discard
 
-proc isCastable(conf: ConfigRef; dst, src: PType): bool =
+proc isCastable(c: PContext; dst, src: PType): bool =
   ## Checks whether the source type can be cast to the destination type.
   ## Casting is very unrestrictive; casts are allowed as long as
   ## castDest.size >= src.size, and typeAllowed(dst, skParam)
@@ -189,6 +191,14 @@ proc isCastable(conf: ConfigRef; dst, src: PType): bool =
     return false
   if skipTypes(src, abstractInst-{tyTypeDesc}).kind == tyTypeDesc:
     return false
+  if skipTypes(dst, abstractInst).kind == tyBuiltInTypeClass:
+    return false
+  let conf = c.config
+  if conf.selectedGC in {gcArc, gcOrc}:
+    let d = skipTypes(dst, abstractInst)
+    let s = skipTypes(src, abstractInst)
+    if d.kind == tyRef and s.kind == tyRef and s[0].isFinal != d[0].isFinal:
+      return false
 
   var dstSize, srcSize: BiggestInt
   dstSize = computeSize(conf, dst)
@@ -201,7 +211,7 @@ proc isCastable(conf: ConfigRef; dst, src: PType): bool =
     result = false
   elif srcSize < 0:
     result = false
-  elif typeAllowed(dst, skParam) != nil:
+  elif typeAllowed(dst, skParam, c) != nil:
     result = false
   elif dst.kind == tyProc and dst.callConv == ccClosure:
     result = src.kind == tyProc and src.callConv == ccClosure
@@ -329,7 +339,7 @@ proc semCast(c: PContext, n: PNode): PNode =
   let castedExpr = semExprWithType(c, n[1])
   if tfHasMeta in targetType.flags:
     localError(c.config, n[0].info, "cannot cast to a non concrete type: '$1'" % $targetType)
-  if not isCastable(c.config, targetType, castedExpr.typ):
+  if not isCastable(c, targetType, castedExpr.typ):
     let tar = $targetType
     let alt = typeToString(targetType, preferDesc)
     let msg = if tar != alt: tar & "=" & alt else: tar
@@ -421,7 +431,7 @@ proc isOpImpl(c: PContext, n: PNode, flags: TExprFlags): PNode =
       m.diagnostics = @[]
       m.diagnosticsEnabled = true
     res = typeRel(m, t2, t1) >= isSubtype # isNone
-    # `res = sameType(t1, t2)` would be wrong, eg for `int is (int|float)`
+    # `res = sameType(t1, t2)` would be wrong, e.g. for `int is (int|float)`
 
   result = newIntNode(nkIntLit, ord(res))
   result.typ = n.typ
@@ -622,11 +632,12 @@ proc isAssignable(c: PContext, n: PNode; isUnsafeAddr=false): TAssignableResult
   result = parampatterns.isAssignable(c.p.owner, n, isUnsafeAddr)
 
 proc isUnresolvedSym(s: PSym): bool =
-  return s.kind == skGenericParam or
-         tfInferrableStatic in s.typ.flags or
-         (s.kind == skParam and s.typ.isMetaType) or
-         (s.kind == skType and
-          s.typ.flags * {tfGenericTypeParam, tfImplicitTypeParam} != {})
+  result = s.kind == skGenericParam
+  if not result and s.typ != nil:
+    result = tfInferrableStatic in s.typ.flags or
+        (s.kind == skParam and s.typ.isMetaType) or
+        (s.kind == skType and
+        s.typ.flags * {tfGenericTypeParam, tfImplicitTypeParam} != {})
 
 proc hasUnresolvedArgs(c: PContext, n: PNode): bool =
   # Checks whether an expression depends on generic parameters that
@@ -652,7 +663,7 @@ proc hasUnresolvedArgs(c: PContext, n: PNode): bool =
     return false
 
 proc newHiddenAddrTaken(c: PContext, n: PNode): PNode =
-  if n.kind == nkHiddenDeref and not (c.config.cmd == cmdCompileToCpp or
+  if n.kind == nkHiddenDeref and not (c.config.backend == backendCpp or
                                       sfCompileToCpp in c.module.flags):
     checkSonsLen(n, 1, c.config)
     result = n[0]
@@ -704,7 +715,7 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) =
     # note sometimes this is eval'ed twice so we check for nkHiddenAddr here:
     for i in 1..<n.len:
       if i < t.len and t[i] != nil and
-          skipTypes(t[i], abstractInst-{tyTypeDesc}).kind == tyVar:
+          skipTypes(t[i], abstractInst-{tyTypeDesc}).kind in {tyVar}:
         let it = n[i]
         if isAssignable(c, it) notin {arLValue, arLocalLValue}:
           if it.kind != nkHiddenAddr:
@@ -725,7 +736,7 @@ proc analyseIfAddressTakenInCall(c: PContext, n: PNode) =
       # calls and then they wouldn't be analysed otherwise
       analyseIfAddressTakenInCall(c, n[i])
     if i < t.len and
-        skipTypes(t[i], abstractInst-{tyTypeDesc}).kind == tyVar:
+        skipTypes(t[i], abstractInst-{tyTypeDesc}).kind in {tyVar}:
       if n[i].kind != nkHiddenAddr:
         n[i] = analyseIfAddressTaken(c, n[i])
 
@@ -784,7 +795,7 @@ proc evalAtCompileTime(c: PContext, n: PNode): PNode =
     if callee.kind notin {skProc, skFunc, skConverter, skConst} or callee.isGenericRoutine:
       return
 
-    if n.typ != nil and typeAllowed(n.typ, skConst) != nil: return
+    if n.typ != nil and typeAllowed(n.typ, skConst, c) != nil: return
 
     var call = newNodeIT(nkCall, n.info, n.typ)
     call.add(n[0])
@@ -945,8 +956,17 @@ proc semIndirectOp(c: PContext, n: PNode, flags: TExprFlags): PNode =
             hasErrorType = true
             break
         if not hasErrorType:
+          let typ = n[0].typ
           msg.add(">\nbut expected one of: \n" &
-              typeToString(n[0].typ))
+              typeToString(typ))
+          # prefer notin preferToResolveSymbols
+          # t.sym != nil
+          # sfAnon notin t.sym.flags
+          # t.kind != tySequence(It is tyProc)
+          if typ.sym != nil and sfAnon notin typ.sym.flags and
+                                typ.kind == tyProc:
+            msg.add(" = " &
+                typeToString(typ, preferDesc))
           localError(c.config, n.info, msg)
         return errorNode(c, n)
       result = nil
@@ -1184,17 +1204,6 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     elif sfGenSym in s.flags:
       # the owner should have been set by now by addParamOrResult
       internalAssert c.config, s.owner != nil
-      if c.p.wasForwarded:
-        # gensym'ed parameters that nevertheless have been forward declared
-        # need a special fixup:
-        let realParam = c.p.owner.typ.n[s.position+1]
-        internalAssert c.config, realParam.kind == nkSym and realParam.sym.kind == skParam
-        return newSymNode(c.p.owner.typ.n[s.position+1].sym, n.info)
-      elif c.p.owner.kind == skMacro:
-        # gensym'ed macro parameters need a similar hack (see bug #1944):
-        var u = searchInScopes(c, s.name)
-        internalAssert c.config, u != nil and u.kind == skParam and u.owner == s.owner
-        return newSymNode(u, n.info)
     result = newSymNode(s, n.info)
   of skVar, skLet, skResult, skForVar:
     if s.magic == mNimvm:
@@ -1263,6 +1272,54 @@ proc semSym(c: PContext, n: PNode, sym: PSym, flags: TExprFlags): PNode =
     onUse(info, s)
     result = newSymNode(s, info)
 
+proc tryReadingGenericParam(c: PContext, n: PNode, i: PIdent, t: PType): PNode =
+  case t.kind
+  of tyTypeParamsHolders:
+    result = readTypeParameter(c, t, i, n.info)
+    if result == c.graph.emptyNode:
+      result = n
+      n.typ = makeTypeFromExpr(c, n.copyTree)
+  of tyUserTypeClasses:
+    if t.isResolvedUserTypeClass:
+      result = readTypeParameter(c, t, i, n.info)
+    else:
+      n.typ = makeTypeFromExpr(c, copyTree(n))
+      result = n
+  of tyGenericParam, tyAnything:
+    n.typ = makeTypeFromExpr(c, copyTree(n))
+    result = n
+  else:
+    discard
+
+proc tryReadingTypeField(c: PContext, n: PNode, i: PIdent, ty: PType): PNode =
+  var ty = ty.skipTypes(tyDotOpTransparent)
+  case ty.kind
+  of tyEnum:
+    # look up if the identifier belongs to the enum:
+    var f = PSym(nil)
+    while ty != nil:
+      f = getSymFromList(ty.n, i)
+      if f != nil: break
+      ty = ty.sons[0]         # enum inheritance
+    if f != nil:
+      result = newSymNode(f)
+      result.info = n.info
+      result.typ = ty
+      markUsed(c, n.info, f)
+      onUse(n.info, f)
+  of tyObject, tyTuple:
+    if ty.n != nil and ty.n.kind == nkRecList:
+      let field = lookupInRecord(ty.n, i)
+      if field != nil:
+        n.typ = makeTypeDesc(c, field.typ)
+        result = n
+  of tyGenericInst:
+    result = tryReadingTypeField(c, n, i, ty.lastSon)
+    if result == nil:
+      result = tryReadingGenericParam(c, n, i, ty)
+  else:
+    result = tryReadingGenericParam(c, n, i, ty)
+
 proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
   ## returns nil if it's not a built-in field access
   checkSonsLen(n, 2, c.config)
@@ -1293,28 +1350,6 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
   var f: PSym = nil
   result = nil
 
-  template tryReadingGenericParam(t: PType) =
-    case t.kind
-    of tyTypeParamsHolders:
-      result = readTypeParameter(c, t, i, n.info)
-      if result == c.graph.emptyNode:
-        result = n
-        n.typ = makeTypeFromExpr(c, n.copyTree)
-      return
-    of tyUserTypeClasses:
-      if t.isResolvedUserTypeClass:
-        return readTypeParameter(c, t, i, n.info)
-      else:
-        n.typ = makeTypeFromExpr(c, copyTree(n))
-        return n
-    of tyGenericParam, tyAnything:
-      n.typ = makeTypeFromExpr(c, copyTree(n))
-      return n
-    else:
-      discard
-
-  var argIsType = false
-
   if ty.kind == tyTypeDesc:
     if ty.base.kind == tyNone:
       # This is a still unresolved typedesc parameter.
@@ -1327,40 +1362,10 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
       else:
         return nil
     else:
-      ty = ty.base
-      argIsType = true
-  else:
-    argIsType = isTypeExpr(n[0])
+      return tryReadingTypeField(c, n, i, ty.base)
+  elif isTypeExpr(n.sons[0]):
+    return tryReadingTypeField(c, n, i, ty)
 
-  if argIsType:
-    ty = ty.skipTypes(tyDotOpTransparent)
-    case ty.kind
-    of tyEnum:
-      # look up if the identifier belongs to the enum:
-      while ty != nil:
-        f = getSymFromList(ty.n, i)
-        if f != nil: break
-        ty = ty[0]         # enum inheritance
-      if f != nil:
-        result = newSymNode(f)
-        result.info = n.info
-        result.typ = ty
-        markUsed(c, n.info, f)
-        onUse(n.info, f)
-        return
-    of tyObject, tyTuple:
-      if ty.n != nil and ty.n.kind == nkRecList:
-        let field = lookupInRecord(ty.n, i)
-        if field != nil:
-          n.typ = makeTypeDesc(c, field.typ)
-          return n
-    else:
-      tryReadingGenericParam(ty)
-      return
-    # XXX: This is probably not relevant any more
-    # reset to prevent 'nil' bug: see "tests/reject/tenumitems.nim":
-    ty = n[0].typ
-    return nil
   if ty.kind in tyUserTypeClasses and ty.isResolvedUserTypeClass:
     ty = ty.lastSon
   ty = skipTypes(ty, {tyGenericInst, tyVar, tyLent, tyPtr, tyRef, tyOwned, tyAlias, tySink})
@@ -1404,7 +1409,7 @@ proc builtinFieldAccess(c: PContext, n: PNode, flags: TExprFlags): PNode =
   # we didn't find any field, let's look for a generic param
   if result == nil:
     let t = n[0].typ.skipTypes(tyDotOpTransparent)
-    tryReadingGenericParam(t)
+    result = tryReadingGenericParam(c, n, i, t)
 
 proc dotTransformation(c: PContext, n: PNode): PNode =
   if isSymChoice(n[1]):
@@ -1552,10 +1557,9 @@ proc propertyWriteAccess(c: PContext, n, nOrig, a: PNode): PNode =
   # this is ugly. XXX Semantic checking should use the ``nfSem`` flag for
   # nodes?
   let aOrig = nOrig[0]
-  result = newNode(nkCall, n.info, sons = @[setterId, a[0],
-                                            semExprWithType(c, n[1])])
+  result = newTreeI(nkCall, n.info, setterId, a[0], semExprWithType(c, n[1]))
   result.flags.incl nfDotSetter
-  let orig = newNode(nkCall, n.info, sons = @[setterId, aOrig[0], nOrig[1]])
+  let orig = newTreeI(nkCall, n.info, setterId, aOrig[0], nOrig[1])
   result = semOverloadedCallAnalyseEffects(c, result, orig, {})
 
   if result != nil:
@@ -1568,17 +1572,22 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode =
   # return a view into the first argument (if there is one):
   let root = exprRoot(n)
   if root != nil and root.owner == c.p.owner:
+    template url: string = "var_t_return.html".createDocLink
     if root.kind in {skLet, skVar, skTemp} and sfGlobal notin root.flags:
-      localError(c.config, n.info, "'$1' escapes its stack frame; context: '$2'; see $3/var_t_return.html" % [
-        root.name.s, renderTree(n, {renderNoComments}), explanationsBaseUrl])
+      localError(c.config, n.info, "'$1' escapes its stack frame; context: '$2'; see $3" % [
+        root.name.s, renderTree(n, {renderNoComments}), url])
     elif root.kind == skParam and root.position != 0:
-      localError(c.config, n.info, "'$1' is not the first parameter; context: '$2'; see $3/var_t_return.html" % [
-        root.name.s, renderTree(n, {renderNoComments}), explanationsBaseUrl])
+      localError(c.config, n.info, "'$1' is not the first parameter; context: '$2'; see $3" % [
+        root.name.s, renderTree(n, {renderNoComments}), url])
   case n.kind
   of nkHiddenAddr, nkAddr: return n
-  of nkHiddenDeref, nkDerefExpr: return n[0]
+  of nkDerefExpr: return n[0]
   of nkBracketExpr:
     if n.len == 1: return n[0]
+  of nkHiddenDeref:
+    # issue #13848
+    # `proc fun(a: var int): var int = a`
+    discard
   else: discard
   let valid = isAssignable(c, n, isLent)
   if valid != arLValue:
@@ -1592,7 +1601,7 @@ proc takeImplicitAddr(c: PContext, n: PNode; isLent: bool): PNode =
 proc asgnToResultVar(c: PContext, n, le, ri: PNode) {.inline.} =
   if le.kind == nkHiddenDeref:
     var x = le[0]
-    if x.typ.kind in {tyVar, tyLent} and x.kind == nkSym and x.sym.kind == skResult:
+    if (x.typ.kind in {tyVar, tyLent} or classifyViewType(x.typ) != noView) and x.kind == nkSym and x.sym.kind == skResult:
       n[0] = x # 'result[]' --> 'result'
       n[1] = takeImplicitAddr(c, ri, x.typ.kind == tyLent)
       x.typ.flags.incl tfVarIsPtr
@@ -1702,9 +1711,9 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
   var le = a.typ
   if le == nil:
     localError(c.config, a.info, "expression has no type")
-  elif (skipTypes(le, {tyGenericInst, tyAlias, tySink}).kind != tyVar and
-        isAssignable(c, a) == arNone) or
-      skipTypes(le, abstractVar).kind in {tyOpenArray, tyVarargs}:
+  elif (skipTypes(le, {tyGenericInst, tyAlias, tySink}).kind notin {tyVar} and
+        isAssignable(c, a) in {arNone, arLentValue}) or (
+      skipTypes(le, abstractVar).kind in {tyOpenArray, tyVarargs} and views notin c.features):
     # Direct assignment to a discriminant is allowed!
     localError(c.config, a.info, errXCannotBeAssignedTo %
                renderTree(a, {renderNoComments}))
@@ -1724,7 +1733,7 @@ proc semAsgn(c: PContext, n: PNode; mode=asgnNormal): PNode =
         if cmpTypes(c, lhs.typ, rhsTyp) in {isGeneric, isEqual}:
           internalAssert c.config, c.p.resultSym != nil
           # Make sure the type is valid for the result variable
-          typeAllowedCheck(c.config, n.info, rhsTyp, skResult)
+          typeAllowedCheck(c, n.info, rhsTyp, skResult)
           lhs.typ = rhsTyp
           c.p.resultSym.typ = rhsTyp
           c.p.owner.typ[0] = rhsTyp
@@ -1745,17 +1754,21 @@ proc semReturn(c: PContext, n: PNode): PNode =
   if c.p.owner.kind in {skConverter, skMethod, skProc, skFunc, skMacro} or
       (not c.p.owner.typ.isNil and isClosureIterator(c.p.owner.typ)):
     if n[0].kind != nkEmpty:
-      # transform ``return expr`` to ``result = expr; return``
-      if c.p.resultSym != nil:
+      if n[0].kind == nkAsgn and n[0][0].kind == nkSym and c.p.resultSym == n[0][0].sym:
+        discard "return is already transformed"
+      elif c.p.resultSym != nil:
+        # transform ``return expr`` to ``result = expr; return``
         var a = newNodeI(nkAsgn, n[0].info)
         a.add newSymNode(c.p.resultSym)
         a.add n[0]
-        n[0] = semAsgn(c, a)
-        # optimize away ``result = result``:
-        if n[0][1].kind == nkSym and n[0][1].sym == c.p.resultSym:
-          n[0] = c.graph.emptyNode
+        n[0] = a
       else:
         localError(c.config, n.info, errNoReturnTypeDeclared)
+        return
+      result[0] = semAsgn(c, n[0])
+      # optimize away ``result = result``:
+      if result[0][1].kind == nkSym and result[0][1].sym == c.p.resultSym:
+        result[0] = c.graph.emptyNode
   else:
     localError(c.config, n.info, "'return' not allowed here")
 
@@ -1817,7 +1830,11 @@ proc semYieldVarResult(c: PContext, n: PNode, restype: PType) =
             tupleConstr[i] = takeImplicitAddr(c, tupleConstr[i], e.kind == tyLent)
         else:
           localError(c.config, n[0].info, errXExpected, "tuple constructor")
-  else: discard
+  else:
+    when false:
+      # XXX investigate what we really need here.
+      if isViewType(t):
+        n[0] = takeImplicitAddr(c, n[0], false)
 
 proc semYield(c: PContext, n: PNode): PNode =
   result = n
@@ -1845,29 +1862,32 @@ proc semYield(c: PContext, n: PNode): PNode =
   elif c.p.owner.typ[0] != nil:
     localError(c.config, n.info, errGenerated, "yield statement must yield a value")
 
-proc lookUpForDefined(c: PContext, i: PIdent, onlyCurrentScope: bool): PSym =
-  if onlyCurrentScope:
-    result = localSearchInScope(c, i)
-  else:
-    result = searchInScopes(c, i) # no need for stub loading
+proc semDefined(c: PContext, n: PNode): PNode =
+  checkSonsLen(n, 2, c.config)
+  # we replace this node by a 'true' or 'false' node:
+  result = newIntNode(nkIntLit, 0)
+  result.intVal = ord isDefined(c.config, considerQuotedIdent(c, n[1], n).s)
+  result.info = n.info
+  result.typ = getSysType(c.graph, n.info, tyBool)
 
-proc lookUpForDefined(c: PContext, n: PNode, onlyCurrentScope: bool): PSym =
+proc lookUpForDeclared(c: PContext, n: PNode, onlyCurrentScope: bool): PSym =
   case n.kind
-  of nkIdent:
-    result = lookUpForDefined(c, n.ident, onlyCurrentScope)
+  of nkIdent, nkAccQuoted:
+    result = if onlyCurrentScope:
+               localSearchInScope(c, considerQuotedIdent(c, n))
+             else:
+               searchInScopes(c, considerQuotedIdent(c, n))
   of nkDotExpr:
     result = nil
     if onlyCurrentScope: return
     checkSonsLen(n, 2, c.config)
-    var m = lookUpForDefined(c, n[0], onlyCurrentScope)
+    var m = lookUpForDeclared(c, n[0], onlyCurrentScope)
     if m != nil and m.kind == skModule:
       let ident = considerQuotedIdent(c, n[1], n)
       if m == c.module:
         result = strTableGet(c.topLevelScope.symbols, ident)
       else:
         result = strTableGet(m.tab, ident)
-  of nkAccQuoted:
-    result = lookUpForDefined(c, considerQuotedIdent(c, n), onlyCurrentScope)
   of nkSym:
     result = n.sym
   of nkOpenSymChoice, nkClosedSymChoice:
@@ -1876,15 +1896,11 @@ proc lookUpForDefined(c: PContext, n: PNode, onlyCurrentScope: bool): PSym =
     localError(c.config, n.info, "identifier expected, but got: " & renderTree(n))
     result = nil
 
-proc semDefined(c: PContext, n: PNode, onlyCurrentScope: bool): PNode =
+proc semDeclared(c: PContext, n: PNode, onlyCurrentScope: bool): PNode =
   checkSonsLen(n, 2, c.config)
   # we replace this node by a 'true' or 'false' node:
   result = newIntNode(nkIntLit, 0)
-  if not onlyCurrentScope and considerQuotedIdent(c, n[0], n).s == "defined":
-    let d = considerQuotedIdent(c, n[1], n)
-    result.intVal = ord isDefined(c.config, d.s)
-  elif lookUpForDefined(c, n[1], onlyCurrentScope) != nil:
-    result.intVal = 1
+  result.intVal = ord lookUpForDeclared(c, n[1], onlyCurrentScope) != nil
   result.info = n.info
   result.typ = getSysType(c.graph, n.info, tyBool)
 
@@ -2033,7 +2049,7 @@ proc semQuoteAst(c: PContext, n: PNode): PNode =
     dummyTemplate[paramsPos].add getSysSym(c.graph, n.info, "untyped").newSymNode # return type
     ids.add getSysSym(c.graph, n.info, "untyped").newSymNode # params type
     ids.add c.graph.emptyNode # no default value
-    dummyTemplate[paramsPos].add newNode(nkIdentDefs, n.info, ids)
+    dummyTemplate[paramsPos].add newTreeI(nkIdentDefs, n.info, ids)
 
   var tmpl = semTemplateDef(c, dummyTemplate)
   quotes[0] = tmpl[namePos]
@@ -2045,14 +2061,18 @@ proc semQuoteAst(c: PContext, n: PNode): PNode =
                     newIdentNode(getIdent(c.cache, "newIdentNode"), n.info)
                   else:
                     identNodeSym.newSymNode
-  quotes[1] = newNode(nkCall, n.info, @[identNode, newStrNode(nkStrLit, "result")])
-  result = newNode(nkCall, n.info, @[
+  quotes[1] = newTreeI(nkCall, n.info, identNode, newStrNode(nkStrLit, "result"))
+  result = newTreeI(nkCall, n.info,
      createMagic(c.graph, "getAst", mExpandToAst).newSymNode,
-    newNode(nkCall, n.info, quotes)])
+     newTreeI(nkCall, n.info, quotes))
   result = semExpandToAst(c, result)
 
 proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   # watch out, hacks ahead:
+  when defined(nimsuggest):
+    # Remove the error hook so nimsuggest doesn't report errors there
+    let tempHook = c.graph.config.structuredErrorHook
+    c.graph.config.structuredErrorHook = nil
   let oldErrorCount = c.config.errorCounter
   let oldErrorMax = c.config.errorMax
   let oldCompilesId = c.compilesContextId
@@ -2061,8 +2081,7 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   if c.compilesContextId == 0:
     inc c.compilesContextIdGenerator
     c.compilesContextId = c.compilesContextIdGenerator
-  # do not halt after first error:
-  c.config.errorMax = high(int)
+  c.config.errorMax = high(int) # `setErrorMaxHighMaybe` not appropriate here
 
   # open a scope for temporary symbol inclusions:
   let oldScope = c.currentScope
@@ -2102,6 +2121,9 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   c.config.m.errorOutputs = oldErrorOutputs
   c.config.errorCounter = oldErrorCount
   c.config.errorMax = oldErrorMax
+  when defined(nimsuggest):
+    # Restore the error hook
+    c.graph.config.structuredErrorHook = tempHook
 
 proc semCompiles(c: PContext, n: PNode, flags: TExprFlags): PNode =
   # we replace this node by a 'true' or 'false' node:
@@ -2140,7 +2162,7 @@ proc instantiateCreateFlowVarCall(c: PContext; t: PType;
   # since it's an instantiation, we unmark it as a compilerproc. Otherwise
   # codegen would fail:
   if sfCompilerProc in result.flags:
-    result.flags = result.flags - {sfCompilerProc, sfExportc, sfImportc}
+    result.flags.excl {sfCompilerProc, sfExportc, sfImportc}
     result.loc.r = nil
 
 proc setMs(n: PNode, s: PSym): PNode =
@@ -2172,10 +2194,13 @@ proc semMagic(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
     result = semTypeOf(c, n)
   of mDefined:
     markUsed(c, n.info, s)
-    result = semDefined(c, setMs(n, s), false)
-  of mDefinedInScope:
+    result = semDefined(c, setMs(n, s))
+  of mDeclared:
     markUsed(c, n.info, s)
-    result = semDefined(c, setMs(n, s), true)
+    result = semDeclared(c, setMs(n, s), false)
+  of mDeclaredInScope:
+    markUsed(c, n.info, s)
+    result = semDeclared(c, setMs(n, s), true)
   of mCompiles:
     markUsed(c, n.info, s)
     result = semCompiles(c, setMs(n, s), flags)
@@ -2415,7 +2440,7 @@ proc checkPar(c: PContext; n: PNode): TParKind =
     for i in 0..<n.len:
       if result == paTupleFields:
         if (n[i].kind != nkExprColonExpr) or
-            n[i][0].kind notin {nkSym, nkIdent}:
+            n[i][0].kind notin {nkSym, nkIdent, nkAccQuoted}:
           localError(c.config, n[i].info, errNamedExprExpected)
           return paNone
       else:
@@ -2562,6 +2587,43 @@ proc shouldBeBracketExpr(n: PNode): bool =
           n[0] = be
           return true
 
+proc hoistParamsUsedInDefault(c: PContext, call, letSection, defExpr: var PNode) =
+  # This takes care of complicated signatures such as:
+  # proc foo(a: int, b = a)
+  # proc bar(a: int, b: int, c = a + b)
+  #
+  # The recursion may confuse you. It performs two duties:
+  #
+  # 1) extracting all referenced params from default expressions
+  #    into a let section preceding the call
+  #
+  # 2) replacing the "references" within the default expression
+  #    with these extracted skLet symbols.
+  #
+  # The first duty is carried out directly in the code here, while the second
+  # duty is activated by returning a non-nil value. The caller is responsible
+  # for replacing the input to the function with the returned non-nil value.
+  # (which is the hoisted symbol)
+  if defExpr.kind == nkSym and defExpr.sym.kind == skParam and defExpr.sym.owner == call[0].sym:
+    let paramPos = defExpr.sym.position + 1
+
+    if call[paramPos].kind != nkSym:
+      let hoistedVarSym = newSym(skLet, getIdent(c.graph.cache, genPrefix), c.p.owner, letSection.info, c.p.owner.options)
+      hoistedVarSym.typ = call[paramPos].typ
+
+      letSection.add newTreeI(nkIdentDefs, letSection.info,
+        newSymNode(hoistedVarSym),
+        newNodeI(nkEmpty, letSection.info),
+        call[paramPos])
+
+      call[paramPos] = newSymNode(hoistedVarSym) # Refer the original arg to its hoisted sym
+
+    # arg we refer to is a sym, wether introduced by hoisting or not doesn't matter, we simply reuse it
+    defExpr = call[paramPos]
+  else:
+    for i in 0..<defExpr.safeLen:
+      hoistParamsUsedInDefault(c, call, letSection, defExpr[i])
+
 proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
   when defined(nimCompilerStackraceHints):
     setFrameMsg c.config$n.info & " " & $n.kind
@@ -2690,6 +2752,15 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
       result = semDirectOp(c, n, flags)
     else:
       result = semIndirectOp(c, n, flags)
+
+    if nfDefaultRefsParam in result.flags:
+      result = result.copyTree #XXX: Figure out what causes default param nodes to be shared.. (sigmatch bug?)
+      # We've found a default value that references another param.
+      # See the notes in `hoistParamsUsedInDefault` for more details.
+      var hoistedParams = newNodeI(nkLetSection, result.info)
+      for i in 1..<result.len:
+        hoistParamsUsedInDefault(c, result, hoistedParams, result[i])
+      result = newTreeIT(nkStmtListExpr, result.info, result.typ, hoistedParams, result)
   of nkWhen:
     if efWantStmt in flags:
       result = semWhen(c, n, true)