Remove the use of usrToCell in gcMark [backport:1.2] (#17709)

* Remove the use of usrToCell in gcMark [backport:1.2]

Recently, we've discovered a GC crash resulting from inlining of
the memory allocation procs that allowed the compiler to avoid
maintaining any references to the "user pointer" on the stack.
Instead, a "cell pointer" appeared there and all field accesses
were performed with adjusted offsets. This interfered with the
ability of the GC to mark the correct cell in the conservative
stack scans which lead to premature collection of objects.

More details here:
af69b3ceae

This commit closes another theoretical loophole that may lead to
the same problem. If a short proc is accessing both the object and
its reference count in a short sequence of instructions, the compiler
may be enticed to reduce the number of registers being used by storing
only a single pointer to the object and using offsets when reading
and writing fields. A perfectly good strategy would be to store only
the cell pointer, so the reference count updates can be performed
without applying offsets. Accessing the fields of the object requires
offsets anyway, but these can be adjusted at compile-time without any
loss. Following this strategy will lead to the same problem of marking
a wrong cell during the conservative stack scan, leading to premature
collection.

The problem is avoided by not using `usrToCell` in `gcMark`. Since
the cell discovery logic can already handle interior pointers, the
user pointers don't need to be adjusted for the GC to function correctly.

(cherry picked from commit 3b47a689cf)

.builds/freebsd.yml

@@ -0,0 +1,31 @@
+# see https://man.sr.ht/builds.sr.ht/compatibility.md#freebsd
+image: freebsd/latest
+packages:
+- databases/sqlite3
+- devel/boehm-gc-threaded
+- devel/pcre
+- devel/sdl20
+- devel/sfml
+- www/node
+- devel/gmake
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    gmake -C csources -j $(sysctl -n hw.ncpu)
+    bin/nim c --skipUserCfg --skipParentCfg koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_0.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "0_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_1.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "1_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.builds/openbsd_2.yml

@@ -0,0 +1,34 @@
+## do not edit directly; auto-generated by `nim r tools/ci_generate.nim`
+
+image: openbsd/latest
+packages:
+- gmake
+- sqlite3
+- node
+- boehm-gc
+- pcre
+- sfml
+- sdl2
+- libffi
+sources:
+- https://github.com/nim-lang/Nim
+environment:
+  NIM_TESTAMENT_BATCH: "2_3"
+  CC: /usr/bin/clang
+tasks:
+- setup: |
+    cd Nim
+    git clone --depth 1 -q https://github.com/nim-lang/csources.git
+    gmake -C csources -j $(sysctl -n hw.ncpuonline)
+    bin/nim c koch
+    echo 'export PATH=$HOME/Nim/bin:$PATH' >> $HOME/.buildenv
+- test: |
+    cd Nim
+    if ! ./koch runCI; then
+      nim c -r tools/ci_testresults.nim
+      exit 1
+    fi
+triggers:
+- action: email
+  condition: failure
+  to: Andreas Rumpf <rumpf_a@web.de>

.gitattributes

@@ -0,0 +1,6 @@
+# Avoids changelog conflicts by assuming additions-only, which is by far the common case.
+# In the rare case where branch b1 rebases against branch b2 and both branches
+# modified the same changelog entry, you'll end up with that changelog entry
+# duplicated, which is easily identifiable and fixable.
+/changelog.md merge=union
+

.github/FUNDING.yml

@@ -1,7 +1,7 @@
 # These are supported funding model platforms
 
 github: # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
-patreon: # Replace with a single Patreon username
+patreon: 'araq' # Replace with a single Patreon username
 open_collective: 'nim' # Replace with a single Open Collective username
 ko_fi: # Replace with a single Ko-fi username
 tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel

.github/ISSUE_TEMPLATE/bug_report.md

@@ -1,46 +1,51 @@
 ---
 name: Bug report
 about: Have you found an unexpected behavior? Use this template.
+title: Think about the title, twice
+labels: ''
+assignees: ''
+
 ---
 
-<!-- Think about the title, twice. -->
-
-<!-- Summarize the problem here, keep it short and simple. -->
 Function `echo` outputs the wrong string.
 
-
 ### Example
-<!-- Paste your example in the code-block below. -->
 ```nim
 echo "Hello World!"
+# This code should be a minimum reproducible example:
+# try to simplify and minimize as much as possible. If it's a compiler
+# issue, try to minimize further by removing any imports if possible.
 ```
 
-
 ### Current Output
+please check whether the problem still exists in git head before posting,
+see [rebuilding the compiler](https://nim-lang.github.io/Nim/intern.html#rebuilding-the-compiler).
 ```
 Hola mundo!
 ```
 
-
 ### Expected Output
-<!-- What should be the correct output? -->
 ```
 Hello World!
 ```
 
-
 ### Possible Solution
-<!--- Have you found a possible solution? Post it here. -->
 
+* In file xyz there is a call that might be the cause of it.
 
 ### Additional Information
-<!--- For Example:
-* Your Nim version (output of `nim -v`).
-* Was it working in the previous Nim releases?
-* A link to a related issue or discussion.
-* A project reference where (and how) the issue causes problems.
--->
+If it's a regression, you can help us by identifying which version introduced
+the bug, see [Bisecting for regressions](https://nim-lang.github.io/Nim/intern.html#bisecting-for-regressions),
+or at least try known past releases (eg `choosenim 1.2.0`).
+
+If it's a pre-existing compiler bug, see [Debugging the compiler](https://nim-lang.github.io/Nim/intern.html#debugging-the-compiler)
+which should give more context on a compiler crash.
+
+* Issue #abc is related, but different because of ...
+* This issue is blocking my project xyz
+
 ```
 $ nim -v
 Nim Compiler Version 0.1.2
+# make sure to include the git hash if not using a tagged release
 ```

.github/ISSUE_TEMPLATE/feature_request.md

@@ -1,6 +1,9 @@
 ---
 name: Feature request
 about: Do you want to suggest a new feature? Use this template.
+title: ''
+labels: ''
+assignees: ''
 
 ---
 

.github/stale.yml

@@ -0,0 +1,69 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 365
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 30
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - ARC
+  - bounty
+  - Codegen
+  - Crash
+  - Generics
+  - High Priority
+  - Macros
+  - Next release
+  - Showstopper
+  - Static[T]
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: false
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: stale
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This pull request has been automatically marked as stale because it has not had
+  recent activity.
+  If you think it is still a valid PR, please rebase it on the latest devel;
+  otherwise it will be closed. Thank you for your contributions.
+
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 20
+
+# Limit to only `issues` or `pulls`
+only: pulls
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.github/workflows/ci.yml.disabled

@@ -0,0 +1,122 @@
+name: Continous Integration
+on: [push, pull_request]
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64, i386]
+        cpp: ['false', 'true']
+        pkg: ['false', 'true']
+        exclude:
+          - os: ubuntu-18.04
+            cpp: 'true'
+          - os: ubuntu-18.04
+            pkg: 'true'
+          - os: macos-10.15
+            cpu: i386
+          - os: macos-10.15
+            pkg: 'true'
+          - os: windows-2019
+            cpu: i386
+          - os: windows-2019
+            cpp: 'true'
+    name: '${{ matrix.os }} (${{ matrix.cpu }}, cpp: ${{ matrix.cpp }}, pkg: ${{ matrix.pkg }})'
+    runs-on: ${{ matrix.os }}
+    env:
+      NIM_COMPILE_TO_CPP: ${{ matrix.cpp }}
+      NIM_TEST_PACKAGES: ${{ matrix.pkg }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install node.js 8.x'
+        uses: actions/setup-node@v1
+        with:
+          node-version: '8.x'
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -yq \
+              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
+              valgrind libc6-dbg
+      - name: 'Install dependencies (Linux i386)'
+        if: runner.os == 'Linux' && matrix.cpu == 'i386'
+        run: |
+          sudo dpkg --add-architecture i386
+
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+              g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
+              libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 \
+              libffi-dev:i386
+
+          cat << EOF > bin/gcc
+          #!/bin/bash
+
+          exec $(which gcc) -m32 "\$@"
+          EOF
+          cat << EOF > bin/g++
+          #!/bin/bash
+
+          exec $(which g++) -m32 "\$@"
+          EOF
+
+          chmod 755 bin/gcc
+          chmod 755 bin/g++
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install boehmgc make sfml
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+      - name: 'Run CI'
+        shell: bash
+        run: ./koch runCI
+
+      - name: 'Show failed tests'
+        if: failure()
+        shell: bash
+        run: nim c -r tools/ci_testresults.nim

.github/workflows/ci_docs.yml

@@ -0,0 +1,145 @@
+name: Nim Docs CI
+on:
+  push:
+    paths:
+      - 'compiler/docgen.nim'
+      - 'compiler/renderverbatim.nim'
+      - 'config/nimdoc.cfg'
+      - 'doc/**.rst'
+      - 'doc/nimdoc.css'
+      - 'lib/**.nim'
+      - 'nimdoc/testproject/expected/testproject.html'
+      - 'tools/dochack/dochack.nim'
+      - 'tools/kochdocs.nim'
+      - '.github/workflows/ci_docs.yml'
+      - 'koch.nim'
+  pull_request:
+    # Run only on changes on these files.
+    paths:
+      - 'compiler/docgen.nim'
+      - 'compiler/renderverbatim.nim'
+      - 'config/nimdoc.cfg'
+      - 'doc/**.rst'
+      - 'doc/nimdoc.css'
+      - 'lib/**.nim'
+      - 'nimdoc/testproject/expected/testproject.html'
+      - 'tools/dochack/dochack.nim'
+      - 'tools/kochdocs.nim'
+      - '.github/workflows/ci_docs.yml'
+
+
+jobs:
+  build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
+    strategy:
+      fail-fast: false
+      matrix:
+        target: [linux, windows, osx]
+        include:
+          - target: linux
+            os: ubuntu-18.04
+          - target: windows
+            os: windows-2019
+          - target: osx
+            os: macos-10.15
+
+    name: ${{ matrix.target }}
+    runs-on: ${{ matrix.os }}
+
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+
+      - name: 'Install build dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install make
+
+      - name: 'Install build dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Get current csources version'
+        id: csources-version
+        shell: bash
+        run: |
+          sha=$(git ls-remote https://github.com/nim-lang/csources master | cut -f 1)
+          echo "::set-output name=sha::$sha"
+
+      - name: 'Get prebuilt csources from cache'
+        id: csources-cache
+        uses: actions/cache@v1
+        with:
+          path: bin
+          key: '${{ matrix.os }}-${{ steps.csources-version.outputs.sha }}'
+
+      - name: 'Checkout csources'
+        if: steps.csources-cache.outputs.cache-hit != 'true'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Build 1-stage compiler from csources'
+        shell: bash
+        run: |
+          ext=
+          [[ '${{ runner.os }}' == 'Windows' ]] && ext=.exe
+          if [[ ! -x bin/nim-csources$ext ]]; then
+            ncpu=
+            case '${{ runner.os }}' in
+            'Linux')
+              ncpu=$(nproc)
+              ;;
+            'macOS')
+              ncpu=$(sysctl -n hw.ncpu)
+              ;;
+            'Windows')
+              ncpu=$NUMBER_OF_PROCESSORS
+              ;;
+            esac
+            [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+            make -C csources -j $ncpu CC=gcc
+            cp bin/nim{,-csources}$ext
+          else
+            echo 'Cache hit, using prebuilt csources'
+            cp bin/nim{-csources,}$ext
+          fi
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build the real compiler'
+        shell: bash
+        run: ./koch boot -d:release
+
+      - name: 'Clone fusion'
+        shell: bash
+        run: ./koch fusion
+
+      - name: 'Build documentation'
+        shell: bash
+        run: ./koch doc --git.commit:devel
+
+      - name: 'Publish documentation to Github Pages'
+        if: |
+          github.event_name == 'push' && github.ref == 'refs/heads/devel' &&
+          matrix.target == 'linux'
+        uses: crazy-max/ghaction-github-pages@v1
+        with:
+          build_dir: doc/html
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci_packages.yml

@@ -0,0 +1,85 @@
+name: Packages CI
+on: [push, pull_request]
+
+jobs:
+  build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-10.15]
+        cpu: [amd64]
+        pkg: [1, 2]
+    name: '${{ matrix.os }} (pkg: ${{ matrix.pkg }})'
+    runs-on: ${{ matrix.os }}
+    env:
+      NIM_TEST_PACKAGES: ${{ matrix.pkg }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install node.js 12.x'
+        uses: actions/setup-node@v1
+        with:
+          node-version: '12.x'
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -yq \
+              libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev \
+              valgrind libc6-dbg libblas-dev xorg-dev
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install boehmgc make sfml gtk+3
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+      - name: 'Run CI'
+        shell: bash
+        run: ./koch runCI
+
+      - name: 'Show failed tests'
+        if: failure()
+        shell: bash
+        run: nim c -r tools/ci_testresults.nim

.github/workflows/ci_ssl.yml

@@ -0,0 +1,93 @@
+name: Nim SSL CI
+on:
+  pull_request:
+    # Run only on changes on related files
+    paths:
+      - 'lib/pure/httpclient.nim'
+      - 'lib/pure/net.nim'
+      - 'lib/pure/ssl_certs.nim'
+      - 'lib/wrappers/openssl.nim'
+      - 'tests/stdlib/thttpclient_ssl*'
+      - 'tests/untestable/thttpclient_ssl*'
+
+jobs:
+  build:
+    if: |
+      !contains(format('{0} {1}', github.event.head_commit.message, github.event.pull_request.title), '[skip ci]')
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-18.04, macos-10.15, windows-2019]
+        cpu: [amd64]
+    name: '${{ matrix.os }} (${{ matrix.cpu }})'
+    runs-on: ${{ matrix.os }}
+    steps:
+      - name: 'Checkout'
+        uses: actions/checkout@v2
+
+      - name: 'Checkout csources'
+        uses: actions/checkout@v2
+        with:
+          repository: nim-lang/csources
+          path: csources
+
+      - name: 'Install dependencies (Linux amd64)'
+        if: runner.os == 'Linux' && matrix.cpu == 'amd64'
+        run: |
+          sudo apt-fast update -qq
+          DEBIAN_FRONTEND='noninteractive' \
+            sudo apt-fast install --no-install-recommends -y libssl1.1
+      - name: 'Install dependencies (macOS)'
+        if: runner.os == 'macOS'
+        run: brew install make
+      - name: 'Install dependencies (Windows)'
+        if: runner.os == 'Windows'
+        shell: bash
+        run: |
+          mkdir dist
+          curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+          curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+          7z x dist/mingw64.7z -odist
+          7z x dist/dlls.zip -obin
+          echo "${{ github.workspace }}/dist/mingw64/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Add build binaries to PATH'
+        shell: bash
+        run: echo "${{ github.workspace }}/bin" >> "${GITHUB_PATH}"
+
+      - name: 'Build 1-stage compiler from csources'
+        shell: bash
+        run: |
+          ncpu=
+          case '${{ runner.os }}' in
+          'Linux')
+            ncpu=$(nproc)
+            ;;
+          'macOS')
+            ncpu=$(sysctl -n hw.ncpu)
+            ;;
+          'Windows')
+            ncpu=$NUMBER_OF_PROCESSORS
+            ;;
+          esac
+          [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+          make -C csources -j $ncpu CC=gcc ucpu='${{ matrix.cpu }}'
+
+      - name: 'Build koch'
+        shell: bash
+        run: nim c koch
+
+      - name: 'Build the real compiler'
+        shell: bash
+        run: ./koch boot
+
+      - name: 'Run SSL nimDisableCertificateValidation integration tests'
+        shell: bash
+        run: nim c -d:nimDisableCertificateValidation -d:ssl -r -p:. tests/untestable/thttpclient_ssl_disabled.nim
+
+      - name: 'Run SSL certificate check integration tests'
+        # Not supported on Windows due to old openssl version
+        if: runner.os != 'Windows'
+        shell: bash
+        run: nim c -d:ssl -p:. --threads:on -r tests/untestable/thttpclient_ssl.nim

.gitignore

@@ -65,7 +65,8 @@ testament.db
 /tests/**/*.json
 /tests/**/*.js
 /csources
-dist/
+/dist/
+/lib/fusion
 
 # Private directories and files (IDEs)
 .*/
@@ -82,5 +83,20 @@ megatest.nim
 
 /outputExpected.txt
 /outputGotten.txt
+/t15148.txt
+/tests/vm/tfile_rw.txt
 
 /lib/pure/*.js
+
+!/.builds/
+
+!/.github
+
+# ignore debug dirs generated by dsymutil on OSX
+*.dSYM
+
+# for `nim c -r nimdoc/tester` etc; this can be in multiple places
+htmldocs
+
+## these are not needed anymore unless checkout old older versions
+nimdoc.out.css

.travis.yml

@@ -4,45 +4,12 @@ language: c
 dist: xenial
 
 matrix:
-  fast_finish: true
-
   include:
     - os: linux
       env:
         - NIM_COMPILE_TO_CPP=false
         - CPU=amd64
 
-    - os: linux
-      env:
-        - NIM_COMPILE_TO_CPP=false
-        - CPU=i386
-      addons:
-        apt:
-          packages:
-            - gcc-multilib
-            - g++-multilib
-            - libcurl4-openssl-dev:i386
-            - libgc-dev:i386
-            - libglib2.0-dev:i386
-            - libpulse-dev:i386
-            - libsdl1.2-dev:i386
-            - libsfml-dev:i386
-
-    - os: osx
-      env:
-        - NIM_COMPILE_TO_CPP=false
-        - CPU=amd64
-
-    - os: osx
-      env:
-        - NIM_COMPILE_TO_CPP=true
-        - CPU=amd64
-
-# To allow failures for a failing configuration, use something like:
-#  allow_failures:
-#    - env: NIM_COMPILE_TO_CPP=true
-#    - os: osx
-
 addons:
   apt:
     # update the list above if more deps are introduced
@@ -51,18 +18,8 @@ addons:
     - libsdl1.2-dev
     - libgc-dev
     - libsfml-dev
-  homebrew:
-    packages:
-    - boehmgc
-    - make
-    - sfml
-    update: true
-
-install:
-  - if [[ $CPU = i386 ]]; then echo -e "#!/bin/bash\n$(which gcc) -m32 \"\$@\"" > bin/gcc; fi
-  - if [[ $CPU = i386 ]]; then chmod 755 bin/gcc; fi
-  - if [[ $CPU = i386 ]]; then echo -e "#!/bin/bash\n$(which g++) -m32 \"\$@\"" > bin/g++; fi
-  - if [[ $CPU = i386 ]]; then chmod 755 bin/g++; fi
+    - libc6-dbg
+    - valgrind
 
 before_script:
   - git clone --depth 1 https://github.com/nim-lang/csources.git
@@ -73,11 +30,17 @@ script:
   - echo "travis_fold:start:nim_c_koch"
   - nim c koch
   - echo "travis_fold:end:nim_c_koch"
-  - ./koch runCI
+  - echo "travis_fold:start:koch_boot"
+  - ./koch boot
+  - echo "travis_fold:end:koch_boot"
+  - echo "travis_fold:start:koch_doc"
+  - ./koch doc
+  - echo "travis_fold:end:koch_doc"
 
 before_deploy:
   # Make https://nim-lang.github.io/Nim work the same as https://nim-lang.github.io/Nim/overview.html
   - cp -f ./doc/html/overview.html ./doc/html/index.html
+
 deploy:                         # https://nim-lang.github.io/Nim
   provider: pages
   # local-dir *has* to be a relative path from the repo root.
@@ -87,6 +50,3 @@ deploy:                         # https://nim-lang.github.io/Nim
   keep-history: false
   on:
     branch: devel
-
-# Extract failed tests
-after_failure: nim c -r tools/ci_testresults.nim

appveyor.yml

@@ -4,8 +4,8 @@ environment:
   DLLS_URL: https://nim-lang.org/download/dlls.zip
   DLLS_ARCHIVE: dlls.zip
   MINGW_DIR: mingw64
-  MINGW_URL: https://nim-lang.org/download/mingw64-6.3.0.7z
-  MINGW_ARCHIVE: mingw64-6.3.0.7z
+  MINGW_URL: https://nim-lang.org/download/mingw64.7z
+  MINGW_ARCHIVE: mingw64.7z
 
   matrix:
     - NIM_TEST_PACKAGES: false
@@ -15,10 +15,6 @@ cache:
     - '%MINGW_ARCHIVE%'
     - '%DLLS_ARCHIVE%'
 
-matrix:
-  #allow_failures:
-  #  - NIM_TEST_PACKAGES: true
-  fast_finish: true
 
 install:
   - ps: Install-Product node 8 # node 8 or later is required to test js async stuff
@@ -33,6 +29,8 @@ install:
   - cd ..
 
 build_script:
+  - openssl version
+  - openssl version -d
   - bin\nim c koch
   - koch runCI
 

azure-pipelines.yml

@@ -0,0 +1,180 @@
+trigger:
+  branches:
+    include:
+    - '*'
+pr:
+  branches:
+    include:
+    - '*'
+
+jobs:
+- job: packages
+
+  timeoutInMinutes: 90 # default `60` led to lots of cancelled jobs; use 0 for unlimited (may be undesirable)
+
+  strategy:
+    matrix:
+      Linux_amd64:
+        vmImage: 'ubuntu-16.04'
+        CPU: amd64
+      Linux_i386:
+        vmImage: 'ubuntu-16.04'
+        CPU: i386
+      OSX_amd64:
+        vmImage: 'macOS-10.15'
+        CPU: amd64
+      OSX_amd64_cpp:
+        vmImage: 'macOS-10.15'
+        CPU: amd64
+        NIM_COMPILE_TO_CPP: true
+      Windows_amd64_batch0_3:
+        vmImage: 'windows-2019'
+        CPU: amd64
+        # see also: `NIM_TEST_PACKAGES`
+        NIM_TESTAMENT_BATCH: "0_3"
+      Windows_amd64_batch1_3:
+        vmImage: 'windows-2019'
+        CPU: amd64
+        NIM_TESTAMENT_BATCH: "1_3"
+      Windows_amd64_batch2_3:
+        vmImage: 'windows-2019'
+        CPU: amd64
+        NIM_TESTAMENT_BATCH: "2_3"
+
+  pool:
+    vmImage: $(vmImage)
+
+  workspace:
+    clean: all
+
+  steps:
+    - bash: git config --global core.autocrlf false
+      displayName: 'Disable auto conversion to CRLF by git (Windows-only)'
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+    - checkout: self
+      fetchDepth: 1
+
+    - bash: git clone --depth 1 https://github.com/nim-lang/csources
+      displayName: 'Checkout Nim csources'
+
+    - task: NodeTool@0
+      inputs:
+        versionSpec: '12.x'
+      displayName: 'Install node.js 12.x'
+
+    - bash: |
+        sudo apt-fast update -qq
+        DEBIAN_FRONTEND='noninteractive' \
+          sudo apt-fast install --no-install-recommends -yq \
+            libcurl4-openssl-dev libgc-dev libsdl1.2-dev libsfml-dev valgrind libc6-dbg
+      displayName: 'Install dependencies (amd64 Linux)'
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'amd64'))
+
+    - bash: |
+        sudo dpkg --add-architecture i386
+
+        # Downgrade llvm:
+        # - llvm has to be downgraded to have 32bit version installed for sfml.
+        cat << EOF | sudo tee /etc/apt/preferences.d/pin-to-rel
+        Package: libllvm6.0
+        Pin: origin "azure.archive.ubuntu.com"
+        Pin-Priority: 1001
+        EOF
+
+        sudo apt-fast update -qq
+        # `:i386` (e.g. in `libffi-dev:i386`) is needed otherwise you may get:
+        # `could not load: libffi.so` during dynamic loading.
+        DEBIAN_FRONTEND='noninteractive' \
+          sudo apt-fast install --no-install-recommends --allow-downgrades -yq \
+            g++-multilib gcc-multilib libcurl4-openssl-dev:i386 libgc-dev:i386 \
+            libsdl1.2-dev:i386 libsfml-dev:i386 libglib2.0-dev:i386 libffi-dev:i386
+
+        cat << EOF > bin/gcc
+        #!/bin/bash
+
+        exec $(which gcc) -m32 "\$@"
+        EOF
+        cat << EOF > bin/g++
+        #!/bin/bash
+
+        exec $(which g++) -m32 "\$@"
+        EOF
+
+        chmod 755 bin/gcc
+        chmod 755 bin/g++
+
+      displayName: 'Install dependencies (i386 Linux)'
+      condition: and(eq(variables['Agent.OS'], 'Linux'), eq(variables['CPU'], 'i386'))
+
+    - bash: brew install boehmgc make sfml
+      displayName: 'Install dependencies (OSX)'
+      condition: eq(variables['Agent.OS'], 'Darwin')
+
+    - bash: |
+        mkdir dist
+        curl -L https://nim-lang.org/download/mingw64.7z -o dist/mingw64.7z
+        curl -L https://nim-lang.org/download/dlls.zip -o dist/dlls.zip
+        7z x dist/mingw64.7z -odist
+        7z x dist/dlls.zip -obin
+        echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/dist/mingw64/bin'
+
+      displayName: 'Install dependencies (Windows)'
+      condition: eq(variables['Agent.OS'], 'Windows_NT')
+
+    - bash: echo '##vso[task.prependpath]$(System.DefaultWorkingDirectory)/bin'
+      displayName: 'Add build binaries to PATH'
+
+    - bash: |
+        echo 'PATH:' "$PATH"
+        echo '##[section]gcc version'
+        gcc -v
+        echo '##[section]nodejs version'
+        node -v
+        echo '##[section]make version'
+        make -v
+      displayName: 'System information'
+
+    - bash: echo '##vso[task.setvariable variable=csources_version]'"$(git -C csources rev-parse HEAD)"
+      displayName: 'Get csources version'
+
+    - task: Cache@2
+      inputs:
+        key: 'csources | "$(Agent.OS)" | $(CPU) | $(csources_version)'
+        path: csources/bin
+      displayName: 'Restore built csources'
+
+    - bash: |
+        ncpu=
+        ext=
+        case '$(Agent.OS)' in
+        'Linux')
+          ncpu=$(nproc)
+          ;;
+        'Darwin')
+          ncpu=$(sysctl -n hw.ncpu)
+          ;;
+        'Windows_NT')
+          ncpu=$NUMBER_OF_PROCESSORS
+          ext=.exe
+          ;;
+        esac
+        [[ -z "$ncpu" || $ncpu -le 0 ]] && ncpu=1
+
+        if [[ -x csources/bin/nim$ext ]]; then
+          echo "Found cached compiler, skipping build"
+        else
+          make -C csources -j $ncpu CC=gcc ucpu=$(CPU) koch=no
+        fi
+
+        cp csources/bin/nim$ext bin
+      displayName: 'Build 1-stage compiler from csources'
+
+    - bash: nim c koch
+      displayName: 'Build koch'
+
+      # set result to omit the "bash exited with error code '1'" message
+    - bash: ./koch runCI || echo '##vso[task.complete result=Failed]'
+      displayName: 'Run CI'
+      env:
+        SYSTEM_ACCESSTOKEN: $(System.AccessToken)

bin/nim-gdb

@@ -5,9 +5,15 @@ set -e
 
 which nim > /dev/null || (echo "nim not in PATH"; exit 1)
 which gdb > /dev/null || (echo "gdb not in PATH"; exit 1)
+which readlink > /dev/null || (echo "readlink not in PATH."; exit 1)
+
+if [[ "$(uname -s)" == "Darwin" || "(uname -s)" == *"BSD" ]]; then
+  NIM_SYSROOT=$(dirname $(dirname $(readlink -f $(which nim))))
+else
+  NIM_SYSROOT=$(dirname $(dirname $(readlink -e $(which nim))))
+fi
 
 # Find out where the pretty printer Python module is
-NIM_SYSROOT=$(dirname $(dirname $(readlink -e $(which nim))))
 GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"
 
 # Run GDB with the additional arguments that load the pretty printers
@@ -16,6 +22,4 @@ GDB_PYTHON_MODULE_PATH="$NIM_SYSROOT/tools/nim-gdb.py"
 NIM_GDB="${NIM_GDB:-gdb}"
 # exec replaces the new process of bash with gdb. It is always good to
 # have fewer processes.
-exec ${NIM_GDB} \
-  -eval-command "source $GDB_PYTHON_MODULE_PATH" \
-  "$@"
+exec "${NIM_GDB}" -eval-command="source $GDB_PYTHON_MODULE_PATH" "$@"

bin/nim-gdb.bash

@@ -0,0 +1 @@
+nim-gdb

bin/nim-gdb.bat

@@ -0,0 +1,14 @@
+@echo off
+for %%i in (nim.exe) do (set NIM_BIN=%%~dp$PATH:i)
+
+for %%i in ("%NIM_BIN%\..\") do (set NIM_ROOT=%%~fi)
+
+set @GDB_PYTHON_MODULE_PATH=%NIM_ROOT%\tools\nim-gdb.py
+set @NIM_GDB=gdb.exe
+
+@echo source %@GDB_PYTHON_MODULE_PATH%> wingdbcommand.txt
+%@NIM_GDB% --command="wingdbcommand.txt" %*
+del wingdbcommand.txt /f /q
+
+EXIT /B %ERRORLEVEL%
+@echo on

build_all.bat

@@ -0,0 +1,17 @@
+@echo off
+rem build development version of the compiler; can be rerun safely
+if not exist csources (
+  git clone --depth 1 https://github.com/nim-lang/csources.git
+)
+if not exist bin\nim.exe (
+  cd csources
+  if PROCESSOR_ARCHITECTURE == AMD64 (
+    SET ARCH=64
+  )
+  CALL build.bat
+  cd ..
+)
+bin\nim.exe c --skipUserCfg --skipParentCfg koch
+koch.exe boot -d:release --skipUserCfg --skipParentCfg
+koch.exe tools --skipUserCfg --skipParentCfg
+

build_all.sh

@@ -1,23 +1,41 @@
 #! /bin/sh
 
-# build development version of the compiler; can be rerun safely
+# build development version of the compiler; can be rerun safely.
+# arguments can be passed, e.g. `--os freebsd`
 
 set -u # error on undefined variables
 set -e # exit on first error
 
 echo_run(){
-  printf "\n$*\n"
+  echo "$*"
   "$@"
 }
 
-[ -d csources ] || echo_run git clone --depth 1 https://github.com/nim-lang/csources.git
+[ -d csources ] || echo_run git clone -q --depth 1 https://github.com/nim-lang/csources.git
 
 nim_csources=bin/nim_csources
+
+build_nim_csources_via_script(){
+  echo_run cd csources
+  echo_run sh build.sh "$@"
+}
+
 build_nim_csources(){
-  ## avoid changing dir in case of failure
+  # avoid changing dir in case of failure
   (
-    echo_run cd csources
-    echo_run sh build.sh $@
+    if [ $# -ne 0 ]; then
+      # some args were passed (e.g.: `--cpu i386`), need to call build.sh
+      build_nim_csources_via_script "$@"
+    else
+      # no args, use multiple Make jobs (5X faster on 16 cores: 10s instead of 50s)
+      makeX=make
+      unamestr=$(uname)
+      if [ "$unamestr" = 'FreeBSD' ]; then
+        makeX=gmake
+      fi
+      nCPU=$(nproc 2>/dev/null || sysctl -n hw.logicalcpu 2>/dev/null || getconf _NPROCESSORS_ONLN 2>/dev/null || 1)
+      which $makeX && echo_run $makeX -C csources -j $((nCPU + 2)) -l $nCPU || build_nim_csources_via_script
+    fi
   )
   # keep $nim_csources in case needed to investigate bootstrap issues
   # without having to rebuild from csources
@@ -29,5 +47,6 @@ build_nim_csources(){
 # Note: if fails, may need to `cd csources && git pull`
 echo_run bin/nim c --skipUserCfg --skipParentCfg koch
 
-echo_run ./koch boot -d:release
-echo_run ./koch tools # Compile Nimble and other tools.
+echo_run ./koch boot -d:release --skipUserCfg --skipParentCfg
+echo_run ./koch tools --skipUserCfg --skipParentCfg # Compile Nimble and other tools.
+

changelog.md

@@ -1,37 +1,50 @@
-# v1.1 - XXXX-XX-XX
+# v1.6.x - yyyy-mm-dd
 
 
-## Changes affecting backwards compatibility
+
+## Standard library additions and changes
+
+- On Windows the SSL library now checks for valid certificates.
+  It uses the `cacert.pem` file for this purpose which was extracted
+  from `https://curl.se/ca/cacert.pem`. Besides
+  the OpenSSL DLLs (e.g. libssl-1_1-x64.dll, libcrypto-1_1-x64.dll) you
+  now also need to ship `cacert.pem` with your `.exe` file.
 
 
-### Breaking changes in the standard library
+- Make `{.requiresInit.}` pragma to work for `distinct` types.
 
+- Added a macros `enumLen` for returning the number of items in an enum to the
+  `typetraits.nim` module.
 
-### Breaking changes in the compiler
+- `prelude` now works with the JavaScript target.
 
+- Added `ioutils` module containing `duplicate` and `duplicateTo` to duplicate `FileHandle` using C function `dup` and `dup2`.
 
-## Library additions
+- The JSON module can now handle integer literals and floating point literals of arbitrary length and precision.
+  Numbers that do not fit the underlying `BiggestInt` or `BiggestFloat` fields are kept as string literals and
+  one can use external BigNum libraries to handle these. The `parseFloat` family of functions also has now optional
+  `rawIntegers` and `rawFloats` parameters that can be used to enforce that all integer or float literals remain
+  in the "raw" string form so that client code can easily treat small and large numbers uniformly.
 
+- Added `randState` template that exposes the default random number generator. Useful for library authors.
 
-## Library changes
-
-- Added `os.delEnv` and `nimscript.delEnv`. (#11466)
-
-- Enable Oid usage in hashtables. (#11472)
-
-#
-# Language additions
+- Added `asyncdispatch.activeDescriptors` that returns the number of currently
+  active async event handles/file descriptors
 
 
 ## Language changes
 
+- `nimscript` now handles `except Exception as e`
 
-### Tool changes
+- The `cstring` doesn't support `[]=` operator in JS backend.
 
 
-### Compiler changes
 
-- VM can now cast integer type arbitrarily. (#11459)
+## Compiler changes
 
 
-## Bugfixes
+
+- Added `unsafeIsolate` and `extract` to `std/isolation`.
+
+## Tool changes
+

changelog_v020.md

@@ -1,31 +0,0 @@
-# v0.20.4 - xxxx-xx-xx
-
-
-## Changes affecting backwards compatibility
-
-
-### Breaking changes in the standard library
-
-
-### Breaking changes in the compiler
-
-
-## Library additions
-
-
-## Library changes
-
-
-## Language additions
-
-
-## Language changes
-
-
-### Tool changes
-
-
-### Compiler changes
-
-
-## Bugfixes

changelogs/changelog_0_18_1.md

@@ -100,7 +100,7 @@
 
 - We changed how array accesses "from backwards" like ``a[^1]`` or ``a[0..^1]`` are
   implemented. These are now implemented purely in ``system.nim`` without compiler
-  support. There is a new "heterogenous" slice type ``system.HSlice`` that takes 2
+  support. There is a new "heterogeneous" slice type ``system.HSlice`` that takes 2
   generic parameters which can be ``BackwardsIndex`` indices. ``BackwardsIndex`` is
   produced by ``system.^``.
   This means if you overload ``[]`` or ``[]=`` you need to ensure they also work
@@ -660,4 +660,4 @@ for i in a..b:
 - Fixed "C++: SIGABRT instead of IndexError for out-of-bounds"
   ([#6512](https://github.com/nim-lang/Nim/issues/6512))
 - Fixed "An uncaught exception in cpp mode doesn't show the exception name/msg"
-  ([#6431](https://github.com/nim-lang/Nim/issues/6431))
+  ([#6431](https://github.com/nim-lang/Nim/issues/6431))

changelogs/changelog_1_0_0.md

@@ -0,0 +1,138 @@
+# v1.0 - 2019-09-23
+
+
+## Changes affecting backwards compatibility
+
+- The switch ``-d:nimBinaryStdFiles`` does not exist anymore. Instead
+  stdin/stdout/stderr are binary files again. This change only affects
+  Windows.
+
+- On Windows console applications the code-page is set at program startup
+  to UTF-8. Use the new switch `-d:nimDontSetUtf8CodePage` to disable this
+  feature.
+
+- The language definition and compiler are now stricter about ``gensym``'ed
+  symbols in hygienic templates. See the section in the
+  [manual](https://nim-lang.org/docs/manual.html#templates-hygiene-in-templates)
+  for further details. Use the compiler switch `--oldgensym:on` for a
+  transition period.
+
+
+### Breaking changes in the standard library
+
+- We removed `unicode.Rune16` without any deprecation period as the name
+  was wrong (see the [RFC](https://github.com/nim-lang/RFCs/issues/151) for details)
+  and we didn't find any usages of it in the wild. If you still need it, add this
+  piece of code to your project:
+```nim
+type
+  Rune16* = distinct int16
+```
+
+- `exportc` now uses C instead of C++ mangling with `nim cpp`, matching behavior
+  of `importc`, see #10578.
+  Use the new `exportcpp` to mangle as C++ when using `nim cpp`.
+
+
+### Breaking changes in the compiler
+
+- A bug allowing `int` to be implicitly converted to range types of smaller size
+  (e.g `range[0'i8..10'i8]`) has been fixed.
+
+
+
+## Library additions
+
+- `encodings.getCurrentEncoding` now distinguishes between the console's
+  encoding and the OS's encoding. This distinction is only meaningful on
+  Windows.
+- Added `system.getOsFileHandle` which is usually more useful
+  than `system.getFileHandle`. This distinction is only meaningful on
+  Windows.
+- Added a `json.parseJsonFragments` iterator that can be used to speedup
+  JSON processing substantially when there are JSON fragments separated
+  by whitespace.
+
+
+
+## Library changes
+
+- Added `os.delEnv` and `nimscript.delEnv`. (#11466)
+
+- Enabled Oid usage in hashtables. (#11472)
+
+- Added `unsafeColumnAt` procs, that return unsafe cstring from InstantRow. (#11647)
+
+- Make public `Sha1Digest` and `Sha1State` types and `newSha1State`,
+  `update` and `finalize` procedures from `sha1` module. (#11694)
+
+- Added the `std/monotimes` module which implements monotonic timestamps.
+
+- Consistent error handling of two `exec` overloads. (#10967)
+
+- Officially the following modules now have an unstable API:
+  - std/varints
+  - core/allocators
+  - core/hotcodereloading
+  - asyncstreams
+  - base64
+  - browsers
+  - collections/rtarrays
+  - collections/sharedlist
+  - collections/sharedtable
+  - concurrency/atomics
+  - concurrency/cpuload
+  - concurrency/threadpool
+  - coro
+  - endians
+  - httpcore
+  - parsesql
+  - pathnorm
+  - reservedmem
+  - typetraits
+
+  Every other stdlib module is API stable with respect to version 1.
+
+
+
+## Language additions
+
+- Inline iterators returning `lent T` types are now supported, similarly to
+  iterators returning `var T`:
+```nim
+iterator myitems[T](x: openarray[T]): lent T
+iterator mypairs[T](x: openarray[T]): tuple[idx: int, val: lent T]
+```
+
+- Added an `importjs` pragma that can now be used instead of `importcpp`
+  and `importc` to import symbols from JavaScript. `importjs` for routines always
+  takes a "pattern" for maximum flexibility.
+
+
+
+## Language changes
+
+- `uint64` is now finally a regular ordinal type. This means `high(uint64)` compiles
+  and yields the correct value.
+
+
+### Tool changes
+
+- The Nim compiler now does not recompile the Nim project via ``nim c -r`` if
+  no dependent Nim file changed. This feature can be overridden by
+  the ``--forceBuild`` command line option.
+- The Nim compiler now warns about unused module imports. You can use a
+  top level ``{.used.}`` pragma in the module that you want to be importable
+  without producing this warning.
+- The "testament" testing tool's name was changed
+  from `tester` to `testament` and is generally available as a tool to run Nim
+  tests automatically.
+
+
+### Compiler changes
+
+- VM can now cast integer type arbitrarily. (#11459)
+
+
+
+## Bugfixes

changelogs/changelog_1_0_2.md

@@ -0,0 +1,56 @@
+# v1.0.2 - 2019-10-23
+
+
+## Bugfixes
+
+* fixes the --verbosity:2 regression
+* Fixed "Fail to compile a file twice under Windows (v1.0 bug)." [#12242](https://github.com/nim-lang/Nim/issues/12242)
+* fix nimpretty removing space before pragma
+* JS: gensym is stricter for 'this'
+* Fixed "VM Assertion Error with newruntime" [#12294](https://github.com/nim-lang/Nim/issues/12294)
+* Fixed "Assertion error when running `nim check` on compiler/nim.nim" [#12281](https://github.com/nim-lang/Nim/issues/12281)
+* Fixed "Compiler crash with empty array and generic instantiation with int as parameter" [#12264](https://github.com/nim-lang/Nim/issues/12264)
+* Fixed "Regression in JS backend codegen "Error: request to generate code for .compileTime proc"" [#12240](https://github.com/nim-lang/Nim/issues/12240)
+* Fix how `relativePath` handle case sensitiviy
+* Fixed "SIGSEGV in compiler when using generic types and seqs" [#12336](https://github.com/nim-lang/Nim/issues/12336)
+* Fixed "[1.0.0] weird interaction between `import os` and casting integer to char on macosx trigger bad codegen" [#12291](https://github.com/nim-lang/Nim/issues/12291)
+* VM: no special casing for big endian machines
+* Fixed "`internal error: environment misses` with a simple template inside one of Jester macros" [#12323](https://github.com/nim-lang/Nim/issues/12323)
+* nimsuggest: fix tcp socket leak
+* nimsuggest: fix tcp socket leak for epc backend
+* Fixed "`writeFile` and `write(f, str)` skip null bytes on Windows" [#12315](https://github.com/nim-lang/Nim/issues/12315)
+* Fixed "Crash in intsets symmetric_difference" [#12366](https://github.com/nim-lang/Nim/issues/12366)
+* Fixed "[regression] VM crash when dealing with var param of a proc result" [#12244](https://github.com/nim-lang/Nim/issues/12244)
+* fixes a koch regression that made 'koch boot --listcmd' not work anymore
+* Fixed "[regression] inconsistent signed int `mod` operator between runtime, compiletime, and semfold" [#12332](https://github.com/nim-lang/Nim/issues/12332)
+* Fixed "Boehm disables interior pointer checking" [#12286](https://github.com/nim-lang/Nim/issues/12286)
+* Fixes semCustomPragma when nkSym
+* Fixed yield in nkCheckedFieldExpr
+* Fixed "`randomize()` from `random` not working on JS" [#12418](https://github.com/nim-lang/Nim/issues/12418)
+* Fixed "Compiler crash with invalid object variant" [#12379](https://github.com/nim-lang/Nim/issues/12379)
+* fix type's case in random.nim
+* Fixed "Update docs with a better way to signal unimplemented methods" [#10804](https://github.com/nim-lang/Nim/issues/10804)
+* Fixed "Nim language manual, push pragma is not explained well" [#10824](https://github.com/nim-lang/Nim/issues/10824)
+* Fixed "[regression] Importing more than one module with same name from different packages produce bad codegen" [#12420](https://github.com/nim-lang/Nim/issues/12420)
+* Namespace unittest enums to avoid name conflicts
+* Fixed "VM checks unsigned integers for overflow." [#12310](https://github.com/nim-lang/Nim/issues/12310)
+* Fixed "line directive is not generated for first line of function definition" [#12426](https://github.com/nim-lang/Nim/issues/12426)
+
+
+
+## Documentation improvements
+
+* threadpool: fix link in docs (#12258)
+* Fix spellings (#12277)
+* fix #12278, don't expose internal PCRE documentation
+* Fixed "Documentation of quitprocs is wrong" [#12279(https://github.com/nim-lang/Nim/issues/12279)
+* Fix typo in docs
+* Fix reference to parseSpec proc in readme
+* [doc/tut1] removed discard discussion in comments
+* Documentation improvements around the db interface 
+* Easier build instructions for windows - just run `build_all.bat`.
+* fix a few dead links and a missing sentence in documentation
+* Macro docs additions
+* Updated the code example in the os module to use better grammar.
+* Mention "lambdas" and `=>` in the manual
+* Better documentation on Garbage Collector

changelogs/changelog_1_2_0.md

@@ -0,0 +1,535 @@
+# v1.2.0 - 2020-04-02
+
+
+## Standard library additions and changes
+- Added overloaded `strformat.fmt` macro that use specified characters as
+  delimiter instead of '{' and '}'.
+- Added new procs in `tables.nim`: `OrderedTable.pop`, `CountTable.del`,
+  `CountTable.pop`, `Table.pop`.
+- Added `strtabs.clear` overload that reuses the existing mode.
+- Added `browsers.osOpen` const alias for the operating system specific *"open"* command.
+- Added `sugar.dup` for turning in-place algorithms like `sort` and `shuffle`
+  into operations that work on a copy of the data and return the mutated copy,
+  like the existing `sorted` does.
+- Added `sugar.collect` that does comprehension for seq/set/table collections.
+- Added `sugar.capture` for capturing some local loop variables when creating a
+  closure. This is an enhanced version of `closureScope`.
+- Added `typetraits.tupleLen` to get number of elements of a tuple/type tuple,
+  and `typetraits.get` to get the ith element of a type tuple.
+- Added `typetraits.genericParams` to return a tuple of generic params from a
+  generic instantiation.
+- `options` now treats `proc` like other pointer types, meaning `nil` proc variables
+  are converted to `None`.
+- Added `os.normalizePathEnd` for additional path sanitization.
+- Added `times.fromUnixFloat,toUnixFloat`, sub-second resolution versions of
+  `fromUnix`,`toUnixFloat`.
+- Added `wrapnils` module for chains of field-access and indexing where the LHS
+  can be nil. This simplifies code by reducing need for if-else branches around
+  intermediate maybe nil values. E.g. `echo ?.n.typ.kind`.
+- Added `minIndex`, `maxIndex` and `unzip` to the `sequtils` module.
+- Added `os.isRelativeTo` to tell whether a path is relative to another.
+- Added `resetOutputFormatters` to `unittest`.
+- Added `expectIdent` to the `macros` module.
+- Added `os.isValidFilename` that returns `true` if `filename` argument is valid
+  for cross-platform use.
+- Added `times.isLeapDay`.
+- `base64` adds URL-Safe Base64, implements RFC-4648 Section-7.
+- Added a new module, `std / compilesettings` for querying the compiler about
+  diverse configuration settings.
+- Added `net.getPeerCertificates` and `asyncnet.getPeerCertificates` for
+  retrieving the verified certificate chain of the peer we are connected to
+  through an SSL-wrapped `Socket`/`AsyncSocket`.
+- Added `browsers.openDefaultBrowser` without URL, implements IETF RFC-6694 Section-3.
+- Added `jsconsole.trace`, `jsconsole.table`, `jsconsole.exception` for JavaScript target.
+- Added `distinctBase` overload for values: `assert 12.MyInt.distinctBase == 12`
+- Added new module `std/stackframes`, in particular `setFrameMsg`, which enables
+  custom runtime annotation of stackframes, see #13351 for examples.
+  Turn on/off via `--stackTraceMsgs:on/off`.
+- Added `sequtils.countIt`, allowing for counting items using a predicate.
+- Added a `with` macro for easy function chaining that's available everywhere,
+  there is no need to concern your APIs with returning the first argument
+  to enable "chaining", instead use the dedicated macro `with` that
+  was designed for it. For example:
+
+```nim
+import std/with
+
+type
+  Foo = object
+    col, pos: string
+
+proc setColor(f: var Foo; r, g, b: int) = f.col = $(r, g, b)
+proc setPosition(f: var Foo; x, y: float) = f.pos = $(x, y)
+
+var f: Foo
+with(f, setColor(2, 3, 4), setPosition(0.0, 1.0))
+echo f
+```
+
+- `macros.newLit` now works for ref object types.
+- `macro pragmas` can now be used in type sections.
+- 5 new pragmas were added to Nim in order to make the upcoming tooling more
+  convenient to use. Nim compiler checks these pragmas for syntax but otherwise
+  ignores them. The pragmas are `requires`, `ensures`, `assume`, `assert`, `invariant`.
+- `system.writeFile` has been overloaded to also support `openarray[byte]`.
+- `asyncdispatch.drain` now properly takes into account `selector.hasPendingOperations`
+  and only returns once all pending async operations are guaranteed to have completed.
+- `sequtils.zip` now returns a sequence of anonymous tuples i.e. those tuples
+  now do not have fields named "a" and "b".
+- `distinctBase` has been moved from `sugar` to `typetraits` and now it is
+  implemented as compiler type trait instead of macro. `distinctBase` in sugar
+  module is now deprecated.
+- `CountTable.mget` has been removed from `tables.nim`. It didn't work, and it
+  was an oversight to be included in v1.0.
+- `tables.merge(CountTable, CountTable): CountTable` has been removed.
+  It didn't work well together with the existing inplace version of the same proc
+  (`tables.merge(var CountTable, CountTable)`).
+  It was an oversight to be included in v1.0.
+- `asyncdispatch.drain` now consistently uses the passed timeout value for all
+  iterations of the event loop, and not just the first iteration.
+  This is more consistent with the other asyncdispatch APIs, and allows
+  `asyncdispatch.drain` to be more efficient.
+- `base64.encode` and `base64.decode` were made faster by about 50%.
+- `htmlgen` adds [MathML](https://wikipedia.org/wiki/MathML) support
+  (ISO 40314).
+- `macros.eqIdent` is now invariant to export markers and backtick quotes.
+- `htmlgen.html` allows `lang` in the `<html>` tag and common valid attributes.
+- `macros.basename` and `basename=` got support for `PragmaExpr`,
+  so that an expression like `MyEnum {.pure.}` is handled correctly.
+- `httpclient.maxredirects` changed from `int` to `Natural`, because negative values
+  serve no purpose whatsoever.
+- `httpclient.newHttpClient` and `httpclient.newAsyncHttpClient` added `headers`
+  argument to set initial HTTP Headers, instead of a hardcoded empty `newHttpHeader()`.
+- `parseutils.parseUntil` has now a different behaviour if the `until` parameter is
+  empty. This was required for intuitive behaviour of the strscans module
+  (see bug #13605).
+- `strutils.formatFloat` with `precision = 0` has the same behavior in all
+  backends, and it is compatible with Python's behavior,
+  e.g. `formatFloat(3.14159, precision = 0)` is now `3`, not `3.`.
+- `times.parse` now only uses input to compute its result, and not `now`:
+  `parse("2020", "YYYY", utc())` is now `2020-01-01T00:00:00Z` instead of
+  `2020-03-02T00:00:00Z` if run on 03-02; it also doesn't crash anymore when
+  used on 29th, 30th, 31st of each month.
+- `httpcore.==(string, HttpCode)` is now deprecated due to lack of practical
+  usage. The `$` operator can be used to obtain the string form of `HttpCode`
+  for comparison if desired.
+- `std/oswalkdir` was buggy, it's now deprecated and reuses `std/os` procs.
+- `os.walkDir` and `os.walkDirRec` now have new flag, `checkDir` (default: false).
+  If it is set to true, it will throw if input dir is invalid instead of a noop
+  (which is the default behaviour, as it was before this change),
+  `os.walkDirRec` only throws if top-level dir is invalid, but ignores errors for
+  subdirs, otherwise it would be impossible to resume iteration.
+- The `FD` variant of `selector.unregister` for `ioselector_epoll` and
+  `ioselector_select` now properly handle the `Event.User` select event type.
+- `joinPath` path normalization when `/` is the first argument works correctly:
+  `assert "/" / "/a" == "/a"`. Fixed the edge case: `assert "" / "" == ""`.
+- `xmltree` now adds indentation consistently to child nodes for any number
+  of children nodes.
+- `os.splitPath()` behavior synchronized with `os.splitFile()` to return "/"
+  as the dir component of `/root_sub_dir` instead of the empty string.
+- The deprecated `lc` macro has been removed from `sugar`. It is now replaced
+  with the more powerful `collect` macro.
+- `os.relativePath("foo", "foo")` is now `"."`, not `""`, as `""` means invalid
+  path and shouldn't be conflated with `"."`; use `-d:nimOldRelativePathBehavior`
+  to restore the old behavior.
+- `os.joinPath(a, b)` now honors trailing slashes in `b` (or `a` if `b` = "").
+- `base64.encode` no longer supports `lineLen` and `newLine`.
+  Use `base64.encodeMime` instead.
+
+
+### Breaking changes
+
+- `net.newContext` now performs SSL Certificate checking on Linux and OSX.
+  Define `nimDisableCertificateValidation` to disable it globally.
+
+
+
+## Language changes
+
+- An `align` pragma can now be used for variables and object fields, similar
+  to the `alignas` declaration modifier in C/C++.
+- The `=sink` type bound operator is now optional. The compiler can now use a
+  combination of `=destroy` and `copyMem` to move objects efficiently.
+- Unsigned integer operators have been fixed to allow promotion of the first operand.
+- Conversions to unsigned integers are unchecked at runtime, imitating earlier Nim
+  versions. The documentation was improved to acknowledge this special case.
+  See https://github.com/nim-lang/RFCs/issues/175 for more details.
+- There is a new syntax for lvalue references: `var b {.byaddr.} = expr` enabled by
+  `import std/decls`.
+- `var a {.foo.}: MyType = expr` now lowers to `foo(a, MyType, expr)` for
+  non-builtin pragmas, enabling things like lvalue references (see `decls.byaddr`).
+
+
+
+## Compiler changes
+
+- The generated JS code uses spaces, instead of mixing spaces and tabs.
+- The Nim compiler now supports the ``--asm`` command option for easier
+  inspection of the produced assembler code.
+- The Nim compiler now supports a new pragma called ``.localPassc`` to
+  pass specific compiler options to the C(++) backend for the C(++) file
+  that was produced from the current Nim module.
+- The compiler now inferes "sink parameters". To disable this for a specific routine,
+  annotate it with `.nosinks`. To disable it for a section of code, use
+  `{.push sinkInference: off.}`...`{.pop.}`.
+- The compiler now supports a new switch `--panics:on` that turns runtime
+  errors like `IndexError` or `OverflowError` into fatal errors that **cannot**
+  be caught via Nim's `try` statement. `--panics:on` can improve the
+  runtime efficiency and code size of your program significantly.
+- The compiler now warns about inheriting directly from `system.Exception` as
+  this is **very bad** style. You should inherit from `ValueError`, `IOError`,
+  `OSError` or from a different specific exception type that inherits from
+  `CatchableError` and cannot be confused with a `Defect`.
+- The error reporting for Nim's effect system has been improved.
+- Implicit conversions for `const` behave correctly now, meaning that code like
+  `const SOMECONST = 0.int; procThatTakesInt32(SOMECONST)` will be illegal now.
+  Simply write `const SOMECONST = 0` instead.
+- The `{.dynlib.}` pragma is now required for exporting symbols when making
+  shared objects on POSIX and macOS, which make it consistent with the behavior
+  on Windows.
+- The compiler is now more strict about type conversions concerning proc
+  types: Type conversions cannot be used to hide `.raise` effects or side
+  effects, instead a `cast` must be used. With the flag `--useVersion:1.0` the
+  old behaviour is emulated.
+- The Nim compiler now implements a faster way to detect overflows based
+  on GCC's `__builtin_sadd_overflow` family of functions. (Clang also
+  supports these). Some versions of GCC lack this feature and unfortunately
+  we cannot detect this case reliably. So if you get compilation errors like
+  "undefined reference to `__builtin_saddll_overflow`" compile your programs
+  with `-d:nimEmulateOverflowChecks`.
+
+
+
+
+## Bugfixes
+
+- Fixed "`nimgrep --nocolor` is ignored on posix; should be instead: `--nimgrep --color=[auto]|true|false`"
+  ([#7591](https://github.com/nim-lang/Nim/issues/7591))
+- Fixed "Runtime index on const array (of converted obj) causes C-compiler error"
+  ([#10514](https://github.com/nim-lang/Nim/issues/10514))
+- Fixed "windows x86 with vcc compile error with "asmNoStackFrame""
+  ([#12298](https://github.com/nim-lang/Nim/issues/12298))
+- Fixed "[TODO] regression: Error: Locks requires --threads:on option"
+  ([#12330](https://github.com/nim-lang/Nim/issues/12330))
+- Fixed "Add --cc option to --help or --fullhelp output"
+  ([#12010](https://github.com/nim-lang/Nim/issues/12010))
+- Fixed "questionable `csize` definition in `system.nim`"
+  ([#12187](https://github.com/nim-lang/Nim/issues/12187))
+- Fixed "os.getAppFilename() returns incorrect results on OpenBSD"
+  ([#12389](https://github.com/nim-lang/Nim/issues/12389))
+- Fixed "HashSet[uint64] slow insertion depending on values"
+  ([#11764](https://github.com/nim-lang/Nim/issues/11764))
+- Fixed "Differences between compiling 'classic call syntax' vs 'method call syntax' ."
+  ([#12453](https://github.com/nim-lang/Nim/issues/12453))
+- Fixed "c -d:nodejs --> SIGSEGV: Illegal storage access"
+  ([#12502](https://github.com/nim-lang/Nim/issues/12502))
+- Fixed "Closure iterator crashes on --newruntime due to "dangling references""
+  ([#12443](https://github.com/nim-lang/Nim/issues/12443))
+- Fixed "No `=destroy` for elements of closure environments other than for latest devel --gc:destructors"
+  ([#12577](https://github.com/nim-lang/Nim/issues/12577))
+- Fixed "strutils:formatBiggestFloat() gives different results in JS mode"
+  ([#8242](https://github.com/nim-lang/Nim/issues/8242))
+- Fixed "Regression (devel): the new `csize_t` definition isn't consistently used, nor tested thoroughly..."
+  ([#12597](https://github.com/nim-lang/Nim/issues/12597))
+- Fixed "tables.take() is defined only for `Table` and missed for other table containers"
+  ([#12519](https://github.com/nim-lang/Nim/issues/12519))
+- Fixed "`pthread_key_t` errors on OpenBSD"
+  ([#12135](https://github.com/nim-lang/Nim/issues/12135))
+- Fixed "newruntime: simple seq pop at ct results in compile error"
+  ([#12644](https://github.com/nim-lang/Nim/issues/12644))
+- Fixed "[Windows] finish.exe C:\Users\<USERNAME>\.nimble\bin is not in your PATH environment variable."
+  ([#12319](https://github.com/nim-lang/Nim/issues/12319))
+- Fixed "Error with strformat + asyncdispatch + const"
+  ([#12612](https://github.com/nim-lang/Nim/issues/12612))
+- Fixed "MultipartData needs $"
+  ([#11863](https://github.com/nim-lang/Nim/issues/11863))
+- Fixed "Nim stdlib style issues with --styleCheck:error"
+  ([#12687](https://github.com/nim-lang/Nim/issues/12687))
+- Fixed "new $nimbleDir path substitution yields unexpected search paths"
+  ([#12767](https://github.com/nim-lang/Nim/issues/12767))
+- Fixed "Regression: inlined procs now get multiple rounds of destructor injection"
+  ([#12766](https://github.com/nim-lang/Nim/issues/12766))
+- Fixed "newruntime: compiler generates defective code"
+  ([#12669](https://github.com/nim-lang/Nim/issues/12669))
+- Fixed "broken windows modules path handling because of 'os.relativePath' breaking changes"
+  ([#12734](https://github.com/nim-lang/Nim/issues/12734))
+- Fixed "for loop tuple syntax not rendered correctly"
+  ([#12740](https://github.com/nim-lang/Nim/issues/12740))
+- Fixed "Crash when trying to use `type.name[0]`"
+  ([#12804](https://github.com/nim-lang/Nim/issues/12804))
+- Fixed "Enums should be considered Trivial types in Atomics"
+  ([#12812](https://github.com/nim-lang/Nim/issues/12812))
+- Fixed "Produce static/const initializations for variables when possible"
+  ([#12216](https://github.com/nim-lang/Nim/issues/12216))
+- Fixed "Assigning descriminator field leads to internal assert with --gc:destructors"
+  ([#12821](https://github.com/nim-lang/Nim/issues/12821))
+- Fixed "nimsuggest `use` command does not return all instances of symbol"
+  ([#12832](https://github.com/nim-lang/Nim/issues/12832))
+- Fixed "@[] is a problem for --gc:destructors"
+  ([#12820](https://github.com/nim-lang/Nim/issues/12820))
+- Fixed "Codegen ICE in allPathsAsgnResult"
+  ([#12827](https://github.com/nim-lang/Nim/issues/12827))
+- Fixed "seq[Object with ref and destructor type] doesn't work in old runtime"
+  ([#12882](https://github.com/nim-lang/Nim/issues/12882))
+- Fixed "Destructor not invoked because it is instantiated too late, old runtime"
+  ([#12883](https://github.com/nim-lang/Nim/issues/12883))
+- Fixed "The collect macro does not handle if/case correctly"
+  ([#12874](https://github.com/nim-lang/Nim/issues/12874))
+- Fixed "allow typed/untyped params in magic procs (even if not in stdlib)"
+  ([#12911](https://github.com/nim-lang/Nim/issues/12911))
+- Fixed "ARC/newruntime memory corruption"
+  ([#12899](https://github.com/nim-lang/Nim/issues/12899))
+- Fixed "tasyncclosestall.nim still flaky test: Address already in use"
+  ([#12919](https://github.com/nim-lang/Nim/issues/12919))
+- Fixed "newruntime and computed goto: variables inside the loop are in generated code uninitialised"
+  ([#12785](https://github.com/nim-lang/Nim/issues/12785))
+- Fixed "osx: dsymutil needs to be called for debug builds to keep debug info"
+  ([#12735](https://github.com/nim-lang/Nim/issues/12735))
+- Fixed "codegen ICE with ref objects, gc:destructors"
+  ([#12826](https://github.com/nim-lang/Nim/issues/12826))
+- Fixed "mutable iterator cannot yield named tuples"
+  ([#12945](https://github.com/nim-lang/Nim/issues/12945))
+- Fixed "parsecfg stores "\r\n" line breaks just as "\n""
+  ([#12970](https://github.com/nim-lang/Nim/issues/12970))
+- Fixed "db_postgres.getValue issues warning when no rows found"
+  ([#12973](https://github.com/nim-lang/Nim/issues/12973))
+- Fixed "ARC: Unpacking tuple with seq causes segfault"
+  ([#12989](https://github.com/nim-lang/Nim/issues/12989))
+- Fixed "ARC/newruntime: strutils.join on seq with only empty strings causes segfault"
+  ([#12965](https://github.com/nim-lang/Nim/issues/12965))
+- Fixed "regression (1.0.4): `{.push exportc.}` wrongly affects generic instantiations, causing codegen errors"
+  ([#12985](https://github.com/nim-lang/Nim/issues/12985))
+- Fixed "cdt, crash with --gc:arc, works fine with default gc"
+  ([#12978](https://github.com/nim-lang/Nim/issues/12978))
+- Fixed "ARC: No indexError thrown on out-of-bound seq access, SIGSEGV instead"
+  ([#12961](https://github.com/nim-lang/Nim/issues/12961))
+- Fixed "ARC/async: Returning in a try-block results in wrong codegen"
+  ([#12956](https://github.com/nim-lang/Nim/issues/12956))
+- Fixed "asm keyword is generating wrong output C code when --cc:tcc"
+  ([#12988](https://github.com/nim-lang/Nim/issues/12988))
+- Fixed "Destructor not invoked"
+  ([#13026](https://github.com/nim-lang/Nim/issues/13026))
+- Fixed "ARC/newruntime: Adding inherited var ref object to seq with base type causes segfault"
+  ([#12964](https://github.com/nim-lang/Nim/issues/12964))
+- Fixed "Style check error with JS compiler target"
+  ([#13032](https://github.com/nim-lang/Nim/issues/13032))
+- Fixed "regression(1.0.4): undeclared identifier: 'readLines'; plus another regression and bug"
+  ([#13013](https://github.com/nim-lang/Nim/issues/13013))
+- Fixed "regression(1.04) `invalid pragma: since` with nim js"
+  ([#12996](https://github.com/nim-lang/Nim/issues/12996))
+- Fixed "Sink to MemMove optimization in injectdestructors"
+  ([#13002](https://github.com/nim-lang/Nim/issues/13002))
+- Fixed "--gc:arc: `catch` doesn't work with exception subclassing"
+  ([#13072](https://github.com/nim-lang/Nim/issues/13072))
+- Fixed "nim c --gc:arc --exceptions:{setjmp,goto} incorrectly handles raise; `nim cpp --gc:arc` is ok"
+  ([#13070](https://github.com/nim-lang/Nim/issues/13070))
+- Fixed "typetraits feature request - get subtype of a generic type"
+  ([#6454](https://github.com/nim-lang/Nim/issues/6454))
+- Fixed "CountTable inconsistencies between keys() and len() after setting value to 0"
+  ([#12813](https://github.com/nim-lang/Nim/issues/12813))
+- Fixed "{.align.} pragma is not applied if there is a generic field"
+  ([#13122](https://github.com/nim-lang/Nim/issues/13122))
+- Fixed "ARC, finalizer, allow rebinding the same function multiple times"
+  ([#13112](https://github.com/nim-lang/Nim/issues/13112))
+- Fixed "`nim doc` treats `export localSymbol` incorrectly"
+  ([#13100](https://github.com/nim-lang/Nim/issues/13100))
+- Fixed "--gc:arc SIGSEGV (double free?)"
+  ([#13119](https://github.com/nim-lang/Nim/issues/13119))
+- Fixed "codegen bug with arc"
+  ([#13105](https://github.com/nim-lang/Nim/issues/13105))
+- Fixed "symbols not defined in the grammar"
+  ([#10665](https://github.com/nim-lang/Nim/issues/10665))
+- Fixed "[JS] Move is not defined"
+  ([#9674](https://github.com/nim-lang/Nim/issues/9674))
+- Fixed "[TODO] pathutils.`/` can return invalid AbsoluteFile"
+  ([#13121](https://github.com/nim-lang/Nim/issues/13121))
+- Fixed "regression(1.04) `nim doc main.nim` generates broken html (no css)"
+  ([#12998](https://github.com/nim-lang/Nim/issues/12998))
+- Fixed "Wrong supportsCopyMem on string in type section"
+  ([#13095](https://github.com/nim-lang/Nim/issues/13095))
+- Fixed "Arc, finalizer, out of memory"
+  ([#13157](https://github.com/nim-lang/Nim/issues/13157))
+- Fixed "`--genscript` messes up nimcache and future nim invocations"
+  ([#13144](https://github.com/nim-lang/Nim/issues/13144))
+- Fixed "--gc:arc with --exceptions:goto for "nim c" generate invalid c code"
+  ([#13186](https://github.com/nim-lang/Nim/issues/13186))
+- Fixed "[regression] duplicate member `_i1` codegen bug"
+  ([#13195](https://github.com/nim-lang/Nim/issues/13195))
+- Fixed "RTree investigations with valgrind for --gc:arc"
+  ([#13110](https://github.com/nim-lang/Nim/issues/13110))
+- Fixed "relativePath("foo", ".") returns wrong path"
+  ([#13211](https://github.com/nim-lang/Nim/issues/13211))
+- Fixed "asyncftpclient - problem with welcome.msg"
+  ([#4684](https://github.com/nim-lang/Nim/issues/4684))
+- Fixed "Unclear error message, lowest priority"
+  ([#13256](https://github.com/nim-lang/Nim/issues/13256))
+- Fixed "Channel messages are corrupted"
+  ([#13219](https://github.com/nim-lang/Nim/issues/13219))
+- Fixed "Codegen bug with exportc and case objects"
+  ([#13281](https://github.com/nim-lang/Nim/issues/13281))
+- Fixed "[bugfix] fix #11590: c compiler warnings silently ignored, giving undefined behavior"
+  ([#11591](https://github.com/nim-lang/Nim/issues/11591))
+- Fixed "[CI] tnetdial flaky test"
+  ([#13132](https://github.com/nim-lang/Nim/issues/13132))
+- Fixed "Cross-Compiling with -d:mingw fails to locate compiler under OSX"
+  ([#10717](https://github.com/nim-lang/Nim/issues/10717))
+- Fixed "`nim doc --project` broken with imports below main project file or duplicate names"
+  ([#13150](https://github.com/nim-lang/Nim/issues/13150))
+- Fixed "regression: isNamedTuple(MyGenericTuple[int]) is false, should be true"
+  ([#13349](https://github.com/nim-lang/Nim/issues/13349))
+- Fixed "--gc:arc codegen bug copying objects bound to C structs with missing C struct fields"
+  ([#13269](https://github.com/nim-lang/Nim/issues/13269))
+- Fixed "write requires conversion to string"
+  ([#13182](https://github.com/nim-lang/Nim/issues/13182))
+- Fixed "Some remarks to stdlib documentation"
+  ([#13352](https://github.com/nim-lang/Nim/issues/13352))
+- Fixed "a `check` in unittest generated by template doesn't show actual value"
+  ([#6736](https://github.com/nim-lang/Nim/issues/6736))
+- Fixed "Implicit return with case expression fails with 'var' return."
+  ([#3339](https://github.com/nim-lang/Nim/issues/3339))
+- Fixed "Segfault with closure on arc"
+  ([#13314](https://github.com/nim-lang/Nim/issues/13314))
+- Fixed "[Macro] Crash on malformed case statement with multiple else"
+  ([#13255](https://github.com/nim-lang/Nim/issues/13255))
+- Fixed "regression: `echo 'discard' | nim c -r -` generates a file '-' ; `-` should be treated specially"
+  ([#13374](https://github.com/nim-lang/Nim/issues/13374))
+- Fixed "on OSX, debugging (w gdb or lldb) a nim program crashes at the 1st call to `execCmdEx`"
+  ([#9634](https://github.com/nim-lang/Nim/issues/9634))
+- Fixed "Internal error in getTypeDescAux"
+  ([#13378](https://github.com/nim-lang/Nim/issues/13378))
+- Fixed "gc:arc mode breaks tuple let"
+  ([#13368](https://github.com/nim-lang/Nim/issues/13368))
+- Fixed "Nim compiler hangs for certain C/C++ compiler errors"
+  ([#8648](https://github.com/nim-lang/Nim/issues/8648))
+- Fixed "htmlgen does not support `data-*` attributes"
+  ([#13444](https://github.com/nim-lang/Nim/issues/13444))
+- Fixed "[gc:arc] setLen will cause string not to be null-terminated."
+  ([#13457](https://github.com/nim-lang/Nim/issues/13457))
+- Fixed "joinPath("", "") is "/" ; should be """
+  ([#13455](https://github.com/nim-lang/Nim/issues/13455))
+- Fixed "[CI] flaky test on windows: tests/osproc/texitcode.nim"
+  ([#13449](https://github.com/nim-lang/Nim/issues/13449))
+- Fixed "Casting to float32 on NimVM is broken"
+  ([#13479](https://github.com/nim-lang/Nim/issues/13479))
+- Fixed "`--hints:off` doesn't work (doesn't override ~/.config/nim.cfg)"
+  ([#8312](https://github.com/nim-lang/Nim/issues/8312))
+- Fixed "joinPath("", "") is "/" ; should be """
+  ([#13455](https://github.com/nim-lang/Nim/issues/13455))
+- Fixed "tables.values is broken"
+  ([#13496](https://github.com/nim-lang/Nim/issues/13496))
+- Fixed "global user config can override project specific config"
+  ([#9405](https://github.com/nim-lang/Nim/issues/9405))
+- Fixed "Non deterministic macros and id consistency problem"
+  ([#12627](https://github.com/nim-lang/Nim/issues/12627))
+- Fixed "try expression doesn't work with return on expect branch"
+  ([#13490](https://github.com/nim-lang/Nim/issues/13490))
+- Fixed "CI will break every 4 years on feb 28: times doesn't handle leap years properly"
+  ([#13543](https://github.com/nim-lang/Nim/issues/13543))
+- Fixed "[minor] `nimgrep --word` doesn't work with operators (eg misses  `1 +% 2`)"
+  ([#13528](https://github.com/nim-lang/Nim/issues/13528))
+- Fixed "`as` is usable as infix operator but its existence and precedence are not documented"
+  ([#13409](https://github.com/nim-lang/Nim/issues/13409))
+- Fixed "JSON unmarshalling drops seq's items"
+  ([#13531](https://github.com/nim-lang/Nim/issues/13531))
+- Fixed "os.joinPath returns wrong path when head ends '\' or '/' and tail starts '..'."
+  ([#13579](https://github.com/nim-lang/Nim/issues/13579))
+- Fixed "Block-local types with the same name lead to bad codegen (sighashes regression)"
+  ([#5170](https://github.com/nim-lang/Nim/issues/5170))
+- Fixed "tuple codegen error"
+  ([#12704](https://github.com/nim-lang/Nim/issues/12704))
+- Fixed "newHttpHeaders does not accept repeated headers"
+  ([#13573](https://github.com/nim-lang/Nim/issues/13573))
+- Fixed "regression: --incremental:on fails on simplest example"
+  ([#13319](https://github.com/nim-lang/Nim/issues/13319))
+- Fixed "strscan can't get value of last element in format"
+  ([#13605](https://github.com/nim-lang/Nim/issues/13605))
+- Fixed "hashes_examples crashes with "Bus Error" (unaligned access) on sparc64"
+  ([#12508](https://github.com/nim-lang/Nim/issues/12508))
+- Fixed "gc:arc bug with re-used `seq[T]`"
+  ([#13596](https://github.com/nim-lang/Nim/issues/13596))
+- Fixed "`raise CatchableError` is broken with --gc:arc  when throwing inside a proc"
+  ([#13599](https://github.com/nim-lang/Nim/issues/13599))
+- Fixed "cpp --gc:arc --exceptions:goto fails to raise with discard"
+  ([#13436](https://github.com/nim-lang/Nim/issues/13436))
+- Fixed "terminal doesn't compile with -d:useWinAnsi"
+  ([#13607](https://github.com/nim-lang/Nim/issues/13607))
+- Fixed "Parsing "sink ptr T" - region needs to be an object type"
+  ([#12757](https://github.com/nim-lang/Nim/issues/12757))
+- Fixed "gc:arc + threads:on + closures compilation error"
+  ([#13519](https://github.com/nim-lang/Nim/issues/13519))
+- Fixed "[ARC] segmentation fault"
+  ([#13240](https://github.com/nim-lang/Nim/issues/13240))
+- Fixed "times.toDateTime buggy on 29th, 30th and 31th of each month"
+  ([#13558](https://github.com/nim-lang/Nim/issues/13558))
+- Fixed "Deque misbehaves on VM"
+  ([#13310](https://github.com/nim-lang/Nim/issues/13310))
+- Fixed "Nimscript listFiles should throw exception when path is not found"
+  ([#12676](https://github.com/nim-lang/Nim/issues/12676))
+- Fixed "koch boot fails if even an empty config.nims is present in ~/.config/nims/ [devel regression]"
+  ([#13633](https://github.com/nim-lang/Nim/issues/13633))
+- Fixed "nim doc generates lots of false positive LockLevel warnings"
+  ([#13218](https://github.com/nim-lang/Nim/issues/13218))
+- Fixed "Arrays are passed by copy to iterators, causing crashes, unnecessary allocations and slowdowns"
+  ([#12747](https://github.com/nim-lang/Nim/issues/12747))
+- Fixed "Range types always uses signed integer as a base type"
+  ([#13646](https://github.com/nim-lang/Nim/issues/13646))
+- Fixed "Generate c code cannot compile with recent devel version"
+  ([#13645](https://github.com/nim-lang/Nim/issues/13645))
+- Fixed "[regression] VM: Error: cannot convert -1 to uint64"
+  ([#13661](https://github.com/nim-lang/Nim/issues/13661))
+- Fixed "Spurious raiseException(Exception) detected"
+  ([#13654](https://github.com/nim-lang/Nim/issues/13654))
+- Fixed "gc:arc memory leak"
+  ([#13659](https://github.com/nim-lang/Nim/issues/13659))
+- Fixed "Error: cannot convert -1 to uint (inside tuples)"
+  ([#13671](https://github.com/nim-lang/Nim/issues/13671))
+- Fixed "strformat issue with --gc:arc"
+  ([#13622](https://github.com/nim-lang/Nim/issues/13622))
+- Fixed "astToStr doesn't work inside generics"
+  ([#13524](https://github.com/nim-lang/Nim/issues/13524))
+- Fixed "oswalkdir.walkDirRec wont return folders"
+  ([#11458](https://github.com/nim-lang/Nim/issues/11458))
+- Fixed "`echo 'echo 1' | nim c -r -`  silently gives wrong results (nimBetterRun not updated for stdin)"
+  ([#13412](https://github.com/nim-lang/Nim/issues/13412))
+- Fixed "gc:arc destroys the global variable accidentally."
+  ([#13691](https://github.com/nim-lang/Nim/issues/13691))
+- Fixed "[minor] sigmatch errors should be sorted, for reproducible errors"
+  ([#13538](https://github.com/nim-lang/Nim/issues/13538))
+- Fixed "Exception when converting csize to clong"
+  ([#13698](https://github.com/nim-lang/Nim/issues/13698))
+- Fixed "ARC: variables are no copied on the thread spawn causing crashes"
+  ([#13708](https://github.com/nim-lang/Nim/issues/13708))
+- Fixed "Illegal distinct seq causes compiler crash"
+  ([#13720](https://github.com/nim-lang/Nim/issues/13720))
+- Fixed "cyclic seq definition crashes the compiler"
+  ([#13715](https://github.com/nim-lang/Nim/issues/13715))
+- Fixed "Iterator with openArray parameter make the argument evaluated many times"
+  ([#13417](https://github.com/nim-lang/Nim/issues/13417))
+- Fixed "net/asyncnet: Unable to access peer's certificate chain"
+  ([#13299](https://github.com/nim-lang/Nim/issues/13299))
+- Fixed "Accidentally "SIGSEGV: Illegal storage access" error after arc optimizations (#13325)"
+  ([#13709](https://github.com/nim-lang/Nim/issues/13709))
+- Fixed "Base64 Regression"
+  ([#13722](https://github.com/nim-lang/Nim/issues/13722))
+- Fixed "A regression (?) with --gc:arc and repr"
+  ([#13731](https://github.com/nim-lang/Nim/issues/13731))
+- Fixed "Internal compiler error when using the new variable pragmas"
+  ([#13737](https://github.com/nim-lang/Nim/issues/13737))
+- Fixed "bool conversion produces vcc 2019 warning at cpp compilation stage"
+  ([#13744](https://github.com/nim-lang/Nim/issues/13744))
+- Fixed "Compiler "does not detect" a type recursion error in the wrong code, remaining frozen"
+  ([#13763](https://github.com/nim-lang/Nim/issues/13763))
+- Fixed "[minor] regression: `Foo[0.0] is Foo[-0.0]` is now false"
+  ([#13730](https://github.com/nim-lang/Nim/issues/13730))
+- Fixed "`nim doc` - only whitespace on first line causes segfault"
+  ([#13631](https://github.com/nim-lang/Nim/issues/13631))
+- Fixed "hashset regression"
+  ([#13794](https://github.com/nim-lang/Nim/issues/13794))
+- Fixed "`os.getApplFreebsd` could return incorrect paths in the case of a long path"
+  ([#13806](https://github.com/nim-lang/Nim/issues/13806))
+- Fixed "Destructors are not inherited"
+  ([#13810](https://github.com/nim-lang/Nim/issues/13810))
+- Fixed "io.readLines AssertionError on devel"
+  ([#13829](https://github.com/nim-lang/Nim/issues/13829))
+- Fixed "exceptions:goto accidentally reset the variable during exception handling"
+  ([#13782](https://github.com/nim-lang/Nim/issues/13782))

changelogs/changelog_1_4_0.md

@@ -0,0 +1,874 @@
+# v1.4.0 - 2020-10-16
+
+
+
+## Standard library additions and changes
+
+- Added some enhancements to `std/jsonutils` module.
+  * Added a possibility to deserialize JSON arrays directly to `HashSet` and
+    `OrderedSet` types and respectively to serialize those types to JSON arrays
+    via `jsonutils.fromJson` and `jsonutils.toJson` procedures.
+  * Added a possibility to deserialize JSON `null` objects to Nim option objects
+    and respectively to serialize Nim option object to JSON object if `isSome`
+    or to JSON null object if `isNone` via `jsonutils.fromJson` and
+    `jsonutils.toJson` procedures.
+  * Added a `Joptions` parameter to `jsonutils.fromJson` currently
+    containing two boolean options `allowExtraKeys` and `allowMissingKeys`.
+    - If `allowExtraKeys` is `true` Nim's object to which the JSON is parsed is
+      not required to have a field for every JSON key.
+    - If `allowMissingKeys` is `true` Nim's object to which JSON is parsed is
+      allowed to have fields without corresponding JSON keys.
+- Added `bindParams`, `bindParam` to `db_sqlite` for binding parameters into a `SqlPrepared` statement.
+- Added `tryInsert`,`insert` procs to `db_*` libs which accept primary key column name.
+- Added `xmltree.newVerbatimText` support create `style`'s,`script`'s text.
+- `uri` module now implements RFC-2397.
+- Added [DOM Parser](https://developer.mozilla.org/en-US/docs/Web/API/DOMParser)
+  to the `dom` module for the JavaScript target.
+- The default hash for `Ordinal` has changed to something more bit-scrambling.
+  `import hashes; proc hash(x: myInt): Hash = hashIdentity(x)` recovers the old
+  one in an instantiation context while `-d:nimIntHash1` recovers it globally.
+- `deques.peekFirst` and `deques.peekLast` now have `var Deque[T] -> var T` overloads.
+- File handles created from high-level abstractions in the stdlib will no longer
+  be inherited by child processes. In particular, these modules are affected:
+  `asyncdispatch`, `asyncnet`, `system`, `nativesockets`, `net` and `selectors`.
+
+  For `asyncdispatch`, `asyncnet`, `net` and `nativesockets`, an `inheritable`
+  flag has been added to all `proc`s that create sockets, allowing the user to
+  control whether the resulting socket is inheritable. This flag is provided to
+  ease the writing of multi-process servers, where sockets inheritance is
+  desired.
+
+  For a transition period, define `nimInheritHandles` to enable file handle
+  inheritance by default. This flag does **not** affect the `selectors` module
+  due to the differing semantics between operating systems.
+
+  `asyncdispatch.setInheritable`, `system.setInheritable` and
+  `nativesockets.setInheritable` are also introduced for setting file handle or
+  socket inheritance. Not all platforms have these `proc`s defined.
+
+- The file descriptors created for internal bookkeeping by `ioselector_kqueue`
+  and `ioselector_epoll` will no longer be leaked to child processes.
+
+- `strutils.formatFloat` with `precision = 0` has been restored to the version
+  1 behaviour that produces a trailing dot, e.g. `formatFloat(3.14159, precision = 0)`
+  is now `3.`, not `3`.
+- Added `commonPrefixLen` to `critbits`.
+
+- `relativePath(rel, abs)` and `relativePath(abs, rel)` used to silently give wrong results
+  (see #13222); instead they now use `getCurrentDir` to resolve those cases,
+  and this can now throw in edge cases where `getCurrentDir` throws.
+  `relativePath` also now works for js with `-d:nodejs`.
+
+- JavaScript and NimScript standard library changes: `streams.StringStream` is
+  now supported in JavaScript, with the limitation that any buffer `pointer`s
+  used must be castable to `ptr string`, any incompatible pointer type will not
+  work. The `lexbase` and `streams` modules used to fail to compile on
+  NimScript due to a bug, but this has been fixed.
+
+  The following modules now compile on both JS and NimScript: `parsecsv`,
+  `parsecfg`, `parsesql`, `xmlparser`, `htmlparser` and `ropes`. Additionally
+  supported for JS is `cstrutils.startsWith` and `cstrutils.endsWith`, for
+  NimScript: `json`, `parsejson`, `strtabs` and `unidecode`.
+
+- Added `streams.readStr` and `streams.peekStr` overloads to
+  accept an existing string to modify, which avoids memory
+  allocations, similar to `streams.readLine` (#13857).
+
+- Added high-level `asyncnet.sendTo` and `asyncnet.recvFrom` UDP functionality.
+
+- `dollars.$` now works for unsigned ints with `nim js`.
+
+- Improvements to the `bitops` module, including bitslices, non-mutating versions
+  of the original masking functions, `mask`/`masked`, and varargs support for
+  `bitand`, `bitor`, and `bitxor`.
+
+- `sugar.=>` and `sugar.->` changes: Previously `(x, y: int)` was transformed
+  into `(x: auto, y: int)`, it now becomes `(x: int, y: int)` for consistency
+  with regular proc definitions (although you cannot use semicolons).
+
+  Pragmas and using a name are now allowed on the lefthand side of `=>`. Here
+  is an example of these changes:
+  ```nim
+  import sugar
+
+  foo(x, y: int) {.noSideEffect.} => x + y
+
+  # is transformed into
+
+  proc foo(x: int, y: int): auto {.noSideEffect.} = x + y
+  ```
+
+- The fields of `times.DateTime` are now private, and are accessed with getters and deprecated setters.
+
+- The `times` module now handles the default value for `DateTime` more consistently.
+  Most procs raise an assertion error when given
+  an uninitialized `DateTime`, the exceptions are `==` and `$` (which returns `"Uninitialized DateTime"`).
+  The proc `times.isInitialized` has been added which can be used to check if
+  a `DateTime` has been initialized.
+
+- Fix a bug where calling `close` on io streams in `osproc.startProcess` was a noop and led to
+  hangs if a process had both reads from stdin and writes (e.g. to stdout).
+
+- The callback that is passed to `system.onThreadDestruction` must now be `.raises: []`.
+- The callback that is assigned to `system.onUnhandledException` must now be `.gcsafe`.
+
+- `osproc.execCmdEx` now takes an optional `input` for stdin, `workingDir` and `env`
+  parameters.
+
+- Added a `ssl_config` module containing lists of secure ciphers as recommended by
+  [Mozilla OpSec](https://wiki.mozilla.org/Security/Server_Side_TLS)
+
+- `net.newContext` now defaults to the list of ciphers targeting
+  ["Intermediate compatibility"](https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28recommended.29)
+  per Mozilla's recommendation instead of `ALL`. This change should protect
+  users from the use of weak and insecure ciphers while still provides
+  adequate compatibility with the majority of the Internet.
+
+- A new module `std/jsonutils` with hookable `jsonTo,toJson,fromJson` operations for json
+  serialization/deserialization of custom types was added.
+
+- A new proc `heapqueue.find[T](heap: HeapQueue[T], x: T): int` to get index of element ``x``
+  was added.
+- Added `rstgen.rstToLatex` a convenience proc for `renderRstToOut` and `initRstGenerator`.
+- Added `os.normalizeExe`.
+- `macros.newLit` now preserves named vs unnamed tuples.
+- Added `random.gauss`, that uses the ratio of uniforms method of sampling from a Gaussian distribution.
+- Added `typetraits.elementType` to get the element type of an iterable.
+- `typetraits.$` changes: `$(int,)` is now `"(int,)"` instead of `"(int)"`;
+  `$tuple[]` is now `"tuple[]"` instead of `"tuple"`;
+  `$((int, float), int)` is now `"((int, float), int)"` instead of `"(tuple of (int, float), int)"`
+- Added `macros.extractDocCommentsAndRunnables` helper.
+
+- `strformat.fmt` and `strformat.&` support `specifier =`. `fmt"{expr=}"` now
+  expands to `fmt"expr={expr}"`.
+- Deprecations: instead of `os.existsDir` use `dirExists`, instead of `os.existsFile` use `fileExists`.
+
+- Added the `jsre` module, [Regular Expressions for the JavaScript target.](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions).
+- Made `maxLines` argument `Positive` in `logging.newRollingFileLogger`,
+  because negative values will result in a new file being created for each logged
+  line which doesn't make sense.
+- Changed `log` in `logging` to use proper log level for JavaScript,
+  e.g. `debug` uses `console.debug`, `info` uses `console.info`, `warn` uses `console.warn`, etc.
+- Tables, HashSets, SharedTables and deques don't require anymore that the passed
+  initial size must be a power of two - this is done internally.
+  Proc `rightSize` for Tables and HashSets is deprecated, as it is not needed anymore.
+  `CountTable.inc` takes `val: int` again not `val: Positive`; i.e. it can "count down" again.
+- Removed deprecated symbols from `macros` module, some of which were deprecated already in `0.15`.
+- Removed `sugar.distinctBase`, deprecated since `0.19`. Use `typetraits.distinctBase`.
+- `asyncdispatch.PDispatcher.handles` is exported so that an external low-level libraries can access it.
+
+- `std/with`, `sugar.dup` now support object field assignment expressions:
+  ```nim
+  import std/with
+
+  type Foo = object
+    x, y: int
+
+  var foo = Foo()
+  with foo:
+    x = 10
+    y = 20
+
+  echo foo
+  ```
+
+- Proc `math.round` is no longer deprecated. The advice to use `strformat` instead
+  cannot be applied to every use case. The limitations and the (lack of) reliability
+  of `round` are well documented.
+
+- Added `getprotobyname` to `winlean`. Added `getProtoByname` to `nativesockets` which returns a protocol code
+  from the database that matches the protocol `name`.
+
+- Added missing attributes and methods to `dom.Navigator` like `deviceMemory`, `onLine`, `vibrate()`, etc.
+
+- Added `strutils.indentation` and `strutils.dedent` which enable indented string literals:
+  ```nim
+  import strutils
+  echo dedent """
+    This
+      is
+        cool!
+    """
+  ```
+
+- Added `initUri(isIpv6: bool)` to `uri` module, now `uri` supports parsing ipv6 hostname.
+
+- Added `readLines(p: Process)` to `osproc`.
+
+- Added the below `toX` procs for collections. The usage is similar to procs such as
+  `sets.toHashSet` and `tables.toTable`. Previously, it was necessary to create the
+  respective empty collection and add items manually.
+    * `critbits.toCritBitTree`, which creates a `CritBitTree` from an `openArray` of
+       items or an `openArray` of pairs.
+    * `deques.toDeque`, which creates a `Deque` from an `openArray`.
+    * `heapqueue.toHeapQueue`, which creates a `HeapQueue` from an `openArray`.
+    * `intsets.toIntSet`, which creates an `IntSet` from an `openArray`.
+
+- Added `progressInterval` argument to `asyncftpclient.newAsyncFtpClient` to control the interval
+  at which progress callbacks are called.
+
+- Added `os.copyFileToDir`.
+
+## Language changes
+
+- The `=destroy` hook no longer has to reset its target, as the compiler now automatically inserts
+  `wasMoved` calls where needed.
+- The `=` hook is now called `=copy` for clarity. The old name `=` is still available so there
+  is no need to update your code. This change was backported to 1.2 too so you can use the
+  more readable `=copy` without loss of compatibility.
+
+- In the newruntime it is now allowed to assign to the discriminator field
+  without restrictions as long as the case object doesn't have a custom destructor.
+  The discriminator value doesn't have to be a constant either. If you have a
+  custom destructor for a case object and you do want to freely assign discriminator
+  fields, it is recommended to refactor the object into 2 objects like this:
+
+  ```nim
+  type
+    MyObj = object
+      case kind: bool
+      of true: y: ptr UncheckedArray[float]
+      of false: z: seq[int]
+
+  proc `=destroy`(x: MyObj) =
+    if x.kind and x.y != nil:
+      deallocShared(x.y)
+  ```
+  Refactor into:
+  ```nim
+  type
+    MySubObj = object
+      val: ptr UncheckedArray[float]
+    MyObj = object
+      case kind: bool
+      of true: y: MySubObj
+      of false: z: seq[int]
+
+  proc `=destroy`(x: MySubObj) =
+    if x.val != nil:
+      deallocShared(x.val)
+  ```
+- `getImpl` on enum type symbols now returns field syms instead of idents. This helps
+  with writing typed macros. The old behavior for backwards compatibility can be restored
+  with `--useVersion:1.0`.
+- The typed AST for proc headers will now have the arguments be syms instead of idents.
+  This helps with writing typed macros. The old behaviour for backwards compatibility can
+  be restored with `--useVersion:1.0`.
+- ``let`` statements can now be used without a value if declared with
+  ``importc``/``importcpp``/``importjs``/``importobjc``.
+- The keyword `from` is now usable as an operator.
+- Exceptions inheriting from `system.Defect` are no longer tracked with
+  the `.raises: []` exception tracking mechanism. This is more consistent with the
+  built-in operations. The following always used to compile (and still does):
+  ```nim
+  proc mydiv(a, b): int {.raises: [].} =
+    a div b # can raise an DivByZeroDefect
+  ```
+
+  Now also this compiles:
+  ```nim
+  proc mydiv(a, b): int {.raises: [].} =
+    if b == 0: raise newException(DivByZeroDefect, "division by zero")
+    else: result = a div b
+  ```
+
+  The reason for this is that `DivByZeroDefect` inherits from `Defect` and
+  with `--panics:on` `Defects` become unrecoverable errors.
+
+- Added the `thiscall` calling convention as specified by Microsoft, mostly for hooking purposes.
+- Deprecated the `{.unroll.}` pragma, because it was always ignored by the compiler anyway.
+- Removed the deprecated `strutils.isNilOrWhitespace`.
+- Removed the deprecated `sharedtables.initSharedTable`.
+- Removed the deprecated `asyncdispatch.newAsyncNativeSocket`.
+- Removed the deprecated `dom.releaseEvents` and `dom.captureEvents`.
+
+- Removed `sharedlists.initSharedList`, was deprecated and produces undefined behaviour.
+
+- There is a new experimental feature called "strictFuncs" which makes the definition of
+  `.noSideEffect` stricter. [See here](manual_experimental.html#stricts-funcs)
+  for more information.
+
+- "for-loop macros" (see [the manual](manual.html#macros-for-loop-macros)) are no longer
+  an experimental feature. In other words, you don't have to write pragma
+  `{.experimental: "forLoopMacros".}` if you want to use them.
+
+- Added the ``.noalias`` pragma. It is mapped to C's ``restrict`` keyword for the increased
+  performance this keyword can enable.
+
+- `items` no longer compiles with enums with holes as its behavior was error prone, see #14004.
+- `system.deepcopy` has to be enabled explicitly for `--gc:arc` and `--gc:orc` via
+  `--deepcopy:on`.
+
+- Added the `std/effecttraits` module for introspection of the inferred effects.
+  We hope this enables `async` macros that are precise about the possible exceptions that
+  can be raised.
+- The pragma blocks `{.gcsafe.}: ...` and `{.noSideEffect.}: ...` can now also be
+  written as `{.cast(gcsafe).}: ...` and `{.cast(noSideEffect).}: ...`. This is the new
+  preferred way of writing these, emphasizing their unsafe nature.
+
+
+## Compiler changes
+
+- Specific warnings can now be turned into errors via `--warningAsError[X]:on|off`.
+- The `define` and `undef` pragmas have been de-deprecated.
+- New command: `nim r main.nim [args...]` which compiles and runs main.nim, and implies `--usenimcache`
+  so that the output is saved to $nimcache/main$exeExt, using the same logic as `nim c -r` to
+  avoid recompilations when sources don't change.
+  Example:
+  ```bash
+  nim r compiler/nim.nim --help # only compiled the first time
+  echo 'import os; echo getCurrentCompilerExe()' | nim r - # this works too
+  nim r compiler/nim.nim --fullhelp # no recompilation
+  nim r --nimcache:/tmp main # binary saved to /tmp/main
+  ```
+- `--hint:processing` is now supported and means `--hint:processing:on`
+  (likewise with other hints and warnings), which is consistent with all other bool flags.
+  (since 1.3.3).
+- `nim doc -r main` and `nim rst2html -r main` now call `openDefaultBrowser`.
+- Added the new hint `--hint:msgOrigin` will show where a compiler msg (hint|warning|error)
+  was generated; this helps in particular when it's non obvious where it came from
+  either because multiple locations generate the same message, or because the
+  message involves runtime formatting.
+- Added the new flag `--backend:js|c|cpp|objc` (or -b:js etc), to change the backend; can be
+  used with any command (e.g. nim r, doc, check etc); safe to re-assign.
+- Added the new flag `--doccmd:cmd` to pass additional flags for runnableExamples,
+  e.g.: `--doccmd:-d:foo --threads`
+  use `--doccmd:skip` to skip runnableExamples and rst test snippets.
+- Added the new flag `--usenimcache` to output binary files to nimcache.
+- `runnableExamples "-b:cpp -r:off": code` is now supported, allowing to override
+  how an example is compiled and run, for example to change the backend.
+- `nim doc` now outputs under `$projectPath/htmldocs` when `--outdir` is unspecified
+  (with or without `--project`); passing `--project` now automatically generates
+  an index and enables search.
+  See [docgen](docgen.html#introduction-quick-start) for details.
+- Removed the `--oldNewlines` switch.
+- Removed the `--laxStrings` switch for mutating the internal zero terminator on strings.
+- Removed the `--oldast` switch.
+- Removed the `--oldgensym` switch.
+- `$getType(untyped)` is now "untyped" instead of "expr", `$getType(typed)` is
+  now "typed" instead of "stmt".
+- Sink inference is now disabled per default and has to enabled explicitly via
+  `--sinkInference:on`. *Note*: For the standard library sink inference remains
+  enabled. This change is most relevant for the `--gc:arc`, `--gc:orc` memory
+  management modes.
+
+
+## Tool changes
+
+- `nimsuggest` now returns both the forward declaration and the
+  implementation location upon a `def` query. Previously the behavior was
+  to return the forward declaration only.
+
+
+## Bugfixes
+
+- Fixed "repr() not available for uint{,8,16,32,64} under --gc:arc"
+  ([#13872](https://github.com/nim-lang/Nim/issues/13872))
+- Fixed "Critical: 1 completed Future, multiple await: Only 1 await will be awakened (the last one)"
+  ([#13889](https://github.com/nim-lang/Nim/issues/13889))
+- Fixed "crash on openarray interator with argument in stmtListExpr"
+  ([#13739](https://github.com/nim-lang/Nim/issues/13739))
+- Fixed "Some compilers on Windows don't work"
+  ([#13910](https://github.com/nim-lang/Nim/issues/13910))
+- Fixed "httpclient hangs if it recieves an HTTP 204 (No Content)"
+  ([#13894](https://github.com/nim-lang/Nim/issues/13894))
+- Fixed ""distinct uint64" type corruption on 32-bit, when using {.borrow.} operators"
+  ([#13902](https://github.com/nim-lang/Nim/issues/13902))
+- Fixed "Regression: impossible to use typed pragmas with proc types"
+  ([#13909](https://github.com/nim-lang/Nim/issues/13909))
+- Fixed "openssl wrapper corrupts stack on OpenSSL 1.1.1f + Android"
+  ([#13903](https://github.com/nim-lang/Nim/issues/13903))
+- Fixed "C compile error with --gc:arc on version 1.2.0 "unknown type name 'TGenericSeq'"
+  ([#13863](https://github.com/nim-lang/Nim/issues/13863))
+- Fixed "var return type for proc doesn't work at c++ backend"
+  ([#13848](https://github.com/nim-lang/Nim/issues/13848))
+- Fixed "TimeFormat() should raise an error but craches at compilation time"
+  ([#12864](https://github.com/nim-lang/Nim/issues/12864))
+- Fixed "gc:arc cannot fully support threadpool with FlowVar"
+  ([#13781](https://github.com/nim-lang/Nim/issues/13781))
+- Fixed "simple 'var openarray[char]' assignment crash when the openarray source is a local string and using gc:arc"
+  ([#14003](https://github.com/nim-lang/Nim/issues/14003))
+- Fixed "Cant use expressions with `when` in `type` sections."
+  ([#14007](https://github.com/nim-lang/Nim/issues/14007))
+- Fixed "`for a in MyEnum` gives incorrect results with enum with holes"
+  ([#14001](https://github.com/nim-lang/Nim/issues/14001))
+- Fixed "Trivial crash"
+  ([#12741](https://github.com/nim-lang/Nim/issues/12741))
+- Fixed "Enum with holes cannot be used as Table index"
+  ([#12834](https://github.com/nim-lang/Nim/issues/12834))
+- Fixed "spawn proc that uses typedesc crashes the compiler"
+  ([#14014](https://github.com/nim-lang/Nim/issues/14014))
+- Fixed "Docs Search `Results` box styling is not Dark Mode Friendly"
+  ([#13972](https://github.com/nim-lang/Nim/issues/13972))
+- Fixed "--gc:arc -d:useSysAssert undeclared identifier `cstderr` with newSeq"
+  ([#14038](https://github.com/nim-lang/Nim/issues/14038))
+- Fixed "issues in the manual"
+  ([#12486](https://github.com/nim-lang/Nim/issues/12486))
+- Fixed "Annoying warning: inherit from a more precise exception type like ValueError, IOError or OSError [InheritFromException]"
+  ([#14052](https://github.com/nim-lang/Nim/issues/14052))
+- Fixed "relativePath("foo", "/") and relativePath("/", "foo") is wrong"
+  ([#13222](https://github.com/nim-lang/Nim/issues/13222))
+- Fixed "[regression] `parseEnum` does not work anymore for enums with holes"
+  ([#14030](https://github.com/nim-lang/Nim/issues/14030))
+- Fixed "Exception types in the stdlib should inherit from `CatchableError` or `Defect`, not `Exception`"
+  ([#10288](https://github.com/nim-lang/Nim/issues/10288))
+- Fixed "Make debugSend and debugRecv procs public in smtp.nim"
+  ([#12189](https://github.com/nim-lang/Nim/issues/12189))
+- Fixed "xmltree need add raw text, when add style element"
+  ([#14064](https://github.com/nim-lang/Nim/issues/14064))
+- Fixed "raises requirement does not propagate to derived methods"
+  ([#8481](https://github.com/nim-lang/Nim/issues/8481))
+- Fixed "tests/stdlib/tgetaddrinfo.nim fails on NetBSD"
+  ([#14091](https://github.com/nim-lang/Nim/issues/14091))
+- Fixed "tests/niminaction/Chapter8/sdl/sdl_test.nim fails on NetBSD"
+  ([#14088](https://github.com/nim-lang/Nim/issues/14088))
+- Fixed "Incorrect escape sequence for example in jsffi library documentation"
+  ([#14110](https://github.com/nim-lang/Nim/issues/14110))
+- Fixed "HCR: Can not link exported const, in external library"
+  ([#13915](https://github.com/nim-lang/Nim/issues/13915))
+- Fixed "Cannot import std/unidecode"
+  ([#14112](https://github.com/nim-lang/Nim/issues/14112))
+- Fixed "macOS: dsymutil should not be called on static libraries"
+  ([#14132](https://github.com/nim-lang/Nim/issues/14132))
+- Fixed "nim jsondoc -o:doc.json filename.nim fails when sequences without a type are used"
+  ([#14066](https://github.com/nim-lang/Nim/issues/14066))
+- Fixed "algorithm.sortedByIt template corrupts tuple input under --gc:arc"
+  ([#14079](https://github.com/nim-lang/Nim/issues/14079))
+- Fixed "Invalid C code with lvalue conversion"
+  ([#14160](https://github.com/nim-lang/Nim/issues/14160))
+- Fixed "strformat: doc example fails"
+  ([#14054](https://github.com/nim-lang/Nim/issues/14054))
+- Fixed "Nim doc fail to run for nim 1.2.0 (nim 1.0.4 is ok)"
+  ([#13986](https://github.com/nim-lang/Nim/issues/13986))
+- Fixed "Exception when converting csize to clong"
+  ([#13698](https://github.com/nim-lang/Nim/issues/13698))
+- Fixed "[Documentation] overloading using named arguments works but is not documented"
+  ([#11932](https://github.com/nim-lang/Nim/issues/11932))
+- Fixed "import os + use of existsDir/dirExists/existsFile/fileExists/findExe in config.nims causes "ambiguous call' error"
+  ([#14142](https://github.com/nim-lang/Nim/issues/14142))
+- Fixed "import os + use of existsDir/dirExists/existsFile/fileExists/findExe in config.nims causes "ambiguous call' error"
+  ([#14142](https://github.com/nim-lang/Nim/issues/14142))
+- Fixed "runnableExamples doc gen crashes compiler with `except Exception as e` syntax"
+  ([#14177](https://github.com/nim-lang/Nim/issues/14177))
+- Fixed "[ARC] Segfault with cyclic references (?)"
+  ([#14159](https://github.com/nim-lang/Nim/issues/14159))
+- Fixed "Semcheck regression when accessing a static parameter in proc"
+  ([#14136](https://github.com/nim-lang/Nim/issues/14136))
+- Fixed "iterator walkDir doesn't work with -d:useWinAnsi"
+  ([#14201](https://github.com/nim-lang/Nim/issues/14201))
+- Fixed "cas is wrong for tcc"
+  ([#14151](https://github.com/nim-lang/Nim/issues/14151))
+- Fixed "proc execCmdEx doesn't work with -d:useWinAnsi"
+  ([#14203](https://github.com/nim-lang/Nim/issues/14203))
+- Fixed "Use -d:nimEmulateOverflowChecks by default?"
+  ([#14209](https://github.com/nim-lang/Nim/issues/14209))
+- Fixed "Old sequences with destructor objects bug"
+  ([#14217](https://github.com/nim-lang/Nim/issues/14217))
+- Fixed "[ARC] ICE when changing the discriminant of a return value"
+  ([#14244](https://github.com/nim-lang/Nim/issues/14244))
+- Fixed "[ARC] ICE with static objects"
+  ([#14236](https://github.com/nim-lang/Nim/issues/14236))
+- Fixed "[ARC] "internal error: environment misses: a" in a finalizer"
+  ([#14243](https://github.com/nim-lang/Nim/issues/14243))
+- Fixed "[ARC] compile failure using repr with object containing `ref seq[string]`"
+  ([#14270](https://github.com/nim-lang/Nim/issues/14270))
+- Fixed "[ARC] implicit move on last use happening on non-last use"
+  ([#14269](https://github.com/nim-lang/Nim/issues/14269))
+- Fixed "[ARC] Compiler crash with a recursive non-ref object variant"
+  ([#14294](https://github.com/nim-lang/Nim/issues/14294))
+- Fixed "htmlparser.parseHtml behaves differently using --gc:arc or --gc:orc"
+  ([#13946](https://github.com/nim-lang/Nim/issues/13946))
+- Fixed "Invalid return value of openProcess is NULL rather than INVALID_HANDLE_VALUE(-1) in windows"
+  ([#14289](https://github.com/nim-lang/Nim/issues/14289))
+- Fixed "ARC codegen bug with inline iterators"
+  ([#14219](https://github.com/nim-lang/Nim/issues/14219))
+- Fixed "Building koch on OpenBSD fails unless the Nim directory is in `$PATH`"
+  ([#13758](https://github.com/nim-lang/Nim/issues/13758))
+- Fixed "[gc:arc] case object assignment SIGSEGV: destroy not called for primitive type "
+  ([#14312](https://github.com/nim-lang/Nim/issues/14312))
+- Fixed "Crash when using thread and --gc:arc "
+  ([#13881](https://github.com/nim-lang/Nim/issues/13881))
+- Fixed "Getting "Warning: Cannot prove that 'result' is initialized" for an importcpp'd proc with `var T` return type"
+  ([#14314](https://github.com/nim-lang/Nim/issues/14314))
+- Fixed "`nim cpp -r --gc:arc` segfaults on caught AssertionError"
+  ([#13071](https://github.com/nim-lang/Nim/issues/13071))
+- Fixed "tests/async/tasyncawait.nim is recently very flaky"
+  ([#14320](https://github.com/nim-lang/Nim/issues/14320))
+- Fixed "Documentation nonexistent quitprocs module"
+  ([#14331](https://github.com/nim-lang/Nim/issues/14331))
+- Fixed "SIGSEV encountered when creating threads in a loop w/ --gc:arc"
+  ([#13935](https://github.com/nim-lang/Nim/issues/13935))
+- Fixed "nim-gdb is missing from all released packages"
+  ([#13104](https://github.com/nim-lang/Nim/issues/13104))
+- Fixed "sysAssert error with gc:arc on 3 line program"
+  ([#13862](https://github.com/nim-lang/Nim/issues/13862))
+- Fixed "compiler error with inline async proc and pragma"
+  ([#13998](https://github.com/nim-lang/Nim/issues/13998))
+- Fixed "[ARC] Compiler crash when adding to a seq[ref Object]"
+  ([#14333](https://github.com/nim-lang/Nim/issues/14333))
+- Fixed "nimvm: sysFatal: unhandled exception: 'sons' is not accessible using discriminant 'kind' of type 'TNode' [FieldError]"
+  ([#14340](https://github.com/nim-lang/Nim/issues/14340))
+- Fixed "[Regression] karax events are not firing "
+  ([#14350](https://github.com/nim-lang/Nim/issues/14350))
+- Fixed "odbcsql module has some wrong integer types"
+  ([#9771](https://github.com/nim-lang/Nim/issues/9771))
+- Fixed "db_sqlite needs sqlPrepared"
+  ([#13559](https://github.com/nim-lang/Nim/issues/13559))
+- Fixed "[Regression] `createThread` is not GC-safe"
+  ([#14370](https://github.com/nim-lang/Nim/issues/14370))
+- Fixed "Broken example on hot code reloading"
+  ([#14380](https://github.com/nim-lang/Nim/issues/14380))
+- Fixed "runnableExamples block with `except` on specified error fails with `nim doc`"
+  ([#12746](https://github.com/nim-lang/Nim/issues/12746))
+- Fixed "compiler as a library: findNimStdLibCompileTime fails to find system.nim"
+  ([#12293](https://github.com/nim-lang/Nim/issues/12293))
+- Fixed "5 bugs with importcpp exceptions"
+  ([#14369](https://github.com/nim-lang/Nim/issues/14369))
+- Fixed "Docs shouldn't collapse pragmas inside runnableExamples/code blocks"
+  ([#14174](https://github.com/nim-lang/Nim/issues/14174))
+- Fixed "Bad codegen/emit for hashes.hiXorLo in some contexts."
+  ([#14394](https://github.com/nim-lang/Nim/issues/14394))
+- Fixed "Boehm GC does not scan thread-local storage"
+  ([#14364](https://github.com/nim-lang/Nim/issues/14364))
+- Fixed "RVO not exception safe"
+  ([#14126](https://github.com/nim-lang/Nim/issues/14126))
+- Fixed "runnableExamples that are only compiled"
+  ([#10731](https://github.com/nim-lang/Nim/issues/10731))
+- Fixed "`foldr` raises IndexError when called on sequence"
+  ([#14404](https://github.com/nim-lang/Nim/issues/14404))
+- Fixed "moveFile does not overwrite destination file"
+  ([#14057](https://github.com/nim-lang/Nim/issues/14057))
+- Fixed "doc2 outputs in current work dir"
+  ([#6583](https://github.com/nim-lang/Nim/issues/6583))
+- Fixed "[docgen] proc doc comments silently omitted after 1st runnableExamples"
+  ([#9227](https://github.com/nim-lang/Nim/issues/9227))
+- Fixed "`nim doc --project` shows '@@/' instead of '../' for relative paths to submodules"
+  ([#14448](https://github.com/nim-lang/Nim/issues/14448))
+- Fixed "re, nre have wrong `start` semantics"
+  ([#14284](https://github.com/nim-lang/Nim/issues/14284))
+- Fixed "runnableExamples should preserve source code doc comments, strings, and (maybe) formatting"
+  ([#8871](https://github.com/nim-lang/Nim/issues/8871))
+- Fixed "`nim doc ..` fails when runnableExamples uses `$`  [devel] [regression]"
+  ([#14485](https://github.com/nim-lang/Nim/issues/14485))
+- Fixed "`items` is 20%~30% slower than iteration via an index"
+  ([#14421](https://github.com/nim-lang/Nim/issues/14421))
+- Fixed "ARC: unreliable setLen "
+  ([#14495](https://github.com/nim-lang/Nim/issues/14495))
+- Fixed "lent is unsafe: after #14447 you can modify variables with "items" loop for sequences"
+  ([#14498](https://github.com/nim-lang/Nim/issues/14498))
+- Fixed "`var op = fn()` wrongly gives warning `ObservableStores` with `object of RootObj` type"
+  ([#14514](https://github.com/nim-lang/Nim/issues/14514))
+- Fixed "Compiler assertion"
+  ([#14562](https://github.com/nim-lang/Nim/issues/14562))
+- Fixed "Can't get `ord` of a value of a Range type in the JS backend "
+  ([#14570](https://github.com/nim-lang/Nim/issues/14570))
+- Fixed "js: can't take addr of param (including implicitly via `lent`)"
+  ([#14576](https://github.com/nim-lang/Nim/issues/14576))
+- Fixed "{.noinit.} ignored in for loop -> bad codegen for non-movable types"
+  ([#14118](https://github.com/nim-lang/Nim/issues/14118))
+- Fixed "generic destructor gives: `Error: unresolved generic parameter`"
+  ([#14315](https://github.com/nim-lang/Nim/issues/14315))
+- Fixed "Memory leak with arc gc"
+  ([#14568](https://github.com/nim-lang/Nim/issues/14568))
+- Fixed "escape analysis broken with `lent`"
+  ([#14557](https://github.com/nim-lang/Nim/issues/14557))
+- Fixed "`wrapWords` seems to ignore linebreaks when wrapping, leaving breaks in the wrong place"
+  ([#14579](https://github.com/nim-lang/Nim/issues/14579))
+- Fixed "`lent` gives wrong results with -d:release"
+  ([#14578](https://github.com/nim-lang/Nim/issues/14578))
+- Fixed "Nested await expressions regression: `await a(await expandValue())` doesnt compile"
+  ([#14279](https://github.com/nim-lang/Nim/issues/14279))
+- Fixed "windows CI docs fails with strange errors"
+  ([#14545](https://github.com/nim-lang/Nim/issues/14545))
+- Fixed "[CI] tests/async/tioselectors.nim flaky test for freebsd + OSX CI"
+  ([#13166](https://github.com/nim-lang/Nim/issues/13166))
+- Fixed "seq.setLen sometimes doesn't zero memory"
+  ([#14655](https://github.com/nim-lang/Nim/issues/14655))
+- Fixed "`nim dump` is roughly 100x slower in 1.3 versus 1.2"
+  ([#14179](https://github.com/nim-lang/Nim/issues/14179))
+- Fixed "Regression: devel docgen cannot generate document for method"
+  ([#14691](https://github.com/nim-lang/Nim/issues/14691))
+- Fixed "recently flaky tests/async/t7758.nim"
+  ([#14685](https://github.com/nim-lang/Nim/issues/14685))
+- Fixed "Bind no longer working in generic procs."
+  ([#11811](https://github.com/nim-lang/Nim/issues/11811))
+- Fixed "The pegs module doesn't work with generics!"
+  ([#14718](https://github.com/nim-lang/Nim/issues/14718))
+- Fixed "Defer is not properly working for asynchronous procedures."
+  ([#13899](https://github.com/nim-lang/Nim/issues/13899))
+- Fixed "Add an ARC test with threads in a loop"
+  ([#14690](https://github.com/nim-lang/Nim/issues/14690))
+- Fixed "[goto exceptions] {.noReturn.} pragma is not detected in a case expression"
+  ([#14458](https://github.com/nim-lang/Nim/issues/14458))
+- Fixed "[exceptions:goto] C compiler error with dynlib pragma calling a proc"
+  ([#14240](https://github.com/nim-lang/Nim/issues/14240))
+- Fixed "Cannot borrow var float64 in infix assignment"
+  ([#14440](https://github.com/nim-lang/Nim/issues/14440))
+- Fixed "lib/pure/memfiles.nim: compilation error with --taintMode:on"
+  ([#14760](https://github.com/nim-lang/Nim/issues/14760))
+- Fixed "newWideCString allocates a multiple of the memory needed"
+  ([#14750](https://github.com/nim-lang/Nim/issues/14750))
+- Fixed "Nim source archive install: 'install.sh' fails with error: cp: cannot stat 'bin/nim-gdb': No such file or directory"
+  ([#14748](https://github.com/nim-lang/Nim/issues/14748))
+- Fixed "`nim cpp -r tests/exception/t9657` hangs"
+  ([#10343](https://github.com/nim-lang/Nim/issues/10343))
+- Fixed "Detect tool fails on FreeBSD"
+  ([#14715](https://github.com/nim-lang/Nim/issues/14715))
+- Fixed "compiler crash: `findUnresolvedStatic` "
+  ([#14802](https://github.com/nim-lang/Nim/issues/14802))
+- Fixed "seq namespace (?) regression"
+  ([#4796](https://github.com/nim-lang/Nim/issues/4796))
+- Fixed "Possible out of bounds string access in std/colors parseColor and isColor"
+  ([#14839](https://github.com/nim-lang/Nim/issues/14839))
+- Fixed "compile error on latest devel with orc and ssl"
+  ([#14647](https://github.com/nim-lang/Nim/issues/14647))
+- Fixed "[minor] `$` wrong for type tuple"
+  ([#13432](https://github.com/nim-lang/Nim/issues/13432))
+- Fixed "Documentation missing on devel asyncftpclient"
+  ([#14846](https://github.com/nim-lang/Nim/issues/14846))
+- Fixed "nimpretty is confused with a trailing comma in enum definition"
+  ([#14401](https://github.com/nim-lang/Nim/issues/14401))
+- Fixed "Output arguments get ignored when compiling with --app:staticlib"
+  ([#12745](https://github.com/nim-lang/Nim/issues/12745))
+- Fixed "[ARC] destructive move destroys the object too early"
+  ([#14396](https://github.com/nim-lang/Nim/issues/14396))
+- Fixed "highlite.getNextToken() crashes if the buffer string is "echo "\"""
+  ([#14830](https://github.com/nim-lang/Nim/issues/14830))
+- Fixed "Memory corruption with --gc:arc with a seq of objects with an empty body."
+  ([#14472](https://github.com/nim-lang/Nim/issues/14472))
+- Fixed "Stropped identifiers don't work as field names in tuple literals"
+  ([#14911](https://github.com/nim-lang/Nim/issues/14911))
+- Fixed "Please revert my commit"
+  ([#14930](https://github.com/nim-lang/Nim/issues/14930))
+- Fixed "[ARC] C compiler error with inline iterators and imports"
+  ([#14864](https://github.com/nim-lang/Nim/issues/14864))
+- Fixed "AsyncHttpClient segfaults with gc:orc, possibly memory corruption"
+  ([#14402](https://github.com/nim-lang/Nim/issues/14402))
+- Fixed "[ARC] Template with a block evaluating to a GC'd value results in a compiler crash"
+  ([#14899](https://github.com/nim-lang/Nim/issues/14899))
+- Fixed "[ARC] Weird issue with if expressions and templates"
+  ([#14900](https://github.com/nim-lang/Nim/issues/14900))
+- Fixed "xmlparser does not compile on devel"
+  ([#14805](https://github.com/nim-lang/Nim/issues/14805))
+- Fixed "returning lent T from a var T param gives codegen errors or SIGSEGV"
+  ([#14878](https://github.com/nim-lang/Nim/issues/14878))
+- Fixed "[ARC] Weird issue with if expressions and templates"
+  ([#14900](https://github.com/nim-lang/Nim/issues/14900))
+- Fixed "threads:on + gc:orc + unittest = C compiler errors"
+  ([#14865](https://github.com/nim-lang/Nim/issues/14865))
+- Fixed "mitems, mpairs doesn't work at compile time anymore"
+  ([#12129](https://github.com/nim-lang/Nim/issues/12129))
+- Fixed "strange result from executing code in const expression"
+  ([#10465](https://github.com/nim-lang/Nim/issues/10465))
+- Fixed "Same warning printed 3 times"
+  ([#11009](https://github.com/nim-lang/Nim/issues/11009))
+- Fixed "type alias for generic typeclass doesn't work"
+  ([#4668](https://github.com/nim-lang/Nim/issues/4668))
+- Fixed "exceptions:goto Bug devel codegen lvalue NIM_FALSE=NIM_FALSE"
+  ([#14925](https://github.com/nim-lang/Nim/issues/14925))
+- Fixed "the --useVersion:1.0 no longer works in devel"
+  ([#14912](https://github.com/nim-lang/Nim/issues/14912))
+- Fixed "template declaration of iterator doesn't compile"
+  ([#4722](https://github.com/nim-lang/Nim/issues/4722))
+- Fixed "Compiler crash on type inheritance with static generic parameter and equality check"
+  ([#12571](https://github.com/nim-lang/Nim/issues/12571))
+- Fixed "Nim crashes while handling a cast in async circumstances."
+  ([#13815](https://github.com/nim-lang/Nim/issues/13815))
+- Fixed "[ARC] Internal compiler error when calling an iterator from an inline proc "
+  ([#14383](https://github.com/nim-lang/Nim/issues/14383))
+- Fixed ""Cannot instantiate" error when template uses generic type"
+  ([#5926](https://github.com/nim-lang/Nim/issues/5926))
+- Fixed "Different raises behaviour for newTerminal between Linux and Windows"
+  ([#12759](https://github.com/nim-lang/Nim/issues/12759))
+- Fixed "Expand on a type (that defines a proc type) in error message "
+  ([#6608](https://github.com/nim-lang/Nim/issues/6608))
+- Fixed "unittest require quits program with an exit code of 0"
+  ([#14475](https://github.com/nim-lang/Nim/issues/14475))
+- Fixed "Range type: Generics vs concrete type, semcheck difference."
+  ([#8426](https://github.com/nim-lang/Nim/issues/8426))
+- Fixed "[Macro] Type mismatch when parameter name is the same as a field"
+  ([#13253](https://github.com/nim-lang/Nim/issues/13253))
+- Fixed "Generic instantiation failure when converting a sequence of circular generic types to strings"
+  ([#10396](https://github.com/nim-lang/Nim/issues/10396))
+- Fixed "initOptParser ignores argument after value option with empty value."
+  ([#13086](https://github.com/nim-lang/Nim/issues/13086))
+- Fixed "[ARC] proc with both explicit and implicit return results in a C compiler error"
+  ([#14985](https://github.com/nim-lang/Nim/issues/14985))
+- Fixed "Alias type forgets implicit generic params depending on order"
+  ([#14990](https://github.com/nim-lang/Nim/issues/14990))
+- Fixed "[ARC] sequtils.insert has different behaviour between ARC/refc"
+  ([#14994](https://github.com/nim-lang/Nim/issues/14994))
+- Fixed "The documentation for "hot code reloading" references a non-existent npm package"
+  ([#13621](https://github.com/nim-lang/Nim/issues/13621))
+- Fixed "existsDir deprecated but breaking `dir` undeclared"
+  ([#15006](https://github.com/nim-lang/Nim/issues/15006))
+- Fixed "uri.decodeUrl crashes on incorrectly formatted input"
+  ([#14082](https://github.com/nim-lang/Nim/issues/14082))
+- Fixed "testament incorrectly reports time for tests, leading to wrong conclusions"
+  ([#14822](https://github.com/nim-lang/Nim/issues/14822))
+- Fixed "Calling peekChar with Stream returned from osproc.outputStream generate runtime error"
+  ([#14906](https://github.com/nim-lang/Nim/issues/14906))
+- Fixed "localPassC pragma should come *after* other flags"
+  ([#14194](https://github.com/nim-lang/Nim/issues/14194))
+- Fixed ""Could not load" dynamic library at runtime because of hidden dependency"
+  ([#2408](https://github.com/nim-lang/Nim/issues/2408))
+- Fixed "--gc:arc generate invalid code for {.global.} (`«nimErr_» in NIM_UNLIKELY`)"
+  ([#14480](https://github.com/nim-lang/Nim/issues/14480))
+- Fixed "Using `^` from stdlib/math along with converters gives a match for types that aren't SomeNumber"
+  ([#15033](https://github.com/nim-lang/Nim/issues/15033))
+- Fixed "[ARC] Weird exception behaviour from doAssertRaises"
+  ([#15026](https://github.com/nim-lang/Nim/issues/15026))
+- Fixed "[ARC] Compiler crash declaring a finalizer proc directly in 'new'"
+  ([#15044](https://github.com/nim-lang/Nim/issues/15044))
+- Fixed "[ARC] C compiler error when creating a var of a const seq"
+  ([#15036](https://github.com/nim-lang/Nim/issues/15036))
+- Fixed "code with named arguments in proc of winim/com can not been compiled"
+  ([#15056](https://github.com/nim-lang/Nim/issues/15056))
+- Fixed "javascript backend produces javascript code with syntax error in object syntax"
+  ([#14534](https://github.com/nim-lang/Nim/issues/14534))
+- Fixed "--gc:arc should be ignored in JS mode."
+  ([#14684](https://github.com/nim-lang/Nim/issues/14684))
+- Fixed "arc: C compilation error with imported global code using a closure iterator"
+  ([#12990](https://github.com/nim-lang/Nim/issues/12990))
+- Fixed "[ARC] Crash when modifying a string with mitems iterator"
+  ([#15052](https://github.com/nim-lang/Nim/issues/15052))
+- Fixed "[ARC] SIGSEGV when calling a closure as a tuple field in a seq"
+  ([#15038](https://github.com/nim-lang/Nim/issues/15038))
+- Fixed "pass varargs[seq[T]] to iterator give empty seq "
+  ([#12576](https://github.com/nim-lang/Nim/issues/12576))
+- Fixed "Compiler crashes when using string as object variant selector with else branch"
+  ([#14189](https://github.com/nim-lang/Nim/issues/14189))
+- Fixed "JS compiler error related to implicit return and return var type"
+  ([#11354](https://github.com/nim-lang/Nim/issues/11354))
+- Fixed "`nkRecWhen` causes internalAssert in semConstructFields"
+  ([#14698](https://github.com/nim-lang/Nim/issues/14698))
+- Fixed "Memory leaks with async (closure iterators?) under ORC"
+  ([#15076](https://github.com/nim-lang/Nim/issues/15076))
+- Fixed "strutil.insertSep() fails on negative numbers"
+  ([#11352](https://github.com/nim-lang/Nim/issues/11352))
+- Fixed "Constructing a uint64 range on a 32-bit machine leads to incorrect codegen"
+  ([#14616](https://github.com/nim-lang/Nim/issues/14616))
+- Fixed "heapqueue pushpop() proc doesn't compile"
+  ([#14139](https://github.com/nim-lang/Nim/issues/14139))
+- Fixed "[ARC] SIGSEGV when trying to swap in a literal/const string"
+  ([#15112](https://github.com/nim-lang/Nim/issues/15112))
+- Fixed "Defer and --gc:arc"
+  ([#15071](https://github.com/nim-lang/Nim/issues/15071))
+- Fixed "internal error: compiler/semobjconstr.nim(324, 20) example"
+  ([#15111](https://github.com/nim-lang/Nim/issues/15111))
+- Fixed "[ARC] Sequence "disappears" with a table inside of a table with an object variant"
+  ([#15122](https://github.com/nim-lang/Nim/issues/15122))
+- Fixed "[ARC] SIGSEGV with tuple assignment caused by cursor inference"
+  ([#15130](https://github.com/nim-lang/Nim/issues/15130))
+- Fixed "Issue with --gc:arc at compile time"
+  ([#15129](https://github.com/nim-lang/Nim/issues/15129))
+- Fixed "Writing an empty string to an AsyncFile raises an IndexDefect"
+  ([#15148](https://github.com/nim-lang/Nim/issues/15148))
+- Fixed "Compiler is confused about call convention of function with nested closure"
+  ([#5688](https://github.com/nim-lang/Nim/issues/5688))
+- Fixed "Nil check on each field fails in generic function"
+  ([#15101](https://github.com/nim-lang/Nim/issues/15101))
+- Fixed "{.nimcall.} convention won't avoid the creation of closures"
+  ([#8473](https://github.com/nim-lang/Nim/issues/8473))
+- Fixed "smtp.nim(161, 40) Error: type mismatch: got <typeof(nil)> but expected 'SslContext = void'"
+  ([#15177](https://github.com/nim-lang/Nim/issues/15177))
+- Fixed "[strscans] scanf doesn't match a single character with $+ if it's the end of the string"
+  ([#15064](https://github.com/nim-lang/Nim/issues/15064))
+- Fixed "Crash and incorrect return values when using readPasswordFromStdin on Windows."
+  ([#15207](https://github.com/nim-lang/Nim/issues/15207))
+- Fixed "Possible capture error with fieldPairs and genericParams"
+  ([#15221](https://github.com/nim-lang/Nim/issues/15221))
+- Fixed "The StmtList processing of template parameters can lead to unexpected errors"
+  ([#5691](https://github.com/nim-lang/Nim/issues/5691))
+- Fixed "[ARC] C compiler error when passing a var openArray to a sink openArray"
+  ([#15035](https://github.com/nim-lang/Nim/issues/15035))
+- Fixed "Inconsistent unsigned -> signed RangeDefect usage across integer sizes"
+  ([#15210](https://github.com/nim-lang/Nim/issues/15210))
+- Fixed "toHex results in RangeDefect exception when used with large uint64"
+  ([#15257](https://github.com/nim-lang/Nim/issues/15257))
+- Fixed "Arc sink arg crash"
+  ([#15238](https://github.com/nim-lang/Nim/issues/15238))
+- Fixed "SQL escape in db_mysql is not enough"
+  ([#15219](https://github.com/nim-lang/Nim/issues/15219))
+- Fixed "Mixing 'return' with expressions is allowed in 1.2"
+  ([#15280](https://github.com/nim-lang/Nim/issues/15280))
+- Fixed "os.getFileInfo() causes ICE with --gc:arc on Windows"
+  ([#15286](https://github.com/nim-lang/Nim/issues/15286))
+- Fixed "[ARC] Sequence "disappears" with a table inside of a table with an object variant"
+  ([#15122](https://github.com/nim-lang/Nim/issues/15122))
+- Fixed "Documentation regression jsre module missing"
+  ([#15183](https://github.com/nim-lang/Nim/issues/15183))
+- Fixed "CountTable.smallest/largest() on empty table either asserts or gives bogus answer"
+  ([#15021](https://github.com/nim-lang/Nim/issues/15021))
+- Fixed "[Regression] Parser regression"
+  ([#15305](https://github.com/nim-lang/Nim/issues/15305))
+- Fixed "[ARC] SIGSEGV with tuple unpacking caused by cursor inference"
+  ([#15147](https://github.com/nim-lang/Nim/issues/15147))
+- Fixed "LwIP/FreeRTOS compile error - missing SIGPIPE and more "
+  ([#15302](https://github.com/nim-lang/Nim/issues/15302))
+- Fixed "Memory leaks with async (closure iterators?) under ORC"
+  ([#15076](https://github.com/nim-lang/Nim/issues/15076))
+- Fixed "Bug compiling with --gc:arg or --gc:orc"
+  ([#15325](https://github.com/nim-lang/Nim/issues/15325))
+- Fixed "memory corruption in tmarshall.nim"
+  ([#9754](https://github.com/nim-lang/Nim/issues/9754))
+- Fixed "typed macros break generic proc definitions"
+  ([#15326](https://github.com/nim-lang/Nim/issues/15326))
+- Fixed "nim doc2 ignores --docSeeSrcUrl parameter"
+  ([#6071](https://github.com/nim-lang/Nim/issues/6071))
+- Fixed "The decodeData Iterator from cgi module crash"
+  ([#15369](https://github.com/nim-lang/Nim/issues/15369))
+- Fixed "|| iterator generates invalid code when compiling with --debugger:native"
+  ([#9710](https://github.com/nim-lang/Nim/issues/9710))
+- Fixed "Wrong number of variables"
+  ([#15360](https://github.com/nim-lang/Nim/issues/15360))
+- Fixed "Coercions with distinct types should traverse pointer modifiers transparently."
+  ([#7165](https://github.com/nim-lang/Nim/issues/7165))
+- Fixed "Error with distinct generic TableRef"
+  ([#6060](https://github.com/nim-lang/Nim/issues/6060))
+- Fixed "Support images in nim docgen"
+  ([#6430](https://github.com/nim-lang/Nim/issues/6430))
+- Fixed "Regression. Double sem check for procs."
+  ([#15389](https://github.com/nim-lang/Nim/issues/15389))
+- Fixed "uri.nim url with literal ipv6 address is printed wrong, and cannot parsed again"
+  ([#15333](https://github.com/nim-lang/Nim/issues/15333))
+- Fixed "[ARC] Object variant gets corrupted with cursor inference"
+  ([#15361](https://github.com/nim-lang/Nim/issues/15361))
+- Fixed "`nim doc ..` compiler crash (regression 0.19.6 => 1.0)"
+  ([#14474](https://github.com/nim-lang/Nim/issues/14474))
+- Fixed "cannot borrow result; what it borrows from is potentially mutated"
+  ([#15403](https://github.com/nim-lang/Nim/issues/15403))
+- Fixed "memory corruption for seq.add(seq) with gc:arc and d:useMalloc "
+  ([#14983](https://github.com/nim-lang/Nim/issues/14983))
+- Fixed "DocGen HTML output appears improperly when encountering text immediately after/before inline monospace; in some cases won't compile"
+  ([#11537](https://github.com/nim-lang/Nim/issues/11537))
+- Fixed "Deepcopy in arc crashes"
+  ([#15405](https://github.com/nim-lang/Nim/issues/15405))
+- Fixed "pop pragma takes invalid input"
+  ([#15430](https://github.com/nim-lang/Nim/issues/15430))
+- Fixed "tests/stdlib/tgetprotobyname fails on NetBSD"
+  ([#15452](https://github.com/nim-lang/Nim/issues/15452))
+- Fixed "defer doesnt work with block, break and await"
+  ([#15243](https://github.com/nim-lang/Nim/issues/15243))
+- Fixed "tests/stdlib/tssl failing on NetBSD"
+  ([#15493](https://github.com/nim-lang/Nim/issues/15493))
+- Fixed "strictFuncs doesn't seem to catch simple ref mutation"
+  ([#15508](https://github.com/nim-lang/Nim/issues/15508))
+- Fixed "Sizeof of case object is incorrect. Showstopper"
+  ([#15516](https://github.com/nim-lang/Nim/issues/15516))
+- Fixed "[ARC] Internal error when trying to use a parallel for loop"
+  ([#15512](https://github.com/nim-lang/Nim/issues/15512))
+- Fixed "[ARC] Type-bound assign op is not being generated"
+  ([#15510](https://github.com/nim-lang/Nim/issues/15510))
+- Fixed "[ARC] Crash when adding openArray proc argument to a local seq"
+  ([#15511](https://github.com/nim-lang/Nim/issues/15511))
+- Fixed "VM: const case object gets some fields zeroed out at runtime"
+  ([#13081](https://github.com/nim-lang/Nim/issues/13081))
+- Fixed "regression(1.2.6 => devel): VM: const case object field access gives: 'sons' is not accessible"
+  ([#15532](https://github.com/nim-lang/Nim/issues/15532))
+- Fixed "Csources: huge size increase (x2.3) in 0.20"
+  ([#12027](https://github.com/nim-lang/Nim/issues/12027))
+- Fixed "Out of date error message for GC options"
+  ([#15547](https://github.com/nim-lang/Nim/issues/15547))
+- Fixed "dbQuote additional escape regression"
+  ([#15560](https://github.com/nim-lang/Nim/issues/15560))

changelogs/changelog_X_XX_X.md

@@ -1,33 +1,20 @@
-# vx.xx.x - yyyy-mm-dd
+# v1.xx.x - yyyy-mm-dd
+
+This is an example file.
+The changes should go to changelog.md!
 
 
-## Changes affecting backwards compatibility
+## Standard library additions and changes
 
-- Example item: `Foo` changed to `Bar`.
-
-
-### Breaking changes in the standard library
-
-
-### Breaking changes in the compiler
-
-
-## Library additions
-
-
-## Library changes
-
-
-## Language additions
+- Added `example.exampleProc`.
+- Changed `example.foo` to take additional `bar` parameter.
 
 
 ## Language changes
 
 
-### Tool changes
+## Compiler changes
 
 
-### Compiler changes
+## Tool changes
 
-
-## Bugfixes

compiler/aliases.nim

@@ -22,17 +22,17 @@ proc isPartOfAux(n: PNode, b: PType, marker: var IntSet): TAnalysisResult =
   result = arNo
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      result = isPartOfAux(n.sons[i], b, marker)
+    for i in 0..<n.len:
+      result = isPartOfAux(n[i], b, marker)
       if result == arYes: return
   of nkRecCase:
-    assert(n.sons[0].kind == nkSym)
-    result = isPartOfAux(n.sons[0], b, marker)
+    assert(n[0].kind == nkSym)
+    result = isPartOfAux(n[0], b, marker)
     if result == arYes: return
-    for i in 1 ..< sonsLen(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = isPartOfAux(lastSon(n.sons[i]), b, marker)
+        result = isPartOfAux(lastSon(n[i]), b, marker)
         if result == arYes: return
       else: discard "isPartOfAux(record case branch)"
   of nkSym:
@@ -46,14 +46,14 @@ proc isPartOfAux(a, b: PType, marker: var IntSet): TAnalysisResult =
   if compareTypes(a, b, dcEqIgnoreDistinct): return arYes
   case a.kind
   of tyObject:
-    if a.sons[0] != nil:
-      result = isPartOfAux(a.sons[0].skipTypes(skipPtrs), b, marker)
+    if a[0] != nil:
+      result = isPartOfAux(a[0].skipTypes(skipPtrs), b, marker)
     if result == arNo: result = isPartOfAux(a.n, b, marker)
   of tyGenericInst, tyDistinct, tyAlias, tySink:
     result = isPartOfAux(lastSon(a), b, marker)
   of tyArray, tySet, tyTuple:
-    for i in 0 ..< sonsLen(a):
-      result = isPartOfAux(a.sons[i], b, marker)
+    for i in 0..<a.len:
+      result = isPartOfAux(a[i], b, marker)
       if result == arYes: return
   else: discard
 
@@ -108,7 +108,7 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
           result = arMaybe
     of nkBracketExpr:
       result = isPartOf(a[0], b[0])
-      if len(a) >= 2 and len(b) >= 2:
+      if a.len >= 2 and b.len >= 2:
         # array accesses:
         if result == arYes and isDeepConstExpr(a[1]) and isDeepConstExpr(b[1]):
           # we know it's the same array and we have 2 constant indexes;
@@ -181,16 +181,19 @@ proc isPartOf*(a, b: PNode): TAnalysisResult =
       else: discard
     of nkObjConstr:
       result = arNo
-      for i in 1 ..< b.len:
+      for i in 1..<b.len:
         let res = isPartOf(a, b[i][1])
         if res != arNo:
           result = res
           if res == arYes: break
     of nkCallKinds:
       result = arNo
-      for i in 1 ..< b.len:
+      for i in 1..<b.len:
         let res = isPartOf(a, b[i])
         if res != arNo:
           result = res
           if res == arYes: break
+    of nkBracket:
+      if b.len > 0:
+        result = isPartOf(a, b[0])
     else: discard

compiler/asciitables.nim

@@ -71,7 +71,7 @@ iterator parseTableCells*(s: string, delim = '\t'): Cell =
 proc alignTable*(s: string, delim = '\t', fill = ' ', sep = " "): string =
   ## formats a `delim`-delimited `s` representing a table; each cell is aligned
   ## to a width that's computed for each column; consecutive columns are
-  ## delimted by `sep`, and alignment space is filled using `fill`.
+  ## delimited by `sep`, and alignment space is filled using `fill`.
   ## More customized formatting can be done by calling `parseTableCells` directly.
   for cell in parseTableCells(s, delim):
     result.add cell.text

compiler/astalgo.nim

@@ -24,46 +24,55 @@ proc typeToYaml*(conf: ConfigRef; n: PType, indent: int = 0, maxRecDepth: int =
 proc symToYaml*(conf: ConfigRef; n: PSym, indent: int = 0, maxRecDepth: int = - 1): Rope
 proc lineInfoToStr*(conf: ConfigRef; info: TLineInfo): Rope
 
-when declared(echo):
-  # these are for debugging only: They are not really deprecated, but I want
-  # the warning so that release versions do not contain debugging statements:
-  proc debug*(n: PSym; conf: ConfigRef = nil) {.exportc: "debugSym", deprecated.}
-  proc debug*(n: PType; conf: ConfigRef = nil) {.exportc: "debugType", deprecated.}
-  proc debug*(n: PNode; conf: ConfigRef = nil) {.exportc: "debugNode", deprecated.}
 
-  template debug*(x: PSym|PType|PNode) {.deprecated.} =
-    when compiles(c.config):
-      debug(c.config, x)
-    elif compiles(c.graph.config):
-      debug(c.graph.config, x)
+# these are for debugging only: They are not really deprecated, but I want
+# the warning so that release versions do not contain debugging statements:
+proc debug*(n: PSym; conf: ConfigRef = nil) {.exportc: "debugSym", deprecated.}
+proc debug*(n: PType; conf: ConfigRef = nil) {.exportc: "debugType", deprecated.}
+proc debug*(n: PNode; conf: ConfigRef = nil) {.exportc: "debugNode", deprecated.}
+
+proc typekinds*(t: PType) {.deprecated.} =
+  var t = t
+  var s = ""
+  while t != nil and t.len > 0:
+    s.add $t.kind
+    s.add " "
+    t = t.lastSon
+  echo s
+
+template debug*(x: PSym|PType|PNode) {.deprecated.} =
+  when compiles(c.config):
+    debug(c.config, x)
+  elif compiles(c.graph.config):
+    debug(c.graph.config, x)
+  else:
+    error()
+
+template debug*(x: auto) {.deprecated.} =
+  echo x
+
+template mdbg*: bool {.deprecated.} =
+  when compiles(c.graph):
+    c.module.fileIdx == c.graph.config.projectMainIdx
+  elif compiles(c.module):
+    c.module.fileIdx == c.config.projectMainIdx
+  elif compiles(c.c.module):
+    c.c.module.fileIdx == c.c.config.projectMainIdx
+  elif compiles(m.c.module):
+    m.c.module.fileIdx == m.c.config.projectMainIdx
+  elif compiles(cl.c.module):
+    cl.c.module.fileIdx == cl.c.config.projectMainIdx
+  elif compiles(p):
+    when compiles(p.lex):
+      p.lex.fileIdx == p.lex.config.projectMainIdx
     else:
-      error()
-
-  template debug*(x: auto) {.deprecated.} =
-    echo x
-
-  template mdbg*: bool {.deprecated.} =
-    when compiles(c.graph):
-      c.module.fileIdx == c.graph.config.projectMainIdx
-    elif compiles(c.module):
-      c.module.fileIdx == c.config.projectMainIdx
-    elif compiles(c.c.module):
-      c.c.module.fileIdx == c.c.config.projectMainIdx
-    elif compiles(m.c.module):
-      m.c.module.fileIdx == m.c.config.projectMainIdx
-    elif compiles(cl.c.module):
-      cl.c.module.fileIdx == cl.c.config.projectMainIdx
-    elif compiles(p):
-      when compiles(p.lex):
-        p.lex.fileIdx == p.lex.config.projectMainIdx
-      else:
-        p.module.module.fileIdx == p.config.projectMainIdx
-    elif compiles(m.module.fileIdx):
-      m.module.fileIdx == m.config.projectMainIdx
-    elif compiles(L.fileIdx):
-      L.fileIdx == L.config.projectMainIdx
-    else:
-      error()
+      p.module.module.fileIdx == p.config.projectMainIdx
+  elif compiles(m.module.fileIdx):
+    m.module.fileIdx == m.config.projectMainIdx
+  elif compiles(L.fileIdx):
+    L.fileIdx == L.config.projectMainIdx
+  else:
+    error()
 
 # --------------------------- ident tables ----------------------------------
 proc idTableGet*(t: TIdTable, key: PIdObj): RootRef
@@ -77,7 +86,6 @@ proc idNodeTablePut*(t: var TIdNodeTable, key: PIdObj, val: PNode)
 
 # ---------------------------------------------------------------------------
 
-proc getSymFromList*(list: PNode, ident: PIdent, start: int = 0): PSym
 proc lookupInRecord*(n: PNode, field: PIdent): PSym
 proc mustRehash*(length, counter: int): bool
 proc nextTry*(h, maxHash: Hash): Hash {.inline.}
@@ -102,22 +110,22 @@ proc iiTablePut*(t: var TIITable, key, val: int)
 
 # implementation
 
-proc skipConvAndClosure*(n: PNode): PNode =
+proc skipConvCastAndClosure*(n: PNode): PNode =
   result = n
   while true:
     case result.kind
     of nkObjUpConv, nkObjDownConv, nkChckRange, nkChckRangeF, nkChckRange64,
        nkClosure:
-      result = result.sons[0]
-    of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-      result = result.sons[1]
+      result = result[0]
+    of nkHiddenStdConv, nkHiddenSubConv, nkConv, nkCast:
+      result = result[1]
     else: break
 
 proc sameValue*(a, b: PNode): bool =
   result = false
   case a.kind
   of nkCharLit..nkUInt64Lit:
-    if b.kind in {nkCharLit..nkUInt64Lit}: result = a.intVal == b.intVal
+    if b.kind in {nkCharLit..nkUInt64Lit}: result = getInt(a) == getInt(b)
   of nkFloatLit..nkFloat64Lit:
     if b.kind in {nkFloatLit..nkFloat64Lit}: result = a.floatVal == b.floatVal
   of nkStrLit..nkTripleStrLit:
@@ -131,8 +139,8 @@ proc leValue*(a, b: PNode): bool =
   # a <= b?
   result = false
   case a.kind
-  of nkCharLit..nkUInt32Lit:
-    if b.kind in {nkCharLit..nkUInt32Lit}: result = a.intVal <= b.intVal
+  of nkCharLit..nkUInt64Lit:
+    if b.kind in {nkCharLit..nkUInt64Lit}: result = getInt(a) <= getInt(b)
   of nkFloatLit..nkFloat64Lit:
     if b.kind in {nkFloatLit..nkFloat64Lit}: result = a.floatVal <= b.floatVal
   of nkStrLit..nkTripleStrLit:
@@ -152,17 +160,17 @@ proc lookupInRecord(n: PNode, field: PIdent): PSym =
   result = nil
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      result = lookupInRecord(n.sons[i], field)
+    for i in 0..<n.len:
+      result = lookupInRecord(n[i], field)
       if result != nil: return
   of nkRecCase:
-    if (n.sons[0].kind != nkSym): return nil
-    result = lookupInRecord(n.sons[0], field)
+    if (n[0].kind != nkSym): return nil
+    result = lookupInRecord(n[0], field)
     if result != nil: return
-    for i in 1 ..< sonsLen(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = lookupInRecord(lastSon(n.sons[i]), field)
+        result = lookupInRecord(lastSon(n[i]), field)
         if result != nil: return
       else: return nil
   of nkSym:
@@ -174,14 +182,55 @@ proc getModule*(s: PSym): PSym =
   assert((result.kind == skModule) or (result.owner != result))
   while result != nil and result.kind != skModule: result = result.owner
 
-proc getSymFromList(list: PNode, ident: PIdent, start: int = 0): PSym =
-  for i in start ..< sonsLen(list):
-    if list.sons[i].kind == nkSym:
-      result = list.sons[i].sym
+proc fromSystem*(op: PSym): bool {.inline.} = sfSystemModule in getModule(op).flags
+proc getSymFromList*(list: PNode, ident: PIdent, start: int = 0): PSym =
+  for i in start..<list.len:
+    if list[i].kind == nkSym:
+      result = list[i].sym
       if result.name.id == ident.id: return
     else: return nil
   result = nil
 
+proc sameIgnoreBacktickGensymInfo(a, b: string): bool =
+  if a[0] != b[0]: return false
+  var alen = a.len - 1
+  while alen > 0 and a[alen] != '`': dec(alen)
+  if alen <= 0: alen = a.len
+
+  var i = 1
+  var j = 1
+  while true:
+    while i < alen and a[i] == '_': inc i
+    while j < b.len and b[j] == '_': inc j
+    var aa = if i < alen: toLowerAscii(a[i]) else: '\0'
+    var bb = if j < b.len: toLowerAscii(b[j]) else: '\0'
+    if aa != bb: return false
+
+    # the characters are identical:
+    if i >= alen:
+      # both cursors at the end:
+      if j >= b.len: return true
+      # not yet at the end of 'b':
+      return false
+    elif j >= b.len:
+      return false
+    inc i
+    inc j
+
+proc getNamedParamFromList*(list: PNode, ident: PIdent): PSym =
+  ## Named parameters are special because a named parameter can be
+  ## gensym'ed and then they have '\`<number>' suffix that we need to
+  ## ignore, see compiler / evaltempl.nim, snippet:
+  ##
+  ##..code-block:: nim
+  ##
+  ##    result.add newIdentNode(getIdent(c.ic, x.name.s & "\`gensym" & $x.id),
+  ##            if c.instLines: actual.info else: templ.info)
+  for i in 1..<list.len:
+    let it = list[i].sym
+    if it.name.id == ident.id or
+        sameIgnoreBacktickGensymInfo(it.name.s, ident.s): return it
+
 proc hashNode(p: RootRef): Hash =
   result = hash(cast[pointer](p))
 
@@ -206,15 +255,15 @@ proc makeYamlString*(s: string): Rope =
   const MaxLineLength = 64
   result = nil
   var res = "\""
-  for i in 0 ..< s.len:
+  for i in 0..<s.len:
     if (i + 1) mod MaxLineLength == 0:
-      add(res, '\"')
-      add(res, "\n")
-      add(result, rope(res))
+      res.add('\"')
+      res.add("\n")
+      result.add(rope(res))
       res = "\""              # reset
-    add(res, toYamlChar(s[i]))
-  add(res, '\"')
-  add(result, rope(res))
+    res.add(toYamlChar(s[i]))
+  res.add('\"')
+  result.add(rope(res))
 
 proc flagsToStr[T](flags: set[T]): Rope =
   if flags == {}:
@@ -222,8 +271,8 @@ proc flagsToStr[T](flags: set[T]): Rope =
   else:
     result = nil
     for x in items(flags):
-      if result != nil: add(result, ", ")
-      add(result, makeYamlString($x))
+      if result != nil: result.add(", ")
+      result.add(makeYamlString($x))
     result = "[" & result & "]"
 
 proc lineInfoToStr(conf: ConfigRef; info: TLineInfo): Rope =
@@ -250,25 +299,25 @@ proc symToYamlAux(conf: ConfigRef; n: PSym, marker: var IntSet, indent: int,
                                  #  indent + 2, maxRecDepth - 1),
     let istr = rspaces(indent + 2)
     result = rope("{")
-    addf(result, "$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
-    addf(result, "$N$1\"name\": $2", [istr, makeYamlString(n.name.s)])
-    addf(result, "$N$1\"typ\": $2", [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
+    result.addf("$N$1\"name\": $2", [istr, makeYamlString(n.name.s)])
+    result.addf("$N$1\"typ\": $2", [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth - 1)])
     if conf != nil:
       # if we don't pass the config, we probably don't care about the line info
-      addf(result, "$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
+      result.addf("$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
     if card(n.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
-    addf(result, "$N$1\"magic\": $2", [istr, makeYamlString($n.magic)])
-    addf(result, "$N$1\"ast\": $2", [istr, ast])
-    addf(result, "$N$1\"options\": $2", [istr, flagsToStr(n.options)])
-    addf(result, "$N$1\"position\": $2", [istr, rope(n.position)])
-    addf(result, "$N$1\"k\": $2", [istr, makeYamlString($n.loc.k)])
-    addf(result, "$N$1\"storage\": $2", [istr, makeYamlString($n.loc.storage)])
+      result.addf("$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
+    result.addf("$N$1\"magic\": $2", [istr, makeYamlString($n.magic)])
+    result.addf("$N$1\"ast\": $2", [istr, ast])
+    result.addf("$N$1\"options\": $2", [istr, flagsToStr(n.options)])
+    result.addf("$N$1\"position\": $2", [istr, rope(n.position)])
+    result.addf("$N$1\"k\": $2", [istr, makeYamlString($n.loc.k)])
+    result.addf("$N$1\"storage\": $2", [istr, makeYamlString($n.loc.storage)])
     if card(n.loc.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, makeYamlString($n.loc.flags)])
-    addf(result, "$N$1\"r\": $2", [istr, n.loc.r])
-    addf(result, "$N$1\"lode\": $2", [istr, treeToYamlAux(conf, n.loc.lode, marker, indent + 2, maxRecDepth - 1)])
-    addf(result, "$N$1}", [rspaces(indent)])
+      result.addf("$N$1\"flags\": $2", [istr, makeYamlString($n.loc.flags)])
+    result.addf("$N$1\"r\": $2", [istr, n.loc.r])
+    result.addf("$N$1\"lode\": $2", [istr, treeToYamlAux(conf, n.loc.lode, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1}", [rspaces(indent)])
 
 proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int,
                    maxRecDepth: int): Rope =
@@ -279,27 +328,27 @@ proc typeToYamlAux(conf: ConfigRef; n: PType, marker: var IntSet, indent: int,
     sonsRope = "\"$1 @$2\"" % [rope($n.kind), rope(
         strutils.toHex(cast[ByteAddress](n), sizeof(n) * 2))]
   else:
-    if sonsLen(n) > 0:
+    if n.len > 0:
       sonsRope = rope("[")
-      for i in 0 ..< sonsLen(n):
-        if i > 0: add(sonsRope, ",")
-        addf(sonsRope, "$N$1$2", [rspaces(indent + 4), typeToYamlAux(conf, n.sons[i],
+      for i in 0..<n.len:
+        if i > 0: sonsRope.add(",")
+        sonsRope.addf("$N$1$2", [rspaces(indent + 4), typeToYamlAux(conf, n[i],
             marker, indent + 4, maxRecDepth - 1)])
-      addf(sonsRope, "$N$1]", [rspaces(indent + 2)])
+      sonsRope.addf("$N$1]", [rspaces(indent + 2)])
     else:
       sonsRope = rope("null")
 
     let istr = rspaces(indent + 2)
     result = rope("{")
-    addf(result, "$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
-    addf(result, "$N$1\"sym\": $2",  [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth - 1)])
-    addf(result, "$N$1\"n\": $2",     [istr, treeToYamlAux(conf, n.n, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1\"kind\": $2", [istr, makeYamlString($n.kind)])
+    result.addf("$N$1\"sym\": $2",  [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth - 1)])
+    result.addf("$N$1\"n\": $2",     [istr, treeToYamlAux(conf, n.n, marker, indent + 2, maxRecDepth - 1)])
     if card(n.flags) > 0:
-      addf(result, "$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
-    addf(result, "$N$1\"callconv\": $2", [istr, makeYamlString(CallingConvToStr[n.callConv])])
-    addf(result, "$N$1\"size\": $2", [istr, rope(n.size)])
-    addf(result, "$N$1\"align\": $2", [istr, rope(n.align)])
-    addf(result, "$N$1\"sons\": $2", [istr, sonsRope])
+      result.addf("$N$1\"flags\": $2", [istr, flagsToStr(n.flags)])
+    result.addf("$N$1\"callconv\": $2", [istr, makeYamlString($n.callConv)])
+    result.addf("$N$1\"size\": $2", [istr, rope(n.size)])
+    result.addf("$N$1\"align\": $2", [istr, rope(n.align)])
+    result.addf("$N$1\"sons\": $2", [istr, sonsRope])
 
 proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
                    maxRecDepth: int): Rope =
@@ -310,34 +359,34 @@ proc treeToYamlAux(conf: ConfigRef; n: PNode, marker: var IntSet, indent: int,
     result = "{$N$1\"kind\": $2" % [istr, makeYamlString($n.kind)]
     if maxRecDepth != 0:
       if conf != nil:
-        addf(result, ",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
+        result.addf(",$N$1\"info\": $2", [istr, lineInfoToStr(conf, n.info)])
       case n.kind
       of nkCharLit..nkInt64Lit:
-        addf(result, ",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
+        result.addf(",$N$1\"intVal\": $2", [istr, rope(n.intVal)])
       of nkFloatLit, nkFloat32Lit, nkFloat64Lit:
-        addf(result, ",$N$1\"floatVal\": $2",
+        result.addf(",$N$1\"floatVal\": $2",
             [istr, rope(n.floatVal.toStrMaxPrecision)])
       of nkStrLit..nkTripleStrLit:
-        addf(result, ",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
+        result.addf(",$N$1\"strVal\": $2", [istr, makeYamlString(n.strVal)])
       of nkSym:
-        addf(result, ",$N$1\"sym\": $2",
+        result.addf(",$N$1\"sym\": $2",
              [istr, symToYamlAux(conf, n.sym, marker, indent + 2, maxRecDepth)])
       of nkIdent:
         if n.ident != nil:
-          addf(result, ",$N$1\"ident\": $2", [istr, makeYamlString(n.ident.s)])
+          result.addf(",$N$1\"ident\": $2", [istr, makeYamlString(n.ident.s)])
         else:
-          addf(result, ",$N$1\"ident\": null", [istr])
+          result.addf(",$N$1\"ident\": null", [istr])
       else:
-        if sonsLen(n) > 0:
-          addf(result, ",$N$1\"sons\": [", [istr])
-          for i in 0 ..< sonsLen(n):
-            if i > 0: add(result, ",")
-            addf(result, "$N$1$2", [rspaces(indent + 4), treeToYamlAux(conf, n.sons[i],
+        if n.len > 0:
+          result.addf(",$N$1\"sons\": [", [istr])
+          for i in 0..<n.len:
+            if i > 0: result.add(",")
+            result.addf("$N$1$2", [rspaces(indent + 4), treeToYamlAux(conf, n[i],
                 marker, indent + 4, maxRecDepth - 1)])
-          addf(result, "$N$1]", [istr])
-      addf(result, ",$N$1\"typ\": $2",
+          result.addf("$N$1]", [istr])
+      result.addf(",$N$1\"typ\": $2",
            [istr, typeToYamlAux(conf, n.typ, marker, indent + 2, maxRecDepth)])
-    addf(result, "$N$1}", [rspaces(indent)])
+    result.addf("$N$1}", [rspaces(indent)])
 
 proc treeToYaml(conf: ConfigRef; n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope =
   var marker = initIntSet()
@@ -379,7 +428,7 @@ proc indentLess(this: var DebugPrinter) =
 proc newlineAndIndent(this: var DebugPrinter) =
   this.res.add "\n"
   this.currentLine += 1
-  for i in 0 ..< this.indent:
+  for i in 0..<this.indent:
     this.res.add ' '
 
 proc openCurly(this: var DebugPrinter) =
@@ -515,12 +564,12 @@ proc value(this: var DebugPrinter; value: PType) =
     this.key "n"
     this.value value.n
 
-  if sonsLen(value) > 0:
+  if value.len > 0:
     this.key "sons"
     this.openBracket
-    for i in 0 ..< sonsLen(value):
-      this.value value.sons[i]
-      if i != sonsLen(value) - 1:
+    for i in 0..<value.len:
+      this.value value[i]
+      if i != value.len - 1:
         this.comma
     this.closeBracket
 
@@ -542,10 +591,17 @@ proc value(this: var DebugPrinter; value: PNode) =
   if this.conf != nil:
     this.key "info"
     this.value $lineInfoToStr(this.conf, value.info)
-  if card(value.flags) > 0:
+  if value.flags != {}:
     this.key "flags"
     this.value value.flags
 
+  if value.typ != nil:
+    this.key "typ"
+    this.value value.typ.kind
+  else:
+    this.key "typ"
+    this.value "nil"
+
   case value.kind
   of nkCharLit..nkUInt64Lit:
     this.key "intVal"
@@ -568,41 +624,41 @@ proc value(this: var DebugPrinter; value: PNode) =
     if this.renderSymType and value.typ != nil:
       this.key "typ"
       this.value value.typ
-    if sonsLen(value) > 0:
+    if value.len > 0:
       this.key "sons"
       this.openBracket
-      for i in 0 ..< sonsLen(value):
-        this.value value.sons[i]
-        if i != sonsLen(value) - 1:
+      for i in 0..<value.len:
+        this.value value[i]
+        if i != value.len - 1:
           this.comma
       this.closeBracket
 
   this.closeCurly
 
-when declared(echo):
-  proc debug(n: PSym; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
 
-  proc debug(n: PType; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
+proc debug(n: PSym; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
 
-  proc debug(n: PNode; conf: ConfigRef) =
-    var this: DebugPrinter
-    this.visited = initTable[pointer, int]()
-    #this.renderSymType = true
-    this.useColor = not defined(windows)
-    this.value(n)
-    echo($this.res)
+proc debug(n: PType; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
+
+proc debug(n: PNode; conf: ConfigRef) =
+  var this: DebugPrinter
+  this.visited = initTable[pointer, int]()
+  #this.renderSymType = true
+  this.useColor = not defined(windows)
+  this.value(n)
+  echo($this.res)
 
 proc nextTry(h, maxHash: Hash): Hash =
   result = ((5 * h) + 1) and maxHash
@@ -629,13 +685,13 @@ proc objectSetRawInsert(data: var TObjectSeq, obj: RootRef) =
 
 proc objectSetEnlarge(t: var TObjectSet) =
   var n: TObjectSeq
-  newSeq(n, len(t.data) * GrowthFactor)
-  for i in 0 .. high(t.data):
+  newSeq(n, t.data.len * GrowthFactor)
+  for i in 0..high(t.data):
     if t.data[i] != nil: objectSetRawInsert(n, t.data[i])
   swap(t.data, n)
 
 proc objectSetIncl*(t: var TObjectSet, obj: RootRef) =
-  if mustRehash(len(t.data), t.counter): objectSetEnlarge(t)
+  if mustRehash(t.data.len, t.counter): objectSetEnlarge(t)
   objectSetRawInsert(t.data, obj)
   inc(t.counter)
 
@@ -648,7 +704,7 @@ proc objectSetContainsOrIncl*(t: var TObjectSet, obj: RootRef): bool =
     if it == obj:
       return true             # found it
     h = nextTry(h, high(t.data))
-  if mustRehash(len(t.data), t.counter):
+  if mustRehash(t.data.len, t.counter):
     objectSetEnlarge(t)
     objectSetRawInsert(t.data, obj)
   else:
@@ -691,13 +747,13 @@ proc symTabReplace*(t: var TStrTable, prevSym: PSym, newSym: PSym) =
 
 proc strTableEnlarge(t: var TStrTable) =
   var n: seq[PSym]
-  newSeq(n, len(t.data) * GrowthFactor)
-  for i in 0 .. high(t.data):
+  newSeq(n, t.data.len * GrowthFactor)
+  for i in 0..high(t.data):
     if t.data[i] != nil: strTableRawInsert(n, t.data[i])
   swap(t.data, n)
 
 proc strTableAdd*(t: var TStrTable, n: PSym) =
-  if mustRehash(len(t.data), t.counter): strTableEnlarge(t)
+  if mustRehash(t.data.len, t.counter): strTableEnlarge(t)
   strTableRawInsert(t.data, n)
   inc(t.counter)
 
@@ -724,7 +780,7 @@ proc strTableInclReportConflict*(t: var TStrTable, n: PSym;
     if not onConflictKeepOld:
       t.data[replaceSlot] = n # overwrite it with newer definition!
     return t.data[replaceSlot] # found it
-  elif mustRehash(len(t.data), t.counter):
+  elif mustRehash(t.data.len, t.counter):
     strTableEnlarge(t)
     strTableRawInsert(t.data, n)
   else:
@@ -828,7 +884,7 @@ iterator items*(tab: TStrTable): PSym =
     s = nextIter(it, tab)
 
 proc hasEmptySlot(data: TIdPairSeq): bool =
-  for h in 0 .. high(data):
+  for h in 0..high(data):
     if data[h].key == nil:
       return true
   result = false
@@ -881,9 +937,9 @@ proc idTablePut(t: var TIdTable, key: PIdObj, val: RootRef) =
     assert(t.data[index].key != nil)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(t.data):
+    if mustRehash(t.data.len, t.counter):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(t.data):
         if t.data[i].key != nil:
           idTableRawInsert(n, t.data[i].key, t.data[i].val)
       assert(hasEmptySlot(n))
@@ -892,7 +948,7 @@ proc idTablePut(t: var TIdTable, key: PIdObj, val: RootRef) =
     inc(t.counter)
 
 iterator idTablePairs*(t: TIdTable): tuple[key: PIdObj, val: RootRef] =
-  for i in 0 .. high(t.data):
+  for i in 0..high(t.data):
     if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val)
 
 proc idNodeTableRawGet(t: TIdNodeTable, key: PIdObj): int =
@@ -926,10 +982,10 @@ proc idNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) =
     assert(t.data[index].key != nil)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
+    if mustRehash(t.data.len, t.counter):
       var n: TIdNodePairSeq
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(t.data):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(t.data):
         if t.data[i].key != nil:
           idNodeTableRawInsert(n, t.data[i].key, t.data[i].val)
       swap(t.data, n)
@@ -937,13 +993,13 @@ proc idNodeTablePut(t: var TIdNodeTable, key: PIdObj, val: PNode) =
     inc(t.counter)
 
 iterator pairs*(t: TIdNodeTable): tuple[key: PIdObj, val: PNode] =
-  for i in 0 .. high(t.data):
+  for i in 0..high(t.data):
     if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val)
 
 proc initIITable(x: var TIITable) =
   x.counter = 0
   newSeq(x.data, StartSize)
-  for i in 0 ..< StartSize: x.data[i].key = InvalidKey
+  for i in 0..<StartSize: x.data[i].key = InvalidKey
 
 proc iiTableRawGet(t: TIITable, key: int): int =
   var h: Hash
@@ -974,11 +1030,11 @@ proc iiTablePut(t: var TIITable, key, val: int) =
     assert(t.data[index].key != InvalidKey)
     t.data[index].val = val
   else:
-    if mustRehash(len(t.data), t.counter):
+    if mustRehash(t.data.len, t.counter):
       var n: TIIPairSeq
-      newSeq(n, len(t.data) * GrowthFactor)
-      for i in 0 .. high(n): n[i].key = InvalidKey
-      for i in 0 .. high(t.data):
+      newSeq(n, t.data.len * GrowthFactor)
+      for i in 0..high(n): n[i].key = InvalidKey
+      for i in 0..high(t.data):
         if t.data[i].key != InvalidKey:
           iiTableRawInsert(n, t.data[i].key, t.data[i].val)
       swap(t.data, n)
@@ -992,3 +1048,14 @@ proc isAddrNode*(n: PNode): bool =
       if n[0].kind == nkSym and n[0].sym.magic == mAddr: true
       else: false
     else: false
+
+proc listSymbolNames*(symbols: openArray[PSym]): string =
+  for sym in symbols:
+    if result.len > 0:
+      result.add ", "
+    result.add sym.name.s
+
+proc isDiscriminantField*(n: PNode): bool =
+  if n.kind == nkCheckedFieldExpr: sfDiscriminant in n[0][1].sym.flags
+  elif n.kind == nkDotExpr: sfDiscriminant in n[1].sym.flags
+  else: false

compiler/bitsets.nim

@@ -52,25 +52,25 @@ proc bitSetInit(b: var TBitSet, length: int) =
   newSeq(b, length)
 
 proc bitSetUnion(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] or y[i]
+  for i in 0..high(x): x[i] = x[i] or y[i]
 
 proc bitSetDiff(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] and not y[i]
+  for i in 0..high(x): x[i] = x[i] and not y[i]
 
 proc bitSetSymDiff(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] xor y[i]
+  for i in 0..high(x): x[i] = x[i] xor y[i]
 
 proc bitSetIntersect(x: var TBitSet, y: TBitSet) =
-  for i in 0 .. high(x): x[i] = x[i] and y[i]
+  for i in 0..high(x): x[i] = x[i] and y[i]
 
 proc bitSetEquals(x, y: TBitSet): bool =
-  for i in 0 .. high(x):
+  for i in 0..high(x):
     if x[i] != y[i]:
       return false
   result = true
 
 proc bitSetContains(x, y: TBitSet): bool =
-  for i in 0 .. high(x):
+  for i in 0..high(x):
     if (x[i] and not y[i]) != Zero:
       return false
   result = true

compiler/btrees.nim

@@ -37,34 +37,34 @@ template eq(a, b): bool = cmp(a, b) == 0
 proc getOrDefault*[Key, Val](b: BTree[Key, Val], key: Key): Val =
   var x = b.root
   while x.isInternal:
-    for j in 0 ..< x.entries:
+    for j in 0..<x.entries:
       if j+1 == x.entries or less(key, x.keys[j+1]):
         x = x.links[j]
         break
   assert(not x.isInternal)
-  for j in 0 ..< x.entries:
+  for j in 0..<x.entries:
     if eq(key, x.keys[j]): return x.vals[j]
 
 proc contains*[Key, Val](b: BTree[Key, Val], key: Key): bool =
   var x = b.root
   while x.isInternal:
-    for j in 0 ..< x.entries:
+    for j in 0..<x.entries:
       if j+1 == x.entries or less(key, x.keys[j+1]):
         x = x.links[j]
         break
   assert(not x.isInternal)
-  for j in 0 ..< x.entries:
+  for j in 0..<x.entries:
     if eq(key, x.keys[j]): return true
   return false
 
 proc copyHalf[Key, Val](h, result: Node[Key, Val]) =
-  for j in 0 ..< Mhalf:
+  for j in 0..<Mhalf:
     result.keys[j] = h.keys[Mhalf + j]
   if h.isInternal:
-    for j in 0 ..< Mhalf:
+    for j in 0..<Mhalf:
       result.links[j] = h.links[Mhalf + j]
   else:
-    for j in 0 ..< Mhalf:
+    for j in 0..<Mhalf:
       shallowCopy(result.vals[j], h.vals[Mhalf + j])
 
 proc split[Key, Val](h: Node[Key, Val]): Node[Key, Val] =
@@ -79,6 +79,9 @@ proc insert[Key, Val](h: Node[Key, Val], key: Key, val: Val): Node[Key, Val] =
   var j = 0
   if not h.isInternal:
     while j < h.entries:
+      if eq(key, h.keys[j]):
+        h.vals[j] = val
+        return
       if less(key, h.keys[j]): break
       inc j
     for i in countdown(h.entries, j+1):
@@ -132,8 +135,7 @@ proc `$`[Key, Val](b: BTree[Key, Val]): string =
   result = ""
   toString(b.root, "", result)
 
-proc hasNext*[Key, Val](b: BTree[Key, Val]; index: int): bool =
-  result = index < b.entries
+proc hasNext*[Key, Val](b: BTree[Key, Val]; index: int): bool = index < b.entries
 
 proc countSubTree[Key, Val](it: Node[Key, Val]): int =
   if it.isInternal:
@@ -166,68 +168,3 @@ iterator pairs*[Key, Val](b: BTree[Key, Val]): (Key, Val) =
     yield (k, v)
 
 proc len*[Key, Val](b: BTree[Key, Val]): int {.inline.} = b.entries
-
-when isMainModule:
-
-  import random, tables
-
-  proc main =
-    var st = initBTree[string, string]()
-    st.add("www.cs.princeton.edu", "abc")
-    st.add("www.princeton.edu",    "128.112.128.15")
-    st.add("www.yale.edu",         "130.132.143.21")
-    st.add("www.simpsons.com",     "209.052.165.60")
-    st.add("www.apple.com",        "17.112.152.32")
-    st.add("www.amazon.com",       "207.171.182.16")
-    st.add("www.ebay.com",         "66.135.192.87")
-    st.add("www.cnn.com",          "64.236.16.20")
-    st.add("www.google.com",       "216.239.41.99")
-    st.add("www.nytimes.com",      "199.239.136.200")
-    st.add("www.microsoft.com",    "207.126.99.140")
-    st.add("www.dell.com",         "143.166.224.230")
-    st.add("www.slashdot.org",     "66.35.250.151")
-    st.add("www.espn.com",         "199.181.135.201")
-    st.add("www.weather.com",      "63.111.66.11")
-    st.add("www.yahoo.com",        "216.109.118.65")
-
-    assert st.getOrDefault("www.cs.princeton.edu") == "abc"
-    assert st.getOrDefault("www.harvardsucks.com") == nil
-
-    assert st.getOrDefault("www.simpsons.com") == "209.052.165.60"
-    assert st.getOrDefault("www.apple.com") == "17.112.152.32"
-    assert st.getOrDefault("www.ebay.com") == "66.135.192.87"
-    assert st.getOrDefault("www.dell.com") == "143.166.224.230"
-    assert(st.entries == 16)
-
-    for k, v in st:
-      echo k, ": ", v
-
-    when false:
-      var b2 = initBTree[string, string]()
-      const iters = 10_000
-      for i in 1..iters:
-        b2.add($i, $(iters - i))
-      for i in 1..iters:
-        let x = b2.getOrDefault($i)
-        if x != $(iters - i):
-          echo "got ", x, ", but expected ", iters - i
-      echo b2.entries
-
-    when true:
-      var b2 = initBTree[int, string]()
-      var t2 = initTable[int, string]()
-      const iters = 100_000
-      for i in 1..iters:
-        let x = rand(high(int))
-        if not t2.hasKey(x):
-          doAssert b2.getOrDefault(x).len == 0, " what, tree has this element " & $x
-          t2[x] = $x
-          b2.add(x, $x)
-
-      doAssert b2.entries == t2.len
-      echo "unique entries ", b2.entries
-      for k, v in t2:
-        doAssert $k == v
-        doAssert b2.getOrDefault(k) == $k
-
-  main()

compiler/canonicalizer.nim

@@ -102,7 +102,7 @@ proc hashTree(c: var MD5Context, n: PNode) =
   of nkStrLit..nkTripleStrLit:
     c &= n.strVal
   else:
-    for i in 0..<n.len: hashTree(c, n.sons[i])
+    for i in 0..<n.len: hashTree(c, n[i])
 
 proc hashType(c: var MD5Context, t: PType) =
   # modelled after 'typeToString'
@@ -120,44 +120,44 @@ proc hashType(c: var MD5Context, t: PType) =
 
   case t.kind
   of tyGenericBody, tyGenericInst, tyGenericInvocation:
-    for i in 0 ..< sonsLen(t)-ord(t.kind != tyGenericInvocation):
-      c.hashType t.sons[i]
+    for i in 0..<t.len-ord(t.kind != tyGenericInvocation):
+      c.hashType t[i]
   of tyUserTypeClass:
     internalAssert t.sym != nil and t.sym.owner != nil
     c &= t.sym.owner.name.s
   of tyUserTypeClassInst:
     let body = t.base
     c.hashSym body.sym
-    for i in 1 .. sonsLen(t) - 2:
-      c.hashType t.sons[i]
+    for i in 1..<t.len-1:
+      c.hashType t[i]
   of tyFromExpr:
     c.hashTree(t.n)
   of tyArray:
-    c.hashTree(t.sons[0].n)
-    c.hashType(t.sons[1])
+    c.hashTree(t[0].n)
+    c.hashType(t[1])
   of tyTuple:
     if t.n != nil:
-      assert(sonsLen(t.n) == sonsLen(t))
-      for i in 0 ..< sonsLen(t.n):
-        assert(t.n.sons[i].kind == nkSym)
-        c &= t.n.sons[i].sym.name.s
+      assert(t.n.len == t.len)
+      for i in 0..<t.n.len:
+        assert(t.n[i].kind == nkSym)
+        c &= t.n[i].sym.name.s
         c &= ":"
-        c.hashType(t.sons[i])
+        c.hashType(t[i])
         c &= ","
     else:
-      for i in 0 ..< sonsLen(t): c.hashType t.sons[i]
+      for i in 0..<t.len: c.hashType t[i]
   of tyRange:
     c.hashTree(t.n)
-    c.hashType(t.sons[0])
+    c.hashType(t[0])
   of tyProc:
     c &= (if tfIterator in t.flags: "iterator " else: "proc ")
-    for i in 0..<t.len: c.hashType(t.sons[i])
+    for i in 0..<t.len: c.hashType(t[i])
     md5Update(c, cast[cstring](addr(t.callConv)), 1)
 
     if tfNoSideEffect in t.flags: c &= ".noSideEffect"
     if tfThread in t.flags: c &= ".thread"
   else:
-    for i in 0..<t.len: c.hashType(t.sons[i])
+    for i in 0..<t.len: c.hashType(t[i])
   if tfNotNil in t.flags: c &= "not nil"
 
 proc canonConst(n: PNode): TUid =
@@ -238,35 +238,35 @@ proc encodeNode(w: PRodWriter, fInfo: TLineInfo, n: PNode,
     encodeVInt(n.sym.id, result)
     pushSym(w, n.sym)
   else:
-    for i in 0 ..< sonsLen(n):
-      encodeNode(w, n.info, n.sons[i], result)
-  add(result, ')')
+    for i in 0..<n.len:
+      encodeNode(w, n.info, n[i], result)
+  result.add(')')
 
 proc encodeLoc(w: PRodWriter, loc: TLoc, result: var string) =
   var oldLen = result.len
   result.add('<')
   if loc.k != low(loc.k): encodeVInt(ord(loc.k), result)
   if loc.s != low(loc.s):
-    add(result, '*')
+    result.add('*')
     encodeVInt(ord(loc.s), result)
   if loc.flags != {}:
-    add(result, '$')
+    result.add('$')
     encodeVInt(cast[int32](loc.flags), result)
   if loc.t != nil:
-    add(result, '^')
+    result.add('^')
     encodeVInt(cast[int32](loc.t.id), result)
     pushType(w, loc.t)
   if loc.r != nil:
-    add(result, '!')
+    result.add('!')
     encodeStr($loc.r, result)
   if loc.a != 0:
-    add(result, '?')
+    result.add('?')
     encodeVInt(loc.a, result)
   if oldLen + 1 == result.len:
     # no data was necessary, so remove the '<' again:
     setLen(result, oldLen)
   else:
-    add(result, '>')
+    result.add('>')
 
 proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t == nil:
@@ -277,47 +277,47 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t.kind == tyForward: internalError("encodeType: tyForward")
   # for the new rodfile viewer we use a preceding [ so that the data section
   # can easily be disambiguated:
-  add(result, '[')
+  result.add('[')
   encodeVInt(ord(t.kind), result)
-  add(result, '+')
+  result.add('+')
   encodeVInt(t.id, result)
   if t.n != nil:
-    encodeNode(w, unknownLineInfo(), t.n, result)
+    encodeNode(w, unknownLineInfo, t.n, result)
   if t.flags != {}:
-    add(result, '$')
+    result.add('$')
     encodeVInt(cast[int32](t.flags), result)
   if t.callConv != low(t.callConv):
-    add(result, '?')
+    result.add('?')
     encodeVInt(ord(t.callConv), result)
   if t.owner != nil:
-    add(result, '*')
+    result.add('*')
     encodeVInt(t.owner.id, result)
     pushSym(w, t.owner)
   if t.sym != nil:
-    add(result, '&')
+    result.add('&')
     encodeVInt(t.sym.id, result)
     pushSym(w, t.sym)
   if t.size != - 1:
-    add(result, '/')
+    result.add('/')
     encodeVBiggestInt(t.size, result)
   if t.align != - 1:
-    add(result, '=')
+    result.add('=')
     encodeVInt(t.align, result)
   encodeLoc(w, t.loc, result)
-  for i in 0 ..< sonsLen(t):
-    if t.sons[i] == nil:
-      add(result, "^()")
+  for i in 0..<t.len:
+    if t[i] == nil:
+      result.add("^()")
     else:
-      add(result, '^')
-      encodeVInt(t.sons[i].id, result)
-      pushType(w, t.sons[i])
+      result.add('^')
+      encodeVInt(t[i].id, result)
+      pushType(w, t[i])
 
 proc encodeLib(w: PRodWriter, lib: PLib, info: TLineInfo, result: var string) =
-  add(result, '|')
+  result.add('|')
   encodeVInt(ord(lib.kind), result)
-  add(result, '|')
+  result.add('|')
   encodeStr($lib.name, result)
-  add(result, '|')
+  result.add('|')
   encodeNode(w, info, lib.path, result)
 
 proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
@@ -363,8 +363,8 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) =
   encodeLoc(w, s.loc, result)
   if s.annex != nil: encodeLib(w, s.annex, s.info, result)
   if s.constraint != nil:
-    add(result, '#')
-    encodeNode(w, unknownLineInfo(), s.constraint, result)
+    result.add('#')
+    encodeNode(w, unknownLineInfo, s.constraint, result)
   # lazy loading will soon reload the ast lazily, so the ast needs to be
   # the last entry of a symbol:
   if s.ast != nil:

compiler/ccgcalls.nim

@@ -9,43 +9,95 @@
 #
 # included from cgen.nim
 
-proc leftAppearsOnRightSide(le, ri: PNode): bool =
+proc canRaiseDisp(p: BProc; n: PNode): bool =
+  # we assume things like sysFatal cannot raise themselves
+  if n.kind == nkSym and {sfNeverRaises, sfImportc, sfCompilerProc} * n.sym.flags != {}:
+    result = false
+  elif optPanics in p.config.globalOptions or
+      (n.kind == nkSym and sfSystemModule in getModule(n.sym).flags):
+    # we know we can be strict:
+    result = canRaise(n)
+  else:
+    # we have to be *very* conservative:
+    result = canRaiseConservative(n)
+
+proc preventNrvo(p: BProc; le, ri: PNode): bool =
+  proc locationEscapes(p: BProc; le: PNode; inTryStmt: bool): bool =
+    var n = le
+    while true:
+      # do NOT follow nkHiddenDeref here!
+      case n.kind
+      of nkSym:
+        # we don't own the location so it escapes:
+        if n.sym.owner != p.prc:
+          return true
+        elif inTryStmt and sfUsedInFinallyOrExcept in n.sym.flags:
+          # it is also an observable store if the location is used
+          # in 'except' or 'finally'
+          return true
+        return false
+      of nkDotExpr, nkBracketExpr, nkObjUpConv, nkObjDownConv,
+          nkCheckedFieldExpr:
+        n = n[0]
+      of nkHiddenStdConv, nkHiddenSubConv, nkConv:
+        n = n[1]
+      else:
+        # cannot analyse the location; assume the worst
+        return true
+
   if le != nil:
-    for i in 1 ..< ri.len:
+    for i in 1..<ri.len:
       let r = ri[i]
       if isPartOf(le, r) != arNo: return true
+    # we use the weaker 'canRaise' here in order to prevent too many
+    # annoying warnings, see #14514
+    if canRaise(ri[0]) and
+        locationEscapes(p, le, p.nestedTryStmts.len > 0):
+      message(p.config, le.info, warnObservableStores, $le)
 
 proc hasNoInit(call: PNode): bool {.inline.} =
-  result = call.sons[0].kind == nkSym and sfNoInit in call.sons[0].sym.flags
+  result = call[0].kind == nkSym and sfNoInit in call[0].sym.flags
+
+proc isHarmlessStore(p: BProc; canRaise: bool; d: TLoc): bool =
+  if d.k in {locTemp, locNone} or not canRaise:
+    result = true
+  elif d.k == locLocalVar and p.withinTryWithExcept == 0:
+    # we cannot observe a store to a local variable if the current proc
+    # has no error handler:
+    result = true
+  else:
+    result = false
 
 proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
                callee, params: Rope) =
+  let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
   genLineDir(p, ri)
   var pl = callee & ~"(" & params
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
+  var typ = skipTypes(ri[0].typ, abstractInst)
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
       if params != nil: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
-        if d.k == locNone: getTemp(p, typ.sons[0], d, needsInit=true)
+        if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
           discard "resetLoc(p, d)"
-        add(pl, addrLoc(p.config, d))
-        add(pl, ~");$n")
+        pl.add(addrLoc(p.config, d))
+        pl.add(~");$n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
-        add(pl, ~");$n")
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
+        pl.add(~");$n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
+      if canRaise: raiseExit(p)
     else:
-      add(pl, ~")")
+      pl.add(~")")
       if p.module.compileToCpp:
         if lfSingleUse in d.flags:
           # do not generate spurious temporaries for C++! For C we're better off
@@ -56,65 +108,127 @@ proc fixupCall(p: BProc, le, ri: PNode, d: var TLoc,
           excl d.flags, lfSingleUse
         else:
           if d.k == locNone and p.splitDecls == 0:
-            getTempCpp(p, typ.sons[0], d, pl)
+            getTempCpp(p, typ[0], d, pl)
           else:
-            if d.k == locNone: getTemp(p, typ.sons[0], d)
+            if d.k == locNone: getTemp(p, typ[0], d)
             var list: TLoc
             initLoc(list, locCall, d.lode, OnUnknown)
             list.r = pl
             genAssignment(p, d, list, {}) # no need for deep copying
-      else:
-        if d.k == locNone: getTemp(p, typ.sons[0], d)
+            if canRaise: raiseExit(p)
+
+      elif isHarmlessStore(p, canRaise, d):
+        if d.k == locNone: getTemp(p, typ[0], d)
         assert(d.t != nil)        # generate an assignment to d:
         var list: TLoc
         initLoc(list, locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
+        if canRaise: raiseExit(p)
+      else:
+        var tmp: TLoc
+        getTemp(p, typ[0], tmp, needsInit=true)
+        var list: TLoc
+        initLoc(list, locCall, d.lode, OnUnknown)
+        list.r = pl
+        genAssignment(p, tmp, list, {}) # no need for deep copying
+        if canRaise: raiseExit(p)
+        genAssignment(p, d, tmp, {})
   else:
-    add(pl, ~");$n")
+    pl.add(~");$n")
     line(p, cpsStmts, pl)
+    if canRaise: raiseExit(p)
 
 proc genBoundsCheck(p: BProc; arr, a, b: TLoc)
 
-proc openArrayLoc(p: BProc, n: PNode): Rope =
-  var a: TLoc
+proc reifiedOpenArray(n: PNode): bool {.inline.} =
+  var x = n
+  while x.kind in {nkAddr, nkHiddenAddr, nkHiddenStdConv, nkHiddenDeref}:
+    x = x[0]
+  if x.kind == nkSym and x.sym.kind == skParam:
+    result = false
+  else:
+    result = true
 
-  let q = skipConv(n)
+proc genOpenArraySlice(p: BProc; q: PNode; formalType, destType: PType): (Rope, Rope) =
+  var a, b, c: TLoc
+  initLocExpr(p, q[1], a)
+  initLocExpr(p, q[2], b)
+  initLocExpr(p, q[3], c)
+  # but first produce the required index checks:
+  if optBoundsCheck in p.options:
+    genBoundsCheck(p, a, b, c)
+  let ty = skipTypes(a.t, abstractVar+{tyPtr})
+  let dest = getTypeDesc(p.module, destType)
+  let lengthExpr = "($1)-($2)+1" % [rdLoc(c), rdLoc(b)]
+  case ty.kind
+  of tyArray:
+    let first = toInt64(firstOrd(p.config, ty))
+    if first == 0:
+      result = ("($3*)(($1)+($2))" % [rdLoc(a), rdLoc(b), dest],
+                lengthExpr)
+    else:
+      result = ("($4*)($1)+(($2)-($3))" %
+        [rdLoc(a), rdLoc(b), intLiteral(first), dest],
+        lengthExpr)
+  of tyOpenArray, tyVarargs:
+    if reifiedOpenArray(q[1]):
+      result = ("($3*)($1.Field0)+($2)" % [rdLoc(a), rdLoc(b), dest],
+                lengthExpr)
+    else:
+      result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest],
+                lengthExpr)
+  of tyUncheckedArray, tyCString:
+    result = ("($3*)($1)+($2)" % [rdLoc(a), rdLoc(b), dest],
+              lengthExpr)
+  of tyString, tySequence:
+    let atyp = skipTypes(a.t, abstractInst)
+    if formalType.skipTypes(abstractInst).kind in {tyVar} and atyp.kind == tyString and
+        optSeqDestructors in p.config.globalOptions:
+      linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+    if atyp.kind in {tyVar} and not compileToCpp(p.module):
+      result = ("($4*)(*$1)$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
+                lengthExpr)
+    else:
+      result = ("($4*)$1$3+($2)" % [rdLoc(a), rdLoc(b), dataField(p), dest],
+                lengthExpr)
+  else:
+    internalError(p.config, "openArrayLoc: " & typeToString(a.t))
+
+proc openArrayLoc(p: BProc, formalType: PType, n: PNode): Rope =
+  var q = skipConv(n)
+  var skipped = false
+  while q.kind == nkStmtListExpr and q.len > 0:
+    skipped = true
+    q = q.lastSon
   if getMagic(q) == mSlice:
     # magic: pass slice to openArray:
-    var b, c: TLoc
-    initLocExpr(p, q[1], a)
-    initLocExpr(p, q[2], b)
-    initLocExpr(p, q[3], c)
-    # but first produce the required index checks:
-    if optBoundsCheck in p.options:
-      genBoundsCheck(p, a, b, c)
-    let ty = skipTypes(a.t, abstractVar+{tyPtr})
-    case ty.kind
-    of tyArray:
-      let first = firstOrd(p.config, ty)
-      if first == 0:
-        result = "($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c)]
-      else:
-        result = "($1)+(($2)-($4)), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), intLiteral(first)]
-    of tyOpenArray, tyVarargs, tyUncheckedArray:
-      result = "($1)+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c)]
-    of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-          not compileToCpp(p.module):
-        result = "(*$1)$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p)]
-      else:
-        result = "$1$4+($2), ($3)-($2)+1" % [rdLoc(a), rdLoc(b), rdLoc(c), dataField(p)]
-    else:
-      internalError(p.config, "openArrayLoc: " & typeToString(a.t))
+    if skipped:
+      q = skipConv(n)
+      while q.kind == nkStmtListExpr and q.len > 0:
+        for i in 0..<q.len-1:
+          genStmts(p, q[i])
+        q = q.lastSon
+    let (x, y) = genOpenArraySlice(p, q, formalType, n.typ[0])
+    result = x & ", " & y
   else:
-    initLocExpr(p, n, a)
+    var a: TLoc
+    initLocExpr(p, if n.kind == nkHiddenStdConv: n[1] else: n, a)
     case skipTypes(a.t, abstractVar).kind
     of tyOpenArray, tyVarargs:
-      result = "$1, $1Len_0" % [rdLoc(a)]
+      if reifiedOpenArray(n):
+        if a.t.kind in {tyVar, tyLent}:
+          result = "$1->Field0, $1->Field1" % [rdLoc(a)]
+        else:
+          result = "$1.Field0, $1.Field1" % [rdLoc(a)]
+      else:
+        result = "$1, $1Len_0" % [rdLoc(a)]
     of tyString, tySequence:
-      if skipTypes(n.typ, abstractInst).kind == tyVar and
-            not compileToCpp(p.module):
+      let ntyp = skipTypes(n.typ, abstractInst)
+      if formalType.skipTypes(abstractInst).kind in {tyVar} and ntyp.kind == tyString and
+          optSeqDestructors in p.config.globalOptions:
+        linefmt(p, cpsStmts, "#nimPrepareStrMutationV2($1);$n", [byRefLoc(p, a)])
+      if ntyp.kind in {tyVar} and not compileToCpp(p.module):
         var t: TLoc
         t.r = "(*$1)" % [a.rdLoc]
         result = "(*$1)$3, $2" % [a.rdLoc, lenExpr(p, t), dataField(p)]
@@ -134,27 +248,38 @@ proc openArrayLoc(p: BProc, n: PNode): Rope =
         internalError(p.config, "openArrayLoc: " & typeToString(a.t))
     else: internalError(p.config, "openArrayLoc: " & typeToString(a.t))
 
-proc genArgStringToCString(p: BProc, n: PNode): Rope {.inline.} =
-  var a: TLoc
-  initLocExpr(p, n.sons[0], a)
-  result = ropecg(p.module, "#nimToCStringConv($1)", [a.rdLoc])
+proc withTmpIfNeeded(p: BProc, a: TLoc, needsTmp: bool): TLoc =
+  # Bug https://github.com/status-im/nimbus-eth2/issues/1549
+  # Aliasing is preferred over stack overflows.
+  # Also don't regress for non ARC-builds, too risky.
+  if needsTmp and a.lode.typ != nil and p.config.selectedGC in {gcArc, gcOrc} and
+      getSize(p.config, a.lode.typ) < 1024:
+    getTemp(p, a.lode.typ, result, needsInit=false)
+    genAssignment(p, result, a, {})
+  else:
+    result = a
 
-proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
+proc genArgStringToCString(p: BProc, n: PNode, needsTmp: bool): Rope {.inline.} =
+  var a: TLoc
+  initLocExpr(p, n[0], a)
+  ropecg(p.module, "#nimToCStringConv($1)", [withTmpIfNeeded(p, a, needsTmp).rdLoc])
+
+proc genArg(p: BProc, n: PNode, param: PSym; call: PNode, needsTmp = false): Rope =
   var a: TLoc
   if n.kind == nkStringToCString:
-    result = genArgStringToCString(p, n)
+    result = genArgStringToCString(p, n, needsTmp)
   elif skipTypes(param.typ, abstractVar).kind in {tyOpenArray, tyVarargs}:
-    var n = if n.kind != nkHiddenAddr: n else: n.sons[0]
-    result = openArrayLoc(p, n)
+    var n = if n.kind != nkHiddenAddr: n else: n[0]
+    result = openArrayLoc(p, param.typ, n)
   elif ccgIntroducedPtr(p.config, param, call[0].typ[0]):
     initLocExpr(p, n, a)
-    result = addrLoc(p.config, a)
-  elif p.module.compileToCpp and param.typ.kind == tyVar and
+    result = addrLoc(p.config, withTmpIfNeeded(p, a, needsTmp))
+  elif p.module.compileToCpp and param.typ.kind in {tyVar} and
       n.kind == nkHiddenAddr:
-    initLocExprSingleUse(p, n.sons[0], a)
+    initLocExprSingleUse(p, n[0], a)
     # if the proc is 'importc'ed but not 'importcpp'ed then 'var T' still
     # means '*T'. See posix.nim for lots of examples that do that in the wild.
-    let callee = call.sons[0]
+    let callee = call[0]
     if callee.kind == nkSym and
         {sfImportc, sfInfixCall, sfCompilerProc} * callee.sym.flags == {sfImportc} and
         {lfHeader, lfNoDecl} * callee.sym.loc.flags != {}:
@@ -163,69 +288,124 @@ proc genArg(p: BProc, n: PNode, param: PSym; call: PNode): Rope =
       result = rdLoc(a)
   else:
     initLocExprSingleUse(p, n, a)
-    result = rdLoc(a)
+    result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
 
-proc genArgNoParam(p: BProc, n: PNode): Rope =
+proc genArgNoParam(p: BProc, n: PNode, needsTmp = false): Rope =
   var a: TLoc
   if n.kind == nkStringToCString:
-    result = genArgStringToCString(p, n)
+    result = genArgStringToCString(p, n, needsTmp)
   else:
     initLocExprSingleUse(p, n, a)
-    result = rdLoc(a)
+    result = rdLoc(withTmpIfNeeded(p, a, needsTmp))
 
-template genParamLoop(params) {.dirty.} =
-  if i < sonsLen(typ):
-    assert(typ.n.sons[i].kind == nkSym)
-    let paramType = typ.n.sons[i]
-    if not paramType.typ.isCompileTimeOnly:
-      if params != nil: add(params, ~", ")
-      add(params, genArg(p, ri.sons[i], paramType.sym, ri))
+from dfa import instrTargets, InstrTargetKind
+
+proc potentialAlias(n: PNode, potentialWrites: seq[PNode]): bool =
+  for p in potentialWrites:
+    if instrTargets(p, n) != None:
+      return true
+
+proc skipTrivialIndirections(n: PNode): PNode =
+  result = n
+  while true:
+    case result.kind
+    of {nkDerefExpr, nkHiddenDeref, nkAddr, nkHiddenAddr, nkObjDownConv, nkObjUpConv}:
+      result = result[0]
+    of {nkHiddenStdConv, nkHiddenSubConv}:
+      result = result[1]
+    else: break
+
+proc getPotentialWrites(n: PNode, mutate = false): seq[PNode] =
+  case n.kind:
+  of nkLiterals, nkIdent: discard
+  of nkSym:
+    if mutate: result.add n
+  of nkAsgn, nkFastAsgn:
+    result.add getPotentialWrites(n[0], true)
+    result.add getPotentialWrites(n[1], mutate)
+  of nkAddr, nkHiddenAddr:
+    result.add getPotentialWrites(n[0], true)
+  of nkCallKinds: #TODO: Find out why in f += 1, f is a nkSym and not a nkHiddenAddr
+    for s in n.sons:
+      result.add getPotentialWrites(s, true)
   else:
-    if params != nil: add(params, ~", ")
-    add(params, genArgNoParam(p, ri.sons[i]))
+    for s in n.sons:
+      result.add getPotentialWrites(s, mutate)
 
-proc addActualPrefixForHCR(res: var Rope, module: PSym, sym: PSym) =
-  if sym.flags * {sfImportc, sfNonReloadable} == {} and
+proc getPotentialReads(n: PNode): seq[PNode] =
+  case n.kind:
+  of nkLiterals, nkIdent: discard
+  of nkSym: result.add n
+  else:
+    for s in n.sons:
+      result.add getPotentialReads(s)
+
+proc genParams(p: BProc, ri: PNode, typ: PType): Rope =
+  # We must generate temporaries in cases like #14396
+  # to keep the strict Left-To-Right evaluation
+  var needTmp = newSeq[bool](ri.len - 1)
+  var potentialWrites: seq[PNode]
+  for i in countdown(ri.len - 1, 1):
+    if ri[i].skipTrivialIndirections.kind == nkSym:
+      needTmp[i - 1] = potentialAlias(ri[i], potentialWrites)
+    else:
+      for n in getPotentialReads(ri[i]):
+        if not needTmp[i - 1]:
+          needTmp[i - 1] = potentialAlias(n, potentialWrites)
+      potentialWrites.add getPotentialWrites(ri[i])
+    if ri[i].kind == nkHiddenAddr:
+      # Optimization: don't use a temp, if we would only take the adress anyway
+      needTmp[i - 1] = false
+
+  for i in 1..<ri.len:
+    if i < typ.len:
+      assert(typ.n[i].kind == nkSym)
+      let paramType = typ.n[i]
+      if not paramType.typ.isCompileTimeOnly:
+        if result != nil: result.add(~", ")
+        result.add(genArg(p, ri[i], paramType.sym, ri, needTmp[i-1]))
+    else:
+      if result != nil: result.add(~", ")
+      result.add(genArgNoParam(p, ri[i], needTmp[i-1]))
+
+proc addActualSuffixForHCR(res: var Rope, module: PSym, sym: PSym) =
+  if sym.flags * {sfImportc, sfNonReloadable} == {} and sym.loc.k == locProc and
       (sym.typ.callConv == ccInline or sym.owner.id == module.id):
     res = res & "_actual".rope
 
 proc genPrefixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var op: TLoc
   # this is a hotspot in the compiler
-  initLocExpr(p, ri.sons[0], op)
-  var params: Rope
+  initLocExpr(p, ri[0], op)
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInstOwned)
+  var typ = skipTypes(ri[0].typ, abstractInstOwned)
   assert(typ.kind == tyProc)
-  assert(sonsLen(typ) == sonsLen(typ.n))
-  var length = sonsLen(ri)
-  for i in 1 ..< length:
-    genParamLoop(params)
+  assert(typ.len == typ.n.len)
+
+  var params = genParams(p, ri, typ)
+
   var callee = rdLoc(op)
-  if p.hcrOn and ri.sons[0].kind == nkSym:
-    callee.addActualPrefixForHCR(p.module.module, ri.sons[0].sym)
+  if p.hcrOn and ri[0].kind == nkSym:
+    callee.addActualSuffixForHCR(p.module.module, ri[0].sym)
   fixupCall(p, le, ri, d, callee, params)
 
 proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
 
-  proc getRawProcType(p: BProc, t: PType): Rope =
-    result = getClosureType(p.module, t, clHalf)
-
   proc addComma(r: Rope): Rope =
-    result = if r == nil: r else: r & ~", "
+    if r == nil: r else: r & ~", "
 
   const PatProc = "$1.ClE_0? $1.ClP_0($3$1.ClE_0):(($4)($1.ClP_0))($2)"
   const PatIter = "$1.ClP_0($3$1.ClE_0)" # we know the env exists
-  var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
-  var pl: Rope
 
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var op: TLoc
+  initLocExpr(p, ri[0], op)
+
+  # getUniqueType() is too expensive here:
+  var typ = skipTypes(ri[0].typ, abstractInstOwned)
   assert(typ.kind == tyProc)
-  var length = sonsLen(ri)
-  for i in 1 ..< length:
-    assert(sonsLen(typ) == sonsLen(typ.n))
-    genParamLoop(pl)
+  assert(typ.len == typ.n.len)
+
+  var pl = genParams(p, ri, typ)
 
   template genCallPattern {.dirty.} =
     if tfIterator in typ.flags:
@@ -233,28 +413,30 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
     else:
       lineF(p, cpsStmts, PatProc & ";$n", [rdLoc(op), pl, pl.addComma, rawProc])
 
-  let rawProc = getRawProcType(p, typ)
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
-      if sonsLen(ri) > 1: add(pl, ~", ")
+  let rawProc = getClosureType(p.module, typ, clHalf)
+  let canRaise = p.config.exc == excGoto and canRaiseDisp(p, ri[0])
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
+      if ri.len > 1: pl.add(~", ")
       # beware of 'result = p(result)'. We may need to allocate a temporary:
-      if d.k in {locTemp, locNone} or not leftAppearsOnRightSide(le, ri):
+      if d.k in {locTemp, locNone} or not preventNrvo(p, le, ri):
         # Great, we can use 'd':
         if d.k == locNone:
-          getTemp(p, typ.sons[0], d, needsInit=true)
+          getTemp(p, typ[0], d, needsInit=true)
         elif d.k notin {locTemp} and not hasNoInit(ri):
           # reset before pass as 'result' var:
           discard "resetLoc(p, d)"
-        add(pl, addrLoc(p.config, d))
+        pl.add(addrLoc(p.config, d))
         genCallPattern()
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
         genCallPattern()
+        if canRaise: raiseExit(p)
         genAssignment(p, d, tmp, {}) # no need for deep copying
-    else:
-      if d.k == locNone: getTemp(p, typ.sons[0], d)
+    elif isHarmlessStore(p, canRaise, d):
+      if d.k == locNone: getTemp(p, typ[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
       initLoc(list, locCall, d.lode, OnUnknown)
@@ -262,29 +444,43 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) =
         list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
       else:
         list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
-
       genAssignment(p, d, list, {}) # no need for deep copying
+      if canRaise: raiseExit(p)
+    else:
+      var tmp: TLoc
+      getTemp(p, typ[0], tmp)
+      assert(d.t != nil)        # generate an assignment to d:
+      var list: TLoc
+      initLoc(list, locCall, d.lode, OnUnknown)
+      if tfIterator in typ.flags:
+        list.r = PatIter % [rdLoc(op), pl, pl.addComma, rawProc]
+      else:
+        list.r = PatProc % [rdLoc(op), pl, pl.addComma, rawProc]
+      genAssignment(p, tmp, list, {})
+      if canRaise: raiseExit(p)
+      genAssignment(p, d, tmp, {})
   else:
     genCallPattern()
+    if canRaise: raiseExit(p)
 
 proc genOtherArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
-  if i < sonsLen(typ):
+  if i < typ.len:
     # 'var T' is 'T&' in C++. This means we ignore the request of
     # any nkHiddenAddr when it's a 'var T'.
-    let paramType = typ.n.sons[i]
+    let paramType = typ.n[i]
     assert(paramType.kind == nkSym)
     if paramType.typ.isCompileTimeOnly:
       result = nil
-    elif typ.sons[i].kind == tyVar and ri.sons[i].kind == nkHiddenAddr:
-      result = genArgNoParam(p, ri.sons[i][0])
+    elif typ[i].kind in {tyVar} and ri[i].kind == nkHiddenAddr:
+      result = genArgNoParam(p, ri[i][0])
     else:
-      result = genArgNoParam(p, ri.sons[i]) #, typ.n.sons[i].sym)
+      result = genArgNoParam(p, ri[i]) #, typ.n[i].sym)
   else:
     if tfVarargs notin typ.flags:
       localError(p.config, ri.info, "wrong argument count")
       result = nil
     else:
-      result = genArgNoParam(p, ri.sons[i])
+      result = genArgNoParam(p, ri[i])
 
 discard """
 Dot call syntax in C++
@@ -328,16 +524,16 @@ proc skipAddrDeref(node: PNode): PNode =
   var isAddr = false
   case n.kind
   of nkAddr, nkHiddenAddr:
-    n = n.sons[0]
+    n = n[0]
     isAddr = true
   of nkDerefExpr, nkHiddenDeref:
-    n = n.sons[0]
+    n = n[0]
   else: return n
-  if n.kind == nkObjDownConv: n = n.sons[0]
+  if n.kind == nkObjDownConv: n = n[0]
   if isAddr and n.kind in {nkDerefExpr, nkHiddenDeref}:
-    result = n.sons[0]
+    result = n[0]
   elif n.kind in {nkAddr, nkHiddenAddr}:
-    result = n.sons[0]
+    result = n[0]
   else:
     result = node
 
@@ -345,14 +541,14 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   # for better or worse c2nim translates the 'this' argument to a 'var T'.
   # However manual wrappers may also use 'ptr T'. In any case we support both
   # for convenience.
-  internalAssert p.config, i < sonsLen(typ)
-  assert(typ.n.sons[i].kind == nkSym)
+  internalAssert p.config, i < typ.len
+  assert(typ.n[i].kind == nkSym)
   # if the parameter is lying (tyVar) and thus we required an additional deref,
   # skip the deref:
   var ri = ri[i]
   while ri.kind == nkObjDownConv: ri = ri[0]
-  let t = typ.sons[i].skipTypes({tyGenericInst, tyAlias, tySink})
-  if t.kind == tyVar:
+  let t = typ[i].skipTypes({tyGenericInst, tyAlias, tySink})
+  if t.kind in {tyVar}:
     let x = if ri.kind == nkHiddenAddr: ri[0] else: ri
     if x.typ.kind == tyPtr:
       result = genArgNoParam(p, x)
@@ -373,7 +569,7 @@ proc genThisArg(p: BProc; ri: PNode; i: int; typ: PType): Rope =
   else:
     ri = skipAddrDeref(ri)
     if ri.kind in {nkAddr, nkHiddenAddr}: ri = ri[0]
-    result = genArgNoParam(p, ri) #, typ.n.sons[i].sym)
+    result = genArgNoParam(p, ri) #, typ.n[i].sym)
     result.add(".")
 
 proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
@@ -382,22 +578,25 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
   while i < pat.len:
     case pat[i]
     of '@':
-      if j < ri.len:
-        result.add genOtherArg(p, ri, j, typ)
-        for k in j+1 ..< ri.len:
-          result.add(~", ")
-          result.add genOtherArg(p, ri, k, typ)
+      var first = true
+      for k in j..<ri.len:
+        let arg = genOtherArg(p, ri, k, typ)
+        if arg.len > 0:
+          if not first:
+            result.add(~", ")
+          first = false
+          result.add arg
       inc i
     of '#':
       if i+1 < pat.len and pat[i+1] in {'+', '@'}:
         let ri = ri[j]
         if ri.kind in nkCallKinds:
-          let typ = skipTypes(ri.sons[0].typ, abstractInst)
-          if pat[i+1] == '+': result.add genArgNoParam(p, ri.sons[0])
+          let typ = skipTypes(ri[0].typ, abstractInst)
+          if pat[i+1] == '+': result.add genArgNoParam(p, ri[0])
           result.add(~"(")
           if 1 < ri.len:
             result.add genOtherArg(p, ri, 1, typ)
-          for k in j+1 ..< ri.len:
+          for k in j+1..<ri.len:
             result.add(~", ")
             result.add genOtherArg(p, ri, k, typ)
           result.add(~")")
@@ -408,7 +607,7 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
         result.add genThisArg(p, ri, j, typ)
         inc i
       elif i+1 < pat.len and pat[i+1] == '[':
-        var arg = ri.sons[j].skipAddrDeref
+        var arg = ri[j].skipAddrDeref
         while arg.kind in {nkAddr, nkHiddenAddr, nkObjDownConv}: arg = arg[0]
         result.add genArgNoParam(p, arg)
         #result.add debugTree(arg, 0, 10)
@@ -428,24 +627,23 @@ proc genPatternCall(p: BProc; ri: PNode; pat: string; typ: PType): Rope =
         if pat[i] notin {'@', '#', '\''}: inc(i)
         else: break
       if i - 1 >= start:
-        add(result, substr(pat, start, i - 1))
+        result.add(substr(pat, start, i - 1))
 
 proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
   var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
-  var length = sonsLen(ri)
-  assert(sonsLen(typ) == sonsLen(typ.n))
+  assert(typ.len == typ.n.len)
   # don't call '$' here for efficiency:
-  let pat = ri.sons[0].sym.loc.r.data
+  let pat = ri[0].sym.loc.r.data
   internalAssert p.config, pat.len > 0
   if pat.contains({'#', '(', '@', '\''}):
     var pl = genPatternCall(p, ri, pat, typ)
     # simpler version of 'fixupCall' that works with the pl+params combination:
-    var typ = skipTypes(ri.sons[0].typ, abstractInst)
-    if typ.sons[0] != nil:
+    var typ = skipTypes(ri[0].typ, abstractInst)
+    if typ[0] != nil:
       if p.module.compileToCpp and lfSingleUse in d.flags:
         # do not generate spurious temporaries for C++! For C we're better off
         # with them to prevent undefined behaviour and because the codegen
@@ -454,116 +652,132 @@ proc genInfixCall(p: BProc, le, ri: PNode, d: var TLoc) =
         d.r = pl
         excl d.flags, lfSingleUse
       else:
-        if d.k == locNone: getTemp(p, typ.sons[0], d)
+        if d.k == locNone: getTemp(p, typ[0], d)
         assert(d.t != nil)        # generate an assignment to d:
         var list: TLoc
         initLoc(list, locCall, d.lode, OnUnknown)
         list.r = pl
         genAssignment(p, d, list, {}) # no need for deep copying
     else:
-      add(pl, ~";$n")
+      pl.add(~";$n")
       line(p, cpsStmts, pl)
   else:
     var pl: Rope = nil
-    #var param = typ.n.sons[1].sym
+    #var param = typ.n[1].sym
     if 1 < ri.len:
-      add(pl, genThisArg(p, ri, 1, typ))
-    add(pl, op.r)
+      pl.add(genThisArg(p, ri, 1, typ))
+    pl.add(op.r)
     var params: Rope
-    for i in 2 ..< length:
+    for i in 2..<ri.len:
       if params != nil: params.add(~", ")
-      assert(sonsLen(typ) == sonsLen(typ.n))
-      add(params, genOtherArg(p, ri, i, typ))
+      assert(typ.len == typ.n.len)
+      params.add(genOtherArg(p, ri, i, typ))
     fixupCall(p, le, ri, d, pl, params)
 
 proc genNamedParamCall(p: BProc, ri: PNode, d: var TLoc) =
   # generates a crappy ObjC call
   var op: TLoc
-  initLocExpr(p, ri.sons[0], op)
+  initLocExpr(p, ri[0], op)
   var pl = ~"["
   # getUniqueType() is too expensive here:
-  var typ = skipTypes(ri.sons[0].typ, abstractInst)
+  var typ = skipTypes(ri[0].typ, abstractInst)
   assert(typ.kind == tyProc)
-  var length = sonsLen(ri)
-  assert(sonsLen(typ) == sonsLen(typ.n))
+  assert(typ.len == typ.n.len)
 
   # don't call '$' here for efficiency:
-  let pat = ri.sons[0].sym.loc.r.data
+  let pat = ri[0].sym.loc.r.data
   internalAssert p.config, pat.len > 0
   var start = 3
   if ' ' in pat:
     start = 1
-    add(pl, op.r)
-    if length > 1:
-      add(pl, ~": ")
-      add(pl, genArg(p, ri.sons[1], typ.n.sons[1].sym, ri))
+    pl.add(op.r)
+    if ri.len > 1:
+      pl.add(~": ")
+      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
       start = 2
   else:
-    if length > 1:
-      add(pl, genArg(p, ri.sons[1], typ.n.sons[1].sym, ri))
-      add(pl, ~" ")
-    add(pl, op.r)
-    if length > 2:
-      add(pl, ~": ")
-      add(pl, genArg(p, ri.sons[2], typ.n.sons[2].sym, ri))
-  for i in start ..< length:
-    assert(sonsLen(typ) == sonsLen(typ.n))
-    if i >= sonsLen(typ):
+    if ri.len > 1:
+      pl.add(genArg(p, ri[1], typ.n[1].sym, ri))
+      pl.add(~" ")
+    pl.add(op.r)
+    if ri.len > 2:
+      pl.add(~": ")
+      pl.add(genArg(p, ri[2], typ.n[2].sym, ri))
+  for i in start..<ri.len:
+    assert(typ.len == typ.n.len)
+    if i >= typ.len:
       internalError(p.config, ri.info, "varargs for objective C method?")
-    assert(typ.n.sons[i].kind == nkSym)
-    var param = typ.n.sons[i].sym
-    add(pl, ~" ")
-    add(pl, param.name.s)
-    add(pl, ~": ")
-    add(pl, genArg(p, ri.sons[i], param, ri))
-  if typ.sons[0] != nil:
-    if isInvalidReturnType(p.config, typ.sons[0]):
-      if sonsLen(ri) > 1: add(pl, ~" ")
+    assert(typ.n[i].kind == nkSym)
+    var param = typ.n[i].sym
+    pl.add(~" ")
+    pl.add(param.name.s)
+    pl.add(~": ")
+    pl.add(genArg(p, ri[i], param, ri))
+  if typ[0] != nil:
+    if isInvalidReturnType(p.config, typ[0]):
+      if ri.len > 1: pl.add(~" ")
       # beware of 'result = p(result)'. We always allocate a temporary:
       if d.k in {locTemp, locNone}:
         # We already got a temp. Great, special case it:
-        if d.k == locNone: getTemp(p, typ.sons[0], d, needsInit=true)
-        add(pl, ~"Result: ")
-        add(pl, addrLoc(p.config, d))
-        add(pl, ~"];$n")
+        if d.k == locNone: getTemp(p, typ[0], d, needsInit=true)
+        pl.add(~"Result: ")
+        pl.add(addrLoc(p.config, d))
+        pl.add(~"];$n")
         line(p, cpsStmts, pl)
       else:
         var tmp: TLoc
-        getTemp(p, typ.sons[0], tmp, needsInit=true)
-        add(pl, addrLoc(p.config, tmp))
-        add(pl, ~"];$n")
+        getTemp(p, typ[0], tmp, needsInit=true)
+        pl.add(addrLoc(p.config, tmp))
+        pl.add(~"];$n")
         line(p, cpsStmts, pl)
         genAssignment(p, d, tmp, {}) # no need for deep copying
     else:
-      add(pl, ~"]")
-      if d.k == locNone: getTemp(p, typ.sons[0], d)
+      pl.add(~"]")
+      if d.k == locNone: getTemp(p, typ[0], d)
       assert(d.t != nil)        # generate an assignment to d:
       var list: TLoc
       initLoc(list, locCall, ri, OnUnknown)
       list.r = pl
       genAssignment(p, d, list, {}) # no need for deep copying
   else:
-    add(pl, ~"];$n")
+    pl.add(~"];$n")
     line(p, cpsStmts, pl)
 
-proc genCall(p: BProc, e: PNode, d: var TLoc) =
-  if e.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
-    genClosureCall(p, nil, e, d)
-  elif e.sons[0].kind == nkSym and sfInfixCall in e.sons[0].sym.flags:
-    genInfixCall(p, nil, e, d)
-  elif e.sons[0].kind == nkSym and sfNamedParamCall in e.sons[0].sym.flags:
-    genNamedParamCall(p, e, d)
-  else:
-    genPrefixCall(p, nil, e, d)
-  postStmtActions(p)
+proc notYetAlive(n: PNode): bool {.inline.} =
+  let r = getRoot(n)
+  result = r != nil and r.loc.lode == nil
+
+proc isInactiveDestructorCall(p: BProc, e: PNode): bool =
+  #[ Consider this example.
+
+    var :tmpD_3281815
+    try:
+      if true:
+        return
+      let args_3280013 =
+        wasMoved_3281816(:tmpD_3281815)
+        `=_3280036`(:tmpD_3281815, [1])
+        :tmpD_3281815
+    finally:
+      `=destroy_3280027`(args_3280013)
+
+  We want to return early but the 'finally' section is traversed before
+  the 'let args = ...' statement. We exploit this to generate better
+  code for 'return'. ]#
+  result = e.len == 2 and e[0].kind == nkSym and
+    e[0].sym.name.s == "=destroy" and notYetAlive(e[1].skipAddr)
 
 proc genAsgnCall(p: BProc, le, ri: PNode, d: var TLoc) =
-  if ri.sons[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
+  if p.withinBlockLeaveActions > 0 and isInactiveDestructorCall(p, ri):
+    return
+  if ri[0].typ.skipTypes({tyGenericInst, tyAlias, tySink, tyOwned}).callConv == ccClosure:
     genClosureCall(p, le, ri, d)
-  elif ri.sons[0].kind == nkSym and sfInfixCall in ri.sons[0].sym.flags:
+  elif ri[0].kind == nkSym and sfInfixCall in ri[0].sym.flags:
     genInfixCall(p, le, ri, d)
-  elif ri.sons[0].kind == nkSym and sfNamedParamCall in ri.sons[0].sym.flags:
+  elif ri[0].kind == nkSym and sfNamedParamCall in ri[0].sym.flags:
     genNamedParamCall(p, ri, d)
   else:
     genPrefixCall(p, le, ri, d)
   postStmtActions(p)
+
+proc genCall(p: BProc, e: PNode, d: var TLoc) = genAsgnCall(p, nil, e, d)

compiler/ccgliterals.nim

@@ -7,6 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
+# included from cgen.nim
+
 ## This include file contains the logic to produce constant string
 ## and seq literals. The code here is responsible that
 ## ``const x = ["a", "b"]`` works without hidden runtime creation code.
@@ -19,7 +21,7 @@ template detectVersion(field, corename) =
     if core == nil or core.kind != skConst:
       m.g.field = 1
     else:
-      m.g.field = int ast.getInt(core.ast)
+      m.g.field = toInt(ast.getInt(core.ast))
   result = m.g.field
 
 proc detectStrVersion(m: BModule): int =
@@ -33,8 +35,8 @@ proc detectSeqVersion(m: BModule): int =
 proc genStringLiteralDataOnlyV1(m: BModule, s: string): Rope =
   discard cgsym(m, "TGenericSeq")
   result = getTempName(m)
-  addf(m.s[cfsData], "STRING_LITERAL($1, $2, $3);$n",
-       [result, makeCString(s), rope(len(s))])
+  m.s[cfsData].addf("STRING_LITERAL($1, $2, $3);$n",
+       [result, makeCString(s), rope(s.len)])
 
 proc genStringLiteralV1(m: BModule; n: PNode): Rope =
   if s.isNil:
@@ -51,29 +53,31 @@ proc genStringLiteralV1(m: BModule; n: PNode): Rope =
 
 # ------ Version 2: destructor based strings and seqs -----------------------
 
-proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope) =
-  addf(m.s[cfsData], "static const struct {$n" &
-       "  NI cap; void* allocator; NIM_CHAR data[$2+1];$n" &
-       "} $1 = { $2, NIM_NIL, $3 };$n",
-       [result, rope(len(s)), makeCString(s)])
+proc genStringLiteralDataOnlyV2(m: BModule, s: string; result: Rope; isConst: bool) =
+  m.s[cfsData].addf("static $4 struct {$n" &
+       "  NI cap; NIM_CHAR data[$2+1];$n" &
+       "} $1 = { $2 | NIM_STRLIT_FLAG, $3 };$n",
+       [result, rope(s.len), makeCString(s),
+       rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2(m: BModule; n: PNode): Rope =
+proc genStringLiteralV2(m: BModule; n: PNode; isConst: bool): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   if id == m.labels:
     let pureLit = getTempName(m)
-    genStringLiteralDataOnlyV2(m, n.strVal, pureLit)
+    genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
     result = getTempName(m)
     discard cgsym(m, "NimStrPayload")
     discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    addf(m.s[cfsData], "static const NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [result, rope(len(n.strVal)), pureLit])
+    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [result, rope(n.strVal.len), pureLit, rope(if isConst: "const" else: "")])
   else:
     result = getTempName(m)
-    addf(m.s[cfsData], "static const NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
-          [result, rope(len(n.strVal)), m.tmpBase & rope(id)])
+    m.s[cfsData].addf("static $4 NimStringV2 $1 = {$2, (NimStrPayload*)&$3};$n",
+          [result, rope(n.strVal.len), m.tmpBase & rope(id),
+          rope(if isConst: "const" else: "")])
 
-proc genStringLiteralV2Const(m: BModule; n: PNode): Rope =
+proc genStringLiteralV2Const(m: BModule; n: PNode; isConst: bool): Rope =
   let id = nodeTableTestOrSet(m.dataCache, n, m.labels)
   var pureLit: Rope
   if id == m.labels:
@@ -81,19 +85,20 @@ proc genStringLiteralV2Const(m: BModule; n: PNode): Rope =
     discard cgsym(m, "NimStrPayload")
     discard cgsym(m, "NimStringV2")
     # string literal not found in the cache:
-    genStringLiteralDataOnlyV2(m, n.strVal, pureLit)
+    genStringLiteralDataOnlyV2(m, n.strVal, pureLit, isConst)
   else:
     pureLit = m.tmpBase & rope(id)
-  result = "{$1, (NimStrPayload*)&$2}" % [rope(len(n.strVal)), pureLit]
+  result = "{$1, (NimStrPayload*)&$2}" % [rope(n.strVal.len), pureLit]
 
 # ------ Version selector ---------------------------------------------------
 
-proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo): Rope =
+proc genStringLiteralDataOnly(m: BModule; s: string; info: TLineInfo;
+                              isConst: bool): Rope =
   case detectStrVersion(m)
   of 0, 1: result = genStringLiteralDataOnlyV1(m, s)
   of 2:
     result = getTempName(m)
-    genStringLiteralDataOnlyV2(m, s, result)
+    genStringLiteralDataOnlyV2(m, s, result, isConst)
   else:
     localError(m.config, info, "cannot determine how to produce code for string literal")
 
@@ -103,6 +108,6 @@ proc genNilStringLiteral(m: BModule; info: TLineInfo): Rope =
 proc genStringLiteral(m: BModule; n: PNode): Rope =
   case detectStrVersion(m)
   of 0, 1: result = genStringLiteralV1(m, n)
-  of 2: result = genStringLiteralV2(m, n)
+  of 2: result = genStringLiteralV2(m, n, isConst = true)
   else:
     localError(m.config, n.info, "cannot determine how to produce code for string literal")

compiler/ccgmerge.nim

@@ -28,8 +28,8 @@ const
     cfsFieldInfo: "NIM_merge_FIELD_INFO",
     cfsTypeInfo: "NIM_merge_TYPE_INFO",
     cfsProcHeaders: "NIM_merge_PROC_HEADERS",
-    cfsVars: "NIM_merge_VARS",
     cfsData: "NIM_merge_DATA",
+    cfsVars: "NIM_merge_VARS",
     cfsProcs: "NIM_merge_PROCS",
     cfsInitProc: "NIM_merge_INIT_PROC",
     cfsDatInitProc: "NIM_merge_DATINIT_PROC",
@@ -48,11 +48,15 @@ const
   NimMergeEndMark = "/*\tNIM_merge_END:*/"
 
 proc genSectionStart*(fs: TCFileSection; conf: ConfigRef): Rope =
+  # useful for debugging and only adds at most a few lines in each file
+  result.add("\n/* section: ")
+  result.add(CFileSectionNames[fs])
+  result.add(" */\n")
   if compilationCachePresent(conf):
     result = nil
-    add(result, "\n/*\t")
-    add(result, CFileSectionNames[fs])
-    add(result, ":*/\n")
+    result.add("\n/*\t")
+    result.add(CFileSectionNames[fs])
+    result.add(":*/\n")
 
 proc genSectionEnd*(fs: TCFileSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
@@ -61,9 +65,9 @@ proc genSectionEnd*(fs: TCFileSection; conf: ConfigRef): Rope =
 proc genSectionStart*(ps: TCProcSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
     result = rope("")
-    add(result, "\n/*\t")
-    add(result, CProcSectionNames[ps])
-    add(result, ":*/\n")
+    result.add("\n/*\t")
+    result.add(CProcSectionNames[ps])
+    result.add(":*/\n")
 
 proc genSectionEnd*(ps: TCProcSection; conf: ConfigRef): Rope =
   if compilationCachePresent(conf):
@@ -278,7 +282,7 @@ proc mergeRequired*(m: BModule): bool =
     if m.s[i] != nil:
       #echo "not empty: ", i, " ", m.s[i]
       return true
-  for i in low(TCProcSection)..high(TCProcSection):
+  for i in TCProcSection:
     if m.initProc.s(i) != nil:
       #echo "not empty: ", i, " ", m.initProc.s[i]
       return true
@@ -288,7 +292,7 @@ proc mergeFiles*(cfilename: AbsoluteFile, m: BModule) =
   var old: TMergeSections
   readMergeSections(cfilename, old)
   # do the merge; old section before new section:
-  for i in low(TCFileSection)..high(TCFileSection):
+  for i in TCFileSection:
     m.s[i] = old.f[i] & m.s[i]
-  for i in low(TCProcSection)..high(TCProcSection):
+  for i in TCProcSection:
     m.initProc.s(i) = old.p[i] & m.initProc.s(i)

compiler/ccgreset.nim

@@ -0,0 +1,94 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# included from cgen.nim
+
+## Code specialization instead of the old, incredibly slow 'genericReset'
+## implementation.
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType)
+
+proc specializeResetN(p: BProc, accessor: Rope, n: PNode;
+                     typ: PType) =
+  if n == nil: return
+  case n.kind
+  of nkRecList:
+    for i in 0..<n.len:
+      specializeResetN(p, accessor, n[i], typ)
+  of nkRecCase:
+    if (n[0].kind != nkSym): internalError(p.config, n.info, "specializeResetN")
+    let disc = n[0].sym
+    if disc.loc.r == nil: fillObjectFields(p.module, typ)
+    if disc.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
+    for i in 1..<n.len:
+      let branch = n[i]
+      assert branch.kind in {nkOfBranch, nkElse}
+      if branch.kind == nkOfBranch:
+        genCaseRange(p, branch)
+      else:
+        lineF(p, cpsStmts, "default:$n", [])
+      specializeResetN(p, accessor, lastSon(branch), typ)
+      lineF(p, cpsStmts, "break;$n", [])
+    lineF(p, cpsStmts, "} $n", [])
+    specializeResetT(p, "$1.$2" % [accessor, disc.loc.r], disc.loc.t)
+  of nkSym:
+    let field = n.sym
+    if field.typ.kind == tyVoid: return
+    if field.loc.r == nil: fillObjectFields(p.module, typ)
+    if field.loc.t == nil:
+      internalError(p.config, n.info, "specializeResetN()")
+    specializeResetT(p, "$1.$2" % [accessor, field.loc.r], field.loc.t)
+  else: internalError(p.config, n.info, "specializeResetN()")
+
+proc specializeResetT(p: BProc, accessor: Rope, typ: PType) =
+  if typ == nil: return
+
+  case typ.kind
+  of tyGenericInst, tyGenericBody, tyTypeDesc, tyAlias, tyDistinct, tyInferred,
+     tySink, tyOwned:
+    specializeResetT(p, accessor, lastSon(typ))
+  of tyArray:
+    let arraySize = lengthOrd(p.config, typ[0])
+    var i: TLoc
+    getTemp(p, getSysType(p.module.g.graph, unknownLineInfo, tyInt), i)
+    linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
+            [i.r, arraySize])
+    specializeResetT(p, ropecg(p.module, "$1[$2]", [accessor, i.r]), typ[1])
+    lineF(p, cpsStmts, "}$n", [])
+  of tyObject:
+    for i in 0..<typ.len:
+      var x = typ[i]
+      if x != nil: x = x.skipTypes(skipPtrs)
+      specializeResetT(p, accessor.parentObj(p.module), x)
+    if typ.n != nil: specializeResetN(p, accessor, typ.n, typ)
+  of tyTuple:
+    let typ = getUniqueType(typ)
+    for i in 0..<typ.len:
+      specializeResetT(p, ropecg(p.module, "$1.Field$2", [accessor, i]), typ[i])
+
+  of tyString, tyRef, tySequence:
+    lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1, NIM_NIL);$n", [accessor])
+
+  of tyProc:
+    if typ.callConv == ccClosure:
+      lineCg(p, cpsStmts, "#unsureAsgnRef((void**)&$1.ClE_0, NIM_NIL);$n", [accessor])
+      lineCg(p, cpsStmts, "$1.ClP_0 = NIM_NIL;$n", [accessor])
+    else:
+      lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  of tyChar, tyBool, tyEnum, tyInt..tyUInt64:
+    lineCg(p, cpsStmts, "$1 = 0;$n", [accessor])
+  of tyCString, tyPointer, tyPtr, tyVar, tyLent:
+    lineCg(p, cpsStmts, "$1 = NIM_NIL;$n", [accessor])
+  else:
+    discard
+
+proc specializeReset(p: BProc, a: TLoc) =
+  specializeResetT(p, rdLoc(a), a.t)

compiler/ccgthreadvars.nim

@@ -16,11 +16,11 @@ proc emulatedThreadVars(conf: ConfigRef): bool =
   result = {optThreads, optTlsEmulation} <= conf.globalOptions
 
 proc accessThreadLocalVar(p: BProc, s: PSym) =
-  if emulatedThreadVars(p.config) and not p.threadVarAccessed:
-    p.threadVarAccessed = true
+  if emulatedThreadVars(p.config) and threadVarAccessed notin p.flags:
+    p.flags.incl threadVarAccessed
     incl p.module.flags, usesThreadVars
-    addf(p.procSec(cpsLocals), "\tNimThreadVars* NimTV_;$n", [])
-    add(p.procSec(cpsInit),
+    p.procSec(cpsLocals).addf("\tNimThreadVars* NimTV_;$n", [])
+    p.procSec(cpsInit).add(
       ropecg(p.module, "\tNimTV_ = (NimThreadVars*) #GetThreadLocalVars();$n", []))
 
 proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
@@ -30,23 +30,26 @@ proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
     # allocator for it :-(
     if not containsOrIncl(m.g.nimtvDeclared, s.id):
       m.g.nimtvDeps.add(s.loc.t)
-      addf(m.g.nimtv, "$1 $2;$n", [getTypeDesc(m, s.loc.t), s.loc.r])
+      m.g.nimtv.addf("$1 $2;$n", [getTypeDesc(m, s.loc.t), s.loc.r])
   else:
-    if isExtern: add(m.s[cfsVars], "extern ")
-    if optThreads in m.config.globalOptions: add(m.s[cfsVars], "NIM_THREADVAR ")
-    add(m.s[cfsVars], getTypeDesc(m, s.loc.t))
-    addf(m.s[cfsVars], " $1;$n", [s.loc.r])
+    if isExtern: m.s[cfsVars].add("extern ")
+    elif lfExportLib in s.loc.flags: m.s[cfsVars].add("N_LIB_EXPORT_VAR ")
+    else: m.s[cfsVars].add("N_LIB_PRIVATE ")
+    if optThreads in m.config.globalOptions: m.s[cfsVars].add("NIM_THREADVAR ")
+    m.s[cfsVars].add(getTypeDesc(m, s.loc.t))
+    m.s[cfsVars].addf(" $1;$n", [s.loc.r])
 
 proc generateThreadLocalStorage(m: BModule) =
   if m.g.nimtv != nil and (usesThreadVars in m.flags or sfMainModule in m.module.flags):
     for t in items(m.g.nimtvDeps): discard getTypeDesc(m, t)
-    addf(m.s[cfsSeqTypes], "typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
+    finishTypeDescriptions(m)
+    m.s[cfsSeqTypes].addf("typedef struct {$1} NimThreadVars;$n", [m.g.nimtv])
 
 proc generateThreadVarsSize(m: BModule) =
   if m.g.nimtv != nil:
-    let externc = if m.config.cmd == cmdCompileToCpp or
+    let externc = if m.config.backend == backendCpp or
                        sfCompileToCpp in m.module.flags: "extern \"C\" "
                   else: ""
-    addf(m.s[cfsProcs],
+    m.s[cfsProcs].addf(
       "$#NI NimThreadVarsSize(){return (NI)sizeof(NimThreadVars);}$n",
       [externc.rope])

compiler/ccgtrav.nim

@@ -17,7 +17,7 @@ type
     visitorFrmt: string
 
 const
-  visitorFrmt  = "#nimGCvisit((void*)$1, $2);$n"
+  visitorFrmt = "#nimGCvisit((void*)$1, $2);$n"
 
 proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType)
 proc genCaseRange(p: BProc, branch: PNode)
@@ -28,18 +28,18 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, n: PNode;
   if n == nil: return
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      genTraverseProc(c, accessor, n.sons[i], typ)
+    for i in 0..<n.len:
+      genTraverseProc(c, accessor, n[i], typ)
   of nkRecCase:
-    if (n.sons[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
+    if (n[0].kind != nkSym): internalError(c.p.config, n.info, "genTraverseProc")
     var p = c.p
-    let disc = n.sons[0].sym
+    let disc = n[0].sym
     if disc.loc.r == nil: fillObjectFields(c.p.module, typ)
     if disc.loc.t == nil:
       internalError(c.p.config, n.info, "genTraverseProc()")
     lineF(p, cpsStmts, "switch ($1.$2) {$n", [accessor, disc.loc.r])
-    for i in 1 ..< sonsLen(n):
-      let branch = n.sons[i]
+    for i in 1..<n.len:
+      let branch = n[i]
       assert branch.kind in {nkOfBranch, nkElse}
       if branch.kind == nkOfBranch:
         genCaseRange(c.p, branch)
@@ -73,38 +73,39 @@ proc genTraverseProc(c: TTraversalClosure, accessor: Rope, typ: PType) =
      tySink, tyOwned:
     genTraverseProc(c, accessor, lastSon(typ))
   of tyArray:
-    let arraySize = lengthOrd(c.p.config, typ.sons[0])
+    let arraySize = lengthOrd(c.p.config, typ[0])
     var i: TLoc
-    getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo(), tyInt), i)
+    getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
     let oldCode = p.s(cpsStmts)
     linefmt(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
             [i.r, arraySize])
     let oldLen = p.s(cpsStmts).len
-    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", [accessor, i.r]), typ.sons[1])
+    genTraverseProc(c, ropecg(c.p.module, "$1[$2]", [accessor, i.r]), typ[1])
     if p.s(cpsStmts).len == oldLen:
       # do not emit dummy long loops for faster debug builds:
       p.s(cpsStmts) = oldCode
     else:
       lineF(p, cpsStmts, "}$n", [])
   of tyObject:
-    for i in 0 ..< sonsLen(typ):
-      var x = typ.sons[i]
+    for i in 0..<typ.len:
+      var x = typ[i]
       if x != nil: x = x.skipTypes(skipPtrs)
       genTraverseProc(c, accessor.parentObj(c.p.module), x)
     if typ.n != nil: genTraverseProc(c, accessor, typ.n, typ)
   of tyTuple:
     let typ = getUniqueType(typ)
-    for i in 0 ..< sonsLen(typ):
-      genTraverseProc(c, ropecg(c.p.module, "$1.Field$2", [accessor, i]), typ.sons[i])
+    for i in 0..<typ.len:
+      genTraverseProc(c, ropecg(c.p.module, "$1.Field$2", [accessor, i]), typ[i])
   of tyRef:
     lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
   of tySequence:
-    if tfHasAsgn notin typ.flags:
+    if optSeqDestructors notin c.p.module.config.globalOptions:
       lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
     elif containsGarbageCollectedRef(typ.lastSon):
       # destructor based seqs are themselves not traced but their data is, if
       # they contain a GC'ed type:
-      genTraverseProcSeq(c, accessor, typ)
+      lineCg(p, cpsStmts, "#nimGCvisitSeq((void*)$1, $2);$n", [accessor, c.visitorFrmt])
+      #genTraverseProcSeq(c, accessor, typ)
   of tyString:
     if tfHasAsgn notin typ.flags:
       lineCg(p, cpsStmts, visitorFrmt, [accessor, c.visitorFrmt])
@@ -118,7 +119,7 @@ proc genTraverseProcSeq(c: TTraversalClosure, accessor: Rope, typ: PType) =
   var p = c.p
   assert typ.kind == tySequence
   var i: TLoc
-  getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo(), tyInt), i)
+  getTemp(p, getSysType(c.p.module.g.graph, unknownLineInfo, tyInt), i)
   let oldCode = p.s(cpsStmts)
   var a: TLoc
   a.r = accessor
@@ -126,7 +127,7 @@ proc genTraverseProcSeq(c: TTraversalClosure, accessor: Rope, typ: PType) =
   lineF(p, cpsStmts, "for ($1 = 0; $1 < $2; $1++) {$n",
       [i.r, lenExpr(c.p, a)])
   let oldLen = p.s(cpsStmts).len
-  genTraverseProc(c, "$1$3[$2]" % [accessor, i.r, dataField(c.p)], typ.sons[0])
+  genTraverseProc(c, "$1$3[$2]" % [accessor, i.r, dataField(c.p)], typ[0])
   if p.s(cpsStmts).len == oldLen:
     # do not emit dummy long loops for faster debug builds:
     p.s(cpsStmts) = oldCode
@@ -154,11 +155,11 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope =
   if typ.kind == tySequence:
     genTraverseProcSeq(c, "a".rope, typ)
   else:
-    if skipTypes(typ.sons[0], typedescInst+{tyOwned}).kind == tyArray:
+    if skipTypes(typ[0], typedescInst+{tyOwned}).kind == tyArray:
       # C's arrays are broken beyond repair:
-      genTraverseProc(c, "a".rope, typ.sons[0])
+      genTraverseProc(c, "a".rope, typ[0])
     else:
-      genTraverseProc(c, "(*a)".rope, typ.sons[0])
+      genTraverseProc(c, "(*a)".rope, typ[0])
 
   let generatedProc = "$1 {$n$2$3$4}\n" %
         [header, p.s(cpsLocals), p.s(cpsInit), p.s(cpsStmts)]
@@ -167,12 +168,12 @@ proc genTraverseProc(m: BModule, origTyp: PType; sig: SigHash): Rope =
   m.s[cfsProcs].add(generatedProc)
 
   if hcrOn:
-    addf(m.s[cfsProcHeaders], "N_NIMCALL_PTR(void, $1)(void*, NI);\n", [result])
-    addf(m.s[cfsDynLibInit], "\t$1 = (N_NIMCALL_PTR(void, )(void*, NI)) hcrRegisterProc($3, \"$1\", (void*)$2);\n",
+    m.s[cfsProcHeaders].addf("N_NIMCALL_PTR(void, $1)(void*, NI);\n", [result])
+    m.s[cfsDynLibInit].addf("\t$1 = (N_NIMCALL_PTR(void, )(void*, NI)) hcrRegisterProc($3, \"$1\", (void*)$2);\n",
          [result, markerName, getModuleDllPath(m)])
 
 proc genTraverseProcForGlobal(m: BModule, s: PSym; info: TLineInfo): Rope =
-  discard genTypeInfo(m, s.loc.t, info)
+  discard genTypeInfoV1(m, s.loc.t, info)
 
   var c: TTraversalClosure
   var p = newProc(nil, m)

compiler/ccgutils.nim

@@ -16,11 +16,11 @@ import
 proc getPragmaStmt*(n: PNode, w: TSpecialWord): PNode =
   case n.kind
   of nkStmtList:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       result = getPragmaStmt(n[i], w)
       if result != nil: break
   of nkPragma:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       if whichPragma(n[i]) == w: return n[i]
   else: discard
 
@@ -33,7 +33,7 @@ proc hashString*(conf: ConfigRef; s: string): BiggestInt =
     # we have to use the same bitwidth
     # as the target CPU
     var b = 0'u64
-    for i in 0 ..< len(s):
+    for i in 0..<s.len:
       b = b + uint(s[i])
       b = b + (b shl 10)
       b = b xor (b shr 6)
@@ -43,7 +43,7 @@ proc hashString*(conf: ConfigRef; s: string): BiggestInt =
     result = cast[Hash](b)
   else:
     var a = 0'u32
-    for i in 0 ..< len(s):
+    for i in 0..<s.len:
       a = a + uint32(s[i])
       a = a + (a shl 10)
       a = a xor (a shr 6)
@@ -70,18 +70,18 @@ proc mangle*(name: string): string =
   template special(x) =
     result.add x
     requiresUnderscore = true
-  for i in start..(name.len-1):
+  for i in start..<name.len:
     let c = name[i]
     case c
     of 'a'..'z', '0'..'9', 'A'..'Z':
-      add(result, c)
+      result.add(c)
     of '_':
       # we generate names like 'foo_9' for scope disambiguations and so
       # disallow this here:
       if i > 0 and i < name.len-1 and name[i+1] in Digits:
         discard
       else:
-        add(result, c)
+        result.add(c)
     of '$': special "dollar"
     of '%': special "percent"
     of '&': special "amp"
@@ -102,7 +102,7 @@ proc mangle*(name: string): string =
     of '@': special "at"
     of '|': special "bar"
     else:
-      add(result, "X" & toHex(ord(c), 2))
+      result.add("X" & toHex(ord(c), 2))
       requiresUnderscore = true
   if requiresUnderscore:
     result.add "_"

compiler/cgendata.nim

@@ -11,7 +11,7 @@
 
 import
   ast, ropes, options, intsets,
-  tables, ndi, lineinfos, pathutils, modulegraphs
+  tables, ndi, lineinfos, pathutils, modulegraphs, sets
 
 type
   TLabel* = Rope              # for the C generator a label is just a rope
@@ -25,10 +25,10 @@ type
                               # this is needed for strange type generation
                               # reasons
     cfsFieldInfo,             # section for field information
-    cfsTypeInfo,              # section for type information
+    cfsTypeInfo,              # section for type information (ag ABI checks)
     cfsProcHeaders,           # section for C procs prototypes
-    cfsVars,                  # section for C variable declarations
     cfsData,                  # section for C constant data
+    cfsVars,                  # section for C variable declarations
     cfsProcs,                 # section for C procs that are not inline
     cfsInitProc,              # section for the C init proc
     cfsDatInitProc,           # section for the C datInit proc
@@ -58,22 +58,27 @@ type
     id*: int                  # the ID of the label; positive means that it
     label*: Rope              # generated text for the label
                               # nil if label is not used
-    sections*: TCProcSections # the code beloging
+    sections*: TCProcSections # the code belonging
     isLoop*: bool             # whether block is a loop
     nestedTryStmts*: int16    # how many try statements is it nested into
     nestedExceptStmts*: int16 # how many except statements is it nested into
     frameLen*: int16
 
+  TCProcFlag* = enum
+    beforeRetNeeded,
+    threadVarAccessed,
+    hasCurFramePointer,
+    noSafePoints,
+    nimErrorFlagAccessed,
+    nimErrorFlagDeclared,
+    nimErrorFlagDisabled
+
   TCProc = object             # represents C proc that is currently generated
     prc*: PSym                # the Nim proc that this C proc belongs to
-    beforeRetNeeded*: bool    # true iff 'BeforeRet' label for proc is needed
-    threadVarAccessed*: bool  # true if the proc already accessed some threadvar
-    hasCurFramePointer*: bool # true if _nimCurFrame var needed to recover after
-                              # exception is generated
-    noSafePoints*: bool       # the proc doesn't use safe points in exception handling
+    flags*: set[TCProcFlag]
     lastLineInfo*: TLineInfo  # to avoid generating excessive 'nimln' statements
     currLineInfo*: TLineInfo  # AST codegen will make this superfluous
-    nestedTryStmts*: seq[tuple[n: PNode, inExcept: bool]]
+    nestedTryStmts*: seq[tuple[fin: PNode, inExcept: bool, label: Natural]]
                               # in how many nested try statements we are
                               # (the vars must be volatile then)
                               # bool is true when are in the except part of a try block
@@ -86,14 +91,13 @@ type
     options*: TOptions        # options that should be used for code
                               # generation; this is the same as prc.options
                               # unless prc == nil
-    maxFrameLen*: int         # max length of frame descriptor
     module*: BModule          # used to prevent excessive parameter passing
     withinLoop*: int          # > 0 if we are within a loop
     splitDecls*: int          # > 0 if we are in some context for C++ that
                               # requires 'T x = T()' to become 'T x; x = T()'
                               # (yes, C++ is weird like that)
-    gcFrameId*: Natural       # for the GC stack marking
-    gcFrameType*: Rope        # the struct {} we put the GC markers into
+    withinTryWithExcept*: int # required for goto based exception handling
+    withinBlockLeaveActions*: int # complex to explain
     sigConflicts*: CountTable[string]
 
   TTypeSeq* = seq[PType]
@@ -117,9 +121,8 @@ type
     modulesClosed*: seq[BModule] # list of the same compiled modules, but in the order they were closed
     forwardedProcs*: seq[PSym] # proc:s that did not yet have a body
     generatedHeader*: BModule
-    breakPointId*: int
-    breakpoints*: Rope # later the breakpoints are inserted into the main proc
     typeInfoMarker*: TypeCacheWithOwner
+    typeInfoMarkerV2*: TypeCacheWithOwner
     config*: ConfigRef
     graph*: ModuleGraph
     strVersion*, seqVersion*: int # version of the string/seq implementation to use
@@ -144,15 +147,20 @@ type
                               # without extension)
     tmpBase*: Rope            # base for temp identifier generation
     typeCache*: TypeCache     # cache the generated types
+    typeABICache*: HashSet[SigHash] # cache for ABI checks; reusing typeCache
+                              # would be ideal but for some reason enums
+                              # don't seem to get cached so it'd generate
+                              # 1 ABI check per occurence in code
     forwTypeCache*: TypeCache # cache for forward declarations of types
     declaredThings*: IntSet   # things we have declared in this .c file
     declaredProtos*: IntSet   # prototypes we have declared in this .c file
     headerFiles*: seq[string] # needed headers to include
     typeInfoMarker*: TypeCache # needed for generating type information
+    typeInfoMarkerV2*: TypeCache
     initProc*: BProc          # code for init procedure
     preInitProc*: BProc       # code executed before the init proc
     hcrCreateTypeInfosProc*: Rope # type info globals are in here when HCR=on
-    inHcrInitGuard*: bool     # We are currently withing a HCR reloading guard.
+    inHcrInitGuard*: bool     # We are currently within a HCR reloading guard.
     typeStack*: TTypeSeq      # used for type generation
     dataCache*: TNodeTable
     typeNodes*, nimTypes*: int # used for type info generation
@@ -174,7 +182,7 @@ proc includeHeader*(this: BModule; header: string) =
 
 proc s*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # section in the current block
-  result = p.blocks[p.blocks.len-1].sections[s]
+  result = p.blocks[^1].sections[s]
 
 proc procSec*(p: BProc, s: TCProcSection): var Rope {.inline.} =
   # top level proc sections
@@ -184,8 +192,8 @@ proc newProc*(prc: PSym, module: BModule): BProc =
   new(result)
   result.prc = prc
   result.module = module
-  if prc != nil: result.options = prc.options
-  else: result.options = module.config.options
+  result.options = if prc != nil: prc.options
+                   else: module.config.options
   newSeq(result.blocks, 1)
   result.nestedTryStmts = @[]
   result.finallySafePoints = @[]

compiler/cgmeth.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-## This module implements code generation for multi methods.
+## This module implements code generation for methods.
 
 import
   intsets, options, ast, msgs, idents, renderer, types, magicsys,
@@ -23,11 +23,11 @@ proc genConv(n: PNode, d: PType, downcast: bool; conf: ConfigRef): PNode =
       result = n
     elif diff < 0:
       result = newNodeIT(nkObjUpConv, n.info, d)
-      addSon(result, n)
+      result.add n
       if downcast: internalError(conf, n.info, "cgmeth.genConv: no upcast allowed")
     elif diff > 0:
       result = newNodeIT(nkObjDownConv, n.info, d)
-      addSon(result, n)
+      result.add n
       if not downcast:
         internalError(conf, n.info, "cgmeth.genConv: no downcast allowed")
     else:
@@ -44,26 +44,26 @@ proc getDispatcher*(s: PSym): PSym =
 proc methodCall*(n: PNode; conf: ConfigRef): PNode =
   result = n
   # replace ordinary method by dispatcher method:
-  let disp = getDispatcher(result.sons[0].sym)
+  let disp = getDispatcher(result[0].sym)
   if disp != nil:
-    result.sons[0].sym = disp
+    result[0].sym = disp
     # change the arguments to up/downcasts to fit the dispatcher's parameters:
-    for i in 1 ..< sonsLen(result):
-      result.sons[i] = genConv(result.sons[i], disp.typ.sons[i], true, conf)
+    for i in 1..<result.len:
+      result[i] = genConv(result[i], disp.typ[i], true, conf)
   else:
-    localError(conf, n.info, "'" & $result.sons[0] & "' lacks a dispatcher")
+    localError(conf, n.info, "'" & $result[0] & "' lacks a dispatcher")
 
 type
   MethodResult = enum No, Invalid, Yes
 
 proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
   if a.name.id != b.name.id: return
-  if sonsLen(a.typ) != sonsLen(b.typ):
+  if a.typ.len != b.typ.len:
     return
 
-  for i in 1 ..< sonsLen(a.typ):
-    var aa = a.typ.sons[i]
-    var bb = b.typ.sons[i]
+  for i in 1..<a.typ.len:
+    var aa = a.typ[i]
+    var bb = b.typ[i]
     while true:
       aa = skipTypes(aa, {tyGenericInst, tyAlias})
       bb = skipTypes(bb, {tyGenericInst, tyAlias})
@@ -72,7 +72,7 @@ proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
         bb = bb.lastSon
       else:
         break
-    if sameType(a.typ.sons[i], b.typ.sons[i]):
+    if sameType(a.typ[i], b.typ[i]):
       if aa.kind == tyObject and result != Invalid:
         result = Yes
     elif aa.kind == tyObject and bb.kind == tyObject and (i == 1 or multiMethods):
@@ -90,22 +90,22 @@ proc sameMethodBucket(a, b: PSym; multiMethods: bool): MethodResult =
       return No
   if result == Yes:
     # check for return type:
-    if not sameTypeOrNil(a.typ.sons[0], b.typ.sons[0]):
-      if b.typ.sons[0] != nil and b.typ.sons[0].kind == tyUntyped:
+    if not sameTypeOrNil(a.typ[0], b.typ[0]):
+      if b.typ[0] != nil and b.typ[0].kind == tyUntyped:
         # infer 'auto' from the base to make it consistent:
-        b.typ.sons[0] = a.typ.sons[0]
+        b.typ[0] = a.typ[0]
       else:
         return No
 
 proc attachDispatcher(s: PSym, dispatcher: PNode) =
   if dispatcherPos < s.ast.len:
     # we've added a dispatcher already, so overwrite it
-    s.ast.sons[dispatcherPos] = dispatcher
+    s.ast[dispatcherPos] = dispatcher
   else:
     setLen(s.ast.sons, dispatcherPos+1)
     if s.ast[resultPos] == nil:
       s.ast[resultPos] = newNodeI(nkEmpty, s.info)
-    s.ast.sons[dispatcherPos] = dispatcher
+    s.ast[dispatcherPos] = dispatcher
 
 proc createDispatcher(s: PSym): PSym =
   var disp = copySym(s)
@@ -113,18 +113,18 @@ proc createDispatcher(s: PSym): PSym =
   excl(disp.flags, sfExported)
   disp.typ = copyType(disp.typ, disp.typ.owner, false)
   # we can't inline the dispatcher itself (for now):
-  if disp.typ.callConv == ccInline: disp.typ.callConv = ccDefault
+  if disp.typ.callConv == ccInline: disp.typ.callConv = ccNimCall
   disp.ast = copyTree(s.ast)
-  disp.ast.sons[bodyPos] = newNodeI(nkEmpty, s.info)
+  disp.ast[bodyPos] = newNodeI(nkEmpty, s.info)
   disp.loc.r = nil
-  if s.typ.sons[0] != nil:
-    if disp.ast.sonsLen > resultPos:
-      disp.ast.sons[resultPos].sym = copySym(s.ast.sons[resultPos].sym)
+  if s.typ[0] != nil:
+    if disp.ast.len > resultPos:
+      disp.ast[resultPos].sym = copySym(s.ast[resultPos].sym)
     else:
       # We've encountered a method prototype without a filled-in
       # resultPos slot. We put a placeholder in there that will
       # be updated in fixupDispatcher().
-      disp.ast.addSon(newNodeI(nkEmpty, s.info))
+      disp.ast.add newNodeI(nkEmpty, s.info)
   attachDispatcher(s, newSymNode(disp))
   # attach to itself to prevent bugs:
   attachDispatcher(disp, newSymNode(disp))
@@ -136,9 +136,9 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
   # from later definitions, particularly the resultPos slot. Also,
   # the lock level of the dispatcher needs to be updated/checked
   # against that of the method.
-  if disp.ast.sonsLen > resultPos and meth.ast.sonsLen > resultPos and
-     disp.ast.sons[resultPos].kind == nkEmpty:
-    disp.ast.sons[resultPos] = copyTree(meth.ast.sons[resultPos])
+  if disp.ast.len > resultPos and meth.ast.len > resultPos and
+     disp.ast[resultPos].kind == nkEmpty:
+    disp.ast[resultPos] = copyTree(meth.ast[resultPos])
 
   # The following code works only with lock levels, so we disable
   # it when they're not available.
@@ -158,13 +158,12 @@ proc fixupDispatcher(meth, disp: PSym; conf: ConfigRef) =
         disp.typ.lockLevel = meth.typ.lockLevel
 
 proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
-  let L = len(g.methods)
   var witness: PSym
-  for i in 0 ..< L:
+  for i in 0..<g.methods.len:
     let disp = g.methods[i].dispatcher
     case sameMethodBucket(disp, s, multimethods = optMultiMethods in g.config.globalOptions)
     of Yes:
-      add(g.methods[i].methods, s)
+      g.methods[i].methods.add(s)
       attachDispatcher(s, disp.ast[dispatcherPos])
       fixupDispatcher(s, disp, g.config)
       #echo "fixup ", disp.name.s, " ", disp.id
@@ -178,7 +177,7 @@ proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
     of Invalid:
       if witness.isNil: witness = g.methods[i].methods[0]
   # create a new dispatcher:
-  add(g.methods, (methods: @[s], dispatcher: createDispatcher(s)))
+  g.methods.add((methods: @[s], dispatcher: createDispatcher(s)))
   #echo "adding ", s.info
   #if fromCache:
   #  internalError(s.info, "no method dispatcher found")
@@ -190,32 +189,32 @@ proc methodDef*(g: ModuleGraph; s: PSym, fromCache: bool) =
 
 proc relevantCol(methods: seq[PSym], col: int): bool =
   # returns true iff the position is relevant
-  var t = methods[0].typ.sons[col].skipTypes(skipPtrs)
+  var t = methods[0].typ[col].skipTypes(skipPtrs)
   if t.kind == tyObject:
-    for i in 1 .. high(methods):
-      let t2 = skipTypes(methods[i].typ.sons[col], skipPtrs)
+    for i in 1..high(methods):
+      let t2 = skipTypes(methods[i].typ[col], skipPtrs)
       if not sameType(t2, t):
         return true
 
 proc cmpSignatures(a, b: PSym, relevantCols: IntSet): int =
-  for col in 1 ..< sonsLen(a.typ):
+  for col in 1..<a.typ.len:
     if contains(relevantCols, col):
-      var aa = skipTypes(a.typ.sons[col], skipPtrs)
-      var bb = skipTypes(b.typ.sons[col], skipPtrs)
+      var aa = skipTypes(a.typ[col], skipPtrs)
+      var bb = skipTypes(b.typ[col], skipPtrs)
       var d = inheritanceDiff(aa, bb)
       if (d != high(int)) and d != 0:
         return d
 
 proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
   # we use shellsort here; fast and simple
-  var n = len(a)
+  var n = a.len
   var h = 1
   while true:
     h = 3 * h + 1
     if h > n: break
   while true:
     h = h div 3
-    for i in h ..< n:
+    for i in h..<n:
       var v = a[i]
       var j = i
       while cmpSignatures(a[j - h], v, relevantCols) >= 0:
@@ -228,71 +227,70 @@ proc sortBucket(a: var seq[PSym], relevantCols: IntSet) =
 proc genDispatcher(g: ModuleGraph; methods: seq[PSym], relevantCols: IntSet): PSym =
   var base = methods[0].ast[dispatcherPos].sym
   result = base
-  var paramLen = sonsLen(base.typ)
+  var paramLen = base.typ.len
   var nilchecks = newNodeI(nkStmtList, base.info)
   var disp = newNodeI(nkIfStmt, base.info)
-  var ands = getSysMagic(g, unknownLineInfo(), "and", mAnd)
-  var iss = getSysMagic(g, unknownLineInfo(), "of", mOf)
-  let boolType = getSysType(g, unknownLineInfo(), tyBool)
-  for col in 1 ..< paramLen:
+  var ands = getSysMagic(g, unknownLineInfo, "and", mAnd)
+  var iss = getSysMagic(g, unknownLineInfo, "of", mOf)
+  let boolType = getSysType(g, unknownLineInfo, tyBool)
+  for col in 1..<paramLen:
     if contains(relevantCols, col):
-      let param = base.typ.n.sons[col].sym
+      let param = base.typ.n[col].sym
       if param.typ.skipTypes(abstractInst).kind in {tyRef, tyPtr}:
-        addSon(nilchecks, newTree(nkCall,
-            newSymNode(getCompilerProc(g, "chckNilDisp")), newSymNode(param)))
-  for meth in 0 .. high(methods):
+        nilchecks.add newTree(nkCall,
+            newSymNode(getCompilerProc(g, "chckNilDisp")), newSymNode(param))
+  for meth in 0..high(methods):
     var curr = methods[meth]      # generate condition:
     var cond: PNode = nil
-    for col in 1 ..< paramLen:
+    for col in 1..<paramLen:
       if contains(relevantCols, col):
         var isn = newNodeIT(nkCall, base.info, boolType)
-        addSon(isn, newSymNode(iss))
-        let param = base.typ.n.sons[col].sym
-        addSon(isn, newSymNode(param))
-        addSon(isn, newNodeIT(nkType, base.info, curr.typ.sons[col]))
+        isn.add newSymNode(iss)
+        let param = base.typ.n[col].sym
+        isn.add newSymNode(param)
+        isn.add newNodeIT(nkType, base.info, curr.typ[col])
         if cond != nil:
           var a = newNodeIT(nkCall, base.info, boolType)
-          addSon(a, newSymNode(ands))
-          addSon(a, cond)
-          addSon(a, isn)
+          a.add newSymNode(ands)
+          a.add cond
+          a.add isn
           cond = a
         else:
           cond = isn
-    let retTyp = base.typ.sons[0]
+    let retTyp = base.typ[0]
     let call = newNodeIT(nkCall, base.info, retTyp)
-    addSon(call, newSymNode(curr))
-    for col in 1 ..< paramLen:
-      addSon(call, genConv(newSymNode(base.typ.n.sons[col].sym),
-                           curr.typ.sons[col], false, g.config))
+    call.add newSymNode(curr)
+    for col in 1..<paramLen:
+      call.add genConv(newSymNode(base.typ.n[col].sym),
+                           curr.typ[col], false, g.config)
     var ret: PNode
     if retTyp != nil:
       var a = newNodeI(nkFastAsgn, base.info)
-      addSon(a, newSymNode(base.ast.sons[resultPos].sym))
-      addSon(a, call)
+      a.add newSymNode(base.ast[resultPos].sym)
+      a.add call
       ret = newNodeI(nkReturnStmt, base.info)
-      addSon(ret, a)
+      ret.add a
     else:
       ret = call
     if cond != nil:
       var a = newNodeI(nkElifBranch, base.info)
-      addSon(a, cond)
-      addSon(a, ret)
-      addSon(disp, a)
+      a.add cond
+      a.add ret
+      disp.add a
     else:
       disp = ret
   nilchecks.add disp
   nilchecks.flags.incl nfTransf # should not be further transformed
-  result.ast.sons[bodyPos] = nilchecks
+  result.ast[bodyPos] = nilchecks
 
 proc generateMethodDispatchers*(g: ModuleGraph): PNode =
   result = newNode(nkStmtList)
-  for bucket in 0 ..< len(g.methods):
+  for bucket in 0..<g.methods.len:
     var relevantCols = initIntSet()
-    for col in 1 ..< sonsLen(g.methods[bucket].methods[0].typ):
+    for col in 1..<g.methods[bucket].methods[0].typ.len:
       if relevantCol(g.methods[bucket].methods, col): incl(relevantCols, col)
       if optMultiMethods notin g.config.globalOptions:
         # if multi-methods are not enabled, we are interested only in the first field
         break
     sortBucket(g.methods[bucket].methods, relevantCols)
-    addSon(result,
-           newSymNode(genDispatcher(g, g.methods[bucket].methods, relevantCols)))
+    result.add newSymNode(genDispatcher(g, g.methods[bucket].methods, relevantCols))

compiler/closureiters.nim

@@ -90,9 +90,6 @@
 #  :unrollFinally = true
 #  goto nearestFinally (or -1 if not exists)
 #
-# Every finally block calls closureIterEndFinally() upon its successful
-# completion.
-#
 # Example:
 #
 # try:
@@ -126,13 +123,15 @@
 #     if :curExc.isNil:
 #       return :tmpResult
 #     else:
+#       closureIterSetupExc(nil)
 #       raise
 #   state = -1 # Goto next state. In this case we just exit
 #   break :stateLoop
 
 import
   ast, msgs, idents,
-  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos
+  renderer, magicsys, lowerings, lambdalifting, modulegraphs, lineinfos,
+  tables, options
 
 type
   Ctx = object
@@ -156,8 +155,8 @@ type
                     # is their finally. For finally it is parent finally. Otherwise -1
 
 const
-  nkSkip = { nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
-            nkCommentStmt } + procDefs
+  nkSkip = {nkEmpty..nkNilLit, nkTemplateDef, nkTypeSection, nkStaticStmt,
+            nkCommentStmt, nkMixinStmt, nkBindStmt} + procDefs
 
 proc newStateAccess(ctx: var Ctx): PNode =
   if ctx.stateVarSym.isNil:
@@ -174,7 +173,7 @@ proc newStateAssgn(ctx: var Ctx, toValue: PNode): PNode =
 proc newStateAssgn(ctx: var Ctx, stateNo: int = -2): PNode =
   # Creates state assignment:
   #   :state = stateNo
-  ctx.newStateAssgn(newIntTypeNode(nkIntLit, stateNo, ctx.g.getSysType(TLineInfo(), tyInt)))
+  ctx.newStateAssgn(newIntTypeNode(stateNo, ctx.g.getSysType(TLineInfo(), tyInt)))
 
 proc newEnvVar(ctx: var Ctx, name: string, typ: PType): PSym =
   result = newSym(skVar, getIdent(ctx.g.cache, name), ctx.fn, ctx.fn.info)
@@ -239,7 +238,7 @@ proc toStmtList(n: PNode): PNode =
 proc addGotoOut(n: PNode, gotoOut: PNode): PNode =
   # Make sure `n` is a stmtlist, and ends with `gotoOut`
   result = toStmtList(n)
-  if result.len == 0 or result.sons[^1].kind != nkGotoState:
+  if result.len == 0 or result[^1].kind != nkGotoState:
     result.add(gotoOut)
 
 proc newTempVar(ctx: var Ctx, typ: PType): PSym =
@@ -275,7 +274,7 @@ proc transformBreaksAndContinuesInWhile(ctx: var Ctx, n: PNode, before, after: P
     if ctx.blockLevel == 0:
       result = after
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformBreaksAndContinuesInWhile(n[i], before, after)
 
 proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode =
@@ -295,7 +294,7 @@ proc transformBreaksInBlock(ctx: var Ctx, n: PNode, label, after: PNode): PNode
       if label.kind == nkSym and n[0].sym == label.sym:
         result = after
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformBreaksInBlock(n[i], label, after)
 
 proc newNullifyCurExc(ctx: var Ctx, info: TLineInfo): PNode =
@@ -320,7 +319,7 @@ proc collectExceptState(ctx: var Ctx, n: PNode): PNode {.inline.} =
 
       if c.len > 1:
         var cond: PNode
-        for i in 0 .. c.len - 2:
+        for i in 0..<c.len - 1:
           assert(c[i].kind == nkType)
           let nextCond = newTree(nkCall,
             newSymNode(g.getSysMagic(c.info, "of", mOf)),
@@ -359,7 +358,7 @@ proc addElseToExcept(ctx: var Ctx, n: PNode) =
     block: # :unrollFinally = true
       branchBody.add(newTree(nkAsgn,
         ctx.newUnrollFinallyAccess(n.info),
-        newIntTypeNode(nkIntLit, 1, ctx.g.getSysType(n.info, tyBool))))
+        newIntTypeNode(1, ctx.g.getSysType(n.info, tyBool))))
 
     block: # :curExc = getCurrentException()
       branchBody.add(newTree(nkAsgn,
@@ -390,6 +389,10 @@ proc hasYieldsInExpressions(n: PNode): bool =
           return true
     else:
       result = n.hasYields
+  of nkCast:
+    for i in 1..<n.len:
+      if n[i].hasYieldsInExpressions:
+        return true
   else:
     for c in n:
       if c.hasYieldsInExpressions:
@@ -403,7 +406,7 @@ proc exprToStmtList(n: PNode): tuple[s, res: PNode] =
   var n = n
   while n.kind == nkStmtListExpr:
     result.s.sons.add(n.sons)
-    result.s.sons.setLen(result.s.sons.len - 1) # delete last son
+    result.s.sons.setLen(result.s.len - 1) # delete last son
     n = n[^1]
 
   result.res = n
@@ -437,7 +440,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkYieldStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -451,7 +454,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkPar, nkObjConstr, nkTupleConstr, nkBracket:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -461,7 +464,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       if n.typ.isNil: internalError(ctx.g.config, "lowerStmtListExprs: constr typ.isNil")
       result.typ = n.typ
 
-      for i in 0 ..< n.len:
+      for i in 0..<n.len:
         case n[i].kind
         of nkExprColonExpr:
           if n[i][1].kind == nkStmtListExpr:
@@ -477,13 +480,12 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkIfStmt, nkIfExpr:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
       needsSplit = true
       var tmp: PSym
-      var s: PNode
       let isExpr = not isEmptyType(n.typ)
       if isExpr:
         tmp = ctx.newTempVar(n.typ)
@@ -543,7 +545,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkTryStmt, nkHiddenTryStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -556,7 +558,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         let tmp = ctx.newTempVar(n.typ)
 
         n[0] = ctx.convertExprBodyToAsgn(n[0], tmp)
-        for i in 1 ..< n.len:
+        for i in 1..<n.len:
           let branch = n[i]
           case branch.kind
           of nkExceptBranch:
@@ -573,7 +575,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkCaseStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -591,7 +593,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
           result.add(st)
           n[0] = ex
 
-        for i in 1 ..< n.len:
+        for i in 1..<n.len:
           let branch = n[i]
           case branch.kind
           of nkOfBranch:
@@ -603,9 +605,9 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         result.add(n)
         result.add(ctx.newEnvVarAccess(tmp))
 
-  of nkCallKinds:
+  of nkCallKinds, nkChckRange, nkChckRangeF, nkChckRange64:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -645,7 +647,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         result.add(ifNode)
         result.add(ctx.newEnvVarAccess(tmp))
       else:
-        for i in 0 ..< n.len:
+        for i in 0..<n.len:
           if n[i].kind == nkStmtListExpr:
             let (st, ex) = exprToStmtList(n[i])
             result.add(st)
@@ -674,7 +676,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkDiscardStmt, nkReturnStmt, nkRaiseStmt:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -685,9 +687,10 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       n[0] = ex
       result.add(n)
 
-  of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv, nkObjDownConv:
+  of nkCast, nkHiddenStdConv, nkHiddenSubConv, nkConv, nkObjDownConv,
+      nkDerefExpr, nkHiddenDeref:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in ord(n.kind == nkCast)..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -701,7 +704,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
 
   of nkAsgn, nkFastAsgn:
     var ns = false
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], ns)
 
     if ns:
@@ -739,8 +742,6 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       result.add(n)
 
   of nkWhileStmt:
-    var ns = false
-
     var condNeedsSplit = false
     n[0] = ctx.lowerStmtListExprs(n[0], condNeedsSplit)
     var bodyNeedsSplit = false
@@ -759,7 +760,7 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
         n[0] = newSymNode(ctx.g.getSysSym(n[0].info, "true"))
         n[1] = newBody
 
-  of nkDotExpr:
+  of nkDotExpr, nkCheckedFieldExpr:
     var ns = false
     n[0] = ctx.lowerStmtListExprs(n[0], ns)
     if ns:
@@ -779,14 +780,14 @@ proc lowerStmtListExprs(ctx: var Ctx, n: PNode, needsSplit: var bool): PNode =
       result = newNodeI(nkStmtListExpr, n.info)
       result.typ = n.typ
       let (st, ex) = exprToStmtList(n[1])
-      n.kind = nkBlockStmt
+      n.transitionSonsKind(nkBlockStmt)
       n.typ = nil
       n[1] = st
       result.add(n)
       result.add(ex)
 
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.lowerStmtListExprs(n[i], needsSplit)
 
 proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
@@ -809,10 +810,11 @@ proc newEndFinallyNode(ctx: var Ctx, info: TLineInfo): PNode =
   let retStmt = newTree(nkReturnStmt, asgn)
   let branch = newTree(nkElifBranch, cmp, retStmt)
 
-  # The C++ backend requires `getCurrentException` here.
-  let raiseStmt = newTree(nkRaiseStmt, ctx.g.callCodegenProc("getCurrentException"))
+  let nullifyExc = newTree(nkCall, newSymNode(ctx.g.getCompilerProc("closureIterSetupExc")), nilnode)
+  nullifyExc.info = info
+  let raiseStmt = newTree(nkRaiseStmt, curExc)
   raiseStmt.info = info
-  let elseBranch = newTree(nkElse, raiseStmt)
+  let elseBranch = newTree(nkElse, newTree(nkStmtList, nullifyExc, raiseStmt))
 
   let ifBody = newTree(nkIfStmt, branch, elseBranch)
   let elifBranch = newTree(nkElifBranch, ctx.newUnrollFinallyAccess(info), ifBody)
@@ -832,13 +834,14 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
     block: # :unrollFinally = true
       let asgn = newNodeI(nkAsgn, n.info)
       asgn.add(ctx.newUnrollFinallyAccess(n.info))
-      asgn.add(newIntTypeNode(nkIntLit, 1, ctx.g.getSysType(n.info, tyBool)))
+      asgn.add(newIntTypeNode(1, ctx.g.getSysType(n.info, tyBool)))
       result.add(asgn)
 
     if n[0].kind != nkEmpty:
       let asgnTmpResult = newNodeI(nkAsgn, n.info)
       asgnTmpResult.add(ctx.newTmpResultAccess())
-      asgnTmpResult.add(n[0])
+      let x = if n[0].kind in {nkAsgn, nkFastAsgn}: n[0][1] else: n[0]
+      asgnTmpResult.add(x)
       result.add(asgnTmpResult)
 
     result.add(ctx.newNullifyCurExc(n.info))
@@ -849,7 +852,7 @@ proc transformReturnsInTry(ctx: var Ctx, n: PNode): PNode =
   of nkSkip:
     discard
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformReturnsInTry(n[i])
 
 proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode =
@@ -859,14 +862,14 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
 
   of nkStmtList, nkStmtListExpr:
     result = addGotoOut(result, gotoOut)
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       if n[i].hasYields:
         # Create a new split
         let go = newNodeI(nkGotoState, n[i].info)
         n[i] = ctx.transformClosureIteratorBody(n[i], go)
 
         let s = newNodeI(nkStmtList, n[i + 1].info)
-        for j in i + 1 ..< n.len:
+        for j in i + 1..<n.len:
           s.add(n[j])
 
         n.sons.setLen(i + 1)
@@ -889,7 +892,7 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
     result[^1] = ctx.transformClosureIteratorBody(result[^1], gotoOut)
 
   of nkIfStmt, nkCaseStmt:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
     if n[^1].kind != nkElse:
       # We don't have an else branch, but every possible branch has to end with
@@ -1001,7 +1004,7 @@ proc transformClosureIteratorBody(ctx: var Ctx, n: PNode, gotoOut: PNode): PNode
     internalError(ctx.g.config, "closure iter " & $n.kind)
 
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformClosureIteratorBody(n[i], gotoOut)
 
 proc stateFromGotoState(n: PNode): int =
@@ -1038,18 +1041,18 @@ proc transformStateAssignments(ctx: var Ctx, n: PNode): PNode =
       result.add(ctx.newStateAssgn(stateFromGotoState(n[1])))
 
       var retStmt = newNodeI(nkReturnStmt, n.info)
-      if n[0].sons[0].kind != nkEmpty:
-        var a = newNodeI(nkAsgn, n[0].sons[0].info)
-        var retVal = n[0].sons[0] #liftCapturedVars(n.sons[0], owner, d, c)
-        addSon(a, newSymNode(getClosureIterResult(ctx.g, ctx.fn)))
-        addSon(a, retVal)
+      if n[0][0].kind != nkEmpty:
+        var a = newNodeI(nkAsgn, n[0][0].info)
+        var retVal = n[0][0] #liftCapturedVars(n[0], owner, d, c)
+        a.add newSymNode(getClosureIterResult(ctx.g, ctx.fn))
+        a.add retVal
         retStmt.add(a)
       else:
         retStmt.add(ctx.g.emptyNode)
 
       result.add(retStmt)
     else:
-      for i in 0 ..< n.len:
+      for i in 0..<n.len:
         n[i] = ctx.transformStateAssignments(n[i])
 
   of nkSkip:
@@ -1069,7 +1072,7 @@ proc transformStateAssignments(ctx: var Ctx, n: PNode): PNode =
     result.add(breakState)
 
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.transformStateAssignments(n[i])
 
 proc skipStmtList(ctx: Ctx; n: PNode): PNode =
@@ -1101,7 +1104,7 @@ proc skipEmptyStates(ctx: Ctx, stateIdx: int): int =
 
   result = ctx.states[stateIdx][0].intVal.int
 
-proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
+proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode=
   result = n
   case n.kind
   of nkSkip:
@@ -1110,7 +1113,7 @@ proc skipThroughEmptyStates(ctx: var Ctx, n: PNode): PNode =
     result = copyTree(n)
     result[0].intVal = ctx.skipEmptyStates(result[0].intVal.int)
   else:
-    for i in 0 ..< n.len:
+    for i in 0..<n.len:
       n[i] = ctx.skipThroughEmptyStates(n[i])
 
 proc newArrayType(g: ModuleGraph; n: int, t: PType, owner: PSym): PType =
@@ -1162,7 +1165,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
     let cond = newTree(nkCall,
       ctx.g.getSysMagic(info, "==", mEqI).newSymNode(),
       ctx.newStateAccess(),
-      newIntTypeNode(nkIntLit, 0, intTyp))
+      newIntTypeNode(0, intTyp))
     cond.typ = boolTyp
 
     let raiseStmt = newTree(nkRaiseStmt, ctx.g.emptyNode)
@@ -1174,7 +1177,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
   block:
     let cond = newTree(nkCall,
       ctx.g.getSysMagic(info, "<", mLtI).newSymNode,
-      newIntTypeNode(nkIntLit, 0, intTyp),
+      newIntTypeNode(0, intTyp),
       ctx.newStateAccess())
     cond.typ = boolTyp
 
@@ -1186,7 +1189,7 @@ proc newCatchBody(ctx: var Ctx, info: TLineInfo): PNode {.inline.} =
     let cond = newTree(nkCall,
       ctx.g.getSysMagic(info, "<", mLtI).newSymNode,
       ctx.newStateAccess(),
-      newIntTypeNode(nkIntLit, 0, intTyp))
+      newIntTypeNode(0, intTyp))
     cond.typ = boolTyp
 
     let negateState = newTree(nkCall,
@@ -1281,19 +1284,117 @@ proc deleteEmptyStates(ctx: var Ctx) =
     else:
       inc i
 
+type
+  PreprocessContext = object
+    finallys: seq[PNode]
+    config: ConfigRef
+    blocks: seq[(PNode, int)]
+  FreshVarsContext = object
+    tab: Table[int, PSym]
+    config: ConfigRef
+    info: TLineInfo
+
+proc freshVars(n: PNode; c: var FreshVarsContext): PNode =
+  case n.kind
+  of nkSym:
+    let x = c.tab.getOrDefault(n.sym.id)
+    if x == nil:
+      result = n
+    else:
+      result = newSymNode(x, n.info)
+  of nkSkip - {nkSym}:
+    result = n
+  of nkLetSection, nkVarSection:
+    result = copyNode(n)
+    for it in n:
+      if it.kind in {nkIdentDefs, nkVarTuple}:
+        let idefs = copyNode(it)
+        for v in 0..it.len-3:
+          if it[v].kind == nkSym:
+            let x = copySym(it[v].sym)
+            c.tab[it[v].sym.id] = x
+            idefs.add newSymNode(x)
+          else:
+            idefs.add it[v]
+
+        for rest in it.len-2 ..< it.len: idefs.add it[rest]
+        result.add idefs
+      else:
+        result.add it
+  of nkRaiseStmt:
+    localError(c.config, c.info, "unsupported control flow: 'finally: ... raise' duplicated because of 'break'")
+  else:
+    result = n
+    for i in 0..<n.safeLen:
+      result[i] = freshVars(n[i], c)
+
+proc preprocess(c: var PreprocessContext; n: PNode): PNode =
+  # in order to fix bug #15243 without risking regressions, we preprocess
+  # the AST so that 'break' statements inside a 'try finally' also have the
+  # finally section. We need to duplicate local variables here and also
+  # detect: 'finally: raises X' which is currently not supported. We produce
+  # an error for this case for now. All this will be done properly with Yuriy's
+  # patch.
+  result = n
+  case n.kind
+  of nkTryStmt:
+    let f = n.lastSon
+    if f.kind == nkFinally:
+      c.finallys.add f.lastSon
+
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
+    if f.kind == nkFinally:
+      discard c.finallys.pop()
+
+  of nkWhileStmt, nkBlockStmt:
+    c.blocks.add((n, c.finallys.len))
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+    discard c.blocks.pop()
+
+  of nkBreakStmt:
+    if c.blocks.len == 0:
+      discard
+    else:
+      var fin = -1
+      if n[0].kind == nkEmpty:
+        fin = c.blocks[^1][1]
+      elif n[0].kind == nkSym:
+        for i in countdown(c.blocks.high, 0):
+          if c.blocks[i][0].kind == nkBlockStmt and c.blocks[i][0][0].kind == nkSym and
+              c.blocks[i][0][0].sym == n[0].sym:
+            fin = c.blocks[i][1]
+            break
+
+      if fin >= 0:
+        result = newNodeI(nkStmtList, n.info)
+        for i in countdown(c.finallys.high, fin):
+          var vars = FreshVarsContext(tab: initTable[int, PSym](), config: c.config, info: n.info)
+          result.add freshVars(preprocess(c, c.finallys[i]), vars)
+        result.add n
+  of nkSkip: discard
+  else:
+    for i in 0 ..< n.len:
+      result[i] = preprocess(c, n[i])
+
 proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   var ctx: Ctx
   ctx.g = g
   ctx.fn = fn
 
   if getEnvParam(fn).isNil:
-    # Lambda lifting was not done yet. Use temporary :state sym, which
+    # Lambda lifting was not done yet. Use temporary :state sym, which will
     # be handled specially by lambda lifting. Local temp vars (if needed)
-    # should folllow the same logic.
+    # should follow the same logic.
     ctx.stateVarSym = newSym(skVar, getIdent(ctx.g.cache, ":state"), fn, fn.info)
     ctx.stateVarSym.typ = g.createClosureIterStateType(fn)
   ctx.stateLoopLabel = newSym(skLabel, getIdent(ctx.g.cache, ":stateLoop"), fn, fn.info)
-  var n = n.toStmtList
+  var pc = PreprocessContext(finallys: @[], config: g.config)
+  var n = preprocess(pc, n.toStmtList)
+  #echo "transformed into ", n
+  #var n = n.toStmtList
 
   discard ctx.newState(n, nil)
   let gotoOut = newTree(nkGotoState, g.newIntLit(n.info, -1))
@@ -1310,7 +1411,7 @@ proc transformClosureIterator*(g: ModuleGraph; fn: PSym, n: PNode): PNode =
   # Optimize empty states away
   ctx.deleteEmptyStates()
 
-  # Make new body by concating the list of states
+  # Make new body by concatenating the list of states
   result = newNodeI(nkStmtList, n.info)
   for s in ctx.states:
     assert(s.len == 2)

compiler/cmdlinehelper.nim

@@ -7,18 +7,33 @@
 #    distribution, for details about the copyright.
 #
 
-## Helpers for binaries that use compiler passes, eg: nim, nimsuggest, nimfix
+## Helpers for binaries that use compiler passes, e.g.: nim, nimsuggest, nimfix
 
 import
-  options, idents, nimconf, scriptconfig, extccomp, commands, msgs,
-  lineinfos, modulegraphs, condsyms, os, pathutils
+  options, idents, nimconf, extccomp, commands, msgs,
+  lineinfos, modulegraphs, condsyms, os, pathutils, parseopt
+
+from strutils import normalize
+
+proc prependCurDir*(f: AbsoluteFile): AbsoluteFile =
+  when defined(unix):
+    if os.isAbsolute(f.string): result = f
+    else: result = AbsoluteFile("./" & f.string)
+  else:
+    result = f
+
+proc addCmdPrefix*(result: var string, kind: CmdLineKind) =
+  # consider moving this to std/parseopt
+  case kind
+  of cmdLongOption: result.add "--"
+  of cmdShortOption: result.add "-"
+  of cmdArgument, cmdEnd: discard
 
 type
   NimProg* = ref object
     suggestMode*: bool
     supportsStdinFile*: bool
     processCmdLine*: proc(pass: TCmdLinePass, cmd: string; config: ConfigRef)
-    mainCommand*: proc(graph: ModuleGraph)
 
 proc initDefinesProg*(self: NimProg, conf: ConfigRef, name: string) =
   condsyms.initDefines(conf.symbols)
@@ -27,10 +42,7 @@ proc initDefinesProg*(self: NimProg, conf: ConfigRef, name: string) =
 proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   self.processCmdLine(passCmd1, "", conf)
   if self.supportsStdinFile and conf.projectName == "-":
-    conf.projectName = "stdinfile"
-    conf.projectFull = AbsoluteFile "stdinfile"
-    conf.projectPath = AbsoluteDir getCurrentDir()
-    conf.projectIsStdin = true
+    handleStdinInput(conf)
   elif conf.projectName != "":
     try:
       conf.projectFull = canonicalizePath(conf, AbsoluteFile conf.projectName)
@@ -43,51 +55,28 @@ proc processCmdLineAndProjectPath*(self: NimProg, conf: ConfigRef) =
   else:
     conf.projectPath = AbsoluteDir canonicalizePath(conf, AbsoluteFile getCurrentDir())
 
-proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef): bool =
-  loadConfigs(DefaultConfig, cache, conf) # load all config files
+proc loadConfigsAndRunMainCommand*(self: NimProg, cache: IdentCache; conf: ConfigRef;
+                                   graph: ModuleGraph): bool =
   if self.suggestMode:
     conf.command = "nimsuggest"
+  loadConfigs(DefaultConfig, cache, conf) # load all config files
 
-  template runNimScriptIfExists(path: AbsoluteFile) =
-    let p = path # eval once
-    if fileExists(p):
-      runNimScript(cache, p, freshDefines = false, conf)
-
-  # Caution: make sure this stays in sync with `loadConfigs`
-  if optSkipSystemConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(getSystemConfigPath(conf, DefaultConfigNims))
-
-  if optSkipUserConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(getUserConfigPath(DefaultConfigNims))
-
-  if optSkipParentConfigFiles notin conf.globalOptions:
-    for dir in parentDirs(conf.projectPath.string, fromRoot = true, inclusive = false):
-      runNimScriptIfExists(AbsoluteDir(dir) / DefaultConfigNims)
-
-  if optSkipProjConfigFile notin conf.globalOptions:
-    runNimScriptIfExists(conf.projectPath / DefaultConfigNims)
-  block:
+  if not self.suggestMode:
     let scriptFile = conf.projectFull.changeFileExt("nims")
-    if not self.suggestMode:
-      runNimScriptIfExists(scriptFile)
-      # 'nim foo.nims' means to just run the NimScript file and do nothing more:
-      if fileExists(scriptFile) and scriptFile == conf.projectFull:
+    # 'nim foo.nims' means to just run the NimScript file and do nothing more:
+    if fileExists(scriptFile) and scriptFile == conf.projectFull:
+      if conf.command == "":
+        conf.command = "e"
+        return false
+      elif conf.command.normalize == "e":
         return false
-    else:
-      if scriptFile != conf.projectFull:
-        runNimScriptIfExists(scriptFile)
-      else:
-        # 'nimsuggest foo.nims' means to just auto-complete the NimScript file
-        discard
 
   # now process command line arguments again, because some options in the
-  # command line can overwite the config file's settings
+  # command line can overwrite the config file's settings
   extccomp.initVars(conf)
   self.processCmdLine(passCmd2, "", conf)
   if conf.command == "":
     rawMessage(conf, errGenerated, "command missing")
 
-  let graph = newModuleGraph(cache, conf)
   graph.suggestMode = self.suggestMode
-  self.mainCommand(graph)
   return true

compiler/commands.nim

@@ -18,7 +18,8 @@ template bootSwitch(name, expr, userString) =
 
 bootSwitch(usedRelease, defined(release), "-d:release")
 bootSwitch(usedDanger, defined(danger), "-d:danger")
-bootSwitch(usedGnuReadline, defined(useLinenoise), "-d:useLinenoise")
+# `useLinenoise` deprecated in favor of `nimUseLinenoise`, kept for backward compatibility
+bootSwitch(useLinenoise, defined(nimUseLinenoise) or defined(useLinenoise), "-d:nimUseLinenoise")
 bootSwitch(usedBoehm, defined(boehmgc), "--gc:boehm")
 bootSwitch(usedMarkAndSweep, defined(gcmarkandsweep), "--gc:markAndSweep")
 bootSwitch(usedGenerational, defined(gcgenerational), "--gc:generational")
@@ -31,6 +32,7 @@ import
   pathutils, strtabs
 
 from incremental import nimIncremental
+from ast import eqTypeFlags, tfGcSafe, tfNoSideEffect
 
 # but some have deps to imported modules. Yay.
 bootSwitch(usedTinyC, hasTinyCBackend, "-d:tinyc")
@@ -51,15 +53,14 @@ const
       "Copyright (c) 2006-" & copyrightYear & " by Andreas Rumpf\n"
 
 proc genFeatureDesc[T: enum](t: typedesc[T]): string {.compileTime.} =
-  var x = ""
-  for f in low(T)..high(T):
-    if x.len > 0: x.add "|"
-    x.add $f
-  x
+  result = ""
+  for f in T:
+    if result.len > 0: result.add "|"
+    result.add $f
 
 const
-  Usage = slurp"../doc/basicopt.txt".replace(" //", " ")
-  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", " ") % [genFeatureDesc(Feature), genFeatureDesc(LegacyFeature)]
+  Usage = slurp"../doc/basicopt.txt".replace(" //", "   ")
+  AdvancedUsage = slurp"../doc/advopt.txt".replace(" //", "   ") % [genFeatureDesc(Feature), genFeatureDesc(LegacyFeature)]
 
 proc getCommandLineDesc(conf: ConfigRef): string =
   result = (HelpMessage % [VersionAsString, platform.OS[conf.target.hostOS].name,
@@ -96,12 +97,12 @@ proc writeVersionInfo(conf: ConfigRef; pass: TCmdLinePass) =
                                  CPU[conf.target.hostCPU].name, CompileDate]),
                {msgStdout})
 
-    const gitHash = gorge("git log -n 1 --format=%H").strip
+    const gitHash {.strdefine.} = gorge("git log -n 1 --format=%H").strip
     when gitHash.len == 40:
       msgWriteln(conf, "git hash: " & gitHash, {msgStdout})
 
     msgWriteln(conf, "active boot switches:" & usedRelease & usedDanger &
-      usedTinyC & usedGnuReadline & usedNativeStacktrace &
+      usedTinyC & useLinenoise & usedNativeStacktrace &
       usedFFI & usedBoehm & usedMarkAndSweep & usedGenerational & usedGoGC & usedNoGC,
                {msgStdout})
     msgQuit(0)
@@ -110,7 +111,7 @@ proc writeCommandLineUsage*(conf: ConfigRef) =
   msgWriteln(conf, getCommandLineDesc(conf), {msgStdout})
 
 proc addPrefix(switch: string): string =
-  if len(switch) <= 1: result = "-" & switch
+  if switch.len <= 1: result = "-" & switch
   else: result = "--" & switch
 
 const
@@ -127,14 +128,14 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
                  info: TLineInfo) =
   cmd = ""
   var i = 0
-  if i < len(switch) and switch[i] == '-': inc(i)
-  if i < len(switch) and switch[i] == '-': inc(i)
-  while i < len(switch):
+  if i < switch.len and switch[i] == '-': inc(i)
+  if i < switch.len and switch[i] == '-': inc(i)
+  while i < switch.len:
     case switch[i]
-    of 'a'..'z', 'A'..'Z', '0'..'9', '_', '.': add(cmd, switch[i])
+    of 'a'..'z', 'A'..'Z', '0'..'9', '_', '.': cmd.add(switch[i])
     else: break
     inc(i)
-  if i >= len(switch): arg = ""
+  if i >= switch.len: arg = ""
   # cmd:arg => (cmd,arg)
   elif switch[i] in {':', '='}: arg = substr(switch, i + 1)
   # cmd[sub]:rest => (cmd,[sub]:rest)
@@ -144,24 +145,24 @@ proc splitSwitch(conf: ConfigRef; switch: string, cmd, arg: var string, pass: TC
 proc processOnOffSwitch(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass,
                         info: TLineInfo) =
   case arg.normalize
-  of "","on": conf.options = conf.options + op
-  of "off": conf.options = conf.options - op
+  of "","on": conf.options.incl op
+  of "off": conf.options.excl op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
 proc processOnOffSwitchOrList(conf: ConfigRef; op: TOptions, arg: string, pass: TCmdLinePass,
                               info: TLineInfo): bool =
   result = false
   case arg.normalize
-  of "on": conf.options = conf.options + op
-  of "off": conf.options = conf.options - op
+  of "on": conf.options.incl op
+  of "off": conf.options.excl op
   of "list": result = true
   else: localError(conf, info, errOnOffOrListExpectedButXFound % arg)
 
 proc processOnOffSwitchG(conf: ConfigRef; op: TGlobalOptions, arg: string, pass: TCmdLinePass,
                          info: TLineInfo) =
   case arg.normalize
-  of "", "on": conf.globalOptions = conf.globalOptions + op
-  of "off": conf.globalOptions = conf.globalOptions - op
+  of "", "on": conf.globalOptions.incl op
+  of "off": conf.globalOptions.excl op
   else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
 
 proc expectArg(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, info: TLineInfo) =
@@ -174,41 +175,53 @@ proc expectNoArg(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, info:
 
 proc processSpecificNote*(arg: string, state: TSpecialWord, pass: TCmdLinePass,
                          info: TLineInfo; orig: string; conf: ConfigRef) =
-  var id = ""  # arg = key:val or [key]:val;  with val=on|off
+  var id = ""  # arg = key or [key] or key:val or [key]:val;  with val=on|off
   var i = 0
   var n = hintMin
   var isBracket = false
-  if i < len(arg) and arg[i] == '[':
+  if i < arg.len and arg[i] == '[':
     isBracket = true
     inc(i)
-  while i < len(arg) and (arg[i] notin {':', '=', ']'}):
-    add(id, arg[i])
+  while i < arg.len and (arg[i] notin {':', '=', ']'}):
+    id.add(arg[i])
     inc(i)
   if isBracket:
-    if i < len(arg) and arg[i] == ']': inc(i)
+    if i < arg.len and arg[i] == ']': inc(i)
     else: invalidCmdLineOption(conf, pass, orig, info)
 
-  if i < len(arg) and (arg[i] in {':', '='}): inc(i)
+  if i == arg.len: discard
+  elif i < arg.len and (arg[i] in {':', '='}): inc(i)
   else: invalidCmdLineOption(conf, pass, orig, info)
   if state == wHint:
-    let x = findStr(lineinfos.HintsToStr, id)
-    if x >= 0: n = TNoteKind(x + ord(hintMin))
+    let x = findStr(hintMin..hintMax, id, errUnknown)
+    if x != errUnknown: n = TNoteKind(x)
     else: localError(conf, info, "unknown hint: " & id)
   else:
-    let x = findStr(lineinfos.WarningsToStr, id)
-    if x >= 0: n = TNoteKind(x + ord(warnMin))
+    let x = findStr(warnMin..warnMax, id, errUnknown)
+    if x != errUnknown: n = TNoteKind(x)
     else: localError(conf, info, "unknown warning: " & id)
-  case substr(arg, i).normalize
-  of "on":
-    incl(conf.notes, n)
-    incl(conf.mainPackageNotes, n)
-    incl(conf.enableNotes, n)
-  of "off":
-    excl(conf.notes, n)
-    excl(conf.mainPackageNotes, n)
-    incl(conf.disableNotes, n)
-    excl(conf.foreignPackageNotes, n)
-  else: localError(conf, info, errOnOrOffExpectedButXFound % arg)
+
+  var val = substr(arg, i).normalize
+  if val == "": val = "on"
+  if val notin ["on", "off"]:
+    localError(conf, info, errOnOrOffExpectedButXFound % arg)
+  elif n notin conf.cmdlineNotes or pass == passCmd1:
+    if pass == passCmd1: incl(conf.cmdlineNotes, n)
+    incl(conf.modifiedyNotes, n)
+    case val
+    of "on":
+      if state == wWarningAsError:
+        incl(conf.warningAsErrors, n)
+      else:
+        incl(conf.notes, n)
+        incl(conf.mainPackageNotes, n)
+    of "off":
+      if state == wWarningAsError:
+        excl(conf.warningAsErrors, n)
+      else:
+        excl(conf.notes, n)
+        excl(conf.mainPackageNotes, n)
+        excl(conf.foreignPackageNotes, n)
 
 proc processCompile(conf: ConfigRef; filename: string) =
   var found = findFile(conf, filename)
@@ -216,22 +229,25 @@ proc processCompile(conf: ConfigRef; filename: string) =
   extccomp.addExternalFileToCompile(conf, found)
 
 const
-  errNoneBoehmRefcExpectedButXFound = "'none', 'boehm' or 'refc' expected, but '$1' found"
+  errNoneBoehmRefcExpectedButXFound = "'arc', 'orc', 'markAndSweep', 'boehm', 'go', 'none', 'regions', or 'refc' expected, but '$1' found"
   errNoneSpeedOrSizeExpectedButXFound = "'none', 'speed' or 'size' expected, but '$1' found"
   errGuiConsoleOrLibExpectedButXFound = "'gui', 'console' or 'lib' expected, but '$1' found"
+  errInvalidExceptionSystem = "'goto', 'setjump', 'cpp' or 'quirky' expected, but '$1' found"
 
 proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo): bool =
   case switch.normalize
   of "gc":
     case arg.normalize
-    of "boehm":        result = conf.selectedGC == gcBoehm
-    of "refc":         result = conf.selectedGC == gcRefc
-    of "v2":           result = false
+    of "boehm": result = conf.selectedGC == gcBoehm
+    of "refc": result = conf.selectedGC == gcRefc
+    of "v2": result = false
     of "markandsweep": result = conf.selectedGC == gcMarkAndSweep
     of "generational": result = false
-    of "destructors":  result = conf.selectedGC == gcDestructors
-    of "go":           result = conf.selectedGC == gcGo
-    of "none":         result = conf.selectedGC == gcNone
+    of "destructors", "arc": result = conf.selectedGC == gcArc
+    of "orc": result = conf.selectedGC == gcOrc
+    of "hooks": result = conf.selectedGC == gcHooks
+    of "go": result = conf.selectedGC == gcGo
+    of "none": result = conf.selectedGC == gcNone
     of "stack", "regions": result = conf.selectedGC == gcRegions
     else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "opt":
@@ -243,15 +259,22 @@ proc testCompileOptionArg*(conf: ConfigRef; switch, arg: string, info: TLineInfo
   of "verbosity": result = $conf.verbosity == arg
   of "app":
     case arg.normalize
-    of "gui":       result = contains(conf.globalOptions, optGenGuiApp)
-    of "console":   result = not contains(conf.globalOptions, optGenGuiApp)
-    of "lib":       result = contains(conf.globalOptions, optGenDynLib) and
+    of "gui": result = contains(conf.globalOptions, optGenGuiApp)
+    of "console": result = not contains(conf.globalOptions, optGenGuiApp)
+    of "lib": result = contains(conf.globalOptions, optGenDynLib) and
                       not contains(conf.globalOptions, optGenGuiApp)
     of "staticlib": result = contains(conf.globalOptions, optGenStaticLib) and
                       not contains(conf.globalOptions, optGenGuiApp)
     else: localError(conf, info, errGuiConsoleOrLibExpectedButXFound % arg)
   of "dynliboverride":
     result = isDynlibOverride(conf, arg)
+  of "exceptions":
+    case arg.normalize
+    of "cpp": result = conf.exc == excCpp
+    of "setjmp": result = conf.exc == excSetjmp
+    of "quirky": result = conf.exc == excQuirky
+    of "goto": result = conf.exc == excGoto
+    else: localError(conf, info, errInvalidExceptionSystem % arg)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool =
@@ -265,8 +288,9 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "hints": result = contains(conf.options, optHints)
   of "threadanalysis": result = contains(conf.globalOptions, optThreadAnalysis)
   of "stacktrace": result = contains(conf.options, optStackTrace)
+  of "stacktracemsgs": result = contains(conf.options, optStackTraceMsgs)
   of "linetrace": result = contains(conf.options, optLineTrace)
-  of "debugger": result = contains(conf.options, optEndb)
+  of "debugger": result = contains(conf.globalOptions, optCDebug)
   of "profiler": result = contains(conf.options, optProfiler)
   of "memtracker": result = contains(conf.options, optMemTracker)
   of "checks", "x": result = conf.options * ChecksOptions == ChecksOptions
@@ -274,13 +298,14 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
     result = conf.options * {optNaNCheck, optInfCheck} == {optNaNCheck, optInfCheck}
   of "infchecks": result = contains(conf.options, optInfCheck)
   of "nanchecks": result = contains(conf.options, optNaNCheck)
-  of "nilchecks": result = contains(conf.options, optNilCheck)
+  of "nilchecks": result = false # not a thing
   of "objchecks": result = contains(conf.options, optObjCheck)
   of "fieldchecks": result = contains(conf.options, optFieldCheck)
   of "rangechecks": result = contains(conf.options, optRangeCheck)
   of "boundchecks": result = contains(conf.options, optBoundsCheck)
   of "refchecks": result = contains(conf.options, optRefCheck)
   of "overflowchecks": result = contains(conf.options, optOverflowCheck)
+  of "staticboundchecks": result = contains(conf.options, optStaticBoundsCheck)
   of "stylechecks": result = contains(conf.options, optStyleCheck)
   of "linedir": result = contains(conf.options, optLineDir)
   of "assertions", "a": result = contains(conf.options, optAssert)
@@ -294,7 +319,6 @@ proc testCompileOption*(conf: ConfigRef; switch: string, info: TLineInfo): bool
   of "patterns", "trmacros": result = contains(conf.options, optTrMacros)
   of "excessivestacktrace": result = contains(conf.globalOptions, optExcessiveStackTrace)
   of "nilseqs": result = contains(conf.options, optNilSeqs)
-  of "oldast": result = contains(conf.options, optOldAst)
   else: invalidCmdLineOption(conf, passCmd1, switch, info)
 
 proc processPath(conf: ConfigRef; path: string, info: TLineInfo,
@@ -359,6 +383,14 @@ proc dynlibOverride(conf: ConfigRef; switch, arg: string, pass: TCmdLinePass, in
     expectArg(conf, switch, arg, pass, info)
     options.inclDynlibOverride(conf, arg)
 
+proc handleStdinInput*(conf: ConfigRef) =
+  conf.projectName = "stdinfile"
+  conf.projectFull = conf.projectName.AbsoluteFile
+  conf.projectPath = AbsoluteDir getCurrentDir()
+  conf.projectIsStdin = true
+  if conf.outDir.isEmpty:
+    conf.outDir = getNimcacheDir(conf)
+
 proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
                     conf: ConfigRef) =
   var
@@ -366,8 +398,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   case switch.normalize
   of "path", "p":
     expectArg(conf, switch, arg, pass, info)
-    addPath(conf, if pass == passPP: processCfgPath(conf, arg, info)
-                  else: processPath(conf, arg, info), info)
+    for path in nimbleSubs(conf, arg):
+      addPath(conf, if pass == passPP: processCfgPath(conf, path, info)
+                    else: processPath(conf, path, info), info)
   of "nimblepath", "babelpath":
     # keep the old name for compat
     if pass in {passCmd2, passPP} and optNoNimblePath notin conf.globalOptions:
@@ -380,6 +413,9 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "nonimblepath", "nobabelpath":
     expectNoArg(conf, switch, arg, pass, info)
     disableNimblePath(conf)
+  of "clearnimblepath":
+    expectNoArg(conf, switch, arg, pass, info)
+    clearNimblePath(conf)
   of "excludepath":
     expectArg(conf, switch, arg, pass, info)
     let path = processPath(conf, arg, info)
@@ -387,26 +423,39 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     conf.lazyPaths.keepItIf(it != path)
   of "nimcache":
     expectArg(conf, switch, arg, pass, info)
-    conf.nimcacheDir = processPath(conf, arg, info, true)
+    conf.nimcacheDir = processPath(conf, arg, info, notRelativeToProj=true)
   of "out", "o":
     expectArg(conf, switch, arg, pass, info)
-    let f = splitFile(arg.expandTilde)
+    let f = splitFile(processPath(conf, arg, info, notRelativeToProj=true).string)
     conf.outFile = RelativeFile f.name & f.ext
     conf.outDir = toAbsoluteDir f.dir
   of "outdir":
     expectArg(conf, switch, arg, pass, info)
-    conf.outDir = toAbsoluteDir arg.expandTilde
+    conf.outDir = processPath(conf, arg, info, notRelativeToProj=true)
+  of "usenimcache":
+    processOnOffSwitchG(conf, {optUseNimcache}, arg, pass, info)
   of "docseesrcurl":
     expectArg(conf, switch, arg, pass, info)
     conf.docSeeSrcUrl = arg
+  of "docroot":
+    conf.docRoot = if arg.len == 0: docRootDefault else: arg
+  of "backend", "b":
+    let backend = parseEnum(arg.normalize, TBackend.default)
+    if backend == TBackend.default: localError(conf, info, "invalid backend: '$1'" % arg)
+    conf.backend = backend
+  of "doccmd": conf.docCmd = arg
   of "mainmodule", "m":
     discard "allow for backwards compatibility, but don't do anything"
   of "define", "d":
     expectArg(conf, switch, arg, pass, info)
     if {':', '='} in arg:
       splitSwitch(conf, arg, key, val, pass, info)
+      if cmpIgnoreStyle(key, "nimQuirky") == 0:
+        conf.exc = excQuirky
       defineSymbol(conf.symbols, key, val)
     else:
+      if cmpIgnoreStyle(arg, "nimQuirky") == 0:
+        conf.exc = excQuirky
       defineSymbol(conf.symbols, arg)
   of "undef", "u":
     expectArg(conf, switch, arg, pass, info)
@@ -434,63 +483,88 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "forcebuild", "f":
     processOnOffSwitchG(conf, {optForceFullMake}, arg, pass, info)
   of "project":
-    processOnOffSwitchG(conf, {optWholeProject}, arg, pass, info)
+    processOnOffSwitchG(conf, {optWholeProject, optGenIndex}, arg, pass, info)
   of "gc":
+    if conf.backend == backendJs or conf.command == "js":
+      # for: bug #16033
+      # This might still be imperfect, in rarse corner cases
+      # (where command is reset in nimscript, maybe).
+      return
     expectArg(conf, switch, arg, pass, info)
-    case arg.normalize
-    of "boehm":
-      conf.selectedGC = gcBoehm
-      defineSymbol(conf.symbols, "boehmgc")
-    of "refc":
-      conf.selectedGC = gcRefc
-    of "v2":
-      message(conf, info, warnDeprecated, "--gc:v2 is deprecated; using default gc")
-    of "markandsweep":
-      conf.selectedGC = gcMarkAndSweep
-      defineSymbol(conf.symbols, "gcmarkandsweep")
-    of "destructors":
-      conf.selectedGC = gcDestructors
-      defineSymbol(conf.symbols, "gcdestructors")
-    of "go":
-      conf.selectedGC = gcGo
-      defineSymbol(conf.symbols, "gogc")
-    of "none":
-      conf.selectedGC = gcNone
-      defineSymbol(conf.symbols, "nogc")
-    of "stack", "regions":
-      conf.selectedGC= gcRegions
-      defineSymbol(conf.symbols, "gcregions")
-    else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
+    if pass in {passCmd2, passPP}:
+      case arg.normalize
+      of "boehm":
+        conf.selectedGC = gcBoehm
+        defineSymbol(conf.symbols, "boehmgc")
+        incl conf.globalOptions, optTlsEmulation # Boehm GC doesn't scan the real TLS
+      of "refc":
+        conf.selectedGC = gcRefc
+      of "v2":
+        message(conf, info, warnDeprecated, "--gc:v2 is deprecated; using default gc")
+      of "markandsweep":
+        conf.selectedGC = gcMarkAndSweep
+        defineSymbol(conf.symbols, "gcmarkandsweep")
+      of "destructors", "arc":
+        conf.selectedGC = gcArc
+        defineSymbol(conf.symbols, "gcdestructors")
+        defineSymbol(conf.symbols, "gcarc")
+        incl conf.globalOptions, optSeqDestructors
+        incl conf.globalOptions, optTinyRtti
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+          defineSymbol(conf.symbols, "nimV2")
+      of "orc":
+        conf.selectedGC = gcOrc
+        defineSymbol(conf.symbols, "gcdestructors")
+        defineSymbol(conf.symbols, "gcorc")
+        incl conf.globalOptions, optSeqDestructors
+        incl conf.globalOptions, optTinyRtti
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+          defineSymbol(conf.symbols, "nimV2")
+      of "hooks":
+        conf.selectedGC = gcHooks
+        defineSymbol(conf.symbols, "gchooks")
+        incl conf.globalOptions, optSeqDestructors
+        processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
+        if pass in {passCmd2, passPP}:
+          defineSymbol(conf.symbols, "nimSeqsV2")
+      of "go":
+        conf.selectedGC = gcGo
+        defineSymbol(conf.symbols, "gogc")
+      of "none":
+        conf.selectedGC = gcNone
+        defineSymbol(conf.symbols, "nogc")
+      of "stack", "regions":
+        conf.selectedGC = gcRegions
+        defineSymbol(conf.symbols, "gcregions")
+      else: localError(conf, info, errNoneBoehmRefcExpectedButXFound % arg)
   of "warnings", "w":
     if processOnOffSwitchOrList(conf, {optWarns}, arg, pass, info): listWarnings(conf)
   of "warning": processSpecificNote(arg, wWarning, pass, info, switch, conf)
   of "hint": processSpecificNote(arg, wHint, pass, info, switch, conf)
+  of "warningaserror": processSpecificNote(arg, wWarningAsError, pass, info, switch, conf)
   of "hints":
     if processOnOffSwitchOrList(conf, {optHints}, arg, pass, info): listHints(conf)
   of "threadanalysis": processOnOffSwitchG(conf, {optThreadAnalysis}, arg, pass, info)
   of "stacktrace": processOnOffSwitch(conf, {optStackTrace}, arg, pass, info)
+  of "stacktracemsgs": processOnOffSwitch(conf, {optStackTraceMsgs}, arg, pass, info)
   of "excessivestacktrace": processOnOffSwitchG(conf, {optExcessiveStackTrace}, arg, pass, info)
   of "linetrace": processOnOffSwitch(conf, {optLineTrace}, arg, pass, info)
   of "debugger":
     case arg.normalize
-    of "on", "endb":
-      conf.options.incl optEndb
-      defineSymbol(conf.symbols, "endb")
-    of "off":
-      conf.options.excl optEndb
-      undefSymbol(conf.symbols, "endb")
-    of "native", "gdb":
-      incl(conf.globalOptions, optCDebug)
-      conf.options = conf.options + {optLineDir} - {optEndb}
+    of "on", "native", "gdb":
+      conf.globalOptions.incl optCDebug
+      conf.options.incl optLineDir
       #defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
-      undefSymbol(conf.symbols, "endb")
+    of "off":
+      conf.globalOptions.excl optCDebug
     else:
-      localError(conf, info, "expected endb|gdb but found " & arg)
+      localError(conf, info, "expected native|gdb|on|off but found " & arg)
   of "g": # alias for --debugger:native
-    incl(conf.globalOptions, optCDebug)
-    conf.options = conf.options + {optLineDir} - {optEndb}
+    conf.globalOptions.incl optCDebug
+    conf.options.incl optLineDir
     #defineSymbol(conf.symbols, "nimTypeNames") # type names are used in gdb pretty printing
-    undefSymbol(conf.symbols, "endb")
   of "profiler":
     processOnOffSwitch(conf, {optProfiler}, arg, pass, info)
     if optProfiler in conf.options: defineSymbol(conf.symbols, "profiler")
@@ -511,38 +585,27 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     else:
       undefSymbol(conf.symbols, "hotcodereloading")
       undefSymbol(conf.symbols, "useNimRtl")
-  of "oldnewlines":
-    case arg.normalize
-    of "","on":
-      conf.oldNewlines = true
-      defineSymbol(conf.symbols, "nimOldNewlines")
-    of "off":
-      conf.oldNewlines = false
-      undefSymbol(conf.symbols, "nimOldNewlines")
-    else:
-      localError(conf, info, errOnOrOffExpectedButXFound % arg)
-  of "laxstrings": processOnOffSwitch(conf, {optLaxStrings}, arg, pass, info)
   of "nilseqs": processOnOffSwitch(conf, {optNilSeqs}, arg, pass, info)
-  of "oldast": processOnOffSwitch(conf, {optOldAst}, arg, pass, info)
   of "checks", "x": processOnOffSwitch(conf, ChecksOptions, arg, pass, info)
   of "floatchecks":
     processOnOffSwitch(conf, {optNaNCheck, optInfCheck}, arg, pass, info)
   of "infchecks": processOnOffSwitch(conf, {optInfCheck}, arg, pass, info)
   of "nanchecks": processOnOffSwitch(conf, {optNaNCheck}, arg, pass, info)
-  of "nilchecks": processOnOffSwitch(conf, {optNilCheck}, arg, pass, info)
+  of "nilchecks": discard "no such thing anymore"
   of "objchecks": processOnOffSwitch(conf, {optObjCheck}, arg, pass, info)
   of "fieldchecks": processOnOffSwitch(conf, {optFieldCheck}, arg, pass, info)
   of "rangechecks": processOnOffSwitch(conf, {optRangeCheck}, arg, pass, info)
   of "boundchecks": processOnOffSwitch(conf, {optBoundsCheck}, arg, pass, info)
   of "refchecks": processOnOffSwitch(conf, {optRefCheck}, arg, pass, info)
   of "overflowchecks": processOnOffSwitch(conf, {optOverflowCheck}, arg, pass, info)
+  of "staticboundchecks": processOnOffSwitch(conf, {optStaticBoundsCheck}, arg, pass, info)
   of "stylechecks": processOnOffSwitch(conf, {optStyleCheck}, arg, pass, info)
   of "linedir": processOnOffSwitch(conf, {optLineDir}, arg, pass, info)
   of "assertions", "a": processOnOffSwitch(conf, {optAssert}, arg, pass, info)
   of "deadcodeelim": discard # deprecated, dead code elim always on
   of "threads":
     processOnOffSwitchG(conf, {optThreads}, arg, pass, info)
-    #if optThreads in conf.globalOptions: incl(conf.notes, warnGcUnsafe)
+    #if optThreads in conf.globalOptions: conf.setNote(warnGcUnsafe)
   of "tlsemulation": processOnOffSwitchG(conf, {optTlsEmulation}, arg, pass, info)
   of "taintmode": processOnOffSwitchG(conf, {optTaintMode}, arg, pass, info)
   of "implicitstatic":
@@ -615,28 +678,31 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       conf.implicitIncludes.add findModule(conf, arg, toFullPath(conf, info)).string
   of "listcmd":
     processOnOffSwitchG(conf, {optListCmd}, arg, pass, info)
+  of "asm":
+    processOnOffSwitchG(conf, {optProduceAsm}, arg, pass, info)
   of "genmapping":
     processOnOffSwitchG(conf, {optGenMapping}, arg, pass, info)
   of "os":
     expectArg(conf, switch, arg, pass, info)
-    if pass in {passCmd1, passPP}:
-      let theOS = platform.nameToOS(arg)
-      if theOS == osNone:
-        let osList = platform.listOSnames().join(", ")
-        localError(conf, info, "unknown OS: '$1'. Available options are: $2" % [arg, $osList])
-      elif theOS != conf.target.hostOS:
-        setTarget(conf.target, theOS, conf.target.targetCPU)
+    let theOS = platform.nameToOS(arg)
+    if theOS == osNone:
+      let osList = platform.listOSnames().join(", ")
+      localError(conf, info, "unknown OS: '$1'. Available options are: $2" % [arg, $osList])
+    else:
+      setTarget(conf.target, theOS, conf.target.targetCPU)
   of "cpu":
     expectArg(conf, switch, arg, pass, info)
-    if pass in {passCmd1, passPP}:
-      let cpu = platform.nameToCPU(arg)
-      if cpu == cpuNone:
-        let cpuList = platform.listCPUnames().join(", ")
-        localError(conf, info, "unknown CPU: '$1'. Available options are: $2" % [ arg, cpuList])
-      elif cpu != conf.target.hostCPU:
-        setTarget(conf.target, conf.target.targetOS, cpu)
+    let cpu = platform.nameToCPU(arg)
+    if cpu == cpuNone:
+      let cpuList = platform.listCPUnames().join(", ")
+      localError(conf, info, "unknown CPU: '$1'. Available options are: $2" % [ arg, cpuList])
+    else:
+      setTarget(conf.target, conf.target.targetOS, cpu)
   of "run", "r":
     processOnOffSwitchG(conf, {optRun}, arg, pass, info)
+  of "maxloopiterationsvm":
+    expectArg(conf, switch, arg, pass, info)
+    conf.maxLoopIterationsVM = parseInt(arg)
   of "errormax":
     expectArg(conf, switch, arg, pass, info)
     # Note: `nim check` (etc) can overwrite this.
@@ -651,9 +717,10 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if verbosity notin {0..3}:
       localError(conf, info, "invalid verbosity level: '$1'" % arg)
     conf.verbosity = verbosity
-    conf.notes = NotesVerbosity[conf.verbosity]
-    incl(conf.notes, conf.enableNotes)
-    excl(conf.notes, conf.disableNotes)
+    var verb = NotesVerbosity[conf.verbosity]
+    ## We override the default `verb` by explicitly modified (set/unset) notes.
+    conf.notes = (conf.modifiedyNotes * conf.notes + verb) -
+      (conf.modifiedyNotes * verb - conf.notes)
     conf.mainPackageNotes = conf.notes
   of "parallelbuild":
     expectArg(conf, switch, arg, pass, info)
@@ -752,8 +819,15 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
       localError(conf, info, "unknown obsolete feature")
   of "nocppexceptions":
     expectNoArg(conf, switch, arg, pass, info)
-    incl(conf.globalOptions, optNoCppExceptions)
+    conf.exc = low(ExceptionSystem)
     defineSymbol(conf.symbols, "noCppExceptions")
+  of "exceptions":
+    case arg.normalize
+    of "cpp": conf.exc = excCpp
+    of "setjmp": conf.exc = excSetjmp
+    of "quirky": conf.exc = excQuirky
+    of "goto": conf.exc = excGoto
+    else: localError(conf, info, errInvalidExceptionSystem % arg)
   of "cppdefine":
     expectArg(conf, switch, arg, pass, info)
     if conf != nil:
@@ -763,14 +837,22 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
     if pass in {passCmd2, passPP}:
       doAssert(conf != nil)
       incl(conf.features, destructor)
-      incl(conf.globalOptions, optNimV2)
+      incl(conf.globalOptions, optTinyRtti)
+      incl(conf.globalOptions, optOwnedRefs)
+      incl(conf.globalOptions, optSeqDestructors)
       defineSymbol(conf.symbols, "nimV2")
-      conf.selectedGC = gcDestructors
-      defineSymbol(conf.symbols, "gcdestructors")
+      conf.selectedGC = gcHooks
+      defineSymbol(conf.symbols, "gchooks")
+      defineSymbol(conf.symbols, "nimSeqsV2")
+      defineSymbol(conf.symbols, "nimOwnedEnabled")
+  of "seqsv2":
+    processOnOffSwitchG(conf, {optSeqDestructors}, arg, pass, info)
+    if pass in {passCmd2, passPP}:
+      defineSymbol(conf.symbols, "nimSeqsV2")
   of "stylecheck":
     case arg.normalize
     of "off": conf.globalOptions = conf.globalOptions - {optStyleHint, optStyleError}
-    of "hint": conf.globalOptions = conf.globalOptions + {optStyleHint}
+    of "hint": conf.globalOptions = conf.globalOptions + {optStyleHint} - {optStyleError}
     of "error": conf.globalOptions = conf.globalOptions + {optStyleError}
     else: localError(conf, info, errOffHintsError % arg)
   of "showallmismatches":
@@ -788,14 +870,54 @@ proc processSwitch*(switch, arg: string, pass: TCmdLinePass, info: TLineInfo;
   of "expandmacro":
     expectArg(conf, switch, arg, pass, info)
     conf.macrosToExpand[arg] = "T"
-  of "":
-    conf.projectName = "-"
+  of "expandarc":
+    expectArg(conf, switch, arg, pass, info)
+    conf.arcToExpand[arg] = "T"
+  of "useversion":
+    expectArg(conf, switch, arg, pass, info)
+    case arg
+    of "1.0":
+      defineSymbol(conf.symbols, "NimMajor", "1")
+      defineSymbol(conf.symbols, "NimMinor", "0")
+      # always be compatible with 1.0.100:
+      defineSymbol(conf.symbols, "NimPatch", "100")
+      # old behaviors go here:
+      defineSymbol(conf.symbols, "nimOldRelativePathBehavior")
+      undefSymbol(conf.symbols, "nimDoesntTrackDefects")
+      ast.eqTypeFlags.excl {tfGcSafe, tfNoSideEffect}
+      conf.globalOptions.incl optNimV1Emulation
+    of "1.2":
+      defineSymbol(conf.symbols, "NimMajor", "1")
+      defineSymbol(conf.symbols, "NimMinor", "2")
+      # always be compatible with 1.2.100:
+      defineSymbol(conf.symbols, "NimPatch", "100")
+      conf.globalOptions.incl optNimV12Emulation
+    else:
+      localError(conf, info, "unknown Nim version; currently supported values are: {1.0}")
+  of "benchmarkvm":
+    processOnOffSwitchG(conf, {optBenchmarkVM}, arg, pass, info)
+  of "profilevm":
+    processOnOffSwitchG(conf, {optProfileVM}, arg, pass, info)
+  of "sinkinference":
+    processOnOffSwitch(conf, {optSinkInference}, arg, pass, info)
+  of "cursorinference":
+    # undocumented, for debugging purposes only:
+    processOnOffSwitch(conf, {optCursorInference}, arg, pass, info)
+  of "panics":
+    processOnOffSwitchG(conf, {optPanics}, arg, pass, info)
+    if optPanics in conf.globalOptions:
+      defineSymbol(conf.symbols, "nimPanics")
+  of "sourcemap":
+    conf.globalOptions.incl optSourcemap
+    conf.options.incl optLineDir
+  of "deepcopy":
+    processOnOffSwitchG(conf, {optEnableDeepCopy}, arg, pass, info)
+  of "": # comes from "-" in for example: `nim c -r -` (gets stripped from -)
+    handleStdinInput(conf)
   else:
     if strutils.find(switch, '.') >= 0: options.setConfigVar(conf, switch, arg)
     else: invalidCmdLineOption(conf, pass, switch, info)
 
-template gCmdLineInfo*(): untyped = newLineInfo(config, AbsoluteFile"command line", 1, 1)
-
 proc processCommand*(switch: string, pass: TCmdLinePass; config: ConfigRef) =
   var cmd, arg: string
   splitSwitch(config, switch, cmd, arg, pass, gCmdLineInfo)

compiler/condsyms.nim

@@ -13,7 +13,7 @@ import
   strtabs
 
 from options import Feature
-from lineinfos import HintsToStr, WarningsToStr
+from lineinfos import hintMin, hintMax, warnMin, warnMax
 
 proc defineSymbol*(symbols: StringTableRef; symbol: string, value: string = "true") =
   symbols[symbol] = value
@@ -48,13 +48,11 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimalias")
   defineSymbol("nimlocks")
   defineSymbol("nimnode")
-  defineSymbol("nimnomagic64")
   defineSymbol("nimvarargstyped")
   defineSymbol("nimtypedescfixed")
   defineSymbol("nimKnowsNimvm")
   defineSymbol("nimArrIdx")
   defineSymbol("nimHasalignOf")
-  defineSymbol("nimNewShiftOps")
   defineSymbol("nimDistros")
   defineSymbol("nimHasCppDefine")
   defineSymbol("nimGenericInOutFlags")
@@ -87,13 +85,42 @@ proc initDefines*(symbols: StringTableRef) =
   defineSymbol("nimHasDefault")
   defineSymbol("nimMacrosSizealignof")
   defineSymbol("nimNoZeroExtendMagic")
-  for f in low(Feature)..high(Feature):
+  defineSymbol("nimMacrosGetNodeId")
+  for f in Feature:
     defineSymbol("nimHas" & $f)
 
-  for s in WarningsToStr:
-    defineSymbol("nimHasWarning" & s)
-  for s in HintsToStr:
-    defineSymbol("nimHasHint" & s)
+  for s in warnMin..warnMax:
+    defineSymbol("nimHasWarning" & $s)
+  for s in hintMin..hintMax:
+    defineSymbol("nimHasHint" & $s)
 
   defineSymbol("nimFixedOwned")
   defineSymbol("nimHasStyleChecks")
+  defineSymbol("nimToOpenArrayCString")
+  defineSymbol("nimHasUsed")
+  defineSymbol("nimFixedForwardGeneric")
+  defineSymbol("nimnomagic64")
+  defineSymbol("nimNewShiftOps")
+  defineSymbol("nimHasCursor")
+  defineSymbol("nimAlignPragma")
+  defineSymbol("nimHasExceptionsQuery")
+  defineSymbol("nimHasIsNamedTuple")
+  defineSymbol("nimHashOrdinalFixed")
+
+  when defined(nimHasLibFFI):
+    # Renaming as we can't conflate input vs output define flags; e.g. this
+    # will report the right thing regardless of whether user adds
+    # `-d:nimHasLibFFI` in his user config.
+    defineSymbol("nimHasLibFFIEnabled")
+
+  defineSymbol("nimHasSinkInference")
+  defineSymbol("nimNewIntegerOps")
+  defineSymbol("nimHasInvariant")
+  defineSymbol("nimHasStacktraceMsgs")
+  defineSymbol("nimDoesntTrackDefects")
+  defineSymbol("nimHasLentIterators")
+  defineSymbol("nimHasDeclaredMagic")
+  defineSymbol("nimHasStacktracesModule")
+  defineSymbol("nimHasEffectTraitsModule")
+  defineSymbol("nimHasCastPragmaBlocks")
+  defineSymbol("nimHasJsBigIntBackend")

compiler/configuration.nim

@@ -1,6 +0,0 @@
-## Use the module 'lineinfos' instead!
-
-{.deprecated.}
-
-import lineinfos
-export lineinfos

compiler/cursors.nim

@@ -1,72 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2019 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-import
-  intsets, ast, astalgo, msgs, renderer, magicsys, types, idents, trees,
-  strutils, options, dfa, lowerings, tables, modulegraphs, msgs,
-  lineinfos, parampatterns
-
-##[
-This module implements "cursor" detection. A cursor is a local variable
-that is used for navigation in a datastructure, it does not "own" the
-data it aliases but it might update the underlying datastructure.
-
-Two primary examples for cursors that I have in mind and that are critical
-for optimization:
-
-1. Local string variable introduced by ``for x in a``::
-
-  var i = 0
-  while i < a.len:
-    let cursor = a[i]
-    use cursor
-    inc i
-
-2. Local ``ref`` variable for navigation::
-
-  var cursor = listHead
-  while cursor != nil:
-    use cursor
-    cursor = cursor.next
-
-Cursors are very interesting for the optimizer because they can be copyMem'ed
-and don't need a destructor.
-
-More formally, a cursor is a variable that is set on all paths to
-a *location* or a proc call that produced a ``lent/var`` type. All statements
-that come after these assignments MUST not mutate what the cursor aliases.
-
-Mutations *through* the cursor are allowed if the cursor has ref semantics.
-
-Look at this complex real world example taken from the compiler itself:
-
-.. code-block:: Nim
-
-  proc getTypeName(m: BModule; typ: PType; sig: SigHash): Rope =
-    var t = typ
-    while true:
-      if t.sym != nil and {sfImportc, sfExportc} * t.sym.flags != {}:
-        return t.sym.loc.r
-
-      if t.kind in irrelevantForBackend:
-        t = t.lastSon
-      else:
-        break
-    let typ = if typ.kind in {tyAlias, tySink, tyOwned}: typ.lastSon else: typ
-    if typ.loc.r == nil:
-      typ.loc.r = typ.typeName & $sig
-    result = typ.loc.r
-    if result == nil: internalError(m.config, "getTypeName: " & $typ.kind)
-
-Here `t` is a cursor but without a control flow based analysis we are unlikely
-to detect it.
-
-]##
-
-# Araq: I owe you an implementation. For now use the .cursor pragma. :-/

compiler/depends.nim

@@ -25,7 +25,7 @@ type
     dotGraph: Rope
 
 proc addDependencyAux(b: Backend; importing, imported: string) =
-  addf(b.dotGraph, "$1 -> \"$2\";$n", [rope(importing), rope(imported)])
+  b.dotGraph.addf("$1 -> \"$2\";$n", [rope(importing), rope(imported)])
   # s1 -> s2_4[label="[0-9]"];
 
 proc addDotDependency(c: PPassContext, n: PNode): PNode =
@@ -34,14 +34,14 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode =
   let b = Backend(g.graph.backend)
   case n.kind
   of nkImportStmt:
-    for i in 0 ..< sonsLen(n):
-      var imported = getModuleName(g.config, n.sons[i])
+    for i in 0..<n.len:
+      var imported = getModuleName(g.config, n[i])
       addDependencyAux(b, g.module.name.s, imported)
   of nkFromStmt, nkImportExceptStmt:
-    var imported = getModuleName(g.config, n.sons[0])
+    var imported = getModuleName(g.config, n[0])
     addDependencyAux(b, g.module.name.s, imported)
   of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr:
-    for i in 0 ..< sonsLen(n): discard addDotDependency(c, n.sons[i])
+    for i in 0..<n.len: discard addDotDependency(c, n[i])
   else:
     discard
 
@@ -51,7 +51,10 @@ proc generateDot*(graph: ModuleGraph; project: AbsoluteFile) =
       rope(project.splitFile.name), b.dotGraph],
             changeFileExt(project, "dot"))
 
-proc myOpen(graph: ModuleGraph; module: PSym): PPassContext =
+when not defined(nimHasSinkInference):
+  {.pragma: nosinks.}
+
+proc myOpen(graph: ModuleGraph; module: PSym): PPassContext {.nosinks.} =
   var g: PGen
   new(g)
   g.module = module

compiler/dfa.nim

@@ -29,47 +29,45 @@
 ## "A Graph–Free Approach to Data–Flow Analysis" by Markus Mohnen.
 ## https://link.springer.com/content/pdf/10.1007/3-540-45937-5_6.pdf
 
-import ast, types, intsets, lineinfos, renderer
+import ast, types, intsets, lineinfos, renderer, asciitables
 
 from patterns import sameTrees
 
 type
   InstrKind* = enum
-    goto, fork, join, def, use
+    goto, fork, def, use
   Instr* = object
-    n*: PNode
+    n*: PNode # contains the def/use location.
     case kind*: InstrKind
-    of def, use: sym*: PSym # 'sym' can also be 'nil' and
-                            # then 'n' contains the def/use location.
-                            # This is used so that we can track object
-                            # and tuple field accesses precisely.
-    of goto, fork, join: dest*: int
+    of goto, fork: dest*: int
+    else: discard
 
   ControlFlowGraph* = seq[Instr]
 
   TPosition = distinct int
+
   TBlock = object
-    label: PSym
-    fixups: seq[TPosition]
+    case isTryBlock: bool
+    of false:
+      label: PSym
+      breakFixups: seq[(TPosition, seq[PNode])] #Contains the gotos for the breaks along with their pending finales
+    of true:
+      finale: PNode
+      raiseFixups: seq[TPosition] #Contains the gotos for the raises
 
   Con = object
     code: ControlFlowGraph
-    inCall, inTryStmt: int
+    inTryStmt: int
     blocks: seq[TBlock]
-    tryStmtFixups: seq[TPosition]
-    forks: seq[TPosition]
     owner: PSym
 
-proc debugInfo(info: TLineInfo): string =
-  result = $info.line #info.toFilename & ":" & $info.line
-
-proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
+proc codeListing(c: ControlFlowGraph, start = 0; last = -1): string =
   # for debugging purposes
   # first iteration: compute all necessary labels:
   var jumpTargets = initIntSet()
   let last = if last < 0: c.len-1 else: min(last, c.len-1)
   for i in start..last:
-    if c[i].kind in {goto, fork, join}:
+    if c[i].kind in {goto, fork}:
       jumpTargets.incl(i+c[i].dest)
   var i = start
   while i <= last:
@@ -80,27 +78,22 @@ proc codeListing(c: ControlFlowGraph, result: var string, start=0; last = -1) =
     case c[i].kind
     of def, use:
       result.add renderTree(c[i].n)
-    of goto, fork, join:
+    of goto, fork:
       result.add "L"
       result.addInt c[i].dest+i
     result.add("\t#")
-    result.add(debugInfo(c[i].n.info))
+    result.add($c[i].n.info.line)
     result.add("\n")
     inc i
   if i in jumpTargets: result.add("L" & $i & ": End\n")
-  # consider calling `asciitables.alignTable`
 
-proc echoCfg*(c: ControlFlowGraph; start=0; last = -1) {.deprecated.} =
+proc echoCfg*(c: ControlFlowGraph; start = 0; last = -1) {.deprecated.} =
   ## echos the ControlFlowGraph for debugging purposes.
-  var buf = ""
-  codeListing(c, buf, start, last)
-  when declared(echo):
-    echo buf
+  echo codeListing(c, start, last).alignTable
 
 proc forkI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
   c.code.add Instr(n: n, kind: fork, dest: 0)
-  c.forks.add result
 
 proc gotoI(c: var Con; n: PNode): TPosition =
   result = TPosition(c.code.len)
@@ -108,6 +101,24 @@ proc gotoI(c: var Con; n: PNode): TPosition =
 
 #[
 
+Join is no more
+===============
+Instead of generating join instructions we adapt our traversal of the CFG.
+
+When encountering a fork we split into two paths, we follow the path
+starting at "pc + 1" until it encounters the joinpoint: "pc + forkInstr.dest".
+If we encounter gotos that would jump further than the current joinpoint,
+as can happen with gotos generated by unstructured controlflow such as break, raise or return,
+we simply suspend following the current path, and follow the other path until the new joinpoint
+which is simply the instruction pointer returned to us by the now suspended path.
+If the path we are following now, also encounters a goto that exceeds the joinpoint
+we repeat the process; suspending the current path and evaluating the other one with a new joinpoint.
+If we eventually reach a common joinpoint we join the two paths.
+This new "ping-pong" approach has the obvious advantage of not requiring join instructions, as such
+cutting down on the CFG size but is also mandatory for correctly handling complicated cases
+of unstructured controlflow.
+
+
 Design of join
 ==============
 
@@ -256,33 +267,36 @@ duplicate the 'join' instructions on breaks and return exits!
 
 ]#
 
-proc joinI(c: var Con; fromFork: TPosition; n: PNode) =
-  let dist = fromFork.int - c.code.len
-  c.code.add Instr(n: n, kind: join, dest: dist)
+proc genLabel(c: Con): TPosition = TPosition(c.code.len)
 
-proc genLabel(c: Con): TPosition =
-  result = TPosition(c.code.len)
+template checkedDistance(dist): int =
+  doAssert low(int) div 2 + 1 < dist and dist < high(int) div 2
+  dist
 
 proc jmpBack(c: var Con, n: PNode, p = TPosition(0)) =
-  let dist = p.int - c.code.len
-  doAssert(low(int) div 2 + 1 < dist and dist < high(int) div 2)
-  c.code.add Instr(n: n, kind: goto, dest: dist)
+  c.code.add Instr(n: n, kind: goto, dest: checkedDistance(p.int - c.code.len))
 
 proc patch(c: var Con, p: TPosition) =
   # patch with current index
-  let p = p.int
-  let diff = c.code.len - p
-  doAssert(low(int) div 2 + 1 < diff and diff < high(int) div 2)
-  c.code[p].dest = diff
+  c.code[p.int].dest = checkedDistance(c.code.len - p.int)
+
+proc gen(c: var Con; n: PNode)
 
 proc popBlock(c: var Con; oldLen: int) =
-  for f in c.blocks[oldLen].fixups:
-    c.patch(f)
+  var exits: seq[TPosition]
+  exits.add c.gotoI(newNode(nkEmpty))
+  for f in c.blocks[oldLen].breakFixups:
+    c.patch(f[0])
+    for finale in f[1]:
+      c.gen(finale)
+    exits.add c.gotoI(newNode(nkEmpty))
+  for e in exits:
+    c.patch e
   c.blocks.setLen(oldLen)
 
-template withBlock(labl: PSym; body: untyped) {.dirty.} =
-  var oldLen {.gensym.} = c.blocks.len
-  c.blocks.add TBlock(label: labl, fixups: @[])
+template withBlock(labl: PSym; body: untyped) =
+  let oldLen = c.blocks.len
+  c.blocks.add TBlock(isTryBlock: false, label: labl)
   body
   popBlock(c, oldLen)
 
@@ -290,8 +304,6 @@ proc isTrue(n: PNode): bool =
   n.kind == nkSym and n.sym.kind == skEnumField and n.sym.position != 0 or
     n.kind == nkIntLit and n.intVal != 0
 
-proc gen(c: var Con; n: PNode) # {.noSideEffect.}
-
 when true:
   proc genWhile(c: var Con; n: PNode) =
     # We unroll every loop 3 times. We emulate 0, 1, 2 iterations
@@ -303,12 +315,13 @@ when true:
 
     Becomes:
 
-    if cond:
-      body
+    block:
       if cond:
         body
         if cond:
           body
+          if cond:
+            body
 
     We still need to ensure 'break' resolves properly, so an AST to AST
     translation is impossible.
@@ -331,28 +344,23 @@ when true:
     L4:
       join F1
     ]#
-    if isTrue(n.sons[0]):
+    if isTrue(n[0]):
       # 'while true' is an idiom in Nim and so we produce
       # better code for it:
-      for i in 0..2:
-        withBlock(nil):
-          c.gen(n.sons[1])
+      withBlock(nil):
+        for i in 0..2:
+          c.gen(n[1])
     else:
-      let oldForksLen = c.forks.len
-      var endings: array[3, TPosition]
-      for i in 0..2:
-        withBlock(nil):
-          c.gen(n.sons[0])
+      withBlock(nil):
+        var endings: array[3, TPosition]
+        for i in 0..2:
+          c.gen(n[0])
           endings[i] = c.forkI(n)
-          c.gen(n.sons[1])
-      for i in countdown(endings.high, 0):
-        let endPos = endings[i]
-        c.patch(endPos)
-        c.joinI(c.forks.pop(), n)
-      doAssert(c.forks.len == oldForksLen)
+          c.gen(n[1])
+        for i in countdown(endings.high, 0):
+          c.patch(endings[i])
 
 else:
-
   proc genWhile(c: var Con; n: PNode) =
     # lab1:
     #   cond, tmp
@@ -360,47 +368,21 @@ else:
     #   body
     #   jmp lab1
     # lab2:
-    let oldForksLen = c.forks.len
     let lab1 = c.genLabel
     withBlock(nil):
-      if isTrue(n.sons[0]):
-        c.gen(n.sons[1])
+      if isTrue(n[0]):
+        c.gen(n[1])
         c.jmpBack(n, lab1)
       else:
-        c.gen(n.sons[0])
-        let lab2 = c.forkI(n)
-        c.gen(n.sons[1])
-        c.jmpBack(n, lab1)
-        c.patch(lab2)
-    setLen(c.forks, oldForksLen)
-
-proc genBlock(c: var Con; n: PNode) =
-  withBlock(n.sons[0].sym):
-    c.gen(n.sons[1])
-
-proc genJoins(c: var Con; n: PNode) =
-  for i in countdown(c.forks.high, 0): joinI(c, c.forks[i], n)
-
-proc genBreak(c: var Con; n: PNode) =
-  genJoins(c, n)
-  let lab1 = c.gotoI(n)
-  if n.sons[0].kind == nkSym:
-    #echo cast[int](n.sons[0].sym)
-    for i in countdown(c.blocks.len-1, 0):
-      if c.blocks[i].label == n.sons[0].sym:
-        c.blocks[i].fixups.add lab1
-        return
-    #globalError(n.info, "VM problem: cannot find 'break' target")
-  else:
-    c.blocks[c.blocks.high].fixups.add lab1
+        c.gen(n[0])
+        forkT(n):
+          c.gen(n[1])
+          c.jmpBack(n, lab1)
 
 template forkT(n, body) =
-  let oldLen = c.forks.len
   let lab1 = c.forkI(n)
   body
   c.patch(lab1)
-  c.joinI(lab1, n)
-  setLen(c.forks, oldLen)
 
 proc genIf(c: var Con, n: PNode) =
   #[
@@ -439,21 +421,16 @@ proc genIf(c: var Con, n: PNode) =
     join F1
 
   ]#
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
-  for i in 0 ..< len(n):
-    var it = n.sons[i]
-    c.gen(it.sons[0])
+  for i in 0..<n.len:
+    let it = n[i]
+    c.gen(it[0])
     if it.len == 2:
-      let elsePos = forkI(c, it[1])
-      c.gen(it.sons[1])
-      endings.add(c.gotoI(it.sons[1]))
-      c.patch(elsePos)
+      forkT(it[1]):
+        c.gen(it[1])
+        endings.add c.gotoI(it[1])
   for i in countdown(endings.high, 0):
-    let endPos = endings[i]
-    c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
+    c.patch(endings[i])
 
 proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, a
@@ -461,9 +438,9 @@ proc genAndOr(c: var Con; n: PNode) =
   #   asgn dest, b
   # lab1:
   #   join F1
-  c.gen(n.sons[1])
+  c.gen(n[1])
   forkT(n):
-    c.gen(n.sons[2])
+    c.gen(n[2])
 
 proc genCase(c: var Con; n: PNode) =
   #  if (!expr1) goto lab1;
@@ -476,94 +453,107 @@ proc genCase(c: var Con; n: PNode) =
   #  lab2:
   #    elsePart
   #  Lend:
-  let isExhaustive = skipTypes(n.sons[0].typ,
+  let isExhaustive = skipTypes(n[0].typ,
     abstractVarRange-{tyTypeDesc}).kind notin {tyFloat..tyFloat128, tyString}
 
   var endings: seq[TPosition] = @[]
-  let oldLen = c.forks.len
-  c.gen(n.sons[0])
-  for i in 1 ..< n.len:
-    let it = n.sons[i]
-    if it.len == 1:
-      c.gen(it.sons[0])
-    elif i == n.len-1 and isExhaustive:
+  c.gen(n[0])
+  for i in 1..<n.len:
+    let it = n[i]
+    if it.len == 1 or (i == n.len-1 and isExhaustive):
       # treat the last branch as 'else' if this is an exhaustive case statement.
       c.gen(it.lastSon)
     else:
-      let elsePos = c.forkI(it.lastSon)
-      c.gen(it.lastSon)
-      endings.add(c.gotoI(it.lastSon))
-      c.patch(elsePos)
+      forkT(it.lastSon):
+        c.gen(it.lastSon)
+        endings.add c.gotoI(it.lastSon)
   for i in countdown(endings.high, 0):
     let endPos = endings[i]
     c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-  doAssert(c.forks.len == oldLen)
+
+proc genBlock(c: var Con; n: PNode) =
+  withBlock(n[0].sym):
+    c.gen(n[1])
+
+proc genBreakOrRaiseAux(c: var Con, i: int, n: PNode) =
+  let lab1 = c.gotoI(n)
+  if c.blocks[i].isTryBlock:
+    c.blocks[i].raiseFixups.add lab1
+  else:
+    var trailingFinales: seq[PNode]
+    if c.inTryStmt > 0: #Ok, we are in a try, lets see which (if any) try's we break out from:
+      for b in countdown(c.blocks.high, i):
+        if c.blocks[b].isTryBlock:
+          trailingFinales.add c.blocks[b].finale
+
+    c.blocks[i].breakFixups.add (lab1, trailingFinales)
+
+proc genBreak(c: var Con; n: PNode) =
+  if n[0].kind == nkSym:
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock and c.blocks[i].label == n[0].sym:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    #globalError(n.info, "VM problem: cannot find 'break' target")
+  else:
+    for i in countdown(c.blocks.high, 0):
+      if not c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
 
 proc genTry(c: var Con; n: PNode) =
-  let oldLen = c.forks.len
   var endings: seq[TPosition] = @[]
+
+  let oldLen = c.blocks.len
+  c.blocks.add TBlock(isTryBlock: true, finale: if n[^1].kind == nkFinally: n[^1] else: newNode(nkEmpty))
+
   inc c.inTryStmt
-  let oldFixups = c.tryStmtFixups.len
-
-  #let elsePos = c.forkI(n)
-  c.gen(n.sons[0])
+  c.gen(n[0])
   dec c.inTryStmt
-  for i in oldFixups..c.tryStmtFixups.high:
-    let f = c.tryStmtFixups[i]
+
+  for f in c.blocks[oldLen].raiseFixups:
     c.patch(f)
-    # we also need to produce join instructions
-    # for the 'fork' that might preceed the goto instruction
-    if f.int-1 >= 0 and c.code[f.int-1].kind == fork:
-      c.joinI(TPosition(f.int-1), n)
 
-  setLen(c.tryStmtFixups, oldFixups)
+  c.blocks.setLen oldLen
 
-  #c.patch(elsePos)
-  for i in 1 ..< n.len:
-    let it = n.sons[i]
+  for i in 1..<n.len:
+    let it = n[i]
     if it.kind != nkFinally:
-      var blen = len(it)
-      let endExcept = c.forkI(it)
-      c.gen(it.lastSon)
-      endings.add(c.gotoI(it))
-      c.patch(endExcept)
+      forkT(it):
+        c.gen(it.lastSon)
+        endings.add c.gotoI(it)
   for i in countdown(endings.high, 0):
-    let endPos = endings[i]
-    c.patch(endPos)
-    c.joinI(c.forks.pop(), n)
-
-  # join the 'elsePos' forkI instruction:
-  #c.joinI(c.forks.pop(), n)
+    c.patch(endings[i])
 
   let fin = lastSon(n)
   if fin.kind == nkFinally:
-    c.gen(fin.sons[0])
-  doAssert(c.forks.len == oldLen)
+    c.gen(fin[0])
 
 template genNoReturn(c: var Con; n: PNode) =
   # leave the graph
   c.code.add Instr(n: n, kind: goto, dest: high(int) - c.code.len)
 
 proc genRaise(c: var Con; n: PNode) =
-  genJoins(c, n)
-  gen(c, n.sons[0])
+  gen(c, n[0])
   if c.inTryStmt > 0:
-    c.tryStmtFixups.add c.gotoI(n)
+    for i in countdown(c.blocks.high, 0):
+      if c.blocks[i].isTryBlock:
+        genBreakOrRaiseAux(c, i, n)
+        return
+    assert false #Unreachable
   else:
     genNoReturn(c, n)
 
 proc genImplicitReturn(c: var Con) =
   if c.owner.kind in {skProc, skFunc, skMethod, skIterator, skConverter} and resultPos < c.owner.ast.len:
-    gen(c, c.owner.ast.sons[resultPos])
+    gen(c, c.owner.ast[resultPos])
 
 proc genReturn(c: var Con; n: PNode) =
-  genJoins(c, n)
-  if n.sons[0].kind != nkEmpty:
-    gen(c, n.sons[0])
+  if n[0].kind != nkEmpty:
+    gen(c, n[0])
   else:
     genImplicitReturn(c)
-  genNoReturn(c, n)
+  genBreakOrRaiseAux(c, 0, n)
 
 const
   InterestingSyms = {skVar, skResult, skLet, skParam, skForVar, skTemp}
@@ -573,16 +563,6 @@ const
                 nkObjDownConv, nkObjUpConv}
   PathKinds1 = {nkHiddenStdConv, nkHiddenSubConv}
 
-proc getRoot(n: PNode): PNode =
-  result = n
-  while true:
-    case result.kind
-    of PathKinds0:
-      result = result[0]
-    of PathKinds1:
-      result = result[1]
-    else: break
-
 proc skipConvDfa*(n: PNode): PNode =
   result = n
   while true:
@@ -593,85 +573,62 @@ proc skipConvDfa*(n: PNode): PNode =
       result = result[1]
     else: break
 
-proc genUse(c: var Con; orig: PNode) =
-  let n = dfa.getRoot(orig)
-  if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: orig, kind: use, sym: if orig != n: nil else: n.sym)
-
-proc aliases(obj, field: PNode): bool =
+proc aliases*(obj, field: PNode): bool =
   var n = field
   var obj = obj
-  while obj.kind in {nkHiddenSubConv, nkHiddenStdConv, nkObjDownConv, nkObjUpConv}:
-    obj = obj[0]
+  while true:
+    case obj.kind
+    of {nkObjDownConv, nkObjUpConv, nkAddr, nkHiddenAddr, nkDerefExpr, nkHiddenDeref}:
+      obj = obj[0]
+    of PathKinds1:
+      obj = obj[1]
+    else: break
   while true:
     if sameTrees(obj, n): return true
     case n.kind
-    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-       nkObjDownConv, nkObjUpConv, nkHiddenDeref, nkDerefExpr:
+    of PathKinds0:
       n = n[0]
-    of nkBracketExpr:
-      let x = n[0]
-      if x.typ != nil and x.typ.skipTypes(abstractInst).kind == tyTuple:
-        n = x
-      else:
-        break
-    else:
-      break
-  return false
+    of PathKinds1:
+      n = n[1]
+    else: break
 
-proc useInstrTargets*(ins: Instr; loc: PNode): bool =
-  assert ins.kind == use
-  if ins.sym != nil and loc.kind == nkSym:
-    result = ins.sym == loc.sym
-  else:
-    result = ins.n == loc or sameTrees(ins.n, loc)
-  if not result:
-    # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
-    # def x.f; question: does it affect the full 'x'? No.
-    # def x; question: does it affect the 'x.f'? Yes.
-    # use x.f;  question: does it affect the full 'x'? No.
-    # use x; question does it affect 'x.f'? Yes.
-    result = aliases(ins.n, loc) or aliases(loc, ins.n)
+type InstrTargetKind* = enum
+  None, Full, Partial
 
-proc defInstrTargets*(ins: Instr; loc: PNode): bool =
-  assert ins.kind == def
-  if ins.sym != nil and loc.kind == nkSym:
-    result = ins.sym == loc.sym
-  else:
-    result = ins.n == loc or sameTrees(ins.n, loc)
-  if not result:
-    # We can come here if loc is 'x.f' and ins.n is 'x' or the other way round.
-    # def x.f; question: does it affect the full 'x'? No.
-    # def x; question: does it affect the 'x.f'? Yes.
-    # use x.f;  question: does it affect the full 'x'? No.
-    # use x; question does it affect 'x.f'? Yes.
-    result = aliases(ins.n, loc)
+proc instrTargets*(insloc, loc: PNode): InstrTargetKind =
+  if sameTrees(insloc, loc) or insloc.aliases(loc):
+    Full    # x -> x; x -> x.f
+  elif loc.aliases(insloc):
+    Partial # x.f -> x
+  else: None
 
 proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
   var n = orig
   while true:
     case n.kind
-    of nkDotExpr, nkCheckedFieldExpr, nkHiddenSubConv, nkHiddenStdConv,
-       nkObjDownConv, nkObjUpConv:
+    of PathKinds0 - {nkBracketExpr, nkHiddenDeref, nkDerefExpr}:
       n = n[0]
+    of PathKinds1:
+      n = n[1]
+    of nkBracketExpr:
+      # in a[i] the 'i' must be known
+      if n.len > 1 and n[1].kind in {nkCharLit..nkUInt64Lit}:
+        n = n[0]
+      else:
+        return false
     of nkHiddenDeref, nkDerefExpr:
       # We "own" sinkparam[].loc but not ourVar[].location as it is a nasty
       # pointer indirection.
+      # bug #14159, we cannot reason about sinkParam[].location as it can
+      # still be shared for tyRef.
       n = n[0]
-      return n.kind == nkSym and n.sym.owner == owner and (isSinkParam(n.sym) or
-          n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
-    of nkBracketExpr:
-      let x = n[0]
-      if x.typ != nil and x.typ.skipTypes(abstractInst).kind == tyTuple:
-        n = x
-      else:
-        break
-    else:
-      break
+      return n.kind == nkSym and n.sym.owner == owner and
+         (n.sym.typ.skipTypes(abstractInst-{tyOwned}).kind in {tyOwned})
+    else: break
   # XXX Allow closure deref operations here if we know
   # the owner controlled the closure allocation?
   result = n.kind == nkSym and n.sym.owner == owner and
-    owner.kind != skModule and
+    {sfGlobal, sfThread, sfCursor} * n.sym.flags == {} and
     (n.sym.kind != skParam or isSinkParam(n.sym)) # or n.sym.typ.kind == tyVar)
   # Note: There is a different move analyzer possible that checks for
   # consume(param.key); param.key = newValue  for all paths. Then code like
@@ -684,45 +641,55 @@ proc isAnalysableFieldAccess*(orig: PNode; owner: PSym): bool =
   # free the old data and all is lost! Lesson: Don't be too smart, trust the
   # lower level C++ optimizer to specialize this code.
 
-proc genDef(c: var Con; n: PNode) =
-  if n.kind == nkSym and n.sym.kind in InterestingSyms:
-    c.code.add Instr(n: n, kind: def, sym: n.sym)
-  elif isAnalysableFieldAccess(n, c.owner):
-    c.code.add Instr(n: n, kind: def, sym: nil)
+proc skipTrivials(c: var Con, n: PNode): PNode =
+  result = n
+  while true:
+    case result.kind
+    of PathKinds0 - {nkBracketExpr}:
+      result = result[0]
+    of nkBracketExpr:
+      gen(c, result[1])
+      result = result[0]
+    of PathKinds1:
+      result = result[1]
+    else: break
 
-proc canRaise(fn: PNode): bool =
-  const magicsThatCanRaise = {
-    mNone, mSlurp, mStaticExec, mParseExprToAst, mParseStmtToAst}
-  if fn.kind == nkSym and fn.sym.magic notin magicsThatCanRaise:
-    result = false
-  else:
-    result = true
+proc genUse(c: var Con; orig: PNode) =
+  let n = c.skipTrivials(orig)
+
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    c.code.add Instr(n: orig, kind: use)
+  elif n.kind in nkCallKinds:
+    gen(c, n)
+
+proc genDef(c: var Con; orig: PNode) =
+  let n = c.skipTrivials(orig)
+
+  if n.kind == nkSym and n.sym.kind in InterestingSyms:
+    c.code.add Instr(n: orig, kind: def)
 
 proc genCall(c: var Con; n: PNode) =
   gen(c, n[0])
   var t = n[0].typ
   if t != nil: t = t.skipTypes(abstractInst)
-  inc c.inCall
   for i in 1..<n.len:
     gen(c, n[i])
     when false:
-      if t != nil and i < t.len and t.sons[i].kind == tyVar:
-        # This is wrong! Pass by var is a 'might def', not a 'must def'
-        # like the other defs we emit. This is not good enough for a move
-        # optimizer.
+      if t != nil and i < t.len and t[i].kind == tyOut:
+        # Pass by 'out' is a 'must def'. Good enough for a move optimizer.
         genDef(c, n[i])
   # every call can potentially raise:
-  if c.inTryStmt > 0 and canRaise(n[0]):
+  if c.inTryStmt > 0 and canRaiseConservative(n[0]):
     # we generate the instruction sequence:
     # fork lab1
     # goto exceptionHandler (except or finally)
     # lab1:
     # join F1
-    let endGoto = c.forkI(n)
-    c.tryStmtFixups.add c.gotoI(n)
-    c.patch(endGoto)
-    c.joinI(c.forks.pop(), n)
-  dec c.inCall
+    forkT(n):
+      for i in countdown(c.blocks.high, 0):
+        if c.blocks[i].isTryBlock:
+          genBreakOrRaiseAux(c, i, n)
+          break
 
 proc genMagic(c: var Con; n: PNode; m: TMagic) =
   case m
@@ -739,23 +706,23 @@ proc genVarSection(c: var Con; n: PNode) =
       discard
     elif a.kind == nkVarTuple:
       gen(c, a.lastSon)
-      for i in 0 .. a.len-3: genDef(c, a[i])
+      for i in 0..<a.len-2: genDef(c, a[i])
     else:
       gen(c, a.lastSon)
       if a.lastSon.kind != nkEmpty:
-        genDef(c, a.sons[0])
+        genDef(c, a[0])
 
 proc gen(c: var Con; n: PNode) =
   case n.kind
   of nkSym: genUse(c, n)
   of nkCallKinds:
-    if n.sons[0].kind == nkSym:
-      let s = n.sons[0].sym
+    if n[0].kind == nkSym:
+      let s = n[0].sym
       if s.magic != mNone:
         genMagic(c, n, s.magic)
       else:
         genCall(c, n)
-      if sfNoReturn in n.sons[0].sym.flags:
+      if sfNoReturn in n[0].sym.flags:
         genNoReturn(c, n)
     else:
       genCall(c, n)
@@ -766,12 +733,12 @@ proc gen(c: var Con; n: PNode) =
     # "uses" 'i'. But we are only talking about builtin array indexing so
     # it doesn't matter and 'x = 34' is NOT a usage of 'x'.
     genDef(c, n[0])
-  of PathKinds0 - {nkHiddenStdConv, nkHiddenSubConv, nkObjDownConv, nkObjUpConv}:
+  of PathKinds0 - {nkObjDownConv, nkObjUpConv}:
     genUse(c, n)
   of nkIfStmt, nkIfExpr: genIf(c, n)
   of nkWhenStmt:
     # This is "when nimvm" node. Chose the first branch.
-    gen(c, n.sons[0].sons[1])
+    gen(c, n[0][1])
   of nkCaseStmt: genCase(c, n)
   of nkWhileStmt: genWhile(c, n)
   of nkBlockExpr, nkBlockStmt: genBlock(c, n)
@@ -783,18 +750,21 @@ proc gen(c: var Con; n: PNode) =
      nkBracket, nkCurly, nkPar, nkTupleConstr, nkClosure, nkObjConstr, nkYieldStmt:
     for x in n: gen(c, x)
   of nkPragmaBlock: gen(c, n.lastSon)
-  of nkDiscardStmt, nkObjDownConv, nkObjUpConv: gen(c, n.sons[0])
-  of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast, nkHiddenSubConv, nkHiddenStdConv:
-    gen(c, n.sons[1])
-  of nkStringToCString, nkCStringToString: gen(c, n.sons[0])
+  of nkDiscardStmt, nkObjDownConv, nkObjUpConv, nkStringToCString, nkCStringToString:
+    gen(c, n[0])
+  of nkConv, nkExprColonExpr, nkExprEqExpr, nkCast, PathKinds1:
+    gen(c, n[1])
   of nkVarSection, nkLetSection: genVarSection(c, n)
-  of nkDefer:
-    doAssert false, "dfa construction pass requires the elimination of 'defer'"
+  of nkDefer: doAssert false, "dfa construction pass requires the elimination of 'defer'"
   else: discard
 
 proc constructCfg*(s: PSym; body: PNode): ControlFlowGraph =
   ## constructs a control flow graph for ``body``.
   var c = Con(code: @[], blocks: @[], owner: s)
-  gen(c, body)
-  genImplicitReturn(c)
-  shallowCopy(result, c.code)
+  withBlock(s):
+    gen(c, body)
+    genImplicitReturn(c)
+  when defined(gcArc) or defined(gcOrc):
+    result = c.code # will move
+  else:
+    shallowCopy(result, c.code)

compiler/docgen2.nim

@@ -11,7 +11,7 @@
 # semantic checking.
 
 import
-  options, ast, msgs, idents, passes, docgen, lineinfos, pathutils
+  options, ast, msgs, passes, docgen, lineinfos, pathutils
 
 from modulegraphs import ModuleGraph, PPassContext
 
@@ -22,13 +22,14 @@ type
     config: ConfigRef
   PGen = ref TGen
 
-template shouldProcess(g): bool =
-  (g.module.owner.id == g.doc.conf.mainPackageId and optWholeProject in g.doc.conf.globalOptions) or
+proc shouldProcess(g: PGen): bool =
+  (optWholeProject in g.doc.conf.globalOptions and g.module.getnimblePkgId == g.doc.conf.mainPackageId) or
       sfMainModule in g.module.flags or g.config.projectMainIdx == g.module.info.fileIndex
 
 template closeImpl(body: untyped) {.dirty.} =
   var g = PGen(p)
   let useWarning = sfMainModule notin g.module.flags
+  let groupedToc = true
   if shouldProcess(g):
     body
     try:
@@ -38,7 +39,7 @@ template closeImpl(body: untyped) {.dirty.} =
 
 proc close(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
-    writeOutput(g.doc, useWarning)
+    writeOutput(g.doc, useWarning, groupedToc)
 
 proc closeJson(graph: ModuleGraph; p: PPassContext, n: PNode): PNode =
   closeImpl:
@@ -62,7 +63,7 @@ template myOpenImpl(ext: untyped) {.dirty.} =
   g.module = module
   g.config = graph.config
   var d = newDocumentor(AbsoluteFile toFullPath(graph.config, FileIndex module.position),
-      graph.cache, graph.config, ext)
+      graph.cache, graph.config, ext, module)
   d.hasToc = true
   g.doc = d
   result = g

compiler/enumtostr.nim

@@ -13,7 +13,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
   result.typ = newType(tyProc, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
-  addSon(result.typ.n, newNodeI(nkEffectList, info))
+  result.typ.n.add newNodeI(nkEffectList, info)
 
   result.typ.addParam dest
 
@@ -22,7 +22,7 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
   caseStmt.add(newSymNode dest)
 
   # copy the branches over, but replace the fields with the for loop body:
-  for i in 0 ..< t.n.len:
+  for i in 0..<t.n.len:
     assert(t.n[i].kind == nkSym)
     var field = t.n[i].sym
     let val = if field.ast == nil: field.name.s else: field.ast.strVal
@@ -33,28 +33,35 @@ proc genEnumToStrProc*(t: PType; info: TLineInfo; g: ModuleGraph): PSym =
   body.add(caseStmt)
 
   var n = newNodeI(nkProcDef, info, bodyPos+2)
-  for i in 0 ..< n.len: n.sons[i] = newNodeI(nkEmpty, info)
-  n.sons[namePos] = newSymNode(result)
-  n.sons[paramsPos] = result.typ.n
-  n.sons[bodyPos] = body
-  n.sons[resultPos] = newSymNode(res)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = body
+  n[resultPos] = newSymNode(res)
   result.ast = n
   incl result.flags, sfFromGeneric
+  incl result.flags, sfNeverRaises
 
-proc searchObjCase(obj: PNode; field: PSym): PNode =
+proc searchObjCaseImpl(obj: PNode; field: PSym): PNode =
   case obj.kind
   of nkSym:
     result = nil
   of nkElse, nkOfBranch:
-    result = searchObjCase(obj.lastSon, field)
+    result = searchObjCaseImpl(obj.lastSon, field)
   else:
     if obj.kind == nkRecCase and obj[0].kind == nkSym and obj[0].sym == field:
       result = obj
     else:
       for x in obj:
-        result = searchObjCase(x, field)
+        result = searchObjCaseImpl(x, field)
         if result != nil: break
 
+proc searchObjCase(t: PType; field: PSym): PNode =
+  result = searchObjCaseImpl(t.n, field)
+  if result == nil and t.len > 0:
+    result = searchObjCase(t[0].skipTypes({tyAlias, tyGenericInst, tyRef, tyPtr}), field)
+  doAssert result != nil
+
 proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGraph): PSym =
   result = newSym(skProc, getIdent(g.cache, "objDiscMapping"), t.owner, info)
 
@@ -67,7 +74,7 @@ proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGra
   result.typ = newType(tyProc, t.owner)
   result.typ.n = newNodeI(nkFormalParams, info)
   rawAddSon(result.typ, res.typ)
-  addSon(result.typ.n, newNodeI(nkEffectList, info))
+  result.typ.n.add newNodeI(nkEffectList, info)
 
   result.typ.addParam dest
 
@@ -75,12 +82,11 @@ proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGra
   var caseStmt = newNodeI(nkCaseStmt, info)
   caseStmt.add(newSymNode dest)
 
-  let subObj = searchObjCase(t.n, field)
-  doAssert subObj != nil
-  for i in 1 ..< subObj.len:
+  let subObj = searchObjCase(t, field)
+  for i in 1..<subObj.len:
     let ofBranch = subObj[i]
     var newBranch = newNodeI(ofBranch.kind, ofBranch.info)
-    for j in 0..ofBranch.len-2:
+    for j in 0..<ofBranch.len-1:
       newBranch.add ofBranch[j]
 
     newBranch.add newTree(nkStmtList, newTree(nkFastAsgn, newSymNode(res), newIntNode(nkInt8Lit, i)))
@@ -89,10 +95,11 @@ proc genCaseObjDiscMapping*(t: PType; field: PSym; info: TLineInfo; g: ModuleGra
   body.add(caseStmt)
 
   var n = newNodeI(nkProcDef, info, bodyPos+2)
-  for i in 0 ..< n.len: n.sons[i] = newNodeI(nkEmpty, info)
-  n.sons[namePos] = newSymNode(result)
-  n.sons[paramsPos] = result.typ.n
-  n.sons[bodyPos] = body
-  n.sons[resultPos] = newSymNode(res)
+  for i in 0..<n.len: n[i] = newNodeI(nkEmpty, info)
+  n[namePos] = newSymNode(result)
+  n[paramsPos] = result.typ.n
+  n[bodyPos] = body
+  n[resultPos] = newSymNode(res)
   result.ast = n
   incl result.flags, sfFromGeneric
+  incl result.flags, sfNeverRaises

compiler/evalffi.nim

@@ -9,13 +9,18 @@
 
 ## This file implements the FFI part of the evaluator for Nim code.
 
-import ast, astalgo, ropes, types, options, tables, dynlib, msgs, os, lineinfos
+import ast, types, options, tables, dynlib, msgs, lineinfos
+from os import getAppFilename
 import pkg/libffi
 
 when defined(windows):
   const libcDll = "msvcrt.dll"
 elif defined(linux):
   const libcDll = "libc.so(.6|.5|)"
+elif defined(openbsd):
+  const libcDll = "/usr/lib/libc.so(.95.1|)"
+elif defined(bsd):
+  const libcDll = "/lib/libc.so.7"
 elif defined(osx):
   const libcDll = "/usr/lib/libSystem.dylib"
 else:
@@ -27,7 +32,7 @@ var
   gDllCache = initTable[string, LibHandle]()
 
 when defined(windows):
-  var gExeHandle = loadLib(os.getAppFilename())
+  var gExeHandle = loadLib(getAppFilename())
 else:
   var gExeHandle = loadLib()
 
@@ -46,38 +51,35 @@ proc getDll(conf: ConfigRef, cache: var TDllCache; dll: string; info: TLineInfo)
 const
   nkPtrLit = nkIntLit # hopefully we can get rid of this hack soon
 
-var myerrno {.importc: "errno", header: "<errno.h>".}: cint ## error variable
-
 proc importcSymbol*(conf: ConfigRef, sym: PSym): PNode =
-  let name = $sym.loc.r
+  let name = sym.cname # $sym.loc.r would point to internal name
   # the AST does not support untyped pointers directly, so we use an nkIntLit
   # that contains the address instead:
   result = newNodeIT(nkPtrLit, sym.info, sym.typ)
-  case name
-  of "stdin":  result.intVal = cast[ByteAddress](system.stdin)
-  of "stdout": result.intVal = cast[ByteAddress](system.stdout)
-  of "stderr": result.intVal = cast[ByteAddress](system.stderr)
-  of "vmErrnoWrapper": result.intVal = cast[ByteAddress](myerrno)
-  else:
+  when true:
+    var libPathMsg = ""
     let lib = sym.annex
     if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}:
       globalError(conf, sym.info, "dynlib needs to be a string lit")
     var theAddr: pointer
-    if (lib.isNil or lib.kind == libHeader) and not gExehandle.isNil:
+    if (lib.isNil or lib.kind == libHeader) and not gExeHandle.isNil:
+      libPathMsg = "current exe: " & getAppFilename() & " nor libc: " & libcDll
       # first try this exe itself:
-      theAddr = gExehandle.symAddr(name)
+      theAddr = gExeHandle.symAddr(name)
       # then try libc:
       if theAddr.isNil:
         let dllhandle = getDll(conf, gDllCache, libcDll, sym.info)
         theAddr = dllhandle.symAddr(name)
     elif not lib.isNil:
       let dll = if lib.kind == libHeader: libcDll else: lib.path.strVal
+      libPathMsg = dll
       let dllhandle = getDll(conf, gDllCache, dll, sym.info)
       theAddr = dllhandle.symAddr(name)
-    if theAddr.isNil: globalError(conf, sym.info, "cannot import: " & sym.name.s)
+    if theAddr.isNil: globalError(conf, sym.info,
+      "cannot import symbol: " & name & " from " & libPathMsg)
     result.intVal = cast[ByteAddress](theAddr)
 
-proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType =
+proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.Type =
   if t == nil: return addr libffi.type_void
 
   case t.kind
@@ -94,7 +96,7 @@ proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType =
      tyTyped, tyTypeDesc, tyProc, tyArray, tyStatic, tyNil:
     result = addr libffi.type_pointer
   of tyDistinct, tyAlias, tySink:
-    result = mapType(conf, t.sons[0])
+    result = mapType(conf, t[0])
   else:
     result = nil
   # too risky:
@@ -102,7 +104,7 @@ proc mapType(conf: ConfigRef, t: ast.PType): ptr libffi.TType =
 
 proc mapCallConv(conf: ConfigRef, cc: TCallingConvention, info: TLineInfo): TABI =
   case cc
-  of ccDefault: result = DEFAULT_ABI
+  of ccNimCall: result = DEFAULT_ABI
   of ccStdCall: result = when defined(windows) and defined(x86): STDCALL else: DEFAULT_ABI
   of ccCDecl: result = DEFAULT_ABI
   else:
@@ -120,16 +122,16 @@ proc packSize(conf: ConfigRef, v: PNode, typ: PType): int =
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     else:
-      result = sizeof(pointer) + packSize(conf, v.sons[0], typ.lastSon)
+      result = sizeof(pointer) + packSize(conf, v[0], typ.lastSon)
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    result = packSize(conf, v, typ.sons[0])
+    result = packSize(conf, v, typ[0])
   of tyArray:
     # consider: ptr array[0..1000_000, int] which is common for interfacing;
     # we use the real length here instead
     if v.kind in {nkNilLit, nkPtrLit}:
       result = sizeof(pointer)
     elif v.len != 0:
-      result = v.len * packSize(conf, v.sons[0], typ.sons[1])
+      result = v.len * packSize(conf, v[0], typ[1])
   else:
     result = getSize(conf, typ).int
 
@@ -138,16 +140,16 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer)
 proc getField(conf: ConfigRef, n: PNode; position: int): PSym =
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      result = getField(conf, n.sons[i], position)
+    for i in 0..<n.len:
+      result = getField(conf, n[i], position)
       if result != nil: return
   of nkRecCase:
-    result = getField(conf, n.sons[0], position)
+    result = getField(conf, n[0], position)
     if result != nil: return
-    for i in 1 ..< sonsLen(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = getField(conf, lastSon(n.sons[i]), position)
+        result = getField(conf, lastSon(n[i]), position)
         if result != nil: return
       else: internalError(conf, n.info, "getField(record case branch)")
   of nkSym:
@@ -158,12 +160,12 @@ proc packObject(conf: ConfigRef, x: PNode, typ: PType, res: pointer) =
   internalAssert conf, x.kind in {nkObjConstr, nkPar, nkTupleConstr}
   # compute the field's offsets:
   discard getSize(conf, typ)
-  for i in ord(x.kind == nkObjConstr) ..< sonsLen(x):
-    var it = x.sons[i]
+  for i in ord(x.kind == nkObjConstr)..<x.len:
+    var it = x[i]
     if it.kind == nkExprColonExpr:
-      internalAssert conf, it.sons[0].kind == nkSym
-      let field = it.sons[0].sym
-      pack(conf, it.sons[1], field.typ, res +! field.offset)
+      internalAssert conf, it[0].kind == nkSym
+      let field = it[0].sym
+      pack(conf, it[1], field.typ, res +! field.offset)
     elif typ.n != nil:
       let field = getField(conf, typ.n, i)
       pack(conf, it, field.typ, res +! field.offset)
@@ -224,19 +226,19 @@ proc pack(conf: ConfigRef, v: PNode, typ: PType, res: pointer) =
         packRecCheck = 0
         globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ))
       inc packRecCheck
-      pack(conf, v.sons[0], typ.lastSon, res +! sizeof(pointer))
+      pack(conf, v[0], typ.lastSon, res +! sizeof(pointer))
       dec packRecCheck
       awr(pointer, res +! sizeof(pointer))
   of tyArray:
-    let baseSize = getSize(conf, typ.sons[1])
-    for i in 0 ..< v.len:
-      pack(conf, v.sons[i], typ.sons[1], res +! i * baseSize)
+    let baseSize = getSize(conf, typ[1])
+    for i in 0..<v.len:
+      pack(conf, v[i], typ[1], res +! i * baseSize)
   of tyObject, tyTuple:
     packObject(conf, v, typ, res)
   of tyNil:
     discard
   of tyDistinct, tyGenericInst, tyAlias, tySink:
-    pack(conf, v, typ.sons[0], res)
+    pack(conf, v, typ[0], res)
   else:
     globalError(conf, v.info, "cannot map value to FFI " & typeToString(v.typ))
 
@@ -245,14 +247,14 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode
 proc unpackObjectAdd(conf: ConfigRef, x: pointer, n, result: PNode) =
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      unpackObjectAdd(conf, x, n.sons[i], result)
+    for i in 0..<n.len:
+      unpackObjectAdd(conf, x, n[i], result)
   of nkRecCase:
     globalError(conf, result.info, "case objects cannot be unpacked")
   of nkSym:
     var pair = newNodeI(nkExprColonExpr, result.info, 2)
-    pair.sons[0] = n
-    pair.sons[1] = unpack(conf, x +! n.sym.offset, n.sym.typ, nil)
+    pair[0] = n
+    pair[1] = unpack(conf, x +! n.sym.offset, n.sym.typ, nil)
     #echo "offset: ", n.sym.name.s, " ", n.sym.offset
     result.add pair
   else: discard
@@ -275,28 +277,28 @@ proc unpackObject(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
       globalError(conf, n.info, "cannot map value from FFI")
     if typ.n.isNil:
       globalError(conf, n.info, "cannot unpack unnamed tuple")
-    for i in ord(n.kind == nkObjConstr) ..< sonsLen(n):
-      var it = n.sons[i]
+    for i in ord(n.kind == nkObjConstr)..<n.len:
+      var it = n[i]
       if it.kind == nkExprColonExpr:
-        internalAssert conf, it.sons[0].kind == nkSym
-        let field = it.sons[0].sym
-        it.sons[1] = unpack(conf, x +! field.offset, field.typ, it.sons[1])
+        internalAssert conf, it[0].kind == nkSym
+        let field = it[0].sym
+        it[1] = unpack(conf, x +! field.offset, field.typ, it[1])
       else:
         let field = getField(conf, typ.n, i)
-        n.sons[i] = unpack(conf, x +! field.offset, field.typ, it)
+        n[i] = unpack(conf, x +! field.offset, field.typ, it)
 
 proc unpackArray(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
   if n.isNil:
     result = newNode(nkBracket)
     result.typ = typ
-    newSeq(result.sons, lengthOrd(conf, typ).int)
+    newSeq(result.sons, lengthOrd(conf, typ).toInt)
   else:
     result = n
     if result.kind != nkBracket:
       globalError(conf, n.info, "cannot map value from FFI")
-  let baseSize = getSize(conf, typ.sons[1])
-  for i in 0 ..< result.len:
-    result.sons[i] = unpack(conf, x +! i * baseSize, typ.sons[1], result.sons[i])
+  let baseSize = getSize(conf, typ[1])
+  for i in 0..<result.len:
+    result[i] = unpack(conf, x +! i * baseSize, typ[1], result[i])
 
 proc canonNodeKind(k: TNodeKind): TNodeKind =
   case k
@@ -327,7 +329,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
     else:
       reset n[]
       result = n
-      result.kind = nkNilLit
+      result[] = TNode(kind: nkNilLit)
       result.typ = typ
 
   template awi(kind, v: untyped): untyped = aw(kind, v, intVal)
@@ -376,7 +378,7 @@ proc unpack(conf: ConfigRef, x: pointer, typ: PType, n: PNode): PNode =
       awi(nkPtrLit, cast[ByteAddress](p))
     elif n != nil and n.len == 1:
       internalAssert(conf, n.kind == nkRefTy)
-      n.sons[0] = unpack(conf, p, typ.lastSon, n.sons[0])
+      n[0] = unpack(conf, p, typ.lastSon, n[0])
       result = n
     else:
       globalError(conf, n.info, "cannot map value from FFI " & typeToString(typ))
@@ -419,41 +421,41 @@ proc fficast*(conf: ConfigRef, x: PNode, destTyp: PType): PNode =
     dealloc a
 
 proc callForeignFunction*(conf: ConfigRef, call: PNode): PNode =
-  internalAssert conf, call.sons[0].kind == nkPtrLit
+  internalAssert conf, call[0].kind == nkPtrLit
 
   var cif: TCif
-  var sig: TParamList
+  var sig: ParamList
   # use the arguments' types for varargs support:
-  for i in 1..call.len-1:
-    sig[i-1] = mapType(conf, call.sons[i].typ)
+  for i in 1..<call.len:
+    sig[i-1] = mapType(conf, call[i].typ)
     if sig[i-1].isNil:
       globalError(conf, call.info, "cannot map FFI type")
 
-  let typ = call.sons[0].typ
+  let typ = call[0].typ
   if prep_cif(cif, mapCallConv(conf, typ.callConv, call.info), cuint(call.len-1),
-              mapType(conf, typ.sons[0]), sig) != OK:
+              mapType(conf, typ[0]), sig) != OK:
     globalError(conf, call.info, "error in FFI call")
 
-  var args: TArgList
-  let fn = cast[pointer](call.sons[0].intVal)
-  for i in 1 .. call.len-1:
-    var t = call.sons[i].typ
-    args[i-1] = alloc0(packSize(conf, call.sons[i], t))
-    pack(conf, call.sons[i], t, args[i-1])
-  let retVal = if isEmptyType(typ.sons[0]): pointer(nil)
-               else: alloc(getSize(conf, typ.sons[0]).int)
+  var args: ArgList
+  let fn = cast[pointer](call[0].intVal)
+  for i in 1..<call.len:
+    var t = call[i].typ
+    args[i-1] = alloc0(packSize(conf, call[i], t))
+    pack(conf, call[i], t, args[i-1])
+  let retVal = if isEmptyType(typ[0]): pointer(nil)
+               else: alloc(getSize(conf, typ[0]).int)
 
   libffi.call(cif, fn, retVal, args)
 
   if retVal.isNil:
     result = newNode(nkEmpty)
   else:
-    result = unpack(conf, retVal, typ.sons[0], nil)
+    result = unpack(conf, retVal, typ[0], nil)
     result.info = call.info
 
   if retVal != nil: dealloc retVal
-  for i in 1 .. call.len-1:
-    call.sons[i] = unpack(conf, args[i-1], typ.sons[i], call[i])
+  for i in 1..<call.len:
+    call[i] = unpack(conf, args[i-1], typ[i], call[i])
     dealloc args[i-1]
 
 proc callForeignFunction*(conf: ConfigRef, fn: PNode, fntyp: PType,
@@ -462,39 +464,39 @@ proc callForeignFunction*(conf: ConfigRef, fn: PNode, fntyp: PType,
   internalAssert conf, fn.kind == nkPtrLit
 
   var cif: TCif
-  var sig: TParamList
+  var sig: ParamList
   for i in 0..len-1:
     var aTyp = args[i+start].typ
     if aTyp.isNil:
       internalAssert conf, i+1 < fntyp.len
-      aTyp = fntyp.sons[i+1]
+      aTyp = fntyp[i+1]
       args[i+start].typ = aTyp
     sig[i] = mapType(conf, aTyp)
     if sig[i].isNil: globalError(conf, info, "cannot map FFI type")
 
   if prep_cif(cif, mapCallConv(conf, fntyp.callConv, info), cuint(len),
-              mapType(conf, fntyp.sons[0]), sig) != OK:
+              mapType(conf, fntyp[0]), sig) != OK:
     globalError(conf, info, "error in FFI call")
 
-  var cargs: TArgList
+  var cargs: ArgList
   let fn = cast[pointer](fn.intVal)
-  for i in 0 .. len-1:
+  for i in 0..len-1:
     let t = args[i+start].typ
     cargs[i] = alloc0(packSize(conf, args[i+start], t))
     pack(conf, args[i+start], t, cargs[i])
-  let retVal = if isEmptyType(fntyp.sons[0]): pointer(nil)
-               else: alloc(getSize(conf, fntyp.sons[0]).int)
+  let retVal = if isEmptyType(fntyp[0]): pointer(nil)
+               else: alloc(getSize(conf, fntyp[0]).int)
 
   libffi.call(cif, fn, retVal, cargs)
 
   if retVal.isNil:
     result = newNode(nkEmpty)
   else:
-    result = unpack(conf, retVal, fntyp.sons[0], nil)
+    result = unpack(conf, retVal, fntyp[0], nil)
     result.info = info
 
   if retVal != nil: dealloc retVal
-  for i in 0 .. len-1:
+  for i in 0..len-1:
     let t = args[i+start].typ
     args[i+start] = unpack(conf, cargs[i], t, args[i+start])
     dealloc cargs[i]

compiler/evaltempl.nim

@@ -10,7 +10,7 @@
 ## Template evaluation engine. Now hygienic.
 
 import
-  strutils, options, ast, astalgo, msgs, renderer, lineinfos
+  strutils, options, ast, astalgo, msgs, renderer, lineinfos, idents
 
 type
   TemplCtx = object
@@ -20,6 +20,8 @@ type
     mapping: TIdTable # every gensym'ed symbol needs to be mapped to some
                       # new symbol
     config: ConfigRef
+    ic: IdentCache
+    instID: int
 
 proc copyNode(ctx: TemplCtx, a, b: PNode): PNode =
   result = copyNode(a)
@@ -37,11 +39,11 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
   of nkSym:
     var s = templ.sym
     if (s.owner == nil and s.kind == skParam) or s.owner == c.owner:
-      if s.kind == skParam and sfGenSym notin s.flags:
-        handleParam actual.sons[s.position]
+      if s.kind == skParam and {sfGenSym, sfTemplateParam} * s.flags == {sfTemplateParam}:
+        handleParam actual[s.position]
       elif (s.owner != nil) and (s.kind == skGenericParam or
            s.kind == skType and s.typ != nil and s.typ.kind == tyGenericParam):
-        handleParam actual.sons[s.owner.typ.len + s.position - 1]
+        handleParam actual[s.owner.typ.len + s.position - 1]
       else:
         internalAssert c.config, sfGenSym in s.flags or s.kind == skType
         var x = PSym(idTableGet(c.mapping, s))
@@ -52,7 +54,11 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
           #if x.kind == skParam and x.owner.kind == skModule:
           #  internalAssert c.config, false
           idTablePut(c.mapping, s, x)
-        result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
+        if sfGenSym in s.flags:
+          result.add newIdentNode(getIdent(c.ic, x.name.s & "`gensym" & $c.instID),
+            if c.instLines: actual.info else: templ.info)
+        else:
+          result.add newSymNode(x, if c.instLines: actual.info else: templ.info)
     else:
       result.add copyNode(c, templ, actual)
   of nkNone..nkIdent, nkType..nkNilLit: # atom
@@ -63,8 +69,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
     # "declarative" context (bug #9235).
     if c.isDeclarative:
       var res = copyNode(c, templ, actual)
-      for i in 0 ..< sonsLen(templ):
-        evalTemplateAux(templ.sons[i], actual, c, res)
+      for i in 0..<templ.len:
+        evalTemplateAux(templ[i], actual, c, res)
       result.add res
     else:
       result.add newNodeI(nkEmpty, templ.info)
@@ -77,8 +83,8 @@ proc evalTemplateAux(templ, actual: PNode, c: var TemplCtx, result: PNode) =
       c.isDeclarative = true
       isDeclarative = true
     var res = copyNode(c, templ, actual)
-    for i in 0 ..< sonsLen(templ):
-      evalTemplateAux(templ.sons[i], actual, c, res)
+    for i in 0..<templ.len:
+      evalTemplateAux(templ[i], actual, c, res)
     result.add res
     if isDeclarative: c.isDeclarative = false
 
@@ -117,22 +123,22 @@ proc evalTemplateArgs(n: PNode, s: PSym; conf: ConfigRef; fromHlo: bool): PNode
                 n.renderTree)
 
   result = newNodeI(nkArgList, n.info)
-  for i in 1 .. givenRegularParams:
-    result.addSon n[i]
+  for i in 1..givenRegularParams:
+    result.add n[i]
 
   # handle parameters with default values, which were
   # not supplied by the user
-  for i in givenRegularParams+1 .. expectedRegularParams:
-    let default = s.typ.n.sons[i].sym.ast
+  for i in givenRegularParams+1..expectedRegularParams:
+    let default = s.typ.n[i].sym.ast
     if default.isNil or default.kind == nkEmpty:
       localError(conf, n.info, errWrongNumberOfArguments)
-      addSon(result, newNodeI(nkEmpty, n.info))
+      result.add newNodeI(nkEmpty, n.info)
     else:
-      addSon(result, default.copyTree)
+      result.add default.copyTree
 
-  # add any generic paramaters
-  for i in 1 .. genericParams:
-    result.addSon n.sons[givenRegularParams + i]
+  # add any generic parameters
+  for i in 1..genericParams:
+    result.add n[givenRegularParams + i]
 
 # to prevent endless recursion in template instantiation
 const evalTemplateLimit* = 1000
@@ -150,7 +156,7 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
       if x.kind in nkCallKinds:
         for i in 1..<x.len:
           if x[i].kind in nkCallKinds:
-            x.sons[i].info = info
+            x[i].info = info
   else:
     result = newNodeI(nkStmtListExpr, info)
     var d = newNodeI(nkComesFrom, info)
@@ -160,7 +166,9 @@ proc wrapInComesFrom*(info: TLineInfo; sym: PSym; res: PNode): PNode =
     result.typ = res.typ
 
 proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
-                   conf: ConfigRef; fromHlo=false): PNode =
+                   conf: ConfigRef;
+                   ic: IdentCache; instID: ref int;
+                   fromHlo=false): PNode =
   inc(conf.evalTemplateCounter)
   if conf.evalTemplateCounter > evalTemplateLimit:
     globalError(conf, n.info, errTemplateInstantiationTooNested)
@@ -172,14 +180,16 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
   ctx.owner = tmpl
   ctx.genSymOwner = genSymOwner
   ctx.config = conf
+  ctx.ic = ic
   initIdTable(ctx.mapping)
+  ctx.instID = instID[]
 
   let body = tmpl.getBody
   #echo "instantion of ", renderTree(body, {renderIds})
   if isAtom(body):
     result = newNodeI(nkPar, body.info)
     evalTemplateAux(body, args, ctx, result)
-    if result.len == 1: result = result.sons[0]
+    if result.len == 1: result = result[0]
     else:
       localError(conf, result.info, "illformed AST: " &
                   renderTree(result, {renderNoComments}))
@@ -188,10 +198,12 @@ proc evalTemplate*(n: PNode, tmpl, genSymOwner: PSym;
     ctx.instLines = sfCallsite in tmpl.flags
     if ctx.instLines:
       result.info = n.info
-    for i in 0 ..< safeLen(body):
-      evalTemplateAux(body.sons[i], args, ctx, result)
+    for i in 0..<body.safeLen:
+      evalTemplateAux(body[i], args, ctx, result)
   result.flags.incl nfFromTemplate
   result = wrapInComesFrom(n.info, tmpl, result)
   #if ctx.debugActive:
   #  echo "instantion of ", renderTree(result, {renderIds})
   dec(conf.evalTemplateCounter)
+  # The instID must be unique for every template instantiation, so we increment it here
+  inc instID[]

compiler/filter_tmpl.nim

@@ -22,7 +22,7 @@ type
     info: TLineInfo
     indent, emitPar: int
     x: string                # the current input line
-    outp: PLLStream          # the ouput will be parsed by pnimsyn
+    outp: PLLStream          # the output will be parsed by pnimsyn
     subsChar, nimDirective: char
     emit, conc, toStr: string
     curly, bracket, par: int
@@ -75,7 +75,7 @@ proc parseLine(p: var TTmplParser) =
     let d = j
     var keyw = ""
     while j < len and p.x[j] in PatternChars:
-      add(keyw, p.x[j])
+      keyw.add(p.x[j])
       inc(j)
 
     scanPar(p, j)
@@ -199,8 +199,8 @@ proc parseLine(p: var TTmplParser) =
           inc(j)
     llStreamWrite(p.outp, "\\n\"")
 
-proc filterTmpl*(stdin: PLLStream, filename: AbsoluteFile,
-                 call: PNode; conf: ConfigRef): PLLStream =
+proc filterTmpl*(conf: ConfigRef, stdin: PLLStream, filename: AbsoluteFile,
+                 call: PNode): PLLStream =
   var p: TTmplParser
   p.config = conf
   p.info = newLineInfo(conf, filename, 0, 0)
@@ -214,9 +214,9 @@ proc filterTmpl*(stdin: PLLStream, filename: AbsoluteFile,
   p.x = newStringOfCap(120)
   # do not process the first line which contains the directive:
   if llStreamReadLine(p.inp, p.x):
-    p.info.line = p.info.line + 1'u16
+    inc p.info.line
   while llStreamReadLine(p.inp, p.x):
-    p.info.line = p.info.line + 1'u16
+    inc p.info.line
     parseLine(p)
   newLine(p)
   result = p.outp

compiler/filters.nim

@@ -20,13 +20,13 @@ proc invalidPragma(conf: ConfigRef; n: PNode) =
 proc getArg(conf: ConfigRef; n: PNode, name: string, pos: int): PNode =
   result = nil
   if n.kind in {nkEmpty..nkNilLit}: return
-  for i in 1 ..< sonsLen(n):
-    if n.sons[i].kind == nkExprEqExpr:
-      if n.sons[i].sons[0].kind != nkIdent: invalidPragma(conf, n)
-      if cmpIgnoreStyle(n.sons[i].sons[0].ident.s, name) == 0:
-        return n.sons[i].sons[1]
+  for i in 1..<n.len:
+    if n[i].kind == nkExprEqExpr:
+      if n[i][0].kind != nkIdent: invalidPragma(conf, n)
+      if cmpIgnoreStyle(n[i][0].ident.s, name) == 0:
+        return n[i][1]
     elif i == pos:
-      return n.sons[i]
+      return n[i]
 
 proc charArg*(conf: ConfigRef; n: PNode, name: string, pos: int, default: char): char =
   var x = getArg(conf, n, name, pos)
@@ -55,7 +55,7 @@ proc filterStrip*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, cal
   var line = newStringOfCap(80)
   while llStreamReadLine(stdin, line):
     var stripped = strip(line, leading, trailing)
-    if len(pattern) == 0 or startsWith(stripped, pattern):
+    if pattern.len == 0 or startsWith(stripped, pattern):
       llStreamWriteln(result, stripped)
     else:
       llStreamWriteln(result, line)
@@ -63,7 +63,7 @@ proc filterStrip*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, cal
 
 proc filterReplace*(conf: ConfigRef; stdin: PLLStream, filename: AbsoluteFile, call: PNode): PLLStream =
   var sub = strArg(conf, call, "sub", 1, "")
-  if len(sub) == 0: invalidPragma(conf, call)
+  if sub.len == 0: invalidPragma(conf, call)
   var by = strArg(conf, call, "by", 2, "")
   result = llStreamOpen("")
   var line = newStringOfCap(80)

compiler/forloops.nim

@@ -1,89 +0,0 @@
-#
-#
-#           The Nim Compiler
-#        (c) Copyright 2015 Andreas Rumpf
-#
-#    See the file "copying.txt", included in this
-#    distribution, for details about the copyright.
-#
-
-## This module implements for loop detection for better C code generation.
-
-import ast, astalgo
-
-const
-  someCmp = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
-    mEqUntracedRef, mLeI, mLeF64, mLeU, mLeU64, mLeEnum,
-    mLeCh, mLeB, mLePtr, mLtI, mLtF64, mLtU, mLtU64, mLtEnum,
-    mLtCh, mLtB, mLtPtr}
-
-proc isCounter(s: PSym): bool {.inline.} =
-  s.kind in {skResult, skVar, skLet, skTemp} and
-  {sfGlobal, sfAddrTaken} * s.flags == {}
-
-proc isCall(n: PNode): bool {.inline.} =
-  n.kind in nkCallKinds and n[0].kind == nkSym
-
-proc fromSystem(op: PSym): bool = sfSystemModule in getModule(op).flags
-
-proc getCounter(lastStmt: PNode): PSym =
-  if lastStmt.isCall:
-    let op = lastStmt.sym
-    if op.magic in {mDec, mInc} or
-        ((op.name.s == "+=" or op.name.s == "-=") and op.fromSystem):
-      if op[1].kind == nkSym and isCounter(op[1].sym):
-        result = op[1].sym
-
-proc counterInTree(n, loop: PNode; counter: PSym): bool =
-  # prune the search tree: within the loop the counter may be used:
-  if n == loop: return
-  case n.kind
-  of nkSym:
-    if n.sym == counter: return true
-  of nkVarSection, nkLetSection:
-    # definitions are fine!
-    for it in n:
-      if counterInTree(it.lastSon): return true
-  else:
-    for i in 0 ..< safeLen(n):
-      if counterInTree(n[i], loop, counter): return true
-
-proc copyExcept(n: PNode, x, dest: PNode) =
-  if x == n: return
-  if n.kind in {nkStmtList, nkStmtListExpr}:
-    for i in 0 ..< n.len: copyExcept(n[i], x, dest)
-  else:
-    dest.add n
-
-type
-  ForLoop* = object
-    counter*: PSym
-    init*, cond*, increment*, body*: PNode
-
-proc extractForLoop*(loop, fullTree: PNode): ForLoop =
-  ## returns 'counter == nil' if the while loop 'n' is not a for loop:
-  assert loop.kind == nkWhileStmt
-  let cond == loop[0]
-
-  if not cond.isCall: return
-  if cond[0].sym.magic notin someCmp: return
-
-  var lastStmt = loop[1]
-  while lastStmt.kind in {nkStmtList, nkStmtListExpr}:
-    lastStmt = lastStmt.lastSon
-
-  let counter = getCounter(lastStmt)
-  if counter.isNil or counter.ast.isNil: return
-
-  template `=~`(a, b): expr = a.kind == nkSym and a.sym == b
-
-  if cond[1] =~ counter or cond[2] =~ counter:
-    # ok, now check 'counter' is not used *after* the loop
-    if counterInTree(fullTree, loop, counter): return
-    # ok, success, fill in the fields:
-    result.counter = counter
-    result.init = counter.ast
-    result.cond = cond
-    result.increment = lastStmt
-    result.body = newNodeI(nkStmtList, loop[1].info)
-    copyExcept(loop[1], lastStmt, result.body)

compiler/gorgeimpl.nim

@@ -40,6 +40,7 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
         p.inputStream.write(input)
         p.inputStream.close()
       result = p.readOutput
+      p.close()
       readSuccessful = true
       # only cache successful runs:
       if result[1] == 0:
@@ -54,5 +55,6 @@ proc opGorge*(cmd, input, cache: string, info: TLineInfo; conf: ConfigRef): (str
         p.inputStream.write(input)
         p.inputStream.close()
       result = p.readOutput
+      p.close()
     except IOError, OSError:
       result = ("", -1)

compiler/guards.nim

@@ -10,22 +10,21 @@
 ## This module implements the 'implies' relation for guards.
 
 import ast, astalgo, msgs, magicsys, nimsets, trees, types, renderer, idents,
-  saturate, modulegraphs, options, lineinfos
+  saturate, modulegraphs, options, lineinfos, int128
 
 const
   someEq = {mEqI, mEqF64, mEqEnum, mEqCh, mEqB, mEqRef, mEqProc,
-    mEqUntracedRef, mEqStr, mEqSet, mEqCString}
+    mEqStr, mEqSet, mEqCString}
 
   # set excluded here as the semantics are vastly different:
-  someLe = {mLeI, mLeF64, mLeU, mLeU64, mLeEnum,
+  someLe = {mLeI, mLeF64, mLeU, mLeEnum,
             mLeCh, mLeB, mLePtr, mLeStr}
-  someLt = {mLtI, mLtF64, mLtU, mLtU64, mLtEnum,
+  someLt = {mLtI, mLtF64, mLtU, mLtEnum,
             mLtCh, mLtB, mLtPtr, mLtStr}
 
-  someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq,
-             mXLenStr, mXLenSeq}
+  someLen = {mLengthOpenArray, mLengthStr, mLengthArray, mLengthSeq}
 
-  someIn = {mInRange, mInSet}
+  someIn = {mInSet}
 
   someHigh = {mHigh}
   # we don't list unsigned here because wrap around semantics suck for
@@ -35,8 +34,8 @@ const
   someMul = {mMulI, mMulF64}
   someDiv = {mDivI, mDivF64}
   someMod = {mModI}
-  someMax = {mMaxI, mMaxF64}
-  someMin = {mMinI, mMinF64}
+  someMax = {mMaxI}
+  someMin = {mMinI}
   someBinaryOp = someAdd+someSub+someMul+someMax+someMin
 
 proc isValue(n: PNode): bool = n.kind in {nkCharLit..nkNilLit}
@@ -47,7 +46,7 @@ proc isLet(n: PNode): bool =
     if n.sym.kind in {skLet, skTemp, skForVar}:
       result = true
     elif n.sym.kind == skParam and skipTypes(n.sym.typ,
-                                             abstractInst).kind != tyVar:
+                                             abstractInst).kind notin {tyVar}:
       result = true
 
 proc isVar(n: PNode): bool =
@@ -65,16 +64,16 @@ proc isLetLocation(m: PNode, isApprox: bool): bool =
   while true:
     case n.kind
     of nkDotExpr, nkCheckedFieldExpr, nkObjUpConv, nkObjDownConv:
-      n = n.sons[0]
+      n = n[0]
     of nkDerefExpr, nkHiddenDeref:
-      n = n.sons[0]
+      n = n[0]
       inc derefs
     of nkBracketExpr:
-      if isConstExpr(n.sons[1]) or isLet(n.sons[1]) or isConstExpr(n.sons[1].skipConv):
-        n = n.sons[0]
+      if isConstExpr(n[1]) or isLet(n[1]) or isConstExpr(n[1].skipConv):
+        n = n[0]
       else: return
     of nkHiddenStdConv, nkHiddenSubConv, nkConv:
-      n = n.sons[1]
+      n = n[1]
     else:
       break
   result = n.isLet and derefs <= ord(isApprox)
@@ -85,8 +84,8 @@ proc interestingCaseExpr*(m: PNode): bool = isLetLocation(m, true)
 
 type
   Operators* = object
-    opNot, opContains, opLe, opLt, opAnd, opOr, opIsNil, opEq: PSym
-    opAdd, opSub, opMul, opDiv, opLen: PSym
+    opNot*, opContains*, opLe*, opLt*, opAnd*, opOr*, opIsNil*, opEq*: PSym
+    opAdd*, opSub*, opMul*, opDiv*, opLen*: PSym
 
 proc initOperators*(g: ModuleGraph): Operators =
   result.opLe = createMagic(g, "<=", mLeI)
@@ -105,34 +104,34 @@ proc initOperators*(g: ModuleGraph): Operators =
 
 proc swapArgs(fact: PNode, newOp: PSym): PNode =
   result = newNodeI(nkCall, fact.info, 3)
-  result.sons[0] = newSymNode(newOp)
-  result.sons[1] = fact.sons[2]
-  result.sons[2] = fact.sons[1]
+  result[0] = newSymNode(newOp)
+  result[1] = fact[2]
+  result[2] = fact[1]
 
 proc neg(n: PNode; o: Operators): PNode =
   if n == nil: return nil
   case n.getMagic
   of mNot:
-    result = n.sons[1]
+    result = n[1]
   of someLt:
     # not (a < b)  ==  a >= b  ==  b <= a
     result = swapArgs(n, o.opLe)
   of someLe:
     result = swapArgs(n, o.opLt)
   of mInSet:
-    if n.sons[1].kind != nkCurly: return nil
-    let t = n.sons[2].typ.skipTypes(abstractInst)
+    if n[1].kind != nkCurly: return nil
+    let t = n[2].typ.skipTypes(abstractInst)
     result = newNodeI(nkCall, n.info, 3)
-    result.sons[0] = n.sons[0]
-    result.sons[2] = n.sons[2]
+    result[0] = n[0]
+    result[2] = n[2]
     if t.kind == tyEnum:
-      var s = newNodeIT(nkCurly, n.info, n.sons[1].typ)
+      var s = newNodeIT(nkCurly, n.info, n[1].typ)
       for e in t.n:
         let eAsNode = newIntNode(nkIntLit, e.sym.position)
-        if not inSet(n.sons[1], eAsNode): s.add eAsNode
-      result.sons[1] = s
+        if not inSet(n[1], eAsNode): s.add eAsNode
+      result[1] = s
     #elif t.kind notin {tyString, tySequence} and lengthOrd(t) < 1000:
-    #  result.sons[1] = complement(n.sons[1])
+    #  result[1] = complement(n[1])
     else:
       # not ({2, 3, 4}.contains(x))   x != 2 and x != 3 and x != 4
       # XXX todo
@@ -140,13 +139,13 @@ proc neg(n: PNode; o: Operators): PNode =
   of mOr:
     # not (a or b) --> not a and not b
     let
-      a = n.sons[1].neg(o)
-      b = n.sons[2].neg(o)
+      a = n[1].neg(o)
+      b = n[2].neg(o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
     elif a != nil:
       result = a
     elif b != nil:
@@ -154,19 +153,19 @@ proc neg(n: PNode; o: Operators): PNode =
   else:
     # leave  not (a == 4)  as it is
     result = newNodeI(nkCall, n.info, 2)
-    result.sons[0] = newSymNode(o.opNot)
-    result.sons[1] = n
+    result[0] = newSymNode(o.opNot)
+    result[1] = n
 
-proc buildCall(op: PSym; a: PNode): PNode =
+proc buildCall*(op: PSym; a: PNode): PNode =
   result = newNodeI(nkCall, a.info, 2)
-  result.sons[0] = newSymNode(op)
-  result.sons[1] = a
+  result[0] = newSymNode(op)
+  result[1] = a
 
-proc buildCall(op: PSym; a, b: PNode): PNode =
+proc buildCall*(op: PSym; a, b: PNode): PNode =
   result = newNodeI(nkInfix, a.info, 3)
-  result.sons[0] = newSymNode(op)
-  result.sons[1] = a
-  result.sons[2] = b
+  result[0] = newSymNode(op)
+  result[1] = a
+  result[2] = b
 
 proc `|+|`(a, b: PNode): PNode =
   result = copyNode(a)
@@ -250,35 +249,35 @@ proc pred(n: PNode): PNode =
   else:
     result = n
 
+proc buildLe*(o: Operators; a, b: PNode): PNode =
+  result = o.opLe.buildCall(a, b)
+
 proc canon*(n: PNode; o: Operators): PNode =
-  # XXX for now only the new code in 'semparallel' uses this
   if n.safeLen >= 1:
     result = shallowCopy(n)
-    for i in 0 ..< n.len:
-      result.sons[i] = canon(n.sons[i], o)
+    for i in 0..<n.len:
+      result[i] = canon(n[i], o)
   elif n.kind == nkSym and n.sym.kind == skLet and
       n.sym.astdef.getMagic in (someEq + someAdd + someMul + someMin +
-      someMax + someHigh + {mUnaryLt} + someSub + someLen + someDiv):
+      someMax + someHigh + someSub + someLen + someDiv):
     result = n.sym.astdef.copyTree
   else:
     result = n
   case result.getMagic
   of someEq, someAdd, someMul, someMin, someMax:
     # these are symmetric; put value as last:
-    if result.sons[1].isValue and not result.sons[2].isValue:
-      result = swapArgs(result, result.sons[0].sym)
+    if result[1].isValue and not result[2].isValue:
+      result = swapArgs(result, result[0].sym)
       # (4 + foo) + 2 --> (foo + 4) + 2
   of someHigh:
     # high == len+(-1)
     result = o.opAdd.buildCall(o.opLen.buildCall(result[1]), minusOne())
-  of mUnaryLt:
-    result = buildCall(o.opAdd, result[1], minusOne())
   of someSub:
     # x - 4  -->  x + (-4)
     result = negate(result[1], result[2], result, o)
   of someLen:
-    result.sons[0] = o.opLen.newSymNode
-  of someLt:
+    result[0] = o.opLen.newSymNode
+  of someLt - {mLtF64}:
     # x < y  same as x <= y-1:
     let y = n[2].canon(o)
     let p = pred(y)
@@ -318,16 +317,16 @@ proc canon*(n: PNode; o: Operators): PNode =
         if plus != nil and not isLetLocation(x, true):
           result = buildCall(result[0].sym, plus, y[1])
       else: discard
-    elif x.isValue and y.getMagic in someAdd and y[2].isValue:
+    elif x.isValue and y.getMagic in someAdd and y[2].kind == x.kind:
       # 0 <= a.len + 3
       # -3 <= a.len
-      result.sons[1] = x |-| y[2]
-      result.sons[2] = y[1]
-    elif x.isValue and y.getMagic in someSub and y[2].isValue:
+      result[1] = x |-| y[2]
+      result[2] = y[1]
+    elif x.isValue and y.getMagic in someSub and y[2].kind == x.kind:
       # 0 <= a.len - 3
       # 3 <= a.len
-      result.sons[1] = x |+| y[2]
-      result.sons[2] = y[1]
+      result[1] = x |+| y[2]
+      result[2] = y[1]
   else: discard
 
 proc buildAdd*(a: PNode; b: BiggestInt; o: Operators): PNode =
@@ -336,41 +335,43 @@ proc buildAdd*(a: PNode; b: BiggestInt; o: Operators): PNode =
 proc usefulFact(n: PNode; o: Operators): PNode =
   case n.getMagic
   of someEq:
-    if skipConv(n.sons[2]).kind == nkNilLit and (
-        isLetLocation(n.sons[1], false) or isVar(n.sons[1])):
-      result = o.opIsNil.buildCall(n.sons[1])
+    if skipConv(n[2]).kind == nkNilLit and (
+        isLetLocation(n[1], false) or isVar(n[1])):
+      result = o.opIsNil.buildCall(n[1])
     else:
-      if isLetLocation(n.sons[1], true) or isLetLocation(n.sons[2], true):
+      if isLetLocation(n[1], true) or isLetLocation(n[2], true):
         # XXX algebraic simplifications!  'i-1 < a.len' --> 'i < a.len+1'
         result = n
+      elif n[1].getMagic in someLen or n[2].getMagic in someLen:
+        result = n
   of someLe+someLt:
-    if isLetLocation(n.sons[1], true) or isLetLocation(n.sons[2], true):
+    if isLetLocation(n[1], true) or isLetLocation(n[2], true):
       # XXX algebraic simplifications!  'i-1 < a.len' --> 'i < a.len+1'
       result = n
     elif n[1].getMagic in someLen or n[2].getMagic in someLen:
       # XXX Rethink this whole idea of 'usefulFact' for semparallel
       result = n
   of mIsNil:
-    if isLetLocation(n.sons[1], false) or isVar(n.sons[1]):
+    if isLetLocation(n[1], false) or isVar(n[1]):
       result = n
   of someIn:
-    if isLetLocation(n.sons[1], true):
+    if isLetLocation(n[1], true):
       result = n
   of mAnd:
     let
-      a = usefulFact(n.sons[1], o)
-      b = usefulFact(n.sons[2], o)
+      a = usefulFact(n[1], o)
+      b = usefulFact(n[2], o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
     elif a != nil:
       result = a
     elif b != nil:
       result = b
   of mNot:
-    let a = usefulFact(n.sons[1], o)
+    let a = usefulFact(n[1], o)
     if a != nil:
       result = a.neg(o)
   of mOr:
@@ -381,13 +382,13 @@ proc usefulFact(n: PNode; o: Operators): PNode =
     #  (x == 3) or (y == 2)  ---> not ( not (x==3) and not (y == 2))
     #  not (x != 3 and y != 2)
     let
-      a = usefulFact(n.sons[1], o).neg(o)
-      b = usefulFact(n.sons[2], o).neg(o)
+      a = usefulFact(n[1], o).neg(o)
+      b = usefulFact(n[2], o).neg(o)
     if a != nil and b != nil:
       result = newNodeI(nkCall, n.info, 3)
-      result.sons[0] = newSymNode(o.opAnd)
-      result.sons[1] = a
-      result.sons[2] = b
+      result[0] = newSymNode(o.opAnd)
+      result[1] = a
+      result[2] = b
       result = result.neg(o)
   elif n.kind == nkSym and n.sym.kind == skLet:
     # consider:
@@ -404,10 +405,20 @@ type
   TModel* = object
     s*: seq[PNode] # the "knowledge base"
     o*: Operators
+    beSmart*: bool
 
 proc addFact*(m: var TModel, nn: PNode) =
   let n = usefulFact(nn, m.o)
-  if n != nil: m.s.add n
+  if n != nil:
+    if not m.beSmart:
+      m.s.add n
+    else:
+      let c = canon(n, m.o)
+      if c.getMagic == mAnd:
+        addFact(m, c[1])
+        addFact(m, c[2])
+      else:
+        m.s.add c
 
 proc addFactNeg*(m: var TModel, n: PNode) =
   let n = n.neg(m.o)
@@ -436,24 +447,24 @@ proc sameTree*(a, b: PNode): bool =
       if not result and a.sym.magic != mNone:
         result = a.sym.magic == b.sym.magic or sameOpr(a.sym, b.sym)
     of nkIdent: result = a.ident.id == b.ident.id
-    of nkCharLit..nkInt64Lit: result = a.intVal == b.intVal
+    of nkCharLit..nkUInt64Lit: result = a.intVal == b.intVal
     of nkFloatLit..nkFloat64Lit: result = a.floatVal == b.floatVal
     of nkStrLit..nkTripleStrLit: result = a.strVal == b.strVal
     of nkType: result = a.typ == b.typ
     of nkEmpty, nkNilLit: result = true
     else:
-      if sonsLen(a) == sonsLen(b):
-        for i in 0 ..< sonsLen(a):
-          if not sameTree(a.sons[i], b.sons[i]): return
+      if a.len == b.len:
+        for i in 0..<a.len:
+          if not sameTree(a[i], b[i]): return
         result = true
 
 proc hasSubTree(n, x: PNode): bool =
   if n.sameTree(x): result = true
   else:
-    for i in 0..safeLen(n)-1:
-      if hasSubTree(n.sons[i], x): return true
+    for i in 0..n.safeLen-1:
+      if hasSubTree(n[i], x): return true
 
-proc invalidateFacts*(m: var TModel, n: PNode) =
+proc invalidateFacts*(s: var seq[PNode], n: PNode) =
   # We are able to guard local vars (as opposed to 'let' variables)!
   # 'while p != nil: f(p); p = p.next'
   # This is actually quite easy to do:
@@ -471,29 +482,32 @@ proc invalidateFacts*(m: var TModel, n: PNode) =
   # The same mechanism could be used for more complex data stored on the heap;
   # procs that 'write: []' cannot invalidate 'n.kind' for instance. In fact, we
   # could CSE these expressions then and help C's optimizer.
-  for i in 0..high(m.s):
-    if m.s[i] != nil and m.s[i].hasSubTree(n): m.s[i] = nil
+  for i in 0..high(s):
+    if s[i] != nil and s[i].hasSubTree(n): s[i] = nil
+
+proc invalidateFacts*(m: var TModel, n: PNode) =
+  invalidateFacts(m.s, n)
 
 proc valuesUnequal(a, b: PNode): bool =
   if a.isValue and b.isValue:
     result = not sameValue(a, b)
 
 proc impliesEq(fact, eq: PNode): TImplication =
-  let (loc, val) = if isLocation(eq.sons[1]): (1, 2) else: (2, 1)
+  let (loc, val) = if isLocation(eq[1]): (1, 2) else: (2, 1)
 
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], eq.sons[loc]):
+    if sameTree(fact[1], eq[loc]):
       # this is not correct; consider:  a == b;  a == 1 --> unknown!
-      if sameTree(fact.sons[2], eq.sons[val]): result = impYes
-      elif valuesUnequal(fact.sons[2], eq.sons[val]): result = impNo
-    elif sameTree(fact.sons[2], eq.sons[loc]):
-      if sameTree(fact.sons[1], eq.sons[val]): result = impYes
-      elif valuesUnequal(fact.sons[1], eq.sons[val]): result = impNo
+      if sameTree(fact[2], eq[val]): result = impYes
+      elif valuesUnequal(fact[2], eq[val]): result = impNo
+    elif sameTree(fact[2], eq[loc]):
+      if sameTree(fact[1], eq[val]): result = impYes
+      elif valuesUnequal(fact[1], eq[val]): result = impNo
   of mInSet:
     # remember: mInSet is 'contains' so the set comes first!
-    if sameTree(fact.sons[2], eq.sons[loc]) and isValue(eq.sons[val]):
-      if inSet(fact.sons[1], eq.sons[val]): result = impYes
+    if sameTree(fact[2], eq[loc]) and isValue(eq[val]):
+      if inSet(fact[1], eq[val]): result = impYes
       else: result = impNo
   of mNot, mOr, mAnd: assert(false, "impliesEq")
   else: discard
@@ -522,7 +536,7 @@ proc geImpliesIn(x, c, aSet: PNode): TImplication =
     var value = newIntNode(c.kind, c.intVal)
     let max = lastOrd(nil, x.typ)
     # don't iterate too often:
-    if max - value.intVal < 1000:
+    if max - getInt(value) < toInt128(1000):
       var i, pos, neg: int
       while value.intVal <= max:
         if inSet(aSet, value): inc pos
@@ -536,28 +550,28 @@ proc compareSets(a, b: PNode): TImplication =
   elif intersectSets(nil, a, b).len == 0: result = impNo
 
 proc impliesIn(fact, loc, aSet: PNode): TImplication =
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], loc):
-      if inSet(aSet, fact.sons[2]): result = impYes
+    if sameTree(fact[1], loc):
+      if inSet(aSet, fact[2]): result = impYes
       else: result = impNo
-    elif sameTree(fact.sons[2], loc):
-      if inSet(aSet, fact.sons[1]): result = impYes
+    elif sameTree(fact[2], loc):
+      if inSet(aSet, fact[1]): result = impYes
       else: result = impNo
   of mInSet:
-    if sameTree(fact.sons[2], loc):
-      result = compareSets(fact.sons[1], aSet)
+    if sameTree(fact[2], loc):
+      result = compareSets(fact[1], aSet)
   of someLe:
-    if sameTree(fact.sons[1], loc):
-      result = leImpliesIn(fact.sons[1], fact.sons[2], aSet)
-    elif sameTree(fact.sons[2], loc):
-      result = geImpliesIn(fact.sons[2], fact.sons[1], aSet)
+    if sameTree(fact[1], loc):
+      result = leImpliesIn(fact[1], fact[2], aSet)
+    elif sameTree(fact[2], loc):
+      result = geImpliesIn(fact[2], fact[1], aSet)
   of someLt:
-    if sameTree(fact.sons[1], loc):
-      result = leImpliesIn(fact.sons[1], fact.sons[2].pred, aSet)
-    elif sameTree(fact.sons[2], loc):
+    if sameTree(fact[1], loc):
+      result = leImpliesIn(fact[1], fact[2].pred, aSet)
+    elif sameTree(fact[2], loc):
       # 4 < x  -->  3 <= x
-      result = geImpliesIn(fact.sons[2], fact.sons[1].pred, aSet)
+      result = geImpliesIn(fact[2], fact[1].pred, aSet)
   of mNot, mOr, mAnd: assert(false, "impliesIn")
   else: discard
 
@@ -567,90 +581,93 @@ proc valueIsNil(n: PNode): TImplication =
   else: impUnknown
 
 proc impliesIsNil(fact, eq: PNode): TImplication =
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of mIsNil:
-    if sameTree(fact.sons[1], eq.sons[1]):
+    if sameTree(fact[1], eq[1]):
       result = impYes
   of someEq:
-    if sameTree(fact.sons[1], eq.sons[1]):
-      result = valueIsNil(fact.sons[2].skipConv)
-    elif sameTree(fact.sons[2], eq.sons[1]):
-      result = valueIsNil(fact.sons[1].skipConv)
+    if sameTree(fact[1], eq[1]):
+      result = valueIsNil(fact[2].skipConv)
+    elif sameTree(fact[2], eq[1]):
+      result = valueIsNil(fact[1].skipConv)
   of mNot, mOr, mAnd: assert(false, "impliesIsNil")
   else: discard
 
 proc impliesGe(fact, x, c: PNode): TImplication =
   assert isLocation(x)
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x = 4;  question x >= 56? --> true iff 4 >= 56
-        if leValue(c, fact.sons[2]): result = impYes
+        if leValue(c, fact[2]): result = impYes
         else: result = impNo
-    elif sameTree(fact.sons[2], x):
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impYes
+    elif sameTree(fact[2], x):
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impYes
         else: result = impNo
   of someLt:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x < 4;  question N <= x? --> false iff N <= 4
-        if leValue(fact.sons[2], c): result = impNo
+        if leValue(fact[2], c): result = impNo
         # fact:  x < 4;  question 2 <= x? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 < x; question: N-1 < x ?  --> true iff N-1 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c.pred, fact.sons[1]): result = impYes
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c.pred, fact[1]): result = impYes
   of someLe:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x <= 4;  question x >= 56? --> false iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impNo
+        if leValue(fact[2], c): result = impNo
         # fact:  x <= 4;  question x >= 2? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 <= x; question: x >= 2 ?  --> true iff 2 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impYes
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impYes
   of mNot, mOr, mAnd: assert(false, "impliesGe")
   else: discard
 
 proc impliesLe(fact, x, c: PNode): TImplication =
   if not isLocation(x):
+    if c.isValue:
+      if leValue(x, x): return impYes
+      else: return impNo
     return impliesGe(fact, c, x)
-  case fact.sons[0].sym.magic
+  case fact[0].sym.magic
   of someEq:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x = 4;  question x <= 56? --> true iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impYes
+        if leValue(fact[2], c): result = impYes
         else: result = impNo
-    elif sameTree(fact.sons[2], x):
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(fact.sons[1], c): result = impYes
+    elif sameTree(fact[2], x):
+      if isValue(fact[1]) and isValue(c):
+        if leValue(fact[1], c): result = impYes
         else: result = impNo
   of someLt:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x < 4;  question x <= N? --> true iff N-1 <= 4
-        if leValue(fact.sons[2], c.pred): result = impYes
+        if leValue(fact[2], c.pred): result = impYes
         # fact:  x < 4;  question x <= 2? --> we don't know
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 < x; question: x <= 1 ?  --> false iff 1 <= 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1]): result = impNo
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1]): result = impNo
 
   of someLe:
-    if sameTree(fact.sons[1], x):
-      if isValue(fact.sons[2]) and isValue(c):
+    if sameTree(fact[1], x):
+      if isValue(fact[2]) and isValue(c):
         # fact:  x <= 4;  question x <= 56? --> true iff 4 <= 56
-        if leValue(fact.sons[2], c): result = impYes
+        if leValue(fact[2], c): result = impYes
         # fact:  x <= 4;  question x <= 2? --> we don't know
 
-    elif sameTree(fact.sons[2], x):
+    elif sameTree(fact[2], x):
       # fact: 3 <= x; question: x <= 2 ?  --> false iff 2 < 3
-      if isValue(fact.sons[1]) and isValue(c):
-        if leValue(c, fact.sons[1].pred): result = impNo
+      if isValue(fact[1]) and isValue(c):
+        if leValue(c, fact[1].pred): result = impNo
 
   of mNot, mOr, mAnd: assert(false, "impliesLe")
   else: discard
@@ -686,32 +703,32 @@ proc factImplies(fact, prop: PNode): TImplication =
 
     #  (not a) -> b  compute as  not (a -> b) ???
     #  == not a or not b == not (a and b)
-    let arg = fact.sons[1]
+    let arg = fact[1]
     case arg.getMagic
     of mIsNil, mEqRef:
       return ~factImplies(arg, prop)
     of mAnd:
       # not (a and b)  means  not a or not b:
       # a or b --> both need to imply 'prop'
-      let a = factImplies(arg.sons[1], prop)
-      let b = factImplies(arg.sons[2], prop)
+      let a = factImplies(arg[1], prop)
+      let b = factImplies(arg[2], prop)
       if a == b: return ~a
       return impUnknown
     else:
       return impUnknown
   of mAnd:
-    result = factImplies(fact.sons[1], prop)
+    result = factImplies(fact[1], prop)
     if result != impUnknown: return result
-    return factImplies(fact.sons[2], prop)
+    return factImplies(fact[2], prop)
   else: discard
 
-  case prop.sons[0].sym.magic
-  of mNot: result = ~fact.factImplies(prop.sons[1])
+  case prop[0].sym.magic
+  of mNot: result = ~fact.factImplies(prop[1])
   of mIsNil: result = impliesIsNil(fact, prop)
   of someEq: result = impliesEq(fact, prop)
-  of someLe: result = impliesLe(fact, prop.sons[1], prop.sons[2])
-  of someLt: result = impliesLt(fact, prop.sons[1], prop.sons[2])
-  of mInSet: result = impliesIn(fact, prop.sons[2], prop.sons[1])
+  of someLe: result = impliesLe(fact, prop[1], prop[2])
+  of someLt: result = impliesLt(fact, prop[1], prop[2])
+  of mInSet: result = impliesIn(fact, prop[2], prop[1])
   else: result = impUnknown
 
 proc doesImply*(facts: TModel, prop: PNode): TImplication =
@@ -802,7 +819,7 @@ proc ple(m: TModel; a, b: PNode): TImplication =
     if lastOrd(nil, a.typ) <= b.intVal: return impYes
   # 3 <= x   iff  low(x) <= 3
   if a.isValue and b.typ != nil and b.typ.isOrdinalType:
-    if firstOrd(nil, b.typ) <= a.intVal: return impYes
+    if a.intVal <= firstOrd(nil, b.typ): return impYes
 
   # x <= x
   if sameTree(a, b): return impYes
@@ -813,7 +830,11 @@ proc ple(m: TModel; a, b: PNode): TImplication =
 
   #   x <= y+c  if 0 <= c and x <= y
   #   x <= y+(-c)  if c <= 0  and y >= x
-  if b.getMagic in someAdd and zero() <=? b[2] and a <=? b[1]: return impYes
+  if b.getMagic in someAdd:
+    if zero() <=? b[2] and a <=? b[1]: return impYes
+    # x <= y-c  if x+c <= y
+    if b[2] <=? zero() and (canon(m.o.opSub.buildCall(a, b[2]), m.o) <=? b[1]):
+      return impYes
 
   #   x+c <= y  if c <= 0 and x <= y
   if a.getMagic in someAdd and a[2] <=? zero() and a[1] <=? b: return impYes
@@ -881,8 +902,8 @@ proc replaceSubTree(n, x, by: PNode): PNode =
     result = by
   elif hasSubTree(n, x):
     result = shallowCopy(n)
-    for i in 0 .. safeLen(n)-1:
-      result.sons[i] = replaceSubTree(n.sons[i], x, by)
+    for i in 0..n.safeLen-1:
+      result[i] = replaceSubTree(n[i], x, by)
   else:
     result = n
 
@@ -902,10 +923,13 @@ proc pleViaModelRec(m: var TModel; a, b: PNode): TImplication =
       # --> true  if  (len-100) <= (len-1)
       let x = fact[1]
       let y = fact[2]
-      if sameTree(x, a) and y.getMagic in someAdd and b.getMagic in someAdd and
-         sameTree(y[1], b[1]):
-        if ple(m, b[2], y[2]) == impYes:
-          return impYes
+      # x <= y.
+      # Question: x <= b? True iff y <= b.
+      if sameTree(x, a):
+        if ple(m, y, b) == impYes: return impYes
+        if y.getMagic in someAdd and b.getMagic in someAdd and sameTree(y[1], b[1]):
+          if ple(m, b[2], y[2]) == impYes:
+            return impYes
 
       # x <= y implies a <= b  if  a <= x and y <= b
       if ple(m, a, x) == impYes:
@@ -964,6 +988,10 @@ proc proveLe*(m: TModel; a, b: PNode): TImplication =
 proc addFactLe*(m: var TModel; a, b: PNode) =
   m.s.add canon(m.o.opLe.buildCall(a, b), m.o)
 
+proc addFactLt*(m: var TModel; a, b: PNode) =
+  let bb = m.o.opAdd.buildCall(b, minusOne())
+  addFactLe(m, a, bb)
+
 proc settype(n: PNode): PType =
   result = newType(tySet, n.typ.owner)
   addSonSkipIntLit(result, n.typ)
@@ -971,37 +999,37 @@ proc settype(n: PNode): PType =
 proc buildOf(it, loc: PNode; o: Operators): PNode =
   var s = newNodeI(nkCurly, it.info, it.len-1)
   s.typ = settype(loc)
-  for i in 0..it.len-2: s.sons[i] = it.sons[i]
+  for i in 0..<it.len-1: s[i] = it[i]
   result = newNodeI(nkCall, it.info, 3)
-  result.sons[0] = newSymNode(o.opContains)
-  result.sons[1] = s
-  result.sons[2] = loc
+  result[0] = newSymNode(o.opContains)
+  result[1] = s
+  result[2] = loc
 
 proc buildElse(n: PNode; o: Operators): PNode =
-  var s = newNodeIT(nkCurly, n.info, settype(n.sons[0]))
-  for i in 1..n.len-2:
-    let branch = n.sons[i]
+  var s = newNodeIT(nkCurly, n.info, settype(n[0]))
+  for i in 1..<n.len-1:
+    let branch = n[i]
     assert branch.kind != nkElse
     if branch.kind == nkOfBranch:
-      for j in 0..branch.len-2:
-        s.add(branch.sons[j])
+      for j in 0..<branch.len-1:
+        s.add(branch[j])
   result = newNodeI(nkCall, n.info, 3)
-  result.sons[0] = newSymNode(o.opContains)
-  result.sons[1] = s
-  result.sons[2] = n.sons[0]
+  result[0] = newSymNode(o.opContains)
+  result[1] = s
+  result[2] = n[0]
 
 proc addDiscriminantFact*(m: var TModel, n: PNode) =
   var fact = newNodeI(nkCall, n.info, 3)
-  fact.sons[0] = newSymNode(m.o.opEq)
-  fact.sons[1] = n.sons[0]
-  fact.sons[2] = n.sons[1]
+  fact[0] = newSymNode(m.o.opEq)
+  fact[1] = n[0]
+  fact[2] = n[1]
   m.s.add fact
 
 proc addAsgnFact*(m: var TModel, key, value: PNode) =
   var fact = newNodeI(nkCall, key.info, 3)
-  fact.sons[0] = newSymNode(m.o.opEq)
-  fact.sons[1] = key
-  fact.sons[2] = value
+  fact[0] = newSymNode(m.o.opEq)
+  fact[1] = key
+  fact[2] = value
   m.s.add fact
 
 proc sameSubexprs*(m: TModel; a, b: PNode): bool =
@@ -1015,34 +1043,34 @@ proc sameSubexprs*(m: TModel; a, b: PNode): bool =
   # However, nil checking requires exactly the same mechanism! But for now
   # we simply use sameTree and live with the unsoundness of the analysis.
   var check = newNodeI(nkCall, a.info, 3)
-  check.sons[0] = newSymNode(m.o.opEq)
-  check.sons[1] = a
-  check.sons[2] = b
+  check[0] = newSymNode(m.o.opEq)
+  check[1] = a
+  check[2] = b
   result = m.doesImply(check) == impYes
 
 proc addCaseBranchFacts*(m: var TModel, n: PNode, i: int) =
-  let branch = n.sons[i]
+  let branch = n[i]
   if branch.kind == nkOfBranch:
-    m.s.add buildOf(branch, n.sons[0], m.o)
+    m.s.add buildOf(branch, n[0], m.o)
   else:
     m.s.add n.buildElse(m.o).neg(m.o)
 
 proc buildProperFieldCheck(access, check: PNode; o: Operators): PNode =
-  if check.sons[1].kind == nkCurly:
+  if check[1].kind == nkCurly:
     result = copyTree(check)
     if access.kind == nkDotExpr:
       var a = copyTree(access)
-      a.sons[1] = check.sons[2]
-      result.sons[2] = a
+      a[1] = check[2]
+      result[2] = a
       # 'access.kind != nkDotExpr' can happen for object constructors
       # which we don't check yet
   else:
     # it is some 'not'
     assert check.getMagic == mNot
-    result = buildProperFieldCheck(access, check.sons[1], o).neg(o)
+    result = buildProperFieldCheck(access, check[1], o).neg(o)
 
 proc checkFieldAccess*(m: TModel, n: PNode; conf: ConfigRef) =
-  for i in 1..n.len-1:
-    let check = buildProperFieldCheck(n.sons[0], n.sons[i], m.o)
+  for i in 1..<n.len:
+    let check = buildProperFieldCheck(n[0], n[i], m.o)
     if check != nil and m.doesImply(check) != impYes:
-      message(conf, n.info, warnProveField, renderTree(n.sons[0])); break
+      message(conf, n.info, warnProveField, renderTree(n[0])); break

compiler/hlo.nim

@@ -12,14 +12,14 @@
 proc hlo(c: PContext, n: PNode): PNode
 
 proc evalPattern(c: PContext, n, orig: PNode): PNode =
-  internalAssert c.config, n.kind == nkCall and n.sons[0].kind == nkSym
+  internalAssert c.config, n.kind == nkCall and n[0].kind == nkSym
   # we need to ensure that the resulting AST is semchecked. However, it's
-  # aweful to semcheck before macro invocation, so we don't and treat
+  # awful to semcheck before macro invocation, so we don't and treat
   # templates and macros as immediate in this context.
   var rule: string
-  if optHints in c.config.options and hintPattern in c.config.notes:
+  if c.config.hasHint(hintPattern):
     rule = renderTree(n, {renderNoComments})
-  let s = n.sons[0].sym
+  let s = n[0].sym
   case s.kind
   of skMacro:
     result = semMacroExpr(c, n, orig, s)
@@ -27,7 +27,7 @@ proc evalPattern(c: PContext, n, orig: PNode): PNode =
     result = semTemplateExpr(c, n, s, {efFromHlo})
   else:
     result = semDirectOp(c, n, {})
-  if optHints in c.config.options and hintPattern in c.config.notes:
+  if c.config.hasHint(hintPattern):
     message(c.config, orig.info, hintPattern, rule & " --> '" &
       renderTree(result, {renderNoComments}) & "'")
 
@@ -50,7 +50,7 @@ proc applyPatterns(c: PContext, n: PNode): PNode =
         c.patterns[i] = nil
         if x.kind == nkStmtList:
           assert x.len == 3
-          x.sons[1] = evalPattern(c, x.sons[1], result)
+          x[1] = evalPattern(c, x[1], result)
           result = flattenStmts(x)
         else:
           result = evalPattern(c, x, result)
@@ -68,17 +68,17 @@ proc hlo(c: PContext, n: PNode): PNode =
     result = n
   else:
     if n.kind in {nkFastAsgn, nkAsgn, nkIdentDefs, nkVarTuple} and
-        n.sons[0].kind == nkSym and
-        {sfGlobal, sfPure} * n.sons[0].sym.flags == {sfGlobal, sfPure}:
+        n[0].kind == nkSym and
+        {sfGlobal, sfPure} * n[0].sym.flags == {sfGlobal, sfPure}:
       # do not optimize 'var g {.global} = re(...)' again!
       return n
     result = applyPatterns(c, n)
     if result == n:
       # no optimization applied, try subtrees:
-      for i in 0 ..< safeLen(result):
-        let a = result.sons[i]
+      for i in 0..<result.safeLen:
+        let a = result[i]
         let h = hlo(c, a)
-        if h != a: result.sons[i] = h
+        if h != a: result[i] = h
     else:
       # perform type checking, so that the replacement still fits:
       if isEmptyType(n.typ) and isEmptyType(result.typ):

compiler/idents.nim

@@ -87,7 +87,7 @@ proc getIdent*(ic: IdentCache; identifier: cstring, length: int, h: Hash): PIden
   new(result)
   result.h = h
   result.s = newString(length)
-  for i in 0 ..< length: result.s[i] = identifier[i]
+  for i in 0..<length: result.s[i] = identifier[i]
   result.next = ic.buckets[idx]
   ic.buckets[idx] = result
   if id == 0:
@@ -97,11 +97,11 @@ proc getIdent*(ic: IdentCache; identifier: cstring, length: int, h: Hash): PIden
     result.id = id
 
 proc getIdent*(ic: IdentCache; identifier: string): PIdent =
-  result = getIdent(ic, cstring(identifier), len(identifier),
+  result = getIdent(ic, cstring(identifier), identifier.len,
                     hashIgnoreStyle(identifier))
 
 proc getIdent*(ic: IdentCache; identifier: string, h: Hash): PIdent =
-  result = getIdent(ic, cstring(identifier), len(identifier), h)
+  result = getIdent(ic, cstring(identifier), identifier.len, h)
 
 proc newIdentCache*(): IdentCache =
   result = IdentCache()
@@ -110,8 +110,8 @@ proc newIdentCache*(): IdentCache =
   result.idDelegator = result.getIdent":delegator"
   result.emptyIdent = result.getIdent("")
   # initialize the keywords:
-  for s in succ(low(specialWords)) .. high(specialWords):
-    result.getIdent(specialWords[s], hashIgnoreStyle(specialWords[s])).id = ord(s)
+  for s in succ(low(TSpecialWord))..high(TSpecialWord):
+    result.getIdent($s, hashIgnoreStyle($s)).id = ord(s)
 
 proc whichKeyword*(id: PIdent): TSpecialWord =
   if id.id < 0: result = wInvalid

compiler/importer.nim

@@ -11,12 +11,12 @@
 
 import
   intsets, ast, astalgo, msgs, options, idents, lookups,
-  semdata, modulepaths, sigmatch, lineinfos
+  semdata, modulepaths, sigmatch, lineinfos, sets
 
 proc readExceptSet*(c: PContext, n: PNode): IntSet =
   assert n.kind in {nkImportExceptStmt, nkExportExceptStmt}
   result = initIntSet()
-  for i in 1 ..< n.len:
+  for i in 1..<n.len:
     let ident = lookups.considerQuotedIdent(c, n[i])
     result.incl(ident.id)
 
@@ -27,11 +27,11 @@ proc importPureEnumField*(c: PContext; s: PSym) =
     if checkB == nil:
       strTableAdd(c.pureEnumFields, s)
     else:
-      # mark as ambigous:
+      # mark as ambiguous:
       incl(c.ambiguousSymbols, checkB.id)
       incl(c.ambiguousSymbols, s.id)
 
-proc rawImportSymbol(c: PContext, s: PSym) =
+proc rawImportSymbol(c: PContext, s, origin: PSym) =
   # This does not handle stubs, because otherwise loading on demand would be
   # pointless in practice. So importing stubs is fine here!
   # check if we have already a symbol of the same name:
@@ -47,8 +47,8 @@ proc rawImportSymbol(c: PContext, s: PSym) =
   if s.kind == skType:
     var etyp = s.typ
     if etyp.kind in {tyBool, tyEnum}:
-      for j in 0 ..< sonsLen(etyp.n):
-        var e = etyp.n.sons[j].sym
+      for j in 0..<etyp.n.len:
+        var e = etyp.n[j].sym
         if e.kind != skEnumField:
           internalError(c.config, s.info, "rawImportSymbol")
           # BUGFIX: because of aliases for enums the symbol may already
@@ -63,13 +63,14 @@ proc rawImportSymbol(c: PContext, s: PSym) =
           check = nextIdentIter(it, c.importTable.symbols)
         if e != nil:
           if sfPure notin s.flags:
-            rawImportSymbol(c, e)
+            rawImportSymbol(c, e, origin)
           else:
             importPureEnumField(c, e)
   else:
-    # rodgen assures that converters and patterns are no stubs
     if s.kind == skConverter: addConverter(c, s)
     if hasPattern(s): addPattern(c, s)
+  if s.owner != origin:
+    c.exportIndirections.incl((origin.id, s.id))
 
 proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
   let ident = lookups.considerQuotedIdent(c, n)
@@ -88,10 +89,10 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) =
       while e != nil:
         if e.name.id != s.name.id: internalError(c.config, n.info, "importSymbol: 3")
         if s.kind in ExportableSymKinds:
-          rawImportSymbol(c, e)
+          rawImportSymbol(c, e, fromMod)
         e = nextIdentIter(it, fromMod.tab)
     else:
-      rawImportSymbol(c, s)
+      rawImportSymbol(c, s, fromMod)
     suggestSym(c.config, n.info, s, c.graph.usageSym, false)
 
 proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
@@ -103,14 +104,14 @@ proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: IntSet) =
         if s.kind notin ExportableSymKinds:
           internalError(c.config, s.info, "importAllSymbols: " & $s.kind & " " & s.name.s)
         if exceptSet.isNil or s.name.id notin exceptSet:
-          rawImportSymbol(c, s)
+          rawImportSymbol(c, s, fromMod)
     s = nextIter(i, fromMod.tab)
 
 proc importAllSymbols*(c: PContext, fromMod: PSym) =
   var exceptSet: IntSet
   importAllSymbolsExcept(c, fromMod, exceptSet)
 
-proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet) =
+proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet; fromMod: PSym) =
   if n.isNil: return
   case n.kind
   of nkExportStmt:
@@ -120,21 +121,22 @@ proc importForwarded(c: PContext, n: PNode, exceptSet: IntSet) =
       if s.kind == skModule:
         importAllSymbolsExcept(c, s, exceptSet)
       elif exceptSet.isNil or s.name.id notin exceptSet:
-        rawImportSymbol(c, s)
+        rawImportSymbol(c, s, fromMod)
   of nkExportExceptStmt:
     localError(c.config, n.info, "'export except' not implemented")
   else:
-    for i in 0..safeLen(n)-1:
-      importForwarded(c, n.sons[i], exceptSet)
+    for i in 0..n.safeLen-1:
+      importForwarded(c, n[i], exceptSet, fromMod)
 
 proc importModuleAs(c: PContext; n: PNode, realModule: PSym): PSym =
   result = realModule
+  c.unusedImports.add((realModule, n.info))
   if n.kind != nkImportAs: discard
-  elif n.len != 2 or n.sons[1].kind != nkIdent:
+  elif n.len != 2 or n[1].kind != nkIdent:
     localError(c.config, n.info, "module alias must be an identifier")
-  elif n.sons[1].ident.id != realModule.name.id:
+  elif n[1].ident.id != realModule.name.id:
     # some misguided guy will write 'import abc.foo as foo' ...
-    result = createModuleAlias(realModule, n.sons[1].ident, realModule.info,
+    result = createModuleAlias(realModule, n[1].ident, realModule.info,
                                c.config.options)
 
 proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
@@ -146,12 +148,16 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
     #echo "adding ", toFullPath(f), " at ", L+1
     if recursion >= 0:
       var err = ""
-      for i in recursion ..< L:
+      for i in recursion..<L:
         if i > recursion: err.add "\n"
         err.add toFullPath(c.config, c.graph.importStack[i]) & " imports " &
                 toFullPath(c.config, c.graph.importStack[i+1])
       c.recursiveDep = err
+
+    discard pushOptionEntry(c)
     result = importModuleAs(c, n, c.graph.importModuleCallback(c.graph, c.module, f))
+    popOptionEntry(c)
+
     #echo "set back to ", L
     c.graph.importStack.setLen(L)
     # we cannot perform this check reliably because of
@@ -172,8 +178,8 @@ proc myImportModule(c: PContext, n: PNode; importStmtResult: PNode): PSym =
 proc transformImportAs(c: PContext; n: PNode): PNode =
   if n.kind == nkInfix and considerQuotedIdent(c, n[0]).s == "as":
     result = newNodeI(nkImportAs, n.info)
-    result.add n.sons[1]
-    result.add n.sons[2]
+    result.add n[1]
+    result.add n[2]
   else:
     result = n
 
@@ -181,16 +187,15 @@ proc impMod(c: PContext; it: PNode; importStmtResult: PNode) =
   let it = transformImportAs(c, it)
   let m = myImportModule(c, it, importStmtResult)
   if m != nil:
-    var emptySet: IntSet
     # ``addDecl`` needs to be done before ``importAllSymbols``!
     addDecl(c, m, it.info) # add symbol to symbol table of module
-    importAllSymbolsExcept(c, m, emptySet)
-    #importForwarded(c, m.ast, emptySet)
+    importAllSymbols(c, m)
+    #importForwarded(c, m.ast, emptySet, m)
 
 proc evalImport*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
-  for i in 0 ..< sonsLen(n):
-    let it = n.sons[i]
+  for i in 0..<n.len:
+    let it = n[i]
     if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
       let sep = it[0]
       let dir = it[1]
@@ -200,13 +205,13 @@ proc evalImport*(c: PContext, n: PNode): PNode =
       imp.add sep # dummy entry, replaced in the loop
       for x in it[2]:
         # transform `a/b/[c as d]` to `/a/b/c as d`
-        if x.kind == nkInfix and x.sons[0].ident.s == "as":
+        if x.kind == nkInfix and x[0].ident.s == "as":
           let impAs = copyTree(x)
-          imp.sons[2] = x.sons[1]
-          impAs.sons[1] = imp
+          imp[2] = x[1]
+          impAs[1] = imp
           impMod(c, imp, result)
         else:
-          imp.sons[2] = x
+          imp[2] = x
           impMod(c, imp, result)
     else:
       impMod(c, it, result)
@@ -214,22 +219,22 @@ proc evalImport*(c: PContext, n: PNode): PNode =
 proc evalFrom*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n.sons[0] = transformImportAs(c, n.sons[0])
-  var m = myImportModule(c, n.sons[0], result)
+  n[0] = transformImportAs(c, n[0])
+  var m = myImportModule(c, n[0], result)
   if m != nil:
-    n.sons[0] = newSymNode(m)
+    n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
-    for i in 1 ..< sonsLen(n):
-      if n.sons[i].kind != nkNilLit:
-        importSymbol(c, n.sons[i], m)
+    for i in 1..<n.len:
+      if n[i].kind != nkNilLit:
+        importSymbol(c, n[i], m)
 
 proc evalImportExcept*(c: PContext, n: PNode): PNode =
   result = newNodeI(nkImportStmt, n.info)
   checkMinSonsLen(n, 2, c.config)
-  n.sons[0] = transformImportAs(c, n.sons[0])
-  var m = myImportModule(c, n.sons[0], result)
+  n[0] = transformImportAs(c, n[0])
+  var m = myImportModule(c, n[0], result)
   if m != nil:
-    n.sons[0] = newSymNode(m)
+    n[0] = newSymNode(m)
     addDecl(c, m, n.info)               # add symbol to symbol table of module
     importAllSymbolsExcept(c, m, readExceptSet(c, n))
-    #importForwarded(c, m.ast, exceptSet)
+    #importForwarded(c, m.ast, exceptSet, m)

compiler/incremental.nim

@@ -10,7 +10,7 @@
 ## Basic type definitions the module graph needs in order to support
 ## incremental compilations.
 
-const nimIncremental* = true # defined(nimIncremental)
+const nimIncremental* = defined(nimIncremental)
 
 import options, lineinfos
 

compiler/index.nim

@@ -0,0 +1,18 @@
+##[
+This module only exists to generate docs for the compiler.
+
+## links
+* [main docs](../lib.html)
+* [compiler user guide](../nimc.html)
+* [Internals of the Nim Compiler](../intern.html)
+]##
+
+#[
+note: this is named `index` so that navigating to https://nim-lang.github.io/Nim/compiler/
+will work.
+
+xxx this should also import other modules, not transitively imported by `compiler/nim.nim`,
+eg `evalffi`, otherwise these aren't shown. A glob could be used at CT.
+]#
+
+import nim

compiler/installer.ini

@@ -6,12 +6,12 @@ Name: "Nim"
 Version: "$version"
 Platforms: """
   windows: i386;amd64
-  linux: i386;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv64
-  macosx: i386;amd64;powerpc64
+  linux: i386;hppa;ia64;alpha;amd64;powerpc64;arm;sparc;sparc64;m68k;mips;mipsel;mips64;mips64el;powerpc;powerpc64el;arm64;riscv32;riscv64
+  macosx: i386;amd64;powerpc64;arm64
   solaris: i386;amd64;sparc;sparc64
-  freebsd: i386;amd64
+  freebsd: i386;amd64;powerpc64;arm;arm64;riscv64;sparc64;mips;mipsel;mips64;mips64el;powerpc;powerpc64el
   netbsd: i386;amd64
-  openbsd: i386;amd64
+  openbsd: i386;amd64;arm;arm64
   dragonfly: i386;amd64
   haiku: i386;amd64
   android: i386;arm;arm64
@@ -35,9 +35,8 @@ App: Console
 License: "copying.txt"
 
 [Config]
-Files: "config/nim.cfg"
-Files: "config/nimdoc.cfg"
-Files: "config/nimdoc.tex.cfg"
+Files: "config/*.cfg"
+Files: "config/config.nims"
 
 [Documentation]
 ; Files: "doc/*.html"
@@ -65,6 +64,7 @@ Files: "compiler"
 Files: "doc"
 Files: "doc/html"
 Files: "tools"
+Files: "tools/nim-gdb.py"
 Files: "nimpretty"
 Files: "testament"
 Files: "nimsuggest"
@@ -76,6 +76,7 @@ Files: "lib"
 [Other]
 Files: "examples"
 Files: "dist/nimble"
+Files: "dist/fusion"
 
 Files: "tests"
 
@@ -87,6 +88,8 @@ Files: "bin/nimble.exe"
 Files: "bin/vccexe.exe"
 Files: "bin/nimgrab.exe"
 Files: "bin/nimpretty.exe"
+Files: "bin/testament.exe"
+Files: "bin/nim-gdb.bat"
 
 Files: "koch.exe"
 Files: "finish.exe"
@@ -107,6 +110,7 @@ Download: r"Aporia Text Editor|dist|aporia.zip|97997|https://nim-lang.org/downlo
 Files: "bin/makelink.exe"
 Files: "bin/7zG.exe"
 Files: "bin/*.dll"
+Files: "bin/cacert.pem"
 
 [UnixBin]
 Files: "bin/nim"
@@ -115,6 +119,8 @@ Files: "bin/nim"
 [Unix]
 InstallScript: "yes"
 UninstallScript: "yes"
+Files: "bin/nim-gdb"
+Files: "bin/nim-gdb.bash"
 
 
 [InnoSetup]
@@ -137,4 +143,4 @@ licenses: "bin/nim,MIT;lib/*,MIT;"
 
 [nimble]
 pkgName: "compiler"
-pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt"
+pkgFiles: "compiler/*;doc/basicopt.txt;doc/advopt.txt;doc/nimdoc.css"

compiler/int128.nim

@@ -1,4 +1,9 @@
+## This module is for compiler internal use only. For reliable error
+## messages and range checks, the compiler needs a data type that can
+## hold all from ``low(BiggestInt)`` to ``high(BiggestUInt)``, This
+## type is for that purpose.
 
+from math import trunc
 
 type
   Int128* = object
@@ -11,46 +16,40 @@ template sdata(arg: Int128, idx: int): int32 =
 
 # encoding least significant int first (like LittleEndian)
 
-type
-  InvalidArgument = object of Exception
-
-template require(cond: bool) =
-  if unlikely(not cond):
-    raise newException(InvalidArgument, "")
-
 const
   Zero* = Int128(udata: [0'u32,0,0,0])
   One* = Int128(udata: [1'u32,0,0,0])
   Ten* = Int128(udata: [10'u32,0,0,0])
   Min = Int128(udata: [0'u32,0,0,0x80000000'u32])
   Max = Int128(udata: [high(uint32),high(uint32),high(uint32),uint32(high(int32))])
+  NegOne* = Int128(udata: [0xffffffff'u32,0xffffffff'u32,0xffffffff'u32,0xffffffff'u32])
 
 template low*(t: typedesc[Int128]): Int128 = Min
 template high*(t: typedesc[Int128]): Int128 = Max
 
 proc `$`*(a: Int128): string
 
-proc toInt128*[T: SomeInteger](arg: T): Int128 =
-  when T is SomeUnsignedInt:
+proc toInt128*[T: SomeInteger | bool](arg: T): Int128 =
+  when T is bool: result.sdata(0) = int32(arg)
+  elif T is SomeUnsignedInt:
     when sizeof(arg) <= 4:
       result.udata[0] = uint32(arg)
     else:
       result.udata[0] = uint32(arg and T(0xffffffff))
       result.udata[1] = uint32(arg shr 32)
+  elif sizeof(arg) <= 4:
+    result.sdata(0) = int32(arg)
+    if arg < 0: # sign extend
+      result.sdata(1) = -1
+      result.sdata(2) = -1
+      result.sdata(3) = -1
   else:
-    when sizeof(arg) <= 4:
-      result.sdata(0) = int32(arg)
-      if arg < 0: # sign extend
-        result.sdata(1) = -1
-        result.sdata(2) = -1
-        result.sdata(3) = -1
-    else:
-      let tmp = int64(arg)
-      result.udata[0] = uint32(tmp and 0xffffffff)
-      result.sdata(1) = int32(tmp shr 32)
-      if arg < 0: # sign extend
-        result.sdata(2) = -1
-        result.sdata(3) = -1
+    let tmp = int64(arg)
+    result.udata[0] = uint32(tmp and 0xffffffff)
+    result.sdata(1) = int32(tmp shr 32)
+    if arg < 0: # sign extend
+      result.sdata(2) = -1
+      result.sdata(3) = -1
 
 template isNegative(arg: Int128): bool =
   arg.sdata(3) < 0
@@ -74,13 +73,96 @@ proc toInt64*(arg: Int128): int64 =
 
   cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
 
+proc toInt64Checked*(arg: Int128; onError: int64): int64 =
+  if isNegative(arg):
+    if arg.sdata(3) != -1 or arg.sdata(2) != -1:
+      return onError
+  else:
+    if arg.sdata(3) != 0 or arg.sdata(2) != 0:
+      return onError
+  return cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
+
+proc toInt32*(arg: Int128): int32 =
+  if isNegative(arg):
+    assert(arg.sdata(3) == -1, "out of range")
+    assert(arg.sdata(2) == -1, "out of range")
+    assert(arg.sdata(1) == -1, "out of range")
+  else:
+    assert(arg.sdata(3) == 0, "out of range")
+    assert(arg.sdata(2) == 0, "out of range")
+    assert(arg.sdata(1) == 0, "out of range")
+
+  arg.sdata(0)
+
+proc toInt16*(arg: Int128): int16 =
+  if isNegative(arg):
+    assert(arg.sdata(3) == -1, "out of range")
+    assert(arg.sdata(2) == -1, "out of range")
+    assert(arg.sdata(1) == -1, "out of range")
+  else:
+    assert(arg.sdata(3) == 0, "out of range")
+    assert(arg.sdata(2) == 0, "out of range")
+    assert(arg.sdata(1) == 0, "out of range")
+
+  int16(arg.sdata(0))
+
+proc toInt8*(arg: Int128): int8 =
+  if isNegative(arg):
+    assert(arg.sdata(3) == -1, "out of range")
+    assert(arg.sdata(2) == -1, "out of range")
+    assert(arg.sdata(1) == -1, "out of range")
+  else:
+    assert(arg.sdata(3) == 0, "out of range")
+    assert(arg.sdata(2) == 0, "out of range")
+    assert(arg.sdata(1) == 0, "out of range")
+
+  int8(arg.sdata(0))
+
+proc toInt*(arg: Int128): int =
+  when sizeof(int) == 4:
+    cast[int](toInt32(arg))
+  else:
+    cast[int](toInt64(arg))
+
 proc toUInt64*(arg: Int128): uint64 =
   assert(arg.udata[3] == 0)
   assert(arg.udata[2] == 0)
   bitconcat(arg.udata[1], arg.udata[0])
 
+proc toUInt32*(arg: Int128): uint32 =
+  assert(arg.udata[3] == 0)
+  assert(arg.udata[2] == 0)
+  assert(arg.udata[1] == 0)
+  arg.udata[0]
+
+proc toUInt16*(arg: Int128): uint16 =
+  assert(arg.udata[3] == 0)
+  assert(arg.udata[2] == 0)
+  assert(arg.udata[1] == 0)
+  uint16(arg.udata[0])
+
+proc toUInt8*(arg: Int128): uint8 =
+  assert(arg.udata[3] == 0)
+  assert(arg.udata[2] == 0)
+  assert(arg.udata[1] == 0)
+  uint8(arg.udata[0])
+
+proc toUInt*(arg: Int128): uint =
+  when sizeof(int) == 4:
+    cast[uint](toInt32(arg))
+  else:
+    cast[uint](toInt64(arg))
+
+proc castToInt64*(arg: Int128): int64 =
+  ## Conversion to int64 without range check.
+  cast[int64](bitconcat(arg.udata[1], arg.udata[0]))
+
+proc castToUInt64*(arg: Int128): uint64 =
+  ## Conversion to uint64 without range check.
+  cast[uint64](bitconcat(arg.udata[1], arg.udata[0]))
+
 proc addToHex(result: var string; arg: uint32) =
-  for i in 0 ..< 8:
+  for i in 0..<8:
     let idx = (arg shr ((7-i) * 4)) and 0xf
     result.add "0123456789abcdef"[idx]
 
@@ -168,14 +250,14 @@ proc `shr`*(a: Int128, b: int): Int128 =
     if isNegative(a):
       result.sdata(3) = -1
     result.sdata(2) = a.sdata(3) shr (b and 31)
-    result.udata[1] = cast[uint32](bitconcat(a.udata[2], a.udata[1]) shr (b and 31))
-    result.udata[0] = cast[uint32](bitconcat(a.udata[1], a.udata[0]) shr (b and 31))
+    result.udata[1] = cast[uint32](bitconcat(a.udata[3], a.udata[2]) shr (b and 31))
+    result.udata[0] = cast[uint32](bitconcat(a.udata[2], a.udata[1]) shr (b and 31))
   elif b < 96:
     if isNegative(a):
       result.sdata(3) = -1
       result.sdata(2) = -1
     result.sdata(1) = a.sdata(3) shr (b and 31)
-    result.udata[0] = cast[uint32](bitconcat(a.udata[1], a.udata[0]) shr (b and 31))
+    result.udata[0] = cast[uint32](bitconcat(a.udata[3], a.udata[2]) shr (b and 31))
   else: # b < 128
     if isNegative(a):
       result.sdata(3) = -1
@@ -206,7 +288,6 @@ proc `shl`*(a: Int128, b: int): Int128 =
     result.udata[2] = 0
     result.udata[3] = a.udata[0] shl (b and 31)
 
-
 proc `+`*(a,b: Int128): Int128 =
   let tmp0 = uint64(a.udata[0]) + uint64(b.udata[0])
   result.udata[0] = cast[uint32](tmp0)
@@ -254,7 +335,6 @@ proc `*`(a: Int128, b: uint32): Int128 =
   result.udata[3] = cast[uint32](tmp3) + cast[uint32](tmp2 shr 32)
 
 proc `*`*(a: Int128, b: int32): Int128 =
-  let isNegative = isNegative(a) xor isNegative(b)
   result = a * cast[uint32](abs(b))
   if b < 0:
     result = -result
@@ -275,8 +355,6 @@ proc low64(a: Int128): uint64 =
   bitconcat(a.udata[1], a.udata[0])
 
 proc `*`*(lhs,rhs: Int128): Int128 =
-  let isNegative = isNegative(lhs) xor isNegative(rhs)
-
   let
     a = cast[uint64](lhs.udata[0])
     b = cast[uint64](lhs.udata[1])
@@ -295,12 +373,8 @@ proc `*`*(lhs,rhs: Int128): Int128 =
   let b00 = cast[uint64](rhs.udata[0])
 
   result = makeInt128(high64(lhs) * low64(rhs) + low64(lhs) * high64(rhs) + a32 * b32, a00 * b00)
-  result = result + toInt128(a32 * b00) shl 32
-  result = result + toInt128(a00 * b32) shl 32
-
-  if isNegative != isNegative(result):
-    echo result
-    assert(false, "overflow")
+  result += toInt128(a32 * b00) shl 32
+  result += toInt128(a00 * b32) shl 32
 
 proc `*=`*(a: var Int128, b: Int128) =
   a = a * b
@@ -319,18 +393,25 @@ proc fastLog2*(a: Int128): int =
 
 proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
   assert(divisor != Zero)
-  let isNegative = isNegative(dividend) xor isNegative(divisor)
+  let isNegativeA = isNegative(dividend)
+  let isNegativeB = isNegative(divisor)
 
   var dividend = abs(dividend)
   let divisor = abs(divisor)
 
   if divisor > dividend:
     result.quotient = Zero
-    result.remainder = dividend
+    if isNegativeA:
+      result.remainder = -dividend
+    else:
+      result.remainder = dividend
     return
 
   if divisor == dividend:
-    result.quotient = One
+    if isNegativeA xor isNegativeB:
+      result.quotient = NegOne
+    else:
+      result.quotient = One
     result.remainder = Zero
     return
 
@@ -343,16 +424,22 @@ proc divMod*(dividend, divisor: Int128): tuple[quotient, remainder: Int128] =
 
   # Uses shift-subtract algorithm to divide dividend by denominator. The
   # remainder will be left in dividend.
-  for i in 0 .. shift:
+  for i in 0..shift:
     quotient = quotient shl 1
     if dividend >= denominator:
-      dividend = dividend - denominator
+      dividend -= denominator
       quotient = bitor(quotient, One)
 
     denominator = denominator shr 1
 
-  result.quotient = quotient
-  result.remainder = dividend
+  if isNegativeA xor isNegativeB:
+    result.quotient = -quotient
+  else:
+    result.quotient = quotient
+  if isNegativeA:
+    result.remainder = -dividend
+  else:
+    result.remainder = dividend
 
 proc `div`*(a,b: Int128): Int128 =
   let (a,b) = divMod(a,b)
@@ -362,28 +449,32 @@ proc `mod`*(a,b: Int128): Int128 =
   let (a,b) = divMod(a,b)
   return b
 
-proc `$`*(a: Int128): string =
-  if a == Zero:
-    result = "0"
-  elif a == low(Int128):
-    result = "-170141183460469231731687303715884105728"
+proc addInt128*(result: var string; value: Int128) =
+  let initialSize = result.len
+  if value == Zero:
+    result.add "0"
+  elif value == low(Int128):
+    result.add "-170141183460469231731687303715884105728"
   else:
-    let isNegative = isNegative(a)
-    var a = abs(a)
-    while a > Zero:
-      let (quot, rem) = divMod(a, Ten)
+    let isNegative = isNegative(value)
+    var value = abs(value)
+    while value > Zero:
+      let (quot, rem) = divMod(value, Ten)
       result.add "0123456789"[rem.toInt64]
-      a = quot
+      value = quot
     if isNegative:
       result.add '-'
 
-    var i = 0
+    var i = initialSize
     var j = high(result)
     while i < j:
       swap(result[i], result[j])
       i += 1
       j -= 1
 
+proc `$`*(a: Int128): string =
+  result.addInt128(a)
+
 proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =
   assert(pos < arg.len)
   assert(arg[pos] in {'-','0'..'9'})
@@ -395,7 +486,7 @@ proc parseDecimalInt128*(arg: string, pos: int = 0): Int128 =
     pos += 1
 
   result = Zero
-  while pos < arg.len and arg[pos] in '0' .. '9':
+  while pos < arg.len and arg[pos] in '0'..'9':
     result = result * Ten
     result.inc(uint32(arg[pos]) - uint32('0'))
     pos += 1
@@ -435,82 +526,69 @@ proc `+`*(a: BiggestInt, b: Int128): Int128 =
 proc `+`*(a: Int128, b: BiggestInt): Int128 =
   a + toInt128(b)
 
+proc toFloat64*(arg: Int128): float64 =
+  let isNegative = isNegative(arg)
+  let arg = abs(arg)
 
-when isMainModule:
-  let (a,b) = divMod(Ten,Ten)
+  let a = float64(bitconcat(arg.udata[1], arg.udata[0]))
+  let b = float64(bitconcat(arg.udata[3], arg.udata[2]))
 
-  doAssert $One == "1"
-  doAssert $Ten == "10"
-  doAssert $Zero == "0"
-  let c = parseDecimalInt128("12345678989876543210123456789")
-  doAssert $c == "12345678989876543210123456789"
+  result = a + 18446744073709551616'f64 * b # a + 2^64 * b
+  if isNegative:
+    result = -result
 
-  var d : array[39, Int128]
-  d[0] =  parseDecimalInt128("1")
-  d[1] =  parseDecimalInt128("10")
-  d[2] =  parseDecimalInt128("100")
-  d[3] =  parseDecimalInt128("1000")
-  d[4] =  parseDecimalInt128("10000")
-  d[5] =  parseDecimalInt128("100000")
-  d[6] =  parseDecimalInt128("1000000")
-  d[7] =  parseDecimalInt128("10000000")
-  d[8] =  parseDecimalInt128("100000000")
-  d[9] =  parseDecimalInt128("1000000000")
-  d[10] = parseDecimalInt128("10000000000")
-  d[11] = parseDecimalInt128("100000000000")
-  d[12] = parseDecimalInt128("1000000000000")
-  d[13] = parseDecimalInt128("10000000000000")
-  d[14] = parseDecimalInt128("100000000000000")
-  d[15] = parseDecimalInt128("1000000000000000")
-  d[16] = parseDecimalInt128("10000000000000000")
-  d[17] = parseDecimalInt128("100000000000000000")
-  d[18] = parseDecimalInt128("1000000000000000000")
-  d[19] = parseDecimalInt128("10000000000000000000")
-  d[20] = parseDecimalInt128("100000000000000000000")
-  d[21] = parseDecimalInt128("1000000000000000000000")
-  d[22] = parseDecimalInt128("10000000000000000000000")
-  d[23] = parseDecimalInt128("100000000000000000000000")
-  d[24] = parseDecimalInt128("1000000000000000000000000")
-  d[25] = parseDecimalInt128("10000000000000000000000000")
-  d[26] = parseDecimalInt128("100000000000000000000000000")
-  d[27] = parseDecimalInt128("1000000000000000000000000000")
-  d[28] = parseDecimalInt128("10000000000000000000000000000")
-  d[29] = parseDecimalInt128("100000000000000000000000000000")
-  d[30] = parseDecimalInt128("1000000000000000000000000000000")
-  d[31] = parseDecimalInt128("10000000000000000000000000000000")
-  d[32] = parseDecimalInt128("100000000000000000000000000000000")
-  d[33] = parseDecimalInt128("1000000000000000000000000000000000")
-  d[34] = parseDecimalInt128("10000000000000000000000000000000000")
-  d[35] = parseDecimalInt128("100000000000000000000000000000000000")
-  d[36] = parseDecimalInt128("1000000000000000000000000000000000000")
-  d[37] = parseDecimalInt128("10000000000000000000000000000000000000")
-  d[38] = parseDecimalInt128("100000000000000000000000000000000000000")
+proc ldexp(x: float64, exp: cint): float64 {.importc: "ldexp", header: "<math.h>".}
 
-  for i in 0 ..< d.len:
-    for j in 0 ..< d.len:
-      doAssert(cmp(d[i], d[j]) == cmp(i,j))
-      if i + j < d.len:
-        doAssert d[i] * d[j] == d[i+j]
-      if i - j >= 0:
-        doAssert d[i] div d[j] == d[i-j]
+template bitor(a,b,c: Int128): Int128 = bitor(bitor(a,b), c)
 
-  var sum: Int128
+proc toInt128*(arg: float64): Int128 =
+  let isNegative = arg < 0
+  let v0 = ldexp(abs(arg), -100)
+  let w0 = uint64(trunc(v0))
+  let v1 = ldexp(v0 - float64(w0), 50)
+  let w1 = uint64(trunc(v1))
+  let v2 = ldexp(v1 - float64(w1), 50)
+  let w2 = uint64(trunc(v2))
 
-  for it in d:
-    sum += it
+  let res = bitor(toInt128(w0) shl 100, toInt128(w1) shl 50, toInt128(w2))
+  if isNegative:
+    return -res
+  else:
+    return res
 
-  doAssert $sum == "111111111111111111111111111111111111111"
+proc maskUInt64*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0]
+  result.udata[1] = arg.udata[1]
+  result.udata[2] = 0
+  result.udata[3] = 0
 
-  for it in d.mitems:
-    it = -it
+proc maskUInt32*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0]
+  result.udata[1] = 0
+  result.udata[2] = 0
+  result.udata[3] = 0
 
-  for i in 0 ..< d.len:
-    for j in 0 ..< d.len:
-      doAssert(cmp(d[i], d[j]) == -cmp(i,j))
-      if i + j < d.len:
-        doAssert d[i] * d[j] == -d[i+j]
-      if i - j >= 0:
-        doAssert d[i] div d[j] == -d[i-j]
+proc maskUInt16*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0] and 0xffff
+  result.udata[1] = 0
+  result.udata[2] = 0
+  result.udata[3] = 0
 
-  doAssert $high(Int128) == "170141183460469231731687303715884105727"
-  doAssert $low(Int128) == "-170141183460469231731687303715884105728"
+proc maskUInt8*(arg: Int128): Int128 {.noinit, inline.} =
+  result.udata[0] = arg.udata[0] and 0xff
+  result.udata[1] = 0
+  result.udata[2] = 0
+  result.udata[3] = 0
+
+proc maskBytes*(arg: Int128, numbytes: int): Int128 {.noinit.} =
+  case numbytes
+  of 1:
+    return maskUInt8(arg)
+  of 2:
+    return maskUInt16(arg)
+  of 4:
+    return maskUInt32(arg)
+  of 8:
+    return maskUInt64(arg)
+  else:
+    assert(false, "masking only implemented for 1, 2, 4 and 8 bytes")

compiler/isolation_check.nim

@@ -0,0 +1,126 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2020 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Implementation of the check that `recover` needs, see
+## https://github.com/nim-lang/RFCs/issues/244 for more details.
+
+import
+  ast, types, renderer, idents, intsets
+
+proc canAlias(arg, ret: PType; marker: var IntSet): bool
+
+proc canAliasN(arg: PType; n: PNode; marker: var IntSet): bool =
+  case n.kind
+  of nkRecList:
+    for i in 0..<n.len:
+      result = canAliasN(arg, n[i], marker)
+      if result: return
+  of nkRecCase:
+    assert(n[0].kind == nkSym)
+    result = canAliasN(arg, n[0], marker)
+    if result: return
+    for i in 1..<n.len:
+      case n[i].kind
+      of nkOfBranch, nkElse:
+        result = canAliasN(arg, lastSon(n[i]), marker)
+        if result: return
+      else: discard
+  of nkSym:
+    result = canAlias(arg, n.sym.typ, marker)
+  else: discard
+
+proc canAlias(arg, ret: PType; marker: var IntSet): bool =
+  if containsOrIncl(marker, ret.id):
+    return false
+
+  if ret.kind in {tyPtr, tyPointer}:
+    # unsafe so we don't care:
+    return false
+  if compareTypes(arg, ret, dcEqIgnoreDistinct):
+    return true
+  case ret.kind
+  of tyObject:
+    if isFinal(ret):
+      result = canAliasN(arg, ret.n, marker)
+      if not result and ret.len > 0 and ret[0] != nil:
+        result = canAlias(arg, ret[0], marker)
+    else:
+      result = true
+  of tyTuple:
+    for i in 0..<ret.len:
+      result = canAlias(arg, ret[i], marker)
+      if result: break
+  of tyArray, tySequence, tyDistinct, tyGenericInst,
+     tyAlias, tyInferred, tySink, tyLent, tyOwned, tyRef:
+    result = canAlias(arg, ret.lastSon, marker)
+  of tyProc:
+    result = ret.callConv == ccClosure
+  else:
+    result = false
+
+proc isValueOnlyType(t: PType): bool = 
+  # t doesn't contain pointers and references
+  proc wrap(t: PType): bool {.nimcall.} = t.kind in {tyRef, tyPtr, tyVar, tyLent}
+  result = not types.searchTypeFor(t, wrap)
+
+proc canAlias*(arg, ret: PType): bool =
+  if isValueOnlyType(arg):
+    # can alias only with unsafeAddr(arg.x) and we don't care if it is not safe
+    result = false
+  else:
+    var marker = initIntSet()
+    result = canAlias(arg, ret, marker)
+
+proc checkIsolate*(n: PNode): bool =
+  if types.containsTyRef(n.typ):
+    # XXX Maybe require that 'n.typ' is acyclic. This is not much
+    # worse than the already exisiting inheritance and closure restrictions.
+    case n.kind
+    of nkCharLit..nkNilLit:
+      result = true
+    of nkCallKinds:
+      if n[0].typ.flags * {tfGcSafe, tfNoSideEffect} == {}:
+        return false
+      for i in 1..<n.len:
+        if checkIsolate(n[i]):
+          discard "fine, it is isolated already"
+        else:
+          let argType = n[i].typ
+          if argType != nil and not isCompileTimeOnly(argType) and containsTyRef(argType):
+            if argType.canAlias(n.typ):
+              return false
+      result = true
+    of nkIfStmt, nkIfExpr:
+      for it in n:
+        result = checkIsolate(it.lastSon)
+        if not result: break
+    of nkCaseStmt, nkObjConstr:
+      for i in 1..<n.len:
+        result = checkIsolate(n[i].lastSon)
+        if not result: break
+    of nkBracket, nkTupleConstr, nkPar:
+      for it in n:
+        result = checkIsolate(it)
+        if not result: break
+    of nkHiddenStdConv, nkHiddenSubConv, nkCast, nkConv:
+      result = checkIsolate(n[1])
+    of nkObjUpConv, nkObjDownConv, nkDotExpr:
+      result = checkIsolate(n[0])
+    of nkStmtList, nkStmtListExpr:
+      if n.len > 0:
+        result = checkIsolate(n[^1])
+      else:
+        result = false
+    else:
+      # unanalysable expression:
+      result = false
+  else:
+    # no ref, no cry:
+    result = true
+

compiler/jstypes.nim

@@ -7,28 +7,30 @@
 #    distribution, for details about the copyright.
 #
 
+# included from jsgen.nim
+
 ## Type info generation for the JS backend.
 
+proc rope(arg: Int128): Rope = rope($arg)
+
 proc genTypeInfo(p: PProc, typ: PType): Rope
 proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
   var
     s, u: Rope
-    length: int
     field: PSym
     b: PNode
   result = nil
   case n.kind
   of nkRecList:
-    length = sonsLen(n)
-    if length == 1:
-      result = genObjectFields(p, typ, n.sons[0])
+    if n.len == 1:
+      result = genObjectFields(p, typ, n[0])
     else:
       s = nil
-      for i in 0 ..< length:
-        if i > 0: add(s, ", \L")
-        add(s, genObjectFields(p, typ, n.sons[i]))
+      for i in 0..<n.len:
+        if i > 0: s.add(", \L")
+        s.add(genObjectFields(p, typ, n[i]))
       result = ("{kind: 2, len: $1, offset: 0, " &
-          "typ: null, name: null, sons: [$2]}") % [rope(length), s]
+          "typ: null, name: null, sons: [$2]}") % [rope(n.len), s]
   of nkSym:
     field = n.sym
     s = genTypeInfo(p, field.typ)
@@ -37,29 +39,28 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
                    [mangleName(p.module, field), s,
                     makeJSString(field.name.s)]
   of nkRecCase:
-    length = sonsLen(n)
-    if (n.sons[0].kind != nkSym): internalError(p.config, n.info, "genObjectFields")
-    field = n.sons[0].sym
+    if (n[0].kind != nkSym): internalError(p.config, n.info, "genObjectFields")
+    field = n[0].sym
     s = genTypeInfo(p, field.typ)
-    for i in 1 ..< length:
-      b = n.sons[i]           # branch
+    for i in 1..<n.len:
+      b = n[i]           # branch
       u = nil
       case b.kind
       of nkOfBranch:
-        if sonsLen(b) < 2:
+        if b.len < 2:
           internalError(p.config, b.info, "genObjectFields; nkOfBranch broken")
-        for j in 0 .. sonsLen(b) - 2:
-          if u != nil: add(u, ", ")
-          if b.sons[j].kind == nkRange:
-            addf(u, "[$1, $2]", [rope(getOrdValue(b.sons[j].sons[0])),
-                                 rope(getOrdValue(b.sons[j].sons[1]))])
+        for j in 0..<b.len - 1:
+          if u != nil: u.add(", ")
+          if b[j].kind == nkRange:
+            u.addf("[$1, $2]", [rope(getOrdValue(b[j][0])),
+                                 rope(getOrdValue(b[j][1]))])
           else:
-            add(u, rope(getOrdValue(b.sons[j])))
+            u.add(rope(getOrdValue(b[j])))
       of nkElse:
         u = rope(lengthOrd(p.config, field.typ))
       else: internalError(p.config, n.info, "genObjectFields(nkRecCase)")
-      if result != nil: add(result, ", \L")
-      addf(result, "[setConstr($1), $2]",
+      if result != nil: result.add(", \L")
+      result.addf("[setConstr($1), $2]",
            [u, genObjectFields(p, typ, lastSon(b))])
     result = ("{kind: 3, offset: \"$1\", len: $3, " &
         "typ: $2, name: $4, sons: [$5]}") % [
@@ -68,27 +69,27 @@ proc genObjectFields(p: PProc, typ: PType, n: PNode): Rope =
   else: internalError(p.config, n.info, "genObjectFields")
 
 proc objHasTypeField(t: PType): bool {.inline.} =
-  tfInheritable in t.flags or t.sons[0] != nil
+  tfInheritable in t.flags or t[0] != nil
 
 proc genObjectInfo(p: PProc, typ: PType, name: Rope) =
   let kind = if objHasTypeField(typ): tyObject else: tyTuple
   var s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
            "finalizer: null};$n") % [name, rope(ord(kind))]
   prepend(p.g.typeInfo, s)
-  addf(p.g.typeInfo, "var NNI$1 = $2;$n",
+  p.g.typeInfo.addf("var NNI$1 = $2;$n",
        [rope(typ.id), genObjectFields(p, typ, typ.n)])
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
-  if (typ.kind == tyObject) and (typ.sons[0] != nil):
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [name, genTypeInfo(p, typ.sons[0].skipTypes(skipPtrs))])
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
+  if (typ.kind == tyObject) and (typ[0] != nil):
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [name, genTypeInfo(p, typ[0].skipTypes(skipPtrs))])
 
 proc genTupleFields(p: PProc, typ: PType): Rope =
   var s: Rope = nil
-  for i in 0 ..< typ.len:
-    if i > 0: add(s, ", \L")
+  for i in 0..<typ.len:
+    if i > 0: s.add(", \L")
     s.addf("{kind: 1, offset: \"Field$1\", len: 0, " &
            "typ: $2, name: \"Field$1\", sons: null}",
-           [i.rope, genTypeInfo(p, typ.sons[i])])
+           [i.rope, genTypeInfo(p, typ[i])])
   result = ("{kind: 2, len: $1, offset: 0, " &
             "typ: null, name: null, sons: [$2]}") % [rope(typ.len), s]
 
@@ -96,30 +97,29 @@ proc genTupleInfo(p: PProc, typ: PType, name: Rope) =
   var s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
            "finalizer: null};$n") % [name, rope(ord(typ.kind))]
   prepend(p.g.typeInfo, s)
-  addf(p.g.typeInfo, "var NNI$1 = $2;$n",
+  p.g.typeInfo.addf("var NNI$1 = $2;$n",
        [rope(typ.id), genTupleFields(p, typ)])
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
 
 proc genEnumInfo(p: PProc, typ: PType, name: Rope) =
-  let length = sonsLen(typ.n)
   var s: Rope = nil
-  for i in 0 ..< length:
-    if (typ.n.sons[i].kind != nkSym): internalError(p.config, typ.n.info, "genEnumInfo")
-    let field = typ.n.sons[i].sym
-    if i > 0: add(s, ", \L")
+  for i in 0..<typ.n.len:
+    if (typ.n[i].kind != nkSym): internalError(p.config, typ.n.info, "genEnumInfo")
+    let field = typ.n[i].sym
+    if i > 0: s.add(", \L")
     let extName = if field.ast == nil: field.name.s else: field.ast.strVal
-    addf(s, "\"$1\": {kind: 1, offset: $1, typ: $2, name: $3, len: 0, sons: null}",
+    s.addf("\"$1\": {kind: 1, offset: $1, typ: $2, name: $3, len: 0, sons: null}",
          [rope(field.position), name, makeJSString(extName)])
   var n = ("var NNI$1 = {kind: 2, offset: 0, typ: null, " &
-      "name: null, len: $2, sons: {$3}};$n") % [rope(typ.id), rope(length), s]
+      "name: null, len: $2, sons: {$3}};$n") % [rope(typ.id), rope(typ.n.len), s]
   s = ("var $1 = {size: 0, kind: $2, base: null, node: null, " &
        "finalizer: null};$n") % [name, rope(ord(typ.kind))]
   prepend(p.g.typeInfo, s)
-  add(p.g.typeInfo, n)
-  addf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, rope(typ.id)])
-  if typ.sons[0] != nil:
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [name, genTypeInfo(p, typ.sons[0])])
+  p.g.typeInfo.add(n)
+  p.g.typeInfo.addf("$1.node = NNI$2;$n", [name, rope(typ.id)])
+  if typ[0] != nil:
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [name, genTypeInfo(p, typ[0])])
 
 proc genTypeInfo(p: PProc, typ: PType): Rope =
   let t = typ.skipTypes({tyGenericInst, tyDistinct, tyAlias, tySink, tyOwned})
@@ -127,7 +127,7 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
   if containsOrIncl(p.g.typeInfoGenerated, t.id): return
   case t.kind
   of tyDistinct:
-    result = genTypeInfo(p, t.sons[0])
+    result = genTypeInfo(p, t[0])
   of tyPointer, tyProc, tyBool, tyChar, tyCString, tyString, tyInt..tyUInt64:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
@@ -138,15 +138,15 @@ proc genTypeInfo(p: PProc, typ: PType): Rope =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-    addf(p.g.typeInfo, "$1.base = $2;$n",
+    p.g.typeInfo.addf("$1.base = $2;$n",
          [result, genTypeInfo(p, t.lastSon)])
   of tyArray:
     var s =
       "var $1 = {size: 0,kind: $2,base: null,node: null,finalizer: null};$n" %
               [result, rope(ord(t.kind))]
     prepend(p.g.typeInfo, s)
-    addf(p.g.typeInfo, "$1.base = $2;$n",
-         [result, genTypeInfo(p, t.sons[1])])
+    p.g.typeInfo.addf("$1.base = $2;$n",
+         [result, genTypeInfo(p, t[1])])
   of tyEnum: genEnumInfo(p, t, result)
   of tyObject: genObjectInfo(p, t, result)
   of tyTuple: genTupleInfo(p, t, result)

compiler/lambdalifting.nim

@@ -128,8 +128,8 @@ proc newCall(a: PSym, b: PNode): PNode =
 
 proc createClosureIterStateType*(g: ModuleGraph; iter: PSym): PType =
   var n = newNodeI(nkRange, iter.info)
-  addSon(n, newIntNode(nkIntLit, -1))
-  addSon(n, newIntNode(nkIntLit, 0))
+  n.add newIntNode(nkIntLit, -1)
+  n.add newIntNode(nkIntLit, 0)
   result = newType(tyRange, iter)
   result.n = n
   var intType = nilOrSysInt(g)
@@ -148,27 +148,27 @@ proc createEnvObj(g: ModuleGraph; owner: PSym; info: TLineInfo): PType =
 
 proc getClosureIterResult*(g: ModuleGraph; iter: PSym): PSym =
   if resultPos < iter.ast.len:
-    result = iter.ast.sons[resultPos].sym
+    result = iter.ast[resultPos].sym
   else:
     # XXX a bit hacky:
     result = newSym(skResult, getIdent(g.cache, ":result"), iter, iter.info, {})
-    result.typ = iter.typ.sons[0]
+    result.typ = iter.typ[0]
     incl(result.flags, sfUsed)
     iter.ast.add newSymNode(result)
 
 proc addHiddenParam(routine: PSym, param: PSym) =
   assert param.kind == skParam
-  var params = routine.ast.sons[paramsPos]
+  var params = routine.ast[paramsPos]
   # -1 is correct here as param.position is 0 based but we have at position 0
   # some nkEffect node:
   param.position = routine.typ.n.len-1
-  addSon(params, newSymNode(param))
+  params.add newSymNode(param)
   #incl(routine.typ.flags, tfCapturesEnv)
   assert sfFromGeneric in param.flags
   #echo "produced environment: ", param.id, " for ", routine.id
 
 proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
-  let params = routine.ast.sons[paramsPos]
+  let params = routine.ast[paramsPos]
   let hidden = lastSon(params)
   if hidden.kind == nkSym and hidden.sym.kind == skParam and hidden.sym.name.s == paramName:
     result = hidden.sym
@@ -179,7 +179,7 @@ proc getHiddenParam(g: ModuleGraph; routine: PSym): PSym =
     result = routine
 
 proc getEnvParam*(routine: PSym): PSym =
-  let params = routine.ast.sons[paramsPos]
+  let params = routine.ast[paramsPos]
   let hidden = lastSon(params)
   if hidden.kind == nkSym and hidden.sym.name.s == paramName:
     result = hidden.sym
@@ -206,8 +206,8 @@ proc newAsgnStmt(le, ri: PNode, info: TLineInfo): PNode =
   # which is however the only case when we generate an assignment in the first
   # place.
   result = newNodeI(nkAsgn, info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
 
 proc makeClosure*(g: ModuleGraph; prc: PSym; env: PNode; info: TLineInfo): PNode =
   result = newNodeIT(nkClosure, info, prc.typ)
@@ -218,6 +218,10 @@ proc makeClosure*(g: ModuleGraph; prc: PSym; env: PNode; info: TLineInfo): PNode
     if env.skipConv.kind == nkClosure:
       localError(g.config, info, "internal error: taking closure of closure")
     result.add(env)
+  #if isClosureIterator(result.typ):
+  createTypeBoundOps(g, nil, result.typ, info)
+  if tfHasAsgn in result.typ.flags or optSeqDestructors in g.config.globalOptions:
+    prc.flags.incl sfInjectDestructors
 
 proc interestingIterVar(s: PSym): bool {.inline.} =
   # XXX optimization: Only lift the variable if it lives across
@@ -231,7 +235,14 @@ template isIterator*(owner: PSym): bool =
 proc liftingHarmful(conf: ConfigRef; owner: PSym): bool {.inline.} =
   ## lambda lifting can be harmful for JS-like code generators.
   let isCompileTime = sfCompileTime in owner.flags or owner.kind == skMacro
-  result = conf.cmd == cmdCompileToJS and not isCompileTime
+  result = conf.backend == backendJs and not isCompileTime
+
+proc createTypeBoundOpsLL(g: ModuleGraph; refType: PType; info: TLineInfo; owner: PSym) =
+  if owner.kind != skMacro:
+    createTypeBoundOps(g, nil, refType.lastSon, info)
+    createTypeBoundOps(g, nil, refType, info)
+    if tfHasAsgn in refType.flags or optSeqDestructors in g.config.globalOptions:
+      owner.flags.incl sfInjectDestructors
 
 proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   # transforms  (iter)  to  (let env = newClosure[iter](); (iter, env))
@@ -245,7 +256,7 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   var env: PNode
   if owner.isIterator:
     let it = getHiddenParam(g, owner)
-    addUniqueField(it.typ.sons[0], hp, g.cache)
+    addUniqueField(it.typ.skipTypes({tyOwned})[0], hp, g.cache)
     env = indirectAccess(newSymNode(it), hp, hp.info)
   else:
     let e = newSym(skLet, iter.name, owner, n.info)
@@ -257,13 +268,12 @@ proc liftIterSym*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
     result.add(v)
   # add 'new' statement:
   result.add newCall(getSysSym(g, n.info, "internalNew"), env)
-  if optNimV2 in g.config.globalOptions:
-    createTypeBoundOps(g, nil, env.typ, n.info)
+  createTypeBoundOpsLL(g, env.typ, n.info, owner)
   result.add makeClosure(g, iter, env, n.info)
 
 proc freshVarForClosureIter*(g: ModuleGraph; s, owner: PSym): PNode =
   let envParam = getHiddenParam(g, owner)
-  let obj = envParam.typ.skipTypes({tyOwned, tyRef})
+  let obj = envParam.typ.skipTypes({tyOwned, tyRef, tyPtr})
   addField(obj, s, g.cache)
 
   var access = newSymNode(envParam)
@@ -282,10 +292,11 @@ proc markAsClosure(g: ModuleGraph; owner: PSym; n: PNode) =
   if illegalCapture(s):
     localError(g.config, n.info,
       ("'$1' is of type <$2> which cannot be captured as it would violate memory" &
-       " safety, declared here: $3") % [s.name.s, typeToString(s.typ), g.config$s.info])
-  elif owner.typ.callConv notin {ccClosure, ccDefault}:
+       " safety, declared here: $3; using '-d:nimWorkaround14447' helps in some cases") %
+      [s.name.s, typeToString(s.typ), g.config$s.info])
+  elif not (owner.typ.callConv == ccClosure or owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags):
     localError(g.config, n.info, "illegal capture '$1' because '$2' has the calling convention: <$3>" %
-      [s.name.s, owner.name.s, CallingConvToStr[owner.typ.callConv]])
+      [s.name.s, owner.name.s, $owner.typ.callConv])
   incl(owner.typ.flags, tfCapturesEnv)
   owner.typ.callConv = ccClosure
 
@@ -293,7 +304,7 @@ type
   DetectionPass = object
     processed, capturedVars: IntSet
     ownerToType: Table[int, PType]
-    somethingToDo, noDestructors: bool
+    somethingToDo: bool
     graph: ModuleGraph
 
 proc initDetectionPass(g: ModuleGraph; fn: PSym): DetectionPass =
@@ -323,7 +334,7 @@ proc getEnvTypeForOwner(c: var DetectionPass; owner: PSym;
     c.ownerToType[owner.id] = result
 
 proc asOwnedRef(c: DetectionPass; t: PType): PType =
-  if optNimV2 in c.graph.config.globalOptions:
+  if optOwnedRefs in c.graph.config.globalOptions:
     assert t.kind == tyRef
     result = newType(tyOwned, t.owner)
     result.flags.incl tfHasOwned
@@ -335,16 +346,17 @@ proc getEnvTypeForOwnerUp(c: var DetectionPass; owner: PSym;
                           info: TLineInfo): PType =
   var r = c.getEnvTypeForOwner(owner, info)
   result = newType(tyPtr, owner)
-  rawAddSon(result, r.skipTypes({tyOwned, tyRef}))
+  rawAddSon(result, r.skipTypes({tyOwned, tyRef, tyPtr}))
 
 proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
   let refObj = c.getEnvTypeForOwner(dest, info) # getHiddenParam(dest).typ
-  let obj = refObj.skipTypes({tyOwned, tyRef})
+  let obj = refObj.skipTypes({tyOwned, tyRef, tyPtr})
   # The assumption here is that gcDestructors means we cannot deal
   # with cycles properly, so it's better to produce a weak ref (=ptr) here.
-  # This seems to be generally correct but since it's a bit risky it's only
-  # enabled for gcDestructors.
-  let fieldType = if false: # c.graph.config.selectedGC == gcDestructors:
+  # This seems to be generally correct but since it's a bit risky it's disabled
+  # for now.
+  # XXX This is wrong for the 'hamming' test, so remove this logic again.
+  let fieldType = if isDefined(c.graph.config, "nimCycleBreaker"):
                     c.getEnvTypeForOwnerUp(dep, info) #getHiddenParam(dep).typ
                   else:
                     c.getEnvTypeForOwner(dep, info)
@@ -354,11 +366,18 @@ proc createUpField(c: var DetectionPass; dest, dep: PSym; info: TLineInfo) =
   let upIdent = getIdent(c.graph.cache, upName)
   let upField = lookupInRecord(obj.n, upIdent)
   if upField != nil:
-    if upField.typ.skipTypes({tyOwned, tyRef}) != fieldType.skipTypes({tyOwned, tyRef}):
+    if upField.typ.skipTypes({tyOwned, tyRef, tyPtr}) != fieldType.skipTypes({tyOwned, tyRef, tyPtr}):
       localError(c.graph.config, dep.info, "internal error: up references do not agree")
+
+    when false:
+      if c.graph.config.selectedGC == gcDestructors and sfCursor notin upField.flags:
+        localError(c.graph.config, dep.info, "internal error: up reference is not a .cursor")
   else:
     let result = newSym(skField, upIdent, obj.owner, obj.owner.info)
     result.typ = fieldType
+    when false:
+      if c.graph.config.selectedGC == gcDestructors:
+        result.flags.incl sfCursor
     rawAddField(obj, result)
 
 discard """
@@ -415,8 +434,7 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
     if innerProc:
       if s.isIterator: c.somethingToDo = true
       if not c.processed.containsOrIncl(s.id):
-        let body = transformBody(c.graph, s, cache = true,
-                                 noDestructors = c.noDestructors)
+        let body = transformBody(c.graph, s, cache = true)
         detectCapturedVars(body, s, c)
     let ow = s.skipGenericOwner
     if ow == owner:
@@ -425,8 +443,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
         addClosureParam(c, owner, n.info)
         if interestingIterVar(s):
           if not c.capturedVars.containsOrIncl(s.id):
-            let obj = getHiddenParam(c.graph, owner).typ.skipTypes({tyOwned, tyRef})
-            #let obj = c.getEnvTypeForOwner(s.owner).skipTypes({tyOwned, tyRef})
+            let obj = getHiddenParam(c.graph, owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
+            #let obj = c.getEnvTypeForOwner(s.owner).skipTypes({tyOwned, tyRef, tyPtr})
 
             if s.name.id == getIdent(c.graph.cache, ":state").id:
               obj.n[0].sym.id = -s.id
@@ -451,8 +469,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
       #echo "capturing ", n.info
       # variable 's' is actually captured:
       if interestingVar(s) and not c.capturedVars.containsOrIncl(s.id):
-        let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef})
-        #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef})
+        let obj = c.getEnvTypeForOwner(ow, n.info).skipTypes({tyOwned, tyRef, tyPtr})
+        #getHiddenParam(owner).typ.skipTypes({tyOwned, tyRef, tyPtr})
         addField(obj, s, c.graph.cache)
       # create required upFields:
       var w = owner.skipGenericOwner
@@ -477,7 +495,8 @@ proc detectCapturedVars(n: PNode; owner: PSym; c: var DetectionPass) =
           w = up
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit,
      nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef,
-     nkConverterDef, nkMacroDef, nkFuncDef, nkCommentStmt:
+     nkConverterDef, nkMacroDef, nkFuncDef, nkCommentStmt,
+     nkTypeOfExpr, nkMixinStmt, nkBindStmt:
     discard
   of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil:
@@ -507,7 +526,7 @@ proc accessViaEnvParam(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   if not envParam.isNil:
     var access = newSymNode(envParam)
     while true:
-      let obj = access.typ.sons[0]
+      let obj = access.typ[0]
       assert obj.kind == tyObject
       let field = getFieldFromObj(obj, s)
       if field != nil:
@@ -526,7 +545,7 @@ proc newEnvVar(cache: IdentCache; owner: PSym; typ: PType; info: TLineInfo): PNo
   when false:
     if owner.kind == skIterator and owner.typ.callConv == ccClosure:
       let it = getHiddenParam(owner)
-      addUniqueField(it.typ.sons[0], v)
+      addUniqueField(it.typ[0], v)
       result = indirectAccess(newSymNode(it), v, v.info)
     else:
       result = newSymNode(v)
@@ -542,7 +561,7 @@ proc setupEnvVar(owner: PSym; d: DetectionPass;
       localError d.graph.config, owner.info, "internal error: could not determine closure type"
     result = newEnvVar(d.graph.cache, owner, asOwnedRef(d, envVarType), info)
     c.envVars[owner.id] = result
-    if optNimV2 in d.graph.config.globalOptions:
+    if optOwnedRefs in d.graph.config.globalOptions:
       var v = newSym(skVar, getIdent(d.graph.cache, envName & "Alt"), owner, info)
       v.flags = {sfShadowed, sfGeneratedOp}
       v.typ = envVarType
@@ -552,7 +571,7 @@ proc getUpViaParam(g: ModuleGraph; owner: PSym): PNode =
   let p = getHiddenParam(g, owner)
   result = p.newSymNode
   if owner.isIterator:
-    let upField = lookupInRecord(p.typ.skipTypes({tyOwned, tyRef}).n, getIdent(g.cache, upName))
+    let upField = lookupInRecord(p.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(g.cache, upName))
     if upField == nil:
       localError(g.config, owner.info, "could not find up reference for closure iter")
     else:
@@ -572,20 +591,20 @@ proc rawClosureCreation(owner: PSym;
       var v = newNodeI(nkVarSection, env.info)
       addVar(v, env)
       result.add(v)
-      if optNimV2 in d.graph.config.globalOptions:
+      if optOwnedRefs in d.graph.config.globalOptions:
         let unowned = c.unownedEnvVars[owner.id]
         assert unowned != nil
         addVar(v, unowned)
 
     # add 'new' statement:
     result.add(newCall(getSysSym(d.graph, env.info, "internalNew"), env))
-    if optNimV2 in d.graph.config.globalOptions:
+    if optOwnedRefs in d.graph.config.globalOptions:
       let unowned = c.unownedEnvVars[owner.id]
       assert unowned != nil
       let env2 = copyTree(env)
       env2.typ = unowned.typ
       result.add newAsgnStmt(unowned, env2, env.info)
-      createTypeBoundOps(d.graph, nil, unowned.typ, env.info)
+      createTypeBoundOpsLL(d.graph, unowned.typ, env.info, owner)
 
     # add assignment statements for captured parameters:
     for i in 1..<owner.typ.n.len:
@@ -594,11 +613,15 @@ proc rawClosureCreation(owner: PSym;
         let fieldAccess = indirectAccess(env, local, env.info)
         # add ``env.param = param``
         result.add(newAsgnStmt(fieldAccess, newSymNode(local), env.info))
+        if owner.kind != skMacro:
+          createTypeBoundOps(d.graph, nil, fieldAccess.typ, env.info)
+        if tfHasAsgn in fieldAccess.typ.flags or optSeqDestructors in d.graph.config.globalOptions:
+          owner.flags.incl sfInjectDestructors
 
-  let upField = lookupInRecord(env.typ.skipTypes({tyOwned, tyRef}).n, getIdent(d.graph.cache, upName))
+  let upField = lookupInRecord(env.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
     let up = getUpViaParam(d.graph, owner)
-    if up != nil and upField.typ.skipTypes({tyOwned, tyRef}) == up.typ.skipTypes({tyOwned, tyRef}):
+    if up != nil and upField.typ.skipTypes({tyOwned, tyRef, tyPtr}) == up.typ.skipTypes({tyOwned, tyRef, tyPtr}):
       result.add(newAsgnStmt(rawIndirectAccess(env, upField, env.info),
                  up, env.info))
     #elif oldenv != nil and oldenv.typ == upField.typ:
@@ -608,16 +631,16 @@ proc rawClosureCreation(owner: PSym;
       localError(d.graph.config, env.info, "internal error: cannot create up reference")
   # we are not in the sem'check phase anymore! so pass 'nil' for the PContext
   # and hope for the best:
-  if optNimV2 in d.graph.config.globalOptions:
-    createTypeBoundOps(d.graph, nil, env.typ, owner.info)
+  createTypeBoundOpsLL(d.graph, env.typ, owner.info, owner)
 
 proc finishClosureCreation(owner: PSym; d: DetectionPass; c: LiftingPass;
                            info: TLineInfo; res: PNode) =
-  if optNimV2 in d.graph.config.globalOptions:
+  if optOwnedRefs in d.graph.config.globalOptions:
     let unowned = c.unownedEnvVars[owner.id]
     assert unowned != nil
     let nilLit = newNodeIT(nkNilLit, info, unowned.typ)
     res.add newAsgnStmt(unowned, nilLit, info)
+    createTypeBoundOpsLL(d.graph, unowned.typ, info, owner)
 
 proc closureCreationForIter(iter: PNode;
                             d: DetectionPass; c: var LiftingPass): PNode =
@@ -629,7 +652,7 @@ proc closureCreationForIter(iter: PNode;
   var vnode: PNode
   if owner.isIterator:
     let it = getHiddenParam(d.graph, owner)
-    addUniqueField(it.typ.skipTypes({tyOwned, tyRef}), v, d.graph.cache)
+    addUniqueField(it.typ.skipTypes({tyOwned, tyRef, tyPtr}), v, d.graph.cache)
     vnode = indirectAccess(newSymNode(it), v, v.info)
   else:
     vnode = v.newSymNode
@@ -637,13 +660,12 @@ proc closureCreationForIter(iter: PNode;
     addVar(vs, vnode)
     result.add(vs)
   result.add(newCall(getSysSym(d.graph, iter.info, "internalNew"), vnode))
-  if optNimV2 in d.graph.config.globalOptions:
-    createTypeBoundOps(d.graph, nil, vnode.typ, iter.info)
+  createTypeBoundOpsLL(d.graph, vnode.typ, iter.info, owner)
 
-  let upField = lookupInRecord(v.typ.skipTypes({tyOwned, tyRef}).n, getIdent(d.graph.cache, upName))
+  let upField = lookupInRecord(v.typ.skipTypes({tyOwned, tyRef, tyPtr}).n, getIdent(d.graph.cache, upName))
   if upField != nil:
     let u = setupEnvVar(owner, d, c, iter.info)
-    if u.typ.skipTypes({tyOwned, tyRef}) == upField.typ.skipTypes({tyOwned, tyRef}):
+    if u.typ.skipTypes({tyOwned, tyRef, tyPtr}) == upField.typ.skipTypes({tyOwned, tyRef, tyPtr}):
       result.add(newAsgnStmt(rawIndirectAccess(vnode, upField, iter.info),
                  u, iter.info))
     else:
@@ -653,9 +675,9 @@ proc closureCreationForIter(iter: PNode;
 proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
                      c: var LiftingPass): PNode =
   var access = setupEnvVar(owner, d, c, n.info)
-  if optNimV2 in d.graph.config.globalOptions:
+  if optOwnedRefs in d.graph.config.globalOptions:
     access = c.unownedEnvVars[owner.id]
-  let obj = access.typ.skipTypes({tyOwned, tyRef})
+  let obj = access.typ.skipTypes({tyOwned, tyRef, tyPtr})
   let field = getFieldFromObj(obj, n.sym)
   if field != nil:
     result = rawIndirectAccess(access, field, n.info)
@@ -664,7 +686,7 @@ proc accessViaEnvVar(n: PNode; owner: PSym; d: DetectionPass;
     result = n
 
 proc getStateField*(g: ModuleGraph; owner: PSym): PSym =
-  getHiddenParam(g, owner).typ.skipTypes({tyOwned, tyRef}).n.sons[0].sym
+  getHiddenParam(g, owner).typ.skipTypes({tyOwned, tyRef, tyPtr}).n[0].sym
 
 proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
                       c: var LiftingPass): PNode
@@ -689,7 +711,7 @@ proc symToClosure(n: PNode; owner: PSym; d: DetectionPass;
     while true:
       if access.typ == wanted:
         return makeClosure(d.graph, s, access, n.info)
-      let obj = access.typ.skipTypes({tyOwned, tyRef})
+      let obj = access.typ.skipTypes({tyOwned, tyRef, tyPtr})
       let upField = lookupInRecord(obj.n, getIdent(d.graph.cache, upName))
       if upField == nil:
         localError(d.graph.config, n.info, "internal error: no environment found")
@@ -708,7 +730,7 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         #  echo renderTree(s.getBody, {renderIds})
         let oldInContainer = c.inContainer
         c.inContainer = 0
-        var body = transformBody(d.graph, s)
+        var body = transformBody(d.graph, s, cache = false)
         body = liftCapturedVars(body, s, d, c)
         if c.envVars.getOrDefault(s.id).isNil:
           s.transformedBody = body
@@ -729,17 +751,17 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
         result = accessViaEnvVar(n, owner, d, c)
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit, nkComesFrom,
      nkTemplateDef, nkTypeSection, nkProcDef, nkMethodDef, nkConverterDef,
-     nkMacroDef, nkFuncDef:
+     nkMacroDef, nkFuncDef, nkMixinStmt, nkBindStmt:
     discard
   of nkClosure:
     if n[1].kind == nkNilLit:
-      n.sons[0] = liftCapturedVars(n[0], owner, d, c)
-      let x = n.sons[0].skipConv
+      n[0] = liftCapturedVars(n[0], owner, d, c)
+      let x = n[0].skipConv
       if x.kind == nkClosure:
         #localError(n.info, "internal error: closure to closure created")
         # now we know better, so patch it:
-        n.sons[0] = x.sons[0]
-        n.sons[1] = x.sons[1]
+        n[0] = x[0]
+        n[1] = x[1]
   of nkLambdaKinds, nkIteratorDef:
     if n.typ != nil and n[namePos].kind == nkSym:
       let oldInContainer = c.inContainer
@@ -750,27 +772,29 @@ proc liftCapturedVars(n: PNode; owner: PSym; d: DetectionPass;
       c.inContainer = oldInContainer
   of nkHiddenStdConv:
     if n.len == 2:
-      n.sons[1] = liftCapturedVars(n[1], owner, d, c)
+      n[1] = liftCapturedVars(n[1], owner, d, c)
       if n[1].kind == nkClosure: result = n[1]
   of nkReturnStmt:
     if n[0].kind in {nkAsgn, nkFastAsgn}:
       # we have a `result = result` expression produced by the closure
       # transform, let's not touch the LHS in order to make the lifting pass
       # correct when `result` is lifted
-      n[0].sons[1] = liftCapturedVars(n[0].sons[1], owner, d, c)
+      n[0][1] = liftCapturedVars(n[0][1], owner, d, c)
     else:
-      n.sons[0] = liftCapturedVars(n[0], owner, d, c)
+      n[0] = liftCapturedVars(n[0], owner, d, c)
+  of nkTypeOfExpr:
+    result = n
   else:
     if owner.isIterator:
       if nfLL in n.flags:
         # special case 'when nimVm' due to bug #3636:
-        n.sons[1] = liftCapturedVars(n[1], owner, d, c)
+        n[1] = liftCapturedVars(n[1], owner, d, c)
         return
 
     let inContainer = n.kind in {nkObjConstr, nkBracket}
     if inContainer: inc c.inContainer
     for i in 0..<n.len:
-      n.sons[i] = liftCapturedVars(n[i], owner, d, c)
+      n[i] = liftCapturedVars(n[i], owner, d, c)
     if inContainer: dec c.inContainer
 
 # ------------------ old stuff -------------------------------------------
@@ -801,7 +825,8 @@ proc semCaptureSym*(s, owner: PSym) =
       var o = owner.skipGenericOwner
       while o != nil and o.kind != skModule:
         if s.owner == o:
-          if owner.typ.callConv in {ccClosure, ccDefault} or owner.kind == skIterator:
+          if owner.typ.callConv == ccClosure or owner.kind == skIterator or
+             owner.typ.callConv == ccNimCall and tfExplicitCallConv notin owner.typ.flags:
             owner.typ.callConv = ccClosure
             propagateClosure(owner.skipGenericOwner, s.owner)
           else:
@@ -817,24 +842,23 @@ proc liftIterToProc*(g: ModuleGraph; fn: PSym; body: PNode; ptrType: PType): PNo
   # pretend 'fn' is a closure iterator for the analysis:
   let oldKind = fn.kind
   let oldCC = fn.typ.callConv
-  fn.kind = skIterator
+  fn.transitionRoutineSymKind(skIterator)
   fn.typ.callConv = ccClosure
   d.ownerToType[fn.id] = ptrType
   detectCapturedVars(body, fn, d)
   result = liftCapturedVars(body, fn, d, c)
-  fn.kind = oldKind
+  fn.transitionRoutineSymKind(oldKind)
   fn.typ.callConv = oldCC
 
-proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
-                  noDestructors: bool): PNode =
-  # XXX gCmd == cmdCompileToJS does not suffice! The compiletime stuff needs
+proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool): PNode =
+  # XXX backend == backendJs does not suffice! The compiletime stuff needs
   # the transformation even when compiling to JS ...
 
   # However we can do lifting for the stuff which is *only* compiletime.
   let isCompileTime = sfCompileTime in fn.flags or fn.kind == skMacro
 
   if body.kind == nkEmpty or (
-      g.config.cmd == cmdCompileToJS and not isCompileTime) or
+      g.config.backend == backendJs and not isCompileTime) or
       fn.skipGenericOwner.kind != skModule:
 
     # ignore forward declaration:
@@ -842,7 +866,6 @@ proc liftLambdas*(g: ModuleGraph; fn: PSym, body: PNode; tooEarly: var bool;
     tooEarly = true
   else:
     var d = initDetectionPass(g, fn)
-    d.noDestructors = noDestructors
     detectCapturedVars(body, fn, d)
     if not d.somethingToDo and fn.isIterator:
       addClosureParam(d, fn, body.info)
@@ -896,11 +919,10 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
         ...
     """
   if liftingHarmful(g.config, owner): return body
-  var L = body.len
-  if not (body.kind == nkForStmt and body[L-2].kind in nkCallKinds):
+  if not (body.kind == nkForStmt and body[^2].kind in nkCallKinds):
     localError(g.config, body.info, "ignored invalid for loop")
     return body
-  var call = body[L-2]
+  var call = body[^2]
 
   result = newNodeI(nkStmtList, body.info)
 
@@ -921,40 +943,39 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
     result.add(v)
     # add 'new' statement:
     result.add(newCall(getSysSym(g, env.info, "internalNew"), env.newSymNode))
-    if optNimV2 in g.config.globalOptions:
-      createTypeBoundOps(g, nil, env.typ, body.info)
+    createTypeBoundOpsLL(g, env.typ, body.info, owner)
 
   elif op.kind == nkStmtListExpr:
     let closure = op.lastSon
     if closure.kind == nkClosure:
-      call.sons[0] = closure
-      for i in 0 .. op.len-2:
+      call[0] = closure
+      for i in 0..<op.len-1:
         result.add op[i]
 
   var loopBody = newNodeI(nkStmtList, body.info, 3)
   var whileLoop = newNodeI(nkWhileStmt, body.info, 2)
-  whileLoop.sons[0] = newIntTypeNode(nkIntLit, 1, getSysType(g, body.info, tyBool))
-  whileLoop.sons[1] = loopBody
+  whileLoop[0] = newIntTypeNode(1, getSysType(g, body.info, tyBool))
+  whileLoop[1] = loopBody
   result.add whileLoop
 
   # setup loopBody:
   # gather vars in a tuple:
   var v2 = newNodeI(nkLetSection, body.info)
-  var vpart = newNodeI(if L == 3: nkIdentDefs else: nkVarTuple, body.info)
-  for i in 0 .. L-3:
+  var vpart = newNodeI(if body.len == 3: nkIdentDefs else: nkVarTuple, body.info)
+  for i in 0..<body.len-2:
     if body[i].kind == nkSym:
-      body[i].sym.kind = skLet
-    addSon(vpart, body[i])
+      body[i].sym.transitionToLet()
+    vpart.add body[i]
 
-  addSon(vpart, newNodeI(nkEmpty, body.info)) # no explicit type
+  vpart.add newNodeI(nkEmpty, body.info) # no explicit type
   if not env.isNil:
-    call.sons[0] = makeClosure(g, call.sons[0].sym, env.newSymNode, body.info)
-  addSon(vpart, call)
-  addSon(v2, vpart)
+    call[0] = makeClosure(g, call[0].sym, env.newSymNode, body.info)
+  vpart.add call
+  v2.add vpart
 
-  loopBody.sons[0] = v2
+  loopBody[0] = v2
   var bs = newNodeI(nkBreakState, body.info)
-  bs.addSon(call.sons[0])
+  bs.add call[0]
 
   let ibs = newNodeI(nkIfStmt, body.info)
   let elifBranch = newNodeI(nkElifBranch, body.info)
@@ -966,5 +987,5 @@ proc liftForLoop*(g: ModuleGraph; body: PNode; owner: PSym): PNode =
   elifBranch.add(br)
   ibs.add(elifBranch)
 
-  loopBody.sons[1] = ibs
-  loopBody.sons[2] = body[L-1]
+  loopBody[1] = ibs
+  loopBody[2] = body[^1]

compiler/layouter.nim

@@ -9,10 +9,7 @@
 
 ## Layouter for nimpretty.
 
-import idents, lexer, lineinfos, llstream, options, msgs, strutils,
-  pathutils
-from os import changeFileExt
-from sequtils import delete
+import idents, lexer, lineinfos, llstream, options, msgs, strutils, pathutils
 
 const
   MinLineLen = 15
@@ -24,7 +21,7 @@ type
   SemicolonKind = enum
     detectSemicolonKind, useSemicolon, dontTouch
 
-  LayoutToken = enum
+  LayoutToken* = enum
     ltSpaces,
     ltCrucialNewline, ## a semantically crucial newline (indentation!)
     ltSplittingNewline, ## newline used for splitting up long
@@ -38,14 +35,14 @@ type
   Emitter* = object
     config: ConfigRef
     fid: FileIndex
-    lastTok: TTokType
+    lastTok: TokType
     inquote, lastTokWasTerse: bool
     semicolons: SemicolonKind
     col, lastLineNumber, lineSpan, indentLevel, indWidth*, inSection: int
     keepIndents*: int
     doIndentMore*: int
-    kinds: seq[LayoutToken]
-    tokens: seq[string]
+    kinds*: seq[LayoutToken]
+    tokens*: seq[string]
     indentStack: seq[int]
     fixedUntil: int # marks where we must not go in the content
     altSplitPos: array[SplitKind, int] # alternative split positions
@@ -149,7 +146,7 @@ proc optionalIsGood(em: var Emitter; pos, currentLen: int): bool =
 
 proc lenOfNextTokens(em: Emitter; pos: int): int =
   result = 0
-  for i in 1 ..< em.tokens.len-pos:
+  for i in 1..<em.tokens.len-pos:
     if em.kinds[pos+i] in {ltCrucialNewline, ltSplittingNewline, ltOptionalNewline}: break
     inc result, em.tokens[pos+i].len
 
@@ -163,13 +160,11 @@ proc guidingInd(em: Emitter; pos: int): int =
     inc i
   result = -1
 
-proc closeEmitter*(em: var Emitter) =
+proc renderTokens*(em: var Emitter): string =
+  ## Render Emitter tokens to a string of code
   template defaultCase() =
     content.add em.tokens[i]
     inc lineLen, em.tokens[i].len
-
-  let outFile = em.config.absOutFile
-
   var content = newStringOfCap(16_000)
   var maxLhs = 0
   var lineLen = 0
@@ -246,6 +241,11 @@ proc closeEmitter*(em: var Emitter) =
       defaultCase()
     inc i
 
+  return content
+
+proc writeOut*(em: Emitter, content: string)  =
+  ## Write to disk
+  let outFile = em.config.absOutFile
   if fileExists(outFile) and readFile(outFile.string) == content:
     discard "do nothing, see #9499"
     return
@@ -256,6 +256,11 @@ proc closeEmitter*(em: var Emitter) =
   f.llStreamWrite content
   llStreamClose(f)
 
+proc closeEmitter*(em: var Emitter) =
+  ## Renders emitter tokens and write to a file
+  let content = renderTokens(em)
+  em.writeOut(content)
+
 proc wr(em: var Emitter; x: string; lt: LayoutToken) =
   em.tokens.add x
   em.kinds.add lt
@@ -314,15 +319,15 @@ proc removeSpaces(em: var Emitter) =
 
 const
   openPars = {tkParLe, tkParDotLe,
-              tkBracketLe, tkBracketLeColon, tkCurlyDotLe,
-              tkCurlyLe}
+              tkBracketLe, tkBracketDotLe, tkBracketLeColon,
+              tkCurlyDotLe, tkCurlyLe}
   closedPars = {tkParRi, tkParDotRi,
-              tkBracketRi, tkCurlyDotRi,
-              tkCurlyRi, tkBracketDotRi}
+                tkBracketRi, tkBracketDotRi,
+                tkCurlyDotRi, tkCurlyRi}
 
   splitters = openPars + {tkComma, tkSemiColon} # do not add 'tkColon' here!
   oprSet = {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
-            tkIsnot, tkNot, tkOf, tkAs, tkDotDot, tkAnd, tkOr, tkXor}
+            tkIsnot, tkNot, tkOf, tkAs, tkFrom, tkDotDot, tkAnd, tkOr, tkXor}
 
 template goodCol(col): bool = col >= em.maxLineLen div 2
 
@@ -397,7 +402,7 @@ proc endsInAlpha(em: Emitter): bool =
   while i >= 0 and em.kinds[i] in {ltBeginSection, ltEndSection}: dec(i)
   result = if i >= 0: em.tokens[i].lastChar in SymChars+{'_'} else: false
 
-proc emitComment(em: var Emitter; tok: TToken; dontIndent: bool) =
+proc emitComment(em: var Emitter; tok: Token; dontIndent: bool) =
   var col = em.col
   let lit = strip fileSection(em.config, em.fid, tok.commentOffsetA, tok.commentOffsetB)
   em.lineSpan = countNewlines(lit)
@@ -412,7 +417,7 @@ proc emitComment(em: var Emitter; tok: TToken; dontIndent: bool) =
       inc col
     emitMultilineComment(em, lit, col, dontIndent)
 
-proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
+proc emitTok*(em: var Emitter; L: Lexer; tok: Token) =
   template wasExportMarker(em): bool =
     em.kinds.len > 0 and em.kinds[^1] == ltExportMarker
 
@@ -444,6 +449,12 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
       em.indentLevel = tok.indent
     elif (em.lastTok in (splitters + oprSet) and
         tok.tokType notin (closedPars - {tkBracketDotRi})):
+      if tok.tokType in openPars and tok.indent > em.indentStack[^1]:
+        while em.indentStack[^1] < tok.indent:
+          em.indentStack.add(em.indentStack[^1] + em.indWidth)
+      while em.indentStack[^1] > tok.indent:
+        discard em.indentStack.pop()
+
       # aka: we are in an expression context:
       let alignment = max(tok.indent - em.indentStack[^1], 0)
       em.indentLevel = alignment + em.indentStack.high * em.indWidth
@@ -483,7 +494,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
       wrSpace em
 
     if not em.inquote:
-      wr(em, TokTypeToStr[tok.tokType], ltKeyword)
+      wr(em, $tok.tokType, ltKeyword)
       if tok.tokType in {tkAnd, tkOr, tkIn, tkNotin}:
         rememberSplit(splitIn)
         wrSpace em
@@ -492,27 +503,28 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
       wr(em, tok.ident.s, ltIdent)
 
   of tkColon:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     wrSpace em
   of tkSemiColon, tkComma:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     rememberSplit(splitComma)
     wrSpace em
   of openPars:
-    if tok.strongSpaceA > 0 and not em.endsInWhite and not em.wasExportMarker:
+    if tok.strongSpaceA > 0 and not em.endsInWhite and
+        (not em.wasExportMarker or tok.tokType == tkCurlyDotLe):
       wrSpace em
-    wr(em, TokTypeToStr[tok.tokType], ltSomeParLe)
+    wr(em, $tok.tokType, ltSomeParLe)
     rememberSplit(splitParLe)
   of closedPars:
-    wr(em, TokTypeToStr[tok.tokType], ltSomeParRi)
+    wr(em, $tok.tokType, ltSomeParRi)
   of tkColonColon:
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
   of tkDot:
     lastTokWasTerse = true
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
   of tkEquals:
     if not em.inquote and not em.endsInWhite: wrSpace(em)
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     if not em.inquote: wrSpace(em)
   of tkOpr, tkDotDot:
     if em.inquote or ((tok.strongSpaceA == 0 and tok.strongSpaceB == 0) and
@@ -532,7 +544,7 @@ proc emitTok*(em: var Emitter; L: TLexer; tok: TToken) =
         wrSpace(em)
   of tkAccent:
     if not em.inquote and endsInAlpha(em): wrSpace(em)
-    wr(em, TokTypeToStr[tok.tokType], ltOther)
+    wr(em, $tok.tokType, ltOther)
     em.inquote = not em.inquote
   of tkComment:
     if not preventComment:
@@ -561,7 +573,7 @@ proc endsWith(em: Emitter; k: varargs[string]): bool =
   return true
 
 proc rfind(em: Emitter, t: string): int =
-  for i in 1 .. 5:
+  for i in 1..5:
     if em.tokens[^i] == t:
       return i
 

compiler/lexer.nim

@@ -30,41 +30,49 @@ const
 # don't forget to update the 'highlite' module if these charsets should change
 
 type
-  TTokType* = enum
-    tkInvalid, tkEof,         # order is important here!
-    tkSymbol, # keywords:
-    tkAddr, tkAnd, tkAs, tkAsm,
-    tkBind, tkBlock, tkBreak, tkCase, tkCast,
-    tkConcept, tkConst, tkContinue, tkConverter,
-    tkDefer, tkDiscard, tkDistinct, tkDiv, tkDo,
-    tkElif, tkElse, tkEnd, tkEnum, tkExcept, tkExport,
-    tkFinally, tkFor, tkFrom, tkFunc,
-    tkIf, tkImport, tkIn, tkInclude, tkInterface,
-    tkIs, tkIsnot, tkIterator,
-    tkLet,
-    tkMacro, tkMethod, tkMixin, tkMod, tkNil, tkNot, tkNotin,
-    tkObject, tkOf, tkOr, tkOut,
-    tkProc, tkPtr, tkRaise, tkRef, tkReturn,
-    tkShl, tkShr, tkStatic,
-    tkTemplate,
-    tkTry, tkTuple, tkType, tkUsing,
-    tkVar, tkWhen, tkWhile, tkXor,
-    tkYield, # end of keywords
-    tkIntLit, tkInt8Lit, tkInt16Lit, tkInt32Lit, tkInt64Lit,
-    tkUIntLit, tkUInt8Lit, tkUInt16Lit, tkUInt32Lit, tkUInt64Lit,
-    tkFloatLit, tkFloat32Lit, tkFloat64Lit, tkFloat128Lit,
-    tkStrLit, tkRStrLit, tkTripleStrLit,
-    tkGStrLit, tkGTripleStrLit, tkCharLit, tkParLe, tkParRi, tkBracketLe,
-    tkBracketRi, tkCurlyLe, tkCurlyRi,
-    tkBracketDotLe, tkBracketDotRi, # [. and  .]
-    tkCurlyDotLe, tkCurlyDotRi, # {.  and  .}
-    tkParDotLe, tkParDotRi,   # (. and .)
-    tkComma, tkSemiColon,
-    tkColon, tkColonColon, tkEquals, tkDot, tkDotDot, tkBracketLeColon,
-    tkOpr, tkComment, tkAccent,
+  TokType* = enum
+    tkInvalid = "tkInvalid", tkEof = "[EOF]", # order is important here!
+    tkSymbol = "tkSymbol", # keywords:
+    tkAddr = "addr", tkAnd = "and", tkAs = "as", tkAsm = "asm",
+    tkBind = "bind", tkBlock = "block", tkBreak = "break", tkCase = "case", tkCast = "cast",
+    tkConcept = "concept", tkConst = "const", tkContinue = "continue", tkConverter = "converter",
+    tkDefer = "defer", tkDiscard = "discard", tkDistinct = "distinct", tkDiv = "div", tkDo = "do",
+    tkElif = "elif", tkElse = "else", tkEnd = "end", tkEnum = "enum", tkExcept = "except", tkExport = "export",
+    tkFinally = "finally", tkFor = "for", tkFrom = "from", tkFunc = "func",
+    tkIf = "if", tkImport = "import", tkIn = "in", tkInclude = "include", tkInterface = "interface",
+    tkIs = "is", tkIsnot = "isnot", tkIterator = "iterator",
+    tkLet = "let",
+    tkMacro = "macro", tkMethod = "method", tkMixin = "mixin", tkMod = "mod", tkNil = "nil", tkNot = "not", tkNotin = "notin",
+    tkObject = "object", tkOf = "of", tkOr = "or", tkOut = "out",
+    tkProc = "proc", tkPtr = "ptr", tkRaise = "raise", tkRef = "ref", tkReturn = "return",
+    tkShl = "shl", tkShr = "shr", tkStatic = "static",
+    tkTemplate = "template",
+    tkTry = "try", tkTuple = "tuple", tkType = "type", tkUsing = "using",
+    tkVar = "var", tkWhen = "when", tkWhile = "while", tkXor = "xor",
+    tkYield = "yield", # end of keywords
+
+    tkIntLit = "tkIntLit", tkInt8Lit = "tkInt8Lit", tkInt16Lit = "tkInt16Lit", 
+    tkInt32Lit = "tkInt32Lit", tkInt64Lit = "tkInt64Lit",
+    tkUIntLit = "tkUIntLit", tkUInt8Lit = "tkUInt8Lit", tkUInt16Lit = "tkUInt16Lit", 
+    tkUInt32Lit = "tkUInt32Lit", tkUInt64Lit = "tkUInt64Lit",
+    tkFloatLit = "tkFloatLit", tkFloat32Lit = "tkFloat32Lit",
+    tkFloat64Lit = "tkFloat64Lit", tkFloat128Lit = "tkFloat128Lit",
+    tkStrLit = "tkStrLit", tkRStrLit = "tkRStrLit", tkTripleStrLit = "tkTripleStrLit",
+    tkGStrLit = "tkGStrLit", tkGTripleStrLit = "tkGTripleStrLit", tkCharLit = "tkCharLit", 
+    
+    tkParLe = "(", tkParRi = ")", tkBracketLe = "[",
+    tkBracketRi = "]", tkCurlyLe = "{", tkCurlyRi = "}",
+    tkBracketDotLe = "[.", tkBracketDotRi = ".]",
+    tkCurlyDotLe = "{.", tkCurlyDotRi = ".}",
+    tkParDotLe = "(.", tkParDotRi = ".)",
+    tkComma = ",", tkSemiColon = ";",
+    tkColon = ":", tkColonColon = "::", tkEquals = "=", 
+    tkDot = ".", tkDotDot = "..", tkBracketLeColon = "[:",
+    tkOpr, tkComment, tkAccent = "`",
+    # these are fake tokens used by renderer.nim
     tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr
 
-  TTokTypes* = set[TTokType]
+  TokTypes* = set[TokType]
 
 const
   weakTokens = {tkComma, tkSemiColon, tkColon,
@@ -73,53 +81,21 @@ const
     # tokens that should not be considered for previousToken
   tokKeywordLow* = succ(tkSymbol)
   tokKeywordHigh* = pred(tkIntLit)
-  TokTypeToStr*: array[TTokType, string] = ["tkInvalid", "[EOF]",
-    "tkSymbol",
-    "addr", "and", "as", "asm",
-    "bind", "block", "break", "case", "cast",
-    "concept", "const", "continue", "converter",
-    "defer", "discard", "distinct", "div", "do",
-    "elif", "else", "end", "enum", "except", "export",
-    "finally", "for", "from", "func", "if",
-    "import", "in", "include", "interface", "is", "isnot", "iterator",
-    "let",
-    "macro", "method", "mixin", "mod",
-    "nil", "not", "notin", "object", "of", "or",
-    "out", "proc", "ptr", "raise", "ref", "return",
-    "shl", "shr", "static",
-    "template",
-    "try", "tuple", "type", "using",
-    "var", "when", "while", "xor",
-    "yield",
-    "tkIntLit", "tkInt8Lit", "tkInt16Lit", "tkInt32Lit", "tkInt64Lit",
-    "tkUIntLit", "tkUInt8Lit", "tkUInt16Lit", "tkUInt32Lit", "tkUInt64Lit",
-    "tkFloatLit", "tkFloat32Lit", "tkFloat64Lit", "tkFloat128Lit",
-    "tkStrLit", "tkRStrLit",
-    "tkTripleStrLit", "tkGStrLit", "tkGTripleStrLit", "tkCharLit", "(",
-    ")", "[", "]", "{", "}", "[.", ".]", "{.", ".}", "(.", ".)",
-    ",", ";",
-    ":", "::", "=", ".", "..", "[:",
-    "tkOpr", "tkComment", "`",
-    "tkSpaces", "tkInfixOpr",
-    "tkPrefixOpr", "tkPostfixOpr"]
 
 type
-  TNumericalBase* = enum
+  NumericalBase* = enum
     base10,                   # base10 is listed as the first element,
                               # so that it is the correct default value
     base2, base8, base16
 
-  CursorPosition* {.pure.} = enum ## XXX remove this again
-    None, InToken, BeforeToken, AfterToken
-
-  TToken* = object            # a Nim token
-    tokType*: TTokType        # the type of the token
+  Token* = object             # a Nim token
+    tokType*: TokType         # the type of the token
     indent*: int              # the indentation; != -1 if the token has been
                               # preceded with indentation
     ident*: PIdent            # the parsed identifier
     iNumber*: BiggestInt      # the parsed integer literal
     fNumber*: BiggestFloat    # the parsed floating point literal
-    base*: TNumericalBase     # the numerical base; only valid for int
+    base*: NumericalBase      # the numerical base; only valid for int
                               # or float literals
     strongSpaceA*: int8       # leading spaces of an operator
     strongSpaceB*: int8       # trailing spaces of an operator
@@ -127,26 +103,25 @@ type
                               # documentation comments are here too
     line*, col*: int
     when defined(nimpretty):
-      offsetA*, offsetB*: int   # used for pretty printing so that literals
-                                # like 0b01 or  r"\L" are unaffected
+      offsetA*, offsetB*: int # used for pretty printing so that literals
+                              # like 0b01 or  r"\L" are unaffected
       commentOffsetA*, commentOffsetB*: int
 
-  TErrorHandler* = proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string)
-  TLexer* = object of TBaseLexer
+  ErrorHandler* = proc (conf: ConfigRef; info: TLineInfo; msg: TMsgKind; arg: string)
+  Lexer* = object of TBaseLexer
     fileIdx*: FileIndex
-    indentAhead*: int         # if > 0 an indendation has already been read
+    indentAhead*: int         # if > 0 an indentation has already been read
                               # this is needed because scanning comments
                               # needs so much look-ahead
     currLineIndent*: int
     strongSpaces*, allowTabs*: bool
-    cursor*: CursorPosition
-    errorHandler*: TErrorHandler
+    errorHandler*: ErrorHandler
     cache*: IdentCache
     when defined(nimsuggest):
       previousToken: TLineInfo
     config*: ConfigRef
 
-proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
+proc getLineInfo*(L: Lexer, tok: Token): TLineInfo {.inline.} =
   result = newLineInfo(L.fileIdx, tok.line, tok.col)
   when defined(nimpretty):
     result.offsetA = tok.offsetA
@@ -154,8 +129,8 @@ proc getLineInfo*(L: TLexer, tok: TToken): TLineInfo {.inline.} =
     result.commentOffsetA = tok.commentOffsetA
     result.commentOffsetB = tok.commentOffsetB
 
-proc isKeyword*(kind: TTokType): bool =
-  result = (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)
+proc isKeyword*(kind: TokType): bool =
+  (kind >= tokKeywordLow) and (kind <= tokKeywordHigh)
 
 template ones(n): untyped = ((1 shl n)-1) # for utf-8 conversion
 
@@ -169,28 +144,26 @@ proc isNimIdentifier*(s: string): bool =
       inc(i)
     result = true
 
-proc `$`*(tok: TToken): string =
+proc `$`*(tok: Token): string =
   case tok.tokType
-  of tkIntLit..tkInt64Lit: result = $tok.iNumber
-  of tkFloatLit..tkFloat64Lit: result = $tok.fNumber
-  of tkInvalid, tkStrLit..tkCharLit, tkComment: result = tok.literal
-  of tkParLe..tkColon, tkEof, tkAccent:
-    result = TokTypeToStr[tok.tokType]
+  of tkIntLit..tkInt64Lit: $tok.iNumber
+  of tkFloatLit..tkFloat64Lit: $tok.fNumber
+  of tkInvalid, tkStrLit..tkCharLit, tkComment: tok.literal
+  of tkParLe..tkColon, tkEof, tkAccent: $tok.tokType
   else:
     if tok.ident != nil:
-      result = tok.ident.s
+      tok.ident.s
     else:
-      result = ""
+      ""
 
-proc prettyTok*(tok: TToken): string =
-  if isKeyword(tok.tokType): result = "keyword " & tok.ident.s
-  else: result = $tok
+proc prettyTok*(tok: Token): string =
+  if isKeyword(tok.tokType): "keyword " & tok.ident.s
+  else: $tok
 
-proc printTok*(conf: ConfigRef; tok: TToken) =
-  msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" &
-      TokTypeToStr[tok.tokType] & " " & $tok)
+proc printTok*(conf: ConfigRef; tok: Token) =
+  msgWriteln(conf, $tok.line & ":" & $tok.col & "\t" & $tok.tokType & " " & $tok)
 
-proc initToken*(L: var TToken) =
+proc initToken*(L: var Token) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
@@ -203,7 +176,7 @@ proc initToken*(L: var TToken) =
     L.commentOffsetA = 0
     L.commentOffsetB = 0
 
-proc fillToken(L: var TToken) =
+proc fillToken(L: var Token) =
   L.tokType = tkInvalid
   L.iNumber = 0
   L.indent = 0
@@ -216,7 +189,7 @@ proc fillToken(L: var TToken) =
     L.commentOffsetA = 0
     L.commentOffsetB = 0
 
-proc openLexer*(lex: var TLexer, fileIdx: FileIndex, inputstream: PLLStream;
+proc openLexer*(lex: var Lexer, fileIdx: FileIndex, inputstream: PLLStream;
                  cache: IdentCache; config: ConfigRef) =
   openBaseLexer(lex, inputstream)
   lex.fileIdx = fileIdx
@@ -228,36 +201,36 @@ proc openLexer*(lex: var TLexer, fileIdx: FileIndex, inputstream: PLLStream;
     lex.previousToken.fileIndex = fileIdx
   lex.config = config
 
-proc openLexer*(lex: var TLexer, filename: AbsoluteFile, inputstream: PLLStream;
+proc openLexer*(lex: var Lexer, filename: AbsoluteFile, inputstream: PLLStream;
                 cache: IdentCache; config: ConfigRef) =
   openLexer(lex, fileInfoIdx(config, filename), inputstream, cache, config)
 
-proc closeLexer*(lex: var TLexer) =
+proc closeLexer*(lex: var Lexer) =
   if lex.config != nil:
     inc(lex.config.linesCompiled, lex.lineNumber)
   closeBaseLexer(lex)
 
-proc getLineInfo(L: TLexer): TLineInfo =
+proc getLineInfo(L: Lexer): TLineInfo =
   result = newLineInfo(L.fileIdx, L.lineNumber, getColNumber(L, L.bufpos))
 
-proc dispMessage(L: TLexer; info: TLineInfo; msg: TMsgKind; arg: string) =
+proc dispMessage(L: Lexer; info: TLineInfo; msg: TMsgKind; arg: string) =
   if L.errorHandler.isNil:
     msgs.message(L.config, info, msg, arg)
   else:
     L.errorHandler(L.config, info, msg, arg)
 
-proc lexMessage*(L: TLexer, msg: TMsgKind, arg = "") =
+proc lexMessage*(L: Lexer, msg: TMsgKind, arg = "") =
   L.dispMessage(getLineInfo(L), msg, arg)
 
-proc lexMessageTok*(L: TLexer, msg: TMsgKind, tok: TToken, arg = "") =
+proc lexMessageTok*(L: Lexer, msg: TMsgKind, tok: Token, arg = "") =
   var info = newLineInfo(L.fileIdx, tok.line, tok.col)
   L.dispMessage(info, msg, arg)
 
-proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
+proc lexMessagePos(L: var Lexer, msg: TMsgKind, pos: int, arg = "") =
   var info = newLineInfo(L.fileIdx, L.lineNumber, pos - L.lineStart)
   L.dispMessage(info, msg, arg)
 
-proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
+proc matchTwoChars(L: Lexer, first: char, second: set[char]): bool =
   result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
 
 template tokenBegin(tok, pos) {.dirty.} =
@@ -271,7 +244,6 @@ template tokenEnd(tok, pos) {.dirty.} =
     let colB = getColNumber(L, pos)+1
     if L.fileIdx == L.config.m.trackPos.fileIndex and L.config.m.trackPos.col in colA..colB and
         L.lineNumber == L.config.m.trackPos.line.int and L.config.ideCmd in {ideSug, ideCon}:
-      L.cursor = CursorPosition.InToken
       L.config.m.trackPos.col = colA.int16
     colA = 0
   when defined(nimpretty):
@@ -296,28 +268,27 @@ template tokenEndPrevious(tok, pos) =
     let colB = getColNumber(L, pos)
     if L.fileIdx == L.config.m.trackPos.fileIndex and L.config.m.trackPos.col in colA..colB and
         L.lineNumber == L.config.m.trackPos.line.int and L.config.ideCmd in {ideSug, ideCon}:
-      L.cursor = CursorPosition.BeforeToken
       L.config.m.trackPos = L.previousToken
       L.config.m.trackPosAttached = true
     colA = 0
   when defined(nimpretty):
     tok.offsetB = L.offsetBase + pos
 
-template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
-  add(t.literal, replacementChar)
+template eatChar(L: var Lexer, t: var Token, replacementChar: char) =
+  t.literal.add(replacementChar)
   inc(L.bufpos)
 
-template eatChar(L: var TLexer, t: var TToken) =
-  add(t.literal, L.buf[L.bufpos])
+template eatChar(L: var Lexer, t: var Token) =
+  t.literal.add(L.buf[L.bufpos])
   inc(L.bufpos)
 
-proc getNumber(L: var TLexer, result: var TToken) =
-  proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]): Natural =
+proc getNumber(L: var Lexer, result: var Token) =
+  proc matchUnderscoreChars(L: var Lexer, tok: var Token, chars: set[char]): Natural =
     var pos = L.bufpos              # use registers for pos, buf
     result = 0
     while true:
       if L.buf[pos] in chars:
-        add(tok.literal, L.buf[pos])
+        tok.literal.add(L.buf[pos])
         inc(pos)
         inc(result)
       else:
@@ -328,35 +299,35 @@ proc getNumber(L: var TLexer, result: var TToken) =
             "only single underscores may occur in a token and token may not " &
             "end with an underscore: e.g. '1__1' and '1_' are invalid")
           break
-        add(tok.literal, '_')
+        tok.literal.add('_')
         inc(pos)
     L.bufpos = pos
 
-  proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
+  proc matchChars(L: var Lexer, tok: var Token, chars: set[char]) =
     var pos = L.bufpos              # use registers for pos, buf
     while L.buf[pos] in chars:
-      add(tok.literal, L.buf[pos])
+      tok.literal.add(L.buf[pos])
       inc(pos)
     L.bufpos = pos
 
-  proc lexMessageLitNum(L: var TLexer, msg: string, startpos: int, msgKind = errGenerated) =
+  proc lexMessageLitNum(L: var Lexer, msg: string, startpos: int, msgKind = errGenerated) =
     # Used to get slightly human friendlier err messages.
     const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O',
       'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
     var msgPos = L.bufpos
-    var t: TToken
+    var t: Token
     t.literal = ""
     L.bufpos = startpos # Use L.bufpos as pos because of matchChars
     matchChars(L, t, literalishChars)
     # We must verify +/- specifically so that we're not past the literal
     if  L.buf[L.bufpos] in {'+', '-'} and
         L.buf[L.bufpos - 1] in {'e', 'E'}:
-      add(t.literal, L.buf[L.bufpos])
+      t.literal.add(L.buf[L.bufpos])
       inc(L.bufpos)
       matchChars(L, t, literalishChars)
     if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
       inc(L.bufpos)
-      add(t.literal, L.buf[L.bufpos])
+      t.literal.add(L.buf[L.bufpos])
       matchChars(L, t, {'0'..'9'})
     L.bufpos = msgPos
     lexMessage(L, msgKind, msg % t.literal)
@@ -565,13 +536,13 @@ proc getNumber(L: var TLexer, result: var TToken) =
       case result.tokType
       of floatTypes:
         result.fNumber = parseFloat(result.literal)
-      of tkUInt64Lit:
+      of tkUInt64Lit, tkUIntLit:
         var iNumber: uint64
         var len: int
         try:
           len = parseBiggestUInt(result.literal, iNumber)
         except ValueError:
-          raise newException(OverflowError, "number out of range: " & $result.literal)
+          raise newException(OverflowDefect, "number out of range: " & $result.literal)
         if len != result.literal.len:
           raise newException(ValueError, "invalid integer: " & $result.literal)
         result.iNumber = cast[int64](iNumber)
@@ -581,7 +552,7 @@ proc getNumber(L: var TLexer, result: var TToken) =
         try:
           len = parseBiggestInt(result.literal, iNumber)
         except ValueError:
-          raise newException(OverflowError, "number out of range: " & $result.literal)
+          raise newException(OverflowDefect, "number out of range: " & $result.literal)
         if len != result.literal.len:
           raise newException(ValueError, "invalid integer: " & $result.literal)
         result.iNumber = iNumber
@@ -607,12 +578,12 @@ proc getNumber(L: var TLexer, result: var TToken) =
 
   except ValueError:
     lexMessageLitNum(L, "invalid number: '$1'", startpos)
-  except OverflowError, RangeError:
+  except OverflowDefect, RangeDefect:
     lexMessageLitNum(L, "number out of range: '$1'", startpos)
   tokenEnd(result, postPos-1)
   L.bufpos = postPos
 
-proc handleHexChar(L: var TLexer, xi: var int; position: range[0..4]) =
+proc handleHexChar(L: var Lexer, xi: var int; position: range[0..4]) =
   template invalid() =
     lexMessage(L, errGenerated,
       "expected a hex digit, but found: " & L.buf[L.bufpos] &
@@ -637,7 +608,7 @@ proc handleHexChar(L: var TLexer, xi: var int; position: range[0..4]) =
     # Need to progress for `nim check`
     inc(L.bufpos)
 
-proc handleDecChars(L: var TLexer, xi: var int) =
+proc handleDecChars(L: var Lexer, xi: var int) =
   while L.buf[L.bufpos] in {'0'..'9'}:
     xi = (xi * 10) + (ord(L.buf[L.bufpos]) - ord('0'))
     inc(L.bufpos)
@@ -680,58 +651,53 @@ proc addUnicodeCodePoint(s: var string, i: int) =
     s[pos+4] = chr(i shr 6 and ones(6) or 0b10_0000_00)
     s[pos+5] = chr(i and ones(6) or 0b10_0000_00)
 
-proc getEscapedChar(L: var TLexer, tok: var TToken) =
+proc getEscapedChar(L: var Lexer, tok: var Token) =
   inc(L.bufpos)               # skip '\'
   case L.buf[L.bufpos]
   of 'n', 'N':
-    if L.config.oldNewlines:
-      if tok.tokType == tkCharLit:
-        lexMessage(L, errGenerated, "\\n not allowed in character literal")
-      add(tok.literal, L.config.target.tnl)
-    else:
-      add(tok.literal, '\L')
+    tok.literal.add('\L')
     inc(L.bufpos)
   of 'p', 'P':
     if tok.tokType == tkCharLit:
       lexMessage(L, errGenerated, "\\p not allowed in character literal")
-    add(tok.literal, L.config.target.tnl)
+    tok.literal.add(L.config.target.tnl)
     inc(L.bufpos)
   of 'r', 'R', 'c', 'C':
-    add(tok.literal, CR)
+    tok.literal.add(CR)
     inc(L.bufpos)
   of 'l', 'L':
-    add(tok.literal, LF)
+    tok.literal.add(LF)
     inc(L.bufpos)
   of 'f', 'F':
-    add(tok.literal, FF)
+    tok.literal.add(FF)
     inc(L.bufpos)
   of 'e', 'E':
-    add(tok.literal, ESC)
+    tok.literal.add(ESC)
     inc(L.bufpos)
   of 'a', 'A':
-    add(tok.literal, BEL)
+    tok.literal.add(BEL)
     inc(L.bufpos)
   of 'b', 'B':
-    add(tok.literal, BACKSPACE)
+    tok.literal.add(BACKSPACE)
     inc(L.bufpos)
   of 'v', 'V':
-    add(tok.literal, VT)
+    tok.literal.add(VT)
     inc(L.bufpos)
   of 't', 'T':
-    add(tok.literal, '\t')
+    tok.literal.add('\t')
     inc(L.bufpos)
   of '\'', '\"':
-    add(tok.literal, L.buf[L.bufpos])
+    tok.literal.add(L.buf[L.bufpos])
     inc(L.bufpos)
   of '\\':
-    add(tok.literal, '\\')
+    tok.literal.add('\\')
     inc(L.bufpos)
   of 'x', 'X':
     inc(L.bufpos)
     var xi = 0
     handleHexChar(L, xi, 1)
     handleHexChar(L, xi, 2)
-    add(tok.literal, chr(xi))
+    tok.literal.add(chr(xi))
   of 'u', 'U':
     if tok.tokType == tkCharLit:
       lexMessage(L, errGenerated, "\\u not allowed in character literal")
@@ -761,22 +727,17 @@ proc getEscapedChar(L: var TLexer, tok: var TToken) =
       lexMessage(L, warnOctalEscape)
     var xi = 0
     handleDecChars(L, xi)
-    if (xi <= 255): add(tok.literal, chr(xi))
+    if (xi <= 255): tok.literal.add(chr(xi))
     else: lexMessage(L, errGenerated, "invalid character constant")
   else: lexMessage(L, errGenerated, "invalid character constant")
 
-proc newString(s: cstring, len: int): string =
-  ## XXX, how come there is no support for this?
-  result = newString(len)
-  for i in 0 ..< len:
-    result[i] = s[i]
-
-proc handleCRLF(L: var TLexer, pos: int): int =
+proc handleCRLF(L: var Lexer, pos: int): int =
   template registerLine =
     let col = L.getColNumber(pos)
 
-    if col > MaxLineLength:
-      lexMessagePos(L, hintLineTooLong, pos)
+    when not defined(nimpretty):
+      if col > MaxLineLength:
+        lexMessagePos(L, hintLineTooLong, pos)
 
   case L.buf[pos]
   of CR:
@@ -793,7 +754,7 @@ type
     raw,
     generalized
 
-proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
+proc getString(L: var Lexer, tok: var Token, mode: StringMode) =
   var pos = L.bufpos
   var line = L.lineNumber         # save linenumber for better error message
   tokenBegin(tok, pos - ord(mode == raw))
@@ -815,12 +776,12 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
           tokenEndIgnore(tok, pos+2)
           L.bufpos = pos + 3 # skip the three """
           break
-        add(tok.literal, '\"')
+        tok.literal.add('\"')
         inc(pos)
       of CR, LF:
         tokenEndIgnore(tok, pos)
         pos = handleCRLF(L, pos)
-        add(tok.literal, "\n")
+        tok.literal.add("\n")
       of nimlexbase.EndOfFile:
         tokenEndIgnore(tok, pos)
         var line2 = L.lineNumber
@@ -830,7 +791,7 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
         L.bufpos = pos
         break
       else:
-        add(tok.literal, L.buf[pos])
+        tok.literal.add(L.buf[pos])
         inc(pos)
   else:
     # ordinary string literal
@@ -841,7 +802,7 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
       if c == '\"':
         if mode != normal and L.buf[pos+1] == '\"':
           inc(pos, 2)
-          add(tok.literal, '"')
+          tok.literal.add('"')
         else:
           tokenEndIgnore(tok, pos)
           inc(pos) # skip '"'
@@ -855,16 +816,18 @@ proc getString(L: var TLexer, tok: var TToken, mode: StringMode) =
         getEscapedChar(L, tok)
         pos = L.bufpos
       else:
-        add(tok.literal, c)
+        tok.literal.add(c)
         inc(pos)
     L.bufpos = pos
 
-proc getCharacter(L: var TLexer, tok: var TToken) =
+proc getCharacter(L: var Lexer, tok: var Token) =
   tokenBegin(tok, L.bufpos)
   inc(L.bufpos)               # skip '
   var c = L.buf[L.bufpos]
   case c
-  of '\0'..pred(' '), '\'': lexMessage(L, errGenerated, "invalid character literal")
+  of '\0'..pred(' '), '\'':
+    lexMessage(L, errGenerated, "invalid character literal")
+    tok.literal = $c
   of '\\': getEscapedChar(L, tok)
   else:
     tok.literal = $c
@@ -874,7 +837,7 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
   tokenEndIgnore(tok, L.bufpos)
   inc(L.bufpos)               # skip '
 
-proc getSymbol(L: var TLexer, tok: var TToken) =
+proc getSymbol(L: var Lexer, tok: var Token) =
   var h: Hash = 0
   var pos = L.bufpos
   tokenBegin(tok, pos)
@@ -904,21 +867,21 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
       (tok.ident.id > ord(tokKeywordHigh) - ord(tkSymbol)):
     tok.tokType = tkSymbol
   else:
-    tok.tokType = TTokType(tok.ident.id + ord(tkSymbol))
+    tok.tokType = TokType(tok.ident.id + ord(tkSymbol))
     if suspicious and {optStyleHint, optStyleError} * L.config.globalOptions != {}:
       lintReport(L.config, getLineInfo(L), tok.ident.s.normalize, tok.ident.s)
   L.bufpos = pos
 
 
-proc endOperator(L: var TLexer, tok: var TToken, pos: int,
+proc endOperator(L: var Lexer, tok: var Token, pos: int,
                  hash: Hash) {.inline.} =
   var h = !$hash
   tok.ident = L.cache.getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
   if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr
-  else: tok.tokType = TTokType(tok.ident.id - oprLow + ord(tkColon))
+  else: tok.tokType = TokType(tok.ident.id - oprLow + ord(tkColon))
   L.bufpos = pos
 
-proc getOperator(L: var TLexer, tok: var TToken) =
+proc getOperator(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tokenBegin(tok, pos)
   var h: Hash = 0
@@ -938,22 +901,18 @@ proc getOperator(L: var TLexer, tok: var TToken) =
   if L.buf[pos] in {CR, LF, nimlexbase.EndOfFile}:
     tok.strongSpaceB = -1
 
-proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
+proc getPrecedence*(tok: Token): int =
   ## Calculates the precedence of the given token.
-  template considerStrongSpaces(x): untyped =
-    x + (if strongSpaces: 100 - tok.strongSpaceA.int*10 else: 0)
-
   case tok.tokType
   of tkOpr:
-    let L = tok.ident.s.len
     let relevantChar = tok.ident.s[0]
 
     # arrow like?
-    if L > 1 and tok.ident.s[L-1] == '>' and
-      tok.ident.s[L-2] in {'-', '~', '='}: return considerStrongSpaces(1)
+    if tok.ident.s.len > 1 and tok.ident.s[^1] == '>' and
+      tok.ident.s[^2] in {'-', '~', '='}: return 1
 
     template considerAsgn(value: untyped) =
-      result = if tok.ident.s[L-1] == '=': 1 else: value
+      result = if tok.ident.s[^1] == '=': 1 else: value
 
     case relevantChar
     of '$', '^': considerAsgn(10)
@@ -966,15 +925,13 @@ proc getPrecedence*(tok: TToken, strongSpaces: bool): int =
     of '?': result = 2
     else: considerAsgn(2)
   of tkDiv, tkMod, tkShl, tkShr: result = 9
-  of tkIn, tkNotin, tkIs, tkIsnot, tkOf, tkAs: result = 5
   of tkDotDot: result = 6
+  of tkIn, tkNotin, tkIs, tkIsnot, tkOf, tkAs, tkFrom: result = 5
   of tkAnd: result = 4
   of tkOr, tkXor, tkPtr, tkRef: result = 3
   else: return -10
-  result = considerStrongSpaces(result)
 
-
-proc newlineFollows*(L: TLexer): bool =
+proc newlineFollows*(L: Lexer): bool =
   var pos = L.bufpos
   while true:
     case L.buf[pos]
@@ -990,7 +947,7 @@ proc newlineFollows*(L: TLexer): bool =
     else:
       break
 
-proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
+proc skipMultiLineComment(L: var Lexer; tok: var Token; start: int;
                           isDoc: bool) =
   var pos = start
   var toStrip = 0
@@ -1055,14 +1012,14 @@ proc skipMultiLineComment(L: var TLexer; tok: var TToken; start: int;
   when defined(nimpretty):
     tok.commentOffsetB = L.offsetBase + pos - 1
 
-proc scanComment(L: var TLexer, tok: var TToken) =
+proc scanComment(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tok.tokType = tkComment
   # iNumber contains the number of '\n' in the token
   tok.iNumber = 0
   assert L.buf[pos+1] == '#'
   when defined(nimpretty):
-    tok.commentOffsetA = L.offsetBase + pos - 1
+    tok.commentOffsetA = L.offsetBase + pos
 
   if L.buf[pos+2] == '[':
     skipMultiLineComment(L, tok, pos+3, true)
@@ -1079,7 +1036,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
     var lastBackslash = -1
     while L.buf[pos] notin {CR, LF, nimlexbase.EndOfFile}:
       if L.buf[pos] == '\\': lastBackslash = pos+1
-      add(tok.literal, L.buf[pos])
+      tok.literal.add(L.buf[pos])
       inc(pos)
     tokenEndIgnore(tok, pos)
     pos = handleCRLF(L, pos)
@@ -1105,7 +1062,7 @@ proc scanComment(L: var TLexer, tok: var TToken) =
   when defined(nimpretty):
     tok.commentOffsetB = L.offsetBase + pos - 1
 
-proc skip(L: var TLexer, tok: var TToken) =
+proc skip(L: var Lexer, tok: var Token) =
   var pos = L.bufpos
   tokenBegin(tok, pos)
   tok.strongSpaceA = 0
@@ -1177,7 +1134,7 @@ proc skip(L: var TLexer, tok: var TToken) =
       tok.tokType = tkComment
       tok.indent = commentIndent
 
-proc rawGetTok*(L: var TLexer, tok: var TToken) =
+proc rawGetTok*(L: var Lexer, tok: var Token) =
   template atTokenEnd() {.dirty.} =
     when defined(nimsuggest):
       # we attach the cursor to the last *strong* token
@@ -1185,8 +1142,6 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
         L.previousToken.line = tok.line.uint16
         L.previousToken.col = tok.col.int16
 
-  when defined(nimsuggest):
-    L.cursor = CursorPosition.None
   fillToken(tok)
   if L.indentAhead >= 0:
     tok.indent = L.indentAhead
@@ -1257,7 +1212,6 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
         if L.fileIdx == L.config.m.trackPos.fileIndex and tok.col+1 == L.config.m.trackPos.col and
             tok.line == L.config.m.trackPos.line.int and L.config.ideCmd == ideSug:
           tok.tokType = tkDot
-          L.cursor = CursorPosition.InToken
           L.config.m.trackPos.col = tok.col.int16
           inc(L.bufpos)
           atTokenEnd()
@@ -1330,8 +1284,8 @@ proc rawGetTok*(L: var TLexer, tok: var TToken) =
 
 proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
                      cache: IdentCache; config: ConfigRef): int =
-  var lex: TLexer
-  var tok: TToken
+  var lex: Lexer
+  var tok: Token
   initToken(tok)
   openLexer(lex, fileIdx, inputstream, cache, config)
   var prevToken = tkEof
@@ -1345,11 +1299,11 @@ proc getIndentWidth*(fileIdx: FileIndex, inputstream: PLLStream;
 
 proc getPrecedence*(ident: PIdent): int =
   ## assumes ident is binary operator already
-  var tok: TToken
+  var tok: Token
   initToken(tok)
   tok.ident = ident
   tok.tokType =
-    if tok.ident.id in ord(tokKeywordLow) - ord(tkSymbol) .. ord(tokKeywordHigh) - ord(tkSymbol):
-      TTokType(tok.ident.id + ord(tkSymbol))
+    if tok.ident.id in ord(tokKeywordLow) - ord(tkSymbol)..ord(tokKeywordHigh) - ord(tkSymbol):
+      TokType(tok.ident.id + ord(tkSymbol))
     else: tkOpr
-  getPrecedence(tok, false)
+  getPrecedence(tok)

compiler/liftlocals.nim

@@ -30,10 +30,10 @@ proc lookupOrAdd(c: var Ctx; s: PSym; info: TLineInfo): PNode =
   let field = addUniqueField(c.objType, s, c.cache)
   var deref = newNodeI(nkHiddenDeref, info)
   deref.typ = c.objType
-  add(deref, newSymNode(c.partialParam, info))
+  deref.add(newSymNode(c.partialParam, info))
   result = newNodeI(nkDotExpr, info)
-  add(result, deref)
-  add(result, newSymNode(field))
+  result.add(deref)
+  result.add(newSymNode(field))
   result.typ = field.typ
 
 proc liftLocals(n: PNode; i: int; c: var Ctx) =
@@ -42,14 +42,14 @@ proc liftLocals(n: PNode; i: int; c: var Ctx) =
   of nkSym:
     if interestingVar(it.sym):
       n[i] = lookupOrAdd(c, it.sym, it.info)
-  of procDefs, nkTypeSection: discard
+  of procDefs, nkTypeSection, nkMixinStmt, nkBindStmt: discard
   else:
-    for i in 0 ..< it.safeLen:
+    for i in 0..<it.safeLen:
       liftLocals(it, i, c)
 
 proc lookupParam(params, dest: PNode): PSym =
   if dest.kind != nkIdent: return nil
-  for i in 1 ..< params.len:
+  for i in 1..<params.len:
     if params[i].kind == nkSym and params[i].sym.name.id == dest.ident.id:
       return params[i].sym
 

compiler/lineinfos.nim

@@ -10,10 +10,20 @@
 ## This module contains the ``TMsgKind`` enum as well as the
 ## ``TLineInfo`` object.
 
-import ropes, tables, pathutils
+import ropes, tables, pathutils, hashes
 
 const
-  explanationsBaseUrl* = "https://nim-lang.org/docs/manual"
+  explanationsBaseUrl* = "https://nim-lang.github.io/Nim"
+    # was: "https://nim-lang.org/docs" but we're now usually showing devel docs
+    # instead of latest release docs.
+
+proc createDocLink*(urlSuffix: string): string =
+  # os.`/` is not appropriate for urls.
+  result = explanationsBaseUrl
+  if urlSuffix.len > 0 and urlSuffix[0] == '/':
+    result.add urlSuffix
+  else:
+    result.add "/" & urlSuffix
 
 type
   TMsgKind* = enum
@@ -23,30 +33,42 @@ type
     errGeneralParseError,
     errNewSectionExpected,
     errInvalidDirectiveX,
+    errProveInit, # deadcode
     errGenerated,
     errUser,
-    warnCannotOpenFile,
-    warnOctalEscape, warnXIsNeverRead, warnXmightNotBeenInit,
-    warnDeprecated, warnConfigDeprecated,
-    warnSmallLshouldNotBeUsed, warnUnknownMagic, warnRedefinitionOfLabel,
-    warnUnknownSubstitutionX, warnLanguageXNotSupported,
-    warnFieldXNotSupported, warnCommentXIgnored,
-    warnTypelessParam,
-    warnUseBase, warnWriteToForeignHeap, warnUnsafeCode,
-    warnEachIdentIsTuple,
-    warnProveInit, warnProveField, warnProveIndex, warnGcUnsafe, warnGcUnsafe2,
-    warnUninit, warnGcMem, warnDestructor, warnLockLevel, warnResultShadowed,
-    warnInconsistentSpacing, warnCaseTransition, warnUser,
-    hintSuccess, hintSuccessX, hintCC,
-    hintLineTooLong, hintXDeclaredButNotUsed, hintConvToBaseNotNeeded,
-    hintConvFromXtoItselfNotNeeded, hintExprAlwaysX, hintQuitCalled,
-    hintProcessing, hintCodeBegin, hintCodeEnd, hintConf, hintPath,
-    hintConditionAlwaysTrue, hintConditionAlwaysFalse, hintName, hintPattern,
-    hintExecuting, hintLinking, hintDependency,
-    hintSource, hintPerformance, hintStackTrace, hintGCStats,
-    hintGlobalVar, hintExpandMacro,
-    hintUser, hintUserRaw,
-    hintExtendedContext
+
+    warnCannotOpenFile = "CannotOpenFile", warnOctalEscape = "OctalEscape",
+    warnXIsNeverRead = "XIsNeverRead", warnXmightNotBeenInit = "XmightNotBeenInit",
+    warnDeprecated = "Deprecated", warnConfigDeprecated = "ConfigDeprecated",
+    warnSmallLshouldNotBeUsed = "SmallLshouldNotBeUsed", warnUnknownMagic = "UnknownMagic",
+    warnRedefinitionOfLabel = "RedefinitionOfLabel", warnUnknownSubstitutionX = "UnknownSubstitutionX",
+    warnLanguageXNotSupported = "LanguageXNotSupported", warnFieldXNotSupported = "FieldXNotSupported",
+    warnCommentXIgnored = "CommentXIgnored", warnTypelessParam = "TypelessParam",
+    warnUseBase = "UseBase", warnWriteToForeignHeap = "WriteToForeignHeap",
+    warnUnsafeCode = "UnsafeCode", warnUnusedImportX = "UnusedImport",
+    warnInheritFromException = "InheritFromException", warnEachIdentIsTuple = "EachIdentIsTuple",
+    warnUnsafeSetLen = "UnsafeSetLen", warnUnsafeDefault = "UnsafeDefault",
+    warnProveInit = "ProveInit", warnProveField = "ProveField", warnProveIndex = "ProveIndex",
+    warnUnreachableElse = "UnreachableElse", warnUnreachableCode = "UnreachableCode",
+    warnStaticIndexCheck = "IndexCheck", warnGcUnsafe = "GcUnsafe", warnGcUnsafe2 = "GcUnsafe2",
+    warnUninit = "Uninit", warnGcMem = "GcMem", warnDestructor = "Destructor",
+    warnLockLevel = "LockLevel", warnResultShadowed = "ResultShadowed",
+    warnInconsistentSpacing = "Spacing",  warnCaseTransition = "CaseTransition",
+    warnCycleCreated = "CycleCreated", warnObservableStores = "ObservableStores",
+    warnUser = "User",
+
+    hintSuccess = "Success", hintSuccessX = "SuccessX", hintCC = "CC",
+    hintLineTooLong = "LineTooLong", hintXDeclaredButNotUsed = "XDeclaredButNotUsed",
+    hintXCannotRaiseY = "XCannotRaiseY", hintConvToBaseNotNeeded = "ConvToBaseNotNeeded",
+    hintConvFromXtoItselfNotNeeded = "ConvFromXtoItselfNotNeeded", hintExprAlwaysX = "ExprAlwaysX",
+    hintQuitCalled = "QuitCalled", hintProcessing = "Processing", hintCodeBegin = "CodeBegin",
+    hintCodeEnd = "CodeEnd", hintConf = "Conf", hintPath = "Path",
+    hintConditionAlwaysTrue = "CondTrue", hintConditionAlwaysFalse = "CondFalse", hintName = "Name",
+    hintPattern = "Pattern", hintExecuting = "Exec", hintLinking = "Link", hintDependency = "Dependency",
+    hintSource = "Source", hintPerformance = "Performance", hintStackTrace = "StackTrace",
+    hintGCStats = "GCStats", hintGlobalVar = "GlobalVar", hintExpandMacro = "ExpandMacro",
+    hintUser = "User", hintUserRaw = "UserRaw", hintExtendedContext = "ExtendedContext",
+    hintMsgOrigin = "MsgOrigin", # since 1.3.5
 
 const
   MsgKindToStr*: array[TMsgKind, string] = [
@@ -59,6 +81,7 @@ const
     errGeneralParseError: "general parse error",
     errNewSectionExpected: "new section expected",
     errInvalidDirectiveX: "invalid directive: '$1'",
+    errProveInit: "Cannot prove that '$1' is initialized.",  # deadcode
     errGenerated: "$1",
     errUser: "$1",
     warnCannotOpenFile: "cannot open '$1'",
@@ -78,25 +101,38 @@ const
     warnUseBase: "use {.base.} for base methods; baseless methods are deprecated",
     warnWriteToForeignHeap: "write to foreign heap",
     warnUnsafeCode: "unsafe code: '$1'",
+    warnUnusedImportX: "imported and not used: '$1'",
+    warnInheritFromException: "inherit from a more precise exception type like ValueError, " &
+      "IOError or OSError. If these don't suit, inherit from CatchableError or Defect.",
     warnEachIdentIsTuple: "each identifier is a tuple",
+    warnUnsafeSetLen: "setLen can potentially expand the sequence, " &
+                      "but the element type '$1' doesn't have a valid default value",
+    warnUnsafeDefault: "The '$1' type doesn't have a valid default value",
     warnProveInit: "Cannot prove that '$1' is initialized. This will become a compile time error in the future.",
     warnProveField: "cannot prove that field '$1' is accessible",
     warnProveIndex: "cannot prove index '$1' is valid",
+    warnUnreachableElse: "unreachable else, all cases are already covered",
+    warnUnreachableCode: "unreachable code after 'return' statement or '{.noReturn.}' proc",
+    warnStaticIndexCheck: "$1",
     warnGcUnsafe: "not GC-safe: '$1'",
     warnGcUnsafe2: "$1",
-    warnUninit: "'$1' might not have been initialized",
+    warnUninit: "use explicit initialization of '$1' for clarity",
     warnGcMem: "'$1' uses GC'ed memory",
     warnDestructor: "usage of a type with a destructor in a non destructible context. This will become a compile time error in the future.",
     warnLockLevel: "$1",
     warnResultShadowed: "Special variable 'result' is shadowed.",
     warnInconsistentSpacing: "Number of spaces around '$#' is not consistent",
     warnCaseTransition: "Potential object case transition, instantiate new object instead",
+    warnCycleCreated: "$1",
+    warnObservableStores: "observable stores to '$1'",
     warnUser: "$1",
     hintSuccess: "operation successful: $#",
-    hintSuccessX: "operation successful ($# lines compiled; $# sec total; $#; $#)",
-    hintCC: "CC: \'$1\'", # unused
+    # keep in sync with `testament.isSuccess`
+    hintSuccessX: "${loc} lines; ${sec}s; $mem; $build build; proj: $project; out: $output",
+    hintCC: "CC: $1",
     hintLineTooLong: "line too long",
     hintXDeclaredButNotUsed: "'$1' is declared but not used",
+    hintXCannotRaiseY: "$1",
     hintConvToBaseNotNeeded: "conversion to base object is not needed",
     hintConvFromXtoItselfNotNeeded: "conversion from $1 to itself is pointless",
     hintExprAlwaysX: "expression evaluates always to '$1'",
@@ -111,7 +147,7 @@ const
     hintName: "$1",
     hintPattern: "$1",
     hintExecuting: "$1",
-    hintLinking: "",
+    hintLinking: "$1",
     hintDependency: "$1",
     hintSource: "$1",
     hintPerformance: "$1",
@@ -122,29 +158,7 @@ const
     hintUser: "$1",
     hintUserRaw: "$1",
     hintExtendedContext: "$1",
-  ]
-
-const
-  WarningsToStr* = ["CannotOpenFile", "OctalEscape",
-    "XIsNeverRead", "XmightNotBeenInit",
-    "Deprecated", "ConfigDeprecated",
-    "SmallLshouldNotBeUsed", "UnknownMagic",
-    "RedefinitionOfLabel", "UnknownSubstitutionX",
-    "LanguageXNotSupported", "FieldXNotSupported",
-    "CommentXIgnored",
-    "TypelessParam", "UseBase", "WriteToForeignHeap",
-    "UnsafeCode", "EachIdentIsTuple",
-    "ProveInit", "ProveField", "ProveIndex", "GcUnsafe", "GcUnsafe2", "Uninit",
-    "GcMem", "Destructor", "LockLevel", "ResultShadowed",
-    "Spacing", "CaseTransition", "User"]
-
-  HintsToStr* = [
-    "Success", "SuccessX", "CC", "LineTooLong",
-    "XDeclaredButNotUsed", "ConvToBaseNotNeeded", "ConvFromXtoItselfNotNeeded",
-    "ExprAlwaysX", "QuitCalled", "Processing", "CodeBegin", "CodeEnd", "Conf",
-    "Path", "CondTrue", "CondFalse", "Name", "Pattern", "Exec", "Link", "Dependency",
-    "Source", "Performance", "StackTrace", "GCStats", "GlobalVar", "ExpandMacro",
-    "User", "UserRaw", "ExtendedContext",
+    hintMsgOrigin: "$1",
   ]
 
 const
@@ -157,20 +171,16 @@ const
   hintMin* = hintSuccess
   hintMax* = high(TMsgKind)
 
-static:
-  doAssert HintsToStr.len == ord(hintMax) - ord(hintMin) + 1
-  doAssert WarningsToStr.len == ord(warnMax) - ord(warnMin) + 1
-
 type
   TNoteKind* = range[warnMin..hintMax] # "notes" are warnings or hints
   TNoteKinds* = set[TNoteKind]
 
 proc computeNotesVerbosity(): array[0..3, TNoteKinds] =
-  result[3] = {low(TNoteKind)..high(TNoteKind)} - {}
+  result[3] = {low(TNoteKind)..high(TNoteKind)} - {warnObservableStores}
   result[2] = result[3] - {hintStackTrace, warnUninit, hintExtendedContext}
   result[1] = result[2] - {warnProveField, warnProveIndex,
     warnGcUnsafe, hintPath, hintDependency, hintCodeBegin, hintCodeEnd,
-    hintSource, hintGlobalVar, hintGCStats}
+    hintSource, hintGlobalVar, hintGCStats, hintMsgOrigin}
   result[0] = result[1] - {hintSuccessX, hintSuccess, hintConf,
     hintProcessing, hintPattern, hintExecuting, hintLinking, hintCC}
 
@@ -221,27 +231,28 @@ type
   TErrorOutputs* = set[TErrorOutput]
 
   ERecoverableError* = object of ValueError
-  ESuggestDone* = object of Exception
+  ESuggestDone* = object of ValueError
 
 proc `==`*(a, b: FileIndex): bool {.borrow.}
 
+proc hash*(i: TLineInfo): Hash =
+  hash (i.line.int, i.col.int, i.fileIndex.int)
+
 proc raiseRecoverableError*(msg: string) {.noinline.} =
   raise newException(ERecoverableError, msg)
 
 const
   InvalidFileIdx* = FileIndex(-1)
-
-proc unknownLineInfo*(): TLineInfo =
-  result.line = uint16(0)
-  result.col = int16(-1)
-  result.fileIndex = InvalidFileIdx
+  unknownLineInfo* = TLineInfo(line: 0, col: -1, fileIndex: InvalidFileIdx)
 
 type
   Severity* {.pure.} = enum ## VS Code only supports these three
     Hint, Warning, Error
 
-const trackPosInvalidFileIdx* = FileIndex(-2) # special marker so that no suggestions
-                                   # are produced within comments and string literals
+const
+  trackPosInvalidFileIdx* = FileIndex(-2) # special marker so that no suggestions
+                                          # are produced within comments and string literals
+  commandLineIdx* = FileIndex(-3)
 
 type
   MsgConfig* = object ## does not need to be stored in the incremental cache
@@ -259,7 +270,7 @@ type
 
 proc initMsgConfig*(): MsgConfig =
   result.msgContext = @[]
-  result.lastError = unknownLineInfo()
+  result.lastError = unknownLineInfo
   result.filenameToIndexTbl = initTable[string, FileIndex]()
   result.fileInfos = @[]
   result.errorOutputs = {eStdOut, eStdErr}

compiler/linter.nim

@@ -37,7 +37,7 @@ proc beautifyName(s: string, k: TSymKind): string =
     # Types should start with a capital unless builtins like 'int' etc.:
     if s =~ ["int", "uint", "cint", "cuint", "clong", "cstring", "string",
              "char", "byte", "bool", "openArray", "seq", "array", "void",
-             "pointer", "float", "csize", "cdouble", "cchar", "cschar",
+             "pointer", "float", "csize", "csize_t", "cdouble", "cchar", "cschar",
              "cshort", "cu", "nil", "typedesc", "auto", "any",
              "range", "openarray", "varargs", "set", "cfloat", "ref", "ptr",
              "untyped", "typed", "static", "sink", "lent", "type", "owned"]:
@@ -71,13 +71,9 @@ proc beautifyName(s: string, k: TSymKind): string =
 
 proc differ*(line: string, a, b: int, x: string): string =
   proc substrEq(s: string, pos, last: int, substr: string): bool =
-    var i = 0
-    var length = substr.len
-    while i < length and pos+i <= last and s[pos+i] == substr[i]:
-      inc i
-    return i == length
-
-  let last = min(b, line.len)
+    result = true
+    for i in 0..<substr.len:
+      if pos+i > last or s[pos+i] != substr[i]: return false
 
   result = ""
   if not substrEq(line, a, b, x):
@@ -85,11 +81,6 @@ proc differ*(line: string, a, b: int, x: string): string =
     if cmpIgnoreStyle(y, x) == 0:
       result = y
 
-proc checkStyle(conf: ConfigRef; cache: IdentCache; info: TLineInfo, s: string, k: TSymKind; sym: PSym) =
-  let beau = beautifyName(s, k)
-  if s != beau:
-    lintReport(conf, info, beau, s)
-
 proc nep1CheckDefImpl(conf: ConfigRef; info: TLineInfo; s: PSym; k: TSymKind) =
   # operators stay as they are:
   if k in {skResult, skTemp} or s.name.s[0] notin Letters: return
@@ -137,6 +128,6 @@ proc styleCheckUse*(conf: ConfigRef; info: TLineInfo; s: PSym) =
     lintReport(conf, info, newName, oldName)
 
 proc checkPragmaUse*(conf: ConfigRef; info: TLineInfo; w: TSpecialWord; pragmaName: string) =
-  let wanted = canonPragmaSpelling(w)
+  let wanted = $w
   if pragmaName != wanted:
     lintReport(conf, info, wanted, pragmaName)

compiler/llstream.nim

@@ -12,9 +12,11 @@
 import
   pathutils
 
-# support '-d:useGnuReadline' for backwards compatibility:
-when not defined(windows) and (defined(useGnuReadline) or defined(useLinenoise)):
-  import rdstdin
+# support `useGnuReadline`, `useLinenoise` for backwards compatibility
+const hasRstdin = (defined(nimUseLinenoise) or defined(useLinenoise) or defined(useGnuReadline)) and
+  not defined(windows)
+
+when hasRstdin: import rdstdin
 
 type
   TLLRepl* = proc (s: PLLStream, buf: pointer, bufLen: int): int
@@ -109,12 +111,12 @@ proc llReadFromStdin(s: PLLStream, buf: pointer, bufLen: int): int =
   var line = newStringOfCap(120)
   var triples = 0
   while readLineFromStdin(if s.s.len == 0: ">>> " else: "... ", line):
-    add(s.s, line)
-    add(s.s, "\n")
+    s.s.add(line)
+    s.s.add("\n")
     inc triples, countTriples(line)
     if not continueLine(line, (triples and 1) == 1): break
   inc(s.lineOffset)
-  result = min(bufLen, len(s.s) - s.rd)
+  result = min(bufLen, s.s.len - s.rd)
   if result > 0:
     copyMem(buf, addr(s.s[s.rd]), result)
     inc(s.rd, result)
@@ -124,7 +126,7 @@ proc llStreamRead*(s: PLLStream, buf: pointer, bufLen: int): int =
   of llsNone:
     result = 0
   of llsString:
-    result = min(bufLen, len(s.s) - s.rd)
+    result = min(bufLen, s.s.len - s.rd)
     if result > 0:
       copyMem(buf, addr(s.s[0 + s.rd]), result)
       inc(s.rd, result)
@@ -139,7 +141,7 @@ proc llStreamReadLine*(s: PLLStream, line: var string): bool =
   of llsNone:
     result = true
   of llsString:
-    while s.rd < len(s.s):
+    while s.rd < s.s.len:
       case s.s[s.rd]
       of '\x0D':
         inc(s.rd)
@@ -149,9 +151,9 @@ proc llStreamReadLine*(s: PLLStream, line: var string): bool =
         inc(s.rd)
         break
       else:
-        add(line, s.s[s.rd])
+        line.add(s.s[s.rd])
         inc(s.rd)
-    result = line.len > 0 or s.rd < len(s.s)
+    result = line.len > 0 or s.rd < s.s.len
   of llsFile:
     result = readLine(s.f, line)
   of llsStdIn:
@@ -162,8 +164,8 @@ proc llStreamWrite*(s: PLLStream, data: string) =
   of llsNone, llsStdIn:
     discard
   of llsString:
-    add(s.s, data)
-    inc(s.wr, len(data))
+    s.s.add(data)
+    inc(s.wr, data.len)
   of llsFile:
     write(s.f, data)
 
@@ -177,7 +179,7 @@ proc llStreamWrite*(s: PLLStream, data: char) =
   of llsNone, llsStdIn:
     discard
   of llsString:
-    add(s.s, data)
+    s.s.add(data)
     inc(s.wr)
   of llsFile:
     c = data
@@ -189,7 +191,7 @@ proc llStreamWrite*(s: PLLStream, buf: pointer, buflen: int) =
     discard
   of llsString:
     if buflen > 0:
-      setLen(s.s, len(s.s) + buflen)
+      setLen(s.s, s.s.len + buflen)
       copyMem(addr(s.s[0 + s.wr]), buf, buflen)
       inc(s.wr, buflen)
   of llsFile:
@@ -204,7 +206,7 @@ proc llStreamReadAll*(s: PLLStream): string =
   of llsString:
     if s.rd == 0: result = s.s
     else: result = substr(s.s, s.rd)
-    s.rd = len(s.s)
+    s.rd = s.s.len
   of llsFile:
     result = newString(bufSize)
     var bytes = readBuffer(s.f, addr(result[0]), bufSize)

compiler/lookups.nim

@@ -36,11 +36,11 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
   of nkAccQuoted:
     case n.len
     of 0: handleError(n, origin)
-    of 1: result = considerQuotedIdent(c, n.sons[0], origin)
+    of 1: result = considerQuotedIdent(c, n[0], origin)
     else:
       var id = ""
       for i in 0..<n.len:
-        let x = n.sons[i]
+        let x = n[i]
         case x.kind
         of nkIdent: id.add(x.ident.s)
         of nkSym: id.add(x.sym.name.s)
@@ -49,7 +49,7 @@ proc considerQuotedIdent*(c: PContext; n: PNode, origin: PNode = nil): PIdent =
       result = getIdent(c.cache, id)
   of nkOpenSymChoice, nkClosedSymChoice:
     if n[0].kind == nkSym:
-      result = n.sons[0].sym.name
+      result = n[0].sym.name
     else:
       handleError(n, origin)
   else:
@@ -91,8 +91,15 @@ proc skipAlias*(s: PSym; n: PNode; conf: ConfigRef): PSym =
       message(conf, n.info, warnDeprecated, "use " & result.name.s & " instead; " &
               s.name.s & " is deprecated")
 
+proc isShadowScope*(s: PScope): bool {.inline.} = s.parent != nil and s.parent.depthLevel == s.depthLevel
+
 proc localSearchInScope*(c: PContext, s: PIdent): PSym =
-  result = strTableGet(c.currentScope.symbols, s)
+  var scope = c.currentScope
+  result = strTableGet(scope.symbols, s)
+  while result == nil and scope.isShadowScope:
+    # We are in a shadow scope, check in the parent too
+    scope = scope.parent
+    result = strTableGet(scope.symbols, s)
 
 proc searchInScopes*(c: PContext, s: PIdent): PSym =
   for scope in walkScopes(c.currentScope):
@@ -100,16 +107,15 @@ proc searchInScopes*(c: PContext, s: PIdent): PSym =
     if result != nil: return
   result = nil
 
-when declared(echo):
-  proc debugScopes*(c: PContext; limit=0) {.deprecated.} =
-    var i = 0
-    for scope in walkScopes(c.currentScope):
-      echo "scope ", i
-      for h in 0 .. high(scope.symbols.data):
-        if scope.symbols.data[h] != nil:
-          echo scope.symbols.data[h].name.s
-      if i == limit: break
-      inc i
+proc debugScopes*(c: PContext; limit=0) {.deprecated.} =
+  var i = 0
+  for scope in walkScopes(c.currentScope):
+    echo "scope ", i
+    for h in 0..high(scope.symbols.data):
+      if scope.symbols.data[h] != nil:
+        echo scope.symbols.data[h].name.s
+    if i == limit: break
+    inc i
 
 proc searchInScopes*(c: PContext, s: PIdent, filter: TSymKinds): PSym =
   for scope in walkScopes(c.currentScope):
@@ -124,7 +130,7 @@ proc errorSym*(c: PContext, n: PNode): PSym =
   ## creates an error symbol to avoid cascading errors (for IDE support)
   var m = n
   # ensure that 'considerQuotedIdent' can't fail:
-  if m.kind == nkDotExpr: m = m.sons[1]
+  if m.kind == nkDotExpr: m = m[1]
   let ident = if m.kind in {nkIdent, nkSym, nkAccQuoted}:
       considerQuotedIdent(c, m)
     else:
@@ -232,8 +238,25 @@ proc addInterfaceOverloadableSymAt*(c: PContext, scope: PScope, sym: PSym) =
   addOverloadableSymAt(c, scope, sym)
   addInterfaceDeclAux(c, sym)
 
+proc openShadowScope*(c: PContext) =
+  c.currentScope = PScope(parent: c.currentScope,
+                          symbols: newStrTable(),
+                          depthLevel: c.scopeDepth)
+
+proc closeShadowScope*(c: PContext) =
+  c.closeScope
+
+proc mergeShadowScope*(c: PContext) =
+  let shadowScope = c.currentScope
+  c.rawCloseScope
+  for sym in shadowScope.symbols:
+    if sym.kind in OverloadableSyms:
+      c.addInterfaceOverloadableSymAt(c.currentScope, sym)
+    else:
+      c.addInterfaceDecl(sym)
+
 when defined(nimfix):
-  # when we cannot find the identifier, retry with a changed identifer:
+  # when we cannot find the identifier, retry with a changed identifier:
   proc altSpelling(x: PIdent): PIdent =
     case x.s[0]
     of 'A'..'Z': result = getIdent(toLowerAscii(x.s[0]) & x.s.substr(1))
@@ -317,8 +340,7 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
       fixSpelling(n, ident, searchInScopes)
       errorUndeclaredIdentifier(c, n.info, ident.s)
       result = errorSym(c, n)
-    elif checkAmbiguity in flags and result != nil and
-        contains(c.ambiguousSymbols, result.id):
+    elif checkAmbiguity in flags and result != nil and result.id in c.ambiguousSymbols:
       errorUseQualifier(c, n.info, result)
   of nkSym:
     result = n.sym
@@ -326,29 +348,29 @@ proc qualifiedLookUp*(c: PContext, n: PNode, flags: set[TLookupFlag]): PSym =
       errorUseQualifier(c, n.info, n.sym)
   of nkDotExpr:
     result = nil
-    var m = qualifiedLookUp(c, n.sons[0], (flags*{checkUndeclared})+{checkModule})
+    var m = qualifiedLookUp(c, n[0], (flags*{checkUndeclared})+{checkModule})
     if m != nil and m.kind == skModule:
       var ident: PIdent = nil
-      if n.sons[1].kind == nkIdent:
-        ident = n.sons[1].ident
-      elif n.sons[1].kind == nkAccQuoted:
-        ident = considerQuotedIdent(c, n.sons[1])
+      if n[1].kind == nkIdent:
+        ident = n[1].ident
+      elif n[1].kind == nkAccQuoted:
+        ident = considerQuotedIdent(c, n[1])
       if ident != nil:
         if m == c.module:
           result = strTableGet(c.topLevelScope.symbols, ident).skipAlias(n, c.config)
         else:
           result = strTableGet(m.tab, ident).skipAlias(n, c.config)
         if result == nil and checkUndeclared in flags:
-          fixSpelling(n.sons[1], ident, searchInScopes)
-          errorUndeclaredIdentifier(c, n.sons[1].info, ident.s)
-          result = errorSym(c, n.sons[1])
-      elif n.sons[1].kind == nkSym:
-        result = n.sons[1].sym
+          fixSpelling(n[1], ident, searchInScopes)
+          errorUndeclaredIdentifier(c, n[1].info, ident.s)
+          result = errorSym(c, n[1])
+      elif n[1].kind == nkSym:
+        result = n[1].sym
       elif checkUndeclared in flags and
-           n.sons[1].kind notin {nkOpenSymChoice, nkClosedSymChoice}:
-        localError(c.config, n.sons[1].info, "identifier expected, but got: " &
-                   renderTree(n.sons[1]))
-        result = errorSym(c, n.sons[1])
+           n[1].kind notin {nkOpenSymChoice, nkClosedSymChoice}:
+        localError(c.config, n[1].info, "identifier expected, but got: " &
+                   renderTree(n[1]))
+        result = errorSym(c, n[1])
   else:
     result = nil
   when false:
@@ -372,13 +394,13 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
     o.mode = oimDone
   of nkDotExpr:
     o.mode = oimOtherModule
-    o.m = qualifiedLookUp(c, n.sons[0], {checkUndeclared, checkModule})
+    o.m = qualifiedLookUp(c, n[0], {checkUndeclared, checkModule})
     if o.m != nil and o.m.kind == skModule:
       var ident: PIdent = nil
-      if n.sons[1].kind == nkIdent:
-        ident = n.sons[1].ident
-      elif n.sons[1].kind == nkAccQuoted:
-        ident = considerQuotedIdent(c, n.sons[1], n)
+      if n[1].kind == nkIdent:
+        ident = n[1].ident
+      elif n[1].kind == nkAccQuoted:
+        ident = considerQuotedIdent(c, n[1], n)
       if ident != nil:
         if o.m == c.module:
           # a module may access its private members:
@@ -388,12 +410,12 @@ proc initOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
         else:
           result = initIdentIter(o.it, o.m.tab, ident).skipAlias(n, c.config)
       else:
-        noidentError(c.config, n.sons[1], n)
-        result = errorSym(c, n.sons[1])
+        noidentError(c.config, n[1], n)
+        result = errorSym(c, n[1])
   of nkClosedSymChoice, nkOpenSymChoice:
     o.mode = oimSymChoice
     if n[0].kind == nkSym:
-      result = n.sons[0].sym
+      result = n[0].sym
     else:
       o.mode = oimDone
       return nil
@@ -430,8 +452,8 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
   of oimOtherModule:
     result = nextIdentIter(o.it, o.m.tab).skipAlias(n, c.config)
   of oimSymChoice:
-    if o.symChoiceIndex < sonsLen(n):
-      result = n.sons[o.symChoiceIndex].sym
+    if o.symChoiceIndex < n.len:
+      result = n[o.symChoiceIndex].sym
       incl(o.inSymChoice, result.id)
       inc o.symChoiceIndex
     elif n.kind == nkOpenSymChoice:
@@ -439,19 +461,19 @@ proc nextOverloadIter*(o: var TOverloadIter, c: PContext, n: PNode): PSym =
       o.mode = oimSymChoiceLocalLookup
       o.scope = c.currentScope
       result = firstIdentExcluding(o.it, o.scope.symbols,
-                                   n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
       while result == nil:
         o.scope = o.scope.parent
         if o.scope == nil: break
         result = firstIdentExcluding(o.it, o.scope.symbols,
-                                     n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                     n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
   of oimSymChoiceLocalLookup:
     result = nextIdentExcluding(o.it, o.scope.symbols, o.inSymChoice).skipAlias(n, c.config)
     while result == nil:
       o.scope = o.scope.parent
       if o.scope == nil: break
       result = firstIdentExcluding(o.it, o.scope.symbols,
-                                   n.sons[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
+                                   n[0].sym.name, o.inSymChoice).skipAlias(n, c.config)
 
   when false:
     if result != nil and result.kind == skStub: loadStub(result)

compiler/lowerings.nim

@@ -16,41 +16,55 @@ import ast, astalgo, types, idents, magicsys, msgs, options, modulegraphs,
   lineinfos
 
 proc newDeref*(n: PNode): PNode {.inline.} =
-  result = newNodeIT(nkHiddenDeref, n.info, n.typ.sons[0])
-  addSon(result, n)
+  result = newNodeIT(nkHiddenDeref, n.info, n.typ[0])
+  result.add n
 
 proc newTupleAccess*(g: ModuleGraph; tup: PNode, i: int): PNode =
   if tup.kind == nkHiddenAddr:
-    result = newNodeIT(nkHiddenAddr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar}))
-    result.addSon(newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar}).sons[i]))
-    addSon(result[0], tup[0])
+    result = newNodeIT(nkHiddenAddr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar, tyLent}))
+    result.add newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(abstractInst+{tyPtr, tyVar, tyLent})[i])
+    result[0].add tup[0]
     var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
     lit.intVal = i
-    addSon(result[0], lit)
+    result[0].add lit
   else:
     result = newNodeIT(nkBracketExpr, tup.info, tup.typ.skipTypes(
-                       abstractInst).sons[i])
-    addSon(result, copyTree(tup))
+                       abstractInst)[i])
+    result.add copyTree(tup)
     var lit = newNodeIT(nkIntLit, tup.info, getSysType(g, tup.info, tyInt))
     lit.intVal = i
-    addSon(result, lit)
+    result.add lit
 
 proc addVar*(father, v: PNode) =
   var vpart = newNodeI(nkIdentDefs, v.info, 3)
-  vpart.sons[0] = v
-  vpart.sons[1] = newNodeI(nkEmpty, v.info)
-  vpart.sons[2] = vpart[1]
-  addSon(father, vpart)
+  vpart[0] = v
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = vpart[1]
+  father.add vpart
+
+proc addVar*(father, v, value: PNode) =
+  var vpart = newNodeI(nkIdentDefs, v.info, 3)
+  vpart[0] = v
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = value
+  father.add vpart
 
 proc newAsgnStmt*(le, ri: PNode): PNode =
   result = newNodeI(nkAsgn, le.info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
 
 proc newFastAsgnStmt*(le, ri: PNode): PNode =
   result = newNodeI(nkFastAsgn, le.info, 2)
-  result.sons[0] = le
-  result.sons[1] = ri
+  result[0] = le
+  result[1] = ri
+
+proc newFastMoveStmt*(g: ModuleGraph, le, ri: PNode): PNode =
+  result = newNodeI(nkFastAsgn, le.info, 2)
+  result[0] = le
+  result[1] = newNodeIT(nkCall, ri.info, ri.typ)
+  result[1].add newSymNode(getSysMagic(g, ri.info, "move", mMove))
+  result[1].add ri
 
 proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   assert n.kind == nkVarTuple
@@ -63,13 +77,13 @@ proc lowerTupleUnpacking*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
 
   var v = newNodeI(nkVarSection, value.info)
   let tempAsNode = newSymNode(temp)
-  v.addVar(tempAsNode)
+  v.addVar(tempAsNode, value)
   result.add(v)
 
-  result.add newAsgnStmt(tempAsNode, value)
-  for i in 0 .. n.len-3:
-    if n.sons[i].kind == nkSym: v.addVar(n.sons[i])
-    result.add newAsgnStmt(n.sons[i], newTupleAccess(g, tempAsNode, i))
+  for i in 0..<n.len-2:
+    let val = newTupleAccess(g, tempAsNode, i)
+    if n[i].kind == nkSym: v.addVar(n[i], val)
+    else: result.add newAsgnStmt(n[i], val)
 
 proc evalOnce*(g: ModuleGraph; value: PNode; owner: PSym): PNode =
   ## Turns (value) into (let tmp = value; tmp) so that 'value' can be re-used
@@ -89,10 +103,10 @@ proc evalOnce*(g: ModuleGraph; value: PNode; owner: PSym): PNode =
 
 proc newTupleAccessRaw*(tup: PNode, i: int): PNode =
   result = newNodeI(nkBracketExpr, tup.info)
-  addSon(result, copyTree(tup))
+  result.add copyTree(tup)
   var lit = newNodeI(nkIntLit, tup.info)
   lit.intVal = i
-  addSon(result, lit)
+  result.add lit
 
 proc newTryFinally*(body, final: PNode): PNode =
   result = newTree(nkHiddenTryStmt, body, newTree(nkFinally, final))
@@ -106,31 +120,32 @@ proc lowerTupleUnpackingForAsgn*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   let tempAsNode = newSymNode(temp) #newIdentNode(getIdent(genPrefix & $temp.id), value.info)
 
   var vpart = newNodeI(nkIdentDefs, tempAsNode.info, 3)
-  vpart.sons[0] = tempAsNode
-  vpart.sons[1] = newNodeI(nkEmpty, value.info)
-  vpart.sons[2] = value
-  addSon(v, vpart)
+  vpart[0] = tempAsNode
+  vpart[1] = newNodeI(nkEmpty, value.info)
+  vpart[2] = value
+  v.add vpart
   result.add(v)
 
-  let lhs = n.sons[0]
-  for i in 0 .. lhs.len-1:
-    result.add newAsgnStmt(lhs.sons[i], newTupleAccessRaw(tempAsNode, i))
+  let lhs = n[0]
+  for i in 0..<lhs.len:
+    result.add newAsgnStmt(lhs[i], newTupleAccessRaw(tempAsNode, i))
 
 proc lowerSwap*(g: ModuleGraph; n: PNode; owner: PSym): PNode =
   result = newNodeI(nkStmtList, n.info)
   # note: cannot use 'skTemp' here cause we really need the copy for the VM :-(
   var temp = newSym(skVar, getIdent(g.cache, genPrefix), owner, n.info, owner.options)
-  temp.typ = n.sons[1].typ
+  temp.typ = n[1].typ
   incl(temp.flags, sfFromGeneric)
+  incl(temp.flags, sfGenSym)
 
   var v = newNodeI(nkVarSection, n.info)
   let tempAsNode = newSymNode(temp)
 
   var vpart = newNodeI(nkIdentDefs, v.info, 3)
-  vpart.sons[0] = tempAsNode
-  vpart.sons[1] = newNodeI(nkEmpty, v.info)
-  vpart.sons[2] = n[1]
-  addSon(v, vpart)
+  vpart[0] = tempAsNode
+  vpart[1] = newNodeI(nkEmpty, v.info)
+  vpart[2] = n[1]
+  v.add vpart
 
   result.add(v)
   result.add newFastAsgnStmt(n[1], n[2])
@@ -144,52 +159,59 @@ proc createObj*(g: ModuleGraph; owner: PSym, info: TLineInfo; final=true): PType
   else:
     rawAddSon(result, getCompilerProc(g, "RootObj").typ)
   result.n = newNodeI(nkRecList, info)
-  let s = newSym(skType, getIdent(g.cache, "Env_" & toFilename(g.config, info)),
+  let s = newSym(skType, getIdent(g.cache, "Env_" & toFilename(g.config, info) & "_" & $owner.name.s),
                   owner, info, owner.options)
   incl s.flags, sfAnon
   s.typ = result
   result.sym = s
 
+template fieldCheck {.dirty.} =
+  when false:
+    if tfCheckedForDestructor in obj.flags:
+      echo "missed field ", field.name.s
+      writeStackTrace()
+
 proc rawAddField*(obj: PType; field: PSym) =
   assert field.kind == skField
-  field.position = sonsLen(obj.n)
-  addSon(obj.n, newSymNode(field))
+  field.position = obj.n.len
+  obj.n.add newSymNode(field)
   propagateToOwner(obj, field.typ)
+  fieldCheck()
 
 proc rawIndirectAccess*(a: PNode; field: PSym; info: TLineInfo): PNode =
   # returns a[].field as a node
   assert field.kind == skField
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
-  addSon(deref, a)
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc rawDirectAccess*(obj, field: PSym): PNode =
   # returns a.field as a node
   assert field.kind == skField
   result = newNodeI(nkDotExpr, field.info)
-  addSon(result, newSymNode obj)
-  addSon(result, newSymNode field)
+  result.add newSymNode(obj)
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc lookupInRecord(n: PNode, id: int): PSym =
   result = nil
   case n.kind
   of nkRecList:
-    for i in 0 ..< sonsLen(n):
-      result = lookupInRecord(n.sons[i], id)
+    for i in 0..<n.len:
+      result = lookupInRecord(n[i], id)
       if result != nil: return
   of nkRecCase:
-    if n.sons[0].kind != nkSym: return
-    result = lookupInRecord(n.sons[0], id)
+    if n[0].kind != nkSym: return
+    result = lookupInRecord(n[0], id)
     if result != nil: return
-    for i in 1 ..< sonsLen(n):
-      case n.sons[i].kind
+    for i in 1..<n.len:
+      case n[i].kind
       of nkOfBranch, nkElse:
-        result = lookupInRecord(lastSon(n.sons[i]), id)
+        result = lookupInRecord(lastSon(n[i]), id)
         if result != nil: return
       else: discard
   of nkSym:
@@ -206,8 +228,10 @@ proc addField*(obj: PType; s: PSym; cache: IdentCache) =
   field.typ = t
   assert t.kind != tyTyped
   propagateToOwner(obj, t)
-  field.position = sonsLen(obj.n)
-  addSon(obj.n, newSymNode(field))
+  field.position = obj.n.len
+  field.flags = s.flags * {sfCursor}
+  obj.n.add newSymNode(field)
+  fieldCheck()
 
 proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache): PSym {.discardable.} =
   result = lookupInRecord(obj.n, s.id)
@@ -219,45 +243,45 @@ proc addUniqueField*(obj: PType; s: PSym; cache: IdentCache): PSym {.discardable
     field.typ = t
     assert t.kind != tyTyped
     propagateToOwner(obj, t)
-    field.position = sonsLen(obj.n)
-    addSon(obj.n, newSymNode(field))
+    field.position = obj.n.len
+    obj.n.add newSymNode(field)
     result = field
 
 proc newDotExpr*(obj, b: PSym): PNode =
   result = newNodeI(nkDotExpr, obj.info)
   let field = lookupInRecord(obj.typ.n, b.id)
   assert field != nil, b.name.s
-  addSon(result, newSymNode(obj))
-  addSon(result, newSymNode(field))
+  result.add newSymNode(obj)
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc indirectAccess*(a: PNode, b: int, info: TLineInfo): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
   while true:
     assert t.kind == tyObject
     field = lookupInRecord(t.n, b)
     if field != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
   #if field == nil:
   #  echo "FIELD ", b
   #  debug deref.typ
   assert field != nil
-  addSon(deref, a)
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc indirectAccess*(a: PNode, b: string, info: TLineInfo; cache: IdentCache): PNode =
   # returns a[].b as a node
   var deref = newNodeI(nkHiddenDeref, info)
-  deref.typ = a.typ.skipTypes(abstractInst).sons[0]
+  deref.typ = a.typ.skipTypes(abstractInst)[0]
   var t = deref.typ.skipTypes(abstractInst)
   var field: PSym
   let bb = getIdent(cache, b)
@@ -265,17 +289,17 @@ proc indirectAccess*(a: PNode, b: string, info: TLineInfo; cache: IdentCache): P
     assert t.kind == tyObject
     field = getSymFromList(t.n, bb)
     if field != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
   #if field == nil:
   #  echo "FIELD ", b
   #  debug deref.typ
   assert field != nil
-  addSon(deref, a)
+  deref.add a
   result = newNodeI(nkDotExpr, info)
-  addSon(result, deref)
-  addSon(result, newSymNode(field))
+  result.add deref
+  result.add newSymNode(field)
   result.typ = field.typ
 
 proc getFieldFromObj*(t: PType; v: PSym): PSym =
@@ -285,7 +309,7 @@ proc getFieldFromObj*(t: PType; v: PSym): PSym =
     assert t.kind == tyObject
     result = lookupInRecord(t.n, v.id)
     if result != nil: break
-    t = t.sons[0]
+    t = t[0]
     if t == nil: break
     t = t.skipTypes(skipPtrs)
 
@@ -296,19 +320,19 @@ proc indirectAccess*(a: PNode, b: PSym, info: TLineInfo): PNode =
 proc indirectAccess*(a, b: PSym, info: TLineInfo): PNode =
   result = indirectAccess(newSymNode(a), b, info)
 
-proc genAddrOf*(n: PNode): PNode =
+proc genAddrOf*(n: PNode, typeKind = tyPtr): PNode =
   result = newNodeI(nkAddr, n.info, 1)
-  result.sons[0] = n
-  result.typ = newType(tyPtr, n.typ.owner)
+  result[0] = n
+  result.typ = newType(typeKind, n.typ.owner)
   result.typ.rawAddSon(n.typ)
 
 proc genDeref*(n: PNode; k = nkHiddenDeref): PNode =
   result = newNodeIT(k, n.info,
-                     n.typ.skipTypes(abstractInst).sons[0])
+                     n.typ.skipTypes(abstractInst)[0])
   result.add n
 
 proc callCodegenProc*(g: ModuleGraph; name: string;
-                      info: TLineInfo = unknownLineInfo();
+                      info: TLineInfo = unknownLineInfo;
                       arg1, arg2, arg3, optionalArgs: PNode = nil): PNode =
   result = newNodeI(nkCall, info)
   let sym = magicsys.getCompilerProc(g, name)
@@ -320,9 +344,9 @@ proc callCodegenProc*(g: ModuleGraph; name: string;
     if arg2 != nil: result.add arg2
     if arg3 != nil: result.add arg3
     if optionalArgs != nil:
-      for i in 1..optionalArgs.len-3:
+      for i in 1..<optionalArgs.len-2:
         result.add optionalArgs[i]
-    result.typ = sym.typ.sons[0]
+    result.typ = sym.typ[0]
 
 proc newIntLit*(g: ModuleGraph; info: TLineInfo; value: BiggestInt): PNode =
   result = nkIntLit.newIntNode(value)
@@ -330,30 +354,19 @@ proc newIntLit*(g: ModuleGraph; info: TLineInfo; value: BiggestInt): PNode =
 
 proc genHigh*(g: ModuleGraph; n: PNode): PNode =
   if skipTypes(n.typ, abstractVar).kind == tyArray:
-    result = newIntLit(g, n.info, lastOrd(g.config, skipTypes(n.typ, abstractVar)))
+    result = newIntLit(g, n.info, toInt64(lastOrd(g.config, skipTypes(n.typ, abstractVar))))
   else:
     result = newNodeI(nkCall, n.info, 2)
     result.typ = getSysType(g, n.info, tyInt)
-    result.sons[0] = newSymNode(getSysMagic(g, n.info, "high", mHigh))
-    result.sons[1] = n
+    result[0] = newSymNode(getSysMagic(g, n.info, "high", mHigh))
+    result[1] = n
 
 proc genLen*(g: ModuleGraph; n: PNode): PNode =
   if skipTypes(n.typ, abstractVar).kind == tyArray:
-    result = newIntLit(g, n.info, lastOrd(g.config, skipTypes(n.typ, abstractVar)) + 1)
+    result = newIntLit(g, n.info, toInt64(lastOrd(g.config, skipTypes(n.typ, abstractVar)) + 1))
   else:
     result = newNodeI(nkCall, n.info, 2)
     result.typ = getSysType(g, n.info, tyInt)
-    result.sons[0] = newSymNode(getSysMagic(g, n.info, "len", mLengthSeq))
-    result.sons[1] = n
+    result[0] = newSymNode(getSysMagic(g, n.info, "len", mLengthSeq))
+    result[1] = n
 
-proc hoistExpr*(varSection, expr: PNode, name: PIdent, owner: PSym): PSym =
-  result = newSym(skLet, name, owner, varSection.info, owner.options)
-  result.flags.incl sfHoisted
-  result.typ = expr.typ
-
-  var varDef = newNodeI(nkIdentDefs, varSection.info, 3)
-  varDef.sons[0] = newSymNode(result)
-  varDef.sons[1] = newNodeI(nkEmpty, varSection.info)
-  varDef.sons[2] = expr
-
-  varSection.add varDef

compiler/magicsys.nim

@@ -40,7 +40,7 @@ proc getSysMagic*(g: ModuleGraph; info: TLineInfo; name: string, m: TMagic): PSy
   while r != nil:
     if r.magic == m:
       # prefer the tyInt variant:
-      if r.typ.sons[0] != nil and r.typ.sons[0].kind == tyInt: return r
+      if r.typ[0] != nil and r.typ[0].kind == tyInt: return r
       result = r
     r = nextIdentIter(ti, g.systemModule.tab)
   if result != nil: return result
@@ -68,7 +68,7 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
     of tyUInt64: result = sysTypeFromName("uint64")
     of tyFloat: result = sysTypeFromName("float")
     of tyFloat32: result = sysTypeFromName("float32")
-    of tyFloat64: return sysTypeFromName("float64")
+    of tyFloat64: result = sysTypeFromName("float64")
     of tyFloat128: result = sysTypeFromName("float128")
     of tyBool: result = sysTypeFromName("bool")
     of tyChar: result = sysTypeFromName("char")
@@ -79,7 +79,9 @@ proc getSysType*(g: ModuleGraph; info: TLineInfo; kind: TTypeKind): PType =
     else: internalError(g.config, "request for typekind: " & $kind)
     g.sysTypes[kind] = result
   if result.kind != kind:
-    internalError(g.config, "wanted: " & $kind & " got: " & $result.kind)
+    if kind == tyFloat64 and result.kind == tyFloat: discard # because of aliasing
+    else:
+      internalError(g.config, "wanted: " & $kind & " got: " & $result.kind)
   if result == nil: internalError(g.config, "type not found: " & $kind)
 
 proc resetSysTypes*(g: ModuleGraph) =
@@ -123,7 +125,7 @@ proc addSonSkipIntLit*(father, son: PType) =
   when not defined(nimNoNilSeqs):
     if isNil(father.sons): father.sons = @[]
   let s = son.skipIntLit
-  add(father.sons, s)
+  father.sons.add(s)
   propagateToOwner(father, s)
 
 proc setIntLitType*(g: ModuleGraph; result: PNode) =
@@ -175,3 +177,28 @@ proc getNimScriptSymbol*(g: ModuleGraph; name: string): PSym =
   strTableGet(g.exposed, getIdent(g.cache, name))
 
 proc resetNimScriptSymbols*(g: ModuleGraph) = initStrTable(g.exposed)
+
+proc getMagicEqSymForType*(g: ModuleGraph; t: PType; info: TLineInfo): PSym =
+  case t.kind
+  of tyInt,  tyInt8, tyInt16, tyInt32, tyInt64,
+     tyUInt, tyUInt8, tyUInt16, tyUInt32, tyUInt64: 
+    result = getSysMagic(g, info, "==", mEqI)
+  of tyEnum: 
+    result = getSysMagic(g, info, "==", mEqEnum)
+  of tyBool: 
+    result = getSysMagic(g, info, "==", mEqB)
+  of tyRef, tyPtr, tyPointer: 
+    result = getSysMagic(g, info, "==", mEqRef)
+  of tyString:
+    result = getSysMagic(g, info, "==", mEqStr)
+  of tyChar:
+    result = getSysMagic(g, info, "==", mEqCh)
+  of tySet:
+    result = getSysMagic(g, info, "==", mEqSet)
+  of tyProc:
+    result = getSysMagic(g, info, "==", mEqProc)
+  else:
+    globalError(g.config, info,
+      "can't find magic equals operator for type kind " & $t.kind)
+
+

compiler/main.nim

@@ -13,13 +13,13 @@ when not defined(nimcore):
   {.error: "nimcore MUST be defined for Nim's core tooling".}
 
 import
-  llstream, strutils, ast, lexer, syntaxes, options, msgs,
+  llstream, strutils, os, ast, lexer, syntaxes, options, msgs,
   condsyms, times,
   sem, idents, passes, extccomp,
   cgen, json, nversion,
   platform, nimconf, passaux, depends, vm, idgen,
-  parser, modules,
-  modulegraphs, tables, rod, lineinfos, pathutils
+  modules,
+  modulegraphs, tables, rod, lineinfos, pathutils, vmprofiler
 
 when not defined(leanCompiler):
   import jsgen, docgen, docgen2
@@ -51,7 +51,7 @@ proc commandGenDepend(graph: ModuleGraph) =
       ' ' & changeFileExt(project, "dot").string)
 
 proc commandCheck(graph: ModuleGraph) =
-  graph.config.errorMax = high(int)  # do not stop after first error
+  graph.config.setErrorMaxHighMaybe
   defineSymbol(graph.config.symbols, "nimcheck")
   semanticPasses(graph)  # use an empty backend for semantic checking only
   compileProject(graph)
@@ -59,7 +59,7 @@ proc commandCheck(graph: ModuleGraph) =
 when not defined(leanCompiler):
   proc commandDoc2(graph: ModuleGraph; json: bool) =
     handleDocOutputOptions graph.config
-    graph.config.errorMax = high(int)  # do not stop after first error
+    graph.config.setErrorMaxHighMaybe
     semanticPasses(graph)
     if json: registerPass(graph, docgen2JsonPass)
     else: registerPass(graph, docgen2Pass)
@@ -68,20 +68,21 @@ when not defined(leanCompiler):
 
 proc commandCompileToC(graph: ModuleGraph) =
   let conf = graph.config
-
-  if conf.outDir.isEmpty:
-    conf.outDir = conf.projectPath
-  if conf.outFile.isEmpty:
-    let targetName = if optGenDynLib in conf.globalOptions:
-      platform.OS[conf.target.targetOS].dllFrmt % conf.projectName
-    else:
-      conf.projectName & platform.OS[conf.target.targetOS].exeExt
-    conf.outFile = RelativeFile targetName
-
+  setOutFile(conf)
   extccomp.initVars(conf)
   semanticPasses(graph)
   registerPass(graph, cgenPass)
 
+  if {optRun, optForceFullMake} * conf.globalOptions == {optRun} or isDefined(conf, "nimBetterRun"):
+    let proj = changeFileExt(conf.projectFull, "")
+    if not changeDetectedViaJsonBuildInstructions(conf, proj):
+      # nothing changed
+      graph.config.notes = graph.config.mainPackageNotes
+      return
+
+  if not extccomp.ccHasSaneOverflow(conf):
+    conf.symbols.defineSymbol("nimEmulateOverflowChecks")
+
   compileProject(graph)
   if graph.config.errorCounter > 0:
     return # issue #9933
@@ -98,12 +99,13 @@ proc commandJsonScript(graph: ModuleGraph) =
   let proj = changeFileExt(graph.config.projectFull, "")
   extccomp.runJsonBuildInstructions(graph.config, proj)
 
-when not defined(leanCompiler):
-  proc commandCompileToJS(graph: ModuleGraph) =
+proc commandCompileToJS(graph: ModuleGraph) =
+  when defined(leanCompiler):
+    globalError(graph.config, unknownLineInfo, "compiler wasn't built with JS code generator")
+  else:
     let conf = graph.config
+    conf.exc = excCpp
 
-    if conf.outDir.isEmpty:
-      conf.outDir = conf.projectPath
     if conf.outFile.isEmpty:
       conf.outFile = RelativeFile(conf.projectName & ".js")
 
@@ -111,7 +113,6 @@ when not defined(leanCompiler):
     setTarget(graph.config.target, osJS, cpuJS)
     #initDefines()
     defineSymbol(graph.config.symbols, "ecmascript") # For backward compatibility
-    defineSymbol(graph.config.symbols, "js")
     semanticPasses(graph)
     registerPass(graph, JSgenPass)
     compileProject(graph)
@@ -128,7 +129,7 @@ proc interactivePasses(graph: ModuleGraph) =
   registerPass(graph, evalPass)
 
 proc commandInteractive(graph: ModuleGraph) =
-  graph.config.errorMax = high(int)  # do not stop after first error
+  graph.config.setErrorMaxHighMaybe
   interactivePasses(graph)
   compileSystemModule(graph)
   if graph.config.commandArgs.len > 0:
@@ -143,21 +144,13 @@ const evalPasses = [verbosePass, semPass, evalPass]
 proc evalNim(graph: ModuleGraph; nodes: PNode, module: PSym) =
   carryPasses(graph, nodes, module, evalPasses)
 
-proc commandEval(graph: ModuleGraph; exp: string) =
-  if graph.systemModule == nil:
-    interactivePasses(graph)
-    compileSystemModule(graph)
-  let echoExp = "echo \"eval\\t\", " & "repr(" & exp & ")"
-  evalNim(graph, echoExp.parseString(graph.cache, graph.config),
-    makeStdinModule(graph))
-
 proc commandScan(cache: IdentCache, config: ConfigRef) =
   var f = addFileExt(AbsoluteFile mainCommandArg(config), NimExt)
   var stream = llStreamOpen(f, fmRead)
   if stream != nil:
     var
-      L: TLexer
-      tok: TToken
+      L: Lexer
+      tok: Token
     initToken(tok)
     openLexer(L, f, stream, cache, config)
     while true:
@@ -181,56 +174,105 @@ proc mainCommand*(graph: ModuleGraph) =
   conf.lastCmdTime = epochTime()
   conf.searchPaths.add(conf.libpath)
   setId(100)
-  case conf.command.normalize
-  of "c", "cc", "compile", "compiletoc":
-    # compile means compileToC currently
-    conf.cmd = cmdCompileToC
-    defineSymbol(graph.config.symbols, "c")
-    commandCompileToC(graph)
-  of "cpp", "compiletocpp":
-    conf.cmd = cmdCompileToCpp
-    defineSymbol(graph.config.symbols, "cpp")
-    commandCompileToC(graph)
-  of "objc", "compiletooc":
-    conf.cmd = cmdCompileToOC
-    defineSymbol(graph.config.symbols, "objc")
-    commandCompileToC(graph)
-  of "run":
-    conf.cmd = cmdRun
-    when hasTinyCBackend:
-      extccomp.setCC("tcc")
-      commandCompileToC(graph)
-    else:
-      rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
-  of "js", "compiletojs":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with JS code generator"
-    else:
-      conf.cmd = cmdCompileToJS
+
+  proc customizeForBackend(backend: TBackend) =
+    ## Sets backend specific options but don't compile to backend yet in
+    ## case command doesn't require it. This must be called by all commands.
+    if conf.backend == backendInvalid:
+      # only set if wasn't already set, to allow override via `nim c -b:cpp`
+      conf.backend = backend
+
+    defineSymbol(graph.config.symbols, $conf.backend)
+    case conf.backend
+    of backendC:
+      if conf.exc == excNone: conf.exc = excSetjmp
+    of backendCpp:
+      if conf.exc == excNone: conf.exc = excCpp
+    of backendObjc: discard
+    of backendJs:
       if conf.hcrOn:
         # XXX: At the moment, system.nim cannot be compiled in JS mode
         # with "-d:useNimRtl". The HCR option has been processed earlier
         # and it has added this define implictly, so we must undo that here.
         # A better solution might be to fix system.nim
         undefSymbol(conf.symbols, "useNimRtl")
-      commandCompileToJS(graph)
-  of "doc0":
+    of backendInvalid: doAssert false
+    if conf.selectedGC in {gcArc, gcOrc} and conf.backend != backendCpp:
+      conf.exc = excGoto
+
+  var commandAlreadyProcessed = false
+
+  proc compileToBackend(backend: TBackend, cmd = cmdCompileToBackend) =
+    commandAlreadyProcessed = true
+    conf.cmd = cmd
+    customizeForBackend(backend)
+    case conf.backend
+    of backendC: commandCompileToC(graph)
+    of backendCpp: commandCompileToC(graph)
+    of backendObjc: commandCompileToC(graph)
+    of backendJs: commandCompileToJS(graph)
+    of backendInvalid: doAssert false
+
+  template docLikeCmd(body) =
     when defined(leanCompiler):
       quit "compiler wasn't built with documentation generator"
     else:
       wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandDoc(cache, conf)
-  of "doc2", "doc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
       conf.cmd = cmdDoc
       loadConfigs(DocConfig, cache, conf)
       defineSymbol(conf.symbols, "nimdoc")
+      body
+
+  block: ## command prepass
+    var docLikeCmd2 = false # includes what calls `docLikeCmd` + some more
+    case conf.command.normalize
+    of "r": conf.globalOptions.incl {optRun, optUseNimcache}
+    of "doc0",  "doc2", "doc", "rst2html", "rst2tex", "jsondoc0", "jsondoc2",
+      "jsondoc", "ctags", "buildindex": docLikeCmd2 = true
+    else: discard
+    if conf.outDir.isEmpty:
+      # doc like commands can generate a lot of files (especially with --project)
+      # so by default should not end up in $PWD nor in $projectPath.
+      conf.outDir = block:
+        var ret = if optUseNimcache in conf.globalOptions: getNimcacheDir(conf)
+                  else: conf.projectPath
+        doAssert ret.string.isAbsolute # `AbsoluteDir` is not a real guarantee
+        if docLikeCmd2: ret = ret / htmldocsDir
+        ret
+
+  ## process all backend commands
+  case conf.command.normalize
+  of "c", "cc", "compile", "compiletoc": compileToBackend(backendC) # compile means compileToC currently
+  of "cpp", "compiletocpp": compileToBackend(backendCpp)
+  of "objc", "compiletooc": compileToBackend(backendObjc)
+  of "js", "compiletojs": compileToBackend(backendJs)
+  of "r": compileToBackend(backendC) # different from `"run"`!
+  of "run":
+    when hasTinyCBackend:
+      extccomp.setCC(conf, "tcc", unknownLineInfo)
+      if conf.backend notin {backendC, backendInvalid}:
+        rawMessage(conf, errGenerated, "'run' requires c backend, got: '$1'" % $conf.backend)
+      compileToBackend(backendC, cmd = cmdRun)
+    else:
+      rawMessage(conf, errGenerated, "'run' command not available; rebuild with -d:tinyc")
+  else: customizeForBackend(backendC) # fallback for other commands
+
+  ## process all other commands
+  case conf.command.normalize # synchronize with `cmdUsingHtmlDocs`
+  of "doc0": docLikeCmd commandDoc(cache, conf)
+  of "doc2", "doc":
+    docLikeCmd():
+      conf.setNoteDefaults(warnLockLevel, false) # issue #13218
+      conf.setNoteDefaults(warnRedefinitionOfLabel, false) # issue #13218
+        # because currently generates lots of false positives due to conflation
+        # of labels links in doc comments, e.g. for random.rand:
+        #  ## * `rand proc<#rand,Rand,Natural>`_ that returns an integer
+        #  ## * `rand proc<#rand,Rand,range[]>`_ that returns a float
       commandDoc2(graph, false)
+      if optGenIndex in conf.globalOptions and optWholeProject in conf.globalOptions:
+        commandBuildIndex(conf, $conf.outDir)
   of "rst2html":
+    conf.setNoteDefaults(warnRedefinitionOfLabel, false) # similar to issue #13218
     when defined(leanCompiler):
       quit "compiler wasn't built with documentation generator"
     else:
@@ -244,41 +286,10 @@ proc mainCommand*(graph: ModuleGraph) =
       conf.cmd = cmdRst2tex
       loadConfigs(DocTexConfig, cache, conf)
       commandRst2TeX(cache, conf)
-  of "jsondoc0":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandJson(cache, conf)
-  of "jsondoc2", "jsondoc":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      wantMainModule(conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandDoc2(graph, true)
-  of "ctags":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      wantMainModule(conf)
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      defineSymbol(conf.symbols, "nimdoc")
-      commandTags(cache, conf)
-  of "buildindex":
-    when defined(leanCompiler):
-      quit "compiler wasn't built with documentation generator"
-    else:
-      conf.cmd = cmdDoc
-      loadConfigs(DocConfig, cache, conf)
-      commandBuildIndex(cache, conf)
+  of "jsondoc0": docLikeCmd commandJson(cache, conf)
+  of "jsondoc2", "jsondoc": docLikeCmd commandDoc2(graph, true)
+  of "ctags": docLikeCmd commandTags(cache, conf)
+  of "buildindex": docLikeCmd commandBuildIndex(conf, $conf.projectFull, conf.outFile)
   of "gendepend":
     conf.cmd = cmdGenDepend
     commandGenDepend(graph)
@@ -291,22 +302,26 @@ proc mainCommand*(graph: ModuleGraph) =
       for s in definedSymbolNames(conf.symbols): definedSymbols.elems.add(%s)
 
       var libpaths = newJArray()
+      var lazyPaths = newJArray()
       for dir in conf.searchPaths: libpaths.elems.add(%dir.string)
+      for dir in conf.lazyPaths: lazyPaths.elems.add(%dir.string)
 
       var hints = newJObject() # consider factoring with `listHints`
       for a in hintMin..hintMax:
-        let key = lineinfos.HintsToStr[ord(a) - ord(hintMin)]
-        hints[key] = %(a in conf.notes)
+        hints[$a] = %(a in conf.notes)
       var warnings = newJObject()
       for a in warnMin..warnMax:
-        let key = lineinfos.WarningsToStr[ord(a) - ord(warnMin)]
-        warnings[key] = %(a in conf.notes)
+        warnings[$a] = %(a in conf.notes)
 
       var dumpdata = %[
         (key: "version", val: %VersionAsString),
+        (key: "nimExe", val: %(getAppFilename())),
+        (key: "prefixdir", val: %conf.getPrefixDir().string),
+        (key: "libpath", val: %conf.libpath.string),
         (key: "project_path", val: %conf.projectFull.string),
         (key: "defined_symbols", val: definedSymbols),
         (key: "lib_paths", val: %libpaths),
+        (key: "lazyPaths", val: %lazyPaths),
         (key: "outdir", val: %conf.outDir.string),
         (key: "out", val: %conf.outFile.string),
         (key: "nimcache", val: %getNimcacheDir(conf).string),
@@ -338,29 +353,53 @@ proc mainCommand*(graph: ModuleGraph) =
     conf.cmd = cmdInteractive
     commandInteractive(graph)
   of "e":
-    incl conf.globalOptions, optWasNimscript
-    commandEval(graph, mainCommandArg(conf))
+    if not fileExists(conf.projectFull):
+      rawMessage(conf, errGenerated, "NimScript file does not exist: " & conf.projectFull.string)
+    elif not conf.projectFull.string.endsWith(".nims"):
+      rawMessage(conf, errGenerated, "not a NimScript file: " & conf.projectFull.string)
+    # main NimScript logic handled in cmdlinehelper.nim.
   of "nop", "help":
     # prevent the "success" message:
     conf.cmd = cmdDump
   of "jsonscript":
     conf.cmd = cmdJsonScript
+    setOutFile(graph.config)
     commandJsonScript(graph)
+  elif commandAlreadyProcessed: discard # already handled
   else:
     rawMessage(conf, errGenerated, "invalid command: " & conf.command)
 
   if conf.errorCounter == 0 and
      conf.cmd notin {cmdInterpret, cmdRun, cmdDump}:
-    when declared(system.getMaxMem):
-      let usedMem = formatSize(getMaxMem()) & " peakmem"
+    let mem =
+      when declared(system.getMaxMem): formatSize(getMaxMem()) & " peakmem"
+      else: formatSize(getTotalMem()) & " totmem"
+    let loc = $conf.linesCompiled
+    let build = if isDefined(conf, "danger"): "Dangerous Release"
+                elif isDefined(conf, "release"): "Release"
+                else: "Debug"
+    let sec = formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3)
+    let project = if optListFullPaths in conf.globalOptions: $conf.projectFull else: $conf.projectName
+
+    var output: string
+    if optCompileOnly in conf.globalOptions and conf.cmd != cmdJsonScript:
+      output = $conf.jsonBuildFile
+    elif conf.outFile.isEmpty and conf.cmd notin {cmdJsonScript, cmdCompileToBackend, cmdDoc}:
+      # for some cmd we expect a valid absOutFile
+      output = "unknownOutput"
     else:
-      let usedMem = formatSize(getTotalMem())
-    rawMessage(conf, hintSuccessX, [$conf.linesCompiled,
-               formatFloat(epochTime() - conf.lastCmdTime, ffDecimal, 3),
-               usedMem,
-               if isDefined(conf, "danger"): "Dangerous Release Build"
-               elif isDefined(conf, "release"): "Release Build"
-               else: "Debug Build"])
+      output = $conf.absOutFile
+    if optListFullPaths notin conf.globalOptions: output = output.AbsoluteFile.extractFilename
+    if optProfileVM in conf.globalOptions:
+      echo conf.dump(conf.vmProfileData)
+    rawMessage(conf, hintSuccessX, [
+      "loc", loc,
+      "sec", sec,
+      "mem", mem,
+      "build", build,
+      "project", project,
+      "output", output,
+      ])
 
   when PrintRopeCacheStats:
     echo "rope cache stats: "
@@ -369,5 +408,3 @@ proc mainCommand*(graph: ModuleGraph) =
     echo "  int tries: ", gCacheIntTries
     echo "  efficiency: ", formatFloat(1-(gCacheMisses.float/gCacheTries.float),
                                        ffDecimal, 3)
-
-  resetAttributes(conf)

compiler/modulegraphs.nim

@@ -64,7 +64,7 @@ type
     importModuleCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PSym {.nimcall.}
     includeFileCallback*: proc (graph: ModuleGraph; m: PSym, fileIdx: FileIndex): PNode {.nimcall.}
     recordStmt*: proc (graph: ModuleGraph; m: PSym; n: PNode) {.nimcall.}
-    cacheSeqs*: Table[string, PNode] # state that is shared to suppor the 'macrocache' API
+    cacheSeqs*: Table[string, PNode] # state that is shared to support the 'macrocache' API
     cacheCounters*: Table[string, BiggestInt]
     cacheTables*: Table[string, BTree[string, PNode]]
     passes*: seq[TPass]
@@ -72,6 +72,8 @@ type
     onDefinitionResolveForward*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     onUsage*: proc (graph: ModuleGraph; s: PSym; info: TLineInfo) {.nimcall.}
     globalDestructors*: seq[PNode]
+    strongSemCheck*: proc (graph: ModuleGraph; owner: PSym; body: PNode) {.nimcall.}
+    compatibleProps*: proc (graph: ModuleGraph; formal, actual: PType): bool {.nimcall.}
 
   TPassContext* = object of RootObj # the pass's context
   PPassContext* = ref TPassContext
@@ -98,7 +100,7 @@ const
 proc toBase64a(s: cstring, len: int): string =
   ## encodes `s` into base64 representation.
   result = newStringOfCap(((len + 2) div 3) * 4)
-  result.add '_'
+  result.add "__"
   var i = 0
   while i < len - 2:
     let a = ord(s[i])
@@ -163,8 +165,9 @@ proc stopCompile*(g: ModuleGraph): bool {.inline.} =
   result = g.doStopCompile != nil and g.doStopCompile()
 
 proc createMagic*(g: ModuleGraph; name: string, m: TMagic): PSym =
-  result = newSym(skProc, getIdent(g.cache, name), nil, unknownLineInfo(), {})
+  result = newSym(skProc, getIdent(g.cache, name), nil, unknownLineInfo, {})
   result.magic = m
+  result.flags = {sfNeverRaises}
 
 proc newModuleGraph*(cache: IdentCache; config: ConfigRef): ModuleGraph =
   result = ModuleGraph()
@@ -215,7 +218,7 @@ proc addDep*(g: ModuleGraph; m: PSym, dep: FileIndex) =
   addModuleDep(g.incr, g.config, m.info.fileIndex, dep, isIncludeFile = false)
   if g.suggestMode:
     g.deps.incl m.position.dependsOn(dep.int)
-    # we compute the transitive closure later when quering the graph lazily.
+    # we compute the transitive closure later when querying the graph lazily.
     # this improves efficiency quite a lot:
     #invalidTransitiveClosure = true
 

compiler/modulepaths.nim

@@ -91,8 +91,7 @@ when false:
             if result.len > 0: return result
 
   proc scriptableImport(pkg, sub: string; info: TLineInfo): string =
-    result = resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info)
-    if result.isNil: result = ""
+    resolveDollar(gProjectFull, info.toFullPath(), pkg, sub, info)
 
   proc lookupPackage(pkg, subdir: PNode): string =
     let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: ""
@@ -112,8 +111,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
   case n.kind
   of nkStrLit, nkRStrLit, nkTripleStrLit:
     try:
-      result =
-        pathSubs(conf, n.strVal, toFullPath(conf, n.info).splitFile().dir)
+      result = pathSubs(conf, n.strVal, toFullPath(conf, n.info).splitFile().dir)
     except ValueError:
       localError(conf, n.info, "invalid path: " & n.strVal)
       result = n.strVal
@@ -139,7 +137,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
       result.add modname
   of nkPrefix:
     when false:
-      if n.sons[0].kind == nkIdent and n.sons[0].ident.s == "$":
+      if n[0].kind == nkIdent and n[0].ident.s == "$":
         result = lookupPackage(n[1], nil)
       else:
         discard
@@ -149,7 +147,7 @@ proc getModuleName*(conf: ConfigRef; n: PNode): string =
     localError(conf, n.info, warnDeprecated, "using '.' instead of '/' in import paths is deprecated")
     result = renderTree(n, {renderNoComments}).replace(".", "/")
   of nkImportAs:
-    result = getModuleName(conf, n.sons[0])
+    result = getModuleName(conf, n[0])
   else:
     localError(conf, n.info, "invalid module name: '$1'" % n.renderTree)
     result = ""

compiler/modules.nim

@@ -17,32 +17,43 @@ import
 proc resetSystemArtifacts*(g: ModuleGraph) =
   magicsys.resetSysTypes(g)
 
-proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
+template getModuleIdent(graph: ModuleGraph, filename: AbsoluteFile): PIdent =
+  getIdent(graph.cache, splitFile(filename).name)
+
+proc getPackage(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+  ## returns package symbol (skPackage) for yet to be defined module for fileIdx
+  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
+  let name = getModuleIdent(graph, filename)
+  let info = newLineInfo(fileIdx, 1, 1)
   let
     pck = getPackageName(graph.config, filename.string)
     pck2 = if pck.len > 0: pck else: "unknown"
     pack = getIdent(graph.cache, pck2)
   var packSym = graph.packageSyms.strTableGet(pack)
   if packSym == nil:
-    packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, result.info)
+    packSym = newSym(skPackage, getIdent(graph.cache, pck2), nil, info)
     initStrTable(packSym.tab)
     graph.packageSyms.strTableAdd(packSym)
   else:
-    let existing = strTableGet(packSym.tab, result.name)
-    if existing != nil and existing.info.fileIndex != result.info.fileIndex:
+    let existing = strTableGet(packSym.tab, name)
+    if existing != nil and existing.info.fileIndex != info.fileIndex:
       when false:
         # we used to produce an error:
-        localError(graph.config, result.info,
+        localError(graph.config, info,
           "module names need to be unique per Nimble package; module clashes with " &
             toFullPath(graph.config, existing.info.fileIndex))
       else:
         # but starting with version 0.20 we now produce a fake Nimble package instead
         # to resolve the conflicts:
         let pck3 = fakePackageName(graph.config, filename)
-        packSym = newSym(skPackage, getIdent(graph.cache, pck3), nil, result.info)
+        # this makes the new `packSym`'s owner be the original `packSym`
+        packSym = newSym(skPackage, getIdent(graph.cache, pck3), packSym, info)
         initStrTable(packSym.tab)
         graph.packageSyms.strTableAdd(packSym)
+  result = packSym
 
+proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; filename: AbsoluteFile) =
+  let packSym = getPackage(graph, fileIdx)
   result.owner = packSym
   result.position = int fileIdx
 
@@ -50,25 +61,24 @@ proc partialInitModule(result: PSym; graph: ModuleGraph; fileIdx: FileIndex; fil
     setLen(graph.modules, int(fileIdx) + 1)
   graph.modules[result.position] = result
 
-  incl(result.flags, sfUsed)
   initStrTable(result.tab)
   strTableAdd(result.tab, result) # a module knows itself
   strTableAdd(packSym.tab, result)
 
 proc newModule(graph: ModuleGraph; fileIdx: FileIndex): PSym =
+  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
   # We cannot call ``newSym`` here, because we have to circumvent the ID
   # mechanism, which we do in order to assign each module a persistent ID.
-  new(result)
-  result.id = -1             # for better error checking
-  result.kind = skModule
-  let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
-  result.name = getIdent(graph.cache, splitFile(filename).name)
+  result = PSym(kind: skModule, id: -1, # for better error checking
+                name: getModuleIdent(graph, filename),
+                info: newLineInfo(fileIdx, 1, 1))
   if not isNimIdentifier(result.name.s):
     rawMessage(graph.config, errGenerated, "invalid module name: " & result.name.s)
-  result.info = newLineInfo(fileIdx, 1, 1)
   partialInitModule(result, graph, fileIdx, filename)
 
 proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): PSym =
+  var flags = flags
+  if fileIdx == graph.config.projectMainIdx2: flags.incl sfMainModule
   result = graph.getModule(fileIdx)
   if result == nil:
     let filename = AbsoluteFile toFullPath(graph.config, fileIdx)
@@ -76,7 +86,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
     result = r
     if result == nil:
       result = newModule(graph, fileIdx)
-      result.flags = result.flags + flags
+      result.flags.incl flags
       result.id = id
       registerModule(graph, result)
     else:
@@ -94,7 +104,7 @@ proc compileModule*(graph: ModuleGraph; fileIdx: FileIndex; flags: TSymFlags): P
       if sfMainModule in flags and graph.config.projectIsStdin: stdin.llStreamOpen else: nil)
     graph.markClientsDirty(fileIdx)
 
-proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym {.procvar.} =
+proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym =
   # this is called by the semantic checking phase
   assert graph.config != nil
   result = compileModule(graph, fileIdx, {})
@@ -106,10 +116,10 @@ proc importModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PSym {.proc
   #  localError(result.info, errAttemptToRedefine, result.name.s)
   # restore the notes for outer module:
   graph.config.notes =
-    if s.owner.id == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
+    if s.getnimblePkgId == graph.config.mainPackageId or isDefined(graph.config, "booting"): graph.config.mainPackageNotes
     else: graph.config.foreignPackageNotes
 
-proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode {.procvar.} =
+proc includeModule*(graph: ModuleGraph; s: PSym, fileIdx: FileIndex): PNode =
   result = syntaxes.parseFile(fileIdx, graph.cache, graph.config)
   graph.addDep(s, fileIdx)
   graph.addIncludeDep(s.position.FileIndex, fileIdx)
@@ -127,7 +137,7 @@ proc compileSystemModule*(graph: ModuleGraph) =
 
 proc wantMainModule*(conf: ConfigRef) =
   if conf.projectFull.isEmpty:
-    fatal(conf, newLineInfo(conf, AbsoluteFile"command line", 1, 1), errGenerated,
+    fatal(conf, newLineInfo(conf, AbsoluteFile(commandLineDesc), 1, 1), errGenerated,
         "command expects a filename")
   conf.projectMainIdx = fileInfoIdx(conf, addFileExt(conf.projectFull, NimExt))
 
@@ -137,7 +147,12 @@ proc compileProject*(graph: ModuleGraph; projectFileIdx = InvalidFileIdx) =
   wantMainModule(conf)
   let systemFileIdx = fileInfoIdx(conf, conf.libpath / RelativeFile"system.nim")
   let projectFile = if projectFileIdx == InvalidFileIdx: conf.projectMainIdx else: projectFileIdx
+  conf.projectMainIdx2 = projectFile
+
+  let packSym = getPackage(graph, projectFile)
+  graph.config.mainPackageId = packSym.getnimblePkgId
   graph.importStack.add projectFile
+
   if projectFile == systemFileIdx:
     discard graph.compileModule(projectFile, {sfMainModule, sfSystemModule})
   else:

compiler/msgs.nim

@@ -10,6 +10,17 @@
 import
   options, strutils, os, tables, ropes, terminal, macros,
   lineinfos, pathutils
+import std/private/miscdollars
+import strutils2
+
+type InstantiationInfo* = typeof(instantiationInfo())
+template instLoc(): InstantiationInfo = instantiationInfo(-2, fullPaths = true)
+
+template flushDot(conf, stdorr) =
+  ## safe to call multiple times
+  if conf.lastMsgWasDot:
+    conf.lastMsgWasDot = false
+    write(stdorr, "\n")
 
 proc toCChar*(c: char; result: var string) =
   case c
@@ -26,14 +37,13 @@ proc makeCString*(s: string): Rope =
   const MaxLineLength = 64
   result = nil
   var res = newStringOfCap(int(s.len.toFloat * 1.1) + 1)
-  add(res, "\"")
-  for i in 0 ..< len(s):
+  res.add("\"")
+  for i in 0..<s.len:
     if (i + 1) mod MaxLineLength == 0:
-      add(res, "\"\L\"")
+      res.add("\"\L\"")
     toCChar(s[i], res)
-  add(res, '\"')
-  add(result, rope(res))
-
+  res.add('\"')
+  result.add(rope(res))
 
 proc newFileInfo(fullPath: AbsoluteFile, projPath: RelativeFile): TFileInfo =
   result.fullPath = fullPath
@@ -57,6 +67,12 @@ when defined(nimpretty):
   proc fileSection*(conf: ConfigRef; fid: FileIndex; a, b: int): string =
     substr(conf.m.fileInfos[fid.int].fullContent, a, b)
 
+proc canonicalCase(path: var string) =
+  ## the idea is to only use this for checking whether a path is already in
+  ## the table but otherwise keep the original case
+  when FileSystemCaseSensitive: discard
+  else: toLowerAscii(path)
+
 proc fileInfoKnown*(conf: ConfigRef; filename: AbsoluteFile): bool =
   var
     canon: AbsoluteFile
@@ -64,6 +80,7 @@ proc fileInfoKnown*(conf: ConfigRef; filename: AbsoluteFile): bool =
     canon = canonicalizePath(conf, filename)
   except OSError:
     canon = filename
+  canon.string.canonicalCase
   result = conf.m.filenameToIndexTbl.hasKey(canon.string)
 
 proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile; isKnownFile: var bool): FileIndex =
@@ -80,14 +97,19 @@ proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile; isKnownFile: var bool
     # This flag indicates that we are working with such a path here
     pseudoPath = true
 
-  if conf.m.filenameToIndexTbl.hasKey(canon.string):
-    result = conf.m.filenameToIndexTbl[canon.string]
+  var canon2: string
+  forceCopy(canon2, canon.string) # because `canon` may be shallow
+  canon2.canonicalCase
+
+  if conf.m.filenameToIndexTbl.hasKey(canon2):
+    isKnownFile = true
+    result = conf.m.filenameToIndexTbl[canon2]
   else:
     isKnownFile = false
     result = conf.m.fileInfos.len.FileIndex
     conf.m.fileInfos.add(newFileInfo(canon, if pseudoPath: RelativeFile filename
                                             else: relativeTo(canon, conf.projectPath)))
-    conf.m.filenameToIndexTbl[canon.string] = result
+    conf.m.filenameToIndexTbl[canon2] = result
 
 proc fileInfoIdx*(conf: ConfigRef; filename: AbsoluteFile): FileIndex =
   var dummy: bool
@@ -107,6 +129,7 @@ proc newLineInfo*(fileInfoIdx: FileIndex, line, col: int): TLineInfo =
 proc newLineInfo*(conf: ConfigRef; filename: AbsoluteFile, line, col: int): TLineInfo {.inline.} =
   result = newLineInfo(fileInfoIdx(conf, filename), line, col)
 
+const gCmdLineInfo* = newLineInfo(commandLineIdx, 1, 1)
 
 proc concat(strings: openArray[string]): string =
   var totalLen = 0
@@ -131,7 +154,6 @@ proc suggestQuit*() =
 # this format is understood by many text editors: it is the same that
 # Borland and Freepascal use
 const
-  PosFormat    = "$1($2, $3) "
   KindFormat   = " [$1]"
   KindColor    = fgCyan
   ErrorTitle   = "Error: "
@@ -144,6 +166,7 @@ const
   # but column numbers start with 0, however most editors expect
   # first column to be 1, so we need to +1 here
   ColOffset*   = 1
+  commandLineDesc* = "command line"
 
 proc getInfoContextLen*(conf: ConfigRef): int = return conf.m.msgContext.len
 proc setInfoContextLen*(conf: ConfigRef; L: int) = setLen(conf.m.msgContext, L)
@@ -152,27 +175,29 @@ proc pushInfoContext*(conf: ConfigRef; info: TLineInfo; detail: string = "") =
   conf.m.msgContext.add((info, detail))
 
 proc popInfoContext*(conf: ConfigRef) =
-  setLen(conf.m.msgContext, len(conf.m.msgContext) - 1)
+  setLen(conf.m.msgContext, conf.m.msgContext.len - 1)
 
 proc getInfoContext*(conf: ConfigRef; index: int): TLineInfo =
-  let L = conf.m.msgContext.len
-  let i = if index < 0: L + index else: index
-  if i >=% L: result = unknownLineInfo()
+  let i = if index < 0: conf.m.msgContext.len + index else: index
+  if i >=% conf.m.msgContext.len: result = unknownLineInfo
   else: result = conf.m.msgContext[i].info
 
 template toFilename*(conf: ConfigRef; fileIdx: FileIndex): string =
   if fileIdx.int32 < 0 or conf == nil:
-    "???"
+    (if fileIdx == commandLineIdx: commandLineDesc else: "???")
   else:
     conf.m.fileInfos[fileIdx.int32].shortName
 
 proc toProjPath*(conf: ConfigRef; fileIdx: FileIndex): string =
-  if fileIdx.int32 < 0 or conf == nil: "???"
+  if fileIdx.int32 < 0 or conf == nil:
+    (if fileIdx == commandLineIdx: commandLineDesc else: "???")
   else: conf.m.fileInfos[fileIdx.int32].projPath.string
 
 proc toFullPath*(conf: ConfigRef; fileIdx: FileIndex): string =
-  if fileIdx.int32 < 0 or conf == nil: result = "???"
-  else: result = conf.m.fileInfos[fileIdx.int32].fullPath.string
+  if fileIdx.int32 < 0 or conf == nil:
+    result = (if fileIdx == commandLineIdx: commandLineDesc else: "???")
+  else:
+    result = conf.m.fileInfos[fileIdx.int32].fullPath.string
 
 proc setDirtyFile*(conf: ConfigRef; fileIdx: FileIndex; filename: AbsoluteFile) =
   assert fileIdx.int32 >= 0
@@ -189,7 +214,7 @@ proc getHash*(conf: ConfigRef; fileIdx: FileIndex): string =
 
 proc toFullPathConsiderDirty*(conf: ConfigRef; fileIdx: FileIndex): AbsoluteFile =
   if fileIdx.int32 < 0:
-    result = AbsoluteFile"???"
+    result = AbsoluteFile(if fileIdx == commandLineIdx: commandLineDesc else: "???")
   elif not conf.m.fileInfos[fileIdx.int32].dirtyFile.isEmpty:
     result = conf.m.fileInfos[fileIdx.int32].dirtyFile
   else:
@@ -207,16 +232,36 @@ template toFullPath*(conf: ConfigRef; info: TLineInfo): string =
 template toFullPathConsiderDirty*(conf: ConfigRef; info: TLineInfo): string =
   string toFullPathConsiderDirty(conf, info.fileIndex)
 
+type FilenameOption* = enum
+  foAbs # absolute path, e.g.: /pathto/bar/foo.nim
+  foRelProject # relative to project path, e.g.: ../foo.nim
+  foMagicSauce # magic sauce, shortest of (foAbs, foRelProject)
+  foName # lastPathPart, e.g.: foo.nim
+  foStacktrace # if optExcessiveStackTrace: foAbs else: foName
+
+proc toFilenameOption*(conf: ConfigRef, fileIdx: FileIndex, opt: FilenameOption): string =
+  case opt
+  of foAbs: result = toFullPath(conf, fileIdx)
+  of foRelProject: result = toProjPath(conf, fileIdx)
+  of foMagicSauce:
+    let
+      absPath = toFullPath(conf, fileIdx)
+      relPath = toProjPath(conf, fileIdx)
+    result = if (optListFullPaths in conf.globalOptions) or
+                (relPath.len > absPath.len) or
+                (relPath.count("..") > 2):
+               absPath
+             else:
+               relPath
+  of foName: result = toProjPath(conf, fileIdx).lastPathPart
+  of foStacktrace:
+    if optExcessiveStackTrace in conf.globalOptions:
+      result = toFilenameOption(conf, fileIdx, foAbs)
+    else:
+      result = toFilenameOption(conf, fileIdx, foName)
+
 proc toMsgFilename*(conf: ConfigRef; info: FileIndex): string =
-  let
-    absPath = toFullPath(conf, info)
-    relPath = toProjPath(conf, info)
-  result = if (optListFullPaths in conf.globalOptions) or
-              (relPath.len > absPath.len) or
-              (relPath.count("..") > 2):
-             absPath
-           else:
-             relPath
+  toFilenameOption(conf, info, foMagicSauce)
 
 template toMsgFilename*(conf: ConfigRef; info: TLineInfo): string =
   toMsgFilename(conf, info.fileIndex)
@@ -227,10 +272,11 @@ proc toLinenumber*(info: TLineInfo): int {.inline.} =
 proc toColumn*(info: TLineInfo): int {.inline.} =
   result = info.col
 
+proc toFileLineCol(info: InstantiationInfo): string {.inline.} =
+  result.toLocation(info.filename, info.line, info.column + ColOffset)
+
 proc toFileLineCol*(conf: ConfigRef; info: TLineInfo): string {.inline.} =
-  # consider calling `helpers.lineInfoToString` instead
-  result = toMsgFilename(conf, info) & "(" & $info.line & ", " &
-    $(info.col + ColOffset) & ")"
+  result.toLocation(toMsgFilename(conf, info), info.line.int, info.col.int + ColOffset)
 
 proc `$`*(conf: ConfigRef; info: TLineInfo): string = toFileLineCol(conf, info)
 
@@ -255,15 +301,16 @@ proc msgWriteln*(conf: ConfigRef; s: string, flags: MsgFlags = {}) =
   ## This is used for 'nim dump' etc. where we don't have nimsuggest
   ## support.
   #if conf.cmd == cmdIdeTools and optCDebug notin gGlobalOptions: return
-
   if not isNil(conf.writelnHook) and msgSkipHook notin flags:
     conf.writelnHook(s)
   elif optStdout in conf.globalOptions or msgStdout in flags:
     if eStdOut in conf.m.errorOutputs:
+      flushDot(conf, stdout)
       writeLine(stdout, s)
       flushFile(stdout)
   else:
     if eStdErr in conf.m.errorOutputs:
+      flushDot(conf, stderr)
       writeLine(stderr, s)
       # On Windows stderr is fully-buffered when piped, regardless of C std.
       when defined(windows):
@@ -292,63 +339,73 @@ macro callStyledWriteLineStderr(args: varargs[typed]): untyped =
   result.add(bindSym"stderr")
   for arg in children(args[0][1]):
     result.add(arg)
+  when false:
+    # not needed because styledWriteLine already ends with resetAttributes
+    result = newStmtList(result, newCall(bindSym"resetAttributes", bindSym"stderr"))
 
 template callWritelnHook(args: varargs[string, `$`]) =
   conf.writelnHook concat(args)
 
+proc msgWrite(conf: ConfigRef; s: string) =
+  if conf.m.errorOutputs != {}:
+    let stdOrr =
+      if optStdout in conf.globalOptions:
+        stdout
+      else:
+        stderr
+    write(stdOrr, s)
+    flushFile(stdOrr)
+    conf.lastMsgWasDot = true # subsequent writes need `flushDot`
+
 template styledMsgWriteln*(args: varargs[typed]) =
   if not isNil(conf.writelnHook):
     callIgnoringStyle(callWritelnHook, nil, args)
   elif optStdout in conf.globalOptions:
     if eStdOut in conf.m.errorOutputs:
+      flushDot(conf, stdout)
       callIgnoringStyle(writeLine, stdout, args)
       flushFile(stdout)
-  else:
-    if eStdErr in conf.m.errorOutputs:
-      if optUseColors in conf.globalOptions:
-        callStyledWriteLineStderr(args)
-      else:
-        callIgnoringStyle(writeLine, stderr, args)
-      # On Windows stderr is fully-buffered when piped, regardless of C std.
-      when defined(windows):
-        flushFile(stderr)
+  elif eStdErr in conf.m.errorOutputs:
+    flushDot(conf, stderr)
+    if optUseColors in conf.globalOptions:
+      callStyledWriteLineStderr(args)
+    else:
+      callIgnoringStyle(writeLine, stderr, args)
+    # On Windows stderr is fully-buffered when piped, regardless of C std.
+    when defined(windows):
+      flushFile(stderr)
 
-proc coordToStr(coord: int): string =
-  if coord == -1: result = "???"
-  else: result = $coord
-
-proc msgKindToString*(kind: TMsgKind): string =
+proc msgKindToString*(kind: TMsgKind): string = MsgKindToStr[kind]
   # later versions may provide translated error messages
-  result = MsgKindToStr[kind]
 
-proc getMessageStr(msg: TMsgKind, arg: string): string =
-  result = msgKindToString(msg) % [arg]
+proc getMessageStr(msg: TMsgKind, arg: string): string = msgKindToString(msg) % [arg]
 
-type
-  TErrorHandling = enum doNothing, doAbort, doRaise
+type TErrorHandling* = enum doNothing, doAbort, doRaise
 
-proc log*(s: string) {.procvar.} =
+proc log*(s: string) =
   var f: File
   if open(f, getHomeDir() / "nimsuggest.log", fmAppend):
     f.writeLine(s)
     close(f)
 
 proc quit(conf: ConfigRef; msg: TMsgKind) {.gcsafe.} =
-  if defined(debug) or msg == errInternal or hintStackTrace in conf.notes:
+  if defined(debug) or msg == errInternal or conf.hasHint(hintStackTrace):
     {.gcsafe.}:
       if stackTraceAvailable() and isNil(conf.writelnHook):
         writeStackTrace()
       else:
-        styledMsgWriteln(fgRed, "No stack traceback available\n" &
-            "To create a stacktrace, rerun compilation with ./koch temp " &
-            conf.command & " <file>")
+        styledMsgWriteln(fgRed, """
+No stack traceback available
+To create a stacktrace, rerun compilation with './koch temp $1 <file>', see $2 for details""" %
+          [conf.command, "intern.html#debugging-the-compiler".createDocLink])
   quit 1
 
 proc handleError(conf: ConfigRef; msg: TMsgKind, eh: TErrorHandling, s: string) =
   if msg >= fatalMin and msg <= fatalMax:
     if conf.cmd == cmdIdeTools: log(s)
     quit(conf, msg)
-  if msg >= errMin and msg <= errMax:
+  if msg >= errMin and msg <= errMax or
+      (msg in warnMin..warnMax and msg in conf.warningAsErrors):
     inc(conf.errorCounter)
     conf.exitcode = 1'i8
     if conf.errorCounter >= conf.errorMax:
@@ -368,93 +425,45 @@ proc writeContext(conf: ConfigRef; lastinfo: TLineInfo) =
   const instantiationFrom = "template/generic instantiation from here"
   const instantiationOfFrom = "template/generic instantiation of `$1` from here"
   var info = lastinfo
-  for i in 0 ..< len(conf.m.msgContext):
+  for i in 0..<conf.m.msgContext.len:
     let context = conf.m.msgContext[i]
     if context.info != lastinfo and context.info != info:
       if conf.structuredErrorHook != nil:
         conf.structuredErrorHook(conf, context.info, instantiationFrom,
-                                 Severity.Error)
+                                 Severity.Hint)
       else:
         let message = if context.detail == "":
           instantiationFrom
         else:
           instantiationOfFrom.format(context.detail)
-        styledMsgWriteln(styleBright,
-                         PosFormat % [toMsgFilename(conf, context.info),
-                                      coordToStr(context.info.line.int),
-                                      coordToStr(context.info.col+ColOffset)],
-                         resetStyle,
-                         message)
+        styledMsgWriteln(styleBright, conf.toFileLineCol(context.info), " ", resetStyle, message)
     info = context.info
 
 proc ignoreMsgBecauseOfIdeTools(conf: ConfigRef; msg: TMsgKind): bool =
   msg >= errGenerated and conf.cmd == cmdIdeTools and optIdeDebug notin conf.globalOptions
 
-proc rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
-  var
-    title: string
-    color: ForegroundColor
-    kind: string
-    sev: Severity
-  case msg
-  of errMin..errMax:
-    sev = Severity.Error
-    writeContext(conf, unknownLineInfo())
-    title = ErrorTitle
-    color = ErrorColor
-  of warnMin..warnMax:
-    sev = Severity.Warning
-    if optWarns notin conf.options: return
-    if msg notin conf.notes: return
-    writeContext(conf, unknownLineInfo())
-    title = WarningTitle
-    color = WarningColor
-    kind = WarningsToStr[ord(msg) - ord(warnMin)]
-    inc(conf.warnCounter)
-  of hintMin..hintMax:
-    sev = Severity.Hint
-    if optHints notin conf.options: return
-    if msg notin conf.notes: return
-    title = HintTitle
-    color = HintColor
-    if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
-    inc(conf.hintCounter)
-  let s = msgKindToString(msg) % args
-
-  if conf.structuredErrorHook != nil:
-    conf.structuredErrorHook(conf, unknownLineInfo(),
-      s & (if kind.len > 0: KindFormat % kind else: ""), sev)
-
-  if not ignoreMsgBecauseOfIdeTools(conf, msg):
-    if kind.len > 0:
-      styledMsgWriteln(color, title, resetStyle, s,
-                       KindColor, `%`(KindFormat, kind))
-    else:
-      styledMsgWriteln(color, title, resetStyle, s)
-  handleError(conf, msg, doAbort, s)
-
-proc rawMessage*(conf: ConfigRef; msg: TMsgKind, arg: string) =
-  rawMessage(conf, msg, [arg])
-
-proc resetAttributes*(conf: ConfigRef) =
-  if {optUseColors, optStdout} * conf.globalOptions == {optUseColors}:
-    terminal.resetAttributes(stderr)
-
 proc addSourceLine(conf: ConfigRef; fileIdx: FileIndex, line: string) =
   conf.m.fileInfos[fileIdx.int32].lines.add line
 
-proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
-  if i.fileIndex.int32 < 0: return ""
-
-  if conf.m.fileInfos[i.fileIndex.int32].lines.len == 0:
+proc numLines*(conf: ConfigRef, fileIdx: FileIndex): int =
+  ## xxx there's an off by 1 error that should be fixed; if a file ends with "foo" or "foo\n"
+  ## it will return same number of lines (ie, a trailing empty line is discounted)
+  result = conf.m.fileInfos[fileIdx.int32].lines.len
+  if result == 0:
     try:
-      for line in lines(toFullPathConsiderDirty(conf, i)):
-        addSourceLine conf, i.fileIndex, line.string
+      for line in lines(toFullPathConsiderDirty(conf, fileIdx).string):
+        addSourceLine conf, fileIdx, line.string
     except IOError:
       discard
-  assert i.fileIndex.int32 < conf.m.fileInfos.len
+    result = conf.m.fileInfos[fileIdx.int32].lines.len
+
+proc sourceLine*(conf: ConfigRef; i: TLineInfo): string =
+  ## 1-based index (matches editor line numbers); 1st line is for i.line = 1
+  ## last valid line is `numLines` inclusive
+  if i.fileIndex.int32 < 0: return ""
+  let num = numLines(conf, i.fileIndex)
   # can happen if the error points to EOF:
-  if i.line.int > conf.m.fileInfos[i.fileIndex.int32].lines.len: return ""
+  if i.line.int > num: return ""
 
   result = conf.m.fileInfos[i.fileIndex.int32].lines[i.line.int-1]
 
@@ -469,101 +478,127 @@ proc formatMsg*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string): s
               of warnMin..warnMax: WarningTitle
               of hintMin..hintMax: HintTitle
               else: ErrorTitle
-  result = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                        coordToStr(info.col+ColOffset)] &
-           title &
-           getMessageStr(msg, arg)
+  conf.toFileLineCol(info) & " " & title & getMessageStr(msg, arg)
 
-proc liMessage(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
-               eh: TErrorHandling) =
+proc liMessage*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg: string,
+               eh: TErrorHandling, info2: InstantiationInfo, isRaw = false) {.noinline.} =
   var
     title: string
     color: ForegroundColor
-    kind:  string
     ignoreMsg = false
     sev: Severity
+  let kind = if msg in warnMin..hintMax and msg != hintUserRaw: $msg else: "" # xxx not sure why hintUserRaw is special
   case msg
   of errMin..errMax:
     sev = Severity.Error
     writeContext(conf, info)
     title = ErrorTitle
     color = ErrorColor
-    # we try to filter error messages so that not two error message
-    # in the same file and line are produced:
-    #ignoreMsg = lastError == info and eh != doAbort
-    conf.m.lastError = info
+    when false:
+      # we try to filter error messages so that not two error message
+      # in the same file and line are produced:
+      # xxx `lastError` is only used in this disabled code; but could be useful to revive
+      ignoreMsg = conf.m.lastError == info and info != unknownLineInfo and eh != doAbort
+    if info != unknownLineInfo: conf.m.lastError = info
   of warnMin..warnMax:
     sev = Severity.Warning
-    ignoreMsg = optWarns notin conf.options or msg notin conf.notes
+    ignoreMsg = not conf.hasWarn(msg)
+    if msg in conf.warningAsErrors:
+      ignoreMsg = false
+      title = ErrorTitle
+    else:
+      title = WarningTitle
     if not ignoreMsg: writeContext(conf, info)
-    title = WarningTitle
     color = WarningColor
-    kind = WarningsToStr[ord(msg) - ord(warnMin)]
     inc(conf.warnCounter)
   of hintMin..hintMax:
     sev = Severity.Hint
-    ignoreMsg = optHints notin conf.options or msg notin conf.notes
+    ignoreMsg = not conf.hasHint(msg)
     title = HintTitle
     color = HintColor
-    if msg != hintUserRaw: kind = HintsToStr[ord(msg) - ord(hintMin)]
     inc(conf.hintCounter)
-  let x = PosFormat % [toMsgFilename(conf, info), coordToStr(info.line.int),
-                       coordToStr(info.col+ColOffset)]
-  let s = getMessageStr(msg, arg)
 
+  let s = if isRaw: arg else: getMessageStr(msg, arg)
   if not ignoreMsg:
+    let loc = if info != unknownLineInfo: conf.toFileLineCol(info) & " " else: ""
+    var kindmsg = if kind.len > 0: KindFormat % kind else: ""
     if conf.structuredErrorHook != nil:
-      conf.structuredErrorHook(conf, info, s & (if kind.len > 0: KindFormat % kind else: ""), sev)
+      conf.structuredErrorHook(conf, info, s & kindmsg, sev)
     if not ignoreMsgBecauseOfIdeTools(conf, msg):
-      if kind.len > 0:
-        styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s,
-                         KindColor, `%`(KindFormat, kind))
+      if msg == hintProcessing:
+        msgWrite(conf, ".")
       else:
-        styledMsgWriteln(styleBright, x, resetStyle, color, title, resetStyle, s)
-      if hintSource in conf.notes:
-        conf.writeSurroundingSrc(info)
+        styledMsgWriteln(styleBright, loc, resetStyle, color, title, resetStyle, s, KindColor, kindmsg)
+        if conf.hasHint(hintSource) and info != unknownLineInfo:
+          conf.writeSurroundingSrc(info)
+        if hintMsgOrigin in conf.mainPackageNotes:
+          styledMsgWriteln(styleBright, toFileLineCol(info2), resetStyle,
+            " compiler msg initiated here", KindColor,
+            KindFormat % $hintMsgOrigin,
+            resetStyle)
   handleError(conf, msg, eh, s)
 
-proc fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  # this fixes bug #7080 so that it is at least obvious 'fatal'
-  # was executed.
+template rawMessage*(conf: ConfigRef; msg: TMsgKind, args: openArray[string]) =
+  let arg = msgKindToString(msg) % args
+  liMessage(conf, unknownLineInfo, msg, arg, eh = doAbort, instLoc(), isRaw = true)
+
+template rawMessage*(conf: ConfigRef; msg: TMsgKind, arg: string) =
+  liMessage(conf, unknownLineInfo, msg, arg, eh = doAbort, instLoc())
+
+template fatal*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  # this fixes bug #7080 so that it is at least obvious 'fatal' was executed.
   conf.m.errorOutputs = {eStdOut, eStdErr}
-  liMessage(conf, info, msg, arg, doAbort)
+  liMessage(conf, info, msg, arg, doAbort, instLoc())
 
-proc globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  liMessage(conf, info, msg, arg, doRaise)
+template globalAssert*(conf: ConfigRef; cond: untyped, info: TLineInfo = unknownLineInfo, arg = "") =
+  ## avoids boilerplate
+  if not cond:
+    var arg2 = "'$1' failed" % [astToStr(cond)]
+    if arg.len > 0: arg2.add "; " & astToStr(arg) & ": " & arg
+    liMessage(conf, info, errGenerated, arg2, doRaise, instLoc())
 
-proc globalError*(conf: ConfigRef; info: TLineInfo, arg: string) =
-  liMessage(conf, info, errGenerated, arg, doRaise)
+template globalError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  ## `local` means compilation keeps going until errorMax is reached (via `doNothing`),
+  ## `global` means it stops.
+  liMessage(conf, info, msg, arg, doRaise, instLoc())
 
-proc localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  liMessage(conf, info, msg, arg, doNothing)
+template globalError*(conf: ConfigRef; info: TLineInfo, arg: string) =
+  liMessage(conf, info, errGenerated, arg, doRaise, instLoc())
 
-proc localError*(conf: ConfigRef; info: TLineInfo, arg: string) =
-  liMessage(conf, info, errGenerated, arg, doNothing)
+template localError*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  liMessage(conf, info, msg, arg, doNothing, instLoc())
 
-proc localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openArray[string]) =
-  localError(conf, info, format % params)
+template localError*(conf: ConfigRef; info: TLineInfo, arg: string) =
+  liMessage(conf, info, errGenerated, arg, doNothing, instLoc())
 
-proc message*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
-  liMessage(conf, info, msg, arg, doNothing)
+template localError*(conf: ConfigRef; info: TLineInfo, format: string, params: openArray[string]) =
+  liMessage(conf, info, errGenerated, format % params, doNothing, instLoc())
 
-proc internalError*(conf: ConfigRef; info: TLineInfo, errMsg: string) =
+template message*(conf: ConfigRef; info: TLineInfo, msg: TMsgKind, arg = "") =
+  liMessage(conf, info, msg, arg, doNothing, instLoc())
+
+proc internalErrorImpl(conf: ConfigRef; info: TLineInfo, errMsg: string, info2: InstantiationInfo) =
   if conf.cmd == cmdIdeTools and conf.structuredErrorHook.isNil: return
   writeContext(conf, info)
-  liMessage(conf, info, errInternal, errMsg, doAbort)
+  liMessage(conf, info, errInternal, errMsg, doAbort, info2)
 
-proc internalError*(conf: ConfigRef; errMsg: string) =
-  if conf.cmd == cmdIdeTools and conf.structuredErrorHook.isNil: return
-  writeContext(conf, unknownLineInfo())
-  rawMessage(conf, errInternal, errMsg)
+template internalError*(conf: ConfigRef; info: TLineInfo, errMsg: string) =
+  internalErrorImpl(conf, info, errMsg, instLoc())
 
-template assertNotNil*(conf: ConfigRef; e): untyped =
-  if e == nil: internalError(conf, $instantiationInfo())
-  e
+template internalError*(conf: ConfigRef; errMsg: string) =
+  internalErrorImpl(conf, unknownLineInfo, errMsg, instLoc())
 
 template internalAssert*(conf: ConfigRef, e: bool) =
-  if not e: internalError(conf, $instantiationInfo())
+  # xxx merge with `globalAssert`
+  if not e:
+    const info2 = instLoc()
+    let arg = info2.toFileLineCol
+    internalErrorImpl(conf, unknownLineInfo, arg, info2)
+
+template lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string) =
+  let m = "'$2' should be: '$1'" % [beau, got]
+  let msg = if optStyleError in conf.globalOptions: errGenerated else: hintName
+  liMessage(conf, info, msg, m, doNothing, instLoc())
 
 proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
   if i.fileIndex.int32 < 0:
@@ -573,25 +608,9 @@ proc quotedFilename*(conf: ConfigRef; i: TLineInfo): Rope =
   else:
     result = conf.m.fileInfos[i.fileIndex.int32].quotedName
 
-proc listWarnings*(conf: ConfigRef) =
-  msgWriteln(conf, "Warnings:")
-  for warn in warnMin..warnMax:
-    msgWriteln(conf, "  [$1] $2" % [
-      if warn in conf.notes: "x" else: " ",
-      lineinfos.WarningsToStr[ord(warn) - ord(warnMin)]
-    ])
+template listMsg(title, r) =
+  msgWriteln(conf, title)
+  for a in r: msgWriteln(conf, "  [$1] $2" % [if a in conf.notes: "x" else: " ", $a])
 
-proc listHints*(conf: ConfigRef) =
-  msgWriteln(conf, "Hints:")
-  for hint in hintMin..hintMax:
-    msgWriteln(conf, "  [$1] $2" % [
-      if hint in conf.notes: "x" else: " ",
-      lineinfos.HintsToStr[ord(hint) - ord(hintMin)]
-    ])
-
-proc lintReport*(conf: ConfigRef; info: TLineInfo, beau, got: string) =
-  let m = "'$2' should be: '$1'" % [beau, got]
-  if optStyleError in conf.globalOptions:
-    localError(conf, info, m)
-  else:
-    message(conf, info, hintName, m)
+proc listWarnings*(conf: ConfigRef) = listMsg("Warnings:", warnMin..warnMax)
+proc listHints*(conf: ConfigRef) = listMsg("Hints:", hintMin..hintMax)

compiler/ndi.nim

@@ -17,6 +17,8 @@ type
     enabled: bool
     f: File
     buf: string
+    filename: AbsoluteFile
+    syms: seq[PSym]
 
 proc doWrite(f: var NdiFile; s: PSym; conf: ConfigRef) =
   f.buf.setLen 0
@@ -28,13 +30,20 @@ proc doWrite(f: var NdiFile; s: PSym; conf: ConfigRef) =
   f.f.writeLine("\t", toFullPath(conf, s.info), "\t", f.buf)
 
 template writeMangledName*(f: NdiFile; s: PSym; conf: ConfigRef) =
-  if f.enabled: doWrite(f, s, conf)
+  if f.enabled: f.syms.add s
 
 proc open*(f: var NdiFile; filename: AbsoluteFile; conf: ConfigRef) =
   f.enabled = not filename.isEmpty
   if f.enabled:
-    f.f = open(filename.string, fmWrite, 8000)
+    f.filename = filename
     f.buf = newStringOfCap(20)
 
-proc close*(f: var NdiFile) =
-  if f.enabled: close(f.f)
+proc close*(f: var NdiFile, conf: ConfigRef) =
+  if f.enabled:
+    f.f = open(f.filename.string, fmWrite, 8000)
+    doAssert f.f != nil, f.filename.string
+    for s in f.syms:
+      doWrite(f, s, conf)
+    close(f.f)
+    f.syms.reset
+    f.filename.reset

compiler/nim.cfg

@@ -7,20 +7,22 @@ define:nimcore
 #define:nimIncremental
 #import:"$projectpath/testability"
 
+#define:staticSqlite
+
 @if windows:
   cincludes: "$lib/wrappers/libffi/common"
 @end
 
 define:useStdoutAsStdmsg
-define:nimOldCaseObjects
 
 @if nimHasStyleChecks:
   styleCheck:error
 @end
 
+
 #define:useNodeIds
 #gc:markAndSweep
 
-@if nimHasWarningCaseTransition:
-warning[CaseTransition]:off
+@if nimHasWarningObservableStores:
+  warning[ObservableStores]: off
 @end

compiler/nim.nim

@@ -22,9 +22,10 @@ import
   commands, options, msgs,
   extccomp, strutils, os, main, parseopt,
   idents, lineinfos, cmdlinehelper,
-  pathutils
+  pathutils, modulegraphs
 
-include nodejs
+from std/browsers import openDefaultBrowser
+from nodejs import findNodeJs
 
 when hasTinyCBackend:
   import tccgen
@@ -33,38 +34,43 @@ when defined(profiler) or defined(memProfiler):
   {.hint: "Profiling support is turned on!".}
   import nimprof
 
-proc prependCurDir(f: AbsoluteFile): AbsoluteFile =
-  when defined(unix):
-    if os.isAbsolute(f.string): result = f
-    else: result = AbsoluteFile("./" & f.string)
-  else:
-    result = f
-
 proc processCmdLine(pass: TCmdLinePass, cmd: string; config: ConfigRef) =
   var p = parseopt.initOptParser(cmd)
   var argsCount = 0
+
+  config.commandLine.setLen 0
+    # bugfix: otherwise, config.commandLine ends up duplicated
+
   while true:
     parseopt.next(p)
     case p.kind
     of cmdEnd: break
     of cmdLongOption, cmdShortOption:
-      if p.key == " ":
+      config.commandLine.add " "
+      config.commandLine.addCmdPrefix p.kind
+      config.commandLine.add p.key.quoteShell # quoteShell to be future proof
+      if p.val.len > 0:
+        config.commandLine.add ':'
+        config.commandLine.add p.val.quoteShell
+
+      if p.key == "": # `-` was passed to indicate main project is stdin
         p.key = "-"
         if processArgument(pass, p, argsCount, config): break
       else:
         processSwitch(pass, p, config)
     of cmdArgument:
+      config.commandLine.add " "
+      config.commandLine.add p.key.quoteShell
       if processArgument(pass, p, argsCount, config): break
   if pass == passCmd2:
     if {optRun, optWasNimscript} * config.globalOptions == {} and
-        config.arguments.len > 0 and config.command.normalize notin ["run", "e"]:
+        config.arguments.len > 0 and config.command.normalize notin ["run", "e", "r"]:
       rawMessage(config, errGenerated, errArgsNeedRunOption)
 
 proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
   let self = NimProg(
     supportsStdinFile: true,
-    processCmdLine: processCmdLine,
-    mainCommand: mainCommand
+    processCmdLine: processCmdLine
   )
   self.initDefinesProg(conf, "nim_compiler")
   if paramCount() == 0:
@@ -72,19 +78,33 @@ proc handleCmdLine(cache: IdentCache; conf: ConfigRef) =
     return
 
   self.processCmdLineAndProjectPath(conf)
-  if not self.loadConfigsAndRunMainCommand(cache, conf): return
-  if optHints in conf.options and hintGCStats in conf.notes: echo(GC_getStatistics())
+  var graph = newModuleGraph(cache, conf)
+  if not self.loadConfigsAndRunMainCommand(cache, conf, graph): return
+  mainCommand(graph)
+  if conf.hasHint(hintGCStats): echo(GC_getStatistics())
   #echo(GC_getStatistics())
   if conf.errorCounter != 0: return
   when hasTinyCBackend:
     if conf.cmd == cmdRun:
-      tccgen.run(conf.arguments)
+      tccgen.run(conf, conf.arguments)
   if optRun in conf.globalOptions:
-    var ex = quoteShell conf.absOutFile
-    if conf.cmd == cmdCompileToJS:
-      execExternalProgram(conf, findNodeJs() & " " & ex & ' ' & conf.arguments)
+    let output = conf.absOutFile
+    case conf.cmd
+    of cmdCompileToBackend:
+      var cmdPrefix = ""
+      case conf.backend
+      of backendC, backendCpp, backendObjc: discard
+      of backendJs: cmdPrefix = findNodeJs() & " "
+      else: doAssert false, $conf.backend
+      execExternalProgram(conf, cmdPrefix & output.quoteShell & ' ' & conf.arguments)
+    of cmdDoc, cmdRst2html:
+      if conf.arguments.len > 0:
+        # reserved for future use
+        rawMessage(conf, errGenerated, "'$1 cannot handle arguments" % [$conf.cmd])
+      openDefaultBrowser($output)
     else:
-      execExternalProgram(conf, ex & ' ' & conf.arguments)
+      # support as needed
+      rawMessage(conf, errGenerated, "'$1 cannot handle --run" % [$conf.cmd])
 
 when declared(GC_setMaxPause):
   GC_setMaxPause 2_000

compiler/nimblecmd.nim

@@ -48,7 +48,7 @@ proc `<`*(ver: Version, ver2: Version): bool =
   # Handling for normal versions such as "0.1.0" or "1.0".
   var sVer = string(ver).split('.')
   var sVer2 = string(ver2).split('.')
-  for i in 0..max(sVer.len, sVer2.len)-1:
+  for i in 0..<max(sVer.len, sVer2.len):
     var sVerI = 0
     if i < sVer.len:
       discard parseInt(sVer[i], sVerI)
@@ -69,9 +69,10 @@ proc getPathVersion*(p: string): tuple[name, version: string] =
   result.version = ""
 
   const specialSeparator = "-#"
-  var sepIdx = p.find(specialSeparator)
+  let last = p.rfind(p.lastPathPart) # the index where the last path part begins
+  var sepIdx = p.find(specialSeparator, start = last)
   if sepIdx == -1:
-    sepIdx = p.rfind('-')
+    sepIdx = p.rfind('-', start = last)
 
   if sepIdx == -1:
     result.name = p
@@ -82,7 +83,7 @@ proc getPathVersion*(p: string): tuple[name, version: string] =
       result.name = p
       return
 
-  result.name = p[0 .. sepIdx - 1]
+  result.name = p[0..sepIdx - 1]
   result.version = p.substr(sepIdx + 1)
 
 proc addPackage(conf: ConfigRef; packages: StringTableRef, p: string; info: TLineInfo) =
@@ -129,30 +130,7 @@ proc addPathRec(conf: ConfigRef; dir: string, info: TLineInfo) =
 proc nimblePath*(conf: ConfigRef; path: AbsoluteDir, info: TLineInfo) =
   addPathRec(conf, path.string, info)
   addNimblePath(conf, path.string, info)
-
-when isMainModule:
-  proc v(s: string): Version = s.newVersion
-  # #head is special in the sense that it's assumed to always be newest.
-  doAssert v"1.0" < v"#head"
-  doAssert v"1.0" < v"1.1"
-  doAssert v"1.0.1" < v"1.1"
-  doAssert v"1" < v"1.1"
-  doAssert v"#aaaqwe" < v"1.1" # We cannot assume that a branch is newer.
-  doAssert v"#a111" < v"#head"
-
-  let conf = newConfigRef()
-  var rr = newStringTable()
-  addPackage conf, rr, "irc-#a111", unknownLineInfo()
-  addPackage conf, rr, "irc-#head", unknownLineInfo()
-  addPackage conf, rr, "irc-0.1.0", unknownLineInfo()
-  #addPackage conf, rr, "irc", unknownLineInfo()
-  #addPackage conf, rr, "another", unknownLineInfo()
-  addPackage conf, rr, "another-0.1", unknownLineInfo()
-
-  addPackage conf, rr, "ab-0.1.3", unknownLineInfo()
-  addPackage conf, rr, "ab-0.1", unknownLineInfo()
-  addPackage conf, rr, "justone-1.0", unknownLineInfo()
-
-  doAssert toSeq(rr.chosen) ==
-    @["irc-#head", "another-0.1", "ab-0.1.3", "justone-1.0"]
-
+  let i = conf.nimblePaths.find(path)
+  if i != -1:
+    conf.nimblePaths.delete(i)
+  conf.nimblePaths.insert(path, 0)